Linux內(nèi)核之ISP驅(qū)動流程分析

ISP驅(qū)動分析

Linux版本: 4.19

芯片平臺: RK3399/RK3288

源碼路徑:

drivers/media/platform/rk-isp10/cif_isp10_v4l2.c

drivers/media/platform/rk-isp10/cif_isp10.c

(1)裝載和卸載函數(shù)

//DTS匹配

static const struct of_device_id cif_isp10_v4l2_of_match[] = {

{.compatible = "rockchip,rk3288-cif-isp",

.data = (void *)&rk3288_cfg},

{.compatible = "rockchip,rk3399-cif-isp",

.data = (void *)&rk3399_cfg},

{},

};

static struct platform_driver cif_isp10_v4l2_plat_drv = {

.driver = {

.name = DRIVER_NAME,

.of_match_table = of_match_ptr(cif_isp10_v4l2_of_match),

.pm = &cif_isp10_dev_pm_ops,

},

.probe = cif_isp10_v4l2_drv_probe,

.remove = cif_isp10_v4l2_drv_remove,

.suspend = cif_isp10_v4l2_drv_suspend,

.resume = cif_isp10_v4l2_drv_resume,

};

static int cif_isp10_v4l2_init(void)

{

int ret;

g_cif_isp10_v4l2_dev_cnt = 0;

ret = platform_driver_register(&cif_isp10_v4l2_plat_drv); //注冊platform_driver

if (ret) {

cif_isp10_pltfrm_pr_err(NULL,

"cannot register platform driver, failed with %d\n",

ret);

return -ENODEV;

}

return ret;

}

static void __exit cif_isp10_v4l2_exit(void)

{

platform_driver_unregister(&cif_isp10_v4l2_plat_drv);

}

上面就是簡單地注冊了一個platform設備。

(2)probe()

static int cif_isp10_v4l2_drv_probe(struct platform_device *pdev)

{

const struct of_device_id *match;

struct device_node *node = pdev->dev.of_node;

struct cif_isp10_device *dev = NULL;

struct cif_isp10_v4l2_device *cif_isp10_v4l2_dev;

int ret;

//........

//分配cif_isp10_v4l2_device

cif_isp10_v4l2_dev = devm_kzalloc(

&pdev->dev,

sizeof(struct cif_isp10_v4l2_device),

GFP_KERNEL);

//.....

match = of_match_node(cif_isp10_v4l2_of_match, node); //獲取匹配的是RK3288還是RK3399

dev = cif_isp10_create(&pdev->dev, //創(chuàng)建cif_isp10_device

cif_isp10_v4l2_event,

cif_isp10_v4l2_requeue_bufs,

(struct pltfrm_soc_cfg *)match->data);

//......

dev->dev_id = g_cif_isp10_v4l2_dev_cnt;

dev->isp_dev.dev_id = &dev->dev_id;

dev->nodes = (void *)cif_isp10_v4l2_dev;

dev->isp_state = CIF_ISP10_STATE_IDLE;

spin_lock_init(&dev->vbq_lock);

spin_lock_init(&dev->vbreq_lock);

spin_lock_init(&dev->iowrite32_verify_lock);

spin_lock_init(&dev->isp_state_lock);

init_waitqueue_head(&dev->isp_stop_wait);

mutex_init(&dev->api_mutex);

ret = v4l2_device_register(dev->dev, &dev->v4l2_dev); //注冊v4l2_device

//......

ret = cif_isp10_v4l2_register_video_device( //注冊video_device, 即生成/dev/videox節(jié)點，該節(jié)點具有VIDEO_OVERLAY功能

dev,

&cif_isp10_v4l2_dev->node[SP_DEV].vdev, //SP:selfpath

SP_VDEV_NAME,

V4L2_CAP_VIDEO_OVERLAY,

CIF_ISP10_V4L2_SP_DEV_MAJOR,

&cif_isp10_v4l2_fops,

&cif_isp10_v4l2_sp_ioctlops);

if (ret)

goto err;

ret = register_cifisp_device(&dev->isp_dev, //注冊ISP video_device

&cif_isp10_v4l2_dev->node[ISP_DEV].vdev,

&dev->v4l2_dev,

dev->config.base_addr);

if (ret)

goto err;

ret = cif_isp10_v4l2_register_video_device( //注冊video_device, 即生成/dev/videox節(jié)點，該節(jié)點具有VIDEO_CAPTURE功能

dev,

&cif_isp10_v4l2_dev->node[MP_DEV].vdev, //MP:mainpath

MP_VDEV_NAME,

V4L2_CAP_VIDEO_CAPTURE,

CIF_ISP10_V4L2_MP_DEV_MAJOR,

&cif_isp10_v4l2_fops,

&cif_isp10_v4l2_mp_ioctlops);

if (ret)

goto err;

ret = cif_isp10_v4l2_register_video_device( //注冊video_device, 即生成/dev/videox節(jié)點，該節(jié)點具有VIDEO_OUTPUT功能

dev,

&cif_isp10_v4l2_dev->node[DMA_DEV].vdev,

DMA_VDEV_NAME,

V4L2_CAP_VIDEO_OUTPUT,

CIF_ISP10_V4L2_DMA_DEV_MAJOR,

&cif_isp10_v4l2_fops,

&cif_isp10_v4l2_dma_ioctlops);

if (ret)

goto err;

cif_isp10_v4l2_register_imgsrc_subdev( //注冊v4l2_subdev，關聯(lián)v4l2_device和v4l2_subdev

dev);

pm_runtime_enable(&pdev->dev);

g_cif_isp10_v4l2_dev[g_cif_isp10_v4l2_dev_cnt] =

cif_isp10_v4l2_dev;

g_cif_isp10_v4l2_dev_cnt++;

return 0;

err:

cif_isp10_destroy(dev);

return ret;

}

上面主要做了:

(1)創(chuàng)建和初始化cif_isp10_device，該結構體中保存著從DTS中解析出來的信息。

(2)注冊v4l2_device

(3)注冊了4個video_device：

rkisp1_ispdev：ISP設備

rkisp1_selfpath: 圖像捕獲設備

rkisp1_mainpath: 圖像捕獲設備，用于高分辨率

rkisp1_dmapath: DMA設備

(4)注冊v4l2_subdev, 將v4l2_device和v4l2_subdev關聯(lián)到一起。

*注意: 應用層就是通過訪問video_device生成的節(jié)點來進行操作Camera，所以video_device注冊時指定了很多ioctl函數(shù)。

(3)創(chuàng)建cif_isp10_device

struct cif_isp10_device *cif_isp10_create(

CIF_ISP10_PLTFRM_DEVICE pdev,

void (*sof_event)(struct cif_isp10_device *dev, __u32 frame_sequence),

void (*requeue_bufs)(struct cif_isp10_device *dev,

enum cif_isp10_stream_id stream_id),

struct pltfrm_soc_cfg *soc_cfg)

{

int ret;

struct cif_isp10_device *dev;

cif_isp10_pltfrm_pr_dbg(NULL, "\n");

//分配結構體

dev = kzalloc(sizeof(*dev), GFP_KERNEL);

//......

dev->sof_event = sof_event;

dev->requeue_bufs = requeue_bufs;

ret = cif_isp10_pltfrm_dev_init(dev,

&pdev, &dev->config.base_addr); //平臺初始化(重映射寄存器地址，申請中斷等)

cif_isp10_pltfrm_soc_init(dev, soc_cfg); //soc相關初始化(ISP,DPHY等時鐘初始化)

ret = cif_isp10_img_srcs_init(dev); //初始化圖像源，即ISP連接的Camera

ret = cif_isp10_register_isrs(dev); //注冊中斷處理函數(shù) (ISR, Interrupt Service Routine),

//......

dev->pm_state = CIF_ISP10_PM_STATE_OFF;

dev->sp_stream.state = CIF_ISP10_STATE_DISABLED;

dev->sp_stream.id = CIF_ISP10_STREAM_SP;

dev->mp_stream.state = CIF_ISP10_STATE_DISABLED;

dev->mp_stream.id = CIF_ISP10_STREAM_MP;

dev->dma_stream.state = CIF_ISP10_STATE_DISABLED;

dev->dma_stream.id = CIF_ISP10_STREAM_DMA;

dev->config.mi_config.async_updt = 0;

(void)cif_isp10_init(dev, CIF_ISP10_ALL_STREAMS); //初始化所有的流(SP,MP,DMA)

cif_isp10_pltfrm_event_init(dev->dev, &dev->dma_stream.done);

cif_isp10_pltfrm_event_init(dev->dev, &dev->sp_stream.done);

cif_isp10_pltfrm_event_init(dev->dev, &dev->mp_stream.done);

dev->img_src_exps.exp_valid_frms[VALID_FR_EXP_T_INDEX] = 4;

dev->img_src_exps.exp_valid_frms[VALID_FR_EXP_G_INDEX] = 4;

dev->img_src_exps.inited = false;

mutex_init(&dev->img_src_exps.mutex);

memset(&dev->img_src_exps.data, 0x00, sizeof(dev->img_src_exps.data));

spin_lock_init(&dev->img_src_exps.lock);

INIT_LIST_HEAD(&dev->img_src_exps.list);

dev->vs_wq = alloc_workqueue("cif isp10 vs workqueue",

WQ_UNBOUND | WQ_MEM_RECLAIM, 1);

/* TBD: clean this up */

init_output_formats();

return dev;

//省略異常處理.....

}

上面是主要的初始化，包括dts解析，時鐘設置，關聯(lián)Camera等配置。

主要的幾個初始化函數(shù)就是上面注釋的位置，我們分析一下 cif_isp10_img_srcs_init，它會獲取ISP上關聯(lián)的Camera。

static int cif_isp10_img_srcs_init(

struct cif_isp10_device *dev)

{

int ret = 0;

memset(dev->img_src_array, 0x00, sizeof(dev->img_src_array));

dev->img_src_cnt = cif_isp10_pltfrm_get_img_src_device(dev->dev, //獲取ISP上的Camera，最多10個

dev->img_src_array, CIF_ISP10_NUM_INPUTS);

if (dev->img_src_cnt > 0)

return 0;

dev->img_src_cnt = 0;

ret = -EFAULT;

cif_isp10_pltfrm_pr_err(dev->dev,

"failed with error %d\n", ret);

return ret;

}

int cif_isp10_pltfrm_get_img_src_device(

struct device *dev,

struct cif_isp10_img_src **img_src_array,

unsigned int array_len)

{

struct device_node *node = NULL;

struct device_node *camera_list_node = NULL;

struct i2c_client *client = NULL;

int ret = 0;

int index, size = 0;

const __be32 *phandle;

int num_cameras = 0;

struct cif_isp10_device *cif_isp10_dev = dev_get_drvdata(dev);

node = of_node_get(dev->of_node);

//.......

//獲取ISP上關聯(lián)的Camera

phandle = of_get_property(node,

"rockchip,camera-modules-attached", &size);

//.......

for (index = 0; index < size / sizeof(*phandle); index++) {

camera_list_node = of_parse_phandle(node,

"rockchip,camera-modules-attached", index);

of_node_put(node);

//......

//判斷是不是I2C subdev，是的話就加入到數(shù)組中

if (!strcmp(camera_list_node->type,

"v4l2-i2c-subdev")) {

client = of_find_i2c_device_by_node(

camera_list_node);

//......

} else {

//......

continue;

}

//加到數(shù)組中

img_src_array[num_cameras] =

cif_isp10_img_src_to_img_src(

&client->dev,

&(cif_isp10_dev->soc_cfg));

if (!IS_ERR_OR_NULL(img_src_array[num_cameras])) {

cif_isp10_pltfrm_pr_info(dev,

"%s attach to cif isp10 img_src_array[%d]\n",

cif_isp10_img_src_g_name(

img_src_array[num_cameras]),

num_cameras);

num_cameras++;

if (num_cameras >= array_len) {

cif_isp10_pltfrm_pr_err(dev,

"cif isp10 isn't support > %d 'camera modules attached'\n",

array_len);

break;

}

}

}

return num_cameras;

//省略異常處理.....

}

上面就是獲取DTS中的如下定義:

&cif_isp0 {

rockchip,camera-modules-attached = <&camera0>;

status = "okay";

};

獲取完后保存在數(shù)組中，cif_isp10_v4l2_register_imgsrc_subdev函數(shù)中會將這些v4l2_subdev和v4l2_device關聯(lián)。

(4)ioctl

應用層調(diào)用ioctl會先調(diào)用到v4l2_file_operations中的unlocked_ioctl或compat_ioctl32，然后最終會調(diào)用到v4l2_ioctl_ops中的各個ioctl。

所以應用層對Camera的控制主要就是通過ioctl，我們隨便找兩個看看:

查詢V4L2功能

static int v4l2_querycap(struct file *file,

void *priv, struct v4l2_capability *cap)

{

struct vb2_queue *queue = to_vb2_queue(file);

struct video_device *vdev = video_devdata(file);

struct cif_isp10_device *dev = to_cif_isp10_device(queue);

u32 stream_ids = to_stream_id(file); //獲取id

strcpy(cap->driver, DRIVER_NAME);

strlcpy(cap->card, vdev->name, sizeof(cap->card));

snprintf(cap->bus_info, sizeof(cap->bus_info),

"platform:" DRIVER_NAME "-%03i",

dev->dev_id);

//根據(jù)ID(SP,MP,DMA)返回對應的功能

if (stream_ids == CIF_ISP10_STREAM_SP) {

cap->capabilities = V4L2_CAP_VIDEO_CAPTURE |

V4L2_CAP_STREAMING;

cap->device_caps |= V4L2_CAP_VIDEO_CAPTURE |

V4L2_CAP_STREAMING;

} else if (stream_ids == CIF_ISP10_STREAM_MP) {

cap->capabilities = V4L2_CAP_VIDEO_CAPTURE |

V4L2_CAP_STREAMING;

cap->device_caps |= V4L2_CAP_VIDEO_CAPTURE |

V4L2_CAP_STREAMING;

}

else if (stream_ids == CIF_ISP10_STREAM_DMA)

cap->capabilities = V4L2_CAP_VIDEO_M2M_MPLANE |

V4L2_CAP_VIDEO_M2M;

cap->capabilities |= V4L2_CAP_DEVICE_CAPS;

cap->device_caps |= V4L2_CAP_DEVICE_CAPS;

return 0;

}

這只是某個ioctl的處理函數(shù)，在內(nèi)部還有非常多，我們就不一一舉例了。因為ISP和Camera已經(jīng)關聯(lián)在一起了，所以ISP中的ioctl會去調(diào)用Camera驅(qū)動中的ioctl。這也就串聯(lián)起來了！

建議大家可以去網(wǎng)上找個Camera拍照的應用demo，就會清楚為什么驅(qū)動會分析到ioctl了。

2. 打開數(shù)據(jù)流

static int cif_isp10_v4l2_streamon(

struct file *file,

void *priv,

enum v4l2_buf_type buf_type)

{

//......

//打開buffer, 準備接收數(shù)據(jù)流

ret = vb2_streamon(queue, buf_type);

//開啟Camera數(shù)據(jù)流。讓數(shù)據(jù)流從Camera流到ISP

ret = cif_isp10_streamon(dev, stream_ids);

if (IS_ERR_VALUE(ret)) {

goto err;

}

return 0;

//......

}

開啟隊列中的buffer，然后調(diào)用Camera中的接口開啟數(shù)據(jù)流，讓數(shù)據(jù)流從Camera流到ISP。

int cif_isp10_streamon(

struct cif_isp10_device *dev,

u32 stream_ids)

{

int ret = 0;

bool streamon_sp = stream_ids & CIF_ISP10_STREAM_SP;

bool streamon_mp = stream_ids & CIF_ISP10_STREAM_MP;

bool streamon_dma = stream_ids & CIF_ISP10_STREAM_DMA;

//......

stream_ids = 0;

if (streamon_mp && dev->mp_stream.updt_cfg)

stream_ids |= CIF_ISP10_STREAM_MP;

if (streamon_sp && dev->sp_stream.updt_cfg)

stream_ids |= CIF_ISP10_STREAM_SP;

ret = cif_isp10_config_cif(dev, stream_ids);

if (IS_ERR_VALUE(ret))

goto err;

//開啟數(shù)據(jù)傳輸

ret = cif_isp10_start(dev, streamon_sp, streamon_mp);

if (IS_ERR_VALUE(ret))

goto err;

//......

}

static int cif_isp10_start(

struct cif_isp10_device *dev,

bool start_sp,

bool start_mp)

{

if (!CIF_ISP10_INP_IS_DMA(dev->config.input_sel)) {

//調(diào)用Camera中的ioctl開啟數(shù)據(jù)流

mutex_lock(&dev->img_src_exps.mutex);

cif_isp10_img_src_ioctl(dev->img_src,

RK_VIDIOC_SENSOR_MODE_DATA,

&dev->img_src_exps.data[0].data);

cif_isp10_img_src_ioctl(dev->img_src,

RK_VIDIOC_SENSOR_MODE_DATA,

&dev->img_src_exps.data[1].data);

dev->img_src_exps.data[0].v_frame_id = 0;

dev->img_src_exps.data[1].v_frame_id = 0;

mutex_unlock(&dev->img_src_exps.mutex);

//.......

}

//.......

return ret;

}

long cif_isp10_img_src_ioctl(

struct cif_isp10_img_src *img_src,

unsigned int cmd,

void *arg)

{

if (!img_src) {

cif_isp10_pltfrm_pr_err(NULL, "img_src is NULL\n");

return -EINVAL;

}

//調(diào)用Camera的ioctl

return img_src->ops->ioctl(img_src->img_src, cmd, arg);

}

通過上面的一些列調(diào)用關系可以看出，最終調(diào)用了Camera的ioctl。這里img_src指的就是sensor。數(shù)據(jù)就開始從Camera一直流向ISP。

總結

我們分析了ISP驅(qū)動的一個大致流程，Camera的很多核心算法不是放在驅(qū)動上的，大部分都是放在應用層上面的。所以我們在驅(qū)動上看到的更多是一些控制，參數(shù)配置等接口。



審核編輯：劉清