Video capture analysis based on xilinx heterogeneous platform

1. Device tree structure

xilinx platform side

		vcap_csi {
			compatible = "xlnx,video";
			dmas = <&Video_IN_1ch_v_frmbuf_wr_0 0>;
			dma-names = "port0";
 
			ports {
				#address-cells = <1>;
				#size-cells = <0>;
 
				port@0 {
					reg = <0>;
					direction = "input";
 
					vcap_csi_in: endpoint {
						remote-endpoint = <&sensor_out>;
					};
				};
			};
		};

PL side frame buffer 

		Video_IN_1ch_v_frmbuf_wr_0: v_frmbuf_wr@b0020000 {
			#dma-cells = <1>;
			clock-names = "ap_clk";
			clocks = <&clk 72>;
			compatible = "xlnx,v-frmbuf-wr-2.1", "xlnx,axi-frmbuf-wr-v2.1";
			interrupt-names = "interrupt";
			interrupt-parent = <&gic>;
			interrupts = <0 104 4>;
			reg = <0x0 0xb0020000 0x0 0x10000>;
			reset-gpios = <&gpio 80 1>;
			xlnx,dma-addr-width = <64>;
			xlnx,dma-align = <8>;
			xlnx,max-height = <2160>;
			xlnx,max-width = <3840>;
			xlnx,pixels-per-clock = <1>;
			xlnx,s-axi-ctrl-addr-width = <0x7>;
			xlnx,s-axi-ctrl-data-width = <0x20>;
			xlnx,vid-formats = "yuyv";
			xlnx,video-width = <8>;
		};

Sub-device driver end: 

			nvp6124: sensor@1a{
				compatible = "nextchip,nvp6124";
				reg = <0x1a>;
				#address-cells = <1>;
				#size-cells = <0>;
 
				port@0 {
					reg = <0>;
 
					sensor_out: endpoint {
						remote-endpoint = <&vcap_csi_in>;
					};
				};
			};

2. Source code analysis

1) Platform side

After the platform device driver matches the device tree, execute the probe function and do the following:

 
static int xvip_graph_init(struct xvip_composite_device *xdev)
{
	struct xvip_graph_entity *entity;
	struct v4l2_async_subdev **subdevs = NULL;
	unsigned int num_subdevs;
	unsigned int i;
	int ret;
 
	/* Init the DMA channels. */
	ret = xvip_graph_dma_init(xdev);
	if (ret < 0) {
		dev_err(xdev->dev, "DMA initialization failed\n");
		goto done;
	}
 
	/* Parse the graph to extract a list of subdevice DT nodes. */
	ret = xvip_graph_parse(xdev);
	if (ret < 0) {
		dev_err(xdev->dev, "graph parsing failed\n");
		goto done;
	}
 
	if (!xdev->num_subdevs) {
		dev_err(xdev->dev, "no subdev found in graph\n");
		goto done;
	}
 
	/* Register the subdevices notifier. */
	num_subdevs = xdev->num_subdevs;
	subdevs = devm_kzalloc(xdev->dev, sizeof(*subdevs) * num_subdevs,
			       GFP_KERNEL);
	if (subdevs == NULL) {
		ret = -ENOMEM;
		goto done;
	}
 
	i = 0;
	list_for_each_entry(entity, &xdev->entities, list)
		subdevs[i++] = &entity->asd;
 
	xdev->notifier.subdevs = subdevs;
	xdev->notifier.num_subdevs = num_subdevs;
	xdev->notifier.bound = xvip_graph_notify_bound;
	xdev->notifier.complete = xvip_graph_notify_complete;
 
	ret = v4l2_async_notifier_register(&xdev->v4l2_dev, &xdev->notifier);
	if (ret < 0) {
		dev_err(xdev->dev, "notifier registration failed\n");
		goto done;
	}
 
	ret = 0;
 
done:
	if (ret < 0)
		xvip_graph_cleanup(xdev);
 
	return ret;
}

Mainly did DMA channel initialization and v4l2_async_notifier_register

Take a look at what v4l2_async_notifier_register does;

 
int v4l2_async_notifier_register(struct v4l2_device *v4l2_dev,
				 struct v4l2_async_notifier *notifier)
{
	struct v4l2_subdev *sd, *tmp;
	struct v4l2_async_subdev *asd;
	int i;
 
	if (!v4l2_dev || !notifier->num_subdevs ||
	    notifier->num_subdevs > V4L2_MAX_SUBDEVS)
		return -EINVAL;
 
	notifier->v4l2_dev = v4l2_dev;
	INIT_LIST_HEAD(&notifier->waiting);
	INIT_LIST_HEAD(&notifier->done);
 
	for (i = 0; i < notifier->num_subdevs; i++) {
		asd = notifier->subdevs[i];
 
		switch (asd->match_type) {
		case V4L2_ASYNC_MATCH_CUSTOM:
		case V4L2_ASYNC_MATCH_DEVNAME:
		case V4L2_ASYNC_MATCH_I2C:
		case V4L2_ASYNC_MATCH_FWNODE:
			break;
		default:
			dev_err(notifier->v4l2_dev ? notifier->v4l2_dev->dev : NULL,
				"Invalid match type %u on %p\n",
				asd->match_type, asd);
			return -EINVAL;
		}
		list_add_tail(&asd->list, &notifier->waiting);
	}
 
	mutex_lock(&list_lock);
	//获取是否有注册的subdev
	list_for_each_entry_safe(sd, tmp, &subdev_list, async_list) {
		int ret;
		printk("___________________v4l2_async_notifier_register___________notifier_list\n");
 
		asd = v4l2_async_belongs(notifier, sd);
		if (!asd)
			continue;
 
		ret = v4l2_async_test_notify(notifier, sd, asd);
		if (ret < 0) {
			mutex_unlock(&list_lock);
			return ret;
		}
	}
	printk("______________________________notifier_list\n");
 
	/* Keep also completed notifiers on the list */
	list_add(&notifier->list, &notifier_list);
 
	mutex_unlock(&list_lock);
 
	return 0;
}

Assuming that the child device is matched late, the existing child device registration cannot be obtained in the linked list. At this time, the notification message is added to the notifier_list linked list, and the child device driver side will obtain it;

If there are sub-devices registered, the v4l2_async_test_notify function will be executed;

 
static int v4l2_async_test_notify(struct v4l2_async_notifier *notifier,
				  struct v4l2_subdev *sd,
				  struct v4l2_async_subdev *asd)
{
	int ret;
 
	if (notifier->bound) {
		ret = notifier->bound(notifier, sd, asd);
		if (ret < 0)
			return ret;
	}
 
	ret = v4l2_device_register_subdev(notifier->v4l2_dev, sd);
	if (ret < 0) {
		if (notifier->unbind)
			notifier->unbind(notifier, sd, asd);
		return ret;
	}
 
	/* Remove from the waiting list */
	list_del(&asd->list);
	sd->asd = asd;
	sd->notifier = notifier;
 
	/* Move from the global subdevice list to notifier's done */
	list_move(&sd->async_list, &notifier->done);
 
	if (list_empty(&notifier->waiting) && notifier->complete)
		return notifier->complete(notifier);
 
	return 0;
}

Here callback bound and execute v4l2_device_register_subdev to register the sub-device to the V4L2 core layer;

 

2) Sub-device end

Register through v4l2_async_register_subdev;

 
int v4l2_async_register_subdev(struct v4l2_subdev *sd)
{
	struct v4l2_async_notifier *notifier;
 
	/*
	 * No reference taken. The reference is held by the device
	 * (struct v4l2_subdev.dev), and async sub-device does not
	 * exist independently of the device at any point of time.
	 */
	if (!sd->fwnode && sd->dev)
		sd->fwnode = dev_fwnode(sd->dev);
 
	mutex_lock(&list_lock);
 
	INIT_LIST_HEAD(&sd->async_list);
	//查找是否平台已注册了通知subdev事件，紧接着将执行注册v4l2_device_register_subdev
	list_for_each_entry(notifier, &notifier_list, list) {
		printk("______________________v4l2_async_register_subdev___________notifier_list\n");
		struct v4l2_async_subdev *asd = v4l2_async_belongs(notifier, sd);
		if (asd) {
			int ret = v4l2_async_test_notify(notifier, sd, asd);
			mutex_unlock(&list_lock);
			return ret;
		}
	}
 
	/* None matched, wait for hot-plugging */
	list_add(&sd->async_list, &subdev_list);
 
	mutex_unlock(&list_lock);
 
	return 0;
}

The operation here is complementary to the above v4l2_async_notifier_register; no matter who registers first, the v4l2_async_test_notify function operation will be realized in the end;