media: staging/imx: get CSI bus type from nearest upstream entity

The imx-media driver currently supports a device tree graph of
limited complexity. This patch is a first step in allowing imx-media
to work with more general OF graphs.

The CSI subdevice assumes the originating upstream subdevice (the
"sensor") is connected directly to either the CSI mux or the MIPI
CSI-2 receiver. But for more complex graphs, the sensor can be distant,
with possible bridge entities in between. Thus the sensor's bus type
could be quite different from what is entering the CSI. For example
a distant sensor could have a parallel interface, but the stream
entering the i.MX is MIPI CSI-2.

To remove this assumption, get the entering bus config from the entity
that is directly upstream from either the CSI mux, or the CSI-2 receiver.
If the CSI-2 receiver is not in the enabled pipeline, the bus type to the
CSI is parallel, otherwise the CSI is receiving MIPI CSI-2.

Note that we can't use the direct upstream source connected to CSI
(which is either the CSI mux or the CSI-2 receiver) to determine
bus type. The bus entering the CSI from the CSI-2 receiver is a 32-bit
parallel bus containing the demultiplexed MIPI CSI-2 virtual channels.
But the CSI and its IDMAC channels must be configured based on whether
it is receiving data from the CSI-2 receiver or from the CSI mux's
parallel interface pins.

The function csi_get_upstream_endpoint() is used to find this
endpoint. It makes use of a new utility function
imx_media_find_upstream_pad(), that if given a grp_id of 0, will
return the closest upstream pad from start_entity.

With these changes, imx_media_find_sensor() is no longer used and
is removed. As a result there is also no longer a need to identify
any sensor or set the sensor subdev's group id as a method to search
for it. So IMX_MEDIA_GRP_ID_SENSOR is removed. Also the video-mux group
id IMX_MEDIA_GRP_ID_VIDMUX was never used so that is removed as well.
The remaining IMX_MEDIA_GRP_ID_* definitions are entities internal
to the i.MX.

Another use of imx_media_find_sensor() in the CSI was to call the
sensor's g_skip_frames op to determine if a delay was needed before
enabling the CSI at stream on. If necessary this will have to be
re-addressed at a later time.

Signed-off-by: Steve Longerbeam <[email protected]>
Signed-off-by: Hans Verkuil <[email protected]>
Signed-off-by: Mauro Carvalho Chehab <[email protected]>

Author: slongerbeam

drivers/staging/media/imx/imx-media-csi.c

@@ -13,6 +13,7 @@
 #include <linux/gcd.h>
 #include <linux/interrupt.h>
 #include <linux/module.h>
+#include <linux/of_graph.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
 #include <media/v4l2-ctrls.h>
@@ -99,8 +100,8 @@ struct csi_priv {
 	/* the mipi virtual channel number at link validate */
 	int vc_num;
 
-	/* the attached sensor at stream on */
-	struct imx_media_subdev *sensor;
+	/* the upstream endpoint CSI is receiving from */
+	struct v4l2_fwnode_endpoint upstream_ep;
 
 	spinlock_t irqlock; /* protect eof_irq handler */
 	struct timer_list eof_timeout_timer;
@@ -120,6 +121,71 @@ static inline struct csi_priv *sd_to_dev(struct v4l2_subdev *sdev)
 	return container_of(sdev, struct csi_priv, sd);
 }
 
+static inline bool is_parallel_16bit_bus(struct v4l2_fwnode_endpoint *ep)
+{
+	return ep->bus_type != V4L2_MBUS_CSI2 &&
+		ep->bus.parallel.bus_width >= 16;
+}
+
+/*
+ * Parses the fwnode endpoint from the source pad of the entity
+ * connected to this CSI. This will either be the entity directly
+ * upstream from the CSI-2 receiver, or directly upstream from the
+ * video mux. The endpoint is needed to determine the bus type and
+ * bus config coming into the CSI.
+ */
+static int csi_get_upstream_endpoint(struct csi_priv *priv,
+				     struct v4l2_fwnode_endpoint *ep)
+{
+	struct device_node *endpoint, *port;
+	struct imx_media_subdev *imxsd;
+	struct media_entity *src;
+	struct v4l2_subdev *sd;
+	struct media_pad *pad;
+
+	if (!priv->src_sd)
+		return -EPIPE;
+
+	src = &priv->src_sd->entity;
+
+	if (src->function == MEDIA_ENT_F_VID_MUX) {
+		/*
+		 * CSI is connected directly to video mux, skip up to
+		 * CSI-2 receiver if it is in the path, otherwise stay
+		 * with video mux.
+		 */
+		imxsd = imx_media_find_upstream_subdev(priv->md, src,
+						       IMX_MEDIA_GRP_ID_CSI2);
+		if (!IS_ERR(imxsd))
+			src = &imxsd->sd->entity;
+	}
+
+	/* get source pad of entity directly upstream from src */
+	pad = imx_media_find_upstream_pad(priv->md, src, 0);
+	if (IS_ERR(pad))
+		return PTR_ERR(pad);
+
+	sd = media_entity_to_v4l2_subdev(pad->entity);
+
+	/*
+	 * NOTE: this assumes an OF-graph port id is the same as a
+	 * media pad index.
+	 */
+	port = of_graph_get_port_by_id(sd->dev->of_node, pad->index);
+	if (!port)
+		return -ENODEV;
+
+	endpoint = of_get_next_child(port, NULL);
+	of_node_put(port);
+	if (!endpoint)
+		return -ENODEV;
+
+	v4l2_fwnode_endpoint_parse(of_fwnode_handle(endpoint), ep);
+	of_node_put(endpoint);
+
+	return 0;
+}
+
 static void csi_idmac_put_ipu_resources(struct csi_priv *priv)
 {
 	if (priv->idmac_ch)
@@ -302,7 +368,6 @@ static void csi_idmac_unsetup_vb2_buf(struct csi_priv *priv,
 static int csi_idmac_setup_channel(struct csi_priv *priv)
 {
 	struct imx_media_video_dev *vdev = priv->vdev;
-	struct v4l2_fwnode_endpoint *sensor_ep;
 	struct v4l2_mbus_framefmt *infmt;
 	struct ipu_image image;
 	u32 passthrough_bits;
@@ -312,7 +377,6 @@ static int csi_idmac_setup_channel(struct csi_priv *priv)
 	int ret;
 
 	infmt = &priv->format_mbus[CSI_SINK_PAD];
-	sensor_ep = &priv->sensor->sensor_ep;
 
 	ipu_cpmem_zero(priv->idmac_ch);
 
@@ -330,7 +394,7 @@ static int csi_idmac_setup_channel(struct csi_priv *priv)
 	 * Check for conditions that require the IPU to handle the
 	 * data internally as generic data, aka passthrough mode:
 	 * - raw bayer formats
-	 * - the sensor bus is 16-bit parallel
+	 * - the CSI is receiving from a 16-bit parallel bus
 	 */
 	switch (image.pix.pixelformat) {
 	case V4L2_PIX_FMT_SBGGR8:
@@ -354,8 +418,7 @@ static int csi_idmac_setup_channel(struct csi_priv *priv)
 		burst_size = (image.pix.width & 0x3f) ?
 			     ((image.pix.width & 0x1f) ?
 			      ((image.pix.width & 0xf) ? 8 : 16) : 32) : 64;
-		passthrough = (sensor_ep->bus_type != V4L2_MBUS_CSI2 &&
-			       sensor_ep->bus.parallel.bus_width >= 16);
+		passthrough = is_parallel_16bit_bus(&priv->upstream_ep);
 		passthrough_bits = 16;
 		/* Skip writing U and V components to odd rows */
 		ipu_cpmem_skip_odd_chroma_rows(priv->idmac_ch);
@@ -364,14 +427,12 @@ static int csi_idmac_setup_channel(struct csi_priv *priv)
 	case V4L2_PIX_FMT_UYVY:
 		burst_size = (image.pix.width & 0x1f) ?
 			     ((image.pix.width & 0xf) ? 8 : 16) : 32;
-		passthrough = (sensor_ep->bus_type != V4L2_MBUS_CSI2 &&
-			       sensor_ep->bus.parallel.bus_width >= 16);
+		passthrough = is_parallel_16bit_bus(&priv->upstream_ep);
 		passthrough_bits = 16;
 		break;
 	default:
 		burst_size = (image.pix.width & 0xf) ? 8 : 16;
-		passthrough = (sensor_ep->bus_type != V4L2_MBUS_CSI2 &&
-			       sensor_ep->bus.parallel.bus_width >= 16);
+		passthrough = is_parallel_16bit_bus(&priv->upstream_ep);
 		passthrough_bits = 16;
 		break;
 	}
@@ -568,22 +629,20 @@ static void csi_idmac_stop(struct csi_priv *priv)
 static int csi_setup(struct csi_priv *priv)
 {
 	struct v4l2_mbus_framefmt *infmt, *outfmt;
-	struct v4l2_mbus_config sensor_mbus_cfg;
-	struct v4l2_fwnode_endpoint *sensor_ep;
+	struct v4l2_mbus_config mbus_cfg;
 	struct v4l2_mbus_framefmt if_fmt;
 
 	infmt = &priv->format_mbus[CSI_SINK_PAD];
 	outfmt = &priv->format_mbus[priv->active_output_pad];
-	sensor_ep = &priv->sensor->sensor_ep;
 
-	/* compose mbus_config from sensor endpoint */
-	sensor_mbus_cfg.type = sensor_ep->bus_type;
-	sensor_mbus_cfg.flags = (sensor_ep->bus_type == V4L2_MBUS_CSI2) ?
-		sensor_ep->bus.mipi_csi2.flags :
-		sensor_ep->bus.parallel.flags;
+	/* compose mbus_config from the upstream endpoint */
+	mbus_cfg.type = priv->upstream_ep.bus_type;
+	mbus_cfg.flags = (priv->upstream_ep.bus_type == V4L2_MBUS_CSI2) ?
+		priv->upstream_ep.bus.mipi_csi2.flags :
+		priv->upstream_ep.bus.parallel.flags;
 
 	/*
-	 * we need to pass input sensor frame to CSI interface, but
+	 * we need to pass input frame to CSI interface, but
 	 * with translated field type from output format
 	 */
 	if_fmt = *infmt;
@@ -595,7 +654,7 @@ static int csi_setup(struct csi_priv *priv)
 			     priv->crop.width == 2 * priv->compose.width,
 			     priv->crop.height == 2 * priv->compose.height);
 
-	ipu_csi_init_interface(priv->csi, &sensor_mbus_cfg, &if_fmt);
+	ipu_csi_init_interface(priv->csi, &mbus_cfg, &if_fmt);
 
 	ipu_csi_set_dest(priv->csi, priv->dest);
 
@@ -611,35 +670,11 @@ static int csi_setup(struct csi_priv *priv)
 static int csi_start(struct csi_priv *priv)
 {
 	struct v4l2_fract *output_fi, *input_fi;
-	u32 bad_frames = 0;
 	int ret;
 
-	if (!priv->sensor) {
-		v4l2_err(&priv->sd, "no sensor attached\n");
-		return -EINVAL;
-	}
-
 	output_fi = &priv->frame_interval[priv->active_output_pad];
 	input_fi = &priv->frame_interval[CSI_SINK_PAD];
 
-	ret = v4l2_subdev_call(priv->sensor->sd, sensor,
-			       g_skip_frames, &bad_frames);
-	if (!ret && bad_frames) {
-		u32 delay_usec;
-
-		/*
-		 * This sensor has bad frames when it is turned on,
-		 * add a delay to avoid them before enabling the CSI
-		 * hardware. Especially for sensors with a bt.656 interface,
-		 * any shifts in the SAV/EAV sync codes will cause the CSI
-		 * to lose vert/horiz sync.
-		 */
-		delay_usec = DIV_ROUND_UP_ULL(
-			(u64)USEC_PER_SEC * input_fi->numerator * bad_frames,
-			input_fi->denominator);
-		usleep_range(delay_usec, delay_usec + 1000);
-	}
-
 	if (priv->dest == IPU_CSI_DEST_IDMAC) {
 		ret = csi_idmac_start(priv);
 		if (ret)
@@ -971,9 +1006,8 @@ static int csi_link_validate(struct v4l2_subdev *sd,
 			     struct v4l2_subdev_format *sink_fmt)
 {
 	struct csi_priv *priv = v4l2_get_subdevdata(sd);
-	struct v4l2_fwnode_endpoint *sensor_ep;
+	struct v4l2_fwnode_endpoint upstream_ep;
 	const struct imx_media_pixfmt *incc;
-	struct imx_media_subdev *sensor;
 	bool is_csi2;
 	int ret;
 
@@ -982,22 +1016,20 @@ static int csi_link_validate(struct v4l2_subdev *sd,
 	if (ret)
 		return ret;
 
-	sensor = __imx_media_find_sensor(priv->md, &priv->sd.entity);
-	if (IS_ERR(sensor)) {
-		v4l2_err(&priv->sd, "no sensor attached\n");
-		return PTR_ERR(sensor);
+	ret = csi_get_upstream_endpoint(priv, &upstream_ep);
+	if (ret) {
+		v4l2_err(&priv->sd, "failed to find upstream endpoint\n");
+		return ret;
 	}
 
 	mutex_lock(&priv->lock);
 
-	priv->sensor = sensor;
-	sensor_ep = &priv->sensor->sensor_ep;
-	is_csi2 = (sensor_ep->bus_type == V4L2_MBUS_CSI2);
+	priv->upstream_ep = upstream_ep;
+	is_csi2 = (upstream_ep.bus_type == V4L2_MBUS_CSI2);
 	incc = priv->cc[CSI_SINK_PAD];
 
 	if (priv->dest != IPU_CSI_DEST_IDMAC &&
-	    (incc->bayer || (!is_csi2 &&
-			     sensor_ep->bus.parallel.bus_width >= 16))) {
+	    (incc->bayer || is_parallel_16bit_bus(&upstream_ep))) {
 		v4l2_err(&priv->sd,
 			 "bayer/16-bit parallel buses must go to IDMAC pad\n");
 		ret = -EINVAL;
@@ -1067,12 +1099,8 @@ static void csi_try_crop(struct csi_priv *priv,
 			 struct v4l2_rect *crop,
 			 struct v4l2_subdev_pad_config *cfg,
 			 struct v4l2_mbus_framefmt *infmt,
-			 struct imx_media_subdev *sensor)
+			 struct v4l2_fwnode_endpoint *upstream_ep)
 {
-	struct v4l2_fwnode_endpoint *sensor_ep;
-
-	sensor_ep = &sensor->sensor_ep;
-
 	crop->width = min_t(__u32, infmt->width, crop->width);
 	if (crop->left + crop->width > infmt->width)
 		crop->left = infmt->width - crop->width;
@@ -1086,7 +1114,7 @@ static void csi_try_crop(struct csi_priv *priv,
 	 * sync, so fix it to NTSC/PAL active lines. NTSC contains
 	 * 2 extra lines of active video that need to be cropped.
 	 */
-	if (sensor_ep->bus_type == V4L2_MBUS_BT656 &&
+	if (upstream_ep->bus_type == V4L2_MBUS_BT656 &&
 	    (V4L2_FIELD_HAS_BOTH(infmt->field) ||
 	     infmt->field == V4L2_FIELD_ALTERNATE)) {
 		crop->height = infmt->height;
@@ -1236,7 +1264,7 @@ static int csi_get_fmt(struct v4l2_subdev *sd,
 }
 
 static void csi_try_fmt(struct csi_priv *priv,
-			struct imx_media_subdev *sensor,
+			struct v4l2_fwnode_endpoint *upstream_ep,
 			struct v4l2_subdev_pad_config *cfg,
 			struct v4l2_subdev_format *sdformat,
 			struct v4l2_rect *crop,
@@ -1304,7 +1332,7 @@ static void csi_try_fmt(struct csi_priv *priv,
 		crop->top = 0;
 		crop->width = sdformat->format.width;
 		crop->height = sdformat->format.height;
-		csi_try_crop(priv, crop, cfg, &sdformat->format, sensor);
+		csi_try_crop(priv, crop, cfg, &sdformat->format, upstream_ep);
 		compose->left = 0;
 		compose->top = 0;
 		compose->width = crop->width;
@@ -1333,20 +1361,20 @@ static int csi_set_fmt(struct v4l2_subdev *sd,
 {
 	struct csi_priv *priv = v4l2_get_subdevdata(sd);
 	struct imx_media_video_dev *vdev = priv->vdev;
+	struct v4l2_fwnode_endpoint upstream_ep;
 	const struct imx_media_pixfmt *cc;
-	struct imx_media_subdev *sensor;
 	struct v4l2_pix_format vdev_fmt;
 	struct v4l2_mbus_framefmt *fmt;
 	struct v4l2_rect *crop, *compose;
-	int ret = 0;
+	int ret;
 
 	if (sdformat->pad >= CSI_NUM_PADS)
 		return -EINVAL;
 
-	sensor = imx_media_find_sensor(priv->md, &priv->sd.entity);
-	if (IS_ERR(sensor)) {
-		v4l2_err(&priv->sd, "no sensor attached\n");
-		return PTR_ERR(sensor);
+	ret = csi_get_upstream_endpoint(priv, &upstream_ep);
+	if (ret) {
+		v4l2_err(&priv->sd, "failed to find upstream endpoint\n");
+		return ret;
 	}
 
 	mutex_lock(&priv->lock);
@@ -1359,7 +1387,7 @@ static int csi_set_fmt(struct v4l2_subdev *sd,
 	crop = __csi_get_crop(priv, cfg, sdformat->which);
 	compose = __csi_get_compose(priv, cfg, sdformat->which);
 
-	csi_try_fmt(priv, sensor, cfg, sdformat, crop, compose, &cc);
+	csi_try_fmt(priv, &upstream_ep, cfg, sdformat, crop, compose, &cc);
 
 	fmt = __csi_get_fmt(priv, cfg, sdformat->pad, sdformat->which);
 	*fmt = sdformat->format;
@@ -1376,8 +1404,8 @@ static int csi_set_fmt(struct v4l2_subdev *sd,
 			format.pad = pad;
 			format.which = sdformat->which;
 			format.format = sdformat->format;
-			csi_try_fmt(priv, sensor, cfg, &format, NULL, compose,
-				    &outcc);
+			csi_try_fmt(priv, &upstream_ep, cfg, &format,
+				    NULL, compose, &outcc);
 
 			outfmt = __csi_get_fmt(priv, cfg, pad, sdformat->which);
 			*outfmt = format.format;
@@ -1472,18 +1500,18 @@ static int csi_set_selection(struct v4l2_subdev *sd,
 			     struct v4l2_subdev_selection *sel)
 {
 	struct csi_priv *priv = v4l2_get_subdevdata(sd);
+	struct v4l2_fwnode_endpoint upstream_ep;
 	struct v4l2_mbus_framefmt *infmt;
 	struct v4l2_rect *crop, *compose;
-	struct imx_media_subdev *sensor;
-	int pad, ret = 0;
+	int pad, ret;
 
 	if (sel->pad != CSI_SINK_PAD)
 		return -EINVAL;
 
-	sensor = imx_media_find_sensor(priv->md, &priv->sd.entity);
-	if (IS_ERR(sensor)) {
-		v4l2_err(&priv->sd, "no sensor attached\n");
-		return PTR_ERR(sensor);
+	ret = csi_get_upstream_endpoint(priv, &upstream_ep);
+	if (ret) {
+		v4l2_err(&priv->sd, "failed to find upstream endpoint\n");
+		return ret;
 	}
 
 	mutex_lock(&priv->lock);
@@ -1511,7 +1539,7 @@ static int csi_set_selection(struct v4l2_subdev *sd,
 			goto out;
 		}
 
-		csi_try_crop(priv, &sel->r, cfg, infmt, sensor);
+		csi_try_crop(priv, &sel->r, cfg, infmt, &upstream_ep);
 
 		*crop = sel->r;
 

drivers/staging/media/imx/imx-media-dev.c

@@ -222,18 +222,6 @@ static int imx_media_subdev_bound(struct v4l2_async_notifier *notifier,
 		ret = imx_media_get_ipu(imxmd, sd);
 		if (ret)
 			goto out_unlock;
-	} else if (sd->entity.function == MEDIA_ENT_F_VID_MUX) {
-		/* this is a video mux */
-		sd->grp_id = IMX_MEDIA_GRP_ID_VIDMUX;
-	} else if (imxsd->num_sink_pads == 0) {
-		/*
-		 * this is an original source of video frames, it
-		 * could be a camera sensor, an analog decoder, or
-		 * a bridge device (HDMI -> MIPI CSI-2 for example).
-		 * This group ID is used to locate the entity that
-		 * is the original source of video in a pipeline.
-		 */
-		sd->grp_id = IMX_MEDIA_GRP_ID_SENSOR;
 	}
 
 	/* attach the subdev */