feat: add esp32s3 yuv2rgb conversion support (#414)

Author: WangYuxin-esp

Kconfig

@@ -164,5 +164,24 @@ menu "Camera configuration"
         help
             Maximum value of DMA buffer
             Larger values may fail to allocate due to insufficient contiguous memory blocks, and smaller value may cause DMA interrupt to be too frequent
+    
+    config CAMERA_CONVERTER_ENABLED
+        bool "Enable camera RGB/YUV converter"
+        depends on IDF_TARGET_ESP32S3
+        default n
+        help
+            Enable this option if you want to use RGB565/YUV422/YUV420/YUV411 format conversion.
 
+    choice CAMERA_CONV_PROTOCOL
+        bool "Camera converter protocol"
+        depends on CAMERA_CONVERTER_ENABLED
+        default LCD_CAM_CONV_BT601_ENABLED
+        help
+            Supports format conversion under both BT601 and BT709 standards.
+
+        config LCD_CAM_CONV_BT601_ENABLED
+            bool "BT601"
+        config LCD_CAM_CONV_BT709_ENABLED
+            bool "BT709"
+    endchoice
 endmenu

driver/esp_camera.c

@@ -236,6 +236,23 @@ static esp_err_t camera_probe(const camera_config_t *config, camera_model_t *out
     return ESP_OK;
 }
 
+#if CONFIG_CAMERA_CONVERTER_ENABLED
+static pixformat_t get_output_data_format(camera_conv_mode_t conv_mode)
+{
+    pixformat_t format = PIXFORMAT_RGB565;
+    switch (conv_mode) {
+    case YUV422_TO_YUV420:
+        format = PIXFORMAT_YUV420;
+        break;
+    case YUV422_TO_RGB565: // default format is RGB565
+    default:
+        break;
+    }
+    ESP_LOGD(TAG, "Convert to %d format enabled", format);
+    return format;
+}
+#endif
+
 esp_err_t esp_camera_init(const camera_config_t *config)
 {
     esp_err_t err;
@@ -274,13 +291,19 @@ esp_err_t esp_camera_init(const camera_config_t *config)
 
     s_state->sensor.status.framesize = frame_size;
     s_state->sensor.pixformat = pix_format;
+
     ESP_LOGD(TAG, "Setting frame size to %dx%d", resolution[frame_size].width, resolution[frame_size].height);
     if (s_state->sensor.set_framesize(&s_state->sensor, frame_size) != 0) {
         ESP_LOGE(TAG, "Failed to set frame size");
         err = ESP_ERR_CAMERA_FAILED_TO_SET_FRAME_SIZE;
         goto fail;
     }
     s_state->sensor.set_pixformat(&s_state->sensor, pix_format);
+#if CONFIG_CAMERA_CONVERTER_ENABLED
+    if(config->conv_mode) {
+        s_state->sensor.pixformat = get_output_data_format(config->conv_mode); // If conversion enabled, change the out data format by conversion mode
+    }
+#endif
 
     if (s_state->sensor.id.PID == OV2640_PID) {
         s_state->sensor.set_gainceiling(&s_state->sensor, GAINCEILING_2X);

driver/include/esp_camera.h

@@ -70,6 +70,7 @@
 #include "driver/ledc.h"
 #include "sensor.h"
 #include "sys/time.h"
+#include "sdkconfig.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -91,6 +92,19 @@ typedef enum {
     CAMERA_FB_IN_DRAM           /*!< Frame buffer is placed in internal DRAM */
 } camera_fb_location_t;
 
+#if CONFIG_CAMERA_CONVERTER_ENABLED
+/**
+ * @brief Camera RGB\YUV conversion mode
+ */
+typedef enum {
+    CONV_DISABLE,
+    RGB565_TO_YUV422,
+        
+    YUV422_TO_RGB565,
+    YUV422_TO_YUV420
+} camera_conv_mode_t;
+#endif
+
 /**
  * @brief Configuration structure for camera initialization
  */
@@ -124,6 +138,9 @@ typedef struct {
     size_t fb_count;                /*!< Number of frame buffers to be allocated. If more than one, then each frame will be acquired (double speed)  */
     camera_fb_location_t fb_location; /*!< The location where the frame buffer will be allocated */
     camera_grab_mode_t grab_mode;   /*!< When buffers should be filled */
+#if CONFIG_CAMERA_CONVERTER_ENABLED
+    camera_conv_mode_t conv_mode;   /*!< RGB<->YUV Conversion mode */
+#endif
 } camera_config_t;
 
 /**

driver/include/sensor.h

@@ -69,6 +69,7 @@ typedef enum {
 typedef enum {
     PIXFORMAT_RGB565,    // 2BPP/RGB565
     PIXFORMAT_YUV422,    // 2BPP/YUV422
+    PIXFORMAT_YUV420,    // 1.5BPP/YUV420
     PIXFORMAT_GRAYSCALE, // 1BPP/GRAYSCALE
     PIXFORMAT_JPEG,      // JPEG/COMPRESSED
     PIXFORMAT_RGB888,    // 3BPP/RGB888

target/esp32s3/ll_cam.c

@@ -175,6 +175,7 @@ static esp_err_t ll_cam_dma_init(cam_obj_t *cam)
     }
 
     GDMA.channel[cam->dma_num].in.conf1.in_check_owner = 0;
+    // GDMA.channel[cam->dma_num].in.conf1.in_ext_mem_bk_size = 2;
 
     GDMA.channel[cam->dma_num].in.peri_sel.sel = 5;
     //GDMA.channel[cam->dma_num].in.pri.rx_pri = 1;//rx prio 0-15
@@ -183,8 +184,52 @@ static esp_err_t ll_cam_dma_init(cam_obj_t *cam)
     return ESP_OK;
 }
 
+#if CONFIG_CAMERA_CONVERTER_ENABLED
+static esp_err_t ll_cam_converter_config(cam_obj_t *cam, const camera_config_t *config)
+{
+    esp_err_t ret = ESP_OK;
+
+    switch (config->conv_mode) {
+    case YUV422_TO_YUV420:
+        if (config->pixel_format != PIXFORMAT_YUV422) {
+            ret = ESP_FAIL;
+        } else {
+            ESP_LOGI(TAG, "YUV422 to YUV420 mode");
+            LCD_CAM.cam_rgb_yuv.cam_conv_yuv2yuv_mode = 1;
+            LCD_CAM.cam_rgb_yuv.cam_conv_yuv_mode = 0;
+            LCD_CAM.cam_rgb_yuv.cam_conv_trans_mode = 1;
+        }
+        break;
+    case YUV422_TO_RGB565:
+        if (config->pixel_format != PIXFORMAT_YUV422) {
+            ret = ESP_FAIL;
+        } else {
+            ESP_LOGI(TAG, "YUV422 to RGB565 mode");
+            LCD_CAM.cam_rgb_yuv.cam_conv_yuv2yuv_mode = 3;
+            LCD_CAM.cam_rgb_yuv.cam_conv_yuv_mode = 0;
+            LCD_CAM.cam_rgb_yuv.cam_conv_trans_mode = 0;
+        }
+        break;
+    default:
+        break;
+    }
+#if CONFIG_LCD_CAM_CONV_BT709_ENABLED
+    LCD_CAM.cam_rgb_yuv.cam_conv_protocol_mode = 1;
+#else
+    LCD_CAM.cam_rgb_yuv.cam_conv_protocol_mode = 0;
+#endif
+    LCD_CAM.cam_rgb_yuv.cam_conv_data_out_mode = 0;
+    LCD_CAM.cam_rgb_yuv.cam_conv_data_in_mode = 0;
+    LCD_CAM.cam_rgb_yuv.cam_conv_mode_8bits_on = 1;
+    LCD_CAM.cam_rgb_yuv.cam_conv_bypass = 1;
+    cam->conv_mode = config->conv_mode;
+    return ret;
+}
+#endif
+
 esp_err_t ll_cam_config(cam_obj_t *cam, const camera_config_t *config)
 {
+    esp_err_t ret = ESP_OK;
     if (REG_GET_BIT(SYSTEM_PERIP_CLK_EN1_REG, SYSTEM_LCD_CAM_CLK_EN) == 0) {
         REG_CLR_BIT(SYSTEM_PERIP_CLK_EN1_REG, SYSTEM_LCD_CAM_CLK_EN);
         REG_SET_BIT(SYSTEM_PERIP_CLK_EN1_REG, SYSTEM_LCD_CAM_CLK_EN);
@@ -220,15 +265,21 @@ esp_err_t ll_cam_config(cam_obj_t *cam, const camera_config_t *config)
 
     LCD_CAM.cam_rgb_yuv.val = 0;
 
+#if CONFIG_CAMERA_CONVERTER_ENABLED
+    if (config->conv_mode) {
+        ret = ll_cam_converter_config(cam, config);
+        if(ret != ESP_OK) {
+            return ret;
+        }
+    }
+#endif
+
     LCD_CAM.cam_ctrl.cam_update = 1;
     LCD_CAM.cam_ctrl1.cam_start = 1;
 
-    esp_err_t err = ll_cam_dma_init(cam);
-    if(err != ESP_OK) {
-        return err;
-    }
+    ret = ll_cam_dma_init(cam);
 
-    return ESP_OK;
+    return ret;
 }
 
 void ll_cam_vsync_intr_enable(cam_obj_t *cam, bool en)
@@ -422,6 +473,7 @@ size_t IRAM_ATTR ll_cam_memcpy(cam_obj_t *cam, uint8_t *out, const uint8_t *in,
         }
         return len / 2;
     }
+    
 
     // just memcpy
     memcpy(out, in, len);
@@ -438,8 +490,22 @@ esp_err_t ll_cam_set_sample_mode(cam_obj_t *cam, pixformat_t pix_format, uint32_
         }
         cam->fb_bytes_per_pixel = 1;       // frame buffer stores Y8
     } else if (pix_format == PIXFORMAT_YUV422 || pix_format == PIXFORMAT_RGB565) {
-            cam->in_bytes_per_pixel = 2;       // camera sends YU/YV
+#if CONFIG_CAMERA_CONVERTER_ENABLED
+        switch (cam->conv_mode) {
+        case YUV422_TO_YUV420:
+            cam->in_bytes_per_pixel = 1.5;       // for DMA receive
+            cam->fb_bytes_per_pixel = 1.5;       // frame buffer stores YUV420
+            break;
+        case YUV422_TO_RGB565:
+        default:
+            cam->in_bytes_per_pixel = 2;       // for DMA receive
             cam->fb_bytes_per_pixel = 2;       // frame buffer stores YU/YV/RGB565
+            break;
+        }
+#else 
+        cam->in_bytes_per_pixel = 2;       // for DMA receive
+        cam->fb_bytes_per_pixel = 2;       // frame buffer stores YU/YV/RGB565
+#endif
     } else if (pix_format == PIXFORMAT_JPEG) {
         cam->in_bytes_per_pixel = 1;
         cam->fb_bytes_per_pixel = 1;

target/private_include/ll_cam.h

@@ -116,8 +116,14 @@ typedef struct {
     //for RGB/YUV modes
     uint16_t width;
     uint16_t height;
+#if CONFIG_CAMERA_CONVERTER_ENABLED
+    float in_bytes_per_pixel;
+    float fb_bytes_per_pixel;
+    camera_conv_mode_t conv_mode;
+#else 
     uint8_t in_bytes_per_pixel;
     uint8_t fb_bytes_per_pixel;
+#endif
     uint32_t fb_size;
 
     cam_state_t state;