Add Video class support

examples/Video/video_capture/usb_descriptors.h

@@ -0,0 +1,152 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2020 Jerzy Kasenbreg
+ * Copyright (c) 2021 Koji KITAYAMA
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+#ifndef _USB_DESCRIPTORS_H_
+#define _USB_DESCRIPTORS_H_
+
+/* Time stamp base clock. It is a deprecated parameter. */
+#define UVC_CLOCK_FREQUENCY    27000000
+/* video capture path */
+#define UVC_ENTITY_CAP_INPUT_TERMINAL  0x01
+#define UVC_ENTITY_CAP_OUTPUT_TERMINAL 0x02
+
+enum {
+  ITF_NUM_VIDEO_CONTROL,
+  ITF_NUM_VIDEO_STREAMING,
+  ITF_NUM_TOTAL
+};
+
+#define TUD_VIDEO_CAPTURE_DESC_UNCOMPR_LEN (\
+    TUD_VIDEO_DESC_IAD_LEN\
+    /* control */\
+    + TUD_VIDEO_DESC_STD_VC_LEN\
+    + (TUD_VIDEO_DESC_CS_VC_LEN + 1/*bInCollection*/)\
+    + TUD_VIDEO_DESC_CAMERA_TERM_LEN\
+    + TUD_VIDEO_DESC_OUTPUT_TERM_LEN\
+    /* Interface 1, Alternate 0 */\
+    + TUD_VIDEO_DESC_STD_VS_LEN\
+    + (TUD_VIDEO_DESC_CS_VS_IN_LEN + 1/*bNumFormats x bControlSize*/)\
+    + TUD_VIDEO_DESC_CS_VS_FMT_UNCOMPR_LEN\
+    + TUD_VIDEO_DESC_CS_VS_FRM_UNCOMPR_CONT_LEN\
+    + TUD_VIDEO_DESC_CS_VS_COLOR_MATCHING_LEN\
+    /* Interface 1, Alternate 1 */\
+    + TUD_VIDEO_DESC_STD_VS_LEN\
+    + 7/* Endpoint */\
+  )
+
+#define TUD_VIDEO_CAPTURE_DESC_MJPEG_LEN (\
+    TUD_VIDEO_DESC_IAD_LEN\
+    /* control */\
+    + TUD_VIDEO_DESC_STD_VC_LEN\
+    + (TUD_VIDEO_DESC_CS_VC_LEN + 1/*bInCollection*/)\
+    + TUD_VIDEO_DESC_CAMERA_TERM_LEN\
+    + TUD_VIDEO_DESC_OUTPUT_TERM_LEN\
+    /* Interface 1, Alternate 0 */\
+    + TUD_VIDEO_DESC_STD_VS_LEN\
+    + (TUD_VIDEO_DESC_CS_VS_IN_LEN + 1/*bNumFormats x bControlSize*/)\
+    + TUD_VIDEO_DESC_CS_VS_FMT_MJPEG_LEN\
+    + TUD_VIDEO_DESC_CS_VS_FRM_MJPEG_CONT_LEN\
+    + TUD_VIDEO_DESC_CS_VS_COLOR_MATCHING_LEN\
+    /* Interface 1, Alternate 1 */\
+    + TUD_VIDEO_DESC_STD_VS_LEN\
+    + 7/* Endpoint */\
+  )
+
+#define TUD_VIDEO_CAPTURE_DESC_UNCOMPR_BULK_LEN (\
+    TUD_VIDEO_DESC_IAD_LEN\
+    /* control */\
+    + TUD_VIDEO_DESC_STD_VC_LEN\
+    + (TUD_VIDEO_DESC_CS_VC_LEN + 1/*bInCollection*/)\
+    + TUD_VIDEO_DESC_CAMERA_TERM_LEN\
+    + TUD_VIDEO_DESC_OUTPUT_TERM_LEN\
+    /* Interface 1, Alternate 0 */\
+    + TUD_VIDEO_DESC_STD_VS_LEN\
+    + (TUD_VIDEO_DESC_CS_VS_IN_LEN + 1/*bNumFormats x bControlSize*/)\
+    + TUD_VIDEO_DESC_CS_VS_FMT_UNCOMPR_LEN\
+    + TUD_VIDEO_DESC_CS_VS_FRM_UNCOMPR_CONT_LEN\
+    + TUD_VIDEO_DESC_CS_VS_COLOR_MATCHING_LEN\
+    + 7/* Endpoint */\
+  )
+
+#define TUD_VIDEO_CAPTURE_DESC_MJPEG_BULK_LEN (\
+    TUD_VIDEO_DESC_IAD_LEN\
+    /* control */\
+    + TUD_VIDEO_DESC_STD_VC_LEN\
+    + (TUD_VIDEO_DESC_CS_VC_LEN + 1/*bInCollection*/)\
+    + TUD_VIDEO_DESC_CAMERA_TERM_LEN\
+    + TUD_VIDEO_DESC_OUTPUT_TERM_LEN\
+    /* Interface 1, Alternate 0 */\
+    + TUD_VIDEO_DESC_STD_VS_LEN\
+    + (TUD_VIDEO_DESC_CS_VS_IN_LEN + 1/*bNumFormats x bControlSize*/)\
+    + TUD_VIDEO_DESC_CS_VS_FMT_MJPEG_LEN\
+    + TUD_VIDEO_DESC_CS_VS_FRM_MJPEG_CONT_LEN\
+    + TUD_VIDEO_DESC_CS_VS_COLOR_MATCHING_LEN\
+    + 7/* Endpoint */\
+  )
+
+/* Windows support YUY2 and NV12
+ * https://docs.microsoft.com/en-us/windows-hardware/drivers/stream/usb-video-class-driver-overview */
+
+#define TUD_VIDEO_DESC_CS_VS_FMT_YUY2(_fmtidx, _numfmtdesc, _frmidx, _asrx, _asry, _interlace, _cp) \
+  TUD_VIDEO_DESC_CS_VS_FMT_UNCOMPR(_fmtidx, _numfmtdesc, TUD_VIDEO_GUID_YUY2, 16, _frmidx, _asrx, _asry, _interlace, _cp)
+#define TUD_VIDEO_DESC_CS_VS_FMT_NV12(_fmtidx, _numfmtdesc, _frmidx, _asrx, _asry, _interlace, _cp) \
+  TUD_VIDEO_DESC_CS_VS_FMT_UNCOMPR(_fmtidx, _numfmtdesc, TUD_VIDEO_GUID_NV12, 12, _frmidx, _asrx, _asry, _interlace, _cp)
+#define TUD_VIDEO_DESC_CS_VS_FMT_M420(_fmtidx, _numfmtdesc, _frmidx, _asrx, _asry, _interlace, _cp) \
+  TUD_VIDEO_DESC_CS_VS_FMT_UNCOMPR(_fmtidx, _numfmtdesc, TUD_VIDEO_GUID_M420, 12, _frmidx, _asrx, _asry, _interlace, _cp)
+#define TUD_VIDEO_DESC_CS_VS_FMT_I420(_fmtidx, _numfmtdesc, _frmidx, _asrx, _asry, _interlace, _cp) \
+  TUD_VIDEO_DESC_CS_VS_FMT_UNCOMPR(_fmtidx, _numfmtdesc, TUD_VIDEO_GUID_I420, 12, _frmidx, _asrx, _asry, _interlace, _cp)
+
+
+#define TUD_VIDEO_CAPTURE_DESCRIPTOR_UNCOMPR_BULK(_stridx, _epin, _width, _height, _fps, _epsize) \
+  TUD_VIDEO_DESC_IAD(ITF_NUM_VIDEO_CONTROL, /* 2 Interfaces */ 0x02, _stridx), \
+  /* Video control 0 */ \
+  TUD_VIDEO_DESC_STD_VC(ITF_NUM_VIDEO_CONTROL, 0, _stridx), \
+    /* Header: UVC 1.5, wTotalLength - bLength */ \
+    TUD_VIDEO_DESC_CS_VC(0x0150, TUD_VIDEO_DESC_CAMERA_TERM_LEN + TUD_VIDEO_DESC_OUTPUT_TERM_LEN, UVC_CLOCK_FREQUENCY, ITF_NUM_VIDEO_STREAMING), \
+      /* Camera Terminal: ID, bAssocTerminal, iTerminal, focal min, max, length, bmControl */                                                                                              \
+      TUD_VIDEO_DESC_CAMERA_TERM(UVC_ENTITY_CAP_INPUT_TERMINAL, 0, 0, 0, 0, 0, 0), \
+      TUD_VIDEO_DESC_OUTPUT_TERM(UVC_ENTITY_CAP_OUTPUT_TERMINAL, VIDEO_TT_STREAMING, 0, 1, 0), \
+  /* Video stream alt. 0 */ \
+  TUD_VIDEO_DESC_STD_VS(ITF_NUM_VIDEO_STREAMING, 0, 1, _stridx), \
+    /* Video stream header for without still image capture */ \
+    TUD_VIDEO_DESC_CS_VS_INPUT( /*bNumFormats*/1, \
+        /*wTotalLength - bLength */\
+        TUD_VIDEO_DESC_CS_VS_FMT_UNCOMPR_LEN + TUD_VIDEO_DESC_CS_VS_FRM_UNCOMPR_CONT_LEN + TUD_VIDEO_DESC_CS_VS_COLOR_MATCHING_LEN,\
+        _epin, /*bmInfo*/0, /*bTerminalLink*/UVC_ENTITY_CAP_OUTPUT_TERMINAL, \
+        /*bStillCaptureMethod*/0, /*bTriggerSupport*/0, /*bTriggerUsage*/0, \
+        /*bmaControls(1)*/0), \
+      /* Video stream format */ \
+      TUD_VIDEO_DESC_CS_VS_FMT_YUY2(/*bFormatIndex*/1, /*bNumFrameDescriptors*/1, \
+        /*bDefaultFrameIndex*/1, 0, 0, 0, /*bCopyProtect*/0), \
+        /* Video stream frame format */ \
+        TUD_VIDEO_DESC_CS_VS_FRM_UNCOMPR_CONT(/*bFrameIndex */1, 0, _width, _height, \
+            _width * _height * 16, _width * _height * 16 * _fps, \
+            _width * _height * 16, \
+            (10000000/_fps), (10000000/_fps), (10000000/_fps)*_fps, (10000000/_fps)), \
+        TUD_VIDEO_DESC_CS_VS_COLOR_MATCHING(VIDEO_COLOR_PRIMARIES_BT709, VIDEO_COLOR_XFER_CH_BT709, VIDEO_COLOR_COEF_SMPTE170M), \
+        TUD_VIDEO_DESC_EP_BULK(_epin, _epsize, 1)
+
+#endif

examples/Video/video_capture/video_capture.ino

@@ -0,0 +1,124 @@
+/*********************************************************************
+ Adafruit invests time and resources providing this open source code,
+ please support Adafruit and open-source hardware by purchasing
+ products from Adafruit!
+
+ MIT license, check LICENSE for more information
+ Copyright (c) 2019 Ha Thach for Adafruit Industries
+ All text above, and the splash screen below must be included in
+ any redistribution
+*********************************************************************/
+
+#include "Adafruit_TinyUSB.h"
+#include "usb_descriptors.h"
+
+//--------------------------------------------------------------------+
+//
+//--------------------------------------------------------------------+
+#define FRAME_WIDTH   128
+#define FRAME_HEIGHT  96
+#define FRAME_RATE    30
+
+uint8_t const desc_video[] = {
+  TUD_VIDEO_CAPTURE_DESCRIPTOR_UNCOMPR_BULK(0, 0x80, FRAME_WIDTH, FRAME_HEIGHT, FRAME_RATE, 64)
+};
+
+Adafruit_USBD_Video usb_video(desc_video, sizeof(desc_video));
+
+// YUY2 frame buffer
+static uint8_t frame_buffer[FRAME_WIDTH * FRAME_HEIGHT * 16 / 8];
+
+static unsigned frame_num = 0;
+static unsigned tx_busy = 0;
+static unsigned interval_ms = 1000 / FRAME_RATE;
+static unsigned start_ms = 0;
+static unsigned already_sent = 0;
+
+//--------------------------------------------------------------------+
+//
+//--------------------------------------------------------------------+
+static void fill_color_bar(uint8_t* buffer, unsigned start_position);
+
+void setup() {
+  Serial.begin(115200);
+  usb_video.begin();
+}
+
+void loop() {
+  if (!tud_video_n_streaming(0, 0)) {
+    already_sent = 0;
+    frame_num = 0;
+    return;
+  }
+
+  if (!already_sent) {
+    already_sent = 1;
+    start_ms = millis();
+    fill_color_bar(frame_buffer, frame_num);
+    tud_video_n_frame_xfer(0, 0, (void*) frame_buffer, FRAME_WIDTH * FRAME_HEIGHT * 16 / 8);
+  }
+
+  unsigned cur = millis();
+  if (cur - start_ms < interval_ms) return; // not enough time
+  if (tx_busy) return;
+  start_ms += interval_ms;
+
+  fill_color_bar(frame_buffer, frame_num);
+  tud_video_n_frame_xfer(0, 0, (void*) frame_buffer, FRAME_WIDTH * FRAME_HEIGHT * 16 / 8);
+}
+
+void tud_video_frame_xfer_complete_cb(uint_fast8_t ctl_idx, uint_fast8_t stm_idx) {
+  (void) ctl_idx;
+  (void) stm_idx;
+  tx_busy = 0;
+  /* flip buffer */
+  ++frame_num;
+}
+
+int tud_video_commit_cb(uint_fast8_t ctl_idx, uint_fast8_t stm_idx,
+                        video_probe_and_commit_control_t const* parameters) {
+  (void) ctl_idx;
+  (void) stm_idx;
+  /* convert unit to ms from 100 ns */
+  interval_ms = parameters->dwFrameInterval / 10000;
+  return VIDEO_ERROR_NONE;
+}
+
+//------------- Helper -------------//
+static void fill_color_bar(uint8_t* buffer, unsigned start_position) {
+  /* EBU color bars
+   * See also https://stackoverflow.com/questions/6939422 */
+  static uint8_t const bar_color[8][4] = {
+      /*  Y,   U,   Y,   V */
+      { 235, 128, 235, 128}, /* 100% White */
+      { 219,  16, 219, 138}, /* Yellow */
+      { 188, 154, 188,  16}, /* Cyan */
+      { 173,  42, 173,  26}, /* Green */
+      {  78, 214,  78, 230}, /* Magenta */
+      {  63, 102,  63, 240}, /* Red */
+      {  32, 240,  32, 118}, /* Blue */
+      {  16, 128,  16, 128}, /* Black */
+  };
+  uint8_t* p;
+
+  /* Generate the 1st line */
+  uint8_t* end = &buffer[FRAME_WIDTH * 2];
+  unsigned idx = (FRAME_WIDTH / 2 - 1) - (start_position % (FRAME_WIDTH / 2));
+  p = &buffer[idx * 4];
+  for (unsigned i = 0; i < 8; ++i) {
+    for (int j = 0; j < FRAME_WIDTH / (2 * 8); ++j) {
+      memcpy(p, &bar_color[i], 4);
+      p += 4;
+      if (end <= p) {
+        p = buffer;
+      }
+    }
+  }
+
+  /* Duplicate the 1st line to the others */
+  p = &buffer[FRAME_WIDTH * 2];
+  for (unsigned i = 1; i < FRAME_HEIGHT; ++i) {
+    memcpy(p, buffer, FRAME_WIDTH * 2);
+    p += FRAME_WIDTH * 2;
+  }
+}

src/Adafruit_TinyUSB.h

@@ -64,6 +64,10 @@
 #include "arduino/webusb/Adafruit_USBD_WebUSB.h"
 #endif
 
+#if CFG_TUD_VIDEO
+#include "arduino/video/Adafruit_USBD_Video.h"
+#endif
+
 // Initialize device hardware, stack, also Serial as CDC
 // Wrapper for TinyUSBDevice.begin(rhport)
 void TinyUSB_Device_Init(uint8_t rhport);

src/arduino/Adafruit_USBD_Device.cpp

@@ -244,11 +244,49 @@ bool Adafruit_USBD_Device::addInterface(Adafruit_USBD_Interface &itf) {
   }
 
   // Parse interface descriptor to update
-  // - Interface Number & string descriptor
-  // - Endpoint address
+  // - IAD: interface number
+  // - Interface: number & string descriptor
+  // - Endpoint: address
   while (desc < desc_end) {
-    if (tu_desc_type(desc) == TUSB_DESC_INTERFACE) {
+    switch (tu_desc_type(desc)) {
+    case TUSB_DESC_INTERFACE_ASSOCIATION: {
+      tusb_desc_interface_assoc_t *desc_iad =
+          (tusb_desc_interface_assoc_t *)desc;
+      desc_iad->bFirstInterface = _itf_count;
+      break;
+    }
+
+    case TUSB_DESC_INTERFACE: {
       tusb_desc_interface_t *desc_itf = (tusb_desc_interface_t *)desc;
+      desc_itf->bInterfaceNumber = _itf_count;
+
+#if CFG_TUD_VIDEO && CFG_TUD_VIDEO_STREAMING
+      if (TUSB_CLASS_VIDEO == desc_itf->bInterfaceClass) {
+        desc += tu_desc_len(desc); // next to CS Interface
+
+        if (TUSB_DESC_CS_INTERFACE == tu_desc_type(desc)) {
+          uint8_t const subtype = desc[2];
+
+          if (VIDEO_SUBCLASS_CONTROL == desc_itf->bInterfaceSubClass) {
+            // Adjust stream interface number in VC Header
+            if (subtype == VIDEO_CS_ITF_VC_HEADER) {
+              uint8_t const vs_count = desc[11];
+              for (uint8_t i = 0; i < vs_count; i++) {
+                desc[12 + i] += _itf_count;
+              }
+            }
+          } else if (VIDEO_SUBCLASS_STREAMING == desc_itf->bInterfaceSubClass) {
+            // Adjust the endpoint address in CS VS Input/Output Header
+            if (subtype == VIDEO_CS_ITF_VS_INPUT_HEADER) {
+              desc[6] = 0x80 | _epin_count;
+            } else if (subtype == VIDEO_CS_ITF_VS_OUTPUT_HEADER) {
+              desc[6] = _epout_count;
+            }
+          }
+        }
+      }
+#endif
+
       if (desc_itf->bAlternateSetting == 0) {
         _itf_count++;
         if (desc_str && (_desc_str_count < STRING_DESCRIPTOR_MAX)) {
@@ -258,12 +296,23 @@ bool Adafruit_USBD_Device::addInterface(Adafruit_USBD_Interface &itf) {
 
           // only assign string index to first interface
           desc_str = NULL;
+        } else {
+          desc_itf->iInterface = 0;
         }
       }
-    } else if (tu_desc_type(desc) == TUSB_DESC_ENDPOINT) {
+
+      break;
+    }
+
+    case TUSB_DESC_ENDPOINT: {
       tusb_desc_endpoint_t *desc_ep = (tusb_desc_endpoint_t *)desc;
       desc_ep->bEndpointAddress |=
           (desc_ep->bEndpointAddress & 0x80) ? _epin_count++ : _epout_count++;
+      break;
+    }
+
+    default:
+      break;
     }
 
     if (desc[0] == 0) {