USB Host UVC driver version 2

.build-test-rules.yml

@@ -6,15 +6,23 @@ host/class:
   enable:
     - if: SOC_USB_OTG_SUPPORTED == 1
 
+host/class/uvc/usb_host_uvc/examples/camera_display:
+  enable:
+    - if: (IDF_VERSION_MAJOR >= 5 and IDF_VERSION_MINOR >= 3)
+      reason: This example uses esp_lcd API introduced in v5.3
+
 # Host tests
 host/class/cdc/usb_host_cdc_acm/host_test:
   enable:
     - if: IDF_TARGET in ["linux"] and (IDF_VERSION_MAJOR >= 5 and IDF_VERSION_MINOR >= 4)
 
+
 host/class/hid/usb_host_hid/host_test:
   enable:
     - if: IDF_TARGET in ["linux"] and (IDF_VERSION_MAJOR >= 5 and IDF_VERSION_MINOR >= 4)
+      reason: USB mock was added in v5.4
 
 host/class/uvc/usb_host_uvc/host_test:
   enable:
-    - if: IDF_TARGET in ["linux"] and (IDF_VERSION_MAJOR >= 5 and IDF_VERSION_MINOR >= 4)
+    - if: IDF_TARGET in ["linux"] and (IDF_VERSION_MAJOR >= 5 and IDF_VERSION_MINOR >= 4)
+      reason: USB mock was added in v5.4

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "host/usb_host_uvc/libuvc"]
-	path = host/class/uvc/usb_host_uvc/libuvc
-	url = ../../espressif/libuvc.git

host/class/uvc/usb_host_uvc/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.0.0
+
+- New version of the driver, native to Espressif's USB Host Library
+- Removed libuvc dependency
+
 ## 1.0.4
 
 - Support printf frame based descriptor

host/class/uvc/usb_host_uvc/CMakeLists.txt

@@ -1,28 +1,13 @@
-configure_file(${CMAKE_CURRENT_LIST_DIR}/libuvc/include/libuvc/libuvc_config.h.in
-               ${CMAKE_CURRENT_BINARY_DIR}/include/libuvc/libuvc_config.h
-               @ONLY)
-
-set(LIBUVC_SOURCES libuvc/src/ctrl.c
-                   libuvc/src/ctrl-gen.c
-                   libuvc/src/device.c
-                   libuvc/src/diag.c
-                   libuvc/src/frame.c
-                   libuvc/src/init.c
-                   libuvc/src/misc.c
-                   libuvc/src/stream.c)
-
-idf_component_register(
-    SRCS ${LIBUVC_SOURCES} src/descriptor.c src/libusb_adapter.c
-    INCLUDE_DIRS include libuvc/include
-    PRIV_INCLUDE_DIRS private_include
-    REQUIRES usb pthread)
-
-set_source_files_properties(
-    ${CMAKE_CURRENT_LIST_DIR}/libuvc/src/device.c PROPERTIES COMPILE_FLAGS -Wno-implicit-fallthrough)
-set_source_files_properties(
-    ${CMAKE_CURRENT_LIST_DIR}/libuvc/src/stream.c PROPERTIES COMPILE_FLAGS -Wno-unused-variable)
-    set_source_files_properties(
-        ${CMAKE_CURRENT_LIST_DIR}/libuvc/src/diag.c PROPERTIES COMPILE_FLAGS -Wno-format)
-
-target_compile_definitions(${COMPONENT_LIB} PRIVATE LIBUVC_NUM_TRANSFER_BUFS=4)
-target_include_directories(${COMPONENT_LIB} PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/include/)
+idf_component_register(SRCS
+                        "uvc_host.c"
+                        "uvc_descriptor_parsing.c"
+                        "uvc_descriptor_printing.c"
+                        "uvc_frame.c"
+                        "uvc_control.c"
+                        "uvc_isoc.c"
+                        "uvc_bulk.c"
+                       INCLUDE_DIRS include
+                       PRIV_INCLUDE_DIRS private_include include/esp_private
+                       PRIV_REQUIRES heap
+                       REQUIRES usb
+                       )

host/class/uvc/usb_host_uvc/LICENCE → host/class/uvc/usb_host_uvc/LICENSE

@@ -1,3 +1,4 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/

host/class/uvc/usb_host_uvc/README.md

@@ -1,24 +1,26 @@
-# USB Host UVC Driver
+# USB Host UVC Class Driver
 
 [![Component Registry](https://components.espressif.com/components/espressif/usb_host_uvc/badge.svg)](https://components.espressif.com/components/espressif/usb_host_uvc)
+![maintenance-status](https://img.shields.io/badge/maintenance-experimental-blue.svg)
 
-This directory contains USB host UVC driver based on [libuvc](https://github.com/libuvc/libuvc) library. Support for `libuvc` is achieved by implementing adapter between [libusb](https://github.com/libusb/libusb) (underling host library used by `libuvc`) and [usb_host](https://docs.espressif.com/projects/esp-idf/en/latest/esp32s2/api-reference/peripherals/usb_host.html) library targeted for ESP SOCs.    
+This component contains an implementation of a USB Host UVC Class Driver that is implemented on top of the [USB Host Library](https://docs.espressif.com/projects/esp-idf/en/latest/esp32s2/api-reference/peripherals/usb_host.html).
 
-## Usage
+The UVC driver allows video streaming from USB cameras.
 
-Reference [uvc_host_example](https://github.com/espressif/esp-idf/tree/master/examples/peripherals/usb/host/uvc) is similar to one found in `libuvc` repository with few additions:
-1. Before calling `uvc_init()`, `initialize_usb_host_lib()` has to be called in order to initialize usb host library.
-2. Since `libuvc` selects highest possible `dwMaxPayloadTransferSize` by default, user has to manually overwrite obatained value to 512 bytes (maximum transfer size supported by ESP32-S2/S3) before passing it to `uvc_print_stream_ctrl()` function.
-3. Optionally, user can configure `libusb adapter` by passing appropriate parameters to `libuvc_adapter_set_config()`.
+### Features
+- Isochronous and Bulk transfers streaming
+- Multiple video streams
+- Frame buffers in PSRAM
+- Video Stream format negotiation
+- Stream overflow and underflow management
 
-## Known limitations
+### Usage
 
-Having only Full Speed USB peripheral and hardware limited MPS (maximum packet size) to 512 bytes, ESP32-S2/S3 is capable of reading about 0.5 MB of data per second. When connected to Full Speed USB host, cameras normally provide resolution no larger than 640x480 pixels. 
-Following two supported formats are the most common (both encoded in MJPEG):
- * 320x240  30 FPS
- * 640x480  15 FPS
+Following sequence diagram represents public API usage of the UVC driver. New frames are passed to user in a callback. This design offers flexible interface upon which more complex frame processing components can be built.
 
-## Tested cameras
- * Logitech C980
- * CANYON CNE-CWC2
- * Logitech C270
+![UVC public API](docs/uvc_public_api.png)
+
+### Additional information
+- [Frequently Asked Questions](docs/FAQ.md)
+- [Examples](examples/)
+- [Architectural notes](docs/arch_notes.md)

host/class/uvc/usb_host_uvc/docs/FAQ.md

@@ -0,0 +1,58 @@
+## FAQ H265 encoding
+
+Q1: I have two input frames inputI.hevc, which is full Intra-coded frame and a second inputP.hevc which is Predicted frame. How do I decode the second frame to png?
+
+---
+
+### Decoding a P-frame Using an I-frame in HEVC
+
+To decode a P-frame using the corresponding I-frame, you need to combine both frames into a single stream since a P-frame requires the reference frame (I-frame) to be properly decoded. Here’s how you can do it using FFmpeg:
+
+#### Step 1: Combine the I-frame and P-frame into a Single Stream
+
+1. **Concatenate the frames**: Create a text file with the paths to the input HEVC files in the order they should be decoded.
+
+   Create a file `inputs.txt` with the following content:
+   ```plaintext
+   file 'inputI.hevc'
+   file 'inputP.hevc'
+   ```
+
+2. **Concatenate the HEVC files using FFmpeg**:
+   ```sh
+   ffmpeg -f concat -safe 0 -i inputs.txt -c copy combined.hevc
+   ```
+
+#### Step 2: Extract and Decode the Second Frame
+
+1. **Decode the combined HEVC stream and extract the second frame**:
+   ```sh
+   ffmpeg -i combined.hevc -vf "select=eq(n\,1)" -vsync vfr second_frame.png
+   ```
+
+#### Explanation
+
+- The `-f concat -safe 0 -i inputs.txt -c copy combined.hevc` command concatenates the I-frame and P-frame into a single HEVC file.
+- The `-vf "select=eq(n\,1)" -vsync vfr second_frame.png` command extracts and decodes the second frame (P-frame) from the combined stream.
+
+#### Complete Command Sequence
+
+1. **Create `inputs.txt`**:
+   ```plaintext
+   file 'inputI.hevc'
+   file 'inputP.hevc'
+   ```
+
+2. **Combine the HEVC files**:
+   ```sh
+   ffmpeg -f concat -safe 0 -i inputs.txt -c copy combined.hevc
+   ```
+
+3. **Extract and decode the second frame**:
+   ```sh
+   ffmpeg -i combined.hevc -vf "select=eq(n\,1)" -vsync vfr second_frame.png
+   ```
+
+By following these steps, you should be able to decode the second frame (P-frame) using the initial I-frame and convert it to PNG.
+
+---

host/class/uvc/usb_host_uvc/docs/arch_notes.md

@@ -0,0 +1,38 @@
+## Architectural Notes
+
+This driver utilizes two distinct types of memory buffers, both of which can be configured per UVC stream during stream initialization via the `uvc_host_stream_config_t.advanced` structure. The buffer types are as follows:
+
+### 1. URB (USB Request Block)
+- **Definition:** Provided by the USB Host Library in ESP-IDF.
+- **Ownership:** Managed by the USB Host Library.
+- **Placement:** Determined by ESP-IDF settings (internal or external RAM).
+- **Purpose:** Transfers raw data from the USB device to the ESP host.
+- **Behavior:** 
+  - The driver continuously resubmits the URB until streaming stops.
+  - After submission, the CPU is interrupted when the URB is completed. 
+  - The buffer size impacts interrupt frequency—larger buffers result in fewer interrupts.
+- **Recommendation:** Use triple buffering for optimal performance:
+  - One buffer is processed by the driver.
+  - One buffer is actively transferring data.
+  - One buffer is queued for submission.
+- **Driver Role:** Processes data from the URB and reconstructs video frames.
+
+### 2. FB (Frame Buffer)
+- **Definition:** Custom buffer type defined within this driver.
+- **Ownership:** Dynamic ownership:
+  - Empty FBs are owned by the driver.
+  - Once a full frame is reconstructed and stored in the FB, it is passed to the user via `uvc_host_frame_callback_t`.
+  - After processing the frame, the user must return the FB to the driver, either by:
+    - Returning `true` from the callback, or
+    - Explicitly calling `uvc_host_frame_return()`.
+- **Recommendation:** Use triple buffering for optimal performance:
+  - One buffer is used for frame reconstruction.
+  - One buffer is processed by the user.
+  - One buffer is queued for use.
+- **Frame Size Considerations:** 
+  - UVC cameras report the maximum frame buffer size during stream format negotiation.
+  - These sizes are often overly large, leading to inefficient RAM usage.
+  - This driver allows the allocation of smaller FBs to optimize memory usage.
+
+### Frame buffer state transitions
+![Frame buffer state transitions](./uvc_frames_state_transitions.png)  

host/class/uvc/usb_host_uvc/docs/uvc_frames_state_transitions.uxf

@@ -0,0 +1,111 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<diagram program="umlet" version="15.1">
+  <zoom_level>10</zoom_level>
+  <element>
+    <id>UMLState</id>
+    <coordinates>
+      <x>310</x>
+      <y>550</y>
+      <w>180</w>
+      <h>110</h>
+    </coordinates>
+    <panel_attributes>*Frame in Empty Queue*
+--
+Frame is owned
+by the driver
+-.
+There can be more frames
+in the Empty Queue
+valign=top
+</panel_attributes>
+    <additional_attributes/>
+  </element>
+  <element>
+    <id>UMLState</id>
+    <coordinates>
+      <x>700</x>
+      <y>550</y>
+      <w>180</w>
+      <h>110</h>
+    </coordinates>
+    <panel_attributes>*Active Frame*
+--
+Frame is owned
+by the driver
+-.
+There can be only
+1 active frame
+valign=top
+</panel_attributes>
+    <additional_attributes/>
+  </element>
+  <element>
+    <id>Relation</id>
+    <coordinates>
+      <x>480</x>
+      <y>610</y>
+      <w>240</w>
+      <h>40</h>
+    </coordinates>
+    <panel_attributes>lt=-&gt;
+uvc_frame_get_empty()</panel_attributes>
+    <additional_attributes>10.0;20.0;220.0;20.0</additional_attributes>
+  </element>
+  <element>
+    <id>Relation</id>
+    <coordinates>
+      <x>570</x>
+      <y>470</y>
+      <w>240</w>
+      <h>140</h>
+    </coordinates>
+    <panel_attributes>lt=-&gt;
+uvc_frame_add_data()</panel_attributes>
+    <additional_attributes>130.0;120.0;50.0;120.0;50.0;20.0;220.0;20.0;220.0;80.0</additional_attributes>
+  </element>
+  <element>
+    <id>Relation</id>
+    <coordinates>
+      <x>870</x>
+      <y>610</y>
+      <w>260</w>
+      <h>60</h>
+    </coordinates>
+    <panel_attributes>lt=-&gt;
+uvc_host_frame_callback_t()
+&lt;&lt; callback&gt;&gt;
+</panel_attributes>
+    <additional_attributes>10.0;20.0;240.0;20.0</additional_attributes>
+  </element>
+  <element>
+    <id>UMLState</id>
+    <coordinates>
+      <x>1110</x>
+      <y>550</y>
+      <w>180</w>
+      <h>110</h>
+    </coordinates>
+    <panel_attributes>*Process Frame*
+--
+Frame is owned
+by the user
+-.
+There can be more frames
+that are being processed
+valign=top
+</panel_attributes>
+    <additional_attributes/>
+  </element>
+  <element>
+    <id>Relation</id>
+    <coordinates>
+      <x>380</x>
+      <y>650</y>
+      <w>830</w>
+      <h>110</h>
+    </coordinates>
+    <panel_attributes>lt=-&gt;
+uvc_host_frame_return()</panel_attributes>
+    <additional_attributes>810.0;10.0;810.0;90.0;10.0;90.0;10.0;10.0</additional_attributes>
+  </element>
+</diagram>