Fixed UVC camera module.

src/core_v2/cuda_helper.cpp

@@ -11,11 +11,12 @@ namespace {
         stream_stack() {
             assert(cuda_ctx != nullptr);
             CUDA_API_CHECK(cuCtxSetCurrent(cuda_ctx));
-            push(default_stream);
+            default_stream.emplace();
+            push(*default_stream);
         }
 
     private:
-        cuda_stream_proxy default_stream = {};
+        std::optional<cuda_stream_proxy> default_stream;
     };
 
     thread_local std::unique_ptr<stream_stack> stack;

src/core_v2/ndarray_helper.hpp

@@ -144,6 +144,18 @@ struct ndarray_proxy : ndarray_base<N> {
         return true;
     }
 
+    [[nodiscard]] bool empty() const {
+        return elem_count() == 0;
+    }
+
+    struct empty_error final : std::exception {
+        [[nodiscard]] const char *what() const noexcept override { return "Ndarray is empty."; }
+    };
+
+    void ensure_not_empty() const {
+        if (empty()) [[unlikely]] throw empty_error();
+    }
+
     //@formatter:off
     auto host() const { return mem->host(); }
     auto cuda() const { return mem->cuda(); }

src/device/impl/uvc_camera.cpp

@@ -1,9 +1,8 @@
 #include "uvc_camera_impl.h"
-#include "core/image_utility.hpp"
-#include "core/image_utility_v2.h"
 #include "core/utility.hpp"
-#include "image_process/cuda_impl/pixel_convert.cuh"
 #include "third_party/scope_guard.hpp"
+#include "image_process_v5/sp_image.h"
+#include "image_process_v5/image_process.h"
 
 #include <opencv2/imgcodecs.hpp>
 #include <opencv2/imgproc.hpp>
@@ -92,34 +91,23 @@ namespace uvc_camera_impl {
         return dev_info_list;
     }
 
-    auto mjpeg_frame_to_image(uvc_frame_t *frame) { // TODO: accelerate with CUDA
+    sp_image mjpeg_frame_to_image(uvc_frame_t *frame) { // TODO: accelerate with CUDA
         assert(frame->frame_format == UVC_FRAME_FORMAT_MJPEG);
         auto img_data = cv::_InputArray((uchar *) frame->data, frame->data_bytes);
         auto img_bgr = cv::imdecode(img_data, cv::IMREAD_UNCHANGED);
-        if (img_bgr.empty()) return image_u8c3(); // TODO: ugly hacked, find out why
-        auto img_rgb_info = create_image_info<uchar3>(img_bgr.size(), MEM_HOST);
-        cv::cvtColor(img_bgr, img_rgb_info.as_mat(), cv::COLOR_BGR2RGB);
-        return create_image(img_rgb_info);
+        if (img_bgr.empty()) return {};
+        const auto img_rgb = sp_image::create(CV_8UC3, img_bgr.size());
+        const auto helper = write_access_helper(img_rgb.host());
+        auto img_rgb_mat = img_rgb.cv_mat(helper.ptr());
+        cv::cvtColor(img_bgr, img_rgb_mat, cv::COLOR_BGR2RGB);
+        return img_rgb;
     }
 
-    image_ptr yuyv_frame_to_image(uvc_frame_t *frame) {
+    sp_image yuyv_frame_to_image(uvc_frame_t *frame) {
         assert(frame->frame_format == UVC_FRAME_FORMAT_YUYV);
         auto img_size = cv::Size(frame->width, frame->height);
-        auto img_yuyv = create_image(img_size, CV_8UC2);
-        if (img_yuyv->size_in_bytes() != frame->data_bytes) return nullptr;
-        auto yuyv_mem = img_yuyv->memory(MEM_HOST);
-        memcpy(yuyv_mem.start_ptr(), frame->data, frame->data_bytes);
-        return img_yuyv;
-    }
-
-    image_u8c3 image_yuyv_to_rgb(const image_ptr &img_yuyv,
-                                 smart_cuda_stream *stream) {
-        auto img_rgb = create_image(img_yuyv->size(), CV_8UC3);
-        call_yuyv_to_rgb(img_yuyv->cuda<uchar2>(stream),
-                         img_rgb->cuda<uchar3>(stream),
-                         stream->cuda);
-        img_rgb->cuda_modified(stream);
-        return img_rgb->v1<uchar3>();
+        if (img_size.area() * sizeof(uchar2) != frame->data_bytes) return {};
+        return sp_image::create<uchar2>(img_size, frame->data);
     }
 
 }
@@ -178,10 +166,11 @@ void uvc_camera::impl::frame_callback(uvc_frame *frame) {
             break;
         }
         case UVC_FRAME_FORMAT_YUYV: {
-            auto img_yuv = yuyv_frame_to_image(frame);
-            if (img_yuv == nullptr) break;
+            auto img_yuyv = yuyv_frame_to_image(frame);
+            if (img_yuyv.empty()) break;
             boost::asio::post(*conf.ctx, [=, this] { // handle CUDA context problem
-                auto img_rgb = image_yuyv_to_rgb(img_yuv, conf.stream);
+                // TODO: check why CUDA commands cannot be used in this thread
+                auto img_rgb = image_yuyv_to_rgb(img_yuyv);
                 OBJ_SAVE(img_name, img_rgb);
             });
             break;
@@ -190,7 +179,6 @@ void uvc_camera::impl::frame_callback(uvc_frame *frame) {
             RET_ERROR;
         }
     }
-
 }
 
 void uvc_camera::impl::start(start_config conf) {

src/device/impl/uvc_camera_ui.cpp

@@ -4,7 +4,6 @@
 uvc_camera_ui::impl::impl(create_config conf) {
     img_name = conf.img_name;
     ctx = conf.ctx;
-    stream = conf.stream;
 }
 
 void uvc_camera_ui::impl::show_config() {
@@ -58,7 +57,7 @@ void uvc_camera_ui::impl::open_camera() {
     assert(dev_index < dev_list->size());
     auto conf = uvc_camera::create_config{
             .h_dev = dev_list->at(dev_index).h_dev,
-            .stream = stream, .ctx = ctx,
+            .ctx = ctx,
     };
     cam = uvc_camera::create(conf);
 

src/device/impl/uvc_camera_ui_impl.h

@@ -8,7 +8,6 @@ struct uvc_camera_ui::impl {
 
     obj_name_type img_name = invalid_obj_name;
     io_context *ctx = nullptr;
-    smart_cuda_stream *stream = nullptr;
 
     uvc_camera::dev_info_list_type dev_list;
     int dev_index = 0;

src/device/uvc_camera.h

@@ -22,7 +22,6 @@ public:
 
     struct create_config {
         void *h_dev = nullptr; // uvc_device_t *
-        smart_cuda_stream *stream = nullptr;
         io_ctx_type *ctx = nullptr;
     };
 

src/device/uvc_camera_ui.h

@@ -17,7 +17,6 @@ public:
     struct create_config {
         obj_name_type img_name = invalid_obj_name;
         io_context *ctx = nullptr;
-        smart_cuda_stream *stream = nullptr;
     };
 
     explicit uvc_camera_ui(create_config conf);

src/image_process_v5/image_process.cpp

@@ -142,6 +142,15 @@ sp_image image_nv12_to_rgb(const sp_image &img) {
     return ret;
 }
 
+sp_image image_yuyv_to_rgb(const sp_image &img) {
+    assert(img.cv_type() == CV_8UC2);
+    auto ret = sp_image::create(CV_8UC3, img.cv_size());
+    auto helper = image_cuda_v2_helper<uchar2, uchar3>(img, ret);
+    call_yuyv_to_rgb(helper.input(), helper.output(), current_cuda_stream());
+    ret.merge_meta(img);
+    return ret;
+}
+
 #include "render/render_utility.h"
 
 struct image_output_helper::impl {

src/image_process_v5/image_process.h

@@ -18,6 +18,7 @@ sp_image image_flip_y(const sp_image &img);
 sp_image image_rgb_to_bgra(const sp_image &img);
 sp_image image_rgb_to_nv12(const sp_image &img);
 sp_image image_nv12_to_rgb(const sp_image &img);
+sp_image image_yuyv_to_rgb(const sp_image &img);
 
 #include <core_v2/object_manager.h>
 

src/image_process_v5/image_viewer.cpp

@@ -55,6 +55,7 @@ struct image_viewer::impl {
             if (!item.visible) continue;
             try {
                 auto img = OBJ_QUERY(sp_image, item.name);
+                img.ensure_not_empty();
                 item.img_osg->setImageSP(img);
                 item.img_osg->setViewportRange(vp.aspectRatio(), item.flip);
                 item.img_osg->setNodeMask(-1); // enable

src/impl/apps/endo_guide/endo_guide.cpp

@@ -35,7 +35,6 @@ app_endo_guide::app_endo_guide(const create_config &_conf) {
     auto uvc_cam_conf = uvc_camera_ui::create_config{
             .img_name = img_color,
             .ctx = main_conf.asio_ctx,
-            .stream = default_cuda_stream,
     };
     uvc_cam = std::make_unique<uvc_camera_ui>(uvc_cam_conf);
 

src/impl/apps/remote_ar/remote_ar_v2.cpp

@@ -22,12 +22,20 @@ app_remote_ar_v2::app_remote_ar_v2(create_config _conf)
         // });
     }
 
+    if (true) {
+        auto uvc_cam_conf = uvc_camera_ui::create_config{
+            .img_name = uvc_img_id,
+            .ctx = main_conf.asio_ctx,
+        };
+        uvc_cam.emplace(uvc_cam_conf);
+    }
+
     if (true) {
         auto sub_conf = stereo_output_helper::create_config();
         sub_conf.left_name = left_img_id;
         sub_conf.right_name = right_img_id;
         sub_conf.out_name = output_img_id;
-        // sub_conf.size = cv::Size(1920, 1080);
+        // sub_conf.size = cv::Size(1920, 804);
         sub_conf.halve_width = false;
         output_helper.emplace(sub_conf);
     }
@@ -36,7 +44,7 @@ app_remote_ar_v2::app_remote_ar_v2(create_config _conf)
         auto sub_conf = image_viewer_v2::create_config();
         sub_conf.items.emplace_back(left_img_id, "Left", true);
         sub_conf.items.emplace_back(right_img_id, "Right", true);
-        sub_conf.items.emplace_back(output_img_id, "Output", true);
+        sub_conf.items.emplace_back(uvc_img_id, "Endoscope", true);
         bg_viewer.emplace(sub_conf);
     }
 
@@ -58,11 +66,16 @@ void app_remote_ar_v2::show_ui() {
     if (ImGui::Begin("Remote AR Control")) {
         ImGui::PushItemWidth(200);
 
-        if (ImGui::CollapsingHeader("Camera")) {
-            auto id_guard = imgui_id_guard("camera");
+        if (ImGui::CollapsingHeader("MVS Camera")) {
+            auto id_guard = imgui_id_guard("mvs_camera");
             mvs_cam->show();
         }
 
+        if (ImGui::CollapsingHeader("UVC Camera")) {
+            auto id_guard = imgui_id_guard("uvc_camera");
+            uvc_cam->show();
+        }
+
         if (ImGui::CollapsingHeader("Streamer")) {
             auto id_guard = imgui_id_guard("streamer");
             streamer->show();

src/impl/apps/remote_ar/remote_ar_v2.h

@@ -3,6 +3,7 @@
 
 #include "impl/app_base.h"
 #include "device/mvs_camera_ui.h"
+#include "device/uvc_camera_ui.h"
 #include "image_process_v5/image_viewer.h"
 #include "image_process_v5/image_process.h"
 #include "module/image_streamer.h"
@@ -26,9 +27,11 @@ private:
             bg_img_id = 0,
             left_img_id = 1,
             right_img_id = 2,
-            output_img_id = 3;
+            output_img_id = 3,
+            uvc_img_id = 4;
 
     std::optional<mvs_camera_ui> mvs_cam;
+    std::optional<uvc_camera_ui> uvc_cam;
     std::optional<stereo_output_helper> output_helper;
     std::optional<image_viewer_v2> bg_viewer;
     std::optional<image_streamer> streamer;

src/module/impl/image_streamer_impl.h

@@ -46,7 +46,7 @@ struct image_streamer::impl {
     std::unique_ptr<io_context> aux_ctx;
 
     // for UDP and UDP (FEC)
-    uint16_t sender_mtu = 1500;
+    uint16_t sender_mtu = 1450;
 
     // for UDP (FEC)
     float sender_parity_rate = 0.2f;

src/module_v5/CMakeLists.txt

@@ -1,2 +1,2 @@
 target_sources(${PROJECT_NAME} PRIVATE
-        image_viewer.cpp)
+    )

src/network_v3/fec_receive_helper.cpp

@@ -33,7 +33,7 @@ namespace {
         READY
     };
 
-    struct smart_chunk {
+    struct smart_chunk : private boost::noncopyable {
         static constexpr auto max_blocks = DATA_SHARDS_MAX;
 
         using block_ptrs_type = boost::container::static_vector<uint8_t *, max_blocks>;
@@ -45,25 +45,15 @@ namespace {
         uint8_t ready_blocks = 0;
         status_type status = NOT_INIT;
 
-        ~smart_chunk() { deallocate(); }
-
-        void reset() {
-            ready_blocks = 0;
-            status = NOT_INIT;
-        }
-
-        void create(const uint8_t total_blocks, const uint8_t parity_blocks, const uint16_t block_size) {
-            if (total_blocks != block_ptrs.size() ||
-                parity_blocks != last_parity_blocks ||
-                block_size != last_block_size) [[unlikely]] {
-                deallocate();
-                allocate(total_blocks, parity_blocks, block_size);
-            }
+        smart_chunk(const uint8_t total_blocks, const uint8_t parity_blocks, const uint16_t block_size) {
+            allocate(total_blocks, parity_blocks, block_size);
             std::ranges::fill(block_miss, true);
             assert(status == NOT_INIT);
             status = WAITING;
         }
 
+        ~smart_chunk() { deallocate(); }
+
         bool reconstruct() {
             if (ready_blocks + last_parity_blocks < block_ptrs.size()) return false;
             const auto ret = reed_solomon_reconstruct(rs, block_ptrs.data(), (uint8_t *) block_miss.data(),
@@ -122,7 +112,7 @@ namespace {
     };
 
     struct frame_info {
-        using chunks_type = std::vector<smart_chunk>;
+        using chunks_type = std::vector<std::shared_ptr<smart_chunk> >;
         chunks_type chunks;
 
         data_ptr data;
@@ -276,23 +266,23 @@ struct fec_receive_helper::impl {
         assert(frame_cache.frame_id == header.frame_id);
         assert(frame_cache.status == WAITING);
         auto &chunk = frame_cache.chunks[header.chunk_id];
-        if (chunk.status == NOT_INIT) {
+        if (chunk == nullptr) {
             const auto parity_blocks = header.block_count - header.chunk_decode_block_count;
-            chunk.create(header.block_count, parity_blocks, header.block_size);
-        } else if (chunk.status == READY) {
+            chunk = std::make_shared<smart_chunk>(header.block_count, parity_blocks, header.block_size);
+        } else if (chunk->status == READY) {
             return {};
         }
 
-        assert(chunk.status == WAITING);
+        assert(chunk->status == WAITING);
         auto data_ptr = in_buf->data + frag_header_size;
-        memcpy(chunk.block_ptrs[header.block_id], data_ptr, header.block_size);
-        chunk.block_miss[header.block_id] = false;
-        ++chunk.ready_blocks;
-        if (!chunk.reconstruct()) [[likely]] return {}; // need more blocks
-
-        assert(chunk.status == READY);
-        assert(chunk.block_data->length >= header.chunk_length);
-        memcpy(frame_cache.data->data + header.chunk_offset, chunk.block_data->data, header.chunk_length);
+        memcpy(chunk->block_ptrs[header.block_id], data_ptr, header.block_size);
+        chunk->block_miss[header.block_id] = false;
+        ++chunk->ready_blocks;
+        if (!chunk->reconstruct()) [[likely]] return {}; // need more blocks
+
+        assert(chunk->status == READY);
+        assert(chunk->block_data->length >= header.chunk_length);
+        memcpy(frame_cache.data->data + header.chunk_offset, chunk->block_data->data, header.chunk_length);
         ++frame_cache.ready_chunks;
         if (frame_cache.ready_chunks < frame_cache.chunks.size()) return {}; // need more chunks
 

src/network_v3/receiver_udp_fec.cpp

@@ -31,7 +31,7 @@ struct receiver_udp_fec::impl {
     fec_receive_helper helper;
     simple_data::pointer in_buf;
 
-    void send_request() const {
+    void try_send_request() const {
         // send packet
         assert(socket != nullptr);
         const auto out_buf = helper.get_output_packet();
@@ -61,6 +61,7 @@ struct receiver_udp_fec::impl {
 
         auto valid_buf = simple_data::create(length, in_buf->data);
         auto frame = helper.handle_input_packet(valid_buf);
+        try_send_request();
         if (frame.frame_id == 0) [[likely]] return;
         auto ret_frame = frame_info();
         assert(frame.data != nullptr);
@@ -81,7 +82,7 @@ struct receiver_udp_fec::impl {
 
         // initialize
         ret->helper.initialize({});
-        ret->send_request();
+        ret->try_send_request();
         return ret;
     }
 };