Implemented UVC camera module.

CMakeLists.txt

@@ -13,6 +13,7 @@ add_executable(${PROJECT_NAME} src/main.cpp
         src/impl/apps/app_selector/app_selector.cpp
         src/impl/apps/debug/app_debug.cpp
         src/impl/apps/depth_guide/depth_guide.cpp
+        src/impl/apps/endo_guide/endo_guide.cpp
         src/impl/apps/remote_ar/remote_ar.cpp
         src/impl/apps/tiny_player/tiny_player.cpp
         src/core/impl/event_timer.cpp
@@ -189,6 +190,13 @@ target_sources(${PROJECT_NAME} PRIVATE
         src/device/impl/mvs_camera.cpp
         src/device/impl/mvs_camera_ui.cpp)
 
+# libuvc config
+find_package(libuvc REQUIRED)
+target_link_libraries(${PROJECT_NAME} LibUVC::UVCShared LibUSB::LibUSB)
+target_sources(${PROJECT_NAME} PRIVATE
+        src/device/impl/uvc_camera.cpp
+        src/device/impl/uvc_camera_ui.cpp)
+
 # Crypto++ config
 set(CRYPTOPP_DIR /home/tpx/usr)
 set(CRYPTOPP_LIB_DIR ${CRYPTOPP_DIR}/lib)

data/config_endo_guide.yaml

@@ -0,0 +1 @@
+app_name: endo_guide

src/device/impl/uvc_camera.cpp

@@ -0,0 +1,237 @@
+#include "uvc_camera_impl.h"
+#include "core/image_utility.hpp"
+#include "core/utility.hpp"
+#include "third_party/scope_guard.hpp"
+
+#include <opencv2/imgcodecs.hpp>
+#include <opencv2/imgproc.hpp>
+
+#include <libusb.h>
+
+#include <fmt/format.h>
+#include <spdlog/spdlog.h>
+
+namespace uvc_camera_impl {
+
+    bool check_api_call(uvc_error_t api_ret, unsigned int line_number,
+                        const char *file_name, const char *api_call_str) {
+        if (api_ret == UVC_SUCCESS) [[likely]] return true;
+        SPDLOG_ERROR("libuvc api call {} failed at {}:{} with error 0x{:x}",
+                     api_call_str, file_name, line_number, (int) api_ret);
+        return false;
+    }
+
+#define API_CHECK(api_call) \
+    check_api_call( \
+        api_call, __LINE__, __FILE__, #api_call)
+
+#define API_CHECK_P(api_call) \
+    if (!check_api_call( \
+        api_call, __LINE__, __FILE__, #api_call)) [[unlikely]] \
+        return nullptr
+
+    uvc_context_t *uvc_ctx = nullptr;
+
+    void free_device_list(dev_info_list_type::element_type *data) {
+        for (auto &item: *data) {
+            uvc_unref_device((uvc_device_t *) item.h_dev);
+        }
+        delete data;
+    }
+
+    dev_info_list_type get_device_list() {
+        if (uvc_ctx == nullptr) [[unlikely]] {
+            API_CHECK_P(uvc_init(&uvc_ctx, nullptr));
+            std::atexit([] { uvc_exit(uvc_ctx); });
+        }
+        assert(uvc_ctx != nullptr);
+
+        uvc_device_t **dev_list = nullptr;
+        auto dev_list_closer = sg::make_scope_guard([&] {
+            if (dev_list == nullptr) return;
+            uvc_free_device_list(dev_list, 1);
+        });
+        API_CHECK_P(uvc_find_devices(uvc_ctx, &dev_list, 0, 0, nullptr));
+
+        auto dev_info_list =
+                dev_info_list_type(new dev_info_list_type::element_type(),
+                                   [](auto ptr) { free_device_list(ptr); });
+        for (auto p_dev = dev_list; *p_dev != nullptr; ++p_dev) {
+            auto dev = *p_dev;
+
+            uvc_device_descriptor_t *dev_desc = nullptr;
+            auto dev_desc_closer = sg::make_scope_guard([&] {
+                if (dev_desc == nullptr) return;
+                uvc_free_device_descriptor(dev_desc);
+            });
+
+            auto safe_str = [](const char *str) -> auto {
+                return (str == nullptr || *str == 0) ? "??" : str;
+            };
+
+            API_CHECK_P(uvc_get_device_descriptor(dev, &dev_desc));
+            assert(dev_desc != nullptr);
+            std::string dev_name;
+            if (dev_desc->serialNumber != nullptr) {
+                dev_name = fmt::format("{}, {} ({})",
+                                       safe_str(dev_desc->manufacturer),
+                                       safe_str(dev_desc->product),
+                                       safe_str(dev_desc->serialNumber));
+            } else {
+                dev_name = fmt::format("{:x}:{:x}",
+                                       dev_desc->idVendor, dev_desc->idProduct);
+            }
+
+            uvc_ref_device(dev);
+            dev_info_list->emplace_back(dev, dev_name);
+        }
+        return dev_info_list;
+    }
+
+    auto mjpeg_frame_to_image(uvc_frame_t *frame) { // TODO: accelerate with CUDA
+        assert(frame->frame_format == UVC_FRAME_FORMAT_MJPEG);
+        auto img_data = cv::_InputArray((uchar *) frame->data, frame->data_bytes);
+        auto img_bgr = cv::imdecode(img_data, cv::IMREAD_UNCHANGED);
+        if (img_bgr.empty()) return image_u8c3(); // TODO: ugly hacked, find out why
+        auto img_rgb_info = create_image_info<uchar3>(img_bgr.size(), MEM_HOST);
+        cv::cvtColor(img_bgr, img_rgb_info.as_mat(), cv::COLOR_BGR2RGB);
+        return create_image(img_rgb_info);
+    }
+
+}
+
+uvc_camera::impl::~impl() {
+    if (is_capturing) {
+        stop();
+    }
+    if (dev_h != nullptr) {
+        uvc_close(dev_h);
+    }
+    if (dev != nullptr) {
+        uvc_unref_device(dev);
+    }
+}
+
+std::unique_ptr<uvc_camera::impl>
+uvc_camera::impl::create(create_config conf) {
+    assert(uvc_ctx != nullptr);
+
+    auto ret = std::make_unique<impl>();
+    ret->dev = (uvc_device_t *) conf.h_dev;
+    uvc_ref_device(ret->dev);
+    API_CHECK_P(uvc_open(ret->dev, &ret->dev_h));
+
+#ifndef NDEBUG
+    uvc_print_diag(ret->dev_h, stdout);
+#endif
+
+    // create stream list
+    ret->stream_list =
+            std::make_shared<stream_info_list_type::element_type>();
+    for (auto fmt_desc = uvc_get_format_descs(ret->dev_h);
+         fmt_desc != nullptr; fmt_desc = fmt_desc->next) {
+
+        auto fmt_name = std::string((char *) fmt_desc->fourccFormat, 4);
+
+        for (auto f_desc = fmt_desc->frame_descs;
+             f_desc != nullptr; f_desc = f_desc->next) {
+
+            auto fps = 1e7 / f_desc->dwDefaultFrameInterval;
+            ret->stream_list->emplace_back(
+                    fmt_name, f_desc->wWidth, f_desc->wHeight, fps, f_desc);
+        }
+    }
+
+    return ret;
+}
+
+void uvc_camera::impl::frame_callback(uvc_frame *frame) {
+    auto img = image_u8c3();
+    switch (frame->frame_format) {
+        // @formatter:off
+        case UVC_FRAME_FORMAT_MJPEG: { img = mjpeg_frame_to_image(frame); break; }
+        // @formatter:on
+        default: {
+            RET_ERROR;
+        }
+    }
+    OBJ_SAVE(img_name, img);
+}
+
+void uvc_camera::impl::start(start_config conf) {
+    assert(!is_capturing);
+
+    // reset usb device to make it work
+    auto usb_dev_h = uvc_get_libusb_handle(dev_h);
+    auto usb_ret = libusb_reset_device(usb_dev_h);
+    assert(usb_ret == LIBUSB_SUCCESS);
+
+    auto f_desc = (uvc_frame_desc_t *) conf.h_desc;
+    auto fmt_desc = f_desc->parent;
+    uvc_frame_format fmt = UVC_FRAME_FORMAT_ANY;
+    switch (fmt_desc->bDescriptorSubtype) {
+        // @formatter:off
+        case UVC_VS_FORMAT_MJPEG: { fmt = UVC_FRAME_FORMAT_MJPEG; break; }
+        case UVC_VS_FORMAT_UNCOMPRESSED: { fmt = UVC_FRAME_FORMAT_YUYV; break; }
+        // @formatter:on
+        default: {
+            RET_ERROR;
+        }
+    }
+
+    auto fps = 1e7 / f_desc->dwDefaultFrameInterval;
+    API_CHECK(uvc_get_stream_ctrl_format_size(dev_h, &stream_ctrl, fmt,
+                                              f_desc->wWidth, f_desc->wHeight, fps));
+
+#ifndef NDEBUG
+    uvc_print_stream_ctrl(&stream_ctrl, stdout);
+#endif
+
+    img_name = conf.img_name;
+    API_CHECK(uvc_start_streaming(dev_h, &stream_ctrl, [](auto frame, void *user) {
+        ((impl *) user)->frame_callback(frame);
+    }, this, 0));
+
+    is_capturing = true;
+}
+
+void uvc_camera::impl::stop() {
+    assert(is_capturing);
+    uvc_stop_streaming(dev_h);
+    is_capturing = false;
+}
+
+dev_info_list_type uvc_camera::get_device_list() {
+    return ::get_device_list();
+}
+
+uvc_camera::pointer uvc_camera::create(create_config conf) {
+    auto pimpl = impl::create(conf);
+    if (pimpl == nullptr) return nullptr;
+    auto ret = std::make_unique<this_type>();
+    ret->pimpl = std::move(pimpl);
+    return ret;
+}
+
+uvc_camera::~uvc_camera() = default;
+
+uvc_camera::stream_info_list_type
+uvc_camera::get_stream_list() {
+    return pimpl->stream_list;
+}
+
+void uvc_camera::start(start_config conf) {
+    pimpl->start(conf);
+}
+
+bool uvc_camera::is_capturing() const {
+    return pimpl->is_capturing;
+}
+
+void uvc_camera::stop() {
+    pimpl->stop();
+}
+
+uvc_camera::stream_info_type::operator std::string() {
+    return fmt::format("{}x{} @ {}fps ({})", width, height, fps, fmt_name);
+}

src/device/impl/uvc_camera_impl.h

@@ -0,0 +1,46 @@
+#ifndef DEPTHGUIDE_UVC_CAMERA_IMPL_H
+#define DEPTHGUIDE_UVC_CAMERA_IMPL_H
+
+#include "device/uvc_camera.h"
+
+#include "libuvc/libuvc.h"
+
+namespace uvc_camera_impl {
+
+    extern uvc_context_t *uvc_ctx;
+
+    using dev_info_list_type =
+            uvc_camera::dev_info_list_type;
+
+    void free_device_list(dev_info_list_type::element_type *data);
+
+    dev_info_list_type get_device_list();
+
+}
+
+using namespace uvc_camera_impl;
+
+struct uvc_camera::impl {
+
+    uvc_device_t *dev = nullptr;
+    uvc_device_handle_t *dev_h = nullptr;
+
+    stream_info_list_type stream_list;
+    uvc_stream_ctrl_t stream_ctrl;
+
+    bool is_capturing = false;
+    obj_name_type img_name = invalid_obj_name;
+
+    ~impl();
+
+    static std::unique_ptr<impl> create(create_config conf);
+
+    void start(start_config conf);
+
+    void frame_callback(uvc_frame *frame);
+
+    void stop();
+
+};
+
+#endif //DEPTHGUIDE_UVC_CAMERA_IMPL_H

src/device/impl/uvc_camera_ui.cpp

@@ -0,0 +1,126 @@
+#include "uvc_camera_ui_impl.h"
+#include "core/imgui_utility.hpp"
+
+uvc_camera_ui::impl::impl(create_config conf) {
+    img_name = conf.img_name;
+    ctx = conf.ctx;
+}
+
+void uvc_camera_ui::impl::show_config() {
+    // select device by S/N
+    if (!dev_list->empty()) {
+        auto guard = imgui_disable_guard(cam != nullptr);
+        auto dev_name_preview = dev_list->at(dev_index).disp_name.c_str();
+        if (ImGui::BeginCombo("Device", dev_name_preview)) {
+            for (int k = 0; k < dev_list->size(); ++k) {
+                auto is_selected = (dev_index == k);
+                if (ImGui::Selectable(dev_list->at(k).disp_name.c_str(), is_selected)) {
+                    dev_index = k;
+                }
+                if (is_selected) {
+                    ImGui::SetItemDefaultFocus();
+                }
+            }
+            ImGui::EndCombo();
+        }
+    } else { // No device found.
+        auto guard = imgui_disable_guard();
+        if (ImGui::BeginCombo("Device", "No device")) {
+            ImGui::EndCombo();
+        }
+    }
+    ImGui::SameLine();
+    if (ImGui::Button("R")) {
+        post(*ctx, [this] { dev_list = nullptr; });
+    }
+
+    // select video config
+    if (cam != nullptr) {
+        auto guard = imgui_disable_guard(cam->is_capturing());
+        auto stream_preview = stream_list[stream_index].disp_name.c_str();
+        if (ImGui::BeginCombo("Stream", stream_preview)) {
+            for (int k = 0; k < stream_list.size(); ++k) {
+                auto is_selected = (stream_index == k);
+                if (ImGui::Selectable(stream_list[k].disp_name.c_str(), is_selected)) {
+                    stream_index = k;
+                }
+                if (is_selected) {
+                    ImGui::SetItemDefaultFocus();
+                }
+            }
+            ImGui::EndCombo();
+        }
+    }
+}
+
+void uvc_camera_ui::impl::open_camera() {
+    assert(dev_index < dev_list->size());
+    auto conf = uvc_camera::create_config{
+            .h_dev = dev_list->at(dev_index).h_dev,
+    };
+    cam = uvc_camera::create(conf);
+
+    stream_list.clear();
+    auto stream_info_list = cam->get_stream_list();
+    for (auto &item: *stream_info_list) {
+        stream_list.emplace_back(item.h_desc, std::string(item));
+    }
+    if (stream_index >= stream_list.size()) {
+        stream_index = 0;
+    }
+}
+
+void uvc_camera_ui::impl::start_camera() {
+    assert(stream_index < stream_list.size());
+    auto conf = uvc_camera::start_config{
+            .h_desc = stream_list[stream_index].h_desc,
+            .img_name = img_name,
+    };
+    cam->start(conf);
+}
+
+void uvc_camera_ui::impl::show() {
+    // load device list
+    if (dev_list == nullptr) [[unlikely]] {
+        dev_list = uvc_camera::get_device_list();
+        if (dev_index >= dev_list->size()) {
+            dev_index = 0;
+        }
+    }
+
+    ImGui::SeparatorText("Actions");
+    if (cam == nullptr) {
+        auto guard = imgui_disable_guard(dev_list->empty());
+        if (ImGui::Button("Open")) {
+            post(*ctx, [this] { open_camera(); });
+        }
+    } else {
+        assert(cam != nullptr);
+        if (ImGui::Button("Close")) {
+            post(*ctx, [this] { cam = nullptr; });
+        }
+        ImGui::SameLine();
+        if (!cam->is_capturing()) {
+            if (ImGui::Button("Start")) {
+                post(*ctx, [this] { start_camera(); });
+            }
+        } else {
+            if (ImGui::Button("Stop")) {
+                post(*ctx, [this] { cam->stop(); });
+            }
+        }
+    }
+
+    ImGui::SeparatorText("Configs");
+    show_config();
+}
+
+uvc_camera_ui::uvc_camera_ui(create_config conf)
+        : pimpl(std::make_unique<impl>(conf)) {
+}
+
+uvc_camera_ui::~uvc_camera_ui() = default;
+
+void uvc_camera_ui::show() {
+    pimpl->show();
+}

src/device/impl/uvc_camera_ui_impl.h

@@ -0,0 +1,36 @@
+#ifndef DEPTHGUIDE_UVC_CAMERA_UI_IMPL_H
+#define DEPTHGUIDE_UVC_CAMERA_UI_IMPL_H
+
+#include "device/uvc_camera_ui.h"
+#include "device/uvc_camera.h"
+
+struct uvc_camera_ui::impl {
+
+    obj_name_type img_name = invalid_obj_name;
+    io_context *ctx = nullptr;
+
+    uvc_camera::dev_info_list_type dev_list;
+    int dev_index = 0;
+
+    struct stream_item_type {
+        void *h_desc;
+        std::string disp_name;
+    };
+    std::vector<stream_item_type> stream_list;
+    int stream_index = 0;
+
+    std::unique_ptr<uvc_camera> cam;
+
+    explicit impl(create_config conf);
+
+    void open_camera();
+
+    void start_camera();
+
+    void show_config();
+
+    void show();
+
+};
+
+#endif //DEPTHGUIDE_UVC_CAMERA_UI_IMPL_H

src/device/uvc_camera.h

@@ -0,0 +1,61 @@
+#ifndef DEPTHGUIDE_UVC_CAMERA_H
+#define DEPTHGUIDE_UVC_CAMERA_H
+
+#include "core/object_manager.h"
+
+#include <memory>
+#include <vector>
+
+class uvc_camera {
+public:
+
+    struct dev_info_type {
+        void *h_dev;
+        std::string disp_name;
+    };
+
+    using dev_info_list_type =
+            std::shared_ptr<std::vector<dev_info_type>>;
+
+    static dev_info_list_type get_device_list();
+
+    struct create_config {
+        void *h_dev; // uvc_device_t *
+    };
+
+    using this_type = uvc_camera;
+    using pointer = std::unique_ptr<this_type>;
+
+    static pointer create(create_config conf);
+
+    ~uvc_camera();
+
+    struct stream_info_type {
+        std::string fmt_name;
+        uint32_t width, height, fps;
+        void *h_desc; // uvc_frame_desc*
+
+        explicit operator std::string();
+    };
+    using stream_info_list_type =
+            std::shared_ptr<std::vector<stream_info_type>>;
+
+    stream_info_list_type get_stream_list();
+
+    struct start_config {
+        void *h_desc;
+        obj_name_type img_name;
+    };
+
+    void start(start_config conf);
+
+    bool is_capturing() const;
+
+    void stop();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#endif //DEPTHGUIDE_UVC_CAMERA_H

src/device/uvc_camera_ui.h

@@ -0,0 +1,32 @@
+#ifndef DEPTHGUIDE_UVC_CAMERA_UI_H
+#define DEPTHGUIDE_UVC_CAMERA_UI_H
+
+#include "core/object_manager.h"
+
+#include <boost/asio/io_context.hpp>
+
+#include <memory>
+
+class uvc_camera_ui {
+public:
+
+    using io_context =
+            boost::asio::io_context;
+
+    struct create_config {
+        obj_name_type img_name = invalid_obj_name;
+        io_context *ctx = nullptr;
+    };
+
+    explicit uvc_camera_ui(create_config conf);
+
+    ~uvc_camera_ui();
+
+    void show();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#endif //DEPTHGUIDE_UVC_CAMERA_UI_H

src/impl/apps/app_selector/app_selector.cpp

@@ -1,5 +1,6 @@
 #include "app_selector.h"
 #include "impl/apps/debug/app_debug.h"
+#include "impl/apps/endo_guide/endo_guide.h"
 #include "impl/apps/depth_guide/depth_guide.h"
 #include "impl/apps/remote_ar/remote_ar.h"
 #include "impl/apps/tiny_player/tiny_player.h"
@@ -53,6 +54,8 @@ void app_selector::load_app(const std::string &conf_path) {
         app = std::make_unique<app_remote_ar>(create_conf);
     } else if (app_name == "tiny_player") {
         app = std::make_unique<app_tiny_player>(create_conf);
+    } else if (app_name == "endo_guide") {
+        app = std::make_unique<app_endo_guide>(create_conf);
     } else if (app_name == "debug") {
         app = std::make_unique<app_debug>(create_conf);
     }

src/impl/apps/depth_guide/depth_guide.cpp

@@ -81,6 +81,7 @@ app_depth_guide::app_depth_guide(const create_config &_conf) {
     saver_conf.img_list.emplace_back("Image", img_color);
     saver_conf.img_list.emplace_back("Image Out", img_out);
     saver_conf.img_list.emplace_back("PC Raw", pc_raw);
+    saver_conf.img_list.emplace_back("PC Object", pc_obj);
     out_saver = std::make_unique<image_saver>(saver_conf);
 
     auto pc_conf = pc_viewer::create_config{

src/impl/apps/endo_guide/endo_guide.cpp

@@ -0,0 +1,40 @@
+#include "endo_guide.h"
+#include "core/imgui_utility.hpp"
+
+app_endo_guide::app_endo_guide(const create_config &_conf) {
+    conf = _conf;
+
+    // initialize object manager
+    OBJ_SAVE(img_color, image_u8c3());
+
+    // initialize modules
+    auto uvc_cam_conf = uvc_camera_ui::create_config{
+            .img_name = img_color, .ctx = conf.asio_ctx,
+    };
+    uvc_cam = std::make_unique<uvc_camera_ui>(uvc_cam_conf);
+
+    auto bg_viewer_conf = image_viewer::create_config{
+            .mode = VIEW_COLOR_ONLY, .flip_y = true,
+            .stream = default_cuda_stream,
+    };
+    auto &bg_extra_conf = bg_viewer_conf.extra.color;
+    bg_extra_conf.fmt = COLOR_RGB;
+    bg_extra_conf.name = img_color;
+    bg_viewer = std::make_unique<image_viewer>(bg_viewer_conf);
+}
+
+void app_endo_guide::show_ui() {
+    if (ImGui::Begin("Endo Guide Control")) {
+        ImGui::PushItemWidth(200);
+
+        if (ImGui::CollapsingHeader("Camera")) {
+            auto id_guard = imgui_id_guard("camera");
+            uvc_cam->show();
+        }
+    }
+    ImGui::End();
+}
+
+void app_endo_guide::render_background() {
+    bg_viewer->render();
+}

src/impl/apps/endo_guide/endo_guide.h

@@ -0,0 +1,37 @@
+#ifndef DEPTHGUIDE_ENDO_GUIDE_H
+#define DEPTHGUIDE_ENDO_GUIDE_H
+
+#include "core/object_manager.h"
+#include "device/uvc_camera_ui.h"
+#include "module/image_viewer.h"
+#include "impl/app_base.h"
+
+class app_endo_guide : public app_base {
+public:
+
+    explicit app_endo_guide(const create_config &conf);
+
+    ~app_endo_guide() override = default;
+
+    const char *window_name() override { return "EndoGuide V1.-1"; }
+
+    void show_ui() override;
+
+    void render_background() override;
+
+private:
+
+    enum obj_names : object_manager::name_type {
+
+        // images from device
+        img_color,
+
+    };
+
+    create_config conf;
+
+    std::unique_ptr<uvc_camera_ui> uvc_cam;
+    std::unique_ptr<image_viewer> bg_viewer;
+};
+
+#endif //DEPTHGUIDE_ENDO_GUIDE_H