Basic camera capture functions.

CMakeLists.txt

@@ -0,0 +1,84 @@
+cmake_minimum_required(VERSION 3.25)
+project(RemoteAR3)
+
+set(CMAKE_CXX_STANDARD 20)
+
+add_executable(RemoteAR3 src/main.cpp
+        src/main_ext.cpp
+        src/image_process.cpp
+        src/simple_mq.cpp
+        src/simple_opengl.cpp)
+
+# CUDA config
+find_package(CUDAToolkit REQUIRED)
+target_link_directories(${PROJECT_NAME} PRIVATE /usr/local/cuda/lib64)
+target_link_libraries(${PROJECT_NAME} CUDA::cudart CUDA::cuda_driver)
+
+# spdlog config
+find_package(spdlog REQUIRED)
+target_link_libraries(${PROJECT_NAME} spdlog::spdlog)
+target_compile_definitions(${PROJECT_NAME} PRIVATE SPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_TRACE)
+
+# OpenCV config
+find_package(OpenCV REQUIRED COMPONENTS cudaimgproc imgcodecs)
+target_include_directories(${PROJECT_NAME} PRIVATE ${OpenCV_INCLUDE_DIRS})
+target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS})
+
+# glfw config
+if (WIN32)
+    set(GLFW_INCLUDE_DIR C:/BuildEssentials/VS2019Libs/include)
+    set(GLFW_LIB_DIR C:/BuildEssentials/VS2019Libs/lib)
+    find_library(GLFW_LIB glfw3 HINTS ${GLFW_LIB_DIR})
+    target_include_directories(${PROJECT_NAME} PRIVATE ${GLFW_INCLUDE_DIR})
+    target_link_libraries(${PROJECT_NAME} ${GLFW_LIB})
+else ()
+    find_package(glfw3 REQUIRED)
+    target_link_libraries(${PROJECT_NAME} glfw)
+endif ()
+
+# glad config
+if (WIN32)
+    set(GLAD_DIR C:/BuildEssentials/Library/glad)
+else ()
+    set(GLAD_DIR /home/tpx/src/glad)
+endif ()
+target_include_directories(${PROJECT_NAME} PRIVATE ${GLAD_DIR}/include)
+target_sources(${PROJECT_NAME} PRIVATE ${GLAD_DIR}/src/gl.c)
+
+# imgui config
+if (WIN32)
+    set(IMGUI_DIR C:/BuildEssentials/Library/imgui-1.89.5)
+else ()
+    set(IMGUI_DIR /home/tpx/src/imgui-1.90)
+endif ()
+set(IMGUI_BACKENDS_DIR ${IMGUI_DIR}/backends)
+target_include_directories(${PROJECT_NAME} PRIVATE ${IMGUI_DIR} ${IMGUI_BACKENDS_DIR})
+target_sources(${PROJECT_NAME} PRIVATE
+        ${IMGUI_DIR}/imgui.cpp
+        ${IMGUI_DIR}/imgui_draw.cpp
+        ${IMGUI_DIR}/imgui_tables.cpp
+        ${IMGUI_DIR}/imgui_widgets.cpp
+        ${IMGUI_DIR}/imgui_demo.cpp
+        ${IMGUI_BACKENDS_DIR}/imgui_impl_glfw.cpp
+        ${IMGUI_BACKENDS_DIR}/imgui_impl_opengl3.cpp)
+target_compile_definitions(${PROJECT_NAME} PRIVATE HAVE_IMGUI)
+
+# MVS config
+if (WIN32)
+    set(MVS_DIR "C:/BuildEssentials/Library/MVS/Development")
+    set(MVS_INCLUDE_DIR ${MVS_DIR}/Includes)
+    set(MVS_LIB_DIR ${MVS_DIR}/Libraries/win64)
+else ()
+    set(MVS_DIR /opt/MVS)
+    set(MVS_INCLUDE_DIR ${MVS_DIR}/include)
+    set(MVS_LIB_DIR ${MVS_DIR}/lib/64)
+endif ()
+find_library(MVS_LIB MvCameraControl HINTS ${MVS_LIB_DIR})
+target_include_directories(${PROJECT_NAME} PRIVATE ${MVS_INCLUDE_DIR})
+target_link_libraries(${PROJECT_NAME} ${MVS_LIB})
+target_sources(${PROJECT_NAME} PRIVATE src/mvs_camera.cpp)
+
+# yaml-cpp
+find_package(yaml-cpp REQUIRED)
+target_include_directories(${PROJECT_NAME} PRIVATE ${YAML_CPP_INCLUDE_DIR})
+target_link_libraries(${PROJECT_NAME} ${YAML_CPP_LIBRARIES})

src/cuda_helper.hpp

@@ -0,0 +1,51 @@
+#ifndef REMOTEAR3_CUDA_HELPER_H
+#define REMOTEAR3_CUDA_HELPER_H
+
+#include "utility.hpp"
+
+#include <cuda.h>
+#include <cuda_runtime.h>
+#include <nppdefs.h>
+
+#include <spdlog/spdlog.h>
+
+inline bool check_cuda_api_call(CUresult api_ret, unsigned int line_number,
+                                const char *file_name, const char *api_call_str) {
+    if (api_ret == CUDA_SUCCESS) [[likely]] return true;
+    const char *error_name, *error_str;
+    auto ret = cuGetErrorName(api_ret, &error_name);
+    if (ret != CUDA_SUCCESS) [[unlikely]] error_name = "Unknown";
+    ret = cuGetErrorString(api_ret, &error_str);
+    if (ret != CUDA_SUCCESS) [[unlikely]] error_str = "Unknown";
+    SPDLOG_ERROR("CUDA runtime api call {} failed at {}:{} with error 0x{:x}:{}, {}.",
+                 api_call_str, file_name, line_number,
+                 (int) api_ret, error_name, error_str);
+    RET_ERROR_B;
+}
+
+inline bool check_cuda_api_call(cudaError api_ret, unsigned int line_number,
+                                const char *file_name, const char *api_call_str) {
+    if (api_ret == cudaSuccess) [[likely]] return true;
+    SPDLOG_ERROR("CUDA driver api call {} failed at {}:{} with error 0x{:x}.",
+                 api_call_str, file_name, line_number, (int) api_ret);
+    RET_ERROR_B;
+}
+
+inline bool check_cuda_api_call(NppStatus api_ret, unsigned int line_number,
+                                const char *file_name, const char *api_call_str) {
+    if (api_ret == NPP_SUCCESS) [[likely]] return true;
+    SPDLOG_ERROR("NPP api call {} failed at {}:{} with error 0x{:x}.",
+                 api_call_str, file_name, line_number, (int) api_ret);
+    RET_ERROR_B;
+}
+
+#define CUDA_API_CHECK(api_call) \
+    check_cuda_api_call( \
+        api_call, __LINE__, __FILE__, #api_call)
+
+#define CUDA_API_CHECK_P(api_call) \
+    if (!check_cuda_api_call( \
+        api_call, __LINE__, __FILE__, #api_call)) [[unlikely]] \
+        return nullptr
+
+#endif //REMOTEAR3_CUDA_HELPER_H

src/image_process.cpp

@@ -0,0 +1,40 @@
+#include "image_process.h"
+#include "utility.hpp"
+
+#include <opencv2/cudaimgproc.hpp>
+
+struct monocular_processor::impl {
+    cv::cuda::GpuMat raw_dev;
+
+    static void debayer(const cv::cuda::GpuMat &in, cv::cuda::GpuMat *out,
+                        cv::cuda::Stream &stream) {
+        switch (in.type()) {
+            case CV_8UC1: {
+                cv::cuda::cvtColor(in, *out, cv::COLOR_BayerRG2RGB, 3, stream);
+                return;
+            }
+        }
+        unreachable();
+    }
+
+    void process(const cv::Mat &in, cv::cuda::GpuMat *out,
+                 cv::cuda::Stream &stream) {
+        // upload from host to device
+        raw_dev.upload(in, stream);
+
+        // debayer using OpenCV
+        debayer(raw_dev, out, stream);
+
+        // TODO: un-distort
+    }
+};
+
+monocular_processor::monocular_processor()
+        : pimpl(std::make_unique<impl>()) {}
+
+monocular_processor::~monocular_processor() = default;
+
+void monocular_processor::process(const cv::Mat &in, cv::cuda::GpuMat *out,
+                                  cv::cuda::Stream &stream) {
+    pimpl->process(in, out, stream);
+}

src/image_process.h

@@ -0,0 +1,24 @@
+#ifndef REMOTEAR3_IMAGE_PROCESS_H
+#define REMOTEAR3_IMAGE_PROCESS_H
+
+#include <opencv2/core/mat.hpp>
+#include <opencv2/core/cuda.hpp>
+
+#include <memory>
+
+class monocular_processor {
+public:
+
+    monocular_processor();
+
+    ~monocular_processor();
+
+    void process(const cv::Mat &in, cv::cuda::GpuMat *out,
+                 cv::cuda::Stream &stream = cv::cuda::Stream::Null());
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#endif //REMOTEAR3_IMAGE_PROCESS_H

src/main.cpp

@@ -0,0 +1,63 @@
+#include "simple_mq.h"
+#include "variable_defs.h"
+
+#include <spdlog/spdlog.h>
+
+#include <GLFW/glfw3.h>
+
+#include <cassert>
+
+using namespace simple_mq_singleton;
+
+extern GLFWwindow *main_window;
+
+void initialize_mq();
+
+void load_config();
+
+void initialize_cuda();
+
+void initialize_main_window();
+
+void cleanup();
+
+void prepare_imgui_frame();
+
+void handle_imgui_events();
+
+bool is_capturing();
+
+void wait_camera_frames();
+
+void process_camera_frames();
+
+void render_main_window();
+
+int main() {
+
+#ifndef NDEBUG
+    spdlog::set_level(spdlog::level::trace);
+#endif
+
+    // initialize many staffs
+    initialize_mq();
+    initialize_cuda();
+    load_config();
+    initialize_main_window();
+
+    while (!glfwWindowShouldClose(main_window)) {
+
+        prepare_imgui_frame();
+        handle_imgui_events();
+
+        if (is_capturing()) {
+            wait_camera_frames();
+            process_camera_frames();
+        }
+
+        render_main_window();
+    }
+
+    cleanup();
+    return 0;
+}

src/main_ext.cpp

@@ -0,0 +1,399 @@
+#include "cuda_helper.hpp"
+#include "image_process.h"
+#include "mvs_camera.h"
+#include "simple_mq.h"
+#include "simple_opengl.h"
+#include "variable_defs.h"
+
+#include <spdlog/spdlog.h>
+
+#include <yaml-cpp/yaml.h>
+
+#include <glad/gl.h>
+#include <GLFW/glfw3.h>
+
+#include <opencv2/core/mat.hpp>
+#include <opencv2/core/cuda.hpp>
+
+#include <imgui.h>
+#include <imgui_impl_glfw.h>
+#include <imgui_impl_opengl3.h>
+
+#include <cassert>
+#include <queue>
+
+using namespace simple_mq_singleton;
+
+// global variable definition
+CUcontext cuda_ctx;
+GLFWwindow *main_window;
+std::string left_camera_name, right_camera_name;
+std::unique_ptr<mvs::camera> left_camera, right_camera;
+mvs::capture_config capture_conf;
+int preview_camera_index = 0; // 0 for left, 1 for right
+uint64_t left_raw_cnt = 0, right_raw_cnt = 0;
+std::unique_ptr<cv::cuda::GpuMat> left_img_dev, right_img_dev;
+std::unique_ptr<cv::cuda::Stream> left_stream, right_stream;
+cudaStream_t left_cuda_stream = nullptr, right_cuda_stream = nullptr;
+std::unique_ptr<monocular_processor> left_processor, right_processor;
+std::unique_ptr<simple_render> opengl_render;
+
+std::queue<void (*)()> simple_eq;
+
+constexpr auto config_path = "./config.yaml";
+
+void initialize_mq() {
+    mq(); // request the mq to be initialized.
+}
+
+void initialize_cuda() {
+    // create cuda context
+    constexpr auto default_cuda_device_id = 0;
+    cuInit(0);
+    int cuda_device_count;
+    CUDA_API_CHECK(cuDeviceGetCount(&cuda_device_count));
+    assert(cuda_device_count > default_cuda_device_id);
+    CUdevice cuda_device;
+    CUDA_API_CHECK(cuDeviceGet(&cuda_device, default_cuda_device_id));
+    CUDA_API_CHECK(cuCtxCreate(&cuda_ctx, CU_CTX_SCHED_AUTO, cuda_device));
+
+    // elegant cleanup
+    std::atexit([] {
+        cuCtxDestroy(cuda_ctx);
+    });
+
+    // create some cuda objects
+    left_img_dev = std::make_unique<cv::cuda::GpuMat>();
+    right_img_dev = std::make_unique<cv::cuda::GpuMat>();
+    left_stream = std::make_unique<cv::cuda::Stream>();
+    right_stream = std::make_unique<cv::cuda::Stream>();
+    left_cuda_stream = (cudaStream_t) left_stream->cudaPtr();
+    right_cuda_stream = (cudaStream_t) right_stream->cudaPtr();
+    left_processor = std::make_unique<monocular_processor>();
+    right_processor = std::make_unique<monocular_processor>();
+}
+
+void load_config() {
+    auto conf = YAML::LoadFile(config_path);
+
+    // load camera names
+    auto camera_conf = conf["camera"];
+    auto camera_names = camera_conf["names"];
+    left_camera_name = camera_names["left"].as<std::string>();
+    right_camera_name = camera_names["right"].as<std::string>();
+    auto capture_param = camera_conf["capture"];
+    capture_conf.frame_rate = capture_param["frame_rate"].as<float>();
+    capture_conf.expo_time_ms = capture_param["expo_time_ms"].as<float>();
+    capture_conf.gain_db = capture_param["gain_db"].as<float>();
+
+    // load main window config
+    auto window_conf = conf["main_window"];
+    mq().update_variable(MAIN_WINDOW_WIDTH, window_conf["width"].as<int>());
+    mq().update_variable(MAIN_WINDOW_HEIGHT, window_conf["height"].as<int>());
+}
+
+void initialize_main_window() {
+    // set GLFW error handler
+    glfwSetErrorCallback([](int error, const char *desc) {
+        SPDLOG_ERROR("GLFW error: code = {}, description = {}", error, desc);
+    });
+
+    // create main window
+    auto ret = glfwInit();
+    assert(ret == GLFW_TRUE);
+    glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 4);
+    glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 6);
+    glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
+    auto window_width = mq().query_variable<int>(MAIN_WINDOW_WIDTH);
+    auto window_height = mq().query_variable<int>(MAIN_WINDOW_HEIGHT);
+    main_window = glfwCreateWindow(window_width, window_height, "RemoteAR V3.-1", nullptr, nullptr);
+    assert(main_window != nullptr);
+    glfwMakeContextCurrent(main_window);
+
+    // load opengl functions
+    auto version = gladLoadGL(glfwGetProcAddress);
+    assert(version > 0);
+    SPDLOG_INFO("Loaded OpenGL {}.{}", GLAD_VERSION_MAJOR(version), GLAD_VERSION_MINOR(version));
+
+    // enable color blending
+    glEnable(GL_BLEND);
+    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
+
+#ifndef NDEBUG
+    // log opengl error
+    glEnable(GL_DEBUG_OUTPUT);
+    glDebugMessageCallback([](GLenum source, GLenum type, GLuint id, GLenum severity,
+                              GLsizei length, const GLchar *message, const void *user_data) {
+        if (type == GL_DEBUG_TYPE_ERROR) {
+            SPDLOG_ERROR("OpenGL error: type = {}, severity = {}, message = {}", type, severity, message);
+            assert(false);
+        }
+    }, nullptr);
+#endif
+
+    // setup imgui context
+    IMGUI_CHECKVERSION();
+    ImGui::CreateContext();
+    auto io = ImGui::GetIO();
+    io.ConfigFlags |= ImGuiConfigFlags_NavEnableKeyboard;
+    ImGui::StyleColorsDark();
+    ImGui_ImplGlfw_InitForOpenGL(main_window, true);
+    ImGui_ImplOpenGL3_Init();
+
+    // initialize OpenGL render
+    opengl_render = std::make_unique<simple_render>();
+
+    // elegant cleanup
+    std::atexit([] {
+        ImGui_ImplOpenGL3_Shutdown();
+        ImGui_ImplGlfw_Shutdown();
+        ImGui::DestroyContext();
+
+        glfwDestroyWindow(main_window);
+        glfwTerminate();
+    });
+}
+
+bool is_camera_opened() {
+    assert((left_camera == nullptr) == (right_camera == nullptr));
+    return left_camera != nullptr;
+}
+
+bool is_capturing() {
+    if (!is_camera_opened()) return false;
+    assert(left_camera->is_capture() == right_camera->is_capture());
+    return left_camera->is_capture();
+}
+
+bool upload_capture_config_impl() {
+    bool ok = true;
+    ok &= left_camera->set_capture_config(capture_conf);
+    ok &= right_camera->set_capture_config(capture_conf);
+    return ok;
+}
+
+void upload_capture_config() {
+    if (!upload_capture_config_impl()) {
+        // TODO: show error msg
+    }
+}
+
+void stop_capture() {
+    left_camera->stop_capture();
+    right_camera->stop_capture();
+}
+
+void start_capture() {
+    assert(is_camera_opened());
+    bool ok = true;
+    ok &= upload_capture_config_impl();
+    ok &= left_camera->start_capture();
+    ok &= right_camera->start_capture();
+    if (!ok) {
+        // TODO: show error msg
+        stop_capture();
+    }
+}
+
+void close_cameras() {
+    if (is_capturing()) {
+        stop_capture();
+    }
+    left_camera.reset();
+    right_camera.reset();
+}
+
+void open_cameras() {
+    auto left_camera_conf = mvs::create_config{
+            left_camera_name, mvs::RG_8, IMG_RAW_HOST_LEFT
+    };
+    auto right_camera_conf = mvs::create_config{
+            right_camera_name, mvs::RG_8, IMG_RAW_HOST_RIGHT
+    };
+    left_camera.reset(mvs::camera::create(left_camera_conf));
+    right_camera.reset(mvs::camera::create(right_camera_conf));
+    if (left_camera == nullptr || right_camera == nullptr) {
+        // TODO: show error msg
+        close_cameras();
+    }
+}
+
+void cleanup() {
+    close_cameras();
+
+    // avoid cudaErrorCudartUnloading
+    opengl_render.reset();
+}
+
+void prepare_imgui_frame() {
+
+    glfwPollEvents();
+    ImGui_ImplOpenGL3_NewFrame();
+    ImGui_ImplGlfw_NewFrame();
+    ImGui::NewFrame();
+
+    if (ImGui::Begin("Remote AR Control")) {
+        ImGui::PushItemWidth(200);
+
+        // camera control
+        if (ImGui::CollapsingHeader("Camera")) {
+            ImGui::PushID("Camera");
+
+            // camera actions
+            ImGui::SeparatorText("Actions");
+            if (!is_camera_opened()) {
+                if (ImGui::Button("Open")) {
+                    simple_eq.push(open_cameras);
+                }
+            } else { // cameras have been opened
+                if (ImGui::Button("Close")) {
+                    simple_eq.push(close_cameras);
+                }
+                ImGui::SameLine();
+                if (!is_capturing()) {
+                    if (ImGui::Button("Start")) {
+                        simple_eq.push(start_capture);
+                    }
+                } else {
+                    if (ImGui::Button("Stop")) {
+                        simple_eq.push(stop_capture);
+                    }
+//                    if (!auto_save_raw) {
+//                        ImGui::SameLine();
+//                        if (ImGui::Button("Capture")) {
+//                                camera.request_save_raw();
+//                        }
+//                    }
+                }
+            }
+
+            // camera configs
+            ImGui::SeparatorText("Configs");
+            if (ImGui::DragFloat("Frame Rate (fps)", &capture_conf.frame_rate,
+                                 0.5, 1, 60, "%.01f")) {
+                simple_eq.push(upload_capture_config);
+            }
+            if (ImGui::DragFloat("Exposure Time (ms)", &capture_conf.expo_time_ms,
+                                 0.1, 0.1, 1e3f / capture_conf.frame_rate, "%.01f")) {
+                simple_eq.push(upload_capture_config);
+            }
+            if (ImGui::DragFloat("Analog Gain (dB)", &capture_conf.gain_db,
+                                 0.1, 0, 23.4, "%.01f")) {
+                simple_eq.push(upload_capture_config);
+            }
+
+            if (is_capturing()) {
+                // preview config
+                ImGui::SeparatorText("Preview Camera");
+                ImGui::RadioButton("Left", &preview_camera_index, 0);
+                ImGui::SameLine();
+                ImGui::RadioButton("Right", &preview_camera_index, 1);
+
+                // auto save raw config
+//                ImGui::SeparatorText("Auto Shoot");
+//                ImGui::PushID("Auto Shoot");
+//
+//                if (!auto_save_raw) {
+//                    if (ImGui::Button("Start")) {
+//                        auto_save_raw = true;
+//                    }
+//                } else {
+//                    if (ImGui::Button("Stop")) {
+//                        auto_save_raw = false;
+//                    }
+//                }
+//
+//                if (auto_save_raw_interval < 1) {
+//                    auto_save_raw_interval = 1;
+//                }
+//
+//                if (auto_save_raw) {
+//                    ImGui::BeginDisabled();
+//                    auto now_time = std::chrono::system_clock::now();
+//                    if (now_time - last_save_raw_time >
+//                        std::chrono::seconds{auto_save_raw_interval}) {
+//                                camera.request_save_raw();
+//                        last_save_raw_time = now_time;
+//                    }
+//                }
+//                ImGui::InputInt("Shoot Interval (s)", &auto_save_raw_interval, 1, 10);
+//                if (auto_save_raw) {
+//                    ImGui::EndDisabled();
+//                }
+
+
+            }
+
+            ImGui::PopID();
+        }
+
+        ImGui::PopItemWidth();
+        ImGui::End();
+        ImGui::Render();
+    }
+}
+
+void handle_imgui_events() {
+    while (!simple_eq.empty()) {
+        simple_eq.front()();
+        simple_eq.pop();
+    }
+}
+
+void wait_camera_frames() {
+    assert(is_capturing());
+    uint64_t cur_cnt;
+    if (auto ptr = mq().query_variable_ptr<cv::Mat>(IMG_RAW_HOST_LEFT, &cur_cnt);
+            ptr == nullptr || cur_cnt <= left_raw_cnt) {
+        mq().wait_variable(IMG_RAW_HOST_LEFT, left_raw_cnt);
+    }
+    if (auto ptr = mq().query_variable_ptr<cv::Mat>(IMG_RAW_HOST_RIGHT, &cur_cnt);
+            ptr == nullptr || cur_cnt <= right_raw_cnt) {
+        mq().wait_variable(IMG_RAW_HOST_RIGHT, right_raw_cnt);
+    }
+}
+
+void process_camera_frames() {
+    // retrieve new frames
+    uint64_t cur_cnt;
+    auto left_raw_ptr = mq().query_variable_ptr<cv::Mat>(IMG_RAW_HOST_LEFT, &cur_cnt);
+    assert(cur_cnt > left_raw_cnt);
+    left_raw_cnt = cur_cnt;
+    auto right_raw_ptr = mq().query_variable_ptr<cv::Mat>(IMG_RAW_HOST_RIGHT, &cur_cnt);
+    assert(cur_cnt > right_raw_cnt);
+    right_raw_cnt = cur_cnt;
+
+    // process images
+    left_processor->process(*left_raw_ptr, left_img_dev.get(), *left_stream);
+    right_processor->process(*right_raw_ptr, right_img_dev.get(), *right_stream);
+}
+
+void render_main_window() {
+    cv::Size frame_size;
+    glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
+    glfwGetFramebufferSize(main_window, &frame_size.width, &frame_size.height);
+    glViewport(0, 0, frame_size.width, frame_size.height);
+    glClear(GL_COLOR_BUFFER_BIT);
+
+    if (is_capturing()) {
+        // draw preview camera frame
+        assert(left_img_dev->size() == right_img_dev->size());
+        float width_normal = left_img_dev->size().aspectRatio() / frame_size.aspectRatio();
+        auto render_rect = simple_rect{
+                -width_normal, 1, 2 * width_normal, -2
+        };
+        if (preview_camera_index == 0) { // left camera
+            if (!left_img_dev->empty()) {
+                opengl_render->render_rect(*left_img_dev, render_rect, false, left_cuda_stream);
+            }
+        } else { // right camera
+            assert(preview_camera_index == 1);
+            if (!right_img_dev->empty()) {
+                opengl_render->render_rect(*right_img_dev, render_rect, false, right_cuda_stream);
+            }
+        }
+    }
+
+    ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());
+    glfwSwapBuffers(main_window);
+}

src/mvs_camera.cpp

@@ -0,0 +1,202 @@
+#include "mvs_camera.h"
+#include "simple_mq.h"
+#include "utility.hpp"
+
+#include <MvCameraControl.h>
+
+#include <opencv2/core/mat.hpp>
+
+#include <spdlog/spdlog.h>
+
+using namespace simple_mq_singleton;
+
+namespace mvs {
+
+    bool check_api_call(int api_ret, unsigned int line_number,
+                        const char *file_name, const char *api_call_str) {
+        if (api_ret == MV_OK) [[likely]] return true;
+        SPDLOG_ERROR("MVS api call {} failed at {}:{} with error 0x{:x}",
+                     api_call_str, file_name, line_number, api_ret);
+        return false;
+    }
+
+#define API_CHECK(api_call) \
+    check_api_call( \
+        api_call, __LINE__, __FILE__, #api_call)
+
+#define API_CHECK_P(api_call) \
+    if (!check_api_call( \
+        api_call, __LINE__, __FILE__, #api_call)) [[unlikely]] \
+        return nullptr
+
+#define API_CHECK_B(api_call) \
+    if (!check_api_call( \
+        api_call, __LINE__, __FILE__, #api_call)) [[unlikely]] \
+        return false
+
+    struct camera::impl {
+        void *handle = nullptr;
+        std::string name;
+        pixel_type type = RG_8;
+        int image_out_index = -1;
+        int frame_width = -1, frame_height = -1;
+        bool is_capture = false;
+
+        ~impl() {
+            if (is_capture) {
+                API_CHECK(MV_CC_StopGrabbing(handle));
+            }
+            API_CHECK(MV_CC_CloseDevice(handle));
+            API_CHECK(MV_CC_DestroyHandle(handle));
+            SPDLOG_INFO("MVS camera {} closed.", name);
+        };
+
+        void on_error_impl(unsigned int msg_type) const {
+            if (msg_type == 0x8003) return; // stop capture event, not an error
+            SPDLOG_ERROR("MVS camera {} exception 0x{:x}.", name, msg_type);
+        }
+
+        static void on_error(unsigned int msg_type, void *user_data) {
+            ((impl *) user_data)->on_error_impl(msg_type);
+        }
+
+        size_t calc_frame_size() const {
+            switch (type) {
+                case RG_8:
+                    return frame_width * frame_height * sizeof(uint8_t);
+            }
+            unreachable();
+        }
+
+        cv::Mat create_mat_view(unsigned char *data) {
+            switch (type) {
+                case RG_8:
+                    return cv::Mat{frame_height, frame_width, CV_8UC1, data};
+            }
+            unreachable();
+        }
+
+        void on_image_impl(unsigned char *data, MV_FRAME_OUT_INFO_EX *frame_info) {
+            assert(frame_info->nFrameLen == calc_frame_size());
+            auto img_view = create_mat_view(data);
+            mq().update_variable(image_out_index, img_view.clone());
+        }
+
+        static void on_image(unsigned char *data, MV_FRAME_OUT_INFO_EX *frame_info, void *user_data) {
+            ((impl *) user_data)->on_image_impl(data, frame_info);
+        }
+
+        static MvGvspPixelType convert_pixel_type(pixel_type type) {
+            switch (type) {
+                case RG_8:
+                    return PixelType_Gvsp_BayerRG8;
+            }
+            unreachable();
+        }
+
+        static impl *create(const create_config &conf) {
+            // find
+            MV_CC_DEVICE_INFO_LIST dev_info_list;
+            API_CHECK_P(MV_CC_EnumDevices(MV_USB_DEVICE, &dev_info_list));
+            MV_CC_DEVICE_INFO *dev_info = nullptr;
+            for (int i = 0; i < dev_info_list.nDeviceNum; ++i) {
+                auto cur_dev_info = dev_info_list.pDeviceInfo[i];
+                auto cur_dev_name = (char *) cur_dev_info->SpecialInfo.stUsb3VInfo.chUserDefinedName;
+                if (cur_dev_name == conf.name) {
+                    dev_info = cur_dev_info;
+                    break;
+                }
+            }
+            if (dev_info == nullptr) {
+                SPDLOG_ERROR("MVS camera with name {} not found.", conf.name);
+                return nullptr;
+            }
+            void *handle = nullptr;
+            API_CHECK_P(MV_CC_CreateHandle(&handle, dev_info));
+            assert(handle != nullptr);
+
+            // open
+            API_CHECK_P(MV_CC_OpenDevice(handle, MV_ACCESS_Control));
+            API_CHECK_P(MV_CC_CloseDevice(handle)); // close and open again to fix some bug
+            API_CHECK_P(MV_CC_OpenDevice(handle, MV_ACCESS_Control));
+            SPDLOG_INFO("MVS camera {} opened.", conf.name);
+
+            // config
+            API_CHECK_P(MV_CC_SetEnumValue(handle, "PixelFormat",
+                                           convert_pixel_type(conf.pixel)));
+            API_CHECK_P(MV_CC_SetEnumValue(handle, "AcquisitionMode",
+                                           MV_CAM_ACQUISITION_MODE::MV_ACQ_MODE_CONTINUOUS));
+            API_CHECK_P(MV_CC_SetEnumValue(handle, "TriggerMode", MV_TRIGGER_MODE_OFF));
+            API_CHECK_P(MV_CC_SetBoolValue(handle, "AcquisitionFrameRateEnable", true));
+
+            // create impl
+            auto ret = new impl;
+            ret->handle = handle;
+            ret->name = conf.name;
+            ret->type = conf.pixel;
+            ret->image_out_index = conf.image_out_index;
+            MVCC_INTVALUE int_val;
+            API_CHECK_P(MV_CC_GetIntValue(handle, "Width", &int_val));
+            ret->frame_width = int_val.nCurValue;
+            API_CHECK_P(MV_CC_GetIntValue(handle, "Height", &int_val));
+            ret->frame_height = int_val.nCurValue;
+
+            // register callbacks
+            API_CHECK_P(MV_CC_RegisterExceptionCallBack(handle, impl::on_error, ret));
+            API_CHECK_P(MV_CC_RegisterImageCallBackEx(handle, impl::on_image, ret));
+
+            return ret;
+        }
+
+        bool set_capture_config(const capture_config &conf) {
+            API_CHECK_B(MV_CC_SetFloatValue(handle, "AcquisitionFrameRate", conf.frame_rate));
+            API_CHECK_B(MV_CC_SetFloatValue(handle, "ExposureTime", conf.expo_time_ms * 1000)); // ms -> us
+            API_CHECK_B(MV_CC_SetFloatValue(handle, "Gain", conf.gain_db));
+            return true;
+        }
+
+        bool start_capture() {
+            assert(!is_capture);
+            API_CHECK_B(MV_CC_StartGrabbing(handle));
+            is_capture = true;
+            SPDLOG_INFO("MVS camera {} started capturing.", name);
+            return true;
+        }
+
+        bool stop_capture() {
+            assert(is_capture);
+            API_CHECK_B(MV_CC_StopGrabbing(handle));
+            is_capture = false;
+            SPDLOG_INFO("MVS camera {} stopped capturing.", name);
+            return true;
+        }
+
+    };
+
+    camera::~camera() = default;
+
+    bool camera::set_capture_config(const mvs::capture_config &conf) {
+        return pimpl->set_capture_config(conf);
+    }
+
+    bool camera::start_capture() {
+        return pimpl->start_capture();
+    }
+
+    bool camera::stop_capture() {
+        return pimpl->stop_capture();
+    }
+
+    bool camera::is_capture() const {
+        return pimpl->is_capture;
+    }
+
+    camera *camera::create(const mvs::create_config &conf) {
+        auto pimpl = impl::create(conf);
+        if (pimpl == nullptr) return nullptr;
+        auto ret = new camera;
+        ret->pimpl.reset(pimpl);
+        return ret;
+    }
+
+}

src/mvs_camera.h

@@ -0,0 +1,47 @@
+#ifndef REMOTEAR3_MVS_CAMERA_H
+#define REMOTEAR3_MVS_CAMERA_H
+
+#include <memory>
+#include <string_view>
+
+namespace mvs {
+
+    enum pixel_type {
+        RG_8
+    };
+
+    struct create_config {
+        std::string_view name;
+        pixel_type pixel;
+        int image_out_index;
+    };
+
+    struct capture_config {
+        float frame_rate; // frame per second
+        float expo_time_ms;
+        float gain_db;
+    };
+
+    class camera {
+    public:
+
+        ~camera();
+
+        bool set_capture_config(const capture_config &conf);
+
+        bool start_capture();
+
+        bool stop_capture();
+
+        bool is_capture() const;
+
+        static camera *create(const create_config &conf);
+
+    private:
+        struct impl;
+        std::unique_ptr<impl> pimpl;
+    };
+}
+
+
+#endif //REMOTEAR3_MVS_CAMERA_H

src/simple_mq.cpp

@@ -0,0 +1,142 @@
+#include "simple_mq.h"
+
+#include <cassert>
+#include <condition_variable>
+#include <map>
+#include <mutex>
+#include <shared_mutex>
+
+struct simple_mq::impl {
+
+    struct variable_info {
+        std::shared_ptr<void> ptr;
+        std::type_index type;
+        uint64_t update_cnt = 0;
+        std::shared_mutex mu;
+        std::condition_variable_any cv;
+    };
+    using pool_type = std::map<index_type, variable_info *>;
+
+    pool_type pool;
+    std::shared_mutex pool_mu;
+    std::condition_variable_any pool_cv;
+
+    ~impl() {
+        for (auto &info: pool) {
+            delete info.second;
+        }
+    };
+
+    bool update_variable(index_type index,
+                         const std::shared_ptr<void> &ptr,
+                         std::type_index type) {
+        auto iter = pool_type::iterator{};
+        {
+            auto lock_pool = std::shared_lock{pool_mu};
+            iter = pool.find(index);
+        }
+        if (iter == pool.end()) return false;
+        auto &info = iter->second;
+        {
+            auto lock_variable = std::unique_lock{info->mu};
+            info->ptr = ptr;
+            info->type = type;
+            ++info->update_cnt;
+        }
+        info->cv.notify_all();
+        return true;
+    }
+
+    void create_variable(index_type index,
+                         const std::shared_ptr<void> &ptr,
+                         std::type_index type) {
+        auto info = new variable_info{ptr, type};
+        {
+            auto lock_pool = std::unique_lock{pool_mu};
+            pool.emplace(index, info);
+            ++info->update_cnt;
+        }
+        pool_cv.notify_all();
+    }
+
+    std::shared_ptr<void> query_variable(index_type index,
+                                         std::type_index type,
+                                         uint64_t *update_cnt) {
+        auto iter = pool_type::iterator{};
+        {
+            auto lock_pool = std::shared_lock{pool_mu};
+            iter = pool.find(index);
+        }
+        if (iter == pool.end()) {
+            if (update_cnt != nullptr) {
+                *update_cnt = 0;
+            }
+            return nullptr;
+        }
+        auto &info = iter->second;
+        {
+            auto lock_variable = std::shared_lock{info->mu};
+            assert(info->type == type);
+            if (update_cnt != nullptr) {
+                *update_cnt = info->update_cnt;
+            }
+            return info->ptr;
+        }
+    }
+
+    bool wait_variable_update(index_type index, uint64_t old_cnt) {
+        auto iter = pool_type::iterator{};
+        {
+            auto lock_pool = std::shared_lock{pool_mu};
+            iter = pool.find(index);
+        }
+        if (iter == pool.end()) return false;
+        auto &info = iter->second;
+        {
+            auto lock_variable = std::shared_lock{info->mu};
+            info->cv.wait(lock_variable, [=] {
+                return info->update_cnt > old_cnt;
+            });
+        }
+        return true;
+    }
+
+    void wait_variable_create(index_type index) {
+        auto lock_pool = std::shared_lock{pool_mu};
+        pool_cv.wait(lock_pool, [=, this] {
+            return pool.contains(index);
+        });
+    }
+};
+
+simple_mq::simple_mq()
+        : pimpl(std::make_unique<impl>()) {}
+
+simple_mq::~simple_mq() = default;
+
+void simple_mq::update_variable_impl(index_type index,
+                                     const std::shared_ptr<void> &ptr,
+                                     std::type_index type) {
+    auto ret = pimpl->update_variable(index, ptr, type);
+    if (ret) [[likely]] return;
+    pimpl->create_variable(index, ptr, type);
+}
+
+std::shared_ptr<void> simple_mq::query_variable_impl(index_type index,
+                                                     std::type_index type,
+                                                     uint64_t *update_cnt) {
+    return pimpl->query_variable(index, type, update_cnt);
+}
+
+void simple_mq::wait_variable(index_type index, uint64_t old_cnt) {
+    auto ret = pimpl->wait_variable_update(index, old_cnt);
+    if (ret)[[likely]] return;
+    pimpl->wait_variable_create(index);
+}
+
+namespace simple_mq_singleton {
+    simple_mq &mq() {
+        static simple_mq instance;
+        return instance;
+    }
+}

src/simple_mq.h

@@ -0,0 +1,63 @@
+#ifndef REMOTEAR3_SIMPLE_MQ_H
+#define REMOTEAR3_SIMPLE_MQ_H
+
+#include <cstdint>
+#include <memory>
+#include <type_traits>
+#include <typeindex>
+#include <typeinfo>
+
+class simple_mq {
+public:
+    using index_type = int;
+
+    simple_mq();
+
+    ~simple_mq();
+
+    template<typename T>
+    void update_variable_ptr(index_type index,
+                             const std::shared_ptr<T> &ptr) {
+        update_variable_impl(index,
+                             std::static_pointer_cast<void>(ptr),
+                             typeid(T));
+    }
+
+    template<typename T>
+    void update_variable(index_type index, T &&value) {
+        using RT = typename std::remove_cvref_t<T>;
+        update_variable_ptr(index,
+                            std::make_shared<RT>(std::forward<T>(value)));
+    }
+
+    template<typename T>
+    std::shared_ptr<T> query_variable_ptr(index_type index, uint64_t *update_cnt = nullptr) {
+        return std::static_pointer_cast<T>(
+                query_variable_impl(index, typeid(T), update_cnt));
+    }
+
+    template<typename T>
+    T query_variable(index_type index, uint64_t *update_cnt = nullptr) {
+        return *query_variable_ptr<T>(index, update_cnt);
+    }
+
+    void wait_variable(index_type index, uint64_t old_cnt);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+
+    void update_variable_impl(index_type index,
+                              const std::shared_ptr<void> &ptr,
+                              std::type_index type);
+
+    std::shared_ptr<void> query_variable_impl(index_type index,
+                                              std::type_index type,
+                                              uint64_t *update_cnt);
+};
+
+namespace simple_mq_singleton {
+    simple_mq &mq();
+}
+
+#endif //REMOTEAR3_SIMPLE_MQ_H

src/simple_opengl.cpp

@@ -0,0 +1,415 @@
+#include "cuda_helper.hpp"
+#include "simple_opengl.h"
+
+#include <cuda_gl_interop.h>
+
+namespace simple_opengl_impl {
+
+    constexpr auto simple_vert_shader_source = R"(
+        #version 460
+        layout (location = 0) in vec2 pos_in;
+        layout (location = 1) in vec2 tex_coord_in;
+        out vec2 tex_coord;
+        void main() {
+            gl_Position = vec4(pos_in, 0, 1);
+            tex_coord = tex_coord_in;
+        }
+    )";
+
+    constexpr auto simple_frag_shader_source = R"(
+        #version 460
+        layout (location = 0) out vec4 color_out;
+        in vec2 tex_coord;
+        uniform sampler2D tex_sampler;
+        void main() {
+            color_out = texture(tex_sampler, tex_coord);
+        }
+    )";
+
+    constexpr auto remap_frag_shader_source = R"(
+        #version 460
+        layout (location = 0) out vec4 color_out;
+        in vec2 tex_coord;
+        uniform sampler2D image_tex;
+        uniform sampler2D remap_tex;
+        void main() {
+            vec2 tex_coord_real = texture(remap_tex, tex_coord).xy;
+            color_out = texture(image_tex, tex_coord_real);
+        }
+    )";
+
+    constexpr GLuint rect_indices[] = {
+            0, 1, 3, // first triangle
+            1, 2, 3 // second triangle
+    };
+
+    struct smart_pixel_buffer {
+        GLuint id = 0;
+        cudaGraphicsResource *res = nullptr;
+
+        ~smart_pixel_buffer() {
+            deallocate();
+        }
+
+        void create(GLenum target, GLenum flags, GLsizeiptr size) {
+            if (size == last_size) [[likely]] return;
+            deallocate();
+            allocate(target, flags, size);
+        }
+
+        void *map_pointer(cudaStream_t stream) {
+            void *ptr;
+            size_t size;
+            CUDA_API_CHECK_P(cudaGraphicsMapResources(1, &res, stream));
+            CUDA_API_CHECK_P(cudaGraphicsResourceGetMappedPointer(&ptr, &size, res));
+            assert(size == last_size);
+            last_stream = stream;
+            return ptr;
+        }
+
+        void unmap_pointer() {
+            CUDA_API_CHECK(cudaGraphicsUnmapResources(1, &res, last_stream));
+        }
+
+    private:
+        GLsizeiptr last_size = 0;
+        cudaStream_t last_stream = nullptr;
+
+        void allocate(GLenum target, GLenum flags, GLsizeiptr size) {
+            glGenBuffers(1, &id);
+            glBindBuffer(target, id);
+            glBufferStorage(target, size, nullptr, flags);
+            glBindBuffer(target, 0);
+            last_size = size;
+
+            // register CUDA resource
+            if (target == GL_PIXEL_UNPACK_BUFFER) {
+                CUDA_API_CHECK(cudaGraphicsGLRegisterBuffer(
+                        &res, id, cudaGraphicsRegisterFlagsWriteDiscard));
+            } else {
+                assert(target == GL_PIXEL_PACK_BUFFER);
+                CUDA_API_CHECK(cudaGraphicsGLRegisterBuffer(
+                        &res, id, cudaGraphicsRegisterFlagsReadOnly));
+            }
+        }
+
+        void deallocate() {
+            if (id == 0) return;
+            glDeleteBuffers(1, &id);
+            id = 0;
+            last_size = 0;
+
+            // unregister CUDA resource
+            CUDA_API_CHECK(cudaGraphicsUnregisterResource(res));
+            res = nullptr;
+        }
+    };
+
+    struct smart_texture {
+        GLuint id = 0;
+
+        void create(GLenum format, cv::Size size,
+                    GLint min_filter = GL_NEAREST, GLint max_filter = GL_NEAREST) {
+            if (size == last_size) [[likely]] return;
+            deallocate();
+            allocate(format, size, min_filter, max_filter);
+        }
+
+        ~smart_texture() {
+            deallocate();
+        }
+
+    private:
+        cv::Size last_size;
+
+        void allocate(GLenum format, cv::Size size,
+                      GLint min_filter, GLint max_filter) {
+            glGenTextures(1, &id);
+            glBindTexture(GL_TEXTURE_2D, id);
+            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, min_filter);
+            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, max_filter);
+            glTexStorage2D(GL_TEXTURE_2D, 1, format, size.width, size.height);
+            glBindTexture(GL_TEXTURE_2D, 0);
+            last_size = size;
+        }
+
+        void deallocate() {
+            if (id == 0) return;
+            glDeleteTextures(1, &id);
+            id = 0;
+            last_size = {};
+        }
+    };
+
+}
+
+using namespace simple_opengl_impl;
+
+struct simple_render::impl {
+    GLuint vao = 0, vbo = 0, ebo = 0;
+    GLuint simple_program = 0, remap_program = 0;
+    GLint image_tex_loc = 0, remap_tex_loc = 0;
+
+    smart_pixel_buffer image_pbo;
+    smart_texture image_tex, remap_tex;
+
+    impl() {
+        create_program();
+    }
+
+    ~impl() {
+        glDeleteVertexArrays(1, &vao);
+        glDeleteBuffers(1, &vbo);
+        glDeleteBuffers(1, &ebo);
+    }
+
+    static void compile_shader(GLuint shader, const char *source, const char *name) {
+        glShaderSource(shader, 1, &source, nullptr);
+        glCompileShader(shader);
+        GLint status, log_length;
+        glGetShaderiv(shader, GL_COMPILE_STATUS, &status);
+        glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &log_length);
+        auto info_log = (GLchar *) malloc(log_length);
+        glGetShaderInfoLog(shader, log_length, nullptr, info_log);
+        if (status == GL_TRUE) {
+            SPDLOG_INFO("Compile {} shader succeeded: {}", name, info_log);
+        } else {
+            SPDLOG_ERROR("Compile {} shader failed: {}", name, info_log);
+            RET_ERROR;
+        }
+        free(info_log);
+    }
+
+    static void check_program(GLuint program) {
+        GLint status, log_length;
+        glGetProgramiv(program, GL_LINK_STATUS, &status);
+        glGetProgramiv(program, GL_INFO_LOG_LENGTH, &log_length);
+        auto info_log = (GLchar *) malloc(log_length);
+        glGetProgramInfoLog(program, log_length, nullptr, info_log);
+        if (status == GL_TRUE) {
+            SPDLOG_INFO("Link program succeeded: {}", info_log);
+        } else {
+            SPDLOG_ERROR("Link program failed: {}", info_log);
+            RET_ERROR;
+        }
+        free(info_log);
+    }
+
+    void create_program() {
+        auto simple_vert_shader = glCreateShader(GL_VERTEX_SHADER);
+        auto simple_frag_shader = glCreateShader(GL_FRAGMENT_SHADER);
+        auto remap_frag_shader = glCreateShader(GL_FRAGMENT_SHADER);
+        compile_shader(simple_vert_shader, simple_vert_shader_source, "simple_vertex");
+        compile_shader(simple_frag_shader, simple_frag_shader_source, "simple_fragment");
+        compile_shader(remap_frag_shader, remap_frag_shader_source, "remap_fragment");
+
+        simple_program = glCreateProgram();
+        glAttachShader(simple_program, simple_vert_shader);
+        glAttachShader(simple_program, simple_frag_shader);
+        glLinkProgram(simple_program);
+        check_program(simple_program);
+
+        remap_program = glCreateProgram();
+        glAttachShader(remap_program, simple_vert_shader);
+        glAttachShader(remap_program, remap_frag_shader);
+        glLinkProgram(remap_program);
+        check_program(remap_program);
+
+        glDeleteShader(simple_vert_shader);
+        glDeleteShader(simple_frag_shader);
+        glDeleteShader(remap_frag_shader);
+
+        // uniform locations
+        image_tex_loc = glGetUniformLocation(remap_program, "image_tex");
+        remap_tex_loc = glGetUniformLocation(remap_program, "remap_tex");
+
+        // create buffers
+        glGenBuffers(1, &vbo);
+        glGenBuffers(1, &ebo);
+
+        // config vertex buffer
+        glBindBuffer(GL_ARRAY_BUFFER, vbo);
+        glBufferStorage(GL_ARRAY_BUFFER, 16 * sizeof(GLfloat), nullptr, GL_DYNAMIC_STORAGE_BIT);
+
+        // fill element buffer
+        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
+        glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(rect_indices), rect_indices, GL_STATIC_DRAW);
+
+        // config vertex array
+        glGenVertexArrays(1, &vao);
+        glBindVertexArray(vao);
+        glEnableVertexAttribArray(0);
+        glEnableVertexAttribArray(1);
+        glVertexAttribPointer(0, 2, GL_FLOAT, false, 4 * sizeof(GLfloat), (void *) 0);
+        glVertexAttribPointer(1, 2, GL_FLOAT, false, 4 * sizeof(GLfloat), (void *) (2 * sizeof(GLfloat)));
+    }
+
+    void render_texture(GLuint tex, const simple_rect &rect, bool is_remap = false) {
+        // bind buffers
+        glUseProgram(is_remap ? remap_program : simple_program);
+        glBindVertexArray(vao);
+        glBindBuffer(GL_ARRAY_BUFFER, vbo);
+        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
+
+        // bind textures
+        if (is_remap) {
+            assert(remap_tex.id != 0);
+            glUniform1i(image_tex_loc, 0);
+            glUniform1i(remap_tex_loc, 1);
+            glActiveTexture(GL_TEXTURE0 + 0);
+            glBindTexture(GL_TEXTURE_2D, tex);
+            glActiveTexture(GL_TEXTURE0 + 1);
+            glBindTexture(GL_TEXTURE_2D, remap_tex.id);
+        } else {
+            glActiveTexture(GL_TEXTURE0 + 0);
+            glBindTexture(GL_TEXTURE_2D, tex);
+        }
+
+        // fill vertex buffer
+        GLfloat vertices[] = {
+                // 2 for position; 2 for texture
+                rect.x + rect.width, rect.y + rect.height, 1, 1, // top right
+                rect.x + rect.width, rect.y, 1, 0, // bottom right
+                rect.x, rect.y, 0, 0, // bottom left
+                rect.x, rect.y + rect.height, 0, 1 // top left
+        };
+        static_assert(sizeof(vertices) == 16 * sizeof(GLfloat));
+        glBufferSubData(GL_ARRAY_BUFFER, 0, sizeof(vertices), vertices);
+
+        // draw texture
+        glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, nullptr);
+    }
+
+    void upload_remap_data(const cv::Mat &data) {
+        // allocate texture
+        assert(data.type() == CV_32FC2);
+        remap_tex.create(GL_RG32F, data.size());
+
+        // copy data to texture
+        glBindTexture(GL_TEXTURE_2D, remap_tex.id);
+        glTexImage2D(GL_TEXTURE_2D, 0, GL_RG32F, data.cols, data.rows,
+                     0, GL_RG, GL_FLOAT, data.data);
+        glBindTexture(GL_TEXTURE_2D, 0);
+    }
+
+    void upload_gpu_mat(const cv::cuda::GpuMat &img, cudaStream_t stream) {
+        // allocate memory if needed
+        assert(img.type() == CV_8UC3);
+        auto pbo_pitch = img.size().width * img.elemSize() * sizeof(uint8_t);
+        auto img_bytes = img.size().height * pbo_pitch;
+        image_pbo.create(GL_PIXEL_PACK_BUFFER, GL_DYNAMIC_STORAGE_BIT, (GLsizeiptr) img_bytes);
+        image_tex.create(GL_RGBA8, img.size());
+
+        // copy image to texture
+        auto ptr = image_pbo.map_pointer(stream);
+        CUDA_API_CHECK(cudaMemcpy2DAsync(ptr, pbo_pitch, img.cudaPtr(), img.step, pbo_pitch,
+                                         img.size().height, cudaMemcpyDeviceToDevice, stream));
+        image_pbo.unmap_pointer();
+
+        // unpack pbo to texture
+        glBindBuffer(GL_PIXEL_UNPACK_BUFFER, image_pbo.id);
+        glBindTexture(GL_TEXTURE_2D, image_tex.id);
+        glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, img.size().width, img.size().height,
+                        GL_BGR, GL_UNSIGNED_BYTE, nullptr);
+        glBindTexture(GL_TEXTURE_2D, 0);
+        glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
+    }
+};
+
+simple_render::simple_render()
+        : pimpl(std::make_unique<impl>()) {}
+
+simple_render::~simple_render() = default;
+
+void simple_render::set_remap_data(const cv::Mat &data) {
+    pimpl->upload_remap_data(data);
+}
+
+void simple_render::render_rect(GLuint tex, const simple_rect &rect, bool remap) {
+    pimpl->render_texture(tex, rect, remap);
+}
+
+void simple_render::render_rect(const cv::cuda::GpuMat &img, const simple_rect &rect,
+                                bool remap, cudaStream_t stream) {
+    pimpl->upload_gpu_mat(img, stream);
+    pimpl->render_texture(pimpl->image_tex.id, rect, remap);
+}
+
+struct smart_frame_buffer::impl {
+
+    smart_frame_buffer *q_this = nullptr;
+    cv::Size last_size;
+    smart_texture color_tex, depth_tex;
+    smart_pixel_buffer pbo;
+
+    static void check_frame_buffer() {
+        auto status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
+        if (status != GL_FRAMEBUFFER_COMPLETE) [[unlikely]] {
+            SPDLOG_ERROR("Framebuffer is not complete 0x{:x}.", status);
+            RET_ERROR;
+        }
+    }
+
+    void create(cv::Size size) {
+        if (size == last_size) [[likely]] return;
+        deallocate();
+        allocate(size);
+    }
+
+    void allocate(cv::Size size) {
+        // allocate buffer and textures
+        auto pbo_size = size.area() * 4 * sizeof(uint8_t);
+        pbo.create(GL_PIXEL_PACK_BUFFER, GL_DYNAMIC_STORAGE_BIT, (GLsizeiptr) pbo_size);
+        color_tex.create(GL_RGB8, size);
+        depth_tex.create(GL_DEPTH_COMPONENT16, size);
+
+        // create frame buffer
+        glGenFramebuffers(1, &q_this->id);
+        glBindFramebuffer(GL_FRAMEBUFFER, q_this->id);
+        glBindTexture(GL_TEXTURE_2D, color_tex.id);
+        glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, color_tex.id, 0);
+        glBindTexture(GL_TEXTURE_2D, depth_tex.id);
+        glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_2D, depth_tex.id, 0);
+        check_frame_buffer();
+
+        last_size = size;
+    }
+
+    void deallocate() {
+        if (q_this->id == 0) return;
+        glDeleteFramebuffers(1, &q_this->id);
+        last_size = {};
+    }
+
+    void download(cv::cuda::GpuMat *img, cudaStream_t stream) {
+        assert(q_this->id != 0);
+        img->create(last_size, CV_8UC4);
+
+        // pack pixels into pbo
+        glBindFramebuffer(GL_FRAMEBUFFER, q_this->id);
+        glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo.id);
+        glReadPixels(0, 0, last_size.width, last_size.height, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, (void *) 0);
+        glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
+        glBindFramebuffer(GL_FRAMEBUFFER, 0);
+
+        // copy image to gpu mat
+        auto pbo_pitch = last_size.width * 4 * sizeof(uint8_t);
+        auto ptr = pbo.map_pointer(stream);
+        CUDA_API_CHECK(cudaMemcpy2DAsync(img->cudaPtr(), img->step, ptr, pbo_pitch, pbo_pitch,
+                                         last_size.height, cudaMemcpyDeviceToDevice, stream));
+        pbo.unmap_pointer();
+    }
+};
+
+smart_frame_buffer::smart_frame_buffer()
+        : pimpl(std::make_unique<impl>()) {}
+
+smart_frame_buffer::~smart_frame_buffer() = default;
+
+void smart_frame_buffer::create(cv::Size size) {
+    pimpl->create(size);
+}
+
+void smart_frame_buffer::download(cv::cuda::GpuMat *img, cudaStream_t stream) {
+    pimpl->download(img, stream);
+}

src/simple_opengl.h

@@ -0,0 +1,56 @@
+#ifndef REMOTEAR3_SIMPLE_OPENGL_H
+#define REMOTEAR3_SIMPLE_OPENGL_H
+
+#include <opencv2/core/cuda.hpp>
+#include <opencv2/core/mat.hpp>
+
+#include <glad/gl.h>
+
+#include <memory>
+
+struct simple_rect {
+    GLfloat x, y;
+    GLfloat width, height;
+};
+
+class simple_render {
+public:
+    simple_render();
+
+    ~simple_render();
+
+    void set_remap_data(const cv::Mat &data);
+
+    void render_rect(GLuint tex,
+                     const simple_rect &rect,
+                     bool remap = false);
+
+    void render_rect(const cv::cuda::GpuMat &img,
+                     const simple_rect &rect,
+                     bool remap = true,
+                     cudaStream_t stream = nullptr);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+class smart_frame_buffer {
+public:
+    GLuint id = 0;
+
+    smart_frame_buffer();
+
+    ~smart_frame_buffer();
+
+    void create(cv::Size size);
+
+    void download(cv::cuda::GpuMat *img,
+                  cudaStream_t stream = nullptr);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#endif //REMOTEAR3_SIMPLE_OPENGL_H

src/utility.hpp

@@ -0,0 +1,24 @@
+#ifndef REMOTEAR3_UTILITY_HPP
+#define REMOTEAR3_UTILITY_HPP
+
+#define RET_ERROR \
+    assert(false)
+
+#define RET_ERROR_B \
+    assert(false); \
+    return false
+
+// https://en.cppreference.com/w/cpp/utility/unreachable
+[[noreturn]] inline void unreachable() {
+    // Uses compiler specific extensions if possible.
+    // Even if no extension is used, undefined behavior is still raised by
+    // an empty function body and the noreturn attribute.
+#ifdef __GNUC__ // GCC, Clang, ICC
+    __builtin_unreachable();
+// #elifdef _MSC_VER // MSVC
+#else
+    __assume(false);
+#endif
+}
+
+#endif //REMOTEAR3_UTILITY_HPP

src/variable_defs.h

@@ -0,0 +1,13 @@
+#ifndef REMOTEAR3_VARIABLE_DEFS_H
+#define REMOTEAR3_VARIABLE_DEFS_H
+
+constexpr auto IMG_RAW_HOST_LEFT = 0;
+constexpr auto IMG_RAW_HOST_RIGHT = 1;
+
+constexpr auto MAIN_WINDOW_WIDTH = 20;
+constexpr auto MAIN_WINDOW_HEIGHT = 21;
+
+// global variable declaration
+
+
+#endif //REMOTEAR3_VARIABLE_DEFS_H