Implemented OpenGL rendering.

CMakeLists.txt

@@ -3,12 +3,13 @@ project(RemoteAR2 LANGUAGES C CXX CUDA)
 
 set(CMAKE_CXX_STANDARD 20)
 
-add_executable(RemoteAR2 src/main.cpp)
+add_executable(RemoteAR2 src/main.cpp src/augment_renderer.cpp)
 
 # OpenGL config
 find_package(OpenGL REQUIRED)
 target_include_directories(${PROJECT_NAME} PRIVATE ${OPENGL_INCLUDE_DIR})
 target_link_libraries(${PROJECT_NAME} OpenGL::GL)
+target_sources(${PROJECT_NAME} PRIVATE src/texture_renderer.cpp)
 
 # glfw config
 if (WIN32)
@@ -99,6 +100,10 @@ target_link_libraries(${PROJECT_NAME} ${CAPI_LIB})
 find_package(Eigen3 REQUIRED)
 target_link_libraries(${PROJECT_NAME} Eigen3::Eigen)
 
+# CUDA config
+target_include_directories(${PROJECT_NAME} PRIVATE ${CUDA_INCLUDE_DIRS})
+target_link_libraries(${PROJECT_NAME} ${CUDA_LIBRARIES})
+
 # NvEnc config
 if (WIN32)
     set(NVCODEC_DIR C:/BuildEssentials/CUDA/Video_Codec_SDK_12.0.16)

src/augment_renderer.cpp

@@ -0,0 +1,97 @@
+#include "augment_renderer.h"
+#include "cuda_helper.hpp"
+
+#include <cuda_gl_interop.h>
+
+#include <glad/gl.h>
+
+struct augment_renderer::impl {
+    const cv::cuda::GpuMat *bg_img = nullptr;
+    GLuint bg_tex = 0, bg_pbo = 0;
+    cudaGraphicsResource *bg_res = nullptr;
+
+    texture_renderer *tex_renderer = nullptr;
+    const render_config *config = nullptr;
+
+    bool initialize() {
+        // generate and allocate pixel buffer
+        glGenBuffers(1, &bg_pbo);
+        glBindBuffer(GL_PIXEL_UNPACK_BUFFER, bg_pbo);
+        glBufferStorage(GL_PIXEL_UNPACK_BUFFER, rgb_image_size, nullptr, GL_DYNAMIC_STORAGE_BIT);
+        glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
+
+        // generate and allocate texture
+        glGenTextures(1, &bg_tex);
+        glBindTexture(GL_TEXTURE_2D, bg_tex);
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+        glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, image_width, image_height);
+
+        // register background pbo
+        CUDA_API_CHECK(cudaGraphicsGLRegisterBuffer(&bg_res, bg_pbo,
+                                                    cudaGraphicsRegisterFlagsWriteDiscard));
+
+        return true;
+    }
+
+    bool render_background() {
+        if (bg_img == nullptr) return true;
+
+        // upload background image to pbo
+        void *pbo_ptr;
+        size_t pbo_size;
+        CUDA_API_CHECK(cudaGraphicsMapResources(1, &bg_res));
+        CUDA_API_CHECK(cudaGraphicsResourceGetMappedPointer(&pbo_ptr, &pbo_size, bg_res));
+        assert(pbo_size == rgb_image_size);
+        CUDA_API_CHECK(cudaMemcpy2D(pbo_ptr, rgb_image_pitch, bg_img->cudaPtr(),
+                                    bg_img->step, rgb_image_pitch, image_height, cudaMemcpyDeviceToDevice));
+        CUDA_API_CHECK(cudaGraphicsUnmapResources(1, &bg_res));
+
+        // unpack pbo to texture
+        glBindBuffer(GL_PIXEL_UNPACK_BUFFER, bg_pbo);
+        glBindTexture(GL_TEXTURE_2D, bg_tex);
+        glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, image_width, image_height, GL_BGR, GL_UNSIGNED_BYTE, nullptr);
+        glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
+
+        // render texture
+        assert(tex_renderer != nullptr);
+        assert(config != nullptr);
+        texture_renderer::render_config tex_config{
+                .tex = bg_tex,
+                .x = config->x,
+                .y = config->y,
+                .width = config->width,
+                .height = config->height
+        };
+        tex_renderer->render(&tex_config);
+
+        return true;
+    }
+
+    bool render() {
+        CALL_CHECK(render_background());
+        return true;
+    }
+
+};
+
+augment_renderer::augment_renderer()
+        : pimpl(std::make_unique<impl>()) {}
+
+augment_renderer::~augment_renderer() = default;
+
+bool augment_renderer::initialize(texture_renderer *renderer) {
+    pimpl->tex_renderer = renderer;
+    return pimpl->initialize();
+}
+
+void augment_renderer::set_background(const cv::cuda::GpuMat *background) {
+    assert(background->rows == image_height);
+    assert(background->cols == image_width);
+    pimpl->bg_img = background;
+}
+
+bool augment_renderer::render(const render_config *config) {
+    pimpl->config = config;
+    return pimpl->render();
+}

src/augment_renderer.h

@@ -0,0 +1,34 @@
+#ifndef REMOTEAR2_AUGMENT_RENDERER_H
+#define REMOTEAR2_AUGMENT_RENDERER_H
+
+#include "texture_renderer.h"
+
+#include <opencv2/core/cuda.hpp>
+
+#include <memory>
+
+class augment_renderer {
+public:
+
+    augment_renderer();
+
+    ~augment_renderer();
+
+    bool initialize(texture_renderer *renderer);
+
+    void set_background(const cv::cuda::GpuMat *background);
+
+    struct render_config {
+        float x, y;
+        float width, height;
+    };
+
+    bool render(const render_config *config);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+
+#endif //REMOTEAR2_AUGMENT_RENDERER_H

src/config.h

@@ -9,9 +9,11 @@
 static constexpr auto main_window_width = 800;
 static constexpr auto main_window_height = 600;
 
-static constexpr auto camera_image_width = 2448;
-static constexpr auto camera_image_height = 2048;
-static constexpr auto camera_image_size = camera_image_width * camera_image_height * 1; // 1 byte per elem
+static constexpr auto image_width = 2448;
+static constexpr auto image_height = 2048;
+static constexpr auto raw_image_size = image_width * image_height;
+static constexpr auto rgb_image_size = raw_image_size * 3;
+static constexpr auto rgb_image_pitch = image_width * 3;
 
 static constexpr auto left_camera_name = "LeftEye";
 static constexpr auto right_camera_name = "RightEye";
@@ -38,7 +40,7 @@ inline bool check_function_call(bool function_ret, unsigned int line_number,
 
 #define CALL_CHECK(function_call) \
     if (!check_function_call( \
-        function_call, __LINE__, __FILE__, #function_call)) \
+        function_call, __LINE__, __FILE__, #function_call)) [[unlikely]] \
         return false
 
 #endif //REMOTEAR2_CONFIG_H

src/cuda_helper.hpp

@@ -0,0 +1,36 @@
+#ifndef REMOTEAR2_CUDA_HELPER_HPP
+#define REMOTEAR2_CUDA_HELPER_HPP
+
+#include "config.h"
+
+#include <cuda.h>
+#include <cuda_runtime.h>
+
+inline bool check_cuda_api_call(CUresult api_ret, unsigned int line_number,
+                                const char *file_name, const char *api_call_str) {
+    if (api_ret == CUDA_SUCCESS) [[likely]] return true;
+    const char *error_name, *error_str;
+    auto ret = cuGetErrorName(api_ret, &error_name);
+    if (ret != CUDA_SUCCESS) [[unlikely]] error_name = "Unknown";
+    ret = cuGetErrorString(api_ret, &error_str);
+    if (ret != CUDA_SUCCESS) [[unlikely]] error_str = "Unknown";
+    SPDLOG_ERROR("CUDA api call {} failed at {}:{} with error 0x{:x}:{}, {}.",
+                 api_call_str, file_name, line_number,
+                 api_ret, error_name, error_str);
+    RET_ERROR;
+}
+
+inline bool check_cuda_api_call(cudaError api_ret, unsigned int line_number,
+                                const char *file_name, const char *api_call_str) {
+    if (api_ret == cudaSuccess) [[likely]] return true;
+    SPDLOG_ERROR("CUDA api call {} failed at {}:{} with error 0x{:x}.",
+                 api_call_str, file_name, line_number, api_ret);
+    RET_ERROR;
+}
+
+#define CUDA_API_CHECK(api_call) \
+    if (!check_cuda_api_call( \
+        api_call, __LINE__, __FILE__, #api_call)) [[unlikely]] \
+        return false
+
+#endif //REMOTEAR2_CUDA_HELPER_HPP

src/main.cpp

@@ -1,5 +1,7 @@
+#include "augment_renderer.h"
 #include "config.h"
 #include "stereo_camera.hpp"
+#include "texture_renderer.h"
 
 #include <imgui.h>
 #include <imgui_impl_glfw.h>
@@ -27,13 +29,25 @@ int main() {
     auto main_window = glfwCreateWindow(main_window_width, main_window_height, "RemoteAR", nullptr, nullptr);
     assert(main_window != nullptr);
     glfwMakeContextCurrent(main_window);
-    glfwSwapInterval(1);
+//    glfwSwapInterval(1);
 
     // load opengl functions
     auto version = gladLoadGL(glfwGetProcAddress);
     assert(version > 0);
     SPDLOG_INFO("Loaded OpenGL {}.{}", GLAD_VERSION_MAJOR(version), GLAD_VERSION_MINOR(version));
 
+#ifndef NDEBUG
+    // log opengl error
+    glEnable(GL_DEBUG_OUTPUT);
+    glDebugMessageCallback([](GLenum source, GLenum type, GLuint id, GLenum severity,
+                              GLsizei length, const GLchar *message, const void *user_data) {
+        if (type == GL_DEBUG_TYPE_ERROR) {
+            SPDLOG_ERROR("OpenGL error: type = {}, severity = {}, message = {}", type, severity, message);
+            assert(false);
+        }
+    }, nullptr);
+#endif
+
     // setup imgui context
     IMGUI_CHECKVERSION();
     ImGui::CreateContext();
@@ -45,6 +59,13 @@ int main() {
 
     // working staffs
     stereo_camera camera;
+    texture_renderer tex_renderer;
+
+    augment_renderer left_ar, right_ar;
+    left_ar.initialize(&tex_renderer);
+    right_ar.initialize(&tex_renderer);
+    left_ar.set_background(&camera.left_rgb_image);
+    right_ar.set_background(&camera.right_rgb_image);
 
     // main loop
     while (!glfwWindowShouldClose(main_window)) {
@@ -112,14 +133,27 @@ int main() {
 
         }
         ImGui::End();
-
         ImGui::Render();
+
         int frame_width, frame_height;
         glfwGetFramebufferSize(main_window, &frame_width, &frame_height);
         glViewport(0, 0, frame_width, frame_height);
         glClear(GL_COLOR_BUFFER_BIT);
+
+        if (camera.is_capturing()) {
+            camera.retrieve_raw_images();
+            camera.debayer_images();
+
+            augment_renderer::render_config config{-1, 1, 2, -2};
+            left_ar.render(&config);
+        }
+
         ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());
         glfwSwapBuffers(main_window);
+
+        if (camera.is_capturing()) {
+            glFlush();
+        }
     }
 
     // cleanup

src/mvs_camera.cpp

@@ -9,6 +9,8 @@
 
 #include <spdlog/spdlog.h>
 
+#include <atomic>
+
 bool check_mvs_api_call(int api_ret, unsigned int line_number,
                         const char *file_name, const char *api_call_str) {
     if (api_ret == MV_OK) [[likely]] return true;
@@ -19,7 +21,7 @@ bool check_mvs_api_call(int api_ret, unsigned int line_number,
 
 #define MVS_API_CHECK(api_call) \
     if (!check_mvs_api_call( \
-        api_call, __LINE__, __FILE__, #api_call)) \
+        api_call, __LINE__, __FILE__, #api_call)) [[unlikely]] \
         return false
 
 struct mvs_camera::impl {
@@ -38,9 +40,9 @@ struct mvs_camera::impl {
     }
 
     static void on_image(unsigned char *data, MV_FRAME_OUT_INFO_EX *frame_info, void *user_data) {
-        assert(frame_info->nFrameLen == camera_image_size);
+        assert(frame_info->nFrameLen == raw_image_size);
         auto pimpl = (impl *) user_data;
-        auto host_img = cv::Mat{camera_image_height, camera_image_width, CV_8UC1, data};
+        auto host_img = cv::Mat{image_height, image_width, CV_8UC1, data};
 
         // upload image to gpu
         if (pimpl->inner_img == nullptr) [[unlikely]] {

src/mvs_camera.h

@@ -3,7 +3,6 @@
 
 #include <opencv2/core/cuda.hpp>
 
-#include <atomic>
 #include <memory>
 #include <string_view>
 

src/stereo_camera.hpp

@@ -4,12 +4,25 @@
 #include "config.h"
 #include "mvs_camera.h"
 
+#include <opencv2/cudaimgproc.hpp>
+
 #include <atomic>
 #include <thread>
 
 struct stereo_camera {
 
     mvs_camera left_camera, right_camera;
+    cv::cuda::GpuMat *left_raw_image = nullptr, *right_raw_image = nullptr;
+    cv::cuda::GpuMat left_rgb_image, right_rgb_image;
+
+    stereo_camera() {
+        left_rgb_image = cv::cuda::GpuMat{image_height, image_width, CV_8UC3};
+        right_rgb_image = cv::cuda::GpuMat{image_height, image_width, CV_8UC3};
+    }
+
+    ~stereo_camera() {
+        close();
+    }
 
     bool open() {
         if (!open_impl()) { // keep consistency
@@ -20,6 +33,7 @@ struct stereo_camera {
     }
 
     void close() {
+        stop_capture();
         left_camera.close();
         right_camera.close();
     }
@@ -34,9 +48,6 @@ struct stereo_camera {
     }
 
     void stop_capture() {
-        left_camera.stop_capture();
-        right_camera.stop_capture();
-
         // stop trigger thread
         if (trigger_thread != nullptr) {
 
@@ -48,6 +59,9 @@ struct stereo_camera {
             should_stop.clear();
             trigger_thread = nullptr;
         }
+
+        left_camera.stop_capture();
+        right_camera.stop_capture();
     }
 
     bool software_trigger() {
@@ -55,6 +69,26 @@ struct stereo_camera {
                right_camera.software_trigger();
     }
 
+    void retrieve_raw_images() {
+        assert(is_capturing());
+
+        // clean old images
+        if (left_raw_image != nullptr) {
+            delete left_raw_image;
+            left_raw_image = nullptr;
+        }
+        if (right_raw_image != nullptr) {
+            delete right_raw_image;
+            right_raw_image = nullptr;
+        }
+
+        // retrieve new images
+        left_camera.retrieve_image(&left_raw_image);
+        right_camera.retrieve_image(&right_raw_image);
+        assert(left_raw_image != nullptr);
+        assert(right_raw_image != nullptr);
+    }
+
     bool is_opened() {
         assert(left_camera.is_opened() == right_camera.is_opened());
         return left_camera.is_opened();
@@ -65,6 +99,11 @@ struct stereo_camera {
         return left_camera.is_capturing();
     }
 
+    void debayer_images() {
+        cv::cuda::cvtColor(*left_raw_image, left_rgb_image, cv::COLOR_BayerRG2RGB);
+        cv::cuda::cvtColor(*right_raw_image, right_rgb_image, cv::COLOR_BayerRG2RGB);
+    }
+
 private:
 
     std::thread *trigger_thread = nullptr;
@@ -95,7 +134,7 @@ private:
                 if (should_stop.test()) break;
                 software_trigger();
 
-                // resume at (almost) exact time
+                // resume at (almost) exact time // TODO: test performance
                 next_trigger_time += trigger_interval;
                 std::this_thread::sleep_until(next_trigger_time - default_spin_time);
                 while (std::chrono::high_resolution_clock::now() < next_trigger_time)

src/texture_renderer.cpp

@@ -0,0 +1,147 @@
+#include "texture_renderer.h"
+
+#include <spdlog/spdlog.h>
+
+#include <cassert>
+#include <cstddef>
+
+static constexpr auto vertex_shader_source = R"(
+        #version 460
+        layout (location = 0) in vec2 pos_in;
+        layout (location = 1) in vec2 tex_coord_in;
+        out vec2 tex_coord;
+        void main() {
+            gl_Position = vec4(pos_in, 0, 1);
+            tex_coord = tex_coord_in;
+        }
+    )";
+
+static constexpr auto fragment_shader_source = R"(
+        #version 460
+        layout (location = 0) out vec4 color_out;
+        in vec2 tex_coord;
+        uniform sampler2D tex_sampler;
+        void main() {
+            color_out = texture(tex_sampler, tex_coord);
+        }
+    )";
+
+static constexpr GLuint indices[] = {
+        0, 1, 3, // first triangle
+        1, 2, 3 // second triangle
+};
+
+struct texture_renderer::impl {
+
+    GLuint vertex_array = 0;
+    GLuint vertex_buffer = 0, element_buffer = 0;
+    GLuint program = 0;
+
+    impl() {
+        // build program
+        auto vertex_shader = glCreateShader(GL_VERTEX_SHADER);
+        auto fragment_shader = glCreateShader(GL_FRAGMENT_SHADER);
+        compile_shader(vertex_shader, vertex_shader_source, "vertex");
+        compile_shader(fragment_shader, fragment_shader_source, "fragment");
+        program = glCreateProgram();
+        glAttachShader(program, vertex_shader);
+        glAttachShader(program, fragment_shader);
+        glLinkProgram(program);
+        check_program();
+        glDeleteShader(vertex_shader);
+        glDeleteShader(fragment_shader);
+
+        // create buffers
+        static_assert(offsetof(impl, element_buffer) - offsetof(impl, vertex_buffer) == sizeof(GLuint));
+        glGenBuffers(2, &vertex_buffer);
+
+        // config vertex buffer
+        glBindBuffer(GL_ARRAY_BUFFER, vertex_buffer);
+        glBufferStorage(GL_ARRAY_BUFFER, 16 * sizeof(GLfloat), nullptr, GL_DYNAMIC_STORAGE_BIT);
+
+        // fill element buffer
+        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, element_buffer);
+        glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(indices), indices, GL_STATIC_DRAW);
+
+        // config vertex array
+        glGenVertexArrays(1, &vertex_array);
+        glBindVertexArray(vertex_array);
+        glEnableVertexAttribArray(0);
+        glEnableVertexAttribArray(1);
+        glVertexAttribPointer(0, 2, GL_FLOAT, false, 4 * sizeof(GLfloat), (void *) 0);
+        glVertexAttribPointer(1, 2, GL_FLOAT, false, 4 * sizeof(GLfloat), (void *) (2 * sizeof(GLfloat)));
+    }
+
+    ~impl() {
+        glDeleteBuffers(2, &vertex_buffer);
+        glDeleteProgram(program);
+    }
+
+    static void compile_shader(GLuint shader, const char *source, const char *name) {
+        glShaderSource(shader, 1, &source, nullptr);
+        glCompileShader(shader);
+        GLint status, log_length;
+        glGetShaderiv(shader, GL_COMPILE_STATUS, &status);
+        glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &log_length);
+        auto info_log = (GLchar *) malloc(log_length);
+        glGetShaderInfoLog(shader, log_length, nullptr, info_log);
+        if (status == GL_TRUE) {
+            SPDLOG_INFO("Compile {} shader succeeded: {}", name, info_log);
+        } else {
+            SPDLOG_ERROR("Compile {} shader failed: {}", name, info_log);
+            assert(false);
+        }
+        free(info_log);
+    }
+
+    void check_program() {
+        GLint status, log_length;
+        glGetProgramiv(program, GL_LINK_STATUS, &status);
+        glGetProgramiv(program, GL_INFO_LOG_LENGTH, &log_length);
+        auto info_log = (GLchar *) malloc(log_length);
+        glGetProgramInfoLog(program, log_length, nullptr, info_log);
+        if (status == GL_TRUE) {
+            SPDLOG_INFO("Link program succeeded: {}", info_log);
+        } else {
+            SPDLOG_ERROR("Link program failed: {}", info_log);
+            assert(false);
+        }
+        free(info_log);
+    }
+
+    void render(const texture_renderer::render_config *config) {
+        auto x = config->x, y = config->y;
+        auto width = config->width, height = config->height;
+
+        // bindings
+        glUseProgram(program);
+        glBindVertexArray(vertex_array);
+        glBindBuffer(GL_ARRAY_BUFFER, vertex_buffer);
+        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, element_buffer);
+        glBindTexture(GL_TEXTURE_2D, config->tex);
+
+        // fill vertex buffer
+        GLfloat vertices[] = {
+                // 2 for position; 2 for texture
+                x + width, y + height, 1, 1, // top right
+                x + width, y, 1, 0, // bottom right
+                x, y, 0, 0, // bottom left
+                x, y + height, 0, 1 // top left
+        };
+        static_assert(sizeof(vertices) == 16 * sizeof(GLfloat));
+        glBufferSubData(GL_ARRAY_BUFFER, 0, sizeof(vertices), vertices);
+
+        // draw texture
+        glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, nullptr);
+    }
+
+};
+
+texture_renderer::texture_renderer()
+        : pimpl(std::make_unique<impl>()) {}
+
+texture_renderer::~texture_renderer() = default;
+
+void texture_renderer::render(const texture_renderer::render_config *config) {
+    pimpl->render(config);
+}

src/texture_renderer.h

@@ -0,0 +1,28 @@
+#ifndef REMOTEAR2_TEXTURE_RENDERER_H
+#define REMOTEAR2_TEXTURE_RENDERER_H
+
+#include <glad/gl.h>
+
+#include <memory>
+
+class texture_renderer {
+public:
+    texture_renderer();
+
+    ~texture_renderer();
+
+    struct render_config {
+        GLuint tex;
+        GLfloat x, y;
+        GLfloat width, height;
+    };
+
+    void render(const render_config *config);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+
+#endif //REMOTEAR2_TEXTURE_RENDERER_H