Implemented HDR imaging.

src/augment_renderer.cpp

@@ -13,22 +13,29 @@
 #include <vtkTextureObject.h>
 
 struct augment_renderer::impl {
+    cv::Size bg_size;
     const cv::cuda::GpuMat *bg_img = nullptr;
     GLuint bg_tex = 0, remap_tex = 0, bg_pbo = 0;
     cudaGraphicsResource *bg_res = nullptr;
 
-    texture_renderer *tex_renderer = nullptr;
     const render_config *config = nullptr;
 
+    static std::unique_ptr<texture_renderer> tex_renderer;
+
     // Switching between multiple render windows will case OpenGL to flush and waste much time.
     static vtkSmartPointer<vtkRenderer> vtk_render;
     static vtkSmartPointer<vtkGenericOpenGLRenderWindow> vtk_window;
 
     vtkSmartPointer<vtkCamera> vtk_cam;
 
-    const camera_intrinsics *cam_info = nullptr;
+    const camera_info *cam_info = nullptr;
     bool is_augment = true;
 
+    impl(const camera_info *_cam_info, cv::Size _bg_size)
+            : cam_info(_cam_info), bg_size(_bg_size) {
+        CALL_ASSERT(initialize());
+    }
+
     ~impl() {
         cudaGraphicsUnregisterResource(bg_res);
     }
@@ -37,7 +44,7 @@ struct augment_renderer::impl {
         // generate and allocate pixel buffer
         glGenBuffers(1, &bg_pbo);
         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, bg_pbo);
-        glBufferStorage(GL_PIXEL_UNPACK_BUFFER, rgb_image_size, nullptr, GL_DYNAMIC_STORAGE_BIT);
+        glBufferStorage(GL_PIXEL_UNPACK_BUFFER, bg_size.area() * 3, nullptr, GL_DYNAMIC_STORAGE_BIT);
         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
 
         // generate and allocate texture
@@ -46,19 +53,23 @@ struct augment_renderer::impl {
         glBindTexture(GL_TEXTURE_2D, bg_tex);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
-        glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, raw_image_width, raw_image_height);
+        glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, bg_size.width, bg_size.height);
 
         // config remap texture
         glBindTexture(GL_TEXTURE_2D, remap_tex);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
-        glTexImage2D(GL_TEXTURE_2D, 0, GL_RG32F, raw_image_width, raw_image_height,
+        glTexImage2D(GL_TEXTURE_2D, 0, GL_RG32F, bg_size.width, bg_size.height,
                      0, GL_RG, GL_FLOAT, cam_info->remap_data);
 
         // register background pbo
         CUDA_API_CHECK(cudaGraphicsGLRegisterBuffer(&bg_res, bg_pbo,
                                                     cudaGraphicsRegisterFlagsWriteDiscard));
 
+        if (tex_renderer == nullptr) {
+            tex_renderer = std::make_unique<texture_renderer>();
+        }
+
         // setup vtk
         vtk_cam = vtkSmartPointer<vtkCamera>::New();
         vtk_cam->SetViewAngle(cam_info->angle);
@@ -86,15 +97,15 @@ struct augment_renderer::impl {
         size_t pbo_size;
         CUDA_API_CHECK(cudaGraphicsMapResources(1, &bg_res, stream));
         CUDA_API_CHECK(cudaGraphicsResourceGetMappedPointer(&pbo_ptr, &pbo_size, bg_res));
-        assert(pbo_size == rgb_image_size);
-        CUDA_API_CHECK(cudaMemcpy2D(pbo_ptr, rgb_image_width_bytes, bg_img->cudaPtr(),
-                                    bg_img->step, rgb_image_width_bytes, raw_image_height, cudaMemcpyDeviceToDevice));
+        assert(pbo_size == bg_size.area() * 3);
+        CUDA_API_CHECK(cudaMemcpy2D(pbo_ptr, bg_size.width * 3, bg_img->cudaPtr(),
+                                    bg_img->step, bg_size.width * 3, bg_size.height, cudaMemcpyDeviceToDevice));
         CUDA_API_CHECK(cudaGraphicsUnmapResources(1, &bg_res, stream));
 
         // unpack pbo to texture
         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, bg_pbo);
         glBindTexture(GL_TEXTURE_2D, bg_tex);
-        glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, raw_image_width, raw_image_height, GL_BGR, GL_UNSIGNED_BYTE, nullptr);
+        glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, bg_size.width, bg_size.height, GL_BGR, GL_UNSIGNED_BYTE, nullptr);
         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
 
         // render texture
@@ -154,28 +165,23 @@ struct augment_renderer::impl {
 
 };
 
+std::unique_ptr<texture_renderer> augment_renderer::impl::tex_renderer;
 vtkSmartPointer<vtkRenderer> augment_renderer::impl::vtk_render;
 vtkSmartPointer<vtkGenericOpenGLRenderWindow> augment_renderer::impl::vtk_window;
 
-augment_renderer::augment_renderer()
-        : pimpl(std::make_unique<impl>()) {}
+augment_renderer::augment_renderer(const camera_info *cam_info, cv::Size bg_size)
+        : pimpl(std::make_unique<impl>(cam_info, bg_size)) {
+}
 
 augment_renderer::~augment_renderer() = default;
 
-bool augment_renderer::initialize(texture_renderer *renderer,
-                                  const camera_intrinsics *cam_info) {
-    pimpl->tex_renderer = renderer;
-    pimpl->cam_info = cam_info;
-    return pimpl->initialize();
-}
-
 void augment_renderer::set_background(const cv::cuda::GpuMat *background) {
-    assert(background->rows == raw_image_height);
-    assert(background->cols == raw_image_width);
+    assert(background->rows == pimpl->bg_size.height);
+    assert(background->cols == pimpl->bg_size.width);
     pimpl->bg_img = background;
 }
 
-bool augment_renderer::render(const render_config &config, const cv::cuda::Stream &stream) {
+bool augment_renderer::render(const render_config &config, cv::cuda::Stream &stream) {
     pimpl->config = &config;
     return pimpl->render((cudaStream_t) stream.cudaPtr());
 }

src/augment_renderer.h

@@ -16,17 +16,15 @@ class vtkActor;
 class augment_renderer {
 public:
 
-    augment_renderer();
-
-    ~augment_renderer();
-
-    struct camera_intrinsics {
+    struct camera_info {
         int width, height;
         float angle;
         void *remap_data;
     };
 
-    bool initialize(texture_renderer *renderer, const camera_intrinsics *camera);
+    augment_renderer(const camera_info *cam_info, cv::Size bg_size);
+
+    ~augment_renderer();
 
     void set_background(const cv::cuda::GpuMat *background);
 
@@ -41,7 +39,7 @@ public:
         float width, height;
     };
 
-    bool render(const render_config &config, const cv::cuda::Stream &stream = cv::cuda::Stream::Null());
+    bool render(const render_config &config, cv::cuda::Stream &stream = cv::cuda::Stream::Null());
 
 private:
     struct impl;

src/config.h

@@ -10,14 +10,14 @@
 static constexpr auto main_window_width = 800;
 static constexpr auto main_window_height = 600;
 
-static constexpr auto raw_image_width = 2448;
-static constexpr auto raw_image_height = 2048;
-static constexpr auto raw_image_size = raw_image_width * 1 * sizeof(uint8_t) * raw_image_height;
-static constexpr auto rgb_image_width_bytes = raw_image_width * 3 * sizeof(uint8_t);
-static constexpr auto rgb_image_size = rgb_image_width_bytes * raw_image_height;
+//static constexpr auto raw_image_width = 2448;
+//static constexpr auto raw_image_height = 2048;
+//static constexpr auto raw_image_size = raw_image_width * 1 * sizeof(uint8_t) * raw_image_height;
+//static constexpr auto rgb_image_width_bytes = raw_image_width * 3 * sizeof(uint8_t);
+//static constexpr auto rgb_image_size = rgb_image_width_bytes * raw_image_height;
 
-static constexpr auto left_camera_name = "LeftEye";
-static constexpr auto right_camera_name = "RightEye";
+//static constexpr auto left_camera_name = "LeftEye";
+//static constexpr auto right_camera_name = "RightEye";
 
 static constexpr auto default_len_focus = 8; // 8mm, maybe not useful
 
@@ -36,8 +36,8 @@ static constexpr auto default_video_stream_bitrate = 10 * 1e6; // 10mbps
 static constexpr auto output_frame_width = 1920;
 static constexpr auto output_frame_height = 1080;
 
-static constexpr auto ar_width_normal =
-        1.0 * output_frame_height * raw_image_width / output_frame_width / raw_image_height;
+//static constexpr auto ar_width_normal =
+//        1.0 * output_frame_height * raw_image_width / output_frame_width / raw_image_height;
 
 static constexpr std::string_view raw_save_prefix = "raw_save";
 static constexpr auto default_raw_save_interval_s = 10; // 10s
@@ -69,7 +69,16 @@ inline void check_function_call_exception(bool function_ret, unsigned int line_n
         return false
 
 #define CALL_ASSERT(function_call) \
-    check_function_call_exception( \
+    check_function_call( \
         function_call, __LINE__, __FILE__, #function_call)
 
+#define IMGUI_DISABLE_IF(cond, cmd) \
+    if (cond) { \
+        ImGui::BeginDisabled(); \
+        cmd \
+        ImGui::EndDisabled(); \
+    } else { \
+        cmd \
+    }\
+
 #endif //REMOTEAR2_CONFIG_H

src/frame_buffer_helper.hpp

@@ -9,11 +9,15 @@
 #include <glad/gl.h>
 
 struct frame_buffer_helper {
-
-    int tex_width = 2 * raw_image_width, tex_height = raw_image_height;
+    cv::Size tex_size;
     GLuint tex = 0, depth_tex = 0, fbo = 0, pbo = 0;
     cudaGraphicsResource *pbo_res = nullptr;
 
+    explicit frame_buffer_helper(cv::Size size)
+            : tex_size(size) {
+        CALL_ASSERT(initialize());
+    }
+
     ~frame_buffer_helper() {
         if (fbo == 0)return;
         cudaGraphicsUnregisterResource(pbo_res);
@@ -22,19 +26,25 @@ struct frame_buffer_helper {
         glDeleteTextures(2, &tex);
     }
 
-    bool initialize(int width, int height) {
-        tex_width = width;
-        tex_height = height;
+    void download_pixels() {
+        glBindFramebuffer(GL_READ_FRAMEBUFFER, fbo);
+        glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo);
+        glReadPixels(0, 0, tex_size.width, tex_size.height, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, (void *) 0);
+        glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
+    }
+
+private:
 
+    bool initialize() {
         // create and allocate tex
         glGenTextures(2, &tex);
         glBindTexture(GL_TEXTURE_2D, tex);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
-        glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGB8, tex_width, tex_height);
+        glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGB8, tex_size.width, tex_size.height);
 
         glBindTexture(GL_TEXTURE_2D, depth_tex);
-        glTexStorage2D(GL_TEXTURE_2D, 1, GL_DEPTH_COMPONENT16, tex_width, tex_height);
+        glTexStorage2D(GL_TEXTURE_2D, 1, GL_DEPTH_COMPONENT16, tex_size.width, tex_size.height);
 
         // create and config fbo
         glGenFramebuffers(1, &fbo);
@@ -48,22 +58,13 @@ struct frame_buffer_helper {
         // create and config pbo
         glGenBuffers(1, &pbo);
         glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo);
-        glBufferStorage(GL_PIXEL_PACK_BUFFER, tex_width * tex_height * 4, nullptr, GL_DYNAMIC_STORAGE_BIT);
+        glBufferStorage(GL_PIXEL_PACK_BUFFER, tex_size.area() * 4, nullptr, GL_DYNAMIC_STORAGE_BIT);
         glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
         CUDA_API_CHECK(cudaGraphicsGLRegisterBuffer(&pbo_res, pbo, cudaGraphicsRegisterFlagsReadOnly));
 
         return true;
     }
 
-    void download_pixels() {
-        glBindFramebuffer(GL_READ_FRAMEBUFFER, fbo);
-        glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo);
-        glReadPixels(0, 0, tex_width, tex_height, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, (void *) 0);
-        glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
-    }
-
-private:
-
     static bool check_frame_buffer() {
         auto status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
         if (status != GL_FRAMEBUFFER_COMPLETE) [[unlikely]] {

src/image_ringbuffer.hpp

@@ -4,7 +4,6 @@
 #include "cuda_helper.hpp"
 
 #include <boost/lockfree/spsc_queue.hpp>
-#include <boost/smart_ptr/scoped_ptr.hpp>
 
 #include <opencv2/core/cuda.hpp>
 
@@ -20,24 +19,20 @@ struct image_buffer {
 
 class image_ringbuffer {
 public:
-    explicit image_ringbuffer(cv::cuda::Stream *stream = &cv::cuda::Stream::Null())
-            : cuda_stream(stream) {
-        main_queue.reset(new queue_type{queue_size});
-        recycle_queue.reset(new queue_type{queue_size});
+    explicit image_ringbuffer(cv::Size _size, int type,
+                              cv::cuda::Stream &stream = cv::cuda::Stream::Null())
+            : size(_size), cv_type(type), cuda_stream(stream) {
+        main_queue = std::make_unique<queue_type>(queue_size);
+        recycle_queue = std::make_unique<queue_type>(queue_size);
     }
 
-//    ~image_ringbuffer() {
-//        CALL_ASSERT(free_buffers());
-//    }
-
-    bool push_image(const cv::Mat &host_img) {
-        image_buffer *buf;
-        CALL_CHECK(get_valid_buf(&buf, host_img));
+    void push_image(cv::InputArray host_img) {
+        assert(host_img.isMat());
+        auto buf = get_valid_buf();
         host_img.copyTo(buf->host_img);
-        buf->cuda_img.upload(buf->host_img, *cuda_stream);
+        buf->cuda_img.upload(buf->host_img, cuda_stream);
         buf->flag.test_and_set(std::memory_order::release);
-        CALL_CHECK(main_queue->push(buf));
-        return true;
+        CALL_ASSERT(main_queue->push(buf));
     }
 
     void cancel_wait() {
@@ -46,19 +41,23 @@ public:
         CALL_ASSERT(main_queue->push(buf));
     }
 
-    bool pop_image(image_buffer **buf) {
-        while (!main_queue->pop(*buf)) {
+    image_buffer *pop_image() {
+        image_buffer *ret = nullptr;
+        while (!main_queue->pop(ret)) {
             std::this_thread::yield();
         }
-        image_buffer *newer_buf;
+        if (auto extra_cnt = main_queue->read_available(); extra_cnt > 0) {
+            SPDLOG_WARN("Image queued: {}.", extra_cnt);
+        }
+        image_buffer *newer_buf = nullptr;
         while (main_queue->pop(newer_buf)) { // return the newest image
-            recycle_buffer(*buf);
-            *buf = newer_buf;
+            recycle_buffer(ret);
+            ret = newer_buf;
         }
-        while (!(*buf)->flag.test(std::memory_order::acquire)) {
+        while (!ret->flag.test(std::memory_order::acquire)) {
             std::this_thread::yield();
         }
-        return !(*buf)->host_img.empty();
+        return ret;
     }
 
     void recycle_buffer(image_buffer *buf) {
@@ -71,38 +70,22 @@ private:
 
     using queue_type = boost::lockfree::spsc_queue<image_buffer *>;
 
-//    size_t width, height, img_size;
-    boost::scoped_ptr<queue_type> main_queue, recycle_queue;
-    cv::cuda::Stream *cuda_stream;
+    cv::Size size;
+    int cv_type;
+    std::unique_ptr<queue_type> main_queue, recycle_queue;
+    cv::cuda::Stream &cuda_stream;
 
-    bool get_valid_buf(image_buffer **buf, const cv::Mat &host_img_ref) {
-        if (recycle_queue->pop(*buf)) [[likely]] {
-            assert((*buf)->host_img.size == host_img_ref.size); // ensure consistency
-            return true;
+    image_buffer *get_valid_buf() {
+        image_buffer *ret = nullptr;
+        if (recycle_queue->pop(ret)) [[likely]] {
+            return ret;
         }
-        auto ret = new image_buffer;
+        ret = new image_buffer;
         // do not need to pre-allocate host image
-        ret->cuda_img.create(host_img_ref.size(), host_img_ref.type());
+        ret->cuda_img.create(size, cv_type);
         ret->flag.clear(std::memory_order::relaxed);
-        *buf = ret;
-        return true;
+        return ret;
     }
-
-//    static bool free_buf(image_buffer *buf) {
-//        buf->host_img.free();
-//        buf->cuda_img.free();
-//    }
-
-//    bool free_buffers() {
-//        image_buffer *buf;
-//        while (recycle_queue->pop(buf)) {
-//            CALL_CHECK(free_buf(buf));
-//        }
-//        while (main_queue->pop(buf)) {
-//            CALL_CHECK(free_buf(buf));
-//        }
-//    }
-
 };
 
 #endif //REMOTEAR2_IMAGE_RINGBUFFER_HPP

src/main.cpp

@@ -90,21 +90,24 @@ int main() {
     cuInit(0);
     create_cuda_context(&cuda_ctx);
 
-    // working staffs
-    std::unique_ptr<stereo_camera> st_cam;
-    texture_renderer tex_renderer;
-    frame_sender sender;
-    std::atomic_flag idr_flag;
-
-    std::unique_ptr<frame_buffer_helper> output_fbo;
-//    output_fbo.initialize(output_frame_width, output_frame_height);
-
-    video_encoder encoder;
-    encoder.initialize();
+    // make VTK happy
+    auto vtk_external = vtkSmartPointer<ExternalVTKWidget>::New();
 
-//    raw_file_saver raw_saver;
-//    raw_saver.start(raw_save_prefix);
-//    camera.set_raw_saver(&raw_saver);
+    // config augment renderers
+    auto left_remap_file = mapped_file{"left_proj.dat", boost::iostreams::mapped_file_base::readonly};
+    auto right_remap_file = mapped_file{"right_proj.dat", boost::iostreams::mapped_file_base::readonly};
+    auto left_cam_info = augment_renderer::camera_info{
+            .width = 2463, .height = 2048, .angle = 47.53009170028617,
+            .remap_data = (void *) left_remap_file.const_data()
+    };
+    auto right_cam_info = augment_renderer::camera_info{
+            .width = 2427, .height = 2048, .angle = 48.0713681473623,
+            .remap_data = (void *) right_remap_file.const_data()
+    };
+    auto left_ar = augment_renderer{&left_cam_info, stereo_camera::get_output_size()};
+    auto right_ar = augment_renderer{&right_cam_info, stereo_camera::get_output_size()};
+    left_remap_file.close();
+    right_remap_file.close();
 
     // config sophiar
     using namespace sophiar;
@@ -118,8 +121,6 @@ int main() {
 //    auto probe_pose_id = var_io.add_variable("probe_in_tracker", VT_transform, VD_Input);
     var_io.connect("127.0.0.1", 5278);
 
-    auto vtk_external = vtkSmartPointer<ExternalVTKWidget>::New();
-
     scene_manager scene;
     scene.var_io = &var_io;
     auto femur_actor = scene.add_actor("femur.stl", femur_pose_id);
@@ -132,28 +133,32 @@ int main() {
     // for debug;
 //    scene.add_actor("/home/tpx/data/stls/GlassProbe_4Ball_3.STL", probe_pose_id);
 
-    augment_renderer left_ar, right_ar;
-    auto left_remap_file = mapped_file{"left_proj.dat", boost::iostreams::mapped_file_base::readonly};
-    auto right_remap_file = mapped_file{"right_proj.dat", boost::iostreams::mapped_file_base::readonly};
-    auto left_cam_info = augment_renderer::camera_intrinsics{
-            .width = 2463, .height = 2048, .angle = 47.53009170028617,
-            .remap_data = (void *) left_remap_file.const_data()
-    };
-    auto right_cam_info = augment_renderer::camera_intrinsics{
-            .width = 2427, .height = 2048, .angle = 48.0713681473623,
-            .remap_data = (void *) right_remap_file.const_data()
-    };
-    left_ar.initialize(&tex_renderer, &left_cam_info);
-    right_ar.initialize(&tex_renderer, &right_cam_info);
-//    left_ar.set_background(&camera.left_rgb_image);
-//    right_ar.set_background(&camera.right_rgb_image);
+    // for debug; config sophiar
+//    auto probe_pose_id = var_io.add_variable("probe_in_tracker", VT_transform, VD_Input);
+    var_io.connect("127.0.0.1", 5278);
     augment_renderer::add_scene(&scene);
-    left_remap_file.close();
-    right_remap_file.close();
 
+    // immutable staffs
+    auto encoder = video_encoder{};
+    auto sender = frame_sender{};
+
+    // mutable staffs
+    std::unique_ptr<stereo_camera> st_cam;
+    std::unique_ptr<frame_buffer_helper> output_fbo;
+
+//    raw_file_saver raw_saver;
+//    raw_saver.start(raw_save_prefix);
+//    camera.set_raw_saver(&raw_saver);
+
+    std::atomic_flag idr_flag;
     int camera_fps = default_camera_fps;
-    float exposure_time_ms = default_camera_exposure_time_ms;
-    float analog_gain = default_camera_analog_gain;
+    bool use_hdr = false;
+    mvs_camera::capture_config cap_conf{
+            default_camera_exposure_time_ms, default_camera_analog_gain
+    };
+    mvs_camera::capture_config hdr_cap_conf{
+            default_camera_exposure_time_ms, default_camera_analog_gain
+    };
     float output_bitrate_mbps = default_video_stream_bitrate / 1e6f;
     int sender_port = 5279;
 
@@ -183,7 +188,6 @@ int main() {
             }
         }
 
-
 //        ImGui::ShowDemoWindow();
 
         if (ImGui::Begin("Remote AR Control")) {
@@ -199,28 +203,32 @@ int main() {
                     if (ImGui::Button("Open")) {
                         st_cam = std::make_unique<stereo_camera>();
                         auto [left_img, right_img] = st_cam->get_output_image();
-                        left_ar.set_background(&left_img);
-                        right_ar.set_background(&right_img);
+                        left_ar.set_background(left_img);
+                        right_ar.set_background(right_img);
                     }
                 } else { // st_cam != nullptr
                     if (ImGui::Button("Close")) {
                         st_cam.reset();
-                    }
-                    ImGui::SameLine();
-                    if (!st_cam->is_capturing()) {
-                        if (ImGui::Button("Start")) {
-                            st_cam->set_fps(camera_fps);
-                            st_cam->start();
-//                            camera.start_capture(1000 * exposure_time_ms, analog_gain, camera_fps);
-                        }
                     } else {
-                        if (ImGui::Button("Stop")) {
-                            st_cam->stop();
-                        }
-                        if (!auto_save_raw) {
-                            ImGui::SameLine();
-                            if (ImGui::Button("Capture")) {
+                        assert(st_cam != nullptr);
+                        ImGui::SameLine();
+                        if (!st_cam->is_capturing()) {
+                            if (ImGui::Button("Start")) {
+                                st_cam->set_fps(camera_fps);
+                                st_cam->set_hdr_config(0, cap_conf);
+                                st_cam->set_hdr_config(1, hdr_cap_conf);
+                                st_cam->set_hdr_enabled(use_hdr);
+                                st_cam->start();
+                            }
+                        } else {
+                            if (ImGui::Button("Stop")) {
+                                st_cam->stop();
+                            }
+                            if (!auto_save_raw) {
+                                ImGui::SameLine();
+                                if (ImGui::Button("Capture")) {
 //                                camera.request_save_raw();
+                                }
                             }
                         }
                     }
@@ -230,27 +238,35 @@ int main() {
                 if (st_cam != nullptr) {
                     ImGui::SeparatorText("Configs");
 
-                    // don't allow config change wile camera is capturing
-                    if (st_cam->is_capturing()) {
-                        ImGui::BeginDisabled();
+                    if (ImGui::SliderInt("Frame Rate (fps)", &camera_fps, 1, 60)) {
+                        st_cam->set_fps(camera_fps);
                     }
-                    ImGui::SliderInt("Frame Rate (fps)", &camera_fps, 1, 60);
-                    if (st_cam->is_capturing()) {
-                        ImGui::EndDisabled();
+                    if (ImGui::DragFloat("Exposure Time (ms)", &cap_conf.exposure_time_ms,
+                                         0.1, 0.1, 1e3f / (float) camera_fps, "%.01f")) {
+                        st_cam->set_hdr_config(0, cap_conf);
+                    }
+                    if (ImGui::DragFloat("Analog Gain (dB)", &cap_conf.analog_gain_db,
+                                         0.1, 0, 23.5, "%.01f")) {
+                        st_cam->set_hdr_config(0, cap_conf);
                     }
 
-                    ImGui::DragFloat("Exposure Time (ms)", &exposure_time_ms,
-                                     0.1, 0.1, 1e3f / (float) camera_fps, "%.01f");
-                    ImGui::DragFloat("Analog Gain (dB)", &analog_gain, 0.1, 0, 23.5, "%.01f");
+                    if (ImGui::Checkbox("Use HDR", &use_hdr)) {
+                        st_cam->set_hdr_enabled(use_hdr);
+                    }
+                    if (use_hdr) {
+                        ImGui::PushID("HDR Config");
+                        if (ImGui::DragFloat("Exposure Time (ms)", &hdr_cap_conf.exposure_time_ms,
+                                             0.1, 0.1, 1e3f / (float) camera_fps, "%.01f")) {
+                            st_cam->set_hdr_config(1, hdr_cap_conf);
+                        }
+                        if (ImGui::DragFloat("Analog Gain (dB)", &hdr_cap_conf.analog_gain_db,
+                                             0.1, 0, 23.5, "%.01f")) {
+                            st_cam->set_hdr_config(1, hdr_cap_conf);
+                        }
+                        ImGui::PopID();
+                    }
 
                     if (st_cam->is_capturing()) {
-                        // capture config
-                        mvs_camera::capture_config conf{
-                                1000 * exposure_time_ms, analog_gain
-                        };
-                        st_cam->set_hdr_config(0, conf);
-//                        camera.set_capture_config(1000 * exposure_time_ms, analog_gain);
-
                         // preview config
                         ImGui::SeparatorText("Preview Camera");
                         ImGui::RadioButton("Left", &preview_cam, 0);
@@ -296,61 +312,68 @@ int main() {
                 ImGui::PopID();
             }
 
-            if (st_cam != nullptr && st_cam->is_capturing() && ImGui::CollapsingHeader("AR Config")) {
-                ImGui::PushID("AR");
+            // AR configs
+            if (st_cam != nullptr && st_cam->is_capturing()) {
+                if (ImGui::CollapsingHeader("AR Config")) {
+                    ImGui::PushID("AR");
 
-                static int femur_opacity = 100;
-                static int tibia_opacity = 100;
-                ImGui::DragInt("Femur Transparency", &femur_opacity, 1, 0, 100);
-                ImGui::DragInt("Tibia Transparency", &tibia_opacity, 1, 0, 100);
-                femur_actor->GetProperty()->SetOpacity(0.01 * femur_opacity);
-                tibia_actor->GetProperty()->SetOpacity(0.01 * tibia_opacity);
+                    static int femur_opacity = 100;
+                    static int tibia_opacity = 100;
+                    ImGui::DragInt("Femur Transparency", &femur_opacity, 1, 0, 100);
+                    ImGui::DragInt("Tibia Transparency", &tibia_opacity, 1, 0, 100);
+                    femur_actor->GetProperty()->SetOpacity(0.01 * femur_opacity);
+                    tibia_actor->GetProperty()->SetOpacity(0.01 * tibia_opacity);
 
-                ImGui::PopID();
+                    ImGui::PopID();
+                }
             }
 
             // video streamer control
-            if (st_cam != nullptr && st_cam->is_capturing() && ImGui::CollapsingHeader("Video Encoder")) {
-                ImGui::PushID("Encoder");
+            if (st_cam != nullptr && st_cam->is_capturing()) {
+                if (ImGui::CollapsingHeader("Video Encoder")) {
+                    ImGui::PushID("Encoder");
 
-                ImGui::SeparatorText("Actions");
-                if (!encoder.is_encoding()) {
-                    if (ImGui::Button("Start")) {
-                        output_fbo = std::make_unique<frame_buffer_helper>();
-                        if (full_resolution) {
-                            output_fbo->initialize(2 * raw_image_width, raw_image_height);
-                        } else {
-                            output_fbo->initialize(output_frame_width, output_frame_height);
+                    ImGui::SeparatorText("Actions");
+                    if (!encoder.is_encoding()) {
+                        if (ImGui::Button("Start")) {
+                            if (full_resolution) {
+                                auto cam_size = mvs_camera::get_output_size();
+                                output_fbo = std::make_unique<frame_buffer_helper>(
+                                        cv::Size{cam_size.width * 2, cam_size.height});
+                            } else {
+                                output_fbo = std::make_unique<frame_buffer_helper>(
+                                        cv::Size{output_frame_width, output_frame_height});
+                            }
+                            encoder.start_encode(output_fbo->tex_size,
+                                                 camera_fps, (int) (output_bitrate_mbps * 1e6),
+                                                 save_video, save_frame_length);
+                        }
+                    } else {
+                        if (ImGui::Button("Close")) {
+                            encoder.stop_encode();
                         }
-                        encoder.start_encode(output_fbo->tex_width, output_fbo->tex_height,
-                                             camera_fps, (int) (output_bitrate_mbps * 1e6),
-                                             save_video, save_frame_length);
-                    }
-                } else {
-                    if (ImGui::Button("Close")) {
-                        encoder.stop_encode();
                     }
-                }
 
-                ImGui::SeparatorText("Configs");
-                if (encoder.is_encoding()) {
-                    ImGui::BeginDisabled();
-                }
+                    ImGui::SeparatorText("Configs");
+                    if (encoder.is_encoding()) {
+                        ImGui::BeginDisabled();
+                    }
 
-                ImGui::DragFloat("Bitrate (Mbps)", &output_bitrate_mbps, 0.1, 1, 20, "%.01f");
-                ImGui::Checkbox("Full Resolution", &full_resolution);
-                ImGui::SameLine();
-                ImGui::Checkbox("Save Video", &save_video);
-                if (save_video) {
+                    ImGui::DragFloat("Bitrate (Mbps)", &output_bitrate_mbps, 0.1, 1, 20, "%.01f");
+                    ImGui::Checkbox("Full Resolution", &full_resolution);
                     ImGui::SameLine();
-                    ImGui::Checkbox("Save Frame Length", &save_frame_length);
-                }
+                    ImGui::Checkbox("Save Video", &save_video);
+                    if (save_video) {
+                        ImGui::SameLine();
+                        ImGui::Checkbox("Save Frame Length", &save_frame_length);
+                    }
 
-                if (encoder.is_encoding()) {
-                    ImGui::EndDisabled();
-                }
+                    if (encoder.is_encoding()) {
+                        ImGui::EndDisabled();
+                    }
 
-                ImGui::PopID();
+                    ImGui::PopID();
+                }
             }
 
             if (ImGui::CollapsingHeader("Frame Sender")) {
@@ -420,13 +443,15 @@ int main() {
         if (encoder.is_encoding()) {
             // draw frame for streaming
             glBindFramebuffer(GL_DRAW_FRAMEBUFFER, output_fbo->fbo);
-            glViewport(0, 0, output_fbo->tex_width, output_fbo->tex_height);
+            glViewport(0, 0, output_fbo->tex_size.width, output_fbo->tex_size.height);
             if (full_resolution) {
                 left_ar.render({-1, -1, 1, 2});
                 right_ar.render({0, -1, 1, 2});
             } else {
-                left_ar.render({-0.5 - ar_width_normal / 2, -1, ar_width_normal, 2});
-                right_ar.render({0.5 - ar_width_normal / 2, -1, ar_width_normal, 2});
+                float ar_width_normal = stereo_camera::get_output_size().aspectRatio()
+                                        / output_fbo->tex_size.aspectRatio();
+                left_ar.render({-0.5f - ar_width_normal / 2, -1, ar_width_normal, 2});
+                right_ar.render({0.5f - ar_width_normal / 2, -1, ar_width_normal, 2});
             }
 
             // send IDR frame occasionally to prevent broken video frame
@@ -457,15 +482,16 @@ int main() {
             }
         }
 
-        int frame_width, frame_height;
+        cv::Size frame_size;
         glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
-        glfwGetFramebufferSize(main_window, &frame_width, &frame_height);
-        glViewport(0, 0, frame_width, frame_height);
+        glfwGetFramebufferSize(main_window, &frame_size.width, &frame_size.height);
+        glViewport(0, 0, frame_size.width, frame_size.height);
         glClear(GL_COLOR_BUFFER_BIT);
 
         if (st_cam != nullptr && st_cam->is_capturing()) {
             // draw frame in the screen
-            float width_normal = 1.0f * frame_height * raw_image_width / frame_width / raw_image_height;
+            float width_normal = stereo_camera::get_output_size().aspectRatio()
+                                 / frame_size.aspectRatio();
             if (preview_cam == 0) { // left cam
                 left_ar.render({-width_normal, 1, 2 * width_normal, -2});
             } else {

src/mvs_camera.cpp

@@ -1,6 +1,4 @@
 #include "mvs_camera.h"
-#include "config.h"
-#include "cuda_helper.hpp"
 #include "image_ringbuffer.hpp"
 
 #ifdef _MSC_VER
@@ -28,8 +26,6 @@
 
 #endif
 
-#include <atomic>
-
 bool check_mvs_api_call(int api_ret, unsigned int line_number,
                         const char *file_name, const char *api_call_str) {
     if (api_ret == MV_OK) [[likely]] return true;
@@ -44,12 +40,25 @@ bool check_mvs_api_call(int api_ret, unsigned int line_number,
         return false
 
 struct mvs_camera::impl {
+    static constexpr auto image_width = 2448;
+    static constexpr auto image_height = 2048;
+    static constexpr auto image_size = image_width * sizeof(uint8_t) * image_height;
+
     void *handle = nullptr;
     const char *cam_name = nullptr;
     image_ringbuffer *ring_buf = nullptr;
 
     bool is_capturing = false;
 
+    explicit impl(const char *_cam_name)
+            : cam_name(_cam_name) {
+        CALL_ASSERT(open());
+    }
+
+    ~impl() {
+        CALL_ASSERT(close());
+    }
+
     bool open() {
         MV_CC_DEVICE_INFO_LIST dev_info_list;
         MVS_API_CHECK(MV_CC_EnumDevices(MV_USB_DEVICE, &dev_info_list));
@@ -84,6 +93,9 @@ struct mvs_camera::impl {
     }
 
     bool start() {
+        // make sure ring buffer is assigned properly
+        assert(ring_buf != nullptr);
+
         // config camera
         assert(handle != nullptr);
         MVS_API_CHECK(MV_CC_SetEnumValue(handle, "PixelFormat", PixelType_Gvsp_BayerRG8));
@@ -136,21 +148,24 @@ struct mvs_camera::impl {
     }
 
     void on_image_impl(unsigned char *data, MV_FRAME_OUT_INFO_EX *frame_info) {
-        assert(frame_info->nFrameLen == raw_image_size);
-        auto host_img = cv::Mat{raw_image_height, raw_image_width, CV_8UC1, data};
-        CALL_ASSERT(ring_buf->push_image(host_img));
+        assert(frame_info->nFrameLen == image_size);
+        auto host_img = cv::Mat{image_height, image_width, CV_8UC1, data};
+        ring_buf->push_image(host_img);
     }
 };
 
-mvs_camera::mvs_camera(const char *camera_name, image_ringbuffer *ring_buffer)
-        : pimpl(std::make_unique<impl>()) {
-    pimpl->cam_name = camera_name;
-    pimpl->ring_buf = ring_buffer;
-    CALL_ASSERT(pimpl->open());
+mvs_camera::mvs_camera(const char *camera_name)
+        : pimpl(std::make_unique<impl>(camera_name)) {
+}
+
+mvs_camera::~mvs_camera() = default;
+
+cv::Size mvs_camera::get_output_size() {
+    return {impl::image_width, impl::image_height};
 }
 
-mvs_camera::~mvs_camera() {
-    CALL_ASSERT(pimpl->close());
+void mvs_camera::set_ring_buffer(image_ringbuffer *ring_buf) {
+    pimpl->ring_buf = ring_buf;
 }
 
 bool mvs_camera::start() {
@@ -161,13 +176,13 @@ bool mvs_camera::trigger() {
     return pimpl->trigger();
 }
 
-bool mvs_camera::stop() {
-    return pimpl->stop();
+void mvs_camera::stop() {
+    pimpl->stop();
 }
 
 bool mvs_camera::set_capture_config(const capture_config &config) {
-    MVS_API_CHECK(MV_CC_SetFloatValue(pimpl->handle, "ExposureTime", config.exposure_time));
-    MVS_API_CHECK(MV_CC_SetFloatValue(pimpl->handle, "Gain", config.analog_gain));
+    MVS_API_CHECK(MV_CC_SetFloatValue(pimpl->handle, "ExposureTime", config.exposure_time_ms * 1000)); // ms -> us
+    MVS_API_CHECK(MV_CC_SetFloatValue(pimpl->handle, "Gain", config.analog_gain_db));
     return true;
 }
 

src/mvs_camera.h

@@ -1,29 +1,36 @@
 #ifndef REMOTEAR2_MVS_CAMERA_H
 #define REMOTEAR2_MVS_CAMERA_H
 
+#include <opencv2/core/types.hpp>
+
 #include <memory>
+#include <tuple>
 
 class image_ringbuffer;
 
 class mvs_camera {
 public:
 
-    mvs_camera(const char *camera_name, image_ringbuffer *ring_buffer);
+    explicit mvs_camera(const char *camera_name);
 
     ~mvs_camera();
 
+    static cv::Size get_output_size();
+
+    void set_ring_buffer(image_ringbuffer *ring_buf);
+
     bool start();
 
     struct capture_config {
-        float exposure_time;
-        float analog_gain;
+        float exposure_time_ms;
+        float analog_gain_db;
     };
 
     bool set_capture_config(const capture_config &config);
 
     bool trigger();
 
-    bool stop();
+    void stop();
 
     bool is_capturing() const;
 

src/stereo_camera.cpp

@@ -3,34 +3,36 @@
 #include "image_ringbuffer.hpp"
 #include "mvs_camera.h"
 
-#include <memory>
 #include <opencv2/core/cuda.hpp>
 #include <opencv2/cudaimgproc.hpp>
 
-#include <boost/smart_ptr/scoped_ptr.hpp>
-
 #include <atomic>
+#include <memory>
 
 struct advanced_camera {
-    boost::scoped_ptr<cv::cuda::Stream> cuda_stream;
-    boost::scoped_ptr<image_ringbuffer> ring_buf;
-    boost::scoped_ptr<mvs_camera> cam;
+    std::unique_ptr<cv::cuda::Stream> cuda_stream;
+    std::unique_ptr<image_ringbuffer> ring_buf;
+    std::unique_ptr<mvs_camera> mvs_cam;
 
     bool use_hdr = false;
-    boost::scoped_ptr<hdr_synthesizer> hdr;
+    std::unique_ptr<hdr_synthesizer> hdr;
     void *hdr_buf_last = nullptr, *hdr_buf_cur = nullptr;
 
     image_buffer *img_buf = nullptr;
     cv::cuda::GpuMat out_rgb_u8;
 
     explicit advanced_camera(const char *name) {
-        cuda_stream.reset(new cv::cuda::Stream{});
-        ring_buf.reset(new image_ringbuffer{cuda_stream.get()});
-        cam.reset(new mvs_camera{name, ring_buf.get()});
+        cuda_stream = std::make_unique<cv::cuda::Stream>();
+        mvs_cam = std::make_unique<mvs_camera>(name);
+        ring_buf = std::make_unique<image_ringbuffer>(
+                mvs_camera::get_output_size(), CV_8UC1, *cuda_stream);
+        mvs_cam->set_ring_buffer(ring_buf.get());
 
-        hdr.reset(
-                new hdr_synthesizer{raw_image_width, raw_image_height, (cudaStream_t) cuda_stream->cudaPtr()}); // TODO
-        out_rgb_u8.create(raw_image_height, raw_image_width, CV_8UC3); // TODO
+        auto img_size = mvs_camera::get_output_size(); // TODO: optimize signature of hdr_synthesizer
+        hdr = std::make_unique<hdr_synthesizer>(img_size.width, img_size.height,
+                                                (cudaStream_t) cuda_stream->cudaPtr(), 6);
+
+        out_rgb_u8.create(img_size, CV_8UC3); // TODO: get output size from hdr_synthesizer
     }
 
     ~advanced_camera() {
@@ -45,23 +47,22 @@ struct advanced_camera {
         return true;
     }
 
-    bool process_next_img() {
-        if (!ring_buf->pop_image(&img_buf)) return false;
+    void process_next_img() {
+        img_buf = ring_buf->pop_image();
         assert(img_buf != nullptr);
         if (use_hdr) {
             std::swap(hdr_buf_last, hdr_buf_cur);
             if (hdr_buf_cur == nullptr) [[unlikely]] {
-                CALL_CHECK(hdr->malloc_buffer(&hdr_buf_cur));
+                CALL_ASSERT(hdr->malloc_buffer(&hdr_buf_cur)); // TODO: make hdr_synthesizer throw no error
             }
-            CALL_CHECK(hdr->preprocess_image(hdr_buf_cur, img_buf->cuda_img.cudaPtr(), img_buf->cuda_img.step1()));
-            CALL_CHECK(hdr->merge_image(hdr_buf_last ? hdr_buf_last : hdr_buf_cur, hdr_buf_cur,
-                                        (uint8_t *) out_rgb_u8.cudaPtr(), out_rgb_u8.step1()));
+            CALL_ASSERT(hdr->preprocess_image(hdr_buf_cur, img_buf->cuda_img.cudaPtr(), img_buf->cuda_img.step1()));
+            CALL_ASSERT(hdr->merge_image(hdr_buf_last ? hdr_buf_last : hdr_buf_cur, hdr_buf_cur,
+                                         (uint8_t *) out_rgb_u8.cudaPtr(), out_rgb_u8.step1()));
         } else { // hdr disabled
-            CUDA_API_CHECK(cudaStreamSynchronize((cudaStream_t) cuda_stream->cudaPtr()));
+            CALL_ASSERT(cudaStreamSynchronize((cudaStream_t) cuda_stream->cudaPtr()) == cudaSuccess);
             cv::cuda::cvtColor(img_buf->cuda_img, out_rgb_u8,
                                cv::COLOR_BayerRG2RGB, 3, *cuda_stream); // rgb image has 3 channels
         }
-        return true;
     }
 
     void finish_frame() {
@@ -79,22 +80,27 @@ struct trigger_config {
 };
 
 struct stereo_camera::impl {
-
     static constexpr auto queue_size = 8;
+    static constexpr auto left_camera_name = "LeftEye";
+    static constexpr auto right_camera_name = "RightEye";
 
-    boost::scoped_ptr<advanced_camera> left_cam, right_cam;
+    std::unique_ptr<advanced_camera> left_cam, right_cam;
 
     trigger_config trig_conf;
     std::atomic_flag stop_flag;
-    boost::scoped_ptr<std::thread> trig_thread;
+    std::unique_ptr<std::thread> trig_thread;
 
     using trig_conf_queue_type = boost::lockfree::spsc_queue<trigger_config>;
-    boost::scoped_ptr<trig_conf_queue_type> trig_queue;
+    std::unique_ptr<trig_conf_queue_type> trig_queue;
 
     impl() {
-        left_cam.reset(new advanced_camera{left_camera_name});
-        right_cam.reset(new advanced_camera{right_camera_name});
-        trig_queue.reset(new trig_conf_queue_type{queue_size});
+        left_cam = std::make_unique<advanced_camera>(left_camera_name);
+        right_cam = std::make_unique<advanced_camera>(right_camera_name);
+        trig_queue = std::make_unique<trig_conf_queue_type>(queue_size);
+    }
+
+    ~impl() {
+        stop();
     }
 
     void upload_trigger_config() {
@@ -109,6 +115,11 @@ struct stereo_camera::impl {
             std::this_thread::yield();
         }
 
+        // wait for initial config
+        while (!trig_queue->pop(conf)) {
+            std::this_thread::yield();
+        }
+
         uint8_t cur_hdr_index = 0;
         auto next_trigger_time = std::chrono::high_resolution_clock::now();
         while (true) {
@@ -118,11 +129,11 @@ struct stereo_camera::impl {
             while (trig_queue->pop(conf));
 
             // config exposure parameters
-            left_cam->cam->set_capture_config(conf.hdrs[cur_hdr_index]);
-            right_cam->cam->set_capture_config(conf.hdrs[cur_hdr_index]);
+            CALL_ASSERT(left_cam->mvs_cam->set_capture_config(conf.hdrs[cur_hdr_index]));
+            CALL_ASSERT(right_cam->mvs_cam->set_capture_config(conf.hdrs[cur_hdr_index]));
 
-            left_cam->cam->trigger();
-            right_cam->cam->trigger();
+            CALL_ASSERT(left_cam->mvs_cam->trigger());
+            CALL_ASSERT(right_cam->mvs_cam->trigger());
 
             if (conf.use_hdr) {
                 cur_hdr_index = (cur_hdr_index + 1) % hdr_config_cnt;
@@ -140,26 +151,24 @@ struct stereo_camera::impl {
     }
 
     bool start() {
-        CALL_CHECK(left_cam->cam->start());
-        CALL_CHECK(right_cam->cam->start());
-        upload_trigger_config();
+        CALL_CHECK(left_cam->mvs_cam->start());
+        CALL_CHECK(right_cam->mvs_cam->start());
 
-        assert(trig_thread == nullptr);
         stop_flag.clear(std::memory_order::relaxed);
-        trig_thread.reset(new std::thread{&impl::trigger_thread_work, this});
-
+        assert(trig_thread == nullptr);
+        trig_thread = std::make_unique<std::thread>(&impl::trigger_thread_work, this);
+        upload_trigger_config();
         return true;
     }
 
-    bool stop() {
-        assert(trig_thread != nullptr);
+    void stop() {
+        if (trig_thread == nullptr) return;
         stop_flag.test_and_set(std::memory_order::relaxed);
         trig_thread->join();
         trig_thread.reset();
 
-        CALL_CHECK(left_cam->cam->stop());
-        CALL_CHECK(right_cam->cam->stop());
-        return true;
+        left_cam->mvs_cam->stop();
+        right_cam->mvs_cam->stop();
     }
 
 };
@@ -170,9 +179,13 @@ stereo_camera::stereo_camera()
 
 stereo_camera::~stereo_camera() = default;
 
-std::tuple<cv::cuda::GpuMat &, cv::cuda::GpuMat &> stereo_camera::get_output_image() {
-    return std::tie(pimpl->left_cam->out_rgb_u8,
-                    pimpl->right_cam->out_rgb_u8);
+cv::Size stereo_camera::get_output_size() {
+    return mvs_camera::get_output_size(); // TODO: get output size from hdr_synthesizer
+}
+
+std::tuple<cv::cuda::GpuMat *, cv::cuda::GpuMat *> stereo_camera::get_output_image() {
+    return std::make_tuple(&pimpl->left_cam->out_rgb_u8,
+                           &pimpl->right_cam->out_rgb_u8);
 }
 
 std::tuple<cv::cuda::Stream *, cv::cuda::Stream *> stereo_camera::get_cuda_stream() {
@@ -183,6 +196,10 @@ std::tuple<cv::cuda::Stream *, cv::cuda::Stream *> stereo_camera::get_cuda_strea
 void stereo_camera::set_hdr_enabled(bool flag) {
     pimpl->trig_conf.use_hdr = flag;
     pimpl->upload_trigger_config();
+
+    // forward config to each camera
+    pimpl->left_cam->use_hdr = flag;
+    pimpl->right_cam->use_hdr = flag;
 }
 
 void stereo_camera::set_hdr_config(uint8_t index, const mvs_camera::capture_config &config) {
@@ -204,10 +221,9 @@ bool stereo_camera::is_capturing() const {
     return pimpl->trig_thread != nullptr;
 }
 
-bool stereo_camera::retrieve_frame() {
-    CALL_CHECK(pimpl->left_cam->process_next_img());
-    CALL_CHECK(pimpl->right_cam->process_next_img());
-    return true;
+void stereo_camera::retrieve_frame() {
+    pimpl->left_cam->process_next_img();
+    pimpl->right_cam->process_next_img();
 }
 
 std::tuple<cv::Mat &, cv::Mat &> stereo_camera::get_raw_image() {
@@ -220,6 +236,6 @@ void stereo_camera::finish_frame() {
     pimpl->right_cam->finish_frame();
 }
 
-bool stereo_camera::stop() {
-    return pimpl->stop();
+void stereo_camera::stop() {
+    pimpl->stop();
 }

src/stereo_camera.h

@@ -17,7 +17,9 @@ public:
 
     ~stereo_camera();
 
-    std::tuple<cv::cuda::GpuMat &, cv::cuda::GpuMat &> get_output_image();
+    static cv::Size get_output_size();
+
+    std::tuple<cv::cuda::GpuMat *, cv::cuda::GpuMat *> get_output_image();
 
     std::tuple<cv::cuda::Stream *, cv::cuda::Stream *> get_cuda_stream();
 
@@ -31,13 +33,13 @@ public:
 
     bool is_capturing() const;
 
-    bool retrieve_frame();
+    void retrieve_frame();
 
     std::tuple<cv::Mat &, cv::Mat &> get_raw_image();
 
     void finish_frame();
 
-    bool stop();
+    void stop();
 
 private:
 

src/video_encoder.cpp

@@ -44,10 +44,8 @@ struct video_encoder::impl {
     void *encoder = nullptr;
     NV_ENC_OUTPUT_PTR output_buf = nullptr;
 
-    int frame_width = raw_image_width * 2, frame_height = raw_image_height;
-    int frame_pitch = frame_width * 4; // ARGB image
-    int frame_rate = default_camera_fps;
-    int frame_bitrate = default_video_stream_bitrate;
+    int frame_width, frame_height, frame_pitch;
+    int frame_rate, frame_bitrate;
 
     // frame related
     void *frame_ptr = nullptr, **output_ptr = nullptr;
@@ -58,6 +56,14 @@ struct video_encoder::impl {
     bool save_frame_length = true;
     FILE *video_save_file = nullptr;
 
+    impl() {
+        CALL_ASSERT(initialize());
+    }
+
+    ~impl() {
+        cleanup();
+    }
+
     bool initialize() {
         NVENC_API_CHECK(NvEncodeAPICreateInstance(&api));
         CUDA_API_CHECK(cuCtxGetCurrent(&cuda_ctx));
@@ -223,21 +229,16 @@ struct video_encoder::impl {
 };
 
 video_encoder::video_encoder()
-        : pimpl(std::make_unique<impl>()) {}
-
-video_encoder::~video_encoder() {
-    stop_encode();
-};
-
-bool video_encoder::initialize() {
-    return pimpl->initialize();
+        : pimpl(std::make_unique<impl>()) {
 }
 
-bool video_encoder::start_encode(int width, int height, int fps, int bitrate,
+video_encoder::~video_encoder() = default;
+
+bool video_encoder::start_encode(cv::Size size, int fps, int bitrate,
                                  bool save_file, bool save_frame_length) {
-    pimpl->frame_width = width;
-    pimpl->frame_height = height;
-    pimpl->frame_pitch = width * 4; // ARGB image
+    pimpl->frame_width = size.width;
+    pimpl->frame_height = size.height;
+    pimpl->frame_pitch = size.width * 4; // ARGB image
     pimpl->frame_rate = fps;
     pimpl->frame_bitrate = bitrate;
 

src/video_encoder.h

@@ -1,6 +1,8 @@
 #ifndef REMOTEAR2_VIDEO_ENCODER_H
 #define REMOTEAR2_VIDEO_ENCODER_H
 
+#include <opencv2/core/types.hpp>
+
 #include <glad/gl.h> // make windows happy
 #include <cuda_gl_interop.h>
 
@@ -13,9 +15,7 @@ public:
 
     ~video_encoder();
 
-    bool initialize();
-
-    bool start_encode(int width, int height, int fps, int bitrate,
+    bool start_encode(cv::Size size, int fps, int bitrate,
                       bool save_file = false, bool save_frame_length = true);
 
     void stop_encode();