Implemented video encoding function.

CMakeLists.txt

@@ -78,6 +78,20 @@ target_include_directories(${PROJECT_NAME} PRIVATE ${MVS_INCLUDE_DIR})
 target_link_libraries(${PROJECT_NAME} ${MVS_LIB})
 target_sources(${PROJECT_NAME} PRIVATE src/mvs_camera.cpp)
 
+# NvEnc config
+if (WIN32)
+    set(NVCODEC_DIR C:/BuildEssentials/CUDA/Video_Codec_SDK_12.0.16)
+    set(NVENC_LIB_DIR ${NVCODEC_DIR}/Lib/x64)
+    find_library(NVENC_LIB nvencodeapi HINTS ${NVENC_LIB_DIR})
+else ()
+    set(NVCODEC_DIR /home/tpx/src/Video_Codec_SDK_12.0.16)
+    find_library(NVENC_LIB nvidia-encode)
+endif ()
+set(NVCODEC_INCLUDE_DIR ${NVCODEC_DIR}/Interface)
+target_include_directories(${PROJECT_NAME} PRIVATE ${NVCODEC_INCLUDE_DIR})
+target_link_libraries(${PROJECT_NAME} ${NVENC_LIB})
+target_sources(${PROJECT_NAME} PRIVATE src/video_encoder.cpp)
+
 # yaml-cpp
 find_package(yaml-cpp REQUIRED)
 target_include_directories(${PROJECT_NAME} PRIVATE ${YAML_CPP_INCLUDE_DIR})

data/config.yaml

@@ -0,0 +1,17 @@
+camera:
+  names:
+    left: LeftEye
+    right: RightEye
+  capture:
+    frame_rate: 40
+    expo_time_ms: 5
+    gain_db: 0
+
+main_window:
+  width: 800
+  height: 600
+
+output:
+  width: 1920
+  height: 1080
+  bitrate: 5 # Mbps

src/main.cpp

@@ -27,12 +27,16 @@ void handle_imgui_events();
 
 bool is_capturing();
 
+bool is_encoding();
+
 void wait_camera_frames();
 
 void process_camera_frames();
 
 void render_main_window();
 
+void generate_output_frame();
+
 int main() {
 
 #ifndef NDEBUG
@@ -53,6 +57,10 @@ int main() {
         if (is_capturing()) {
             wait_camera_frames();
             process_camera_frames();
+
+            if (is_encoding()) {
+                generate_output_frame();
+            }
         }
 
         render_main_window();

src/main_ext.cpp

@@ -3,7 +3,9 @@
 #include "mvs_camera.h"
 #include "simple_mq.h"
 #include "simple_opengl.h"
+#include "third_party/timestamp_helper.hpp"
 #include "variable_defs.h"
+#include "video_encoder.h"
 
 #include <spdlog/spdlog.h>
 
@@ -20,16 +22,22 @@
 #include <imgui_impl_opengl3.h>
 
 #include <cassert>
+#include <thread>
 #include <queue>
 
 using namespace simple_mq_singleton;
+using namespace sophiar;
+
+// make sophiar happy
+local_time_type sophiar::program_start_time;
 
 // global variable definition
-CUcontext cuda_ctx;
-GLFWwindow *main_window;
+CUcontext cuda_ctx = nullptr;
+int main_window_width = -1, main_window_height = -1;
+GLFWwindow *main_window = nullptr;
 std::string left_camera_name, right_camera_name;
 std::unique_ptr<mvs::camera> left_camera, right_camera;
-mvs::capture_config capture_conf;
+mvs::capture_config capture_conf = {};
 int preview_camera_index = 0; // 0 for left, 1 for right
 uint64_t left_raw_cnt = 0, right_raw_cnt = 0;
 std::unique_ptr<cv::cuda::GpuMat> left_img_dev, right_img_dev;
@@ -37,6 +45,15 @@ std::unique_ptr<cv::cuda::Stream> left_stream, right_stream;
 cudaStream_t left_cuda_stream = nullptr, right_cuda_stream = nullptr;
 std::unique_ptr<monocular_processor> left_processor, right_processor;
 std::unique_ptr<simple_render> opengl_render;
+float process_frame_rate = 0;
+std::unique_ptr<std::thread> encoder_thread;
+bool output_full_frame = false;
+int output_width = -1, output_height = -1;
+encoder_config main_encoder_conf;
+std::unique_ptr<smart_frame_buffer> output_fbo;
+std::unique_ptr<cv::cuda::Stream> output_stream;
+cudaStream_t output_cuda_stream = nullptr;
+std::shared_ptr<cv::cuda::GpuMat> output_frame_dev;
 
 std::queue<void (*)()> simple_eq;
 
@@ -56,6 +73,7 @@ void initialize_cuda() {
     CUdevice cuda_device;
     CUDA_API_CHECK(cuDeviceGet(&cuda_device, default_cuda_device_id));
     CUDA_API_CHECK(cuCtxCreate(&cuda_ctx, CU_CTX_SCHED_AUTO, cuda_device));
+    mq().update_variable(CUDA_CONTEXT, cuda_ctx);
 
     // elegant cleanup
     std::atexit([] {
@@ -71,6 +89,9 @@ void initialize_cuda() {
     right_cuda_stream = (cudaStream_t) right_stream->cudaPtr();
     left_processor = std::make_unique<monocular_processor>();
     right_processor = std::make_unique<monocular_processor>();
+    output_stream = std::make_unique<cv::cuda::Stream>();
+    output_cuda_stream = (cudaStream_t) output_stream->cudaPtr();
+    output_frame_dev = std::make_shared<cv::cuda::GpuMat>();
 }
 
 void load_config() {
@@ -82,14 +103,21 @@ void load_config() {
     left_camera_name = camera_names["left"].as<std::string>();
     right_camera_name = camera_names["right"].as<std::string>();
     auto capture_param = camera_conf["capture"];
-    capture_conf.frame_rate = capture_param["frame_rate"].as<float>();
+    capture_conf.frame_rate = capture_param["frame_rate"].as<int>();
+    main_encoder_conf.frame_rate = capture_conf.frame_rate;
     capture_conf.expo_time_ms = capture_param["expo_time_ms"].as<float>();
     capture_conf.gain_db = capture_param["gain_db"].as<float>();
 
     // load main window config
     auto window_conf = conf["main_window"];
-    mq().update_variable(MAIN_WINDOW_WIDTH, window_conf["width"].as<int>());
-    mq().update_variable(MAIN_WINDOW_HEIGHT, window_conf["height"].as<int>());
+    main_window_width = window_conf["width"].as<int>();
+    main_window_height = window_conf["height"].as<int>();
+
+    // load output config
+    auto output_conf = conf["output"];
+    output_width = output_conf["width"].as<int>();
+    output_height = output_conf["height"].as<int>();
+    main_encoder_conf.bitrate_mbps = output_conf["bitrate"].as<float>();
 }
 
 void initialize_main_window() {
@@ -104,9 +132,8 @@ void initialize_main_window() {
     glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 4);
     glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 6);
     glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
-    auto window_width = mq().query_variable<int>(MAIN_WINDOW_WIDTH);
-    auto window_height = mq().query_variable<int>(MAIN_WINDOW_HEIGHT);
-    main_window = glfwCreateWindow(window_width, window_height, "RemoteAR V3.-1", nullptr, nullptr);
+    main_window = glfwCreateWindow(main_window_width, main_window_height,
+                                   "RemoteAR V3.-1", nullptr, nullptr);
     assert(main_window != nullptr);
     glfwMakeContextCurrent(main_window);
 
@@ -140,8 +167,9 @@ void initialize_main_window() {
     ImGui_ImplGlfw_InitForOpenGL(main_window, true);
     ImGui_ImplOpenGL3_Init();
 
-    // initialize OpenGL render
+    // initialize OpenGL objects
     opengl_render = std::make_unique<simple_render>();
+    output_fbo = std::make_unique<smart_frame_buffer>();
 
     // elegant cleanup
     std::atexit([] {
@@ -218,11 +246,72 @@ void open_cameras() {
     }
 }
 
+void encoder_thread_work() {
+    uint64_t last_conf_cnt;
+    auto conf = mq().query_variable<encoder_config>(ENCODER_CONFIG, &last_conf_cnt);
+    auto encoder = std::unique_ptr<video_encoder>(video_encoder::create(conf));
+
+    uint64_t frame_cnt = 0;
+    auto frame_data = std::make_unique<video_nal>();
+    for (;;) {
+        mq().wait_variable(OUTPUT_FRAME, frame_cnt);
+        auto frame = mq().query_variable_ptr<cv::cuda::GpuMat>(OUTPUT_FRAME, &frame_cnt);
+
+        // test stop flag
+        if (mq().query_variable<bool>(ENCODER_SHOULD_STOP)) break;
+
+        // check for config update
+        uint64_t cur_conf_cnt;
+        conf = mq().query_variable<encoder_config>(ENCODER_CONFIG, &cur_conf_cnt);
+        if (cur_conf_cnt > last_conf_cnt) {
+            encoder->change_config(conf);
+            last_conf_cnt = cur_conf_cnt;
+        }
+
+        assert(frame != nullptr);
+        encoder->encode(*frame, frame_data.get());
+    }
+}
+
+bool is_encoding() {
+    return encoder_thread != nullptr;
+}
+
+void upload_encoder_config() {
+    mq().update_variable(ENCODER_CONFIG, main_encoder_conf);
+}
+
+void start_encoder() {
+    if (output_full_frame) {
+        assert(!left_img_dev->empty() && !right_img_dev->empty());
+        assert(left_img_dev->size() == right_img_dev->size());
+        main_encoder_conf.frame_size = cv::Size{
+                left_img_dev->size().width * 2,
+                left_img_dev->size().height};
+        output_fbo->create(main_encoder_conf.frame_size);
+    } else {
+        main_encoder_conf.frame_size = cv::Size{output_width, output_height};
+        output_fbo->create(main_encoder_conf.frame_size);
+    }
+    upload_encoder_config();
+    mq().update_variable(ENCODER_SHOULD_STOP, false);
+    mq().update_variable(ENCODER_BUSY, false); // make variable exist
+    encoder_thread = std::make_unique<std::thread>(encoder_thread_work);
+}
+
+void stop_encoder() {
+    mq().update_variable(ENCODER_SHOULD_STOP, true);
+    mq().update_variable_ptr<cv::cuda::GpuMat>(OUTPUT_FRAME, nullptr);
+    encoder_thread->join();
+    encoder_thread.reset();
+}
+
 void cleanup() {
     close_cameras();
 
     // avoid cudaErrorCudartUnloading
     opengl_render.reset();
+    output_fbo.reset();
 }
 
 void prepare_imgui_frame() {
@@ -232,6 +321,8 @@ void prepare_imgui_frame() {
     ImGui_ImplGlfw_NewFrame();
     ImGui::NewFrame();
 
+    ImGui::ShowDemoWindow();
+
     if (ImGui::Begin("Remote AR Control")) {
         ImGui::PushItemWidth(200);
 
@@ -269,9 +360,10 @@ void prepare_imgui_frame() {
 
             // camera configs
             ImGui::SeparatorText("Configs");
-            if (ImGui::DragFloat("Frame Rate (fps)", &capture_conf.frame_rate,
-                                 0.5, 1, 60, "%.01f")) {
+            if (ImGui::DragInt("Frame Rate (fps)", &capture_conf.frame_rate, 1, 1, 60)) {
                 simple_eq.push(upload_capture_config);
+                main_encoder_conf.frame_rate = capture_conf.frame_rate;
+                simple_eq.push(upload_encoder_config);
             }
             if (ImGui::DragFloat("Exposure Time (ms)", &capture_conf.expo_time_ms,
                                  0.1, 0.1, 1e3f / capture_conf.frame_rate, "%.01f")) {
@@ -289,6 +381,11 @@ void prepare_imgui_frame() {
                 ImGui::SameLine();
                 ImGui::RadioButton("Right", &preview_camera_index, 1);
 
+                ImGui::SeparatorText("Infos");
+                ImGui::BeginDisabled();
+                ImGui::DragFloat("Process Frame Rate (fps)", &process_frame_rate, 0, 0, 60, "%.01f");
+                ImGui::EndDisabled();
+
                 // auto save raw config
 //                ImGui::SeparatorText("Auto Shoot");
 //                ImGui::PushID("Auto Shoot");
@@ -320,8 +417,43 @@ void prepare_imgui_frame() {
 //                if (auto_save_raw) {
 //                    ImGui::EndDisabled();
 //                }
+            }
+
+            ImGui::PopID();
+        }
 
+        if (is_capturing() && ImGui::CollapsingHeader("Video Encoder")) {
+            ImGui::PushID("Encoder");
 
+            ImGui::SeparatorText("Actions");
+            if (!is_encoding()) {
+                if (ImGui::Button("Start")) {
+                    simple_eq.push(start_encoder);
+                }
+            } else {
+                if (ImGui::Button("Close")) {
+                    simple_eq.push(stop_encoder);
+                }
+            }
+
+            ImGui::SeparatorText("Configs");
+            if (ImGui::DragFloat("Bitrate (Mbps)", &main_encoder_conf.bitrate_mbps,
+                                 0.1, 1, 20, "%.01f")) {
+                simple_eq.push(upload_encoder_config);
+            }
+
+            if (is_encoding()) {
+                ImGui::BeginDisabled();
+            }
+            ImGui::Checkbox("Full Resolution", &output_full_frame);
+            ImGui::SameLine();
+            ImGui::Checkbox("Save Video", &main_encoder_conf.save_file);
+            if (main_encoder_conf.save_file) {
+                ImGui::SameLine();
+                ImGui::Checkbox("Save Frame Length", &main_encoder_conf.save_length);
+            }
+            if (is_encoding()) {
+                ImGui::EndDisabled();
             }
 
             ImGui::PopID();
@@ -396,4 +528,44 @@ void render_main_window() {
 
     ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());
     glfwSwapBuffers(main_window);
+
+    // calculate process frame rate
+    static auto last_ts = current_timestamp();
+    auto cur_ts = current_timestamp();
+    process_frame_rate = 1e6f / (cur_ts - last_ts);
+    last_ts = cur_ts;
+}
+
+void generate_output_frame() {
+    // offline drawing
+    assert(output_fbo->id != 0);
+    glBindFramebuffer(GL_DRAW_FRAMEBUFFER, output_fbo->id);
+    glViewport(0, 0, output_fbo->size().width, output_fbo->size().height);
+    simple_rect left_rect, right_rect;
+    if (output_full_frame) {
+        left_rect = simple_rect{-1, -1, 1, 2};
+        right_rect = simple_rect{0, -1, 1, 2};
+    } else {
+        float width_normal = left_img_dev->size().aspectRatio() /
+                             output_fbo->size().aspectRatio();
+        left_rect = simple_rect{-0.5f - width_normal / 2, -1, width_normal, 2};
+        right_rect = simple_rect{0.5f - width_normal / 2, -1, width_normal, 2};
+    }
+    opengl_render->render_rect(*left_img_dev, left_rect, false, left_cuda_stream);
+    opengl_render->render_rect(*right_img_dev, right_rect, false, right_cuda_stream);
+
+    // wait encoder idle
+    for (uint64_t cur_cnt = 0;;) {
+        if (mq().query_variable<bool>(ENCODER_BUSY, &cur_cnt)) {
+            mq().wait_variable(ENCODER_BUSY, cur_cnt);
+        } else {
+            break;
+        }
+    }
+
+    // download output frame
+    output_fbo->download(output_frame_dev.get(), output_cuda_stream);
+
+    // upload output frame
+    mq().update_variable_ptr(OUTPUT_FRAME, output_frame_dev);
 }

src/mvs_camera.cpp

@@ -1,5 +1,6 @@
 #include "mvs_camera.h"
 #include "simple_mq.h"
+#include "third_party/scope_guard.hpp"
 #include "utility.hpp"
 
 #include <MvCameraControl.h>
@@ -131,6 +132,7 @@ namespace mvs {
 
             // create impl
             auto ret = new impl;
+            auto closer = sg::make_scope_guard([&] { delete ret; });
             ret->handle = handle;
             ret->name = conf.name;
             ret->type = conf.pixel;
@@ -145,6 +147,7 @@ namespace mvs {
             API_CHECK_P(MV_CC_RegisterExceptionCallBack(handle, impl::on_error, ret));
             API_CHECK_P(MV_CC_RegisterImageCallBackEx(handle, impl::on_image, ret));
 
+            closer.dismiss();
             return ret;
         }
 

src/mvs_camera.h

@@ -17,7 +17,7 @@ namespace mvs {
     };
 
     struct capture_config {
-        float frame_rate; // frame per second
+        int frame_rate; // frame per second
         float expo_time_ms;
         float gain_db;
     };

src/simple_opengl.cpp

@@ -338,7 +338,7 @@ void simple_render::render_rect(const cv::cuda::GpuMat &img, const simple_rect &
 struct smart_frame_buffer::impl {
 
     smart_frame_buffer *q_this = nullptr;
-    cv::Size last_size;
+    cv::Size last_size = {};
     smart_texture color_tex, depth_tex;
     smart_pixel_buffer pbo;
 
@@ -402,7 +402,9 @@ struct smart_frame_buffer::impl {
 };
 
 smart_frame_buffer::smart_frame_buffer()
-        : pimpl(std::make_unique<impl>()) {}
+        : pimpl(std::make_unique<impl>()) {
+    pimpl->q_this = this;
+}
 
 smart_frame_buffer::~smart_frame_buffer() = default;
 
@@ -410,6 +412,10 @@ void smart_frame_buffer::create(cv::Size size) {
     pimpl->create(size);
 }
 
+cv::Size smart_frame_buffer::size() const {
+    return pimpl->last_size;
+}
+
 void smart_frame_buffer::download(cv::cuda::GpuMat *img, cudaStream_t stream) {
     pimpl->download(img, stream);
 }

src/simple_opengl.h

@@ -27,7 +27,7 @@ public:
 
     void render_rect(const cv::cuda::GpuMat &img,
                      const simple_rect &rect,
-                     bool remap = true,
+                     bool remap = false,
                      cudaStream_t stream = nullptr);
 
 private:
@@ -45,6 +45,8 @@ public:
 
     void create(cv::Size size);
 
+    cv::Size size() const;
+
     void download(cv::cuda::GpuMat *img,
                   cudaStream_t stream = nullptr);
 

src/third_party/scope_guard.hpp

@@ -0,0 +1,180 @@
+/*
+ *  Created on: 13/02/2018
+ *      Author: ricab
+ *
+ * See README.md for documentation of this header's public interface.
+ */
+
+#ifndef SCOPE_GUARD_HPP_
+#define SCOPE_GUARD_HPP_
+
+#include <type_traits>
+#include <utility>
+
+#if __cplusplus >= 201703L && defined(SG_REQUIRE_NOEXCEPT_IN_CPP17)
+#define SG_REQUIRE_NOEXCEPT
+#endif
+
+namespace sg {
+    namespace detail {
+        /* --- Some custom type traits --- */
+
+        // Type trait determining whether a type is callable with no arguments
+        template<typename T, typename = void>
+        struct is_noarg_callable_t
+                : public std::false_type {
+        }; // in general, false
+
+        template<typename T>
+        struct is_noarg_callable_t<T, decltype(std::declval<T &&>()())>
+                : public std::true_type {
+        }; // only true when call expression valid
+
+        // Type trait determining whether a no-argument callable returns void
+        template<typename T>
+        struct returns_void_t
+                : public std::is_same<void, decltype(std::declval<T &&>()())> {
+        };
+
+        /* Type trait determining whether a no-arg callable is nothrow invocable if
+        required. This is where SG_REQUIRE_NOEXCEPT logic is encapsulated. */
+        template<typename T>
+        struct is_nothrow_invocable_if_required_t
+                : public
+#ifdef SG_REQUIRE_NOEXCEPT
+                  std::is_nothrow_invocable<T> /* Note: _r variants not enough to
+                                          confirm void return: any return can be
+                                          discarded so all returns are
+                                          compatible with void */
+#else
+                  std::true_type
+#endif
+        {
+        };
+
+        // logic AND of two or more type traits
+        template<typename A, typename B, typename... C>
+        struct and_t : public and_t<A, and_t<B, C...>> {
+        }; // for more than two arguments
+
+        template<typename A, typename B>
+        struct and_t<A, B> : public std::conditional<A::value, B, A>::type {
+        }; // for two arguments
+
+        // Type trait determining whether a type is a proper scope_guard callback.
+        template<typename T>
+        struct is_proper_sg_callback_t
+                : public and_t<is_noarg_callable_t<T>,
+                        returns_void_t<T>,
+                        is_nothrow_invocable_if_required_t<T>,
+                        std::is_nothrow_destructible<T>> {
+        };
+
+
+        /* --- The actual scope_guard template --- */
+
+        template<typename Callback,
+                typename = typename std::enable_if<
+                        is_proper_sg_callback_t<Callback>::value>::type>
+        class scope_guard;
+
+
+        /* --- Now the friend maker --- */
+
+        template<typename Callback>
+        detail::scope_guard<Callback> make_scope_guard(Callback &&callback)
+        noexcept(std::is_nothrow_constructible<Callback, Callback &&>::value); /*
+    we need this in the inner namespace due to MSVC bugs preventing
+    sg::detail::scope_guard from befriending a sg::make_scope_guard
+    template instance in the parent namespace (see https://is.gd/xFfFhE). */
+
+
+        /* --- The template specialization that actually defines the class --- */
+
+        template<typename Callback>
+        class scope_guard<Callback> final {
+        public:
+            typedef Callback callback_type;
+
+            scope_guard(scope_guard &&other)
+            noexcept(std::is_nothrow_constructible<Callback, Callback &&>::value);
+
+            ~scope_guard() noexcept; // highlight noexcept dtor
+
+            void dismiss() noexcept;
+
+        public:
+            scope_guard() = delete;
+
+            scope_guard(const scope_guard &) = delete;
+
+            scope_guard &operator=(const scope_guard &) = delete;
+
+            scope_guard &operator=(scope_guard &&) = delete;
+
+        private:
+            explicit scope_guard(Callback &&callback)
+            noexcept(std::is_nothrow_constructible<Callback, Callback &&>::value); /*
+                                                      meant for friends only */
+
+            friend scope_guard<Callback> make_scope_guard<Callback>(Callback &&)
+            noexcept(std::is_nothrow_constructible<Callback, Callback &&>::value); /*
+      only make_scope_guard can create scope_guards from scratch (i.e. non-move)
+      */
+
+        private:
+            Callback m_callback;
+            bool m_active;
+
+        };
+
+    } // namespace detail
+
+
+    /* --- Now the single public maker function --- */
+
+    using detail::make_scope_guard; // see comment on declaration above
+
+} // namespace sg
+
+////////////////////////////////////////////////////////////////////////////////
+template<typename Callback>
+sg::detail::scope_guard<Callback>::scope_guard(Callback &&callback)
+noexcept(std::is_nothrow_constructible<Callback, Callback &&>::value)
+        : m_callback(std::forward<Callback>(callback)) /* use () instead of {} because
+    of DR 1467 (https://is.gd/WHmWuo), which still impacts older compilers
+    (e.g. GCC 4.x and clang <=3.6, see https://godbolt.org/g/TE9tPJ and
+    https://is.gd/Tsmh8G) */
+        , m_active{true} {}
+
+////////////////////////////////////////////////////////////////////////////////
+template<typename Callback>
+sg::detail::scope_guard<Callback>::~scope_guard() noexcept {
+    if (m_active)
+        m_callback();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+template<typename Callback>
+sg::detail::scope_guard<Callback>::scope_guard(scope_guard &&other)
+noexcept(std::is_nothrow_constructible<Callback, Callback &&>::value)
+        : m_callback(std::forward<Callback>(other.m_callback)) // idem
+        , m_active{std::move(other.m_active)} {
+    other.m_active = false;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+template<typename Callback>
+inline void sg::detail::scope_guard<Callback>::dismiss() noexcept {
+    m_active = false;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+template<typename Callback>
+inline auto sg::detail::make_scope_guard(Callback &&callback)
+noexcept(std::is_nothrow_constructible<Callback, Callback &&>::value)
+-> detail::scope_guard <Callback> {
+    return detail::scope_guard<Callback>{std::forward<Callback>(callback)};
+}
+
+#endif /* SCOPE_GUARD_HPP_ */

src/third_party/timestamp_helper.hpp

@@ -0,0 +1,29 @@
+#ifndef SOPHIAR2_TIMESTAMP_HELPER_HPP
+#define SOPHIAR2_TIMESTAMP_HELPER_HPP
+
+#include <chrono>
+
+namespace sophiar {
+
+    inline auto get_local_time() {
+        return std::chrono::high_resolution_clock::now();
+    }
+
+    using local_time_type = decltype(get_local_time());
+
+    extern local_time_type program_start_time;
+
+    inline auto current_timestamp() {
+        auto time_diff = get_local_time() - program_start_time;
+        return std::chrono::duration_cast<std::chrono::microseconds>(time_diff).count();
+    }
+
+    using timestamp_type = decltype(current_timestamp());
+
+    inline auto get_time_from_timestamp(timestamp_type ts) {
+        return program_start_time + std::chrono::microseconds(ts);
+    }
+
+}
+
+#endif //SOPHIAR2_TIMESTAMP_HELPER_HPP

src/variable_defs.h

@@ -4,10 +4,13 @@
 constexpr auto IMG_RAW_HOST_LEFT = 0;
 constexpr auto IMG_RAW_HOST_RIGHT = 1;
 
-constexpr auto MAIN_WINDOW_WIDTH = 20;
-constexpr auto MAIN_WINDOW_HEIGHT = 21;
+constexpr auto ENCODER_BUSY = 2;
+constexpr auto ENCODER_CONFIG = 3;
+constexpr auto OUTPUT_FRAME = 4;
+constexpr auto ENCODER_SHOULD_STOP = 5;
 
-// global variable declaration
+constexpr auto CUDA_CONTEXT = 6;
 
+// global variable declaration
 
 #endif //REMOTEAR3_VARIABLE_DEFS_H

src/video_encoder.cpp

@@ -0,0 +1,276 @@
+#include "cuda_helper.hpp"
+#include "simple_mq.h"
+#include "third_party/scope_guard.hpp"
+#include "variable_defs.h"
+#include "video_encoder.h"
+
+#include <nvEncodeAPI.h>
+
+#include <fmt/chrono.h>
+
+bool check_nvenc_api_call(NVENCSTATUS api_ret, unsigned int line_number,
+                          const char *file_name, const char *api_call_str) {
+    if (api_ret == NV_ENC_SUCCESS) [[likely]] return true;
+    SPDLOG_ERROR("NvEnc api call {} failed at {}:{} with error 0x{:x}.",
+                 api_call_str, file_name, line_number, (int) api_ret);
+    RET_ERROR;
+}
+
+#define API_CHECK(api_call) \
+    check_nvenc_api_call( \
+        api_call, __LINE__, __FILE__, #api_call)
+
+#define API_CHECK_P(api_call) \
+    if (!check_nvenc_api_call( \
+        api_call, __LINE__, __FILE__, #api_call)) [[unlikely]] \
+        return nullptr
+
+namespace video_encoder_impl {
+    constexpr auto frame_buffer_type = NV_ENC_BUFFER_FORMAT_ARGB;
+    static auto codec_guid = NV_ENC_CODEC_HEVC_GUID;
+    static auto preset_guid = NV_ENC_PRESET_P3_GUID;
+    constexpr auto tuning_info = NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY;
+    std::unique_ptr<NV_ENCODE_API_FUNCTION_LIST> api;
+}
+
+using namespace video_encoder_impl;
+using namespace simple_mq_singleton;
+
+struct video_encoder::impl {
+
+    std::unique_ptr<NV_ENC_PRESET_CONFIG> preset_config;
+    std::unique_ptr<NV_ENC_INITIALIZE_PARAMS> init_params;
+    void *encoder;
+    NV_ENC_OUTPUT_PTR output_buf;
+
+    cv::Size frame_size;
+    FILE *save_file = nullptr;
+    bool save_length;
+
+    void *last_frame_ptr = nullptr;
+    NV_ENC_REGISTERED_PTR last_reg_ptr = nullptr;
+
+    ~impl() {
+        // notify the end of stream
+        NV_ENC_PIC_PARAMS pic_params = {NV_ENC_PIC_PARAMS_VER};
+        pic_params.encodePicFlags = NV_ENC_PIC_FLAG_EOS;
+        API_CHECK(api->nvEncEncodePicture(encoder, &pic_params));
+
+        // releasing resources
+        unregister_frame_ptr();
+        API_CHECK(api->nvEncDestroyBitstreamBuffer(encoder, output_buf));
+
+        // close encoder
+        API_CHECK(api->nvEncDestroyEncoder(encoder));
+
+        // close save file
+        if (save_file != nullptr) {
+            fclose(save_file);
+        }
+
+        SPDLOG_INFO("Video encoder stopped.");
+    }
+
+    static impl *create(const encoder_config &conf) {
+        // initialize api
+        if (api == nullptr) [[unlikely]] {
+            api = std::make_unique<NV_ENCODE_API_FUNCTION_LIST>(
+                    NV_ENCODE_API_FUNCTION_LIST_VER);
+            API_CHECK_P(NvEncodeAPICreateInstance(api.get()));
+        }
+
+        // get cuda context
+//        CUcontext cuda_ctx;
+//        CUDA_API_CHECK(cuCtxGetCurrent(&cuda_ctx));
+        auto cuda_ctx = mq().query_variable<CUcontext>(CUDA_CONTEXT);
+
+        // create encoder
+        auto ret = new impl;
+        ret->frame_size = conf.frame_size;
+        auto closer = sg::make_scope_guard([&] {
+            if (ret->save_file != nullptr) {
+                fclose(ret->save_file);
+            }
+            delete ret;
+        });
+        NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS session_params = {
+                NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER};
+        session_params.deviceType = NV_ENC_DEVICE_TYPE_CUDA;
+        session_params.device = cuda_ctx;
+        session_params.apiVersion = NVENCAPI_VERSION;
+        API_CHECK_P(api->nvEncOpenEncodeSessionEx(&session_params, &ret->encoder));
+
+        // get preset config
+        ret->preset_config = std::make_unique<NV_ENC_PRESET_CONFIG>();
+        auto &preset_config = *ret->preset_config;
+        preset_config.version = NV_ENC_PRESET_CONFIG_VER;
+        preset_config.presetCfg.version = NV_ENC_CONFIG_VER;
+        API_CHECK_P(api->nvEncGetEncodePresetConfigEx(
+                ret->encoder, codec_guid, preset_guid, tuning_info, &preset_config));
+        auto &encode_config = preset_config.presetCfg;
+        encode_config.gopLength = NVENC_INFINITE_GOPLENGTH;
+        encode_config.frameIntervalP = 1;
+        auto &rc_params = encode_config.rcParams;
+        rc_params.rateControlMode = NV_ENC_PARAMS_RC_CBR;
+        rc_params.averageBitRate = conf.bitrate_mbps * 1e6;
+        rc_params.enableAQ = true;
+        rc_params.multiPass = NV_ENC_TWO_PASS_QUARTER_RESOLUTION;
+        // TODO; fine tune encoder config
+
+        // start_encode encoder
+        ret->init_params =
+                std::make_unique<NV_ENC_INITIALIZE_PARAMS>(NV_ENC_INITIALIZE_PARAMS_VER);
+        auto &init_params = *ret->init_params;
+        init_params.encodeGUID = codec_guid;
+        init_params.presetGUID = preset_guid;
+        init_params.encodeWidth = conf.frame_size.width;
+        init_params.encodeHeight = conf.frame_size.height;
+        init_params.darWidth = conf.frame_size.width; // TODO; learn more about this
+        init_params.darHeight = conf.frame_size.height; // TODO; learn more about this
+        init_params.frameRateNum = conf.frame_rate;
+        init_params.frameRateDen = 1;
+        init_params.enablePTD = 1;
+        init_params.encodeConfig = &preset_config.presetCfg;
+        init_params.maxEncodeWidth = conf.frame_size.width;
+        init_params.maxEncodeHeight = conf.frame_size.height;
+        init_params.tuningInfo = tuning_info;
+        init_params.bufferFormat = frame_buffer_type;
+        API_CHECK_P(api->nvEncInitializeEncoder(ret->encoder, &init_params));
+
+        // create output buffer
+        NV_ENC_CREATE_BITSTREAM_BUFFER buffer_config = {
+                NV_ENC_CREATE_BITSTREAM_BUFFER_VER};
+        API_CHECK_P(api->nvEncCreateBitstreamBuffer(ret->encoder, &buffer_config));
+        ret->output_buf = buffer_config.bitstreamBuffer;
+
+        // create save file
+        if (conf.save_file) {
+            auto file_name = fmt::format("record_{:%Y_%m_%d_%H_%M_%S}.{}",
+                                         std::chrono::system_clock::now(),
+                                         conf.save_length ? "dat" : "hevc");
+            ret->save_file = fopen(file_name.c_str(), "wb");
+            ret->save_length = conf.save_length;
+        }
+
+        SPDLOG_INFO("Video encoder started.");
+        closer.dismiss();
+        return ret;
+    }
+
+    void unregister_frame_ptr() {
+        if (last_reg_ptr == nullptr) return;
+        API_CHECK(api->nvEncUnregisterResource(encoder, last_reg_ptr));
+        last_reg_ptr = nullptr;
+    }
+
+    void register_frame_ptr(const cv::cuda::GpuMat &img) {
+        NV_ENC_REGISTER_RESOURCE reg_params = {NV_ENC_REGISTER_RESOURCE_VER};
+        reg_params.resourceType = NV_ENC_INPUT_RESOURCE_TYPE_CUDADEVICEPTR;
+        reg_params.width = img.size().width;
+        reg_params.height = img.size().height;
+        reg_params.pitch = img.step;
+        reg_params.resourceToRegister = img.cudaPtr();
+        reg_params.bufferFormat = frame_buffer_type;
+        reg_params.bufferUsage = NV_ENC_INPUT_IMAGE;
+        API_CHECK(api->nvEncRegisterResource(encoder, &reg_params));
+        last_reg_ptr = reg_params.registeredResource;
+    }
+
+    void encode(const cv::cuda::GpuMat &img, video_nal *out, bool force_idr) {
+        // register pointer if needed
+        if (img.cudaPtr() != last_frame_ptr) [[unlikely]] {
+            assert(img.size() == frame_size);
+            unregister_frame_ptr();
+            register_frame_ptr(img);
+        }
+
+        // map input resource
+        NV_ENC_MAP_INPUT_RESOURCE map_params = {
+                NV_ENC_MAP_INPUT_RESOURCE_VER};
+        map_params.registeredResource = last_reg_ptr;
+        API_CHECK(api->nvEncMapInputResource(encoder, &map_params));
+        assert(map_params.mappedBufferFmt == frame_buffer_type);
+
+        // encode frame
+        NV_ENC_PIC_PARAMS pic_params = {NV_ENC_PIC_PARAMS_VER};
+        pic_params.inputWidth = img.size().width;
+        pic_params.inputHeight = img.size().height;
+        pic_params.inputPitch = img.step;
+        if (force_idr) { // request for IDR frame
+            pic_params.encodePicFlags = NV_ENC_PIC_FLAG_FORCEIDR | NV_ENC_PIC_FLAG_OUTPUT_SPSPPS;
+        } else {
+            pic_params.encodePicFlags = 0;
+        }
+        pic_params.inputBuffer = map_params.mappedResource;
+        pic_params.outputBitstream = output_buf;
+        pic_params.bufferFmt = frame_buffer_type;
+        pic_params.pictureStruct = NV_ENC_PIC_STRUCT_FRAME; // TODO; learn more about this
+        API_CHECK(api->nvEncEncodePicture(encoder, &pic_params));
+
+        // get encoded bitstream
+        NV_ENC_LOCK_BITSTREAM lock_config = {NV_ENC_LOCK_BITSTREAM_VER};
+        lock_config.doNotWait = false; // block until encode completed.
+        lock_config.outputBitstream = output_buf;
+        API_CHECK(api->nvEncLockBitstream(encoder, &lock_config));
+
+        // copy bitstream
+        out->create(lock_config.bitstreamBufferPtr,
+                    lock_config.bitstreamSizeInBytes,
+                    lock_config.pictureType == NV_ENC_PIC_TYPE_IDR);
+
+        // save bitstream
+        if (save_file != nullptr) {
+            if (save_length) {
+                fwrite(&out->length, sizeof(size_t), 1, save_file);
+            }
+            fwrite(out->ptr, out->length, 1, save_file);
+        }
+
+        // cleanup
+        API_CHECK(api->nvEncUnlockBitstream(encoder, output_buf));
+        API_CHECK(api->nvEncUnmapInputResource(encoder, map_params.mappedResource));
+    }
+
+    void change_config(const encoder_config &conf) {
+        NV_ENC_RECONFIGURE_PARAMS params = {NV_ENC_RECONFIGURE_PARAMS_VER};
+        init_params->frameRateNum = conf.frame_rate;
+        init_params->encodeConfig->rcParams.averageBitRate = conf.bitrate_mbps * 1e6;
+        params.reInitEncodeParams = *init_params;
+        params.resetEncoder = true;
+        params.forceIDR = true;
+        API_CHECK(api->nvEncReconfigureEncoder(encoder, &params));
+    }
+};
+
+video_encoder::~video_encoder() = default;
+
+video_encoder *video_encoder::create(const encoder_config &conf) {
+    auto pimpl = impl::create(conf);
+    if (pimpl == nullptr) return nullptr;
+    auto ret = new video_encoder;
+    ret->pimpl.reset(pimpl);
+    return ret;
+}
+
+void video_encoder::encode(const cv::cuda::GpuMat &img, video_nal *out, bool force_idr) {
+    mq().update_variable(ENCODER_BUSY, true);
+    pimpl->encode(img, out, force_idr);
+    mq().update_variable(ENCODER_BUSY, false);
+}
+
+void video_encoder::change_config(const encoder_config &conf) {
+    pimpl->change_config(conf);
+}
+
+void video_nal::create(void *_ptr, size_t _length, bool _idr) {
+    free(ptr);
+    length = _length;
+    idr = _idr;
+    ptr = malloc(length);
+    assert(_ptr != nullptr);
+    memcpy(ptr, _ptr, length);
+}
+
+video_nal::~video_nal() {
+    free(ptr);
+}

src/video_encoder.h

@@ -0,0 +1,44 @@
+#ifndef REMOTEAR3_VIDEO_ENCODER_H
+#define REMOTEAR3_VIDEO_ENCODER_H
+
+#include <opencv2/core/cuda.hpp>
+
+#include <memory>
+
+struct encoder_config {
+    cv::Size frame_size;
+    int frame_rate;
+    float bitrate_mbps;
+    bool save_file;
+    bool save_length;
+};
+
+struct video_nal {
+    void *ptr = nullptr;
+    size_t length = 0;
+    bool idr = false;
+
+    void create(void *ptr, size_t length, bool idr);
+
+    ~video_nal();
+};
+
+class video_encoder {
+public:
+
+    ~video_encoder();
+
+    void encode(const cv::cuda::GpuMat &img, video_nal *out, bool force_idr = false);
+
+    // only frame_rate and bitrate can be changed
+    void change_config(const encoder_config &conf);
+
+    static video_encoder *create(const encoder_config &conf);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+
+#endif //REMOTEAR3_VIDEO_ENCODER_H