Implemented nvJPEG encoder.

CMakeLists.txt

@@ -5,6 +5,7 @@ set(CMAKE_CXX_STANDARD 20)
 
 add_executable(RemoteAR3 src/main.cpp
         src/main_ext.cpp
+        src/frame_encoder/encoder_base.cpp
         src/frame_sender/sender_base.cpp
         src/frame_sender/sender_udp_fec.cpp
         src/frame_sender/sender_tcp.cpp
@@ -100,7 +101,11 @@ endif ()
 set(NVCODEC_INCLUDE_DIR ${NVCODEC_DIR}/Interface)
 target_include_directories(${PROJECT_NAME} PRIVATE ${NVCODEC_INCLUDE_DIR})
 target_link_libraries(${PROJECT_NAME} ${NVENC_LIB})
-target_sources(${PROJECT_NAME} PRIVATE src/video_encoder.cpp)
+target_sources(${PROJECT_NAME} PRIVATE src/frame_encoder/encoder_nvenc.cpp)
+
+# nvJPEG config
+target_link_libraries(${PROJECT_NAME} nvjpeg)
+target_sources(${PROJECT_NAME} PRIVATE src/frame_encoder/encoder_jpeg.cpp)
 
 # yaml-cpp
 find_package(yaml-cpp REQUIRED)

data/config.yaml

@@ -33,7 +33,8 @@ main_window:
 output:
   width: 1920
   height: 1080
-  bitrate: 5 # Mbps
+  hevc_bitrate: 5 # Mbps
+  jpeg_quality: 15
 
 sender:
   mtu: 1400

src/cuda_helper.hpp

@@ -17,7 +17,7 @@ inline bool check_cuda_api_call(CUresult api_ret, unsigned int line_number,
     if (ret != CUDA_SUCCESS) [[unlikely]] error_name = "Unknown";
     ret = cuGetErrorString(api_ret, &error_str);
     if (ret != CUDA_SUCCESS) [[unlikely]] error_str = "Unknown";
-    SPDLOG_ERROR("CUDA runtime api call {} failed at {}:{} with error 0x{:x}:{}, {}.",
+    SPDLOG_ERROR("CUDA driver api call {} failed at {}:{} with error 0x{:x}:{}, {}.",
                  api_call_str, file_name, line_number,
                  (int) api_ret, error_name, error_str);
     RET_ERROR_B;
@@ -26,7 +26,7 @@ inline bool check_cuda_api_call(CUresult api_ret, unsigned int line_number,
 inline bool check_cuda_api_call(cudaError api_ret, unsigned int line_number,
                                 const char *file_name, const char *api_call_str) {
     if (api_ret == cudaSuccess) [[likely]] return true;
-    SPDLOG_ERROR("CUDA driver api call {} failed at {}:{} with error 0x{:x}.",
+    SPDLOG_ERROR("CUDA runtime api call {} failed at {}:{} with error 0x{:x}.",
                  api_call_str, file_name, line_number, (int) api_ret);
     RET_ERROR_B;
 }
@@ -48,4 +48,18 @@ inline bool check_cuda_api_call(NppStatus api_ret, unsigned int line_number,
         api_call, __LINE__, __FILE__, #api_call)) [[unlikely]] \
         return nullptr
 
+struct smart_cuda_stream {
+    cudaStream_t obj = nullptr;
+
+    smart_cuda_stream() {
+        CUDA_API_CHECK(cudaStreamCreate(&obj));
+        assert(obj != nullptr);
+    }
+
+    ~smart_cuda_stream() {
+        assert(obj != nullptr);
+        CUDA_API_CHECK(cudaStreamDestroy(obj));
+    }
+};
+
 #endif //REMOTEAR3_CUDA_HELPER_H

src/debug_utility.hpp

@@ -0,0 +1,13 @@
+#ifndef REMOTEAR3_DEBUG_UTILITY_HPP
+#define REMOTEAR3_DEBUG_UTILITY_HPP
+
+#include <opencv2/core/cuda.hpp>
+#include <opencv2/imgcodecs.hpp>
+
+inline void save_image(const cv::cuda::GpuMat &mat, const std::string &path) {
+    cv::Mat host_mat;
+    mat.download(host_mat);
+    cv::imwrite(path, host_mat);
+}
+
+#endif //REMOTEAR3_DEBUG_UTILITY_HPP

src/frame_encoder/encoder_base.cpp

@@ -0,0 +1,18 @@
+#include "encoder_base.h"
+
+#include <cassert>
+#include <cstring>
+
+void video_nal::create(void *_ptr, size_t _length, bool _idr) {
+    free(ptr);
+    length = _length;
+    idr = _idr;
+    ptr = (uint8_t *) malloc(length);
+    if (_ptr != nullptr) {
+        memcpy(ptr, _ptr, length);
+    }
+}
+
+video_nal::~video_nal() {
+    free(ptr);
+}

src/frame_encoder/encoder_base.h

@@ -0,0 +1,31 @@
+#ifndef REMOTEAR3_ENCODER_BASE_H
+#define REMOTEAR3_ENCODER_BASE_H
+
+#include <opencv2/core/cuda.hpp>
+
+enum encoder_type {
+    ENCODER_NVENC,
+    ENCODER_JPEG
+};
+
+struct video_nal {
+    uint8_t *ptr = nullptr;
+    size_t length = 0;
+    bool idr = false;
+
+    void create(void *ptr, size_t length, bool idr);
+
+    ~video_nal();
+};
+
+class encoder_base {
+public:
+
+    virtual ~encoder_base() = default;
+
+    virtual void encode(const cv::cuda::GpuMat &img, video_nal *out, bool force_idr) = 0;
+
+};
+
+
+#endif //REMOTEAR3_ENCODER_BASE_H

src/frame_encoder/encoder_jpeg.cpp

@@ -0,0 +1,169 @@
+#include "encoder_jpeg.h"
+#include "cuda_helper.hpp"
+#include "simple_mq.h"
+#include "third_party/scope_guard.hpp"
+#include "variable_defs.h"
+#include "utility.hpp"
+
+#include <nvjpeg.h>
+
+#include <fmt/chrono.h>
+
+#include <opencv2/cudaimgproc.hpp>
+
+bool check_nvjpeg_api_call(nvjpegStatus_t api_ret, unsigned int line_number,
+                           const char *file_name, const char *api_call_str) {
+    if (api_ret == NVJPEG_STATUS_SUCCESS) [[likely]] return true;
+    SPDLOG_ERROR("nvJPEG api call {} failed at {}:{} with error 0x{:x}.",
+                 api_call_str, file_name, line_number, (int) api_ret);
+    RET_ERROR_B;
+}
+
+#define API_CHECK(api_call) \
+    check_nvjpeg_api_call( \
+        api_call, __LINE__, __FILE__, #api_call)
+
+#define API_CHECK_P(api_call) \
+    if (!check_nvjpeg_api_call( \
+        api_call, __LINE__, __FILE__, #api_call)) [[unlikely]] \
+        return nullptr
+
+namespace encoder_jpeg_impl {
+    nvjpegHandle_t handle = nullptr;
+}
+
+using namespace encoder_jpeg_impl;
+using namespace simple_mq_singleton;
+
+struct encoder_jpeg::impl {
+
+    nvjpegEncoderState_t enc_state = nullptr;
+    nvjpegEncoderParams_t enc_params = nullptr;
+
+    std::unique_ptr<cv::cuda::Stream> cv_stream;
+    cudaStream_t stream; // content of cv_stream
+    cv::cuda::GpuMat img_bgr;
+
+    FILE *save_file = nullptr;
+    bool save_length;
+
+    cudaStream_t output_stream = nullptr;
+    cudaEvent_t output_event = nullptr;
+
+    ~impl() {
+        API_CHECK(nvjpegEncoderParamsDestroy(enc_params));
+        API_CHECK(nvjpegEncoderStateDestroy(enc_state));
+        CUDA_API_CHECK(cudaEventDestroy(output_event));
+
+        // close save file
+        if (save_file != nullptr) {
+            fclose(save_file);
+        }
+
+        SPDLOG_INFO("Video encoder stopped.");
+    }
+
+    static impl *create(nvjpeg_config conf) {
+        // binding cuda context
+        auto cuda_ctx = mq().query_variable<CUcontext>(CUDA_CONTEXT);
+        CUDA_API_CHECK(cuCtxSetCurrent(cuda_ctx));
+
+        // create encoder
+        auto ret = new impl;
+        auto closer = sg::make_scope_guard([&] {
+            if (ret->save_file != nullptr) {
+                fclose(ret->save_file);
+            }
+            delete ret;
+        });
+        if (handle == nullptr) [[unlikely]] {
+            API_CHECK_P(nvjpegCreateSimple(&handle));
+        }
+        ret->cv_stream = std::make_unique<cv::cuda::Stream>();
+        ret->stream = (cudaStream_t) ret->cv_stream->cudaPtr();
+        CUDA_API_CHECK(cudaEventCreateWithFlags(&ret->output_event, cudaEventDisableTiming));
+        API_CHECK_P(nvjpegEncoderStateCreate(handle, &ret->enc_state, ret->stream));
+        API_CHECK_P(nvjpegEncoderParamsCreate(handle, &ret->enc_params, ret->stream));
+        ret->output_stream = mq().query_variable<cudaStream_t>(CUDA_STREAM_OUTPUT);
+
+        // config parameters
+        API_CHECK_P(nvjpegEncoderParamsSetOptimizedHuffman(ret->enc_params, true, ret->stream));
+        API_CHECK_P(nvjpegEncoderParamsSetSamplingFactors(ret->enc_params, NVJPEG_CSS_420, ret->stream));
+        assert(conf.quality >= 1 && conf.quality <= 100);
+        API_CHECK_P(nvjpegEncoderParamsSetQuality(ret->enc_params, conf.quality, ret->stream));
+
+        // create save file
+        if (conf.save_file) {
+            auto file_name = fmt::format("record_{:%Y_%m_%d_%H_%M_%S}.{}",
+                                         std::chrono::system_clock::now(),
+                                         conf.save_length ? "dat" : "mjpeg");
+            ret->save_file = fopen(file_name.c_str(), "wb");
+            ret->save_length = conf.save_length;
+        }
+
+        SPDLOG_INFO("Video encoder started.");
+        closer.dismiss();
+        return ret;
+    }
+
+    void encode(const cv::cuda::GpuMat &img_bgra, video_nal *out) {
+        // wait for output stream
+        CUDA_API_CHECK(cudaEventRecord(output_event, output_stream));
+        CUDA_API_CHECK(cudaStreamWaitEvent(stream, output_event));
+
+        // remove alpha channel
+        cv::cuda::cvtColor(img_bgra, img_bgr, cv::COLOR_BGRA2BGR, 3, *cv_stream);
+
+        // create image
+        nvjpegImage_t img_in{};
+        img_in.channel[0] = (uint8_t *) img_bgr.cudaPtr();
+        img_in.pitch[0] = img_bgr.step;
+
+        // encode frame
+        API_CHECK(nvjpegEncodeImage(handle, enc_state, enc_params, &img_in,
+                                    NVJPEG_INPUT_BGRI, img_bgr.cols, img_bgr.rows, stream));
+
+        // retrieve encoded image
+        size_t length;
+        API_CHECK(nvjpegEncodeRetrieveBitstream(handle, enc_state, nullptr, &length, stream));
+        out->create(nullptr, length, true);
+        API_CHECK(nvjpegEncodeRetrieveBitstream(handle, enc_state, out->ptr, &length, stream));
+
+        // sync stream
+        CUDA_API_CHECK(cudaStreamSynchronize(stream));
+
+        // save bitstream
+        if (save_file != nullptr) {
+            if (save_length) {
+                fwrite(&out->length, sizeof(size_t), 1, save_file);
+            }
+            fwrite(out->ptr, out->length, 1, save_file);
+        }
+    }
+
+    void change_config(const nvjpeg_config &conf) {
+        assert(conf.save_file == (save_file != nullptr));
+        assert(conf.quality >= 1 && conf.quality <= 100);
+        API_CHECK(nvjpegEncoderParamsSetQuality(enc_params, conf.quality, stream));
+    }
+};
+
+encoder_jpeg::~encoder_jpeg() = default;
+
+encoder_jpeg *encoder_jpeg::create(nvjpeg_config conf) {
+    auto pimpl = impl::create(conf);
+    if (pimpl == nullptr) return nullptr;
+    auto ret = new encoder_jpeg;
+    ret->pimpl.reset(pimpl);
+    return ret;
+}
+
+void encoder_jpeg::encode(const cv::cuda::GpuMat &img, video_nal *out, bool) {
+    mq().update_variable(ENCODER_BUSY, true);
+    pimpl->encode(img, out);
+    mq().update_variable(ENCODER_BUSY, false);
+}
+
+void encoder_jpeg::change_config(const nvjpeg_config &conf) {
+    pimpl->change_config(conf);
+}

src/frame_encoder/encoder_jpeg.h

@@ -0,0 +1,32 @@
+#ifndef REMOTEAR3_ENCODER_JPEG_H
+#define REMOTEAR3_ENCODER_JPEG_H
+
+#include "encoder_base.h"
+
+#include <memory>
+
+struct nvjpeg_config {
+    int quality;
+    bool save_file;
+    bool save_length;
+};
+
+class encoder_jpeg : public encoder_base {
+public:
+
+    ~encoder_jpeg() override;
+
+    void encode(const cv::cuda::GpuMat &img, video_nal *out, bool) override;
+
+    // only quality can be changed
+    void change_config(const nvjpeg_config &conf);
+
+    static encoder_jpeg *create(nvjpeg_config conf);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+
+#endif //REMOTEAR3_ENCODER_JPEG_H

src/video_encoder.cpp → src/frame_encoder/encoder_nvenc.cpp

@@ -2,7 +2,7 @@
 #include "simple_mq.h"
 #include "third_party/scope_guard.hpp"
 #include "variable_defs.h"
-#include "video_encoder.h"
+#include "encoder_nvenc.h"
 
 #include <nvEncodeAPI.h>
 
@@ -46,7 +46,7 @@ namespace video_encoder_impl {
 using namespace video_encoder_impl;
 using namespace simple_mq_singleton;
 
-struct video_encoder::impl {
+struct encoder_nvenc::impl {
 
     std::unique_ptr<NV_ENC_PRESET_CONFIG> preset_config;
     std::unique_ptr<NV_ENC_INITIALIZE_PARAMS> init_params;
@@ -81,7 +81,7 @@ struct video_encoder::impl {
         SPDLOG_INFO("Video encoder stopped.");
     }
 
-    static impl *create(const encoder_config &conf) {
+    static impl *create(const nvenc_config &conf) {
         // initialize api
         if (api == nullptr) [[unlikely]] {
             api = std::make_unique<NV_ENCODE_API_FUNCTION_LIST>(
@@ -241,7 +241,7 @@ struct video_encoder::impl {
         API_CHECK(api->nvEncUnmapInputResource(encoder, map_params.mappedResource));
     }
 
-    void change_config(const encoder_config &conf) {
+    void change_config(const nvenc_config &conf) {
         assert(conf.frame_size == frame_size);
         assert(conf.save_file == (save_file != nullptr));
         assert(conf.save_length == save_length);
@@ -255,35 +255,22 @@ struct video_encoder::impl {
     }
 };
 
-video_encoder::~video_encoder() = default;
+encoder_nvenc::~encoder_nvenc() = default;
 
-video_encoder *video_encoder::create(const encoder_config &conf) {
+encoder_nvenc *encoder_nvenc::create(const nvenc_config &conf) {
     auto pimpl = impl::create(conf);
     if (pimpl == nullptr) return nullptr;
-    auto ret = new video_encoder;
+    auto ret = new encoder_nvenc;
     ret->pimpl.reset(pimpl);
     return ret;
 }
 
-void video_encoder::encode(const cv::cuda::GpuMat &img, video_nal *out, bool force_idr) {
+void encoder_nvenc::encode(const cv::cuda::GpuMat &img, video_nal *out, bool force_idr) {
     mq().update_variable(ENCODER_BUSY, true);
     pimpl->encode(img, out, force_idr);
     mq().update_variable(ENCODER_BUSY, false);
 }
 
-void video_encoder::change_config(const encoder_config &conf) {
+void encoder_nvenc::change_config(const nvenc_config &conf) {
     pimpl->change_config(conf);
 }
-
-void video_nal::create(void *_ptr, size_t _length, bool _idr) {
-    free(ptr);
-    length = _length;
-    idr = _idr;
-    ptr = (uint8_t *) malloc(length);
-    assert(_ptr != nullptr);
-    memcpy(ptr, _ptr, length);
-}
-
-video_nal::~video_nal() {
-    free(ptr);
-}

src/frame_encoder/encoder_nvenc.h

@@ -0,0 +1,34 @@
+#ifndef REMOTEAR3_ENCODER_NVENC_H
+#define REMOTEAR3_ENCODER_NVENC_H
+
+#include "encoder_base.h"
+
+#include <memory>
+
+struct nvenc_config {
+    cv::Size frame_size;
+    int frame_rate;
+    float bitrate_mbps;
+    bool save_file;
+    bool save_length;
+};
+
+class encoder_nvenc : public encoder_base {
+public:
+
+    ~encoder_nvenc() override;
+
+    void encode(const cv::cuda::GpuMat &img, video_nal *out, bool force_idr) override;
+
+    // only frame_rate and bitrate can be changed
+    void change_config(const nvenc_config &conf);
+
+    static encoder_nvenc *create(const nvenc_config &conf);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+
+#endif //REMOTEAR3_ENCODER_NVENC_H

src/frame_sender/sender_base.h

@@ -1,7 +1,7 @@
 #ifndef REMOTEAR3_SENDER_BASE_HPP
 #define REMOTEAR3_SENDER_BASE_HPP
 
-#include "video_encoder.h"
+#include "../frame_encoder/encoder_base.h"
 
 #include <boost/asio/io_context.hpp>
 
@@ -19,14 +19,14 @@ public:
 
     virtual ~sender_base();
 
-    void send_frame(std::unique_ptr<video_nal> &&frame);
+    using frame_ptr_type = std::unique_ptr<video_nal>;
+
+    void send_frame(frame_ptr_type &&frame);
 
     void run();
 
 protected:
 
-    using frame_ptr_type = std::unique_ptr<video_nal>;
-
     void request_idr_frame();
 
     virtual void handle_frame(frame_ptr_type &&frame) = 0;

src/main_ext.cpp

@@ -2,6 +2,8 @@
 #include "core/local_connection.h"
 #include "core/timestamp_helper.hpp"
 #include "cuda_helper.hpp"
+#include "frame_encoder/encoder_nvenc.h"
+#include "frame_encoder/encoder_jpeg.h"
 #include "frame_sender/sender_tcp.h"
 #include "frame_sender/sender_udp_fec.h"
 #include "image_process.h"
@@ -11,7 +13,6 @@
 #include "third_party/scope_guard.hpp"
 #include "utility.hpp"
 #include "variable_defs.h"
-#include "video_encoder.h"
 #include "vtk_viewer.h"
 
 #ifdef _MSC_VER
@@ -75,7 +76,11 @@ std::unique_ptr<vtk_viewer> augment_viewer;
 std::unique_ptr<std::thread> encoder_thread;
 bool output_full_frame = false;
 int output_width = -1, output_height = -1;
-encoder_config main_encoder_conf;
+encoder_type chosen_encoder = ENCODER_JPEG;
+nvenc_config main_nvenc_conf;
+nvjpeg_config main_nvjpeg_conf;
+bool encoder_save_file = false;
+bool encoder_save_length = false;
 std::unique_ptr<smart_frame_buffer> output_fbo;
 std::unique_ptr<cv::cuda::Stream> output_stream;
 cudaStream_t output_cuda_stream = nullptr;
@@ -229,6 +234,7 @@ void initialize_cuda() {
     right = std::make_unique<camera_related>();
     output_stream = std::make_unique<cv::cuda::Stream>();
     output_cuda_stream = (cudaStream_t) output_stream->cudaPtr();
+    mq().update_variable(CUDA_STREAM_OUTPUT, output_cuda_stream);
     output_frame_dev = std::make_shared<cv::cuda::GpuMat>();
 }
 
@@ -242,7 +248,7 @@ void load_config() {
     right_camera_name = camera_names["right"].as<std::string>();
     auto capture_param = camera_conf["capture"];
     capture_conf.frame_rate = capture_param["frame_rate"].as<int>();
-    main_encoder_conf.frame_rate = capture_conf.frame_rate;
+    main_nvenc_conf.frame_rate = capture_conf.frame_rate;
     capture_conf.expo_time_ms = capture_param["expo_time_ms"].as<float>();
     capture_conf.gain_db = capture_param["gain_db"].as<float>();
 
@@ -268,7 +274,8 @@ void load_config() {
     auto output_conf = conf["output"];
     output_width = output_conf["width"].as<int>();
     output_height = output_conf["height"].as<int>();
-    main_encoder_conf.bitrate_mbps = output_conf["bitrate"].as<float>();
+    main_nvenc_conf.bitrate_mbps = output_conf["hevc_bitrate"].as<float>();
+    main_nvjpeg_conf.quality = output_conf["jpeg_quality"].as<int>();
     left->process_conf.resample_height = output_height; // use output height as resample height
     right->process_conf.resample_height = output_height;
 
@@ -482,8 +489,7 @@ void sender_thread_work() {
             break;
         }
         default: {
-            assert(false);
-            unreachable();
+            RET_ERROR;
         }
     }
 
@@ -493,8 +499,25 @@ void sender_thread_work() {
 
 void encoder_thread_work() {
     uint64_t last_conf_cnt;
-    auto conf = mq().query_variable<encoder_config>(ENCODER_CONFIG, &last_conf_cnt);
-    auto encoder = std::unique_ptr<video_encoder>(video_encoder::create(conf));
+    std::unique_ptr<encoder_base> encoder;
+    bool save_file = false;
+    switch (chosen_encoder) {
+        case ENCODER_NVENC: {
+            auto conf = mq().query_variable<nvenc_config>(ENCODER_CONFIG, &last_conf_cnt);
+            save_file = conf.save_file;
+            encoder.reset(encoder_nvenc::create(conf));
+            break;
+        }
+        case ENCODER_JPEG: {
+            auto conf = mq().query_variable<nvjpeg_config>(ENCODER_CONFIG, &last_conf_cnt);
+            save_file = conf.save_file;
+            encoder.reset(encoder_jpeg::create(conf));
+            break;
+        }
+        default: {
+            RET_ERROR;
+        }
+    }
 
     uint64_t frame_cnt = 0;
     for (;;) {
@@ -504,17 +527,37 @@ void encoder_thread_work() {
         // test stop flag
         if (mq().query_variable<bool>(ENCODER_SHOULD_STOP)) break;
 
-        if (!(conf.save_file ||
+        if (!(save_file ||
               mq().query_variable<bool>(SENDER_CONNECTED))) {
             continue; // no need to encode frame
         }
 
         // check for config update
         uint64_t cur_conf_cnt;
-        conf = mq().query_variable<encoder_config>(ENCODER_CONFIG, &cur_conf_cnt);
-        if (cur_conf_cnt > last_conf_cnt) {
-            encoder->change_config(conf);
-            last_conf_cnt = cur_conf_cnt;
+        switch (chosen_encoder) {
+            case ENCODER_NVENC: {
+                auto conf = mq().query_variable<nvenc_config>(ENCODER_CONFIG, &cur_conf_cnt);
+                if (cur_conf_cnt > last_conf_cnt) {
+                    auto real_encoder = dynamic_cast<encoder_nvenc *>(encoder.get());
+                    assert(real_encoder != nullptr);
+                    real_encoder->change_config(conf);
+                    last_conf_cnt = cur_conf_cnt;
+                }
+                break;
+            }
+            case ENCODER_JPEG: {
+                auto conf = mq().query_variable<nvjpeg_config>(ENCODER_CONFIG, &cur_conf_cnt);
+                if (cur_conf_cnt > last_conf_cnt) {
+                    auto real_encoder = dynamic_cast<encoder_jpeg *>(encoder.get());
+                    assert(real_encoder != nullptr);
+                    real_encoder->change_config(conf);
+                    last_conf_cnt = cur_conf_cnt;
+                }
+                break;
+            }
+            default : {
+                RET_ERROR;
+            }
         }
 
         bool force_idr = false;
@@ -523,7 +566,7 @@ void encoder_thread_work() {
             mq().update_variable(REQUEST_IDR, false);
         }
 
-        assert(frame != nullptr);
+        if (frame == nullptr) continue;
         auto frame_data = std::make_unique<video_nal>();
         encoder->encode(*frame, frame_data.get(), force_idr);
         if (mq().query_variable<bool>(SENDER_CONNECTED)) {
@@ -556,22 +599,53 @@ bool is_encoding() {
 }
 
 void upload_encoder_config() {
-    mq().update_variable(ENCODER_CONFIG, main_encoder_conf);
+    switch (chosen_encoder) {
+        case ENCODER_NVENC: {
+            main_nvenc_conf.save_file = encoder_save_file;
+            main_nvenc_conf.save_length = encoder_save_length;
+            mq().update_variable(ENCODER_CONFIG, main_nvenc_conf);
+            break;
+        }
+        case ENCODER_JPEG: {
+            main_nvjpeg_conf.save_file = encoder_save_file;
+            main_nvjpeg_conf.save_length = encoder_save_length;
+            mq().update_variable(ENCODER_CONFIG, main_nvjpeg_conf);
+            break;
+        }
+        default: {
+            RET_ERROR;
+        }
+    }
 }
 
 void start_encoder() {
+    // determine output frame size
+    cv::Size frame_size;
     if (output_full_frame) {
         assert(!left->img_dev->empty() && !right->img_dev->empty());
         assert(left->img_dev->size() == right->img_dev->size());
-        main_encoder_conf.frame_size = cv::Size{
-                left->img_dev->size().width * 2,
-                left->img_dev->size().height};
-        output_fbo->create(main_encoder_conf.frame_size);
+        frame_size.width = left->img_dev->size().width * 2;
+        frame_size.height = left->img_dev->size().height;
     } else {
-        main_encoder_conf.frame_size = cv::Size{output_width, output_height};
-        output_fbo->create(main_encoder_conf.frame_size);
+        frame_size = {output_width, output_height};
+    }
+    output_fbo->create(frame_size);
+
+    // update config
+    switch (chosen_encoder) {
+        case ENCODER_NVENC: {
+            main_nvenc_conf.frame_size = frame_size;
+            break;
+        }
+        case ENCODER_JPEG: {
+            break;
+        }
+        default: {
+            RET_ERROR;
+        }
     }
     upload_encoder_config();
+
     mq().update_variable(ENCODER_SHOULD_STOP, false);
     mq().update_variable(ENCODER_BUSY, false); // make variable exist
     encoder_thread = std::make_unique<std::thread>(encoder_thread_work);
@@ -658,7 +732,7 @@ void prepare_imgui_frame() {
             ImGui::SeparatorText("Configs");
             if (ImGui::DragInt("Frame Rate (fps)", &capture_conf.frame_rate, 1, 1, 60)) {
                 simple_eq.emplace(upload_capture_config);
-                main_encoder_conf.frame_rate = capture_conf.frame_rate;
+                main_nvenc_conf.frame_rate = capture_conf.frame_rate;
                 simple_eq.emplace(upload_encoder_config);
             }
             if (ImGui::DragFloat("Exposure Time (ms)", &capture_conf.expo_time_ms,
@@ -783,6 +857,21 @@ void prepare_imgui_frame() {
         if (ImGui::CollapsingHeader("Video Encoder")) {
             ImGui::PushID("Encoder");
 
+            ImGui::SeparatorText("Method");
+            if (is_encoding()) {
+                ImGui::BeginDisabled();
+            }
+            if (ImGui::RadioButton("NvEnc", chosen_encoder == ENCODER_NVENC)) {
+                simple_eq.emplace([] { chosen_encoder = ENCODER_NVENC; });
+            }
+            ImGui::SameLine();
+            if (ImGui::RadioButton("nvJPEG", chosen_encoder == ENCODER_JPEG)) {
+                simple_eq.emplace([] { chosen_encoder = ENCODER_JPEG; });
+            }
+            if (is_encoding()) {
+                ImGui::EndDisabled();
+            }
+
             ImGui::SeparatorText("Actions");
             if (!is_encoding()) {
                 if (ImGui::Button("Start")) {
@@ -795,20 +884,33 @@ void prepare_imgui_frame() {
             }
 
             ImGui::SeparatorText("Configs");
-            if (ImGui::DragFloat("Bitrate (Mbps)", &main_encoder_conf.bitrate_mbps,
-                                 0.1, 1, 20, "%.01f")) {
-                simple_eq.emplace(upload_encoder_config);
+            switch (chosen_encoder) {
+                case ENCODER_NVENC: {
+                    if (ImGui::DragFloat("Bitrate (Mbps)", &main_nvenc_conf.bitrate_mbps,
+                                         0.1, 1, 20, "%.01f")) {
+                        simple_eq.emplace(upload_encoder_config);
+                    }
+                    break;
+                }
+                case ENCODER_JPEG: {
+                    if (ImGui::DragInt("Quality (%)", &main_nvjpeg_conf.quality, 1, 1, 100)) {
+                        simple_eq.emplace(upload_encoder_config);
+                    }
+                    break;
+                }
+                default: {
+                    RET_ERROR;
+                }
             }
-
             if (is_encoding()) {
                 ImGui::BeginDisabled();
             }
             ImGui::Checkbox("Full Resolution", &output_full_frame);
             ImGui::SameLine();
-            ImGui::Checkbox("Save Video", &main_encoder_conf.save_file);
-            if (main_encoder_conf.save_file) {
+            ImGui::Checkbox("Save Video", &encoder_save_file);
+            if (encoder_save_file) {
                 ImGui::SameLine();
-                ImGui::Checkbox("Save Frame Length", &main_encoder_conf.save_length);
+                ImGui::Checkbox("Save Frame Length", &encoder_save_length);
             }
             if (is_encoding()) {
                 ImGui::EndDisabled();

src/simple_opengl.cpp

@@ -25,18 +25,6 @@ namespace simple_opengl_impl {
         }
     )";
 
-    constexpr auto remap_frag_shader_source = R"(
-        #version 460
-        layout (location = 0) out vec4 color_out;
-        in vec2 tex_coord;
-        uniform sampler2D image_tex;
-        uniform sampler2D remap_tex;
-        void main() {
-            vec2 tex_coord_real = texture(remap_tex, tex_coord).xy;
-            color_out = texture(image_tex, tex_coord_real);
-        }
-    )";
-
     constexpr GLuint rect_indices[] = {
             0, 1, 3, // first triangle
             1, 2, 3 // second triangle
@@ -110,8 +98,7 @@ using namespace simple_opengl_impl;
 
 struct simple_render::impl {
     GLuint vao = 0, vbo = 0, ebo = 0;
-    GLuint simple_program = 0, remap_program = 0;
-    GLint image_tex_loc = 0, remap_tex_loc = 0;
+    GLuint simple_program = 0;
 
     smart_pixel_buffer image_pbo;
     smart_texture image_tex;
@@ -164,7 +151,6 @@ struct simple_render::impl {
         auto remap_frag_shader = glCreateShader(GL_FRAGMENT_SHADER);
         compile_shader(simple_vert_shader, simple_vert_shader_source, "simple_vertex");
         compile_shader(simple_frag_shader, simple_frag_shader_source, "simple_fragment");
-        compile_shader(remap_frag_shader, remap_frag_shader_source, "remap_fragment");
 
         simple_program = glCreateProgram();
         glAttachShader(simple_program, simple_vert_shader);
@@ -172,20 +158,10 @@ struct simple_render::impl {
         glLinkProgram(simple_program);
         check_program(simple_program);
 
-        remap_program = glCreateProgram();
-        glAttachShader(remap_program, simple_vert_shader);
-        glAttachShader(remap_program, remap_frag_shader);
-        glLinkProgram(remap_program);
-        check_program(remap_program);
-
         glDeleteShader(simple_vert_shader);
         glDeleteShader(simple_frag_shader);
         glDeleteShader(remap_frag_shader);
 
-        // uniform locations
-        image_tex_loc = glGetUniformLocation(remap_program, "image_tex");
-        remap_tex_loc = glGetUniformLocation(remap_program, "remap_tex");
-
         // create buffers
         glGenBuffers(1, &vbo);
         glGenBuffers(1, &ebo);
@@ -209,26 +185,14 @@ struct simple_render::impl {
 
     void render_texture(GLuint tex, const simple_rect &rect, bool flip_y) {
         // bind buffers
-//        bool is_remap = (remap_tex != 0);
-        constexpr bool is_remap = false;
-        glUseProgram(is_remap ? remap_program : simple_program);
+        glUseProgram(simple_program);
         glBindVertexArray(vao);
         glBindBuffer(GL_ARRAY_BUFFER, vbo);
         glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
 
         // bind textures
-        if (is_remap) {
-//            assert(remap_tex != 0);
-//            glUniform1i(image_tex_loc, 0);
-//            glUniform1i(remap_tex_loc, 1);
-//            glActiveTexture(GL_TEXTURE0 + 0);
-//            glBindTexture(GL_TEXTURE_2D, tex);
-//            glActiveTexture(GL_TEXTURE0 + 1);
-//            glBindTexture(GL_TEXTURE_2D, remap_tex);
-        } else {
-            glActiveTexture(GL_TEXTURE0 + 0);
-            glBindTexture(GL_TEXTURE_2D, tex);
-        }
+        glActiveTexture(GL_TEXTURE0 + 0);
+        glBindTexture(GL_TEXTURE_2D, tex);
 
         // fill vertex buffer
         GLfloat tex_top = flip_y ? 0 : 1;

src/utility.hpp

@@ -3,8 +3,22 @@
 
 #include <spdlog/spdlog.h>
 
+// https://en.cppreference.com/w/cpp/utility/unreachable
+[[noreturn]] inline void unreachable() {
+    // Uses compiler specific extensions if possible.
+    // Even if no extension is used, undefined behavior is still raised by
+    // an empty function body and the noreturn attribute.
+#ifdef __GNUC__ // GCC, Clang, ICC
+    __builtin_unreachable();
+// #elifdef _MSC_VER // MSVC
+#else
+    __assume(false);
+#endif
+}
+
 #define RET_ERROR \
-    assert(false)
+    assert(false);\
+    unreachable()
 
 #define RET_ERROR_B \
     assert(false); \
@@ -22,19 +36,6 @@ inline bool check_function_call(bool function_ret, unsigned int line_number,
     check_function_call( \
         function_call, __LINE__, __FILE__, #function_call)
 
-// https://en.cppreference.com/w/cpp/utility/unreachable
-[[noreturn]] inline void unreachable() {
-    // Uses compiler specific extensions if possible.
-    // Even if no extension is used, undefined behavior is still raised by
-    // an empty function body and the noreturn attribute.
-#ifdef __GNUC__ // GCC, Clang, ICC
-    __builtin_unreachable();
-// #elifdef _MSC_VER // MSVC
-#else
-    __assume(false);
-#endif
-}
-
 struct log_timer {
 
     void reset() {

src/variable_defs.h

@@ -10,6 +10,7 @@ constexpr auto OUTPUT_FRAME = 4;
 constexpr auto ENCODER_SHOULD_STOP = 5;
 
 constexpr auto CUDA_CONTEXT = 6;
+constexpr auto CUDA_STREAM_OUTPUT = 16;
 
 constexpr auto REQUEST_IDR = 7;
 constexpr auto SENDER_CONNECTED = 8;

src/video_encoder.h

@@ -1,44 +0,0 @@
-#ifndef REMOTEAR3_VIDEO_ENCODER_H
-#define REMOTEAR3_VIDEO_ENCODER_H
-
-#include <opencv2/core/cuda.hpp>
-
-#include <memory>
-
-struct encoder_config {
-    cv::Size frame_size;
-    int frame_rate;
-    float bitrate_mbps;
-    bool save_file;
-    bool save_length;
-};
-
-struct video_nal {
-    uint8_t *ptr = nullptr;
-    size_t length = 0;
-    bool idr = false;
-
-    void create(void *ptr, size_t length, bool idr);
-
-    ~video_nal();
-};
-
-class video_encoder {
-public:
-
-    ~video_encoder();
-
-    void encode(const cv::cuda::GpuMat &img, video_nal *out, bool force_idr = false);
-
-    // only frame_rate and bitrate can be changed
-    void change_config(const encoder_config &conf);
-
-    static video_encoder *create(const encoder_config &conf);
-
-private:
-    struct impl;
-    std::unique_ptr<impl> pimpl;
-};
-
-
-#endif //REMOTEAR3_VIDEO_ENCODER_H