Merged Tiny Player.

CMakeLists.txt

@@ -9,9 +9,11 @@ add_executable(${PROJECT_NAME} src/main.cpp
         src/impl/apps/app_selector/app_selector.cpp
         src/impl/apps/depth_guide/depth_guide.cpp
         src/impl/apps/remote_ar/remote_ar.cpp
+        src/impl/apps/tiny_player/tiny_player.cpp
         src/core/impl/event_timer.cpp
         src/core/impl/memory_pool.cpp
         src/core/impl/object_manager.cpp
+        src/module/impl/image_player.cpp
         src/module/impl/image_viewer.cpp
         src/module/impl/image_streamer.cpp
         src/network_v3/sender_tcp.cpp

data/config_tiny_player.yaml

@@ -0,0 +1 @@
+app_name: tiny_player

src/codec/decoder_nvdec.cpp

@@ -14,7 +14,7 @@ struct decoder_nvdec::impl {
     uint8_t decode_surface;
     cv::Size frame_size;
 
-    impl(create_config _conf) {
+    explicit impl(create_config _conf) {
         conf = _conf;
 
         // query decoder capability

src/core/image_utility.hpp

@@ -100,9 +100,9 @@ struct image_info_type {
     cv::Size size = {};
     size_t pitch = 0;
 
-    // start pointer of specific row
-    void *start_ptr(int row = 0) const {
-        return ptr.get() + row * pitch;
+    // start pointer of specific pixel
+    void *start_ptr(int row = 0, int col = 0) const {
+        return ptr.get() + row * pitch + col * sizeof(T);
     }
 
     size_t size_in_bytes() const { return sizeof(T) * size.area(); }
@@ -111,6 +111,29 @@ struct image_info_type {
 
     bool is_continuous() const { return sizeof(T) * size.width == pitch; }
 
+    this_type sub_image(int row = 0, int col = 0,
+                        int width = -1, int height = -1) const {
+        if (width == -1) {
+            width = size.width - col;
+        }
+        if (height == -1) {
+            height = size.height - row;
+        }
+        auto ret_size = cv::Size(width, height);
+        auto ret_ptr = (row == 0 && col == 0) ? ptr :
+                       std::shared_ptr<T>((T *) start_ptr(row, col), [p = ptr](void *) {});
+        return {ret_ptr, loc, ret_size, pitch};
+    }
+
+    template<typename U>
+    image_info_type<U> cast() const {
+        auto ret_width = size.width * sizeof(T) / sizeof(U);
+        assert(size.width * sizeof(T) == ret_width * sizeof(U));
+        auto ret_size = cv::Size(ret_width, size.height);
+        auto ret_ptr = std::reinterpret_pointer_cast<U>(ptr);
+        return {ret_ptr, loc, ret_size, pitch};
+    }
+
     this_type flatten(smart_cuda_stream *stream) const {
         if (is_continuous()) return *this;
         assert(loc == MEM_CUDA); // image in host is always continuous
@@ -305,7 +328,13 @@ auto create_image(image_info_type<T> info) {
     return std::make_shared<smart_image<T>>(info);
 }
 
+using image_info_u8c1 = image_info_type<uchar1>;
+using image_info_u8c2 = image_info_type<uchar2>;
+using image_info_u8c3 = image_info_type<uchar3>;
+using image_info_u8c4 = image_info_type<uchar4>;
+
 using image_u8c1 = std::shared_ptr<smart_image<uchar1>>;
+using image_u8c2 = std::shared_ptr<smart_image<uchar2>>;
 using image_u8c3 = std::shared_ptr<smart_image<uchar3>>;
 using image_u8c4 = std::shared_ptr<smart_image<uchar4>>;
 using image_u16c1 = std::shared_ptr<smart_image<ushort1>>;

src/impl/apps/app_selector/app_selector.cpp

@@ -1,6 +1,7 @@
 #include "app_selector.h"
 #include "impl/apps/depth_guide/depth_guide.h"
 #include "impl/apps/remote_ar/remote_ar.h"
+#include "impl/apps/tiny_player/tiny_player.h"
 
 #include <GLFW/glfw3.h>
 
@@ -49,6 +50,8 @@ void app_selector::load_app(const std::string &conf_path) {
         app = std::make_unique<app_depth_guide>(create_conf);
     } else if (app_name == "remote_ar") {
         app = std::make_unique<app_remote_ar>(create_conf);
+    } else if (app_name == "tiny_player") {
+        app = std::make_unique<app_tiny_player>(create_conf);
     }
 
     // replace application

src/impl/apps/depth_guide/depth_guide.cpp

@@ -2,7 +2,7 @@
 #include "core/image_utility.hpp"
 #include "core/imgui_utility.hpp"
 
-app_depth_guide::app_depth_guide(create_config _conf) {
+app_depth_guide::app_depth_guide(const create_config &_conf) {
     conf = _conf;
 
     // initialize object manager

src/impl/apps/depth_guide/depth_guide.h

@@ -14,7 +14,7 @@
 class app_depth_guide : public app_base {
 public:
 
-    explicit app_depth_guide(create_config conf);
+    explicit app_depth_guide(const create_config &conf);
 
     ~app_depth_guide() override = default;
 

src/impl/apps/tiny_player/tiny_player.cpp

@@ -0,0 +1,32 @@
+#include "tiny_player.h"
+
+app_tiny_player::app_tiny_player(const create_config &_conf) {
+    conf = _conf;
+
+    // initialize object manager
+    OBJ_SAVE(img_bg, image_u8c1());
+
+    // initialize modules
+    auto in_player_conf = image_player::create_config{
+            .img_name = img_bg, .ctx = conf.asio_ctx,
+            .stream = default_cuda_stream
+    };
+    in_player = std::make_unique<image_player>(in_player_conf);
+
+    auto bg_viewer_conf = image_viewer::create_config{
+            .mode = VIEW_COLOR_ONLY, .flip_y = true,
+            .stream = default_cuda_stream,
+    };
+    auto &bg_extra_conf = bg_viewer_conf.extra.color;
+    bg_extra_conf.fmt = COLOR_NV12;
+    bg_extra_conf.name = img_bg;
+    bg_viewer = std::make_unique<image_viewer>(bg_viewer_conf);
+}
+
+void app_tiny_player::show_ui() {
+    in_player->show();
+}
+
+void app_tiny_player::render_background() {
+    bg_viewer->render();
+}

src/impl/apps/tiny_player/tiny_player.h

@@ -0,0 +1,35 @@
+#ifndef DEPTHGUIDE_TINY_PLAYER_H
+#define DEPTHGUIDE_TINY_PLAYER_H
+
+#include "core/object_manager.h"
+#include "module/image_player.h"
+#include "module/image_viewer.h"
+#include "impl/app_base.h"
+
+class app_tiny_player : public app_base {
+public:
+    explicit app_tiny_player(const create_config &conf);
+
+    ~app_tiny_player() override = default;
+
+    const char *window_name() override { return "TinyPlayer V4.-1"; }
+
+    void show_ui() override;
+
+    void render_background() override;
+
+private:
+
+    enum obj_names : object_manager::name_type {
+        img_bg,
+    };
+
+    create_config conf;
+
+    std::unique_ptr<image_viewer> bg_viewer; // background viewer
+    std::unique_ptr<image_player> in_player; // input player
+
+};
+
+
+#endif //DEPTHGUIDE_TINY_PLAYER_H

src/module/image_player.h

@@ -0,0 +1,36 @@
+#ifndef DEPTHGUIDE_IMAGE_PLAYER_H
+#define DEPTHGUIDE_IMAGE_PLAYER_H
+
+#include "core/cuda_helper.hpp"
+#include "core/object_manager.h"
+
+#include <boost/asio/io_context.hpp>
+
+#include <memory>
+
+// does the opposite thing as image_streamer
+class image_player {
+public:
+
+    using io_context = boost::asio::io_context;
+
+    struct create_config {
+        obj_name_type img_name = invalid_obj_name;
+        io_context *ctx = nullptr;
+
+        // for decoder
+        smart_cuda_stream *stream = nullptr;
+    };
+
+    explicit image_player(create_config conf);
+
+    ~image_player();
+
+    void show();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#endif //DEPTHGUIDE_IMAGE_PLAYER_H

src/module/image_streamer.h

@@ -9,10 +9,12 @@
 class image_streamer {
 public:
 
+    using io_context = boost::asio::io_context;
+
     struct create_config {
         // image must be valid before start
         obj_name_type img_name = invalid_obj_name;
-        boost::asio::io_context *asio_ctx = nullptr;
+        io_context *asio_ctx = nullptr;
         // TODO: add frame rate
 
         // for encoder
@@ -28,8 +30,8 @@ public:
 
     // TODO: add change frame rate
 
-    using size_change_sig_type = boost::signals2::signal<void(cv::Size)>;
-    size_change_sig_type sig_size_changed;
+    using sig_req_size_type = boost::signals2::signal<void(cv::Size)>;
+    sig_req_size_type sig_req_size;
 
 private:
     struct impl;

src/module/image_viewer.h

@@ -7,6 +7,8 @@
 #include <memory>
 
 enum image_viewer_mode {
+    VIEW_COLOR_ONLY,
+    VIEW_DEPTH_ONLY,
     VIEW_COLOR_DEPTH,
     VIEW_STEREO
 };
@@ -20,9 +22,18 @@ public:
         smart_cuda_stream *stream = nullptr;
         union {
             struct {
+                color_format fmt;
+                obj_name_type name;
+            } color;
+            struct {
+                obj_name_type name;
+            } depth;
+            struct {
+                color_format c_fmt;
                 obj_name_type c_name, d_name;
             } color_depth;
             struct {
+                color_format c_fmt;
                 obj_name_type left_name, right_name;
             } stereo;
         } extra = {};

src/module/impl/image_player.cpp

@@ -0,0 +1,169 @@
+#include "image_player_impl.h"
+#include "core/imgui_utility.hpp"
+
+void image_player::impl::create_decoder() {
+    switch (chose_decoder_type) {
+        case DECODER_NVDEC: {
+            auto dec_conf = decoder_nvdec::create_config{
+                    .img_name = conf.img_name, .stream = conf.stream
+            };
+            assert(dec_nvdec == nullptr);
+            dec_nvdec = std::make_unique<decoder_nvdec>(dec_conf);
+            assert(dec_nvdec != nullptr);
+            return;
+        }
+        default: {
+            RET_ERROR;
+        }
+    }
+}
+
+void image_player::impl::create_receiver() {
+    if (enable_aux_thread) {
+        assert(aux_ctx != nullptr);
+        recv_ctx = aux_ctx.get();
+    } else {
+        recv_ctx = ctx;
+    }
+
+    assert(receiver == nullptr);
+    switch (chose_receiver_type) {
+        case RECEIVER_UDP_FEC: {
+            auto recv_conf = receiver_udp_fec::create_config();
+            recv_conf.server_addr = recv_server_addr;
+            recv_conf.server_port = recv_server_port;
+            recv_conf.ctx = recv_ctx;
+            recv_conf.cb_func = [this](auto frame) { frame_callback(frame); };
+            recv_conf.enable_log = recv_enable_log;
+            receiver = receiver_udp_fec::create(recv_conf);
+            break;
+        }
+        default: {
+            RET_ERROR;
+        }
+    }
+    assert(receiver != nullptr);
+}
+
+void image_player::impl::frame_callback(const frame_info &frame) {
+    if (enable_aux_thread) {
+        assert(frame_queue != nullptr);
+        frame_queue->push(frame, frame.idr);
+    } else {
+        decode_image(frame);
+    }
+}
+
+void image_player::impl::decode_image(const frame_info &frame) {
+    switch (chose_decoder_type) {
+        case DECODER_NVDEC: {
+            dec_nvdec->decode(frame);
+            return;
+        }
+        default: {
+            RET_ERROR;
+        }
+    }
+}
+
+void image_player::impl::start() {
+    if (enable_aux_thread) {
+        assert(aux_ctx == nullptr);
+        aux_ctx = std::make_unique<io_context>();
+
+        auto queue_conf = frame_queue_type::create_config();
+        queue_conf.ctx = ctx;
+        queue_conf.tid = std::this_thread::get_id();
+        queue_conf.cb_func = [this](auto frame) { decode_image(frame); };
+        assert(frame_queue == nullptr);
+        frame_queue = frame_queue_type::create(queue_conf);
+    }
+
+    create_decoder();
+    create_receiver();
+
+    if (enable_aux_thread) {
+        aux_thread = std::make_unique<std::thread>([this] {
+            auto blocker = boost::asio::make_work_guard(*aux_ctx);
+            aux_ctx->run();
+        });
+    }
+
+    assert(!is_running);
+    is_running = true;
+}
+
+void image_player::impl::stop() {
+    if (enable_aux_thread) {
+        aux_ctx->stop();
+        aux_thread->join();
+        aux_thread = nullptr;
+    }
+
+    dec_nvdec = nullptr;
+    receiver = nullptr;
+    aux_ctx = nullptr;
+
+    assert(is_running);
+    is_running = false;
+}
+
+void image_player::impl::show_config() {
+    auto guard = imgui_disable_guard(is_running);
+
+    ImGui::SeparatorText("Sender Configs");
+    ImGui::InputText("Server IP", recv_server_addr, addr_max_length);
+    ImGui::InputScalar("Server Port", ImGuiDataType_U16, &recv_server_port);
+    // chose receive method
+    if (ImGui::RadioButton("TCP", chose_receiver_type == RECEIVER_TCP)) {
+        chose_receiver_type = RECEIVER_TCP;
+    }
+    if (chose_decoder_type != DECODER_NVDEC) {
+        ImGui::SameLine();
+        if (ImGui::RadioButton("UDP", chose_receiver_type == RECEIVER_UDP)) {
+            chose_receiver_type = RECEIVER_UDP;
+        }
+    }
+    ImGui::SameLine();
+    if (ImGui::RadioButton("UDP (FEC)", chose_receiver_type == RECEIVER_UDP_FEC)) {
+        chose_receiver_type = RECEIVER_UDP_FEC;
+    }
+    ImGui::Checkbox("Enable Log", &recv_enable_log);
+
+    ImGui::SeparatorText("Decoder Configs");
+    // chose decoding method
+    if (ImGui::RadioButton("NvDec", chose_decoder_type == DECODER_NVDEC)) {
+        chose_decoder_type = DECODER_NVDEC;
+        if (chose_receiver_type == RECEIVER_UDP) {
+            chose_receiver_type = RECEIVER_TCP;
+        }
+    }
+    ImGui::SameLine();
+    if (ImGui::RadioButton("nvJPEG", chose_decoder_type == DECODER_JPEG)) {
+        chose_decoder_type = DECODER_JPEG;
+    }
+}
+
+void image_player::impl::show() {
+    ImGui::SeparatorText("Actions");
+    if (!is_running) {
+        if (ImGui::Button("Start")) {
+            post(*ctx, [this] { start(); });
+        }
+    } else {
+        if (ImGui::Button("Close")) {
+            post(*ctx, [this] { stop(); });
+        }
+    }
+    show_config();
+}
+
+image_player::image_player(create_config conf)
+        : pimpl(std::make_unique<impl>(conf)) {
+}
+
+image_player::~image_player() = default;
+
+void image_player::show() {
+    pimpl->show();
+}

src/module/impl/image_player_impl.h

@@ -0,0 +1,69 @@
+#ifndef DEPTHGUIDE_IMAGE_PLAYER_IMPL_H
+#define DEPTHGUIDE_IMAGE_PLAYER_IMPL_H
+
+#include "module/image_player.h"
+#include "codec/decoder_nvdec.h"
+#include "core/async_queue.hpp"
+#include "network_v3/receiver_udp_fec.h"
+
+#include <boost/asio/post.hpp>
+
+using boost::asio::post;
+
+struct image_player::impl {
+
+    create_config conf;
+    io_context *ctx = nullptr;
+    bool is_running = false;
+
+    decoder_type chose_decoder_type = DECODER_NVDEC;
+    std::unique_ptr<decoder_nvdec> dec_nvdec;
+
+    receiver_type chose_receiver_type = RECEIVER_UDP_FEC;
+    std::shared_ptr<receiver_base> receiver;
+
+    // for all receiver
+    static constexpr auto addr_max_length = 256;
+    char recv_server_addr[addr_max_length] = "192.168.1.201";
+    uint16_t recv_server_port = 5279;
+    io_context *recv_ctx = nullptr;
+    bool recv_enable_log = false;
+
+    bool enable_aux_thread = true; // run sender in another thread
+    std::unique_ptr<io_context> aux_ctx;
+
+    // for auxiliary thread
+    using frame_queue_type = async_queue<frame_info>;
+    std::shared_ptr<frame_queue_type> frame_queue;
+    std::unique_ptr<std::thread> aux_thread;
+
+    explicit impl(create_config _conf) {
+        conf = _conf;
+        ctx = conf.ctx;
+    }
+
+    ~impl() {
+        if (is_running) {
+            stop();
+        }
+    }
+
+    void show_config();
+
+    void show();
+
+    void start();
+
+    void stop();
+
+    void create_decoder();
+
+    void create_receiver();
+
+    void frame_callback(const frame_info &frame);
+
+    void decode_image(const frame_info &frame);
+
+};
+
+#endif //DEPTHGUIDE_IMAGE_PLAYER_IMPL_H

src/module/impl/image_streamer.cpp

@@ -46,8 +46,8 @@ void image_streamer::impl::create_sender() {
             sender_conf.ctx = sender_ctx;
             sender_conf.enable_log = sender_enable_log;
             auto sender_fec = sender_udp_fec::create(sender_conf);
-            sender_fec->sig_size_changed.connect([this](auto size) { // forward signal
-                q_this->sig_size_changed(size);
+            sender_fec->sig_req_size.connect([this](auto size) { // forward signal
+                q_this->sig_req_size(size);
             });
             sender_fec->start();
             sender = sender_fec;
@@ -216,7 +216,7 @@ void image_streamer::impl::show_config() {
         chose_sender_type = SENDER_UDP_FEC;
     }
     // configs
-    ImGui::InputInt("Listen Port", &sender_listen_port);
+    ImGui::InputScalar("Listen Port", ImGuiDataType_U16, &sender_listen_port);
     if (chose_sender_type == SENDER_UDP_FEC) {
         ImGui::DragFloat("Parity Rate", &sender_parity_rate, 0.01, 0, 2, "%.02f");
     }

src/module/impl/image_streamer_impl.h

@@ -7,10 +7,8 @@
 #include "network_v3/sender_tcp.h"
 #include "network_v3/sender_udp_fec.h"
 
-#include <boost/asio/io_context.hpp>
 #include <boost/asio/post.hpp>
 
-using boost::asio::io_context;
 using boost::asio::post;
 
 struct image_streamer::impl {
@@ -38,7 +36,7 @@ struct image_streamer::impl {
     std::shared_ptr<sender_base> sender;
 
     // for all sender
-    int sender_listen_port = 5279; // make ImGui happy
+    uint16_t sender_listen_port = 5279; // make ImGui happy
     io_context *sender_ctx = nullptr;
     bool sender_enable_log = false;
 

src/module/impl/image_viewer.cpp

@@ -2,14 +2,7 @@
 #include "core/imgui_utility.hpp"
 #include "render/render_texture.h"
 
-void image_viewer::impl::show_color_depth() {
-    ImGui::RadioButton("Color", &chose_index, 0);
-    ImGui::SameLine();
-    ImGui::RadioButton("Depth", &chose_index, 1);
-    ImGui::SameLine();
-    ImGui::RadioButton("Both", &chose_index, 2);
-
-    ImGui::PushItemWidth(150);
+void image_viewer::impl::show_depth_only() {
     {
         auto guard = imgui_disable_guard(!depth_conf.manual_depth_range);
         static constexpr float dep_hard_min = 0.15; // TODO: config value
@@ -19,11 +12,23 @@ void image_viewer::impl::show_color_depth() {
     }
     ImGui::SameLine();
     ImGui::Checkbox("##manual_dep_range", &depth_conf.manual_depth_range);
+}
+
+void image_viewer::impl::show_color_depth() {
+    ImGui::RadioButton("Color", &chose_index, 0);
+    ImGui::SameLine();
+    ImGui::RadioButton("Depth", &chose_index, 1);
+    ImGui::SameLine();
+    ImGui::RadioButton("Both", &chose_index, 2);
 
+    if (chose_index == 1 || chose_index == 2) { // depth or both
+        show_depth_only();
+    }
+
+    ImGui::PushItemWidth(150);
     if (chose_index == 2) { // both
         ImGui::SliderFloat("Depth Alpha", &depth_overlay_alpha, 0.f, 1.f, "%.2f");
     }
-    ImGui::PopItemWidth();
 }
 
 void image_viewer::impl::show_stereo() {
@@ -33,7 +38,15 @@ void image_viewer::impl::show_stereo() {
 }
 
 void image_viewer::impl::show() {
+    ImGui::PushItemWidth(150);
     switch (conf.mode) {
+        case VIEW_COLOR_ONLY: {
+            break;
+        }
+        case VIEW_DEPTH_ONLY: {
+            show_depth_only();
+            break;
+        }
         case VIEW_COLOR_DEPTH: {
             show_color_depth();
             break;
@@ -46,6 +59,7 @@ void image_viewer::impl::show() {
             RET_ERROR;
         }
     }
+    ImGui::PopItemWidth();
 }
 
 void image_viewer::impl::render_color_obj(obj_name_type name) {
@@ -59,6 +73,9 @@ void image_viewer::impl::render_depth_obj(obj_name_type name, float alpha) {
 
 void image_viewer::impl::render_color_depth() {
     auto info = conf.extra.color_depth;
+    if (chose_index == 0 || chose_index == 2) { // color or both
+        color_conf.fmt = info.c_fmt;
+    }
     switch (chose_index) {
         case 0: { // color
             render_color_obj(info.c_name);
@@ -81,6 +98,7 @@ void image_viewer::impl::render_color_depth() {
 
 void image_viewer::impl::render_stereo() {
     auto info = conf.extra.stereo;
+    color_conf.fmt = info.c_fmt;
     switch (chose_index) {
         case 0: { // left
             render_color_obj(info.left_name);
@@ -98,6 +116,15 @@ void image_viewer::impl::render_stereo() {
 
 void image_viewer::impl::render() {
     switch (conf.mode) {
+        case VIEW_COLOR_ONLY: {
+            color_conf.fmt = conf.extra.color.fmt;
+            render_color_obj(conf.extra.color.name);
+            break;
+        }
+        case VIEW_DEPTH_ONLY: {
+            render_depth_obj(conf.extra.depth.name);
+            break;
+        }
         case VIEW_COLOR_DEPTH: {
             render_color_depth();
             break;

src/module/impl/image_viewer_impl.h

@@ -25,6 +25,8 @@ struct image_viewer::impl {
 
     float depth_overlay_alpha = 0.5;
 
+    void show_depth_only();
+
     void show_color_depth();
 
     void show_stereo();

src/network_v3/receiver_udp_fec.cpp

@@ -331,7 +331,7 @@ struct receiver_udp_fec::impl {
         ret->socket->connect(ret->server_ep);
         ret->socket->set_option(udp::socket::receive_buffer_size{udp_buffer_size});
         ret->async_handle_package();
-        ret->request_idr_frame(0);
+        ret->request_idr_frame(0); // TODO: post to event queue
 
         // initialize reed solomon
         fec_init();

src/network_v3/sender_udp_fec.cpp

@@ -283,7 +283,7 @@ struct sender_udp_fec::impl {
                     uint16_t width, height;
                     ptr = read_binary_number(ptr, &width);
                     ptr = read_binary_number(ptr, &height);
-                    q_this->sig_size_changed(cv::Size(width, height));
+                    q_this->sig_req_size(cv::Size(width, height));
                     SPDLOG_INFO("Output size changed to {}x{}", width, height);
                 }
 

src/network_v3/sender_udp_fec.h

@@ -39,8 +39,8 @@ public:
 
     ~sender_udp_fec() override;
 
-    using size_change_sig_type = boost::signals2::signal<void(cv::Size)>;
-    size_change_sig_type sig_size_changed;
+    using sig_req_size_type = boost::signals2::signal<void(cv::Size)>;
+    sig_req_size_type sig_req_size;
 
     void start();
 

src/render/impl/render_texture.cpp

@@ -13,7 +13,8 @@ namespace render_texture_impl {
     bool init_ok = false;
 
     using pg_type = std::unique_ptr<smart_program>;
-    pg_type pg_color_only;
+    pg_type pg_rgb; // render rgb texture
+    pg_type pg_nv12; // render nv12 textures
 //    pg_type pg_depth_only;
 //    pg_type pg_color_depth;
 //    pg_type pg_depth_alpha;
@@ -71,28 +72,68 @@ namespace render_texture_impl {
         glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, nullptr);
     }
 
-    // render color only
-    void ren_c_only(const tex_render_info &info) {
-        if (pg_color_only == nullptr) {
-            pg_color_only = std::unique_ptr<smart_program>(
+    // render rgb texture
+    void ren_rgb_only(const tex_render_info &info) {
+        if (pg_rgb == nullptr) {
+            pg_rgb = std::unique_ptr<smart_program>(
                     smart_program::create("tex_rgb",
                                           {{GL_VERTEX_SHADER,   "tex.vert"},
                                            {GL_FRAGMENT_SHADER, "tex_rgb.frag"}}));
         }
-        assert(pg_color_only != nullptr);
-        pg_color_only->use();
+        assert(pg_rgb != nullptr);
+        pg_rgb->use();
 
-        pg_color_only->set_uniform_f("alpha", info.color.alpha);
+        pg_rgb->set_uniform_f("alpha", info.color.alpha);
 
         glActiveTexture(GL_TEXTURE0 + 0);
         glBindTexture(GL_TEXTURE_2D, info.color.id);
-        pg_color_only->set_uniform_i("c_tex", 0);
+        pg_rgb->set_uniform_i("c_tex", 0);
 
         glDisable(GL_DEPTH_TEST);
         config_buffers(info);
         draw();
     }
 
+    void ren_nv12_only(const tex_render_info &info) {
+        if (pg_nv12 == nullptr) {
+            pg_nv12 = std::unique_ptr<smart_program>(
+                    smart_program::create("tex_nv12",
+                                          {{GL_VERTEX_SHADER,   "tex.vert"},
+                                           {GL_FRAGMENT_SHADER, "tex_nv12.frag"}}));
+        }
+        assert(pg_nv12 != nullptr);
+        pg_nv12->use();
+
+        pg_nv12->set_uniform_f("alpha", info.color.alpha);
+
+        glActiveTexture(GL_TEXTURE0 + 0);
+        glBindTexture(GL_TEXTURE_2D, info.color.id);
+        pg_nv12->set_uniform_i("luma_tex", 0);
+        glActiveTexture(GL_TEXTURE0 + 1);
+        glBindTexture(GL_TEXTURE_2D, info.color.id_ext[0]);
+        pg_nv12->set_uniform_i("chroma_tex", 1);
+
+        glDisable(GL_DEPTH_TEST);
+        config_buffers(info);
+        draw();
+    }
+
+    void ren_c_only(const tex_render_info &info) {
+        switch (info.color.fmt) {
+            case COLOR_RGB: {
+                ren_rgb_only(info);
+                break;
+            }
+            case COLOR_NV12: {
+                ren_nv12_only(info);
+                break;
+            }
+            default: {
+                RET_ERROR;
+            }
+        }
+    }
+
 }
 
 using namespace render_texture_impl;

src/render/impl/render_texturer_impl.h

@@ -5,6 +5,10 @@
 
 namespace render_texture_impl {
 
+    void ren_rgb_only(const tex_render_info &info);
+
+    void ren_nv12_only(const tex_render_info &info);
+
     void ren_c_only(const tex_render_info &info);
 
 }

src/render/impl/render_tools.cpp

@@ -14,17 +14,81 @@ void color_image_render::render_tex(cv::Size img_size, config_type conf) {
     info.mode = TEX_COLOR_ONLY;
     info.flip_y = conf.flip_y;
     info.range = calc_render_range(img_size);
+    info.color.fmt = conf.fmt;
     info.color.id = img_tex.id;
+    if (conf.fmt == COLOR_NV12) { // chroma tex
+        info.color.id_ext[0] = ext_tex[0].id;
+    }
     info.color.alpha = conf.alpha;
     render_texture(info);
 }
 
-void color_image_render::render(obj_name_type name, config_type conf) {
+template<typename T>
+void color_image_render::render_rgb(const image_info_type<T> &info, config_type conf) {
+    assert(conf.fmt == COLOR_RGB);
+    img_tex.upload(info, conf.stream);
+    render_tex(info.size, conf);
+}
+
+// @formatter:off
+template void color_image_render::render_rgb<uchar1>(const image_info_u8c1 &, config_type);
+template void color_image_render::render_rgb<uchar2>(const image_info_u8c2 &, config_type);
+template void color_image_render::render_rgb<uchar3>(const image_info_u8c3 &, config_type);
+template void color_image_render::render_rgb<uchar4>(const image_info_u8c4 &, config_type);
+// @formatter:on
+
+void color_image_render::render_rgb(obj_name_type name, config_type conf) {
+    assert(conf.fmt == COLOR_RGB);
     auto img_type = OBJ_TYPE(name);
-    assert(img_type == typeid(image_u8c3));
-    auto img = OBJ_QUERY(image_u8c3, name);
+
+    auto impl_func = [&](auto V) {
+        using T = std::remove_cvref_t<decltype(V)>;
+        if (img_type == typeid(T)) {
+            auto img = OBJ_QUERY(T, name);
+            if (img == nullptr) return;
+            render_rgb(img->as_info(), conf);
+        }
+    };
+    impl_func(image_u8c1());
+    impl_func(image_u8c2());
+    impl_func(image_u8c3());
+    impl_func(image_u8c4());
+}
+
+void color_image_render::render_nv12(const image_info_u8c1 &img_info, config_type conf) {
+    assert(conf.fmt == COLOR_NV12);
+    auto img_size = nv12_size_to_img(img_info.size);
+
+    auto luma_img = img_info.sub_image(0, 0, -1, img_size.height);
+    img_tex.upload(luma_img, conf.stream);
+    auto chroma_img = img_info.sub_image(img_size.height).cast<uchar2>();
+    ext_tex[0].upload(chroma_img, conf.stream);
+
+    render_tex(img_size, conf);
+}
+
+void color_image_render::render_nv12(obj_name_type name, config_type conf) {
+    assert(conf.fmt == COLOR_NV12);
+    assert(OBJ_TYPE(name) == typeid(image_u8c1));
+    auto img = OBJ_QUERY(image_u8c1, name);
     if (img == nullptr) return;
-    render(img->as_info(), conf);
+    render_nv12(img->as_info(), conf);
+}
+
+void color_image_render::render(obj_name_type name, config_type conf) {
+    switch (conf.fmt) {
+        case COLOR_RGB: {
+            render_rgb(name, conf);
+            break;
+        }
+        case COLOR_NV12: {
+            render_nv12(name, conf);
+            break;
+        }
+        default: {
+            RET_ERROR;
+        }
+    }
 }
 
 void depth_image_render::render(obj_name_type name, config_type conf) {
@@ -53,8 +117,9 @@ void depth_image_render::render(obj_name_type name, config_type conf) {
     cv::applyColorMap(img_u8.as_mat(), img_color.as_mat(), cv::COLORMAP_TURBO);
 
     auto c_conf = color_image_render::config_type();
+    c_conf.fmt = COLOR_RGB;
     c_conf.flip_y = conf.flip_y;
     c_conf.alpha = conf.alpha;
     c_conf.stream = conf.stream;
-    color_render.render(img_color, c_conf);
+    color_render.render_rgb(img_color, c_conf);
 }

src/render/impl/shader/tex_nv12.frag

@@ -0,0 +1,25 @@
+#version 460
+
+uniform float alpha;
+
+uniform sampler2D luma_tex;
+uniform sampler2D chroma_tex;
+
+in vec2 frag_uv;
+
+// @formatter:off
+const mat3 cvt_mat = mat3(
+    1,       1,        1,
+    0,       -0.3455,  1.7790,
+    1.4075,  -0.7169,  0);
+// @formatter:on
+
+layout (location = 0) out vec4 frag_color;
+
+void main() {
+    vec3 yuv, rgb;
+    yuv.x = texture(luma_tex, frag_uv).x;
+    yuv.yz = texture(chroma_tex, frag_uv).xy - vec2(0.5, 0.5);
+    rgb = cvt_mat * yuv;
+    frag_color = vec4(rgb, alpha);
+}

src/render/render_texture.h

@@ -19,7 +19,9 @@ struct tex_render_info {
 
     // color texture info
     struct {
+        color_format fmt = COLOR_RGB;
         GLuint id = 0;
+        GLuint id_ext[2] = {};
         GLfloat alpha = 1.0;
     } color;
 

src/render/render_tools.h

@@ -10,21 +10,30 @@ class color_image_render {
 public:
 
     struct config_type {
+        color_format fmt = COLOR_RGB;
         bool flip_y = false;
         float alpha = 1.0;
         smart_cuda_stream *stream = nullptr;
     };
 
+    void render(obj_name_type name, config_type conf);
+
+    // T = uchar{1-4}
     template<typename T>
-    void render(const image_info_type<T> &img, config_type conf) {
-        img_tex.upload(img, conf.stream);
-        render_tex(img.size, conf);
-    }
+    void render_rgb(const image_info_type<T> &img, config_type conf);
 
-    void render(obj_name_type name, config_type conf);
+    // supports image_u8c{1-4}
+    void render_rgb(obj_name_type name, config_type conf);
+
+    // supports image_u8c1
+    void render_nv12(const image_info_u8c1 &img, config_type conf);
+
+    // supports image_u8c1
+    void render_nv12(obj_name_type name, config_type conf);
 
 private:
     smart_texture img_tex;
+    smart_texture ext_tex[2];
 
     void render_tex(cv::Size img_size, config_type conf);
 };

src/render/render_utility.h

@@ -24,6 +24,8 @@ void check_program(const char *name, GLuint id);
 template<typename T>
 constexpr inline GLenum get_tex_format() {
     // @formatter:off
+    if constexpr (std::is_same_v<T, uchar1>) { return GL_RED; }
+    if constexpr (std::is_same_v<T, uchar2>) { return GL_RG;  }
     if constexpr (std::is_same_v<T, uchar3>) { return GL_RGB; }
     // @formatter:on
     return 0;
@@ -32,6 +34,8 @@ constexpr inline GLenum get_tex_format() {
 template<typename T>
 constexpr inline GLenum get_tex_type() {
     // @formatter:off
+    if constexpr (std::is_same_v<T, uchar1>) { return GL_UNSIGNED_BYTE; }
+    if constexpr (std::is_same_v<T, uchar2>) { return GL_UNSIGNED_BYTE; }
     if constexpr (std::is_same_v<T, uchar3>) { return GL_UNSIGNED_BYTE; }
     // @formatter:on
     return 0;
@@ -40,11 +44,18 @@ constexpr inline GLenum get_tex_type() {
 template<typename T>
 constexpr inline GLenum get_tex_internal_format() {
     // @formatter:off
+    if constexpr (std::is_same_v<T, uchar1>) { return GL_R8;   }
+    if constexpr (std::is_same_v<T, uchar2>) { return GL_RG8;  }
     if constexpr (std::is_same_v<T, uchar3>) { return GL_RGB8; }
     // @formatter:on
     return 0;
 }
 
+enum color_format : uint8_t {
+    COLOR_RGB,
+    COLOR_NV12
+};
+
 struct simple_rect {
     GLfloat x, y;
     GLfloat width, height;