Re-implemented augment modules.

CMakeLists.txt

@@ -23,7 +23,9 @@ add_executable(${PROJECT_NAME} src/main.cpp
         src/core/impl/pc_utility.cpp
         src/module_v3/registration.cpp
         src/module/impl/augment_manager.cpp
+        src/module/impl/augment_manager_v2.cpp
         src/module/impl/camera_augment_helper.cpp
+        src/module/impl/camera_augment_helper_v2.cpp
         src/module/impl/depth_guide_controller.cpp
         src/module/impl/image_augment_helper.cpp
         src/module/impl/image_player.cpp
@@ -51,6 +53,9 @@ add_executable(${PROJECT_NAME} src/main.cpp
 
 target_include_directories(${PROJECT_NAME} PRIVATE src)
 
+#target_compile_options(${PROJECT_NAME} PRIVATE -g -pg)
+#target_link_options(${PROJECT_NAME} PRIVATE -g -pg)
+
 # image process sub-module
 add_subdirectory(src/image_process/cuda_impl)
 target_link_libraries(${PROJECT_NAME} ImageProcessCuda)

data/sophiar_config.json

@@ -278,7 +278,7 @@
             }
           },
           {
-            "rom_path": "/home/tpx/data/roms/Glass_3Ball_6.rom",
+            "rom_path": "/home/tpx/data/roms/Glass_4Ball_2.rom",
             "outputs": {
               "transform": "femur_ref_in_tracker"
             }

src/ai/impl/fast_sam.cpp

@@ -38,7 +38,8 @@ void fast_sam::impl::image_callback(obj_name_type name) {
 void fast_sam::impl::reply_callback(const zmq_client::reply_type &rep) {
     assert(!rep.img_list.empty());
     auto img_v2 = rep.img_list.front();
-    img_v2->set_meta(META_COLOR_FMT, COLOR_BW);
+    img_v2->set_meta(META_COLOR_FMT, COLOR_BW); // TODO: deprecated, remove this
+    img_v2->set_meta(META_IMAGE_DISPLAY_FMT, DISP_MASK);
 //    auto img = img_v2->v1<uchar1>();
     OBJ_SAVE(conf.mask_out, img_v2);
 }

src/core/image_utility.hpp

@@ -429,4 +429,6 @@ inline cv::Mat image_as_mat(obj_name_type name,
     return ret;
 }
 
+
+
 #endif //DEPTHGUIDE_IMAGE_UTILITY_HPP

src/core/image_utility_v2.h

@@ -18,14 +18,24 @@ enum pixel_format_enum {
 };
 
 enum meta_key_enum {
+    META_IMAGE_DISPLAY_FMT,
     META_COLOR_FMT, // color_format
 };
 
+enum display_fmt : uint8_t {
+    DISP_COLOR = 0,
+    DISP_MASK,
+    DISP_DEPTH,
+};
+
 enum color_format : uint8_t {
     COLOR_BW, // only 0 and 1
+    COLOR_GRAY,
     COLOR_RGB,
     COLOR_RGBA,
-    COLOR_NV12
+    COLOR_ARGB,
+    COLOR_NV12,
+    COLOR_ANY = 0xFF,
 };
 
 class generic_image;
@@ -141,4 +151,6 @@ inline image_ptr create_image(const std::shared_ptr<smart_image<T>> &img) {
     return generic_image::create(img);
 }
 
+image_ptr to_image(obj_name_type name);
+
 #endif //DEPTHGUIDE_IMAGE_UTILITY_V2_H

src/core/impl/image_utility_v2.cpp

@@ -416,4 +416,24 @@ void generic_image::host_modified(smart_cuda_stream *stream) {
 
 void generic_image::cuda_modified(smart_cuda_stream *stream) {
     pimpl->cuda_modified(stream);
+}
+
+image_ptr to_image(obj_name_type name) {
+    auto img_type = OBJ_TYPE(name);
+    if (OBJ_TYPE(name) == typeid(image_ptr)) {
+        return OBJ_QUERY(image_ptr, name);
+    }
+
+    // convert from v1
+    auto ret = image_ptr();
+    auto impl_func = [&](auto V) {
+        using T = std::remove_cvref_t<decltype(V)>;
+        if (img_type == typeid(T)) {
+            auto img = OBJ_QUERY(T, name);
+            if (img == nullptr) return;
+            ret = create_image(img);
+        }
+    };
+    FORALL_IMG_TYPE;
+    return ret;
 }

src/core/impl/memory_pool.cpp

@@ -119,7 +119,7 @@ cudaEvent_t memory_pool::impl::get_event(void *ptr) {
 void memory_pool::impl::deallocate(void *ptr) {
     auto guard = std::lock_guard(mu);
     auto iter = malloc_pool.find(ptr);
-    if (iter == malloc_pool.end()) {
+    if (iter == malloc_pool.end()) [[unlikely]] {
         SPDLOG_WARN("Deallocate unknown pointer: {}.", fmt::ptr(ptr));
         return;
     }

src/core/math_helper.hpp

@@ -19,7 +19,8 @@ inline glm::mat4 to_transform_mat(glm::vec3 t, glm::vec3 r) {
     return offset * glm::mat4_cast(rot);
 }
 
-inline glm::mat4 to_mat4(const Eigen::Isometry3d &m) {
+template<typename Scalar, int Mode>
+inline glm::mat4 to_mat4(const Eigen::Transform<Scalar, 3, Mode> &m) {
     auto ret = glm::mat4();
     auto &mat = m.matrix();
     for (auto j = 0; j < 4; ++j)

src/impl/apps/depth_guide/depth_guide.cpp

@@ -88,6 +88,24 @@ app_depth_guide::app_depth_guide(const create_config &_conf) {
             .name = pc_raw, .stream = default_cuda_stream,
     };
     scene_viewer = std::make_unique<pc_viewer>(pc_conf);
+
+    {
+        auto aug_conf = augment_manager_v2::create_config{
+                .stream = default_cuda_stream,
+        };
+        aug_conf.item_list.push_back(
+                {.type = augment_manager_v2::ITEM_PC,
+                        .disp_name = "PC Raw", .name = pc_raw});
+        aug_manager = std::make_unique<augment_manager_v2>(aug_conf);
+    }
+
+    {
+        auto aug_conf = camera_augment_helper_v2::create_config{
+                .camera = camera_augment_helper_v2::create_config::freedom_camera_config{},
+                .manager = aug_manager.get(),
+        };
+        aug_helper_left = std::make_unique<camera_augment_helper_v2>(aug_conf);
+    }
 }
 
 void app_depth_guide::show_ui() {
@@ -110,6 +128,19 @@ void app_depth_guide::show_ui() {
             out_streamer->show();
         }
 
+        if (ImGui::CollapsingHeader("Augment")) {
+            {
+                ImGui::SeparatorText("Scene");
+                auto id_guard = imgui_id_guard("augment_scene");
+                aug_manager->show();
+            }
+            {
+                ImGui::SeparatorText("Camera");
+                auto id_guard = imgui_id_guard("augment_camera");
+                aug_helper_left->show();
+            }
+        }
+
         if (ImGui::CollapsingHeader("Debug")) {
             if (ImGui::TreeNode("Background")) {
                 ImGui::RadioButton("Image", &enable_scene_viewer, false);
@@ -151,9 +182,10 @@ void app_depth_guide::show_ui() {
 }
 
 void app_depth_guide::render_background() {
-    if (enable_scene_viewer) {
-        scene_viewer->render();
-    } else {
-        bg_viewer->render();
-    }
+    aug_helper_left->render();
+//    if (enable_scene_viewer) {
+//        scene_viewer->render();
+//    } else {
+//        bg_viewer->render();
+//    }
 }

src/impl/apps/depth_guide/depth_guide.h

@@ -6,6 +6,8 @@
 #include "core/event_timer.h"
 #include "core/object_manager.h"
 #include "device/orb_camera_ui.h"
+#include "module/augment_manager_v2.h"
+#include "module/camera_augment_helper_v2.h"
 #include "module/image_augment_helper.h"
 #include "module/image_saver.h"
 #include "module/image_streamer.h"
@@ -64,6 +66,9 @@ private:
     std::unique_ptr<pc_viewer> scene_viewer;
     std::unique_ptr<fast_sam> ai_seg;
 
+    std::unique_ptr<augment_manager_v2> aug_manager;
+    std::unique_ptr<camera_augment_helper_v2> aug_helper_left;
+
     // miscellaneous
     event_timer perf_timer; // performance timer
 

src/impl/apps/remote_ar/remote_ar.cpp

@@ -1,5 +1,6 @@
 #include "remote_ar.h"
 #include "core/imgui_utility.hpp"
+#include "core/math_helper.hpp"
 #include "core/yaml_utility.hpp"
 #include "network/binary_utility.hpp"
 
@@ -50,6 +51,7 @@ app_remote_ar::app_remote_ar(const create_config &_conf) {
     auto stereo_info = stereo_camera_info::from_yaml(LOAD_SUB("stereo_info"));
     float view_angle = 0.0f;
     auto img_range = calc_valid_range(stereo_info.left, stereo_info.right, &view_angle);
+    view_angle = glm::degrees(view_angle);
 
     auto cam_left_conf = image_process_ui::create_config{
             .in_name = raw_left, .out_name = rgb_left, .stream = &cam_left.stream
@@ -98,22 +100,37 @@ app_remote_ar::app_remote_ar(const create_config &_conf) {
     };
     guide_controller = std::make_unique<depth_guide_controller>(guide_control_conf);
 
-    auto aug_conf = augment_manager::create_config{
-            .item_list = augment_manager::item_list_from_yaml(LOAD_LIST("augment_list")),
-            .sophiar_conn = sophiar_conn.get()
+    auto aug_list_v1 = augment_manager::item_list_from_yaml(LOAD_LIST("augment_list"));
+    auto aug_list = augment_manager_v2::item_list_from_v1(aug_list_v1, extra_name);
+    auto aug_conf = augment_manager_v2::create_config{
+            .item_list = aug_list,
+            .sophiar_conn = sophiar_conn.get(),
+            .stream = default_cuda_stream,
     };
-    aug_manager = std::make_unique<augment_manager>(aug_conf);
+    aug_manager = std::make_unique<augment_manager_v2>(aug_conf);
 
-    auto left_aug_conf = camera_augment_helper::create_config{
-            .fov = view_angle, .transform_var = LOAD_STR("left_camera_transform_var"),
+    auto left_aug_conf = camera_augment_helper_v2::create_config{
+            .camera = camera_augment_helper_v2::create_config::fixed_camera_config{
+                    .fov = view_angle, .transform_var = LOAD_STR("left_camera_transform_var"),
+            },
+            .sophiar_conn = sophiar_conn.get(), .manager = aug_manager.get()
+    };
+    cam_left.aug_helper = std::make_unique<camera_augment_helper_v2>(left_aug_conf);
+
+    auto right_aug_conf = camera_augment_helper_v2::create_config{
+            .camera = camera_augment_helper_v2::create_config::relative_camera_config{
+                    .fov = view_angle,
+                    .parent = cam_left.aug_helper.get(),
+                    .transform = glm::inverse(to_mat4(stereo_info.transform)),
+            },
             .sophiar_conn = sophiar_conn.get(), .manager = aug_manager.get()
     };
-    cam_left.aug_helper = std::make_unique<camera_augment_helper>(left_aug_conf);
+    cam_right.aug_helper = std::make_unique<camera_augment_helper_v2>(right_aug_conf);
 
-    auto right_aug_conf = left_aug_conf;
-    right_aug_conf.transform_var = LOAD_STR("right_camera_transform_var");
-    cam_right.aug_helper = std::make_unique<camera_augment_helper>(right_aug_conf);
-    cam_right.aug_helper->sync_with(cam_left.aug_helper.get());
+//    auto right_aug_conf = left_aug_conf;
+//    std::get<2>(right_aug_conf.camera).transform_var = LOAD_STR("right_camera_transform_var");
+//    cam_right.aug_helper = std::make_unique<camera_augment_helper_v2>(right_aug_conf);
+////    cam_right.aug_helper->sync_with(cam_left.aug_helper.get());
 
     auto left_aug_ren_conf = image_augment_helper::create_config{
             .in_name = rgb_left, .out_name = aug_left,

src/impl/apps/remote_ar/remote_ar.h

@@ -4,8 +4,8 @@
 #include "core/event_timer.h"
 #include "core/object_manager.h"
 #include "device/mvs_camera_ui.h"
-#include "module/augment_manager.h"
-#include "module/camera_augment_helper.h"
+#include "module/augment_manager_v2.h"
+#include "module/camera_augment_helper_v2.h"
 #include "module/depth_guide_controller.h"
 #include "module/image_augment_helper.h"
 #include "module/image_player.h"
@@ -46,12 +46,16 @@ private:
         guide_combine_rgb,
         guide_img, guide_depth_fake,
         guide_depth,
-        guide_final
+        guide_final,
+
+        name_end,
     };
 
+    obj_name_type extra_name = name_end;
+
     struct camera_module {
         std::unique_ptr<image_process_ui> img_proc;
-        std::unique_ptr<camera_augment_helper> aug_helper;
+        std::unique_ptr<camera_augment_helper_v2> aug_helper;
         std::unique_ptr<image_augment_helper> aug_render;
         smart_cuda_stream stream;
     };
@@ -69,7 +73,7 @@ private:
     // modules
     std::unique_ptr<mvs_camera_ui> mvs_cam;
     std::unique_ptr<image_viewer> bg_viewer; // background viewer
-    std::unique_ptr<augment_manager> aug_manager;
+    std::unique_ptr<augment_manager_v2> aug_manager;
     std::unique_ptr<stereo_augment_helper> stereo_aug;
     std::unique_ptr<image_streamer> out_streamer;
 

src/module/augment_manager_v2.h

@@ -0,0 +1,64 @@
+#ifndef DEPTHGUIDE_AUGMENT_MANAGER_V2_H
+#define DEPTHGUIDE_AUGMENT_MANAGER_V2_H
+
+#include "core/object_manager.h"
+#include "module/augment_manager.h"
+#include "render/render_scene.h"
+
+// sophiar
+#include "core/local_connection.h"
+
+#include <opencv2/core/types.hpp>
+
+#include <glm/glm.hpp>
+
+#include <memory>
+
+class augment_manager_v2 {
+public:
+
+    enum item_type_enum {
+        ITEM_IMAGE,
+        ITEM_MESH,
+        ITEM_PC,
+    };
+
+    struct create_config {
+
+        struct item_type {
+            item_type_enum type;
+            std::string disp_name;
+            obj_name_type name;
+            std::string transform_var; // for sophiar
+            bool flip_y; // only for image
+        };
+
+        using item_list_type = std::vector<item_type>;
+        item_list_type item_list;
+
+        using sophiar_conn_type = sophiar::local_connection;
+        sophiar_conn_type *sophiar_conn = nullptr;
+
+        smart_cuda_stream *stream = nullptr;
+    };
+
+    using item_list_type = create_config::item_list_type;
+    using item_list_v1_type = augment_manager::item_list_type;
+
+    static item_list_type item_list_from_v1(const item_list_v1_type &items,
+                                            obj_name_type &item_name);
+
+    explicit augment_manager_v2(const create_config &conf);
+
+    ~augment_manager_v2();
+
+    void show();
+
+    void render(const camera_info &info);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#endif //DEPTHGUIDE_AUGMENT_MANAGER_V2_H

src/module/camera_augment_helper_v2.h

@@ -0,0 +1,64 @@
+#ifndef DEPTHGUIDE_CAMERA_AUGMENT_HELPER_V2_H
+#define DEPTHGUIDE_CAMERA_AUGMENT_HELPER_V2_H
+
+#include "augment_manager_v2.h"
+
+// sophiar
+#include "core/local_connection.h"
+
+#include <glm/glm.hpp>
+
+#include <memory>
+#include <variant>
+
+class camera_augment_helper_v2 {
+public:
+
+    struct create_config {
+
+        struct fixed_camera_config {
+            float fov = 60.0f; // field of view
+            std::string transform_var; // for sophiar
+        };
+
+        struct relative_camera_config {
+            float fov = 60.0f; // field of view
+            camera_augment_helper_v2 *parent = nullptr;
+            glm::mat4 transform = glm::mat4(1.0f); // viewed in parent
+        };
+
+        struct freedom_camera_config {
+        };
+
+        using camera_config_type = std::variant<std::monostate,
+                freedom_camera_config, fixed_camera_config, relative_camera_config>;
+        camera_config_type camera;
+
+        // sophiar
+        using sophiar_conn_type = sophiar::local_connection;
+        sophiar_conn_type *sophiar_conn = nullptr;
+
+        augment_manager_v2 *manager = nullptr;
+    };
+
+    explicit camera_augment_helper_v2(const create_config &conf);
+
+    ~camera_augment_helper_v2();
+
+    void render();
+
+    struct output_config {
+        cv::Size size;
+        obj_name_type img_name;
+    };
+
+    void render_image(output_config conf);
+
+    void show();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#endif //DEPTHGUIDE_CAMERA_AUGMENT_HELPER_V2_H

src/module/impl/augment_manager_v2.cpp

@@ -0,0 +1,177 @@
+#include "augment_manager_v2_impl.h"
+#include "core/imgui_utility.hpp"
+#include "core/math_helper.hpp"
+
+#include <glm/gtc/type_ptr.hpp>
+
+glm::mat4 augment_manager_v2::impl::item_store_type::extra_transform() const {
+    return to_transform_mat(extra_offset, extra_rotation);
+}
+
+glm::mat4 augment_manager_v2::impl::item_store_type::get_transform() const {
+    return base_transform * extra_transform();
+}
+
+void augment_manager_v2::impl::item_store_type::update_transform(impl *_pimpl) {
+    if (transform_var.empty()) return;
+    if (_pimpl->sophiar_conn == nullptr) return;
+    auto trans = _pimpl->sophiar_conn->
+            query_transform_variable(transform_var);
+    if (trans.has_value()) {
+        base_transform = to_mat4(*trans);
+        is_tracked = true;
+    } else {
+        is_tracked = false;
+    }
+}
+
+augment_manager_v2::impl::impl(const create_config &conf) {
+    sophiar_conn = conf.sophiar_conn;
+    stream = conf.stream;
+    for (auto &item: conf.item_list) {
+        auto &st = item_list.emplace_back(item);
+
+        // special hacks
+        if (item.type == ITEM_IMAGE) {
+            st.flip_y = item.flip_y;
+        }
+    }
+
+    // load point size information
+    glGetFloatv(GL_POINT_SIZE_RANGE, (GLfloat *) &ps_range);
+    glGetFloatv(GL_POINT_SIZE_GRANULARITY, &ps_interval);
+}
+
+void augment_manager_v2::impl::render(const camera_info &info) {
+    ren_info.camera = info;
+    ren_info.light.follow_camera = enable_light_follow_camera;
+    ren_info.light.direction = to_vec3(light_direction);
+    ren_info.stream = stream;
+
+    ren_info.items.clear();
+    for (auto &item: item_list) {
+        if (!item.visible) continue;
+
+        item.update_transform(this);
+        if (!item.is_tracked && !ignore_missing) continue;
+
+        auto ren_item = scene_render_info::item_type();
+        ren_item.alpha = item.alpha;
+        ren_item.transform = item.get_transform();
+        switch (item.type) {
+            case ITEM_IMAGE: {
+                auto image_info = scene_render_info::image_info{
+                        .img = to_image(item.name),
+                        .flip_y = item.flip_y,
+                };
+                ren_item.info = image_info;
+                break;
+            }
+            case ITEM_MESH: {
+                auto mesh_info = scene_render_info::mesh_info{
+                        .mesh = OBJ_QUERY(mesh_ptr, item.name),
+                };
+                mesh_info.material = {
+                        .ambient = item.color * item.ambient_factor,
+                        .diffuse = item.color
+                };
+                ren_item.info = mesh_info;
+                break;
+            }
+            case ITEM_PC: {
+                auto pc_info = scene_render_info::pc_info{
+                        .pc = OBJ_QUERY(pc_ptr, item.name),
+                        .point_size = item.point_size,
+                        .color = item.color,
+                };
+                ren_item.info = pc_info;
+                break;
+            }
+            default: {
+                RET_ERROR;
+            }
+        }
+        ren_info.items.push_back(ren_item);
+    }
+
+    render_scene(ren_info);
+}
+
+void augment_manager_v2::impl::show() {
+    ImGui::Checkbox("Enable", &enable);
+    if (!enable) return;
+    ImGui::SameLine();
+    ImGui::Checkbox("Ignore Missing", &ignore_missing);
+    for (auto &item: item_list) {
+        if (ImGui::TreeNode(item.disp_name.c_str())) {
+            ImGui::Checkbox("Visibility", &item.visible);
+            if (!item.visible) continue;
+
+            ImGui::DragFloat("Alpha", &item.alpha, 0.005f, 0.0f, 1.0f);
+
+            if (item.type == ITEM_MESH || item.type == ITEM_PC) {
+                ImGui::ColorEdit3("Color", glm::value_ptr(item.color));
+                ImGui::DragFloat3("Offset (mm)", glm::value_ptr(item.extra_offset),
+                                  0.05f, 0.0f, 0.0f, "%.02f");
+                ImGui::DragFloat3("Offset (deg)", glm::value_ptr(item.extra_rotation),
+                                  0.1f, -180.0f, 180.0f, "%.01f");
+            }
+
+            if (item.type == ITEM_MESH) {
+                ImGui::DragFloat("Ambient Factor", &item.ambient_factor, 0.005f, 0.0f, 1.0f);
+            }
+
+            if (item.type == ITEM_PC) {
+                ImGui::DragFloat("Point Size", &item.point_size, ps_interval,
+                                 ps_range.min, ps_range.max, "%.01f");
+            }
+
+            ImGui::TreePop();
+        }
+    }
+    if (ImGui::TreeNode("Light")) {
+        ImGui::Checkbox("Follow Camera", &enable_light_follow_camera);
+        if (!enable_light_follow_camera) {
+            ImGui::gizmo3D("##direction", light_direction, ImGui::CalcItemWidth());
+        }
+        ImGui::TreePop();
+    }
+}
+
+augment_manager_v2::augment_manager_v2(const create_config &conf)
+        : pimpl(std::make_unique<impl>(conf)) {
+}
+
+augment_manager_v2::~augment_manager_v2() = default;
+
+void augment_manager_v2::render(const camera_info &info) {
+    pimpl->render(info);
+}
+
+void augment_manager_v2::show() {
+    pimpl->show();
+}
+
+augment_manager_v2::item_list_type
+augment_manager_v2::item_list_from_v1(const item_list_v1_type &items,
+                                      obj_name_type &item_name) {
+    auto ret = item_list_type();
+    ret.reserve(items.size());
+
+    using item_type = create_config::item_type;
+    for (auto &v1: items) {
+        auto item = item_type{
+                .type = ITEM_MESH,
+                .disp_name = v1.name,
+                .name = item_name++,
+                .transform_var = v1.transform_var,
+        };
+        // TODO: bg_name ignored
+        ret.push_back(item);
+
+        // create mesh
+        auto mesh = mesh_type::create({.path = v1.model_path});
+        OBJ_SAVE(item.name, mesh);
+    }
+    return ret;
+}

src/module/impl/augment_manager_v2_impl.h

@@ -0,0 +1,73 @@
+#ifndef DEPTHGUIDE_AUGMENT_MANAGER_V2_IMPL_H
+#define DEPTHGUIDE_AUGMENT_MANAGER_V2_IMPL_H
+
+#include "module/augment_manager_v2.h"
+
+#include <imGuIZMOquat.h>
+
+struct augment_manager_v2::impl {
+
+    using item_base_type = create_config::item_type;
+
+    struct item_store_type : public item_base_type {
+
+        // for all
+        bool visible = true;
+        float alpha = 1.0f;
+
+        // for image
+        bool flip_y = false;
+
+        // for mesh
+        float ambient_factor = 0.5f;
+
+        // for pc
+        float point_size = 1.0f;
+
+        // for mesh and pc
+        glm::vec3 color = {1.0f, 0.0f, 0.0f};
+        glm::vec3 extra_offset;
+        glm::vec3 extra_rotation;
+        glm::mat4 base_transform = glm::mat4(1.0f);
+        bool is_tracked = true;
+
+        // combined from extra_offset and extra_rotation
+        glm::mat4 extra_transform() const;
+
+        glm::mat4 get_transform() const;
+
+        void update_transform(impl *pimpl);
+    };
+
+    using item_list_type =
+            std::vector<item_store_type>;
+    item_list_type item_list;
+
+    using sophiar_conn_type = create_config::sophiar_conn_type;
+    sophiar_conn_type *sophiar_conn = nullptr;
+
+    smart_cuda_stream *stream = nullptr;
+
+    bool enable = true; // enable this module
+    bool enable_light_follow_camera = false;
+    bool ignore_missing = false; // show item even if missing from the tracker
+
+    vgm::Vec3 light_direction = {0.0f, 0.0f, -1.0f};
+
+    // cached to prevent frequent memory operation
+    scene_render_info ren_info;
+
+    // low-level GL_POINT info
+    struct {
+        GLfloat min, max;
+    } ps_range;
+    GLfloat ps_interval;
+
+    explicit impl(const create_config &conf);
+
+    void render(const camera_info &info);
+
+    void show();
+};
+
+#endif //DEPTHGUIDE_AUGMENT_MANAGER_V2_IMPL_H

src/module/impl/camera_augment_helper_v2.cpp

@@ -0,0 +1,240 @@
+#include "camera_augment_helper_v2_impl.h"
+#include "core/math_helper.hpp"
+#include "core/imgui_utility.hpp"
+
+#include <glm/gtc/matrix_transform.hpp>
+#include <glm/gtc/type_ptr.hpp>
+#include <glm/gtx/rotate_vector.hpp>
+
+glm::mat4 camera_augment_helper_v2::impl::freedom_info_type::to_transform() {
+    // normalize view up
+    view_up -= view_dir() * glm::dot(view_up, view_dir());
+    view_up = glm::normalize(view_up);
+
+    auto ret = glm::lookAt(eye, center, view_up);
+    ret = glm::rotate(ret, glm::radians(180.0f), glm::vec3(1.0f, 0.0f, 0.0f)); // viewport -> camera
+    return ret;
+}
+
+glm::vec3 camera_augment_helper_v2::impl::freedom_info_type::view_dir() const {
+    return glm::normalize(center - eye);
+}
+
+float camera_augment_helper_v2::impl::freedom_info_type::view_dis() const {
+    return glm::distance(center, eye);
+}
+
+void camera_augment_helper_v2::impl::freedom_info_type::translate(glm::vec3 offset) {
+    eye += offset;
+    center += offset;
+}
+
+void camera_augment_helper_v2::impl::freedom_info_type::forward(float dis) {
+    translate(view_dir() * dis);
+}
+
+void camera_augment_helper_v2::impl::freedom_info_type::move_right(float dis) {
+    auto axis = glm::cross(view_up, view_dir());
+    translate(-axis * dis);
+}
+
+void camera_augment_helper_v2::impl::freedom_info_type::move_up(float dis) {
+    translate(view_up * dis);
+}
+
+void camera_augment_helper_v2::impl::freedom_info_type::center_forward(float dis) {
+    center = eye + view_dir() * (view_dis() + dis);
+}
+
+void camera_augment_helper_v2::impl::freedom_info_type::roll(float deg) {
+    view_up = glm::rotate(view_up, glm::radians(deg), view_dir());
+}
+
+void camera_augment_helper_v2::impl::freedom_info_type::yaw(float deg) {
+    auto next_view_dir = glm::rotate(view_dir(), glm::radians(deg), view_up);
+    center = eye + next_view_dir * view_dis();
+}
+
+void camera_augment_helper_v2::impl::freedom_info_type::pitch(float deg) {
+    auto rot_axis = glm::cross(view_up, view_dir());
+    view_up = glm::rotate(view_up, deg, rot_axis);
+    auto next_view_dir = glm::rotate(view_dir(), glm::radians(deg), rot_axis);
+    center = eye + next_view_dir * view_dis();
+}
+
+glm::mat4 camera_augment_helper_v2::impl::fixed_info_type::extra_transform() const {
+    return to_transform_mat(extra_offset, extra_rotation);
+}
+
+camera_augment_helper_v2::impl::impl(const create_config &conf) {
+    manager = conf.manager;
+    sophiar_conn = conf.sophiar_conn;
+
+    switch (conf.camera.index()) {
+        case 1: {
+            mode = MODE_FREEDOM;
+            break;
+        }
+        case 2: {
+            mode = MODE_FIXED;
+            auto info = std::get<2>(conf.camera);
+            fov = info.fov;
+            fixed_info.transform_var = info.transform_var;
+            break;
+        }
+        case 3: {
+            mode = MODE_RELATIVE;
+            auto info = std::get<3>(conf.camera);
+            fov = info.fov;
+            relative_info.parent = info.parent->pimpl.get();
+            relative_info.transform = info.transform;
+            break;
+        }
+        default: {
+            RET_ERROR;
+        }
+    }
+}
+
+void camera_augment_helper_v2::impl::update_freedom() {
+    transform = freedom_info.to_transform();
+    is_missing = false;
+}
+
+void camera_augment_helper_v2::impl::update_fixed() {
+    assert(sophiar_conn != nullptr);
+    auto trans = sophiar_conn->
+            query_transform_variable(fixed_info.transform_var);
+    is_missing = !trans.has_value();
+    if (trans.has_value()) {
+        transform = to_mat4(*trans) * fixed_info.extra_transform();
+    }
+}
+
+void camera_augment_helper_v2::impl::update_relative() {
+    auto par = relative_info.parent;
+    par->update_transform();
+    is_missing = par->is_missing;
+    transform = par->transform * relative_info.transform;
+}
+
+void camera_augment_helper_v2::impl::update_transform() {
+    switch (mode) {
+        // @formatter:off
+        case MODE_FREEDOM: { update_freedom(); break; }
+        case MODE_RELATIVE: { update_relative(); break; }
+        case MODE_FIXED: { update_fixed(); break; }
+        // @formatter:on
+    }
+}
+
+void camera_augment_helper_v2::impl::render() {
+    update_transform();
+    auto ren_info = camera_info{
+            .transform = transform, .fov = fov,
+            .near = near, .far = far
+    };
+    if (!is_missing || ignore_missing) {
+        manager->render(ren_info);
+    }
+}
+
+void camera_augment_helper_v2::impl::render_image(output_config conf) {
+    fbo_conf.size = conf.size;
+    fbo.create(fbo_conf);
+
+    fbo.bind();
+    render();
+    auto out_img = img_downloader->download_v2();
+    OBJ_SAVE(conf.img_name, out_img);
+    fbo.unbind();
+}
+
+void camera_augment_helper_v2::impl::show_freedom() {
+    auto &info = freedom_info;
+
+    if (ImGui::TreeNode("Center")) {
+        if (ImGui::Button("X+")) { info.move_right(info.linear_speed); }
+        ImGui::SameLine();
+        if (ImGui::Button("Y+")) { info.move_up(info.linear_speed); }
+        ImGui::SameLine();
+        if (ImGui::Button("Z+")) { info.forward(info.linear_speed); }
+//        ImGui::SameLine();
+//        if (ImGui::Button("D+")) { info.center_forward(info.linear_speed); }
+
+        if (ImGui::Button("X-")) { info.move_right(-info.linear_speed); }
+        ImGui::SameLine();
+        if (ImGui::Button("Y-")) { info.move_up(-info.linear_speed); }
+        ImGui::SameLine();
+        if (ImGui::Button("Z-")) { info.forward(-info.linear_speed); }
+//        ImGui::SameLine();
+//        if (ImGui::Button("D-")) { info.center_forward(-info.linear_speed); }
+
+        ImGui::TreePop();
+    }
+
+    if (ImGui::TreeNode("Eye")) {
+        if (ImGui::Button("Ro+")) { info.roll(info.angle_speed); } // roll
+        ImGui::SameLine();
+        if (ImGui::Button("Ya+")) { info.yaw(info.angle_speed); } // yaw
+        ImGui::SameLine();
+        if (ImGui::Button("Pi+")) { info.pitch(info.angle_speed); }  // pitch
+
+        if (ImGui::Button("Ro-")) { info.roll(-info.angle_speed); } // roll
+        ImGui::SameLine();
+        if (ImGui::Button("Ya-")) { info.yaw(-info.angle_speed); } // yaw
+        ImGui::SameLine();
+        if (ImGui::Button("Pi-")) { info.pitch(-info.angle_speed); } // pitch
+
+        ImGui::TreePop();
+    }
+
+    if (ImGui::TreeNode("Extra")) {
+        ImGui::SliderFloat("Angular Step", &info.angle_speed, 0.01f, 10.0f, "%.2f");
+        ImGui::SliderFloat("Linear Step", &info.linear_speed, 0.01f, 100.0f, "%.2f");
+        ImGui::DragFloat("FOV (deg)", &fov, 0.5f, 10.0f, 178.0f, "%.01f");
+        ImGui::TreePop();
+    }
+}
+
+void camera_augment_helper_v2::impl::show_fixed() {
+    ImGui::DragFloat3("Offset (mm)", glm::value_ptr(fixed_info.extra_offset),
+                      0.05f, 0.0f, 0.0f, "%.02f");
+    ImGui::DragFloat3("Offset (deg)", glm::value_ptr(fixed_info.extra_rotation),
+                      0.1f, -180.0f, 180.0f, "%.01f");
+}
+
+void camera_augment_helper_v2::impl::show() {
+    ImGui::Checkbox("Ignore Missing", &ignore_missing);
+    ImGui::SliderFloat("Clip Near", &near, 1.0f, far, "%.f", ImGuiSliderFlags_Logarithmic);
+    ImGui::SliderFloat("Clip Far", &far, near, 10000.0f, "%.f", ImGuiSliderFlags_Logarithmic);
+
+    switch (mode) {
+        // @formatter:off
+        case MODE_FIXED: { show_fixed(); break; }
+        case MODE_RELATIVE: { break; }
+        case MODE_FREEDOM: { show_freedom(); break; }
+        // @formatter:on
+        default: {
+            RET_ERROR;
+        }
+    }
+}
+
+camera_augment_helper_v2::camera_augment_helper_v2(const create_config &conf)
+        : pimpl(std::make_unique<impl>(conf)) {
+}
+
+camera_augment_helper_v2::~camera_augment_helper_v2() = default;
+
+void camera_augment_helper_v2::render() {
+    pimpl->render();
+}
+
+void camera_augment_helper_v2::render_image(output_config conf) {
+    pimpl->render_image(conf);
+}
+
+void camera_augment_helper_v2::show() {
+    pimpl->show();
+}

src/module/impl/camera_augment_helper_v2_impl.h

@@ -0,0 +1,101 @@
+#ifndef DEPTHGUIDE_CAMERA_AUGMENT_HELPER_V2_IMPL_H
+#define DEPTHGUIDE_CAMERA_AUGMENT_HELPER_V2_IMPL_H
+
+#include "module/camera_augment_helper_v2.h"
+#include "module/viewport_downloader.hpp"
+
+struct camera_augment_helper_v2::impl {
+
+    bool is_missing = false;
+    bool ignore_missing = false;
+
+    augment_manager_v2 *manager = nullptr;
+    using sophiar_conn_type = create_config::sophiar_conn_type;
+    sophiar_conn_type *sophiar_conn = nullptr;
+
+    // for all
+    float fov = 60.0f; // field of view
+    float near = 10.0f;
+    float far = 1000.0f; // clip range (mm)
+    glm::mat4 transform; // standard camera transform
+
+    struct fixed_info_type {
+        glm::vec3 extra_offset = glm::vec3();
+        glm::vec3 extra_rotation = glm::vec3();
+        std::string transform_var; // sophiar
+
+        glm::mat4 extra_transform() const;
+
+    } fixed_info;
+
+    struct freedom_info_type {
+        glm::vec3 eye = glm::vec3();
+        glm::vec3 center = glm::vec3(0.0f, 0.0f, 100.0f);
+        glm::vec3 view_up = glm::vec3(0.0f, -1.0f, 0.0f);
+
+        float angle_speed = 1.0f; // degree
+        float linear_speed = 0.1f; // mm
+
+        glm::vec3 view_dir() const;
+
+        float view_dis() const;
+
+        void translate(glm::vec3 offset);
+
+        void forward(float dis);
+
+        void move_right(float dis);
+
+        void move_up(float dis);
+
+        void center_forward(float dis);
+
+        void roll(float deg);
+
+        void yaw(float deg);
+
+        void pitch(float deg);
+
+        glm::mat4 to_transform();
+
+    } freedom_info;
+
+    struct {
+        impl *parent;
+        glm::mat4 transform;
+    } relative_info;
+
+    enum {
+        MODE_FIXED,
+        MODE_RELATIVE,
+        MODE_FREEDOM
+    } mode;
+
+    // for off-screen rendering
+    smart_frame_buffer::create_config fbo_conf = {};
+    smart_frame_buffer fbo;
+    std::unique_ptr<viewport_downloader> img_downloader;
+
+    explicit impl(const create_config &conf);
+
+    void update_freedom();
+
+    void update_fixed();
+
+    void update_relative();
+
+    void update_transform();
+
+    void render();
+
+    void render_image(output_config conf);
+
+    void show_freedom();
+
+    void show_fixed();
+
+    void show();
+
+};
+
+#endif //DEPTHGUIDE_CAMERA_AUGMENT_HELPER_V2_IMPL_H

src/module/impl/image_augment_helper.cpp

@@ -34,6 +34,7 @@ void image_augment_helper::impl::process(obj_name_type name) {
     fbo.bind();
 
     auto ren_conf = color_image_render::config_type{
+            .fmt = COLOR_RGB, // TODO: ugly hacked
             .flip_y = conf.flip_image,
             .stream = conf.stream,
     };
@@ -110,6 +111,7 @@ void stereo_augment_helper::impl::process() {
     }
 
     auto ren_conf = color_image_render::config_type{
+            .fmt = COLOR_RGB, // TODO: ugly hacked
             .flip_y = conf.flip_image,
             .stream = conf.stream,
     };

src/module/viewport_downloader.hpp

@@ -35,6 +35,23 @@ public:
         return create_image(info);
     }
 
+    image_ptr download_v2(color_format fmt = COLOR_RGB) {
+        switch (fmt) {
+            case COLOR_RGB: {
+                return pbo.download_viewport(GL_RGB, GL_UNSIGNED_BYTE, stream);
+            }
+            case COLOR_RGBA: {
+                return pbo.download_viewport(GL_RGBA, GL_UNSIGNED_BYTE, stream);
+            }
+            case COLOR_ARGB: {
+                return pbo.download_viewport(GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, stream);
+            }
+            default: {
+                RET_ERROR_E;
+            }
+        }
+    }
+
 private:
     smart_cuda_stream *stream;
     smart_pixel_buffer pbo;

src/render/impl/render_scene.cpp

@@ -1 +1,122 @@
-#include "render_scene_impl.h"
+#include "render_scene_impl.h"
+
+#include <glm/gtc/matrix_access.hpp>
+#include <glm/gtx/transform.hpp>
+
+namespace render_scene_impl {
+
+    color_image_render image_render;
+
+    void render_image(const render_request &req) {
+        using info_type = scene_render_info::image_info;
+        auto info = std::get<info_type>(req.item.info);
+
+        // TODO: color with depth has not been implemented
+        assert(info.depth == nullptr);
+
+        auto ren_conf = color_image_render::config_type{
+                .flip_y = info.flip_y,
+                .alpha = req.item.alpha,
+                .stream = req.stream,
+        };
+        image_render.render_v2(info.img, ren_conf);
+    }
+
+    void render_mesh(const render_request &req) {
+        using info_type = scene_render_info::mesh_info;
+        auto info = std::get<info_type>(req.item.info);
+
+        auto mesh_conf = mesh_info_type{
+                .mesh = info.mesh.get(),
+                .transform = req.item.transform,
+                .material = info.material,
+        };
+        auto ren_conf = mesh_render_info{
+                .mode = MESH_NORMAL,
+                .model = mesh_conf,
+                .camera = req.camera,
+                .light= req.light,
+        };
+        render_mesh(ren_conf);
+    }
+
+    void render_pc(const render_request &req) {
+        using info_type = scene_render_info::pc_info;
+        auto info = std::get<info_type>(req.item.info);
+
+        auto ren_conf = pc_render::config_type{
+                .stream = req.stream,
+        };
+        ren_conf.pc = {
+                .transform = req.item.transform,
+                .color = info.color,
+                .point_size = info.point_size,
+        };
+        ren_conf.camera = {
+                .transform = req.camera.transform,
+                .fov = req.camera_raw.fov,
+                .near = req.camera_raw.near,
+                .far = req.camera_raw.far,
+        };
+        pc_render::render(info.pc, ren_conf);
+    }
+
+    // standard camera transform matrix ->
+    // OpenGL 's camera transform matrix
+    glm::mat4 cvt_camera_transform(const glm::mat4 &transform) {
+        static constexpr auto default_focal_length = 8.0f; // 8mm
+        auto trans_part = glm::vec3(transform[3]);
+        auto rot_part = glm::mat3(transform);
+        auto focal_point = trans_part + rot_part[2] * default_focal_length;
+        auto view_up = -rot_part[1];
+        return glm::lookAt(trans_part, focal_point, view_up);
+    }
+
+}
+
+void render_scene(const scene_render_info &info) {
+    auto vp_size = query_viewport_size();
+
+    auto req = render_request{
+            .camera_raw = info.camera,
+            .stream = info.stream,
+    };
+    req.camera.transform =
+            cvt_camera_transform(info.camera.transform);
+    req.camera.project = glm::perspective(
+            glm::radians(info.camera.fov), (float) vp_size.aspectRatio(),
+            info.camera.near, info.camera.far);
+    if (info.light.follow_camera) {
+        req.light.direction = glm::column(info.camera.transform, 2);
+    } else {
+        req.light.direction = info.light.direction;
+    }
+
+    for (auto &item: info.items) {
+        req.item = item;
+        using info_type = std::remove_cvref_t<typeof(item.info)>;
+        switch (item.info.index()) {
+            case 1: {
+                static_assert(std::is_same_v<std::variant_alternative_t<1, info_type>,
+                        scene_render_info::image_info>);
+                render_image(req);
+                break;
+            }
+            case 2: {
+                static_assert(std::is_same_v<std::variant_alternative_t<2, info_type>,
+                        scene_render_info::mesh_info>);
+                render_mesh(req);
+                break;
+            }
+            case 3: {
+                static_assert(std::is_same_v<std::variant_alternative_t<3, info_type>,
+                        scene_render_info::pc_info>);
+                render_pc(req);
+                break;
+            }
+            default: {
+                RET_ERROR;
+            }
+        }
+    }
+}

src/render/impl/render_scene_impl.h

@@ -2,5 +2,31 @@
 #define DEPTHGUIDE_RENDER_SCENE_IMPL_H
 
 #include "render/render_scene.h"
+#include "render/render_tools.h"
+
+namespace render_scene_impl {
+
+    using item_type = scene_render_info::item_type;
+
+    struct render_request {
+        item_type item;
+        camera_info camera_raw = {};
+        camera_info_type camera = {};
+        light_info_type light = {};
+        smart_cuda_stream *stream = nullptr;
+    };
+
+    extern color_image_render image_render;
+
+    void render_image(const render_request &req);
+
+    void render_mesh(const render_request &req);
+
+    void render_pc(const render_request &req);
+
+    glm::mat4 cvt_camera_transform(const glm::mat4 &transform);
+}
+
+using namespace render_scene_impl;
 
 #endif //DEPTHGUIDE_RENDER_SCENE_IMPL_H

src/render/impl/render_tools.cpp

@@ -42,14 +42,7 @@ template void color_image_render::render_rgba<uchar4>(const image_info_u8c4 &, c
 
 void color_image_render::render_rgba(obj_name_type name, config_type conf) {
     auto img_type = OBJ_TYPE(name);
-
-    // TODO: ugly hacked
-    if (img_type == typeid(image_ptr)) {
-        auto img = OBJ_QUERY(image_ptr, name);
-        img_tex.upload(img, conf.stream);
-        render_tex(img->size(), conf);
-        return;
-    }
+    assert(img_type != typeid(image_ptr));
 
     auto impl_func = [&](auto V) {
         using T = std::remove_cvref_t<decltype(V)>;
@@ -88,6 +81,11 @@ void color_image_render::render_nv12(obj_name_type name, config_type conf) {
 }
 
 void color_image_render::render(obj_name_type name, config_type conf) {
+    if (OBJ_TYPE(name) == typeid(image_ptr)) {
+        render_v2(name, conf);
+        return;
+    }
+
     switch (conf.fmt) {
         case COLOR_BW:
         case COLOR_RGB:
@@ -105,6 +103,56 @@ void color_image_render::render(obj_name_type name, config_type conf) {
     }
 }
 
+void color_image_render::render_pix_normal(const image_ptr &img, config_type conf) {
+    auto disp_fmt =
+            (display_fmt) img->get_meta(META_IMAGE_DISPLAY_FMT).value_or(0);
+    auto img_type = img->cv_type();
+    if (disp_fmt == DISP_MASK) {
+        assert(img_type == CV_8UC1);
+        conf.fmt = COLOR_BW;
+    } else {
+        assert(disp_fmt == DISP_COLOR);
+        switch (CV_MAT_CN(img->cv_type())) {
+            // @formatter:off
+            case 1: { conf.fmt = COLOR_GRAY; break; }
+            case 3: { conf.fmt = COLOR_RGB; break; }
+            case 4: { conf.fmt = COLOR_RGBA; break; }
+            // @formatter:on
+            default: {
+                RET_ERROR;
+            }
+        }
+    }
+
+    // TODO: create a texture pool to prevent frequent allocate and deallocate
+    img_tex.upload(img, conf.stream);
+    render_tex(img->size(), conf);
+}
+
+void color_image_render::render_pix_nv12(const image_ptr &img, config_type conf) {
+    auto img_info = img->v1<uchar1>()->as_info();
+    SYNC_CREATE(img_info.start_ptr(), conf.stream);
+    render_nv12(img_info, conf);
+}
+
+void color_image_render::render_v2(obj_name_type name, config_type conf) {
+    assert(OBJ_TYPE(name) == typeid(image_ptr));
+    auto img = OBJ_QUERY(image_ptr, name);
+    render_v2(img, conf);
+}
+
+void color_image_render::render_v2(const image_ptr &img, config_type conf) {
+    switch (img->pixel_format()) {
+        // @formatter:off
+        case PIX_NORMAL: { render_pix_normal(img, conf); break; }
+        case PIX_NV12: { render_pix_nv12(img, conf); break; }
+        // @formatter:on
+        default: {
+            RET_ERROR;
+        }
+    }
+}
+
 void depth_image_render::render(obj_name_type name, config_type conf) {
     auto img_type = OBJ_TYPE(name);
     assert(img_type == typeid(image_f32c1));
@@ -141,7 +189,11 @@ void depth_image_render::render(obj_name_type name, config_type conf) {
 void pc_render::render(obj_name_type name, const pc_render::config_type &conf) {
     auto pc = OBJ_QUERY(pc_ptr, name);
     if (pc == nullptr) [[unlikely]] return;
+    render(pc, conf);
+}
 
+void pc_render::render(const pc_ptr &pc, const config_type &conf) {
+    if (pc == nullptr) return;
     auto gl_info = pc->get_gl_info(conf.stream);
     auto ren_info = pc_render_info();
     ren_info.pc.format = pc->format();

src/render/impl/render_utility.cpp

@@ -119,15 +119,19 @@ void smart_texture::set_filter(GLint min_filter, GLint max_filter) {
 void smart_texture::create(GLenum _format, cv::Size _size) {
     if (_format == format && _size == size) [[likely]] return;
 
+    // allocate before deallocate
+    GLuint next_id;
+    glGenTextures(1, &next_id);
+    glBindTexture(GL_TEXTURE_2D, next_id);
+    glTexStorage2D(GL_TEXTURE_2D, 1, _format, _size.width, _size.height);
+    glBindTexture(GL_TEXTURE_2D, 0);
+
     deallocate();
 
     // allocate
     format = _format;
     size = _size;
-    glGenTextures(1, &id);
-    glBindTexture(GL_TEXTURE_2D, id);
-    glTexStorage2D(GL_TEXTURE_2D, 1, format, size.width, size.height);
-    glBindTexture(GL_TEXTURE_2D, 0);
+    id = next_id;
 
     // config
     set_filter(GL_NEAREST, GL_NEAREST);
@@ -291,6 +295,20 @@ void smart_pixel_buffer::upload_impl(const std::shared_ptr<void> &data, size_t d
     }
 }
 
+void smart_pixel_buffer::download(const image_ptr &img, smart_cuda_stream *stream) {
+    assert(size == img->size_in_bytes());
+    download_impl(img->memory_v1(stream), stream);
+}
+
+image_ptr smart_pixel_buffer::download_viewport(GLenum format, GLenum type,
+                                                smart_cuda_stream *stream) {
+    auto img_type = CV_MAKETYPE(get_type_depth(type), get_format_channels(format));
+    auto img = create_image(query_viewport_size(), img_type);
+    download_viewport(format, type);
+    download(img, stream);
+    return img;
+}
+
 void smart_pixel_buffer::download_impl(const image_mem_info &img, smart_cuda_stream *stream) {
     if (cuda_res_down == nullptr) {
         CUDA_API_CHECK(cudaGraphicsGLRegisterBuffer(

src/render/render_mesh.h

@@ -35,18 +35,25 @@ private:
     std::unique_ptr<impl> pimpl;
 };
 
+using mesh_ptr = mesh_type::pointer;
+
 enum mesh_render_mode {
     MESH_NORMAL,
     MESH_DEPTH_ALPHA
 };
 
+struct material_type {
+    glm::vec3 ambient;
+    glm::vec3 diffuse;
+};
+
 struct mesh_info_type {
     mesh_type *mesh = nullptr;
     glm::mat4 transform = {};
-    struct material_type {
-        glm::vec3 ambient;
-        glm::vec3 diffuse;
-    } material = {};
+
+    // make scene_render happy
+    using material_type = ::material_type;
+    material_type material = {};
 };
 
 struct camera_info_type {

src/render/render_scene.h

@@ -4,27 +4,57 @@
 #include "core/image_utility_v2.h"
 #include "render_mesh.h"
 
-struct scene_render_info {
+#include <variant>
 
-    image_ptr background_img = nullptr;
+struct camera_info {
+    glm::mat4 transform; // standard camera transform
+    GLfloat fov = 60.0f; // field of view (degree)
+    GLfloat near = 10.0f;
+    GLfloat far = 1000.0f; // clip range (mm)
+};
 
-    enum item_type {
-        ITEM_MESH,
-        ITEM_PC,
+struct scene_render_info {
+
+    struct image_info {
+        image_ptr img = nullptr;
+        image_ptr depth = nullptr;
+        bool flip_y = false;
     };
 
     struct mesh_info {
-        mesh_type *mesh = nullptr;
-        struct material_type {
-            glm::vec3 ambient;
-            glm::vec3 diffuse;
-        } material = {};
+        mesh_ptr mesh = nullptr;
+        material_type material = {};
     };
 
     struct pc_info {
+        pc_ptr pc = nullptr;
+        GLfloat point_size = 1.0f;
+        glm::vec3 color = glm::vec3(0.5f);
+    };
 
+    struct item_type {
+        using item_store_type = std::variant<std::monostate,
+                image_info, mesh_info, pc_info>;
+        item_store_type info;
+
+        GLfloat alpha = 1.0f;
+        glm::mat4 transform = glm::mat4(1.0f); // ignored with ITEM_IMAGE
     };
 
+    using item_list_type =
+            std::vector<item_type>;
+    item_list_type items;
+
+    camera_info camera;
+
+    struct light_info {
+        bool follow_camera = false;
+        glm::vec3 direction = glm::vec3(0.0f, 0.0f, -1.0f);
+    } light;
+
+    smart_cuda_stream *stream = nullptr;
 };
 
+void render_scene(const scene_render_info &info);
+
 #endif //DEPTHGUIDE_RENDER_SCENE_H

src/render/render_tools.h

@@ -15,7 +15,7 @@ class color_image_render {
 public:
 
     struct config_type {
-        color_format fmt = COLOR_RGB;
+        color_format fmt = COLOR_ANY;
         bool flip_y = false;
         float alpha = 1.0;
         std::optional<simple_rect> range;
@@ -41,11 +41,20 @@ public:
     // supports image_u8c1
     void render_nv12(obj_name_type name, config_type conf);
 
+    // render image_ptr, conf.fmt ignored
+    void render_v2(obj_name_type name, config_type conf);
+
+    void render_v2(const image_ptr &img, config_type conf);
+
 private:
     smart_texture img_tex;
     smart_texture ext_tex[2];
 
     void render_tex(cv::Size img_size, config_type conf);
+
+    void render_pix_normal(const image_ptr &img, config_type conf);
+
+    void render_pix_nv12(const image_ptr &img, config_type conf);
 };
 
 class depth_image_render {
@@ -87,7 +96,9 @@ public:
         smart_cuda_stream *stream = nullptr;
     };
 
-    void render(obj_name_type name, const config_type &conf);
+    static void render(obj_name_type name, const config_type &conf);
+
+    static void render(const pc_ptr &pc, const config_type &conf);
 };
 
 #endif //DEPTHGUIDE_RENDER_TOOLS_H

src/render/render_utility.h

@@ -49,6 +49,33 @@ constexpr inline GLenum get_tex_format(int type) {
     }
 }
 
+constexpr inline int get_format_channels(int fmt) {
+    switch (fmt) {
+        // @formatter:off
+        case GL_RED: { return 1; }
+        case GL_RG:  { return 2; }
+        case GL_RGB: { return 3; }
+        case GL_RGBA:
+        case GL_BGRA: { return 4; }
+        // @formatter:on
+        default: {
+            RET_ERROR;
+        }
+    }
+}
+
+constexpr inline int get_type_depth(int type) {
+    switch (type) {
+        // @formatter:off
+        case GL_UNSIGNED_INT_8_8_8_8_REV:
+        case GL_UNSIGNED_BYTE: { return CV_8U; }
+        // @formatter:on
+        default: {
+            RET_ERROR;
+        }
+    }
+}
+
 template<typename T>
 constexpr inline GLenum get_tex_type() {
     // @formatter:off
@@ -141,6 +168,8 @@ public:
         download_impl(img.mem_info(), stream);
     }
 
+    void download(const image_ptr &img, smart_cuda_stream *stream);
+
     // memory location maintains as img->loc.
     template<typename T>
     void download_viewport(image_info_type<T> *img, GLenum format,
@@ -150,6 +179,9 @@ public:
         download(*img, stream);
     }
 
+    image_ptr download_viewport(GLenum format, GLenum type,
+                                smart_cuda_stream *stream);
+
 private:
 
     size_t capacity = 0;