Implemented depth-based alpha blending rendering.

CMakeLists.txt

@@ -63,7 +63,16 @@ else ()
 endif ()
 target_include_directories(${PROJECT_NAME} PRIVATE ${GLAD_DIR}/include)
 target_sources(${PROJECT_NAME} PRIVATE ${GLAD_DIR}/src/gl.c)
-target_sources(${PROJECT_NAME} PRIVATE src/simple_opengl.cpp)
+target_sources(${PROJECT_NAME} PRIVATE
+        src/render/mesh_render.cpp
+        src/render/render_utility.cpp
+        src/render/scene_render.cpp
+        src/render/texture_render.cpp)
+
+# glm config
+find_package(glm REQUIRED)
+target_link_libraries(${PROJECT_NAME} glm::glm)
+target_compile_definitions(${PROJECT_NAME} PRIVATE GLM_ENABLE_EXPERIMENTAL)
 
 # imgui config
 if (WIN32)
@@ -83,6 +92,14 @@ target_sources(${PROJECT_NAME} PRIVATE
         ${IMGUI_BACKENDS_DIR}/imgui_impl_opengl3.cpp)
 target_compile_definitions(${PROJECT_NAME} PRIVATE HAVE_IMGUI)
 
+# imGuIZMO config
+set(IMGUIZMO_DIR /home/tpx/src/imGuIZMO.quat-3.0/imGuIZMO.quat)
+target_include_directories(${PROJECT_NAME} PRIVATE ${IMGUIZMO_DIR})
+target_sources(${PROJECT_NAME} PRIVATE
+        ${IMGUIZMO_DIR}/imGuIZMOquat.cpp)
+target_compile_definitions(${PROJECT_NAME} PRIVATE
+        IMGUIZMO_IMGUI_FOLDER=${IMGUI_DIR}/)
+
 # MVS config
 if (WIN32)
     set(MVS_DIR "C:/BuildEssentials/Library/MVS/Development")
@@ -143,6 +160,10 @@ target_link_libraries(${PROJECT_NAME} Eigen3::Eigen)
 find_package(nlohmann_json REQUIRED)
 target_link_libraries(${PROJECT_NAME} nlohmann_json::nlohmann_json)
 
+# cxxopts config
+find_package(cxxopts REQUIRED)
+target_link_libraries(${PROJECT_NAME} cxxopts::cxxopts)
+
 # Sophiar2 config
 if (WIN32)
     set(Sophiar2DIR D:/Program/Robot/Sophiar2)

data/config.yaml

@@ -39,7 +39,7 @@ main_window:
   height: 600
 
 output:
-  width: 5000
+  width: 1920
   height: 1080
   hevc_bitrate: 5 # Mbps
   jpeg_quality: 15
@@ -58,19 +58,23 @@ sophiar:
     - name: Femur
       trans_var: femur_in_tracker_denoised
       stl_file: ./models/Femur_2.stl
-      registration:
-        collect_obj: point_picker_in_femur_ref
-        collect_var: picked_point_in_femur_ref
-        target_var: femur_in_femur_ref
-        probe_var: probe_in_femur
     - name: Tibia
       trans_var: tibia_in_tracker_denoised
       stl_file: ./models/Tibia_1.stl
-      registration:
-        collect_obj: point_picker_in_tibia_ref
-        collect_var: picked_point_in_tibia_ref
-        target_var: tibia_in_tibia_ref
-        probe_var: probe_in_tibia
     - name: TibiaCut
       trans_var: tibia_in_tracker_denoised
-      stl_file: ./models/Tibia_1_AfterClip.STL
+      stl_file: ./models/Tibia_1_AfterClip.STL
+  registration:
+    - name: Femur
+      collect_obj: point_picker_in_femur_ref
+      collect_var: picked_point_in_femur_ref
+      target_var: femur_in_femur_ref
+      probe_var: probe_in_femur
+    - name: Tibia
+      collect_obj: point_picker_in_tibia_ref
+      collect_var: picked_point_in_tibia_ref
+      target_var: tibia_in_tibia_ref
+      probe_var: probe_in_tibia
+
+render:
+  shader_dir: /home/tpx/project/RemoteAR3/src/render/shader

data/config_dental.yaml

@@ -0,0 +1,72 @@
+camera:
+  names:
+    left: LeftEye
+    right: RightEye
+  intrinsic:
+    left:
+      fx: 3565.41112619664
+      fy: 3564.05573194168
+      cx: 1233.69545638269
+      cy: 1025.51713185245
+      k0: -0.0520040229813935
+      k1: 0.0398974773143781
+      width: 2448
+      height: 2048
+    right:
+      fx: 3571.67418317616
+      fy: 3570.36935016201
+      cx: 1222.10981440059
+      cy: 1034.19592532512
+      k0: -0.0760010312471698
+      k1: 0.196624456130940
+      width: 2448
+      height: 2048
+  stereo_trans:
+    tx: -65.3265350213620
+    ty: 0.245313839615459
+    tz: -0.508569900882309
+    qw: 0.999987680292989
+    qx: -0.000428742826329354
+    qy: -0.00395921570373319
+    qz: -0.00296311539549921
+  capture:
+    frame_rate: 40
+    expo_time_ms: 12
+    gain_db: 20
+
+main_window:
+  width: 800
+  height: 600
+
+opengl:
+  shader_dir: /home/tpx/project/RemoteAR3/src/render/shader
+
+output:
+  width: 1920
+  height: 1080
+  hevc_bitrate: 5 # Mbps
+  jpeg_quality: 15
+
+sender:
+  mtu: 1400
+  port: 5279
+  parity: 0.2
+
+sophiar:
+  config: ./sophiar_config.json
+  left_camera_trans_var: left_camera_in_tracker_denoised
+  right_camera_trans_var: right_camera_in_tracker_denoised
+  probe_model: ./models/Probe.stl
+  augment:
+    - name: Bone
+      trans_var: femur_in_tracker_denoised
+      model_file: /home/tpx/project/TransparentAR/data/E1_AR/E1_Model.obj
+      registration:
+        collect_obj: point_picker_in_femur_ref
+        collect_var: picked_point_in_femur_ref
+        target_var: femur_in_femur_ref
+        probe_var: probe_in_femur
+    - name: Plan
+      trans_var: femur_in_tracker_denoised
+      model_file: /home/tpx/project/TransparentAR/data/E1_AR/E1_Implant.obj
+      background: Bone

data/sophiar_config.json

@@ -14,7 +14,16 @@
     },
     {
       "name": "femur_in_femur_ref",
-      "type": "transform_obj"
+      "type": "transform_obj",
+      "value": [
+        34.34018126820815,
+        -44.83303063330605,
+        24.112401735811442,
+        0.7008713002927913,
+        -0.15030845884955477,
+        0.6856915303702952,
+        0.12654608963776195
+      ]
     },
     {
       "name": "tibia_ref_in_tracker",

src/components/registration.cpp

@@ -15,6 +15,8 @@
 #include <queue>
 #include <vector>
 
+using namespace vtk_viewer_helper;
+
 struct registration::impl {
 
     static constexpr auto MIN_REG_POINTS = 3;
@@ -113,7 +115,7 @@ struct registration::impl {
     void add_target(const registration_target &conf) {
         auto &target = targets.emplace_back();
         target.name = conf.name;
-        target.model = load_stl(conf.model_path);
+        target.model = load_any(conf.model_path);
         target.model_actor = create_actor(target.model);
         target.model_locator = vtkSmartPointer<vtkCellLocator>::New();
         target.model_locator->SetDataSet(target.model);

src/experiment/augment_accuracy.cpp

@@ -12,6 +12,7 @@
 #include <queue>
 
 using namespace sophiar;
+using namespace vtk_viewer_helper;
 
 bool enable_augment_accuracy = false;
 vis_marker_detector::options detect_opt;
@@ -31,7 +32,7 @@ extern int preview_camera_index;
 extern stereo_camera_info stereo_info;
 extern std::unique_ptr<camera_related> left;
 extern std::unique_ptr<camera_related> right;
-extern std::unique_ptr<vtk_viewer> augment_viewer;
+extern std::unique_ptr<scene_render> augment_viewer;
 extern std::queue<std::function<void()>> simple_eq;
 extern std::queue<std::function<void()>> close_funcs;
 extern local_connection sophiar_conn;
@@ -48,8 +49,8 @@ void initialize_augment_accuracy() { // TODO, make this module a class
         augment_marker_points.add_point(p.cast<double>());
     }
     augment_marker_points.get_actor()->GetProperty()->SetColor(1, 0, 0);
-    augment_viewer->add_actor(real_marker_points.get_actor());
-    augment_viewer->add_actor(augment_marker_points.get_actor());
+//    augment_viewer->add_actor(real_marker_points.get_actor()); // TODO: fix this
+//    augment_viewer->add_actor(augment_marker_points.get_actor());
 
     marker_detector = std::make_unique<vis_marker_detector>();
     close_funcs.emplace([&] { marker_detector.reset(); });

src/experiment/probe_p2p.cpp

@@ -9,9 +9,10 @@
 #include <queue>
 
 using namespace sophiar;
+using namespace vtk_viewer_helper;
 
 extern local_connection sophiar_conn;
-extern std::unique_ptr<vtk_viewer> augment_viewer;
+std::unique_ptr<scene_render> augment_viewer;
 extern std::unique_ptr<camera_related> left;
 
 struct probe_p2p::impl {
@@ -36,8 +37,8 @@ struct probe_p2p::impl {
         auto ret = new impl{};
         ret->real_probe_actor = create_actor(conf.probe_model_path);
         ret->augment_probe_actor = create_actor(conf.probe_model_path);
-        augment_viewer->add_actor(ret->real_probe_actor);
-        augment_viewer->add_actor(ret->augment_probe_actor);
+//        augment_viewer->add_actor(ret->real_probe_actor); // TODO: fix this
+//        augment_viewer->add_actor(ret->augment_probe_actor);
         ret->probe_var_name = conf.probe_var_name;
         return ret;
     }

src/frame_sender/sender_udp_fec.cpp

@@ -328,7 +328,7 @@ struct sender_udp_fec::impl {
         ret->q_this = q_this;
         ret->socket = std::make_unique<udp::socket>(*q_this->get_ctx(), local_ep);
         ret->socket->set_option(udp::socket::send_buffer_size{udp_buffer_size});
-        ret->hole_punch();
+//        ret->hole_punch();
         ret->async_handle_request();
 
         // constant configs

src/imgui_utility.cpp

@@ -1,13 +1,29 @@
 #include "imgui_utility.h"
 
+#include <iostream>
+
 static constexpr auto nan_v = std::numeric_limits<float>::quiet_NaN();
 static float nan_arr[] = {nan_v, nan_v, nan_v, nan_v};
 static constexpr auto nan_arr_len = sizeof(nan_arr) / sizeof(nan_arr[0]);
 static auto empty_color = (ImVec4) ImColor::HSV(0, 1, 1);
 
+namespace imgui_utility_impl {
+
+    auto matrix_format = Eigen::IOFormat{Eigen::FullPrecision, 0, ", ", ",\n", "[", "]", "[", "]"};
+
+    void print_variable(const Eigen::Isometry3f &transform) {
+        // TODO: make print better
+        std::cout << transform.matrix().format(matrix_format) << std::endl;
+    }
+
+}
+
+using namespace imgui_utility_impl;
+
 void show_variable(const std::string &name,
                    const std::optional<Eigen::Isometry3f> &transform) {
-    auto guard = imgui_disable_guard{};
+    auto id_guard = imgui_id_guard{name.c_str()};
+    auto disable_guard = imgui_disable_guard{};
 
     auto trans_name = fmt::format("{} (T3f_t)", name);
     auto rot_name = fmt::format("{} (T3f_r)", name);
@@ -29,6 +45,12 @@ void show_variable(const std::string &name,
         };
         ImGui::InputFloat3(trans_name.c_str(), trans_val);
         ImGui::InputFloat4(rot_name.c_str(), rot_val);
+
+        disable_guard.cancel();
+        ImGui::SameLine();
+        if (ImGui::Button("Print")) {
+            print_variable(transform.value());
+        }
     }
 }
 

src/imgui_utility.h

@@ -23,10 +23,25 @@ struct imgui_disable_guard {
         }
     }
 
+    void cancel() {
+        ImGui::EndDisabled();
+        is_disabled = false;
+    }
+
 private:
     bool is_disabled;
 };
 
+struct imgui_id_guard {
+    explicit imgui_id_guard(const char *name) {
+        ImGui::PushID(name);
+    }
+
+    ~imgui_id_guard() {
+        ImGui::PopID();
+    }
+};
+
 void show_variable(const std::string &name,
                    const std::optional<Eigen::Isometry3f> &transform);
 

src/main.cpp

@@ -1,18 +1,12 @@
-#include "simple_mq.h"
 #include "utility.hpp"
-#include "variable_defs.h"
 
 #include <spdlog/spdlog.h>
 
 #include <GLFW/glfw3.h>
 
-#include <cassert>
-
-using namespace simple_mq_singleton;
-
 extern GLFWwindow *main_window;
 
-void initialize_main();
+void initialize_main(int argc, char *argv[]);
 
 void cleanup();
 
@@ -32,14 +26,14 @@ void render_main_window();
 
 void generate_output_frame();
 
-int main() {
+int main(int argc, char *argv[]) {
 
 #ifndef NDEBUG
     spdlog::set_level(spdlog::level::trace);
 #endif
 
     // initialize many staffs
-    initialize_main();
+    initialize_main(argc, argv);
 
     while (!glfwWindowShouldClose(main_window)) {
 

src/main_ext.cpp

@@ -2,6 +2,7 @@
 #include "core/local_connection.h"
 #include "cuda_helper.hpp"
 #include "experiment/probe_p2p.h"
+#include "imgui_utility.h"
 #include "main_impl/impl_types.h"
 #include "simple_mq.h"
 #include "utility.hpp"
@@ -28,11 +29,12 @@
 #include <imgui_impl_glfw.h>
 #include <imgui_impl_opengl3.h>
 
-#include <boost/iostreams/device/mapped_file.hpp>
+#include <cxxopts.hpp>
 
-#include <fstream>
+#include <boost/iostreams/device/mapped_file.hpp>
 
 #include <cassert>
+#include <fstream>
 #include <thread>
 #include <queue>
 #include <vector>
@@ -57,6 +59,7 @@ extern bool mono_mode;
 
 // render related
 extern bool enable_augment_without_nav;
+extern std::unique_ptr<scene_render> augment_render;
 extern std::vector<augment_store_type> augment_items;
 
 std::string sophiar_config_path;
@@ -82,8 +85,6 @@ std::queue<std::function<void()>> close_funcs;
 std::unique_ptr<camera_related> left;
 std::unique_ptr<camera_related> right;
 
-constexpr auto config_path = "./config.yaml";
-
 void initialize_render();
 
 void initialize_augment_accuracy();
@@ -153,8 +154,9 @@ void load_encoder_config(YAML::Node conf);
 
 void load_sender_config(YAML::Node conf);
 
-void load_config() {
+void load_config(const std::string &config_path) {
     auto conf = YAML::LoadFile(config_path);
+    SPDLOG_INFO("Use config file: {}", config_path);
 
     // initialize mq
     mq();
@@ -164,6 +166,11 @@ void load_config() {
     load_encoder_config(conf["output"]);
     load_sender_config(conf["sender"]);
 
+    // load render configs
+    auto opengl_conf = conf["opengl"];
+    auto shader_dir = opengl_conf["shader_dir"].as<std::string>();
+    mq().update_variable(SHADER_DIR, shader_dir);
+
     // load main window config
     auto window_conf = conf["main_window"];
     main_window_width = window_conf["width"].as<int>();
@@ -178,14 +185,18 @@ void load_config() {
 
     // load augment items
     for (auto item: sophiar_conf["augment"]) {
-        augment_items.emplace_back(
-                item["name"].as<std::string>(),
-                item["trans_var"].as<std::string>(),
-                create_actor(item["stl_file"].as<std::string>()));
+        auto &store = augment_items.emplace_back();
+        store.name = item["name"].as<std::string>();
+        store.trans_var = item["trans_var"].as<std::string>();
+        store.model_path = item["model_file"].as<std::string>();
+        if (auto bg_conf = item["background"]; bg_conf) {
+            store.background = bg_conf.as<std::string>();
+        } else {
+            store.background = {};
+        }
         if (auto reg_conf = item["registration"]; reg_conf) {
             reg_targets.emplace_back(
-                    item["name"].as<std::string>(),
-                    item["stl_file"].as<std::string>(),
+                    store.name, store.model_path,
                     std::vector<Eigen::Vector3d>{},
                     reg_conf["target_var"].as<std::string>(),
                     reg_conf["collect_var"].as<std::string>(),
@@ -200,7 +211,14 @@ void load_config() {
     mq().update_variable(ENCODER_SHOULD_RESET, false);
 }
 
-void initialize_main() {
+void initialize_main(int argc, char *argv[]) {
+
+    // parse arguments
+    auto options = cxxopts::Options{"RemoteAR3", "Capture surgery scene and do augmentation works."};
+    options.add_options()
+            ("c,config", "Config file", cxxopts::value<std::string>()->default_value("config.yaml"));
+    auto opts = options.parse(argc, argv);
+
     // initialize cuda
     constexpr auto default_cuda_device_id = 0;
     cuInit(0);
@@ -218,7 +236,7 @@ void initialize_main() {
     left = std::make_unique<camera_related>();
     right = std::make_unique<camera_related>();
 
-    load_config();
+    load_config(opts["config"].as<std::string>());
     initialize_main_window();
 
     // initialize modules
@@ -244,11 +262,11 @@ void initialize_main() {
     // initialize vtk test viewer
     vtk_test1 = std::make_unique<vtk_viewer>();
     vtk_test2 = std::make_unique<vtk_viewer>();
-    vtk_test1->clear_actor();
-    vtk_test1->add_actor(create_actor("./models/femur.stl"));
-    vtk_test1->reset_camera();
-    vtk_test2->add_actor(create_actor("./models/tibia.stl"));
-    vtk_test1->start_picking();
+//    vtk_test1->clear_actor(); // TODO: remove them
+//    vtk_test1->add_actor(create_actor("./models/femur.stl"));
+//    vtk_test1->reset_camera();
+//    vtk_test2->add_actor(create_actor("./models/tibia.stl"));
+//    vtk_test1->start_picking();
 }
 
 void start_tracking() {

src/main_impl/camera_related.cpp

@@ -28,8 +28,7 @@ struct simple_image_saver {
 
     void save_image_png(smart_mat *left, smart_mat *right) const;
 
-    void save_image_png(const smart_texture &left,
-                        const smart_texture &right) const;
+    void save_texture_png(GLuint left_tex, GLuint right_tex) const;
 
 private:
     auto get_save_path(std::string_view ext) const {
@@ -63,11 +62,11 @@ private:
         cv::imwrite(file_path, out_img);
     }
 
-    static void save_tex(const std::string &file_path, const smart_texture &tex) {
+    static void save_tex(const std::string &file_path, GLuint tex) {
         // download texture
         GLenum format;
         GLsizei width, height;
-        glBindTexture(GL_TEXTURE_2D, tex.id);
+        glBindTexture(GL_TEXTURE_2D, tex);
         glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_INTERNAL_FORMAT, (int *) &format);
         glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_WIDTH, &width);
         glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &height);
@@ -75,18 +74,20 @@ private:
             glBindTexture(GL_TEXTURE_2D, 0);
         });
 
+        cv::Mat img;
         switch (format) {
             case GL_RGBA8: {
-                auto img = cv::Mat(height, width, CV_8UC4);
+                img.create(height, width, CV_8UC4);
                 assert(img.isContinuous());
                 glGetTexImage(GL_TEXTURE_2D, 0, GL_BGRA, GL_UNSIGNED_BYTE, img.data);
-                save_png(file_path, img);
-                return;
+                break;
             }
             default: {
                 RET_ERROR;
             }
         }
+        cv::flip(img, img, 0); // flip along Y-axis
+        save_png(file_path, img);
     }
 };
 
@@ -106,11 +107,10 @@ void simple_image_saver::save_image_png(smart_mat *left, smart_mat *right) const
     save_png(right_path, right->host());
 }
 
-void simple_image_saver::save_image_png(const smart_texture &left,
-                                        const smart_texture &right) const {
+void simple_image_saver::save_texture_png(GLuint left_tex, GLuint right_tex) const {
     auto [left_path, right_path] = get_save_path(".t.png");
-    save_tex(left_path, left);
-    save_tex(right_path, right);
+    save_tex(left_path, left_tex);
+    save_tex(right_path, right_tex);
 }
 
 static constexpr auto CAPTURE_RAW_BIT = 1 << 0;
@@ -329,7 +329,8 @@ void process_camera_frames() {
         image_saver->save_image_png(&left->img_mat, &right->img_mat);
     }
     if (capture_flag & CAPTURE_AUGMENT_BIT) {
-        image_saver->save_image_png(left->augment_tex, right->augment_tex);
+        image_saver->save_texture_png(left->augment_fbo.color_tex(),
+                                      right->augment_fbo.color_tex());
     }
 
     // reset capture flag

src/main_impl/impl_types.h

@@ -3,9 +3,11 @@
 
 #include "image_process.h"
 #include "image_utility.hpp"
-#include "simple_opengl.h"
+#include "render/mesh_render.h"
+#include "render/render_utility.h"
+#include "render/scene_render.h"
+#include "render/texture_render.h"
 #include "simple_mq.h"
-#include "vtk_viewer.h"
 
 #include <yaml-cpp/yaml.h>
 
@@ -23,7 +25,7 @@ struct camera_related {
     monocular_processor processor;
 
     bool augment_available = false;
-    smart_texture augment_tex;
+    smart_frame_buffer augment_fbo;
 
     void load_intrinsic(YAML::Node conf);
 
@@ -39,8 +41,16 @@ struct camera_related {
 
 struct augment_store_type {
     std::string name, trans_var;
-    vtkSmartPointer<vtkActor> actor;
-    int opacity = 100;
+    std::string model_path;
+    std::optional<std::string> background;
+    scene_render::actor_token_type actor;
+
+    float color[3] = {1.0f, 0.0f, 0.0f};
+    float ambient_factor = 0.5f;
+    float alpha_factor = 0.25f;
+    bool enable = true;
+
+    Eigen::Vector3f extra_offset = {};
 };
 
 #endif //REMOTEAR3_MAIN_IMPL_TYPES_H

src/main_impl/render_related.cpp

@@ -1,15 +1,17 @@
 #include "core/local_connection.h"
 #include "core/timestamp_helper.hpp"
-#include "impl_types.h"
 #include "simple_mq.h"
 #include "third_party/scope_guard.hpp"
 #include "variable_defs.h"
 
-#include <GLFW/glfw3.h>
-
+#include <imGuIZMOquat.h>
 #include <imgui_impl_opengl3.h>
+#include "impl_types.h" // must include after imGuiZMOquat.h
+#include "imgui_utility.h"
 
-#include <vtkProperty.h>
+#include <GLFW/glfw3.h>
+
+#include <glm/gtc/type_ptr.hpp>
 
 #include <queue>
 
@@ -22,12 +24,18 @@ float augment_render_angle = 0;
 bool output_full_frame = false;
 bool output_halve_width = false;
 
-std::unique_ptr<simple_render> opengl_render;
+std::unique_ptr<texture_render> opengl_render;
 std::unique_ptr<smart_cuda_stream> output_stream;
 std::shared_ptr<cv::cuda::GpuMat> output_frame_dev;
 std::unique_ptr<smart_frame_buffer> output_fbo;
 
-std::unique_ptr<vtk_viewer> augment_viewer;
+bool enable_light_follow_camera = true;
+Eigen::Vector3f camera_extra_offset = {};
+Eigen::Vector3f camera_extra_rotation = {};
+vgm::Vec3 light_direction = {1.0f, 0.0f, 0.0f};
+float augment_opacity = 1.0f;
+float camera_near = 1.0f, camera_far = 1000.0f;
+std::unique_ptr<scene_render> augment_render;
 std::vector<augment_store_type> augment_items;
 
 extern GLFWwindow *main_window;
@@ -39,6 +47,15 @@ extern std::unique_ptr<camera_related> left;
 extern std::unique_ptr<camera_related> right;
 extern std::queue<std::function<void()>> close_funcs;
 
+Eigen::Isometry3f transform_combine_eigen_xyz_deg(
+        const Eigen::Vector3f &pos, const Eigen::Vector3f &rot_angle) {
+    auto rotation = Eigen::AngleAxisf{glm::radians(rot_angle.x()), Eigen::Vector3f::UnitX()}
+                    * Eigen::AngleAxisf{glm::radians(rot_angle.y()), Eigen::Vector3f::UnitY()}
+                    * Eigen::AngleAxisf{glm::radians(rot_angle.z()), Eigen::Vector3f::UnitZ()};
+    auto translation = Eigen::Translation3f{pos};
+    return translation * rotation;
+}
+
 void camera_related::process_augment() {
     if (augment_enable) {
         auto trans = sophiar_conn.query_transform_variable(trans_var);
@@ -48,13 +65,13 @@ void camera_related::process_augment() {
         augment_available = trans.has_value() || enable_augment_without_nav;
         if (augment_available) {
             auto img_size = img_mat.size();
-            augment_viewer->set_camera_pose(transform);
-            augment_viewer->render(img_size);
-
-            // copy rendered image
-            augment_tex.create(GL_RGBA8, img_size);
-            glCopyImageSubData(augment_viewer->get_tex(), GL_TEXTURE_2D, 0, 0, 0, 0,
-                               augment_tex.id, GL_TEXTURE_2D, 0, 0, 0, 0, img_size.width, img_size.height, 1);
+            augment_fbo.create(img_size);
+            augment_fbo.bind();
+            auto extra_transform = transform_combine_eigen_xyz_deg(camera_extra_offset, camera_extra_rotation);
+            augment_render->set_camera_transform(transform.cast<float>() * extra_transform);
+            augment_render->set_camera_clip_range(camera_near, camera_far);
+            augment_render->render();
+            augment_fbo.unbind();
         }
     } else {
         augment_available = false;
@@ -63,9 +80,9 @@ void camera_related::process_augment() {
 
 void camera_related::render(const simple_rect &rect, bool flip_y) {
     assert(!img_mat.empty());
-    opengl_render->render_rect(img_mat.dev(stream.cv), rect, !flip_y, stream.cuda);
+    opengl_render->render(img_mat.dev(stream.cv), 1.0f, rect, !flip_y, stream.cuda);
     if (augment_available) {
-        opengl_render->render_rect(augment_tex.id, rect, flip_y);
+        opengl_render->render(augment_fbo.color_tex(), augment_opacity, rect, flip_y);
     }
 }
 
@@ -75,21 +92,39 @@ void start_encoder();
 
 void stop_encoder();
 
+scene_render::actor_token_type find_augment_actor(const std::string &name) {
+    for (auto &item: augment_items) { // find background item
+        if (item.name == name) {
+            assert(item.actor != nullptr);
+            return item.actor;
+        }
+    }
+    SPDLOG_ERROR("Augment actor {} is not found.", name);
+    return nullptr;
+}
+
 void initialize_render() {
-    opengl_render = std::make_unique<simple_render>();
+    opengl_render = std::make_unique<texture_render>();
     output_stream = std::make_unique<smart_cuda_stream>();
     output_frame_dev = std::make_shared<cv::cuda::GpuMat>();
     output_fbo = std::make_unique<smart_frame_buffer>();
 
-    augment_viewer = std::make_unique<vtk_viewer>();
-    augment_viewer->set_camera_view_angle(augment_render_angle);
+    augment_render = std::make_unique<scene_render>();
+    augment_render->set_camera_fov(augment_render_angle);
+
+    // load augment models
     for (auto &item: augment_items) {
-        item.actor->GetProperty()->SetColor(1, 0, 0); // TODO: make color configurable
-        augment_viewer->add_actor(item.actor);
+        if (item.background.has_value()) {
+            item.actor = augment_render->create_actor(
+                    item.model_path.c_str(), find_augment_actor(item.background.value()));
+        } else {
+            item.actor = augment_render->create_actor(item.model_path.c_str());
+        }
     }
 
     close_funcs.emplace([&] {
         opengl_render.reset();
+        augment_render.reset();
         output_fbo.reset();
     });
 }
@@ -99,7 +134,7 @@ void render_main_window() {
     glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
     glfwGetFramebufferSize(main_window, &frame_size.width, &frame_size.height);
     glViewport(0, 0, frame_size.width, frame_size.height);
-    glClear(GL_COLOR_BUFFER_BIT);
+    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
 
     if (is_capturing()) {
         // draw preview camera frame
@@ -174,7 +209,19 @@ void generate_output_frame() {
 void prepare_augment_info() {
     for (auto &item: augment_items) {
         auto trans = sophiar_conn.query_transform_variable(item.trans_var);
-        update_actor_pose(item.actor, trans);
+        if (!trans.has_value()) { // TODO: add flag show_augment_when_actor_missing
+            augment_render->set_actor_visibility(item.actor, false);
+        } else {
+            augment_render->set_actor_visibility(item.actor, item.enable);
+            auto extra_transform = transform_combine_eigen_xyz_deg(item.extra_offset, Eigen::Vector3f::Zero());
+            augment_render->set_actor_transform(item.actor, trans.value().cast<float>() * extra_transform);
+            augment_render->set_actor_color(item.actor, item.color, item.ambient_factor);
+            augment_render->set_actor_alpha_factor(item.actor, item.alpha_factor);
+        }
+    }
+    if (!enable_light_follow_camera) {
+        auto direction_eigen = Eigen::Vector3f{(float *) light_direction};
+        augment_render->set_light_direction(direction_eigen);
     }
 }
 
@@ -183,10 +230,35 @@ void show_augment_ui() {
     ImGui::Checkbox("Enable", &augment_enable);
     if (augment_enable) {
         for (auto &item: augment_items) {
-            auto opacity_name = fmt::format("{} Opacity (%)", item.name);
-            if (ImGui::DragInt(opacity_name.c_str(), &item.opacity, 1, 0, 100)) {
-                item.actor->GetProperty()->SetOpacity(0.01 * item.opacity);
+            if (ImGui::TreeNode(item.name.c_str())) {
+                ImGui::Checkbox("Show", &item.enable);
+                ImGui::ColorEdit3("Color", item.color);
+                ImGui::DragFloat("Ambient Factor", &item.ambient_factor, 0.005f, 0.0f, 1.0f);
+                if (item.background.has_value()) {
+                    ImGui::DragFloat("Alpha Factor", &item.alpha_factor, 0.005f, 0.0f, 1.0f);
+                }
+                ImGui::DragFloat3("Offset (mm)", item.extra_offset.data(),
+                                  0.05f, 0.0f, 0.0f, "%.02f");
+                ImGui::TreePop();
             }
         }
+        if (ImGui::TreeNode("Light")) {
+            if (ImGui::Checkbox("Follow Camera", &enable_light_follow_camera)) {
+                augment_render->set_light_follow_camera(enable_light_follow_camera);
+            }
+            auto guard = imgui_disable_guard(enable_light_follow_camera);
+            ImGui::gizmo3D("##direction", light_direction, ImGui::CalcItemWidth());
+            ImGui::TreePop();
+        }
+        if (ImGui::TreeNode("Camera")) {
+            ImGui::DragFloat3("Offset (mm)", camera_extra_offset.data(),
+                              0.05f, 0.0f, 0.0f, "%.02f");
+            ImGui::DragFloat3("Offset (deg)", camera_extra_rotation.data(),
+                              0.1f, -180.0f, 180.0f, "%.01f");
+            ImGui::DragFloat("Opacity", &augment_opacity, 0.01f, 0.0f, 1.0f, "%.02f");
+            ImGui::SliderFloat("Clip Near", &camera_near, 1.0f, camera_far, "%.f", ImGuiSliderFlags_Logarithmic);
+            ImGui::SliderFloat("Clip Far", &camera_far, camera_near, 10000.0f, "%.f", ImGuiSliderFlags_Logarithmic);
+            ImGui::TreePop();
+        }
     }
 }

src/render/mesh_render.cpp

@@ -0,0 +1,296 @@
+#include "mesh_render.h"
+#include "render_utility.h"
+#include "simple_mq.h"
+#include "texture_render.h"
+#include "variable_defs.h"
+
+#include <vtkCellArray.h>
+#include <vtkFloatArray.h>
+#include <vtkNew.h>
+#include <vtkOBJReader.h>
+#include <vtkPointData.h>
+#include <vtkPoints.h>
+#include <vtkPolyData.h>
+#include <vtkPolyDataNormals.h>
+#include <vtkSTLReader.h>
+
+#include <glm/gtc/type_ptr.hpp>
+
+#include <spdlog/spdlog.h>
+
+#include <filesystem>
+
+using namespace simple_mq_singleton;
+
+struct mesh_type::impl {
+
+    struct vertex_info {
+        glm::vec3 position;
+        glm::vec3 normal;
+    };
+
+    size_t triangle_num;
+    smart_buffer<vertex_info> vbo_data;
+    smart_buffer<GLuint> ebo_data;
+
+    size_t modify_id = 0, last_upload_id = 0;
+    GLuint vao = 0, vbo = 0, ebo = 0;
+
+    void upload() {
+        if (last_upload_id == modify_id) [[likely]] return;
+        assert(last_upload_id < modify_id);
+
+        // delete old objects
+        glDeleteVertexArrays(1, &vao);
+        glDeleteBuffers(1, &vbo);
+        glDeleteBuffers(1, &ebo);
+
+        // config vertex buffer
+        glGenBuffers(1, &vbo);
+        glBindBuffer(GL_ARRAY_BUFFER, vbo);
+        glBufferData(GL_ARRAY_BUFFER, vbo_data.size(), vbo_data.ptr, GL_STATIC_DRAW);
+
+        // fill element buffer
+        glGenBuffers(1, &ebo);
+        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
+        glBufferData(GL_ELEMENT_ARRAY_BUFFER, ebo_data.size(), ebo_data.ptr, GL_STATIC_DRAW);
+
+        // config vertex array
+        glGenVertexArrays(1, &vao);
+        glBindVertexArray(vao);
+        glEnableVertexAttribArray(0);
+        glEnableVertexAttribArray(1);
+        glVertexAttribPointer(0, 3, GL_FLOAT, false, sizeof(vertex_info), (void *) offsetof(vertex_info, position));
+        glVertexAttribPointer(1, 3, GL_FLOAT, false, sizeof(vertex_info), (void *) offsetof(vertex_info, normal));
+
+        last_upload_id = modify_id;
+    }
+
+    void create_from_poly_data(vtkPolyData *poly) {
+        // calculate vertex normal if needed
+        vtkNew<vtkPolyDataNormals> filter;
+        if (poly->GetPointData()->GetNormals() == nullptr) {
+            filter->SetInputData(poly);
+            filter->Update();
+            poly = filter->GetOutput();
+        }
+
+        // copy vertex data
+        auto points = poly->GetPoints();
+        auto vertex_num = points->GetNumberOfPoints();
+        auto vertex_normal = (vtkFloatArray *) poly->GetPointData()->GetNormals();
+        assert(vertex_normal != nullptr);
+        assert(vertex_normal->GetNumberOfTuples() == vertex_num);
+        vbo_data.create(vertex_num);
+        for (auto k = 0; k < vertex_num; ++k) {
+            glm::dvec3 position, normal;
+            points->GetPoint(k, glm::value_ptr(position));
+            vertex_normal->GetTuple(k, glm::value_ptr(normal));
+            vbo_data.ptr[k].position = position;
+            vbo_data.ptr[k].normal = normal;
+        }
+
+        // copy index data
+        auto cells = poly->GetPolys();
+        triangle_num = cells->GetNumberOfCells();
+        ebo_data.create(triangle_num * 3);
+        vtkNew<vtkIdList> cell;
+        for (auto k = 0; k < triangle_num; ++k) {
+            auto ptr = (GLuint *) ebo_data.ptr + k * 3;
+            cells->GetCellAtId(k, cell);
+            assert(cell->GetNumberOfIds() == 3);
+            for (auto i = 0; i < 3; ++i) {
+                auto vertex_id = cell->GetId(i);
+                assert(vertex_id < vertex_num);
+                ptr[i] = vertex_id;
+            }
+        }
+
+        ++modify_id;
+    }
+
+    void draw() {
+        upload();
+        glBindVertexArray(vao);
+        glBindBuffer(GL_ARRAY_BUFFER, vbo);
+        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
+        glDrawElements(GL_TRIANGLES, 3 * triangle_num, GL_UNSIGNED_INT, nullptr);
+    }
+
+    static impl *from_reader(const char *path,
+                             vtkAbstractPolyDataReader *reader) {
+        reader->SetFileName(path);
+        reader->Update();
+        auto ret = new impl{};
+        ret->create_from_poly_data(reader->GetOutput());
+        return ret;
+    }
+
+    static mesh_type *mesh_type_from_reader(const char *path,
+                                            vtkAbstractPolyDataReader *reader) {
+        auto ret = new mesh_type;
+        ret->pimpl.reset(
+                mesh_type::impl::from_reader(path, reader));
+        return ret;
+    }
+
+};
+
+mesh_type::mesh_type() = default;
+
+mesh_type::~mesh_type() = default;
+
+mesh_type *mesh_type::from_stl(const char *path) {
+    vtkNew<vtkSTLReader> reader;
+    return impl::mesh_type_from_reader(path, reader);
+}
+
+mesh_type *mesh_type::from_obj(const char *path) {
+    vtkNew<vtkOBJReader> reader;
+    return impl::mesh_type_from_reader(path, reader);
+}
+
+mesh_type *mesh_type::from_any(const char *path) {
+    auto path_fs = std::filesystem::path{path};
+    auto ext = path_fs.extension();
+    if (ext == ".stl") {
+        return from_stl(path);
+    } else if (ext == ".obj") {
+        return from_obj(path);
+    }
+    RET_ERROR_P;
+}
+
+namespace mesh_render_impl {
+
+    void ensure_program(smart_program *program) {
+#ifndef NDEBUG
+        GLuint program_id;
+        glGetIntegerv(GL_CURRENT_PROGRAM, (GLint *) &program_id);
+        assert(program_id == program->id);
+#endif
+    }
+
+    void upload_scene_property(smart_program *program,
+                               const scene_property &scene_info) {
+        ensure_program(program);
+        program->set_uniform_mat4("camera_mat", scene_info.camera.projection
+                                                * scene_info.camera.transform);
+        program->set_uniform_vec3("light.direction", scene_info.light.direction);
+    }
+
+    void upload_mesh_property(smart_program *program,
+                              const mesh_property &mesh_info) {
+        ensure_program(program);
+        program->set_uniform_mat4("model_mat", mesh_info.transform);
+        program->set_uniform_vec3("material.ambient", mesh_info.material.ambient);
+        program->set_uniform_vec3("material.diffuse", mesh_info.material.diffuse);
+    }
+
+}
+
+using namespace mesh_render_impl;
+
+struct mesh_render::impl {
+
+    std::unique_ptr<smart_program> program_basic;
+    std::unique_ptr<smart_program> program_mask;
+
+    smart_frame_buffer bg_frame;
+    smart_frame_buffer model_frame;
+    smart_frame_buffer output_frame;
+
+    texture_render tex_render;
+
+    impl() {
+        std::filesystem::path shader_folder = mq().query_variable<std::string>(SHADER_DIR);
+
+        auto vert_basic = shader_folder / "mesh_render.vert";
+        auto frag_basic = shader_folder / "mesh_render.frag";
+        program_basic.reset(smart_program::create(vert_basic.c_str(),
+                                                  frag_basic.c_str()));
+
+        auto vert_mask = shader_folder / "mesh_render_mask.vert";
+        auto frag_mask = shader_folder / "mesh_render_mask.frag";
+        program_mask.reset(smart_program::create(vert_mask.c_str(),
+                                                 frag_mask.c_str()));
+    }
+
+    void render(mesh_type *mesh,
+                const mesh_property &mesh_info,
+                const scene_property &scene_info) {
+        program_basic->use();
+        upload_mesh_property(program_basic.get(), mesh_info);
+        upload_scene_property(program_basic.get(), scene_info);
+        glEnable(GL_DEPTH_TEST);
+        mesh->pimpl->draw();
+    }
+
+    void render(mesh_type *model, const mesh_property &model_info,
+                mesh_type *bg_mesh, const mesh_property &bg_info,
+                const scene_property &scene_info,
+                float alpha_factor, bool show_bg) {
+
+        // config frame buffers
+        GLint viewport[4]; // x, y, width, height
+        glGetIntegerv(GL_VIEWPORT, viewport);
+        auto view_size = cv::Size{viewport[2], viewport[3]};
+        bg_frame.create(view_size);
+        model_frame.create(view_size);
+        output_frame.create(view_size);
+
+        // render background
+        bg_frame.bind();
+        if (show_bg) {
+            render(bg_mesh, bg_info, scene_info);
+        } else { // use a simpler program
+            auto bg_mat = scene_info.camera.projection
+                          * scene_info.camera.transform
+                          * bg_info.transform;
+            program_mask->use();
+            program_mask->set_uniform_mat4("transform_mat", bg_mat);
+            glEnable(GL_DEPTH_TEST);
+            bg_mesh->pimpl->draw();
+        }
+        bg_frame.unbind();
+
+        // render model
+        model_frame.bind();
+        render(model, model_info, scene_info);
+        model_frame.unbind();
+
+        // render output frame
+        output_frame.bind();
+        if (show_bg) {
+            tex_render.render_with_depth(bg_frame.color_tex(), bg_frame.depth_tex());
+        }
+        tex_render.render_depth_alpha(model_frame.color_tex(), model_frame.depth_tex(), bg_frame.depth_tex(),
+                                      scene_info.camera.projection, alpha_factor);
+        tex_render.render_depth_only(model_frame.depth_tex());
+        output_frame.unbind();
+
+        // draw output frame
+        tex_render.render_with_depth(output_frame.color_tex(), output_frame.depth_tex());
+    }
+
+};
+
+mesh_render::mesh_render()
+        : pimpl(std::make_unique<impl>()) {}
+
+mesh_render::~mesh_render() = default;
+
+void mesh_render::render(mesh_type *mesh,
+                         const mesh_property &mesh_info,
+                         const scene_property &scene_info) {
+    pimpl->render(mesh, mesh_info, scene_info);
+}
+
+void
+mesh_render::render(mesh_type *model, const mesh_property &model_info,
+                    mesh_type *bg_mesh, const mesh_property &bg_info,
+                    const scene_property &scene_info,
+                    float alpha_factor, bool show_bg) {
+    pimpl->render(model, model_info, bg_mesh, bg_info,
+                  scene_info, alpha_factor, show_bg);
+}

src/render/mesh_render.h

@@ -0,0 +1,76 @@
+#ifndef TRANSPARENTAR_MESH_RENDER_H
+#define TRANSPARENTAR_MESH_RENDER_H
+
+#include "utility.hpp"
+
+#include <glad/gl.h>
+#include <glm/glm.hpp>
+
+#include <opencv2/core/types.hpp>
+
+#include <memory>
+
+struct mesh_type {
+public:
+    ~mesh_type();
+
+    static mesh_type *from_stl(const char *path);
+
+    static mesh_type *from_obj(const char *path);
+
+    static mesh_type *from_any(const char *path);
+
+private:
+    mesh_type();
+
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+
+    friend class mesh_render;
+};
+
+struct material_type {
+    glm::vec3 ambient;
+    glm::vec3 diffuse;
+};
+
+struct mesh_property {
+    glm::mat4 transform;
+    material_type material;
+};
+
+struct camera_property {
+    glm::mat4 transform;
+    glm::mat4 projection;
+};
+
+struct light_property {
+    glm::vec3 direction; // simulate daylight
+};
+
+struct scene_property {
+    camera_property camera;
+    light_property light;
+};
+
+class mesh_render {
+public:
+    mesh_render();
+
+    ~mesh_render();
+
+    void render(mesh_type *mesh,
+                const mesh_property &mesh_info,
+                const scene_property &scene_info);
+
+    void render(mesh_type *model, const mesh_property &model_info,
+                mesh_type *bg_mesh, const mesh_property &bg_info,
+                const scene_property &scene_info,
+                float alpha_factor = 0.25f, bool show_bg = false);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#endif //TRANSPARENTAR_MESH_RENDER_H

src/render/render_utility.cpp

@@ -0,0 +1,372 @@
+#include "render_utility.h"
+
+#include <cuda_gl_interop.h>
+
+#include <glm/gtc/type_ptr.hpp>
+
+#include <fmt/format.h>
+#include <spdlog/spdlog.h>
+
+#include <boost/iostreams/device/mapped_file.hpp>
+
+using boost::iostreams::mapped_file;
+
+struct smart_texture::impl {
+
+    smart_texture *q_this = nullptr;
+    cv::Size last_size = {};
+
+    void create(GLenum format, cv::Size size, GLint min_filter, GLint max_filter) {
+        if (size == last_size) [[likely]] return;
+        deallocate();
+        allocate(format, size, min_filter, max_filter);
+    }
+
+    void allocate(GLenum format, cv::Size size,
+                  GLint min_filter, GLint max_filter) {
+        glGenTextures(1, &q_this->id);
+        glBindTexture(GL_TEXTURE_2D, q_this->id);
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, min_filter);
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, max_filter);
+        glTexStorage2D(GL_TEXTURE_2D, 1, format, size.width, size.height);
+        glBindTexture(GL_TEXTURE_2D, 0);
+        last_size = size;
+    }
+
+    void deallocate() {
+        if (q_this->id == 0) return;
+        glDeleteTextures(1, &q_this->id);
+        q_this->id = 0;
+        last_size = {};
+    }
+
+};
+
+smart_texture::smart_texture()
+        : pimpl(std::make_unique<impl>()) {
+    pimpl->q_this = this;
+}
+
+smart_texture::~smart_texture() = default;
+
+void smart_texture::create(GLenum format, cv::Size size, GLint min_filter, GLint max_filter) {
+    pimpl->create(format, size, min_filter, max_filter);
+}
+
+cv::Size smart_texture::size() const {
+    return pimpl->last_size;
+}
+
+struct smart_pixel_buffer::impl {
+
+    smart_pixel_buffer *q_this = nullptr;
+    cudaGraphicsResource *res = nullptr;
+    GLsizeiptr last_size = 0;
+    cudaStream_t last_stream = nullptr;
+
+    void allocate(GLenum target, GLenum flags, GLsizeiptr size) {
+        auto &qid = q_this->id;
+        glGenBuffers(1, &qid);
+        glBindBuffer(target, qid);
+        glBufferStorage(target, size, nullptr, flags);
+        glBindBuffer(target, 0);
+        last_size = size;
+
+        // register CUDA resource
+        if (target == GL_PIXEL_UNPACK_BUFFER) {
+            CUDA_API_CHECK(cudaGraphicsGLRegisterBuffer(
+                    &res, qid, cudaGraphicsRegisterFlagsWriteDiscard));
+        } else {
+            assert(target == GL_PIXEL_PACK_BUFFER);
+            CUDA_API_CHECK(cudaGraphicsGLRegisterBuffer(
+                    &res, qid, cudaGraphicsRegisterFlagsReadOnly));
+        }
+    }
+
+    void deallocate() {
+        auto &qid = q_this->id;
+        if (qid == 0) return;
+        glDeleteBuffers(1, &qid);
+        qid = 0;
+        last_size = 0;
+
+        // unregister CUDA resource
+        CUDA_API_CHECK(cudaGraphicsUnregisterResource(res));
+        res = nullptr;
+    }
+
+    void create(GLenum target, GLenum flags, GLsizeiptr size) {
+        if (size == last_size) [[likely]] return;
+        deallocate();
+        allocate(target, flags, size);
+    }
+
+    void *map_pointer(cudaStream_t stream) {
+        void *ptr;
+        size_t size;
+        CUDA_API_CHECK_P(cudaGraphicsMapResources(1, &res, stream));
+        CUDA_API_CHECK_P(cudaGraphicsResourceGetMappedPointer(&ptr, &size, res));
+        assert(size == last_size);
+        last_stream = stream;
+        return ptr;
+    }
+
+    void unmap_pointer() {
+        CUDA_API_CHECK(cudaGraphicsUnmapResources(1, &res, last_stream));
+    }
+
+};
+
+smart_pixel_buffer::smart_pixel_buffer()
+        : pimpl(std::make_unique<impl>()) {
+    pimpl->q_this = this;
+}
+
+smart_pixel_buffer::~smart_pixel_buffer() = default;
+
+void smart_pixel_buffer::create(GLenum target, GLenum flags, GLsizeiptr size) {
+    pimpl->create(target, flags, size);
+}
+
+void *smart_pixel_buffer::map_pointer(cudaStream_t stream) {
+    return pimpl->map_pointer(stream);
+}
+
+void smart_pixel_buffer::unmap_pointer() {
+    pimpl->unmap_pointer();
+}
+
+struct smart_frame_buffer::impl {
+
+    smart_frame_buffer *q_this = nullptr;
+    cv::Size last_size = {};
+    smart_texture color_tex, depth_tex;
+    smart_pixel_buffer pbo;
+
+    GLuint last_bind_fbo = 0;
+
+    static void check_frame_buffer() {
+        auto status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
+        if (status != GL_FRAMEBUFFER_COMPLETE) [[unlikely]] {
+            SPDLOG_ERROR("Framebuffer is not complete 0x{:x}.", status);
+            RET_ERROR;
+        }
+    }
+
+    void create(cv::Size size) {
+        if (size == last_size) [[likely]] return;
+        deallocate();
+        allocate(size);
+    }
+
+    void allocate(cv::Size size) {
+        // allocate buffer and textures
+        auto pbo_size = size.area() * 4 * sizeof(uint8_t);
+        pbo.create(GL_PIXEL_PACK_BUFFER, GL_DYNAMIC_STORAGE_BIT, (GLsizeiptr) pbo_size);
+        color_tex.create(GL_RGBA8, size);
+        depth_tex.create(GL_DEPTH_COMPONENT32, size);
+
+        // create frame buffer
+        glGenFramebuffers(1, &q_this->id);
+        glBindFramebuffer(GL_FRAMEBUFFER, q_this->id);
+        glBindTexture(GL_TEXTURE_2D, color_tex.id);
+        glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, color_tex.id, 0);
+        glBindTexture(GL_TEXTURE_2D, depth_tex.id);
+        glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_2D, depth_tex.id, 0);
+        check_frame_buffer();
+
+        last_size = size;
+    }
+
+    void deallocate() {
+        if (q_this->id == 0) return;
+        glDeleteFramebuffers(1, &q_this->id);
+        last_size = {};
+    }
+
+    void download(cv::cuda::GpuMat *img, cudaStream_t stream) {
+        assert(q_this->id != 0);
+        img->create(last_size, CV_8UC4);
+
+        // pack pixels into pbo
+        glBindFramebuffer(GL_FRAMEBUFFER, q_this->id);
+        glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo.id);
+        glReadPixels(0, 0, last_size.width, last_size.height, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, (void *) 0);
+        glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
+        glBindFramebuffer(GL_FRAMEBUFFER, 0);
+
+        // copy image to gpu mat
+        auto pbo_pitch = last_size.width * 4 * sizeof(uint8_t);
+        auto ptr = pbo.map_pointer(stream);
+        CUDA_API_CHECK(cudaMemcpy2DAsync(img->cudaPtr(), img->step, ptr, pbo_pitch, pbo_pitch,
+                                         last_size.height, cudaMemcpyDeviceToDevice, stream));
+        pbo.unmap_pointer();
+    }
+};
+
+smart_frame_buffer::smart_frame_buffer()
+        : pimpl(std::make_unique<impl>()) {
+    pimpl->q_this = this;
+}
+
+smart_frame_buffer::~smart_frame_buffer() = default;
+
+void smart_frame_buffer::create(cv::Size size) {
+    pimpl->create(size);
+
+    // clear color and depth
+    bind();
+    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+    unbind();
+}
+
+cv::Size smart_frame_buffer::size() const {
+    return pimpl->last_size;
+}
+
+void smart_frame_buffer::download(cv::cuda::GpuMat *img, cudaStream_t stream) {
+    pimpl->download(img, stream);
+}
+
+void smart_frame_buffer::bind() {
+    // save origin framebuffer id
+    glGetIntegerv(GL_DRAW_FRAMEBUFFER_BINDING, (GLint *) &pimpl->last_bind_fbo);
+
+    assert(id != 0);
+    glBindFramebuffer(GL_DRAW_FRAMEBUFFER, id);
+    glViewport(0, 0, pimpl->last_size.width, pimpl->last_size.height);
+}
+
+void smart_frame_buffer::unbind() {
+    glBindFramebuffer(GL_DRAW_FRAMEBUFFER, pimpl->last_bind_fbo);
+}
+
+GLuint smart_frame_buffer::color_tex() const {
+    return pimpl->color_tex.id;
+}
+
+GLuint smart_frame_buffer::depth_tex() const {
+    return pimpl->depth_tex.id;
+}
+
+namespace smart_program_impl {
+
+    void compile_shader(GLuint shader, const char *path, const char *name) {
+        auto file = mapped_file{path, mapped_file::readonly};
+        assert(file.is_open());
+        auto file_content = file.const_data();
+        GLint file_size = file.size();
+        glShaderSource(shader, 1, &file_content, &file_size);
+        glCompileShader(shader);
+        GLint status, log_length;
+        glGetShaderiv(shader, GL_COMPILE_STATUS, &status);
+        glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &log_length);
+        auto info_log = (GLchar *) malloc(log_length);
+        glGetShaderInfoLog(shader, log_length, nullptr, info_log);
+        if (status == GL_TRUE) {
+            SPDLOG_INFO("Compile {} shader succeeded: {}", name, info_log);
+        } else {
+            SPDLOG_ERROR("Compile {} shader failed: {}", name, info_log);
+            RET_ERROR;
+        }
+        free(info_log);
+    }
+
+    static void check_program(GLuint program) {
+        GLint status, log_length;
+        glGetProgramiv(program, GL_LINK_STATUS, &status);
+        glGetProgramiv(program, GL_INFO_LOG_LENGTH, &log_length);
+        auto info_log = (GLchar *) malloc(log_length);
+        glGetProgramInfoLog(program, log_length, nullptr, info_log);
+        if (status == GL_TRUE) {
+            SPDLOG_INFO("Link program succeeded: {}", info_log);
+        } else {
+            SPDLOG_ERROR("Link program failed: {}", info_log);
+            RET_ERROR;
+        }
+        free(info_log);
+    }
+
+    GLuint create_program(const char *vert_shader_path,
+                          const char *frag_shader_path) {
+        static auto index = 0;
+        ++index;
+
+        auto vert_shader = glCreateShader(GL_VERTEX_SHADER);
+        auto frag_shader = glCreateShader(GL_FRAGMENT_SHADER);
+        compile_shader(vert_shader, vert_shader_path, fmt::format("vertex_shader_{}", index).c_str());
+        compile_shader(frag_shader, frag_shader_path, fmt::format("fragment_shader_{}", index).c_str());
+
+        auto program_id = glCreateProgram();
+        glAttachShader(program_id, vert_shader);
+        glAttachShader(program_id, frag_shader);
+        glLinkProgram(program_id);
+        check_program(program_id);
+
+        glDeleteShader(vert_shader);
+        glDeleteShader(frag_shader);
+
+        return program_id;
+    }
+
+}
+
+using namespace smart_program_impl;
+
+struct smart_program::impl {
+
+    smart_program *q_this = nullptr;
+
+    using uniform_locs_type = std::unordered_map<std::string, GLint>;
+    uniform_locs_type uniform_locs;
+
+    GLint query_uniform_loc(const char *name) {
+        if (!uniform_locs.contains(name)) [[unlikely]] {
+            auto loc = glGetUniformLocation(q_this->id, name);
+            uniform_locs.emplace(std::make_pair(name, loc));
+            if (loc == -1) {
+                SPDLOG_WARN("Uniform {} is not found.", name);
+            }
+        }
+        return uniform_locs.at(name);
+    }
+
+};
+
+smart_program::smart_program()
+        : pimpl(std::make_unique<impl>()) {
+    pimpl->q_this = this;
+}
+
+smart_program::~smart_program() = default;
+
+smart_program *smart_program::create(const char *vert_source_path, // TODO: add program name
+                                     const char *frag_source_path) {
+    auto ret = new smart_program{};
+    ret->id = create_program(vert_source_path, frag_source_path);
+    return ret;
+}
+
+void smart_program::use() const {
+    glUseProgram(id);
+}
+
+void smart_program::set_uniform_i(const char *name, GLint val) {
+    auto loc = pimpl->query_uniform_loc(name);
+    glUniform1i(loc, val);
+}
+
+void smart_program::set_uniform_f(const char *name, GLfloat val) {
+    auto loc = pimpl->query_uniform_loc(name);
+    glUniform1f(loc, val);
+}
+
+void smart_program::set_uniform_vec3(const char *name, const glm::vec3 &vec) {
+    auto loc = pimpl->query_uniform_loc(name);
+    glUniform3f(loc, vec.x, vec.y, vec.z);
+}
+
+void smart_program::set_uniform_mat4(const char *name, const glm::mat4 &mat) {
+    auto loc = pimpl->query_uniform_loc(name);
+    glUniformMatrix4fv(loc, 1, false, glm::value_ptr(mat));
+}

src/render/render_utility.h

@@ -0,0 +1,102 @@
+#ifndef REMOTEAR3_RENDER_UTILITY_H
+#define REMOTEAR3_RENDER_UTILITY_H
+
+#include "cuda_helper.hpp"
+
+#include <glad/gl.h>
+#include <glm/glm.hpp>
+
+#include <opencv2/core/types.hpp>
+
+class smart_texture {
+public:
+    GLuint id = 0;
+
+    smart_texture();
+
+    ~smart_texture();
+
+    void create(GLenum format, cv::Size size,
+                GLint min_filter = GL_NEAREST, GLint max_filter = GL_NEAREST);
+
+    cv::Size size() const;
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+class smart_pixel_buffer {
+public:
+    GLuint id = 0;
+
+    smart_pixel_buffer();
+
+    ~smart_pixel_buffer();
+
+    void create(GLenum target, GLenum flags, GLsizeiptr size);
+
+    void *map_pointer(cudaStream_t stream);
+
+    void unmap_pointer();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+class smart_frame_buffer {
+public:
+    GLuint id = 0;
+
+    smart_frame_buffer();
+
+    ~smart_frame_buffer();
+
+    void create(cv::Size size);
+
+    cv::Size size() const;
+
+    void download(cv::cuda::GpuMat *img,
+                  cudaStream_t stream = nullptr);
+
+    void bind();
+
+    void unbind();
+
+    GLuint color_tex() const;
+
+    GLuint depth_tex() const;
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+class smart_program {
+public:
+    GLuint id = 0;
+
+    ~smart_program();
+
+    static smart_program *create(const char *vert_source_path,
+                                 const char *frag_source_path);
+
+    void use() const;
+
+    void set_uniform_i(const char *name, GLint val);
+
+    void set_uniform_f(const char *name, GLfloat val);
+
+    void set_uniform_vec3(const char *name, const glm::vec3 &vec);
+
+    void set_uniform_mat4(const char *name, const glm::mat4 &mat);
+
+private:
+    smart_program();
+
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#endif //REMOTEAR3_RENDER_UTILITY_H

src/render/scene_render.cpp

@@ -0,0 +1,170 @@
+#include "scene_render.h"
+#include "render/mesh_render.h"
+
+#include <glm/gtc/matrix_access.hpp>
+#include <glm/gtx/transform.hpp>
+
+#include <list>
+
+namespace scene_render_impl {
+
+    glm::vec3 vec3_from_ptr(const float *ptr) {
+        static_assert(sizeof(glm::vec3) == 3 * sizeof(float));
+        return *(glm::vec3 *) ptr;
+    }
+
+    glm::vec3 vec3_from_eigen(const Eigen::Vector3f &vec) {
+        static_assert(sizeof(glm::vec3) == sizeof(Eigen::Vector3f));
+        return *(glm::vec3 *) vec.data();
+    }
+
+    glm::mat4 mat4_from_eigen(const Eigen::Isometry3f &transform) {
+        static_assert(sizeof(glm::mat4) == sizeof(Eigen::Isometry3f));
+        return *(glm::mat4 *) transform.data();
+    }
+
+}
+
+using namespace scene_render_impl;
+
+struct scene_render::impl {
+
+    struct actor_store_type {
+        std::unique_ptr<mesh_type> model;
+
+        actor_store_type *background = nullptr;
+        float alpha_factor = 0.25f;
+
+        bool visible = true;
+        glm::mat4 transform = {};
+        material_type material = {};
+
+        mesh_property get_property() const {
+            mesh_property ret = {};
+            ret.transform = transform;
+            ret.material = material;
+            return ret;
+        }
+    };
+
+    bool enable_light_follow_camera = false;
+    glm::vec3 light_direction = {1.0f, 0.0f, 0.0f};
+
+    float camera_fov = 60.0f; // in degree
+    float camera_near = 0.1f, camera_far = 1000.0f;
+    glm::mat4 camera_transform = {};
+
+    std::list<actor_store_type> actor_list;
+
+    std::unique_ptr<mesh_render> render;
+
+    impl() {
+        render = std::make_unique<mesh_render>();
+    }
+
+    void *create_actor(const char *model_path, void *bg_actor) {
+        auto &actor = actor_list.emplace_back();
+        actor.model.reset(mesh_type::from_any(model_path));
+        actor.background = (actor_store_type *) bg_actor;
+        return &actor;
+    }
+
+    scene_property get_scene_info(float aspect) {
+        scene_property ret = {};
+        ret.camera.transform = camera_transform;
+        ret.camera.projection = glm::perspective(glm::radians(camera_fov),
+                                                 aspect, camera_near, camera_far);
+        if (enable_light_follow_camera) {
+            auto camera_rot = glm::inverse(glm::mat3{camera_transform});
+            light_direction = -glm::column(camera_rot, 2);
+        }
+        ret.light.direction = light_direction;
+        return ret;
+    }
+
+    void draw() {
+        // calculate aspect
+        GLint viewport[4]; // x, y, width, height
+        glGetIntegerv(GL_VIEWPORT, viewport);
+        auto width = viewport[2];
+        auto height = viewport[3];
+        auto aspect = 1.0f * width / height;
+
+        auto scene_info = get_scene_info(aspect);
+        for (const auto &actor: actor_list) {
+            if (!actor.visible) continue;
+            if (actor.background == nullptr) { // no background
+                render->render(actor.model.get(), actor.get_property(), scene_info);
+            } else { // with background
+                render->render(actor.model.get(), actor.get_property(),
+                               actor.background->model.get(), actor.background->get_property(),
+                               scene_info, actor.alpha_factor, false);
+            }
+        }
+    }
+
+};
+
+scene_render::scene_render()
+        : pimpl(std::make_unique<impl>()) {}
+
+scene_render::~scene_render() = default;
+
+void *scene_render::create_actor(const char *model_path, void *bg_actor) {
+    return pimpl->create_actor(model_path, bg_actor);
+}
+
+void scene_render::set_actor_visibility(void *actor, bool visible) {
+    auto real_ptr = (impl::actor_store_type *) actor;
+    real_ptr->visible = visible;
+}
+
+void scene_render::set_actor_color(void *actor, const float *color, float ambient_factor) {
+    auto real_ptr = (impl::actor_store_type *) actor;
+    auto color_vec = vec3_from_ptr(color);
+    real_ptr->material.diffuse = color_vec;
+    real_ptr->material.ambient = ambient_factor * color_vec;
+}
+
+void scene_render::set_actor_alpha_factor(void *actor, float alpha_factor) {
+    auto real_ptr = (impl::actor_store_type *) actor;
+    real_ptr->alpha_factor = alpha_factor;
+}
+
+void scene_render::set_actor_transform(void *actor, const Eigen::Isometry3f &transform) {
+    auto real_ptr = (impl::actor_store_type *) actor;
+    real_ptr->transform = mat4_from_eigen(transform);
+}
+
+void scene_render::set_light_direction(const Eigen::Vector3f &direction) {
+    pimpl->light_direction = vec3_from_eigen(direction);
+}
+
+void scene_render::set_light_follow_camera(bool enable) {
+    pimpl->enable_light_follow_camera = enable;
+}
+
+void scene_render::set_camera_fov(float fov) {
+    pimpl->camera_fov = fov;
+}
+
+void scene_render::set_camera_clip_range(float near, float far) {
+    pimpl->camera_near = near;
+    pimpl->camera_far = far;
+}
+
+void scene_render::set_camera_transform(const Eigen::Isometry3f &transform) {
+    static constexpr auto default_focal_length = 8; // 8mm
+    auto trans_part = transform.translation();
+    auto rot_part = transform.rotation();
+    auto focal_point = trans_part + rot_part.col(2) * default_focal_length;
+    auto view_up = -rot_part.col(1);
+    auto view_mat = glm::lookAt(glm::vec3{trans_part.x(), trans_part.y(), trans_part.z()},
+                                glm::vec3{focal_point.x(), focal_point.y(), focal_point.z()},
+                                glm::vec3{view_up.x(), view_up.y(), view_up.z()});
+    pimpl->camera_transform = view_mat;
+}
+
+void scene_render::render() {
+    pimpl->draw();
+}

src/render/scene_render.h

@@ -0,0 +1,48 @@
+#ifndef REMOTEAR3_SCENE_RENDER_H
+#define REMOTEAR3_SCENE_RENDER_H
+
+#include <Eigen/Geometry>
+
+#include <memory>
+
+class scene_render {
+public:
+
+    scene_render();
+
+    ~scene_render();
+
+    using actor_token_type = void *;
+
+    actor_token_type create_actor(const char *model_path,
+                                  actor_token_type bg_actor = nullptr);
+
+    static void set_actor_visibility(actor_token_type actor, bool visible);
+
+    static void set_actor_color(actor_token_type actor,
+                                const float color[3], float ambient_factor = 0.2f);
+
+    static void set_actor_alpha_factor(actor_token_type actor, float alpha_factor = 0.25f);
+
+    static void set_actor_transform(actor_token_type actor, const Eigen::Isometry3f &transform);
+
+    void set_light_direction(const Eigen::Vector3f &direction);
+
+    void set_light_follow_camera(bool enable = true);
+
+    void set_camera_fov(float fov);
+
+    void set_camera_clip_range(float near = 0.1f, float far = 1000.0f);
+
+    // transform: standard camera coordinate (Z points inside image), like VTK
+    void set_camera_transform(const Eigen::Isometry3f &transform);
+
+    void render();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+
+#endif //REMOTEAR3_SCENE_RENDER_H

src/render/shader/mesh_render.frag

@@ -0,0 +1,23 @@
+#version 460
+
+struct material_type {
+    vec3 ambient;
+    vec3 diffuse;
+};
+
+struct light_property {
+    vec3 direction;
+};
+
+uniform material_type material;
+uniform light_property light;
+
+in vec3 frag_normal;
+
+layout (location = 0) out vec4 frag_color;
+
+void main() {
+    float diffuse_weight = max(-dot(frag_normal, light.direction), 0.0);
+    vec3 color = material.ambient + diffuse_weight * material.diffuse;
+    frag_color = vec4(color, 1.0);
+}

src/render/shader/mesh_render.vert

@@ -0,0 +1,14 @@
+#version 460
+
+uniform mat4 model_mat;
+uniform mat4 camera_mat;
+
+layout (location = 0) in vec3 position;
+layout (location = 1) in vec3 normal;
+
+out vec3 frag_normal;
+
+void main() {
+    frag_normal = mat3(model_mat) * normal;
+    gl_Position = camera_mat * model_mat * vec4(position, 1.0);
+}

src/render/shader/mesh_render_mask.frag

@@ -0,0 +1,7 @@
+#version 460
+
+layout (location = 0) out vec4 frag_color;
+
+void main() {
+    frag_color = vec4(1.0);
+}

src/render/shader/mesh_render_mask.vert

@@ -0,0 +1,9 @@
+#version 460
+
+uniform mat4 transform_mat;
+
+layout (location = 0) in vec3 position;
+
+void main() {
+    gl_Position = transform_mat * vec4(position, 1.0);
+}

src/render/shader/texture_render.frag

@@ -0,0 +1,14 @@
+#version 460
+
+uniform float opacity;
+
+uniform sampler2D tex;
+
+in vec2 frag_uv;
+
+layout (location = 0) out vec4 frag_color;
+
+void main() {
+    frag_color = texture(tex, frag_uv);
+    frag_color.a *= opacity;
+}

src/render/shader/texture_render.vert

@@ -0,0 +1,11 @@
+#version 460
+
+layout (location = 0) in vec2 position;
+layout (location = 1) in vec2 tex_uv;
+
+out vec2 frag_uv;
+
+void main() {
+    gl_Position = vec4(position, 1.0, 1.0);
+    frag_uv = tex_uv;
+}

src/render/shader/texture_render_depth_alpha.frag

@@ -0,0 +1,45 @@
+#version 460
+
+uniform float alpha_factor;
+uniform mat4 proj_inv;
+
+uniform sampler2D color_tex;
+uniform sampler2D main_depth_tex;
+uniform sampler2D bg_depth_tex;
+
+in vec2 frag_uv;
+
+layout (location = 0) out vec4 frag_color;
+
+vec3 ndc_to_camera(in vec3 ndc) {
+    vec4 position = proj_inv * vec4(ndc, 1.0);// NDC -> camera
+    return position.xyz / position.w;// homogeneous -> cartesian
+}
+
+vec3 tex_to_ndc(in vec3 coord) {
+    return 2 * coord - 1;
+}
+
+vec3 tex_to_camera(in vec3 coord) {
+    return ndc_to_camera(tex_to_ndc(coord));
+}
+
+void main() {
+    float main_depth = texture(main_depth_tex, frag_uv).x;
+    if (main_depth == 1.0) {
+        frag_color = vec4(0.0);
+        return;
+    }
+
+    float alpha = 1.0;
+    float bg_depth = texture(bg_depth_tex, frag_uv).x;
+    if (main_depth > bg_depth) {
+        vec3 main_position = tex_to_camera(vec3(frag_uv, main_depth));
+        vec3 bg_position = tex_to_camera(vec3(frag_uv, bg_depth));
+        float depth_dis = distance(main_position, bg_position);
+        alpha = exp(-alpha_factor * depth_dis);
+    }
+
+    vec3 color = texture(color_tex, frag_uv).xyz;
+    frag_color = vec4(color, alpha);
+}

src/render/shader/texture_render_depth_only.frag

@@ -0,0 +1,11 @@
+#version 460
+
+uniform sampler2D depth_tex;
+
+in vec2 frag_uv;
+
+layout (depth_less) out float gl_FragDepth;
+
+void main() {
+    gl_FragDepth = texture(depth_tex, frag_uv).x;
+}

src/render/shader/texture_render_with_depth.frag

@@ -0,0 +1,14 @@
+#version 460
+
+uniform sampler2D color_tex;
+uniform sampler2D depth_tex;
+
+in vec2 frag_uv;
+
+layout (location = 0) out vec4 frag_color;
+layout (depth_less) out float gl_FragDepth;
+
+void main() {
+    frag_color = texture(color_tex, frag_uv);
+    gl_FragDepth = texture(depth_tex, frag_uv).x;
+}

src/render/texture_render.cpp

@@ -0,0 +1,242 @@
+#include "texture_render.h"
+#include "render_utility.h"
+#include "simple_mq.h"
+#include "variable_defs.h"
+
+#include <filesystem>
+
+namespace texture_render_impl {
+
+    GLuint rect_indices[] = {
+            0, 1, 3, // first triangle
+            1, 2, 3 // second triangle
+    };
+
+}
+
+using namespace texture_render_impl;
+using namespace simple_mq_singleton;
+
+struct texture_render::impl {
+
+    GLuint vao = 0, vbo = 0, ebo = 0;
+
+    std::unique_ptr<smart_program> program_basic;
+    std::unique_ptr<smart_program> program_depth_alpha;
+    std::unique_ptr<smart_program> program_with_depth;
+    std::unique_ptr<smart_program> program_depth_only;
+
+    smart_pixel_buffer image_pbo;
+    smart_texture image_tex;
+
+    impl() {
+        create_programs();
+        create_buffers();
+    }
+
+    void create_buffers() {
+        // create vertex buffer
+        glGenBuffers(1, &vbo);
+        glBindBuffer(GL_ARRAY_BUFFER, vbo);
+        glBufferStorage(GL_ARRAY_BUFFER, 16 * sizeof(GLfloat), nullptr, GL_DYNAMIC_STORAGE_BIT);
+
+        // create index buffer
+        glGenBuffers(1, &ebo);
+        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
+        glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(rect_indices), rect_indices, GL_STATIC_DRAW);
+
+        // config vertex array
+        glGenVertexArrays(1, &vao);
+        glBindVertexArray(vao);
+        glEnableVertexAttribArray(0);
+        glEnableVertexAttribArray(1);
+        glVertexAttribPointer(0, 2, GL_FLOAT, false, 4 * sizeof(GLfloat), (void *) 0);
+        glVertexAttribPointer(1, 2, GL_FLOAT, false, 4 * sizeof(GLfloat), (void *) (2 * sizeof(GLfloat)));
+    }
+
+    void create_programs() {
+        std::filesystem::path shader_folder = mq().query_variable<std::string>(SHADER_DIR);
+
+        auto vert_basic = shader_folder / "texture_render.vert";
+        auto frag_basic = shader_folder / "texture_render.frag";
+        program_basic.reset(smart_program::create(vert_basic.c_str(),
+                                                  frag_basic.c_str()));
+
+        auto frag_depth_alpha = shader_folder / "texture_render_depth_alpha.frag";
+        program_depth_alpha.reset(smart_program::create(vert_basic.c_str(),
+                                                        frag_depth_alpha.c_str()));
+
+        auto frag_width_depth = shader_folder / "texture_render_with_depth.frag";
+        program_with_depth.reset(smart_program::create(vert_basic.c_str(),
+                                                       frag_width_depth.c_str()));
+
+        auto frag_depth_only = shader_folder / "texture_render_depth_only.frag";
+        program_depth_only.reset(smart_program::create(vert_basic.c_str(),
+                                                       frag_depth_only.c_str()));
+    }
+
+    void config_buffer(const simple_rect &rect, bool flip_y) {
+        // bind buffers
+        glBindVertexArray(vao);
+        glBindBuffer(GL_ARRAY_BUFFER, vbo);
+        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
+
+        // fill vertex buffer
+        GLfloat tex_top = flip_y ? 0 : 1;
+        GLfloat tex_bottom = flip_y ? 1 : 0;
+        GLfloat vertices[] = {
+                // 2 for position; 2 for texture
+                rect.x + rect.width, rect.y + rect.height, 1, tex_top, // top right
+                rect.x + rect.width, rect.y, 1, tex_bottom, // bottom right
+                rect.x, rect.y, 0, tex_bottom, // bottom left
+                rect.x, rect.y + rect.height, 0, tex_top // top left
+        };
+        static_assert(sizeof(vertices) == 16 * sizeof(GLfloat));
+        glBufferSubData(GL_ARRAY_BUFFER, 0, sizeof(vertices), vertices);
+    }
+
+    void draw() {
+        glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, nullptr);
+    }
+
+    void upload_gpu_mat(const cv::cuda::GpuMat &img, cudaStream_t stream) {
+        // allocate memory if needed
+        assert(img.type() == CV_8UC3);
+        auto pbo_pitch = img.size().width * img.elemSize() * sizeof(uint8_t);
+        auto img_bytes = img.size().height * pbo_pitch;
+        image_pbo.create(GL_PIXEL_PACK_BUFFER, GL_DYNAMIC_STORAGE_BIT, (GLsizeiptr) img_bytes);
+        image_tex.create(GL_RGBA8, img.size());
+
+        // copy image to texture
+        auto ptr = image_pbo.map_pointer(stream);
+        CUDA_API_CHECK(cudaMemcpy2DAsync(ptr, pbo_pitch, img.cudaPtr(), img.step, pbo_pitch,
+                                         img.size().height, cudaMemcpyDeviceToDevice, stream));
+        image_pbo.unmap_pointer();
+
+        // unpack pbo to texture
+        glBindBuffer(GL_PIXEL_UNPACK_BUFFER, image_pbo.id);
+        glBindTexture(GL_TEXTURE_2D, image_tex.id);
+        glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, img.size().width, img.size().height,
+                        GL_RGB, GL_UNSIGNED_BYTE, nullptr);
+        glBindTexture(GL_TEXTURE_2D, 0);
+        glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
+    }
+
+    void render(GLuint tex, float opacity, const simple_rect &rect, bool flip_y) {
+        program_basic->use();
+        config_buffer(rect, flip_y);
+
+        program_basic->set_uniform_f("opacity", opacity);
+
+        glActiveTexture(GL_TEXTURE0 + 0);
+        glBindTexture(GL_TEXTURE_2D, tex);
+        program_basic->set_uniform_i("tex", 0);
+
+        glDisable(GL_DEPTH_TEST);
+        draw();
+    }
+
+    void render_with_depth(GLuint color_tex, GLuint depth_tex,
+                           const simple_rect &rect, bool flip_y) {
+        program_with_depth->use();
+        config_buffer(rect, flip_y);
+
+        glActiveTexture(GL_TEXTURE0 + 0);
+        glBindTexture(GL_TEXTURE_2D, color_tex);
+        program_with_depth->set_uniform_i("color_tex", 0);
+        glActiveTexture(GL_TEXTURE0 + 1);
+        glBindTexture(GL_TEXTURE_2D, depth_tex);
+        program_with_depth->set_uniform_i("depth_tex", 1);
+
+        glEnable(GL_DEPTH_TEST);
+        draw();
+    }
+
+    void render_depth_alpha(GLuint color_tex, GLuint main_depth_tex, GLuint bg_depth_tex,
+                            const glm::mat4 &projection, float alpha_factor,
+                            const simple_rect &rect, bool flip_y) {
+        program_depth_alpha->use();
+        config_buffer(rect, flip_y);
+
+        auto project_inv = glm::inverse(projection);
+        program_depth_alpha->set_uniform_mat4("proj_inv", project_inv);
+        alpha_factor = alpha_factor / (1.0f - alpha_factor); // [0, 1] -> [0, +inf)
+        program_depth_alpha->set_uniform_f("alpha_factor", alpha_factor);
+
+        glActiveTexture(GL_TEXTURE0 + 0);
+        glBindTexture(GL_TEXTURE_2D, color_tex);
+        program_depth_alpha->set_uniform_i("color_tex", 0);
+        glActiveTexture(GL_TEXTURE0 + 1);
+        glBindTexture(GL_TEXTURE_2D, main_depth_tex);
+        program_depth_alpha->set_uniform_i("main_depth_tex", 1);
+        glActiveTexture(GL_TEXTURE0 + 2);
+        glBindTexture(GL_TEXTURE_2D, bg_depth_tex);
+        program_depth_alpha->set_uniform_i("bg_depth_tex", 2);
+
+        glDisable(GL_DEPTH_TEST);
+        draw();
+    }
+
+    void render_depth_only(GLuint depth_tex, const simple_rect &rect, bool flip_y) {
+        program_depth_only->use();
+        config_buffer(rect, flip_y);
+
+        glActiveTexture(GL_TEXTURE0 + 0);
+        glBindTexture(GL_TEXTURE_2D, depth_tex);
+        program_depth_only->set_uniform_i("depth_tex", 0);
+
+        glEnable(GL_DEPTH_TEST);
+        draw();
+    }
+
+};
+
+texture_render::texture_render()
+        : pimpl(std::make_unique<impl>()) {}
+
+texture_render::~texture_render() = default;
+
+void texture_render::render(GLuint tex, float opacity,
+                            const simple_rect &rect, bool flip_y) {
+    pimpl->render(tex, opacity, rect, flip_y);
+}
+
+void texture_render::render(const cv::cuda::GpuMat &img, float opacity,
+                            const simple_rect &rect, bool flip_y,
+                            cudaStream_t stream) {
+    pimpl->upload_gpu_mat(img, stream);
+    pimpl->render(pimpl->image_tex.id, opacity, rect, flip_y);
+}
+
+void texture_render::render_with_depth(GLuint color_tex, GLuint depth_tex,
+                                       const simple_rect &rect, bool flip_y) {
+    pimpl->render_with_depth(color_tex, depth_tex, rect, flip_y);
+}
+
+void texture_render::render_depth_alpha(GLuint color_tex, GLuint main_depth_tex, GLuint bg_depth_tex,
+                                        const glm::mat4 &projection, float alpha_factor,
+                                        const simple_rect &rect, bool flip_y) {
+    pimpl->render_depth_alpha(color_tex, main_depth_tex, bg_depth_tex,
+                              projection, alpha_factor, rect, flip_y);
+}
+
+void texture_render::render_depth_only(GLuint depth_tex, const simple_rect &rect, bool flip_y) {
+    pimpl->render_depth_only(depth_tex, rect, flip_y);
+}
+
+simple_rect simple_rect::fit_aspect(float aspect_target) const {
+    simple_rect ret = {};
+    auto aspect_this = width / height;
+    if (aspect_this > aspect_target) { // adjust width
+        ret.height = height;
+        ret.width = height * aspect_target;
+        ret.x = x + 0.5f * (width - ret.width);
+        ret.y = y;
+    } else { // adjust height
+        ret.width = width;
+        ret.height = width / aspect_target;
+        ret.x = x;
+        ret.y = y + 0.5f * (height - ret.height);
+    }
+    return ret;
+}

src/render/texture_render.h

@@ -0,0 +1,52 @@
+#ifndef TRANSPARENTAR_TEXTURE_RENDER_H
+#define TRANSPARENTAR_TEXTURE_RENDER_H
+
+#include "cuda_helper.hpp"
+
+#include <glad/gl.h>
+#include <glm/glm.hpp>
+
+#include <opencv2/core/cuda.hpp>
+
+#include <memory>
+
+struct simple_rect {
+    GLfloat x, y;
+    GLfloat width, height;
+
+    simple_rect fit_aspect(float aspect) const;
+};
+
+class texture_render {
+public:
+    texture_render();
+
+    ~texture_render();
+
+    static constexpr auto full_rect = simple_rect{-1.0f, -1.0f, 2.0f, 2.0f};
+
+    void render(GLuint tex, float opacity = 1.0f,
+                const simple_rect &rect = full_rect, bool flip_y = false);
+
+    void render_with_depth(GLuint color_tex, GLuint depth_tex,
+                           const simple_rect &rect = full_rect, bool flip_y = false);
+
+
+    void render(const cv::cuda::GpuMat &img, float opacity = 1.0f,
+                const simple_rect &rect = full_rect, bool flip_y = false,
+                cudaStream_t stream = nullptr);
+
+    void render_depth_alpha(GLuint color_tex, GLuint main_depth_tex, GLuint bg_depth_tex,
+                            const glm::mat4 &projection, float alpha_factor = 0.2,
+                            const simple_rect &rect = full_rect, bool flip_y = false);
+
+    void render_depth_only(GLuint depth_tex,
+                           const simple_rect &rect = full_rect, bool flip_y = false);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+
+#endif //TRANSPARENTAR_TEXTURE_RENDER_H

src/simple_opengl.cpp

@@ -1,411 +0,0 @@
-#include "simple_opengl.h"
-
-#include <cuda_gl_interop.h>
-
-namespace simple_opengl_impl {
-
-    constexpr auto simple_vert_shader_source = R"(
-        #version 460
-        layout (location = 0) in vec2 pos_in;
-        layout (location = 1) in vec2 tex_coord_in;
-        out vec2 tex_coord;
-        void main() {
-            gl_Position = vec4(pos_in, 0, 1);
-            tex_coord = tex_coord_in;
-        }
-    )";
-
-    constexpr auto simple_frag_shader_source = R"(
-        #version 460
-        layout (location = 0) out vec4 color_out;
-        in vec2 tex_coord;
-        uniform sampler2D tex_sampler;
-        void main() {
-            color_out = texture(tex_sampler, tex_coord);
-        }
-    )";
-
-    constexpr GLuint rect_indices[] = {
-            0, 1, 3, // first triangle
-            1, 2, 3 // second triangle
-    };
-
-    struct smart_pixel_buffer {
-        GLuint id = 0;
-        cudaGraphicsResource *res = nullptr;
-
-        smart_pixel_buffer() = default;
-
-        smart_pixel_buffer(const smart_pixel_buffer &other) = delete;
-
-        ~smart_pixel_buffer() {
-            deallocate();
-        }
-
-        void create(GLenum target, GLenum flags, GLsizeiptr size) {
-            if (size == last_size) [[likely]] return;
-            deallocate();
-            allocate(target, flags, size);
-        }
-
-        void *map_pointer(cudaStream_t stream) {
-            void *ptr;
-            size_t size;
-            CUDA_API_CHECK_P(cudaGraphicsMapResources(1, &res, stream));
-            CUDA_API_CHECK_P(cudaGraphicsResourceGetMappedPointer(&ptr, &size, res));
-            assert(size == last_size);
-            last_stream = stream;
-            return ptr;
-        }
-
-        void unmap_pointer() {
-            CUDA_API_CHECK(cudaGraphicsUnmapResources(1, &res, last_stream));
-        }
-
-    private:
-        GLsizeiptr last_size = 0;
-        cudaStream_t last_stream = nullptr;
-
-        void allocate(GLenum target, GLenum flags, GLsizeiptr size) {
-            glGenBuffers(1, &id);
-            glBindBuffer(target, id);
-            glBufferStorage(target, size, nullptr, flags);
-            glBindBuffer(target, 0);
-            last_size = size;
-
-            // register CUDA resource
-            if (target == GL_PIXEL_UNPACK_BUFFER) {
-                CUDA_API_CHECK(cudaGraphicsGLRegisterBuffer(
-                        &res, id, cudaGraphicsRegisterFlagsWriteDiscard));
-            } else {
-                assert(target == GL_PIXEL_PACK_BUFFER);
-                CUDA_API_CHECK(cudaGraphicsGLRegisterBuffer(
-                        &res, id, cudaGraphicsRegisterFlagsReadOnly));
-            }
-        }
-
-        void deallocate() {
-            if (id == 0) return;
-            glDeleteBuffers(1, &id);
-            id = 0;
-            last_size = 0;
-
-            // unregister CUDA resource
-            CUDA_API_CHECK(cudaGraphicsUnregisterResource(res));
-            res = nullptr;
-        }
-    };
-
-}
-
-using namespace simple_opengl_impl;
-
-struct simple_render::impl {
-    GLuint vao = 0, vbo = 0, ebo = 0;
-    GLuint simple_program = 0;
-
-    smart_pixel_buffer image_pbo;
-    smart_texture image_tex;
-
-    impl() {
-        create_program();
-    }
-
-    ~impl() {
-        glDeleteVertexArrays(1, &vao);
-        glDeleteBuffers(1, &vbo);
-        glDeleteBuffers(1, &ebo);
-    }
-
-    static void compile_shader(GLuint shader, const char *source, const char *name) {
-        glShaderSource(shader, 1, &source, nullptr);
-        glCompileShader(shader);
-        GLint status, log_length;
-        glGetShaderiv(shader, GL_COMPILE_STATUS, &status);
-        glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &log_length);
-        auto info_log = (GLchar *) malloc(log_length);
-        glGetShaderInfoLog(shader, log_length, nullptr, info_log);
-        if (status == GL_TRUE) {
-            SPDLOG_INFO("Compile {} shader succeeded: {}", name, info_log);
-        } else {
-            SPDLOG_ERROR("Compile {} shader failed: {}", name, info_log);
-            RET_ERROR;
-        }
-        free(info_log);
-    }
-
-    static void check_program(GLuint program) {
-        GLint status, log_length;
-        glGetProgramiv(program, GL_LINK_STATUS, &status);
-        glGetProgramiv(program, GL_INFO_LOG_LENGTH, &log_length);
-        auto info_log = (GLchar *) malloc(log_length);
-        glGetProgramInfoLog(program, log_length, nullptr, info_log);
-        if (status == GL_TRUE) {
-            SPDLOG_INFO("Link program succeeded: {}", info_log);
-        } else {
-            SPDLOG_ERROR("Link program failed: {}", info_log);
-            RET_ERROR;
-        }
-        free(info_log);
-    }
-
-    void create_program() {
-        auto simple_vert_shader = glCreateShader(GL_VERTEX_SHADER);
-        auto simple_frag_shader = glCreateShader(GL_FRAGMENT_SHADER);
-        compile_shader(simple_vert_shader, simple_vert_shader_source, "simple_vertex");
-        compile_shader(simple_frag_shader, simple_frag_shader_source, "simple_fragment");
-
-        simple_program = glCreateProgram();
-        glAttachShader(simple_program, simple_vert_shader);
-        glAttachShader(simple_program, simple_frag_shader);
-        glLinkProgram(simple_program);
-        check_program(simple_program);
-
-        glDeleteShader(simple_vert_shader);
-        glDeleteShader(simple_frag_shader);
-
-        // create buffers
-        glGenBuffers(1, &vbo);
-        glGenBuffers(1, &ebo);
-
-        // config vertex buffer
-        glBindBuffer(GL_ARRAY_BUFFER, vbo);
-        glBufferStorage(GL_ARRAY_BUFFER, 16 * sizeof(GLfloat), nullptr, GL_DYNAMIC_STORAGE_BIT);
-
-        // fill element buffer
-        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
-        glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(rect_indices), rect_indices, GL_STATIC_DRAW);
-
-        // config vertex array
-        glGenVertexArrays(1, &vao);
-        glBindVertexArray(vao);
-        glEnableVertexAttribArray(0);
-        glEnableVertexAttribArray(1);
-        glVertexAttribPointer(0, 2, GL_FLOAT, false, 4 * sizeof(GLfloat), (void *) 0);
-        glVertexAttribPointer(1, 2, GL_FLOAT, false, 4 * sizeof(GLfloat), (void *) (2 * sizeof(GLfloat)));
-    }
-
-    void render_texture(GLuint tex, const simple_rect &rect, bool flip_y) {
-        // bind buffers
-        glUseProgram(simple_program);
-        glBindVertexArray(vao);
-        glBindBuffer(GL_ARRAY_BUFFER, vbo);
-        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
-
-        // bind textures
-        glActiveTexture(GL_TEXTURE0 + 0);
-        glBindTexture(GL_TEXTURE_2D, tex);
-
-        // fill vertex buffer
-        GLfloat tex_top = flip_y ? 0 : 1;
-        GLfloat tex_bottom = flip_y ? 1 : 0;
-        GLfloat vertices[] = {
-                // 2 for position; 2 for texture
-                rect.x + rect.width, rect.y + rect.height, 1, tex_top, // top right
-                rect.x + rect.width, rect.y, 1, tex_bottom, // bottom right
-                rect.x, rect.y, 0, tex_bottom, // bottom left
-                rect.x, rect.y + rect.height, 0, tex_top // top left
-        };
-        static_assert(sizeof(vertices) == 16 * sizeof(GLfloat));
-        glBufferSubData(GL_ARRAY_BUFFER, 0, sizeof(vertices), vertices);
-
-        // draw texture
-        glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, nullptr);
-    }
-
-    void upload_gpu_mat(const cv::cuda::GpuMat &img, cudaStream_t stream) {
-        // allocate memory if needed
-        assert(img.type() == CV_8UC3);
-        auto pbo_pitch = img.size().width * img.elemSize() * sizeof(uint8_t);
-        auto img_bytes = img.size().height * pbo_pitch;
-        image_pbo.create(GL_PIXEL_PACK_BUFFER, GL_DYNAMIC_STORAGE_BIT, (GLsizeiptr) img_bytes);
-        image_tex.create(GL_RGBA8, img.size());
-
-        // copy image to texture
-        auto ptr = image_pbo.map_pointer(stream);
-        CUDA_API_CHECK(cudaMemcpy2DAsync(ptr, pbo_pitch, img.cudaPtr(), img.step, pbo_pitch,
-                                         img.size().height, cudaMemcpyDeviceToDevice, stream));
-        image_pbo.unmap_pointer();
-
-        // unpack pbo to texture
-        glBindBuffer(GL_PIXEL_UNPACK_BUFFER, image_pbo.id);
-        glBindTexture(GL_TEXTURE_2D, image_tex.id);
-        glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, img.size().width, img.size().height,
-                        GL_RGB, GL_UNSIGNED_BYTE, nullptr);
-        glBindTexture(GL_TEXTURE_2D, 0);
-        glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
-    }
-};
-
-simple_render::simple_render()
-        : pimpl(std::make_unique<impl>()) {}
-
-simple_render::~simple_render() = default;
-
-void simple_render::render_rect(GLuint tex, const simple_rect &rect, bool flip_y) {
-    pimpl->render_texture(tex, rect, flip_y);
-}
-
-void simple_render::render_rect(const cv::cuda::GpuMat &img, const simple_rect &rect,
-                                bool flip_y, cudaStream_t stream) {
-    pimpl->upload_gpu_mat(img, stream);
-    pimpl->render_texture(pimpl->image_tex.id, rect, flip_y);
-}
-
-struct smart_frame_buffer::impl {
-
-    smart_frame_buffer *q_this = nullptr;
-    cv::Size last_size = {};
-    smart_texture color_tex, depth_tex;
-    smart_pixel_buffer pbo;
-
-    static void check_frame_buffer() {
-        auto status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
-        if (status != GL_FRAMEBUFFER_COMPLETE) [[unlikely]] {
-            SPDLOG_ERROR("Framebuffer is not complete 0x{:x}.", status);
-            RET_ERROR;
-        }
-    }
-
-    void create(cv::Size size) {
-        if (size == last_size) [[likely]] return;
-        deallocate();
-        allocate(size);
-    }
-
-    void allocate(cv::Size size) {
-        // allocate buffer and textures
-        auto pbo_size = size.area() * 4 * sizeof(uint8_t);
-        pbo.create(GL_PIXEL_PACK_BUFFER, GL_DYNAMIC_STORAGE_BIT, (GLsizeiptr) pbo_size);
-        color_tex.create(GL_RGB8, size);
-        depth_tex.create(GL_DEPTH_COMPONENT16, size);
-
-        // create frame buffer
-        glGenFramebuffers(1, &q_this->id);
-        glBindFramebuffer(GL_FRAMEBUFFER, q_this->id);
-        glBindTexture(GL_TEXTURE_2D, color_tex.id);
-        glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, color_tex.id, 0);
-        glBindTexture(GL_TEXTURE_2D, depth_tex.id);
-        glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_2D, depth_tex.id, 0);
-        check_frame_buffer();
-
-        last_size = size;
-    }
-
-    void deallocate() {
-        if (q_this->id == 0) return;
-        glDeleteFramebuffers(1, &q_this->id);
-        last_size = {};
-    }
-
-    void download(cv::cuda::GpuMat *img, cudaStream_t stream) {
-        assert(q_this->id != 0);
-        img->create(last_size, CV_8UC4);
-
-        // pack pixels into pbo
-        glBindFramebuffer(GL_FRAMEBUFFER, q_this->id);
-        glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo.id);
-        glReadPixels(0, 0, last_size.width, last_size.height, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, (void *) 0);
-        glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
-        glBindFramebuffer(GL_FRAMEBUFFER, 0);
-
-        // copy image to gpu mat
-        auto pbo_pitch = last_size.width * 4 * sizeof(uint8_t);
-        auto ptr = pbo.map_pointer(stream);
-        CUDA_API_CHECK(cudaMemcpy2DAsync(img->cudaPtr(), img->step, ptr, pbo_pitch, pbo_pitch,
-                                         last_size.height, cudaMemcpyDeviceToDevice, stream));
-        pbo.unmap_pointer();
-    }
-};
-
-smart_frame_buffer::smart_frame_buffer()
-        : pimpl(std::make_unique<impl>()) {
-    pimpl->q_this = this;
-}
-
-smart_frame_buffer::~smart_frame_buffer() = default;
-
-void smart_frame_buffer::create(cv::Size size) {
-    pimpl->create(size);
-}
-
-cv::Size smart_frame_buffer::size() const {
-    return pimpl->last_size;
-}
-
-void smart_frame_buffer::download(cv::cuda::GpuMat *img, cudaStream_t stream) {
-    pimpl->download(img, stream);
-}
-
-void smart_frame_buffer::bind() {
-    assert(id != 0);
-    glBindFramebuffer(GL_DRAW_FRAMEBUFFER, id);
-    glViewport(0, 0, pimpl->last_size.width, pimpl->last_size.height);
-}
-
-void smart_frame_buffer::unbind() {
-    glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
-}
-
-struct smart_texture::impl {
-
-    smart_texture *q_this = nullptr;
-    cv::Size last_size = {};
-
-    void create(GLenum format, cv::Size size, GLint min_filter, GLint max_filter) {
-        if (size == last_size) [[likely]] return;
-        deallocate();
-        allocate(format, size, min_filter, max_filter);
-    }
-
-    void allocate(GLenum format, cv::Size size,
-                  GLint min_filter, GLint max_filter) {
-        glGenTextures(1, &q_this->id);
-        glBindTexture(GL_TEXTURE_2D, q_this->id);
-        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, min_filter);
-        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, max_filter);
-        glTexStorage2D(GL_TEXTURE_2D, 1, format, size.width, size.height);
-        glBindTexture(GL_TEXTURE_2D, 0);
-        last_size = size;
-    }
-
-    void deallocate() {
-        if (q_this->id == 0) return;
-        glDeleteTextures(1, &q_this->id);
-        q_this->id = 0;
-        last_size = {};
-    }
-
-};
-
-smart_texture::smart_texture()
-        : pimpl(std::make_unique<impl>()) {
-    pimpl->q_this = this;
-}
-
-smart_texture::~smart_texture() = default;
-
-void smart_texture::create(GLenum format, cv::Size size, GLint min_filter, GLint max_filter) {
-    pimpl->create(format, size, min_filter, max_filter);
-}
-
-cv::Size smart_texture::size() const {
-    return pimpl->last_size;
-}
-
-simple_rect simple_rect::fit_aspect(float aspect_target) const {
-    simple_rect ret = {};
-    auto aspect_this = width / height;
-    if (aspect_this > aspect_target) { // adjust width
-        ret.height = height;
-        ret.width = height * aspect_target;
-        ret.x = x + 0.5f * (width - ret.width);
-        ret.y = y;
-    } else { // adjust height
-        ret.width = width;
-        ret.height = width / aspect_target;
-        ret.x = x;
-        ret.y = y + 0.5f * (height - ret.height);
-    }
-    return ret;
-}

src/simple_opengl.h

@@ -1,81 +0,0 @@
-#ifndef REMOTEAR3_SIMPLE_OPENGL_H
-#define REMOTEAR3_SIMPLE_OPENGL_H
-
-#include "cuda_helper.hpp"
-
-#include <opencv2/core/cuda.hpp>
-#include <opencv2/core/mat.hpp>
-
-#include <glad/gl.h>
-
-#include <memory>
-
-struct simple_rect {
-    GLfloat x, y;
-    GLfloat width, height;
-
-    simple_rect fit_aspect(float aspect) const;
-};
-
-struct smart_texture {
-    GLuint id = 0;
-
-    smart_texture();
-
-    ~smart_texture();
-
-    void create(GLenum format, cv::Size size,
-                GLint min_filter = GL_NEAREST, GLint max_filter = GL_NEAREST);
-
-    cv::Size size() const;
-
-private:
-    struct impl;
-    std::unique_ptr<impl> pimpl;
-};
-
-class simple_render {
-public:
-    simple_render();
-
-    ~simple_render();
-
-    void render_rect(GLuint tex,
-                     const simple_rect &rect,
-                     bool flip_y = false);
-
-    void render_rect(const cv::cuda::GpuMat &img,
-                     const simple_rect &rect,
-                     bool flip_y = true,
-                     cudaStream_t stream = nullptr);
-
-private:
-    struct impl;
-    std::unique_ptr<impl> pimpl;
-};
-
-class smart_frame_buffer {
-public:
-    GLuint id = 0;
-
-    smart_frame_buffer();
-
-    ~smart_frame_buffer();
-
-    void create(cv::Size size);
-
-    cv::Size size() const;
-
-    void download(cv::cuda::GpuMat *img,
-                  cudaStream_t stream = nullptr);
-
-    void bind();
-
-    static void unbind();
-
-private:
-    struct impl;
-    std::unique_ptr<impl> pimpl;
-};
-
-#endif //REMOTEAR3_SIMPLE_OPENGL_H

src/utility.hpp

@@ -26,6 +26,10 @@
     assert(false); \
     return false
 
+#define RET_ERROR_P \
+    assert(false); \
+    return nullptr
+
 inline bool check_function_call(bool function_ret, unsigned int line_number,
                                 const char *file_name, const char *function_call_str) {
     if (function_ret) [[likely]] return true;
@@ -94,6 +98,10 @@ struct smart_buffer {
         length = req_length;
     }
 
+    size_t size() const {
+        return length * sizeof(T);
+    }
+
 private:
     size_t capacity = 0;
 };

src/variable_defs.h

@@ -21,6 +21,8 @@ constexpr auto SENDER_ENABLE_LOG = 10;
 constexpr auto OUTPUT_WIDTH = 11;
 constexpr auto OUTPUT_HEIGHT = 12;
 
+constexpr auto SHADER_DIR = 13;
+
 // global variable declaration
 
 #endif //REMOTEAR3_VARIABLE_DEFS_H

src/vtk_viewer.cpp

@@ -1,4 +1,5 @@
 #include "vtk_viewer.h"
+#include "utility.hpp"
 
 #include <vtkCamera.h>
 #include <vtkCellArray.h>
@@ -7,6 +8,7 @@
 #include <vtkInteractorStyleTrackballCamera.h>
 #include <vtkMatrix4x4.h>
 #include <vtkNew.h>
+#include <vtkOBJReader.h>
 #include <vtkOpenGLFramebufferObject.h>
 #include <vtkPointPicker.h>
 #include <vtkPoints.h>
@@ -18,6 +20,7 @@
 
 #include <spdlog/spdlog.h>
 
+#include <filesystem>
 #include <vector>
 
 namespace vtk_viewer_impl {
@@ -279,46 +282,68 @@ void vtk_viewer::reset_camera() {
     pimpl->renderer->ResetCamera();
 }
 
-vtkSmartPointer<vtkPolyData> load_stl(const std::string &path) {
-    vtkNew<vtkSTLReader> reader;
-    reader->SetFileName(path.c_str());
-    reader->Update();
-    return reader->GetOutput();
-}
+namespace vtk_viewer_helper {
 
-vtkSmartPointer<vtkActor> create_actor(vtkPolyData *data) {
-    vtkNew<vtkPolyDataMapper> mapper;
-    mapper->SetInputData(data);
-    vtkNew<vtkActor> actor;
-    actor->SetMapper(mapper);
-    vtkNew<vtkMatrix4x4> pose;
-    actor->SetUserMatrix(pose);
-    return actor;
-}
+    vtkSmartPointer<vtkPolyData> load_stl(const std::string &path) {
+        vtkNew<vtkSTLReader> reader;
+        reader->SetFileName(path.c_str());
+        reader->Update();
+        return reader->GetOutput();
+    }
 
-vtkSmartPointer<vtkActor> create_actor(const std::string &path) {
-    return create_actor(load_stl(path));
-}
+    vtkSmartPointer<vtkPolyData> load_obj(const std::string &path) {
+        vtkNew<vtkOBJReader> reader;
+        reader->SetFileName(path.c_str());
+        reader->Update();
+        return reader->GetOutput();
+    }
 
-void update_actor_pose(vtkActor *actor, const std::optional<Eigen::Isometry3d> &trans) {
-    if (!trans.has_value()) {
-        actor->VisibilityOff();
-        return;
+    vtkSmartPointer<vtkPolyData> load_any(const std::string &path) {
+        auto path_fs = std::filesystem::path{path};
+        auto ext = path_fs.extension();
+        if (ext == ".stl") {
+            return load_stl(path);
+        } else if (ext == ".obj") {
+            return load_obj(path);
+        }
+        RET_ERROR_P;
     }
-    actor->VisibilityOn();
-    auto &real_trans = trans.value();
-    auto matrix = actor->GetUserMatrix();
-    if (matrix == nullptr) {
-        actor->SetUserMatrix(vtkMatrix4x4::New());
-        matrix = actor->GetUserMatrix();
+
+    vtkSmartPointer<vtkActor> create_actor(vtkPolyData *data) {
+        vtkNew<vtkPolyDataMapper> mapper;
+        mapper->SetInputData(data);
+        vtkNew<vtkActor> actor;
+        actor->SetMapper(mapper);
+        vtkNew<vtkMatrix4x4> pose;
+        actor->SetUserMatrix(pose);
+        return actor;
     }
-    for (int i = 0; i < 4; ++i) {
-        for (int j = 0; j < 4; ++j) {
-            matrix->SetElement(i, j, real_trans(i, j));
+
+    vtkSmartPointer<vtkActor> create_actor(const std::string &path) {
+        return create_actor(load_stl(path));
+    }
+
+    void update_actor_pose(vtkActor *actor, const std::optional<Eigen::Isometry3d> &trans) {
+        if (!trans.has_value()) {
+            actor->VisibilityOff();
+            return;
+        }
+        actor->VisibilityOn();
+        auto &real_trans = trans.value();
+        auto matrix = actor->GetUserMatrix();
+        if (matrix == nullptr) {
+            actor->SetUserMatrix(vtkMatrix4x4::New());
+            matrix = actor->GetUserMatrix();
         }
+        for (int i = 0; i < 4; ++i) {
+            for (int j = 0; j < 4; ++j) {
+                matrix->SetElement(i, j, real_trans(i, j));
+            }
+        }
+        matrix->Modified();
+        actor->Modified();
     }
-    matrix->Modified();
-    actor->Modified();
+
 }
 
 struct smart_point_sets::impl {

src/vtk_viewer.h

@@ -1,7 +1,7 @@
 #ifndef REMOTEAR3_VTK_VIEWER_H
 #define REMOTEAR3_VTK_VIEWER_H
 
-#include "simple_opengl.h"
+#include <glad/gl.h>
 
 #include <imgui.h>
 
@@ -17,13 +17,21 @@
 #include <memory>
 #include <optional>
 
-vtkSmartPointer<vtkPolyData> load_stl(const std::string &path);
+namespace vtk_viewer_helper {
 
-vtkSmartPointer<vtkActor> create_actor(vtkPolyData *data);
+    vtkSmartPointer<vtkPolyData> load_stl(const std::string &path);
 
-vtkSmartPointer<vtkActor> create_actor(const std::string &path);
+    vtkSmartPointer<vtkPolyData> load_obj(const std::string &path);
 
-void update_actor_pose(vtkActor *actor, const std::optional<Eigen::Isometry3d> &trans);
+    vtkSmartPointer<vtkPolyData> load_any(const std::string &path);
+
+    vtkSmartPointer<vtkActor> create_actor(vtkPolyData *data);
+
+    vtkSmartPointer<vtkActor> create_actor(const std::string &path);
+
+    void update_actor_pose(vtkActor *actor, const std::optional<Eigen::Isometry3d> &trans);
+
+}
 
 class vtk_viewer {
 public: