Merged augment pipeline.

CMakeLists.txt

@@ -13,12 +13,16 @@ add_executable(${PROJECT_NAME} src/main.cpp
         src/core/impl/event_timer.cpp
         src/core/impl/memory_pool.cpp
         src/core/impl/object_manager.cpp
+        src/module_v3/registration.cpp
+        src/module/impl/augment_manager.cpp
+        src/module/impl/camera_augment_helper.cpp
+        src/module/impl/image_augment_helper.cpp
         src/module/impl/image_player.cpp
         src/module/impl/image_viewer.cpp
         src/module/impl/image_streamer.cpp
         src/network_v3/sender_tcp.cpp
         src/network_v3/sender_udp_fec.cpp
-#        src/network_v3/receiver_tcp.cpp
+        #        src/network_v3/receiver_tcp.cpp
         src/network_v3/receiver_udp_fec.cpp
         #        src/network/impl/crc_checker.cpp
         src/network/impl/fragment/third_party/rs.c
@@ -26,6 +30,8 @@ add_executable(${PROJECT_NAME} src/main.cpp
         #        src/network/impl/fragment/frag_rs.cpp
         #        src/network/impl/multiplexer.cpp
         #        src/network/impl/network.cpp
+        src/render_v3/scene_render.cpp
+        src/render/impl/render_mesh.cpp
         src/render/impl/render_texture.cpp
         src/render/impl/render_tools.cpp
         src/render/impl/render_utility.cpp)
@@ -102,6 +108,14 @@ add_subdirectory(${ImGuiFileDialog_DIR} third_party/imgui_file_dialog)
 target_include_directories(ImGuiFileDialog PRIVATE ${IMGUI_DIR})
 target_link_libraries(${PROJECT_NAME} ImGuiFileDialog)
 
+# imGuIZMO config
+set(IMGUIZMO_DIR /home/tpx/src/imGuIZMO.quat-3.0/imGuIZMO.quat)
+target_include_directories(${PROJECT_NAME} PRIVATE ${IMGUIZMO_DIR})
+target_sources(${PROJECT_NAME} PRIVATE
+        ${IMGUIZMO_DIR}/imGuIZMOquat.cpp)
+target_compile_definitions(${PROJECT_NAME} PRIVATE
+        IMGUIZMO_IMGUI_FOLDER=${IMGUI_DIR}/)
+
 # yaml-cpp
 find_package(yaml-cpp REQUIRED)
 target_include_directories(${PROJECT_NAME} PRIVATE ${YAML_CPP_INCLUDE_DIR})
@@ -115,6 +129,12 @@ find_package(Boost REQUIRED COMPONENTS iostreams)
 target_include_directories(${PROJECT_NAME} PRIVATE ${Boost_INCLUDE_DIRS})
 target_link_libraries(${PROJECT_NAME} ${Boost_LIBRARIES})
 
+# VTK config
+find_package(VTK REQUIRED)
+target_link_libraries(${PROJECT_NAME} ${VTK_LIBRARIES})
+vtk_module_autoinit(TARGETS ${PROJECT_NAME} MODULES ${VTK_LIBRARIES})
+target_sources(${PROJECT_NAME} PRIVATE src/render_v3/vtk_viewer.cpp)
+
 # Eigen3 config
 find_package(Eigen3 REQUIRED)
 target_link_libraries(${PROJECT_NAME} Eigen3::Eigen)
@@ -182,4 +202,18 @@ target_sources(${PROJECT_NAME} PRIVATE src/codec/decoder_nvdec.cpp)
 # nvJPEG config
 target_link_libraries(${PROJECT_NAME} CUDA::nvjpeg)
 #target_sources(${PROJECT_NAME} PRIVATE src/codec/encoder_jpeg.cpp)
-#target_sources(${PROJECT_NAME} PRIVATE src/codec/decoder_nvjpeg.cpp)
+#target_sources(${PROJECT_NAME} PRIVATE src/codec/decoder_nvjpeg.cpp)
+
+# assimp config
+find_package(assimp REQUIRED)
+target_link_libraries(${PROJECT_NAME} assimp::assimp)
+
+# Sophiar2 config
+if (WIN32)
+    set(Sophiar2DIR D:/Program/Robot/Sophiar2)
+else ()
+    set(Sophiar2DIR /home/tpx/project/Sophiar2)
+endif ()
+add_subdirectory(${Sophiar2DIR}/src Sophiar2Lib)
+target_include_directories(${PROJECT_NAME} PRIVATE ${Sophiar2DIR}/src)
+target_link_libraries(${PROJECT_NAME} Sophiar2Lib)

data/config_remote_ar_20240410.yaml

@@ -1,5 +1,8 @@
 app_name: remote_ar
 
+sophiar_config: /home/tpx/project/DepthGuide/data/sophiar_config.json
+sophiar_start_var: tracker_all
+
 left_camera_name: "LeftEye"
 right_camera_name: "RightEye"
 
@@ -29,4 +32,19 @@ stereo_info:
     qw: 0.999987680292989
     qx: -0.000428742826329354
     qy: -0.00395921570373319
-    qz: -0.00296311539549921
+    qz: -0.00296311539549921
+
+left_camera_transform_var: left_camera_in_tracker_denoised
+right_camera_transform_var: right_camera_in_tracker_denoised
+
+augment_list:
+  - name: Bone
+    transform_var: femur_in_tracker_denoised
+    model: /home/tpx/project/TransparentAR/data/E1_AR/E1_Model.obj
+  - name: Plan
+    transform_var: femur_in_tracker_denoised
+    model: /home/tpx/project/TransparentAR/data/E1_AR/E1_Implant.obj
+#    background: Bone
+
+output_width: 1920
+output_height: 1080

data/sophiar_config.json

@@ -0,0 +1,658 @@
+{
+  "variable_list": [
+    {
+      "name": "probe_ref_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "probe_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "femur_ref_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "femur_in_femur_ref",
+      "type": "transform_obj",
+      "value": [
+        78.7201788489362,
+        73.11713596096779,
+        23.49171840405709,
+        -0.21932850052780128,
+        0.6801707950006277,
+        -0.2076550105848499,
+        -0.6679386910963054
+      ]
+    },
+    {
+      "name": "tibia_ref_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "tibia_in_tibia_ref",
+      "type": "transform_obj"
+    },
+    {
+      "name": "camera_ref_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "left_camera_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "right_camera_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "femur_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "tibia_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "probe_in_femur_ref",
+      "type": "transform_obj"
+    },
+    {
+      "name": "probe_in_tibia_ref",
+      "type": "transform_obj"
+    },
+    {
+      "name": "probe_in_femur",
+      "type": "transform_obj"
+    },
+    {
+      "name": "probe_in_tibia",
+      "type": "transform_obj"
+    },
+    {
+      "name": "probe_tip_in_femur_ref",
+      "type": "scalarxyz_obj"
+    },
+    {
+      "name": "probe_tip_in_tibia_ref",
+      "type": "scalarxyz_obj"
+    },
+    {
+      "name": "picked_point_in_femur_ref",
+      "type": "scalarxyz_obj"
+    },
+    {
+      "name": "picked_point_in_tibia_ref",
+      "type": "scalarxyz_obj"
+    },
+    {
+      "name": "femur_in_femur_ref_error",
+      "type": "double_obj"
+    },
+    {
+      "name": "tibia_in_tibia_ref_error",
+      "type": "double_obj"
+    },
+    {
+      "name": "probe_tip_in_femur",
+      "type": "scalarxyz_obj"
+    },
+    {
+      "name": "probe_tip_in_tibia",
+      "type": "scalarxyz_obj"
+    },
+    {
+      "name": "picked_point_in_femur",
+      "type": "scalarxyz_obj"
+    },
+    {
+      "name": "picked_point_in_tibia",
+      "type": "scalarxyz_obj"
+    },
+    {
+      "name": "left_camera_in_tracker_denoised",
+      "type": "transform_obj"
+    },
+    {
+      "name": "right_camera_in_tracker_denoised",
+      "type": "transform_obj"
+    },
+    {
+      "name": "femur_in_tracker_denoised",
+      "type": "transform_obj"
+    },
+    {
+      "name": "tibia_in_tracker_denoised",
+      "type": "transform_obj"
+    }
+  ],
+  "object_list": [
+    {
+      "type": "transform_tree",
+      "name": "transform_tree",
+      "init_config": {
+        "node_list": [
+          {
+            "name": "tracker"
+          },
+          {
+            "name": "probe_ref",
+            "parent": "tracker",
+            "transform_var_name": "probe_ref_in_tracker"
+          },
+          {
+            "name": "probe",
+            "parent": "probe_ref",
+            "transform": [
+              -0.23,
+              -13.98,
+              -119.65,
+              1,
+              0,
+              0,
+              0
+            ]
+          },
+          {
+            "name": "femur_ref",
+            "parent": "tracker",
+            "transform_var_name": "femur_ref_in_tracker"
+          },
+          {
+            "name": "femur",
+            "parent": "femur_ref",
+            "transform_var_name": "femur_in_femur_ref"
+          },
+          {
+            "name": "tibia_ref",
+            "parent": "tracker",
+            "transform_var_name": "tibia_ref_in_tracker"
+          },
+          {
+            "name": "tibia",
+            "parent": "tibia_ref",
+            "transform_var_name": "tibia_in_tibia_ref"
+          },
+          {
+            "name": "camera_ref",
+            "parent": "tracker",
+            "transform_var_name": "camera_ref_in_tracker"
+          },
+          {
+            "name": "left_camera",
+            "parent": "camera_ref",
+            "transform": [
+              33.2854,
+              -37.7917,
+              -24.4682,
+              0.6318,
+              0.3547,
+              0.4719,
+              -0.5023
+            ]
+          },
+          {
+            "name": "right_camera",
+            "parent": "camera_ref",
+            "transform": [
+              37.3992,
+              -56.7108,
+              -85.8847,
+              0.6304,
+              0.3605,
+              0.4738,
+              -0.4982
+            ]
+          }
+        ]
+      },
+      "start_config": {
+        "watch_list": [
+          {
+            "target": "left_camera",
+            "observer": "tracker",
+            "transform_var_name": "left_camera_in_tracker"
+          },
+          {
+            "target": "right_camera",
+            "observer": "tracker",
+            "transform_var_name": "right_camera_in_tracker"
+          },
+          {
+            "target": "femur",
+            "observer": "tracker",
+            "transform_var_name": "femur_in_tracker"
+          },
+          {
+            "target": "tibia",
+            "observer": "tracker",
+            "transform_var_name": "tibia_in_tracker"
+          },
+          {
+            "target": "probe",
+            "observer": "tracker",
+            "transform_var_name": "probe_in_tracker"
+          },
+          {
+            "target": "probe",
+            "observer": "femur_ref",
+            "transform_var_name": "probe_in_femur_ref"
+          },
+          {
+            "target": "probe",
+            "observer": "tibia_ref",
+            "transform_var_name": "probe_in_tibia_ref"
+          },
+          {
+            "target": "probe",
+            "observer": "femur",
+            "transform_var_name": "probe_in_femur"
+          },
+          {
+            "target": "probe",
+            "observer": "tibia",
+            "transform_var_name": "probe_in_tibia"
+          }
+        ]
+      }
+    },
+    {
+      "type": "ndi_interface",
+      "name": "ndi",
+      "init_config": {
+        "address_type": "ethernet",
+        "ip": "10.0.0.5",
+        "tcp_port": 8765,
+        "com_port": "/dev/ttyUSB0",
+        "tool_list": [
+          {
+            "rom_path": "/home/tpx/data/roms/GlassProbe_4Ball_4.rom",
+            "outputs": {
+              "transform": "probe_ref_in_tracker"
+            }
+          },
+          {
+            "rom_path": "/home/tpx/data/roms/Glass_4Ball_1_Camera_20240312.rom",
+            "outputs": {
+              "transform": "camera_ref_in_tracker"
+            }
+          },
+          {
+            "rom_path": "/home/tpx/data/roms/Glass_3Ball_6.rom",
+            "outputs": {
+              "transform": "femur_ref_in_tracker"
+            }
+          },
+          {
+            "rom_path": "/home/tpx/data/roms/Glass_3Ball_5_Combined.rom",
+            "outputs": {
+              "transform": "tibia_ref_in_tracker"
+            }
+          }
+        ]
+      },
+      "start_config": {
+        "allow_unreliable": true,
+        "prefer_stream_tracking": false
+      }
+    },
+    {
+      "type": "variable_validity_watcher",
+      "name": "probe_visibility_watcher",
+      "start_config": {
+        "variable_name": "probe_in_tracker"
+      }
+    },
+    {
+      "type": "transform_obj_validity_watcher",
+      "name": "camera_visibility_watcher",
+      "start_config": {
+        "variable_name": "camera_ref_in_tracker"
+      }
+    },
+    {
+      "type": "transform_obj_validity_watcher",
+      "name": "femur_visibility_watcher",
+      "start_config": {
+        "variable_name": "femur_ref_in_tracker"
+      }
+    },
+    {
+      "type": "transform_obj_validity_watcher",
+      "name": "tibia_visibility_watcher",
+      "start_config": {
+        "variable_name": "tibia_ref_in_tracker"
+      }
+    },
+    {
+      "type": "scalarxyz_transformer",
+      "name": "probe_tip_in_femur_ref_transformer",
+      "start_config": {
+        "transform_type": "point",
+        "transform_var_name": "probe_in_femur_ref",
+        "target_value": [
+          0,
+          0,
+          0
+        ],
+        "output_var_name": "probe_tip_in_femur_ref"
+      },
+      "dependencies": [
+        "ndi",
+        "transform_tree"
+      ]
+    },
+    {
+      "type": "scalarxyz_transformer",
+      "name": "probe_tip_in_tibia_ref_transformer",
+      "start_config": {
+        "transform_type": "point",
+        "transform_var_name": "probe_in_tibia_ref",
+        "target_value": [
+          0,
+          0,
+          0
+        ],
+        "output_var_name": "probe_tip_in_tibia_ref"
+      },
+      "dependencies": [
+        "ndi",
+        "transform_tree"
+      ]
+    },
+    {
+      "type": "transform_stabilizer",
+      "name": "point_picker_in_femur_ref",
+      "start_config": {
+        "stable_type": "point",
+        "input_var_name": "probe_tip_in_femur_ref",
+        "output_var_name": "picked_point_in_femur_ref",
+        "linear_tolerance_mm": 0.05,
+        "temporal_interval_s": 3
+      },
+      "dependencies": [
+        "probe_tip_in_femur_ref_transformer"
+      ]
+    },
+    {
+      "type": "transform_stabilizer",
+      "name": "point_picker_in_tibia_ref",
+      "start_config": {
+        "stable_type": "point",
+        "input_var_name": "probe_tip_in_tibia_ref",
+        "output_var_name": "picked_point_in_tibia_ref",
+        "linear_tolerance_mm": 0.05,
+        "temporal_interval_s": 3
+      },
+      "dependencies": [
+        "probe_tip_in_tibia_ref_transformer"
+      ]
+    },
+    {
+      "type": "scalarxyz_obj_watcher",
+      "name": "picked_point_watcher_for_femur_ref",
+      "start_config": {
+        "variable_name": "picked_point_in_femur_ref"
+      }
+    },
+    {
+      "type": "scalarxyz_obj_watcher",
+      "name": "picked_point_watcher_for_tibia_ref",
+      "start_config": {
+        "variable_name": "picked_point_in_tibia_ref"
+      }
+    },
+    {
+      "type": "landmark_registration",
+      "name": "femur_landmark",
+      "start_config": {
+        "fiducial_points": [
+          [
+            1.68412,
+            22.5766,
+            202.831
+          ],
+          [
+            13.5083,
+            -24.1831,
+            200.386
+          ],
+          [
+            26.621,
+            2.0151,
+            191.341
+          ]
+        ],
+        "point_var_name": "picked_point_in_femur_ref",
+        "transform_var_name": "femur_in_femur_ref",
+        "error_var_name": "femur_in_femur_ref_error"
+      },
+      "dependencies": [
+        "point_picker_in_femur_ref"
+      ]
+    },
+    {
+      "type": "landmark_registration",
+      "name": "tibia_landmark",
+      "start_config": {
+        "fiducial_points": [
+          [
+            47.351,
+            46.6799,
+            -70.5804
+          ],
+          [
+            -45.3668,
+            23.3452,
+            1.94304
+          ],
+          [
+            -55.3382,
+            -4.55638,
+            54.7446
+          ]
+        ],
+        "point_var_name": "picked_point_in_tibia_ref",
+        "transform_var_name": "tibia_in_tibia_ref",
+        "error_var_name": "tibia_in_tibia_ref_error"
+      },
+      "dependencies": [
+        "point_picker_in_tibia_ref"
+      ]
+    },
+    {
+      "type": "transform_obj_watcher",
+      "name": "femur_registration_result_watcher",
+      "start_config": {
+        "variable_name": "femur_in_femur_ref"
+      }
+    },
+    {
+      "type": "transform_obj_watcher",
+      "name": "tibia_registration_result_watcher",
+      "start_config": {
+        "variable_name": "tibia_in_tibia_ref"
+      }
+    },
+    {
+      "type": "double_obj_watcher",
+      "name": "femur_registration_error_watcher",
+      "start_config": {
+        "variable_name": "femur_in_femur_ref_error"
+      }
+    },
+    {
+      "type": "double_obj_watcher",
+      "name": "tibia_registration_error_watcher",
+      "start_config": {
+        "variable_name": "tibia_in_tibia_ref_error"
+      }
+    },
+    {
+      "type": "scalarxyz_transformer",
+      "name": "probe_tip_in_femur_transformer",
+      "start_config": {
+        "transform_type": "point",
+        "transform_var_name": "probe_in_femur",
+        "target_value": [
+          0,
+          0,
+          0
+        ],
+        "output_var_name": "probe_tip_in_femur"
+      },
+      "dependencies": [
+        "ndi",
+        "transform_tree"
+      ]
+    },
+    {
+      "type": "scalarxyz_transformer",
+      "name": "probe_tip_in_tibia_transformer",
+      "start_config": {
+        "transform_type": "point",
+        "transform_var_name": "probe_in_tibia",
+        "target_value": [
+          0,
+          0,
+          0
+        ],
+        "output_var_name": "probe_tip_in_tibia"
+      },
+      "dependencies": [
+        "ndi",
+        "transform_tree"
+      ]
+    },
+    {
+      "type": "transform_stabilizer",
+      "name": "point_picker_in_femur",
+      "start_config": {
+        "stable_type": "point",
+        "input_var_name": "probe_tip_in_femur",
+        "output_var_name": "picked_point_in_femur",
+        "linear_tolerance_mm": 0.05,
+        "temporal_interval_s": 3
+      },
+      "dependencies": [
+        "probe_tip_in_femur_transformer"
+      ]
+    },
+    {
+      "type": "transform_stabilizer",
+      "name": "point_picker_in_tibia",
+      "start_config": {
+        "stable_type": "point",
+        "input_var_name": "probe_tip_in_tibia",
+        "output_var_name": "picked_point_in_tibia",
+        "linear_tolerance_mm": 0.05,
+        "temporal_interval_s": 3
+      },
+      "dependencies": [
+        "probe_tip_in_tibia_transformer"
+      ]
+    },
+    {
+      "type": "scalarxyz_obj_watcher",
+      "name": "picked_point_watcher_for_femur",
+      "start_config": {
+        "variable_name": "picked_point_in_femur"
+      }
+    },
+    {
+      "type": "scalarxyz_obj_watcher",
+      "name": "picked_point_watcher_for_tibia",
+      "start_config": {
+        "variable_name": "picked_point_in_tibia"
+      }
+    },
+    {
+      "type": "kalman_denoiser",
+      "name": "left_camera_denoiser",
+      "init_config": {
+        "variable_in": "left_camera_in_tracker",
+        "variable_out": "left_camera_in_tracker_denoised"
+      }
+    },
+    {
+      "type": "kalman_denoiser",
+      "name": "right_camera_denoiser",
+      "init_config": {
+        "variable_in": "right_camera_in_tracker",
+        "variable_out": "right_camera_in_tracker_denoised"
+      }
+    },
+    {
+      "type": "kalman_denoiser",
+      "name": "femur_denoiser",
+      "init_config": {
+        "variable_in": "femur_in_tracker",
+        "variable_out": "femur_in_tracker_denoised"
+      }
+    },
+    {
+      "type": "kalman_denoiser",
+      "name": "tibia_denoiser",
+      "init_config": {
+        "variable_in": "tibia_in_tracker",
+        "variable_out": "tibia_in_tracker_denoised"
+      }
+    },
+    {
+      "type": "empty_object",
+      "name": "tracker_all",
+      "dependencies": [
+        "transform_tree",
+        "ndi",
+        "probe_visibility_watcher",
+        "camera_visibility_watcher",
+        "femur_visibility_watcher",
+        "tibia_visibility_watcher",
+        "left_camera_denoiser",
+        "right_camera_denoiser",
+        "femur_denoiser",
+        "tibia_denoiser"
+      ]
+    },
+    {
+      "type": "empty_object",
+      "name": "femur_landmark_all",
+      "dependencies": [
+        "tracker_all",
+        "femur_landmark",
+        "picked_point_watcher_for_femur_ref",
+        "femur_registration_result_watcher",
+        "femur_registration_error_watcher"
+      ]
+    },
+    {
+      "type": "empty_object",
+      "name": "tibia_landmark_all",
+      "dependencies": [
+        "tracker_all",
+        "tibia_landmark",
+        "picked_point_watcher_for_tibia_ref",
+        "tibia_registration_result_watcher",
+        "tibia_registration_error_watcher"
+      ]
+    },
+    {
+      "type": "empty_object",
+      "name": "femur_icp_all",
+      "dependencies": [
+        "tracker_all",
+        "point_picker_in_femur",
+        "picked_point_watcher_for_femur"
+      ]
+    },
+    {
+      "type": "empty_object",
+      "name": "tibia_icp_all",
+      "dependencies": [
+        "tracker_all",
+        "point_picker_in_tibia",
+        "picked_point_watcher_for_tibia"
+      ]
+    }
+  ]
+}

src/core/image_utility.hpp

@@ -3,6 +3,7 @@
 
 #include "cuda_helper.hpp"
 #include "memory_pool.h"
+#include "object_manager.h"
 
 #include <boost/integer.hpp>
 
@@ -340,4 +341,22 @@ using image_u8c4 = std::shared_ptr<smart_image<uchar4>>;
 using image_u16c1 = std::shared_ptr<smart_image<ushort1>>;
 using image_f32c1 = std::shared_ptr<smart_image<float1>>;
 
+inline cv::Size get_image_size(obj_name_type name) { // TODO: better dispatch method
+    auto ret = cv::Size();
+    auto img_type = OBJ_TYPE(name);
+    auto impl_func = [&](auto V) {
+        using T = std::remove_cvref_t<decltype(V)>;
+        if (img_type == typeid(T)) {
+            auto img = OBJ_QUERY(T, name);
+            if (img == nullptr) return;
+            ret = img->size();
+        }
+    };
+    impl_func(image_u8c1());
+    impl_func(image_u8c2());
+    impl_func(image_u8c3());
+    impl_func(image_u8c4());
+    return ret;
+}
+
 #endif //DEPTHGUIDE_IMAGE_UTILITY_HPP

src/core/math_helper.hpp

@@ -0,0 +1,43 @@
+#ifndef DEPTHGUIDE_MATH_HELPER_HPP
+#define DEPTHGUIDE_MATH_HELPER_HPP
+
+#include <glm/glm.hpp>
+#include <glm/gtc/quaternion.hpp>
+#include <glm/gtx/transform.hpp>
+
+#include <Eigen/Geometry>
+
+inline glm::mat4 to_transform_mat(glm::vec3 t, glm::vec3 r) {
+    static constexpr auto unit_x = glm::vec3(1.0f, 0.0f, 0.0f);
+    static constexpr auto unit_y = glm::vec3(0.0f, 1.0f, 0.0f);
+    static constexpr auto unit_z = glm::vec3(0.0f, 0.0f, 1.0f);
+    auto rot = glm::angleAxis(r.x, unit_x)
+               * glm::angleAxis(r.y, unit_y)
+               * glm::angleAxis(r.z, unit_z);
+    auto offset = glm::translate(t);
+    return offset * glm::mat4_cast(rot);
+}
+
+inline glm::mat4 to_mat4(const Eigen::Isometry3d &m) {
+    auto ret = glm::mat4();
+    auto &mat = m.matrix();
+    for (auto j = 0; j < 4; ++j)
+        for (auto i = 0; i < 4; ++i) {
+            ret[j][i] = mat(i, j);
+        }
+    return ret;
+}
+
+template<typename T>
+concept Vec3Type = requires(T t) {
+    { t.x } -> std::convertible_to<float>;
+    { t.y } -> std::convertible_to<float>;
+    { t.z } -> std::convertible_to<float>;
+};
+
+template<Vec3Type T>
+inline auto to_vec3(const T &v) {
+    return glm::vec3(v.x, v.y, v.z);
+}
+
+#endif //DEPTHGUIDE_MATH_HELPER_HPP

src/core/object_manager.h

@@ -134,7 +134,7 @@ extern object_manager *main_ob;
 #define OBJ_TYPE(name) \
     main_ob->query_type(name)
 
-#define OBJ_SAVE_TS(name) \
+#define OBJ_TS(name) \
     main_ob->query_save_ts(name)
 
 #define OBJ_STATS(name) \

src/core/yaml_utility.hpp

@@ -3,6 +3,9 @@
 
 #include <yaml-cpp/yaml.h>
 
+#include <cassert>
+#include <optional>
+
 template<typename T>
 inline auto yaml_load_number(const YAML::Node &conf, const char *name) {
     static_assert(std::is_arithmetic_v<T>);
@@ -15,20 +18,39 @@ inline auto yaml_load_str(const YAML::Node &conf, const char *name) {
     return conf[name].as<std::string>();
 }
 
+inline std::optional<std::string> yaml_try_load_str(const YAML::Node &conf, const char *name) {
+    if (auto item = conf[name]; item) {
+        assert(item.IsScalar());
+        return item.as<std::string>();
+    }
+    return {};
+}
+
 inline auto yaml_load_sub(const YAML::Node &conf, const char *name) {
     assert(conf[name].IsMap());
     return conf[name];
 }
 
+inline auto yaml_load_list(const YAML::Node &conf, const char *name) {
+    assert(conf[name].IsSequence());
+    return conf[name];
+}
+
 #define LOAD_NUMBER(type, name) \
     yaml_load_number<type>(conf, name)
 
 #define LOAD_STR(name) \
     yaml_load_str(conf, name)
 
+#define TRY_LOAD_STR(name) \
+    yaml_try_load_str(conf, name)
+
 #define LOAD_SUB(name) \
     yaml_load_sub(conf, name)
 
+#define LOAD_LIST(name) \
+    yaml_load_list(conf, name)
+
 #define FROM_YAML_IMPL(type) \
     static auto from_yaml(const YAML::Node &conf) { \
         auto ret = type(); \

src/impl/apps/depth_guide/depth_guide.cpp

@@ -32,7 +32,7 @@ app_depth_guide::app_depth_guide(const create_config &_conf) {
     bg_viewer = std::make_unique<image_viewer>(bg_viewer_conf);
 
     auto out_down_conf = viewport_downloader::create_config{
-            .type = PIX_RGBA, .stream = default_cuda_stream
+            .stream = default_cuda_stream
     };
     out_downloader = std::make_unique<viewport_downloader>(out_down_conf);
 
@@ -85,6 +85,6 @@ void app_depth_guide::render_background() {
     bg_viewer->render();
 
     // TODO: for test
-    auto bg_img = out_downloader->download_rgba();
+    auto bg_img = out_downloader->download_argb();
     OBJ_SAVE(img_out, bg_img);
 }

src/impl/apps/remote_ar/remote_ar.cpp

@@ -12,10 +12,20 @@ app_remote_ar::app_remote_ar(const create_config &_conf) {
     OBJ_SAVE(raw_right, image_u8c1());
     OBJ_SAVE(rgb_left, image_u8c3());
     OBJ_SAVE(rgb_right, image_u8c3());
+    OBJ_SAVE(aug_left, image_u8c3());
+    OBJ_SAVE(aug_right, image_u8c3());
+    OBJ_SAVE(img_out, image_u8c4()); // ARGB
 
     // process callbacks caused by OBJ_SAVE
     asio_ctx->poll();
 
+    // initialize sophiar
+    sophiar_thread = std::make_unique<std::thread>([conf_path = LOAD_STR("sophiar_config")] {
+        sophiar::run_sophiar(conf_path);
+    });
+    sophiar_conn = std::make_unique<sophiar_conn_type>();
+    sophiar_start_var = LOAD_STR("sophiar_start_var");
+
     // initialize modules
     auto mvs_conf = mvs_camera_ui::create_config{.ctx = asio_ctx};
     mvs_conf.cameras.push_back({.dev_name = LOAD_STR("left_camera_name"), .img_name = raw_left});
@@ -44,16 +54,74 @@ app_remote_ar::app_remote_ar(const create_config &_conf) {
         cam_right.img_proc->change_config({.is_mono = info.is_mono});
     });
 
+    auto aug_conf = augment_manager::create_config{
+            .item_list = augment_manager::item_list_from_yaml(LOAD_LIST("augment_list")),
+            .sophiar_conn = sophiar_conn.get()
+    };
+    aug_manager = std::make_unique<augment_manager>(aug_conf);
+
+    auto left_aug_conf = camera_augment_helper::create_config{
+            .fov = view_angle, .transform_var = LOAD_STR("left_camera_transform_var"),
+            .sophiar_conn = sophiar_conn.get(), .manager = aug_manager.get()
+    };
+    cam_left.aug_helper = std::make_unique<camera_augment_helper>(left_aug_conf);
+
+    auto right_aug_conf = left_aug_conf;
+    right_aug_conf.transform_var = LOAD_STR("right_camera_transform_var");
+    cam_right.aug_helper = std::make_unique<camera_augment_helper>(right_aug_conf);
+    cam_right.aug_helper->sync_with(cam_left.aug_helper.get());
+
+    auto left_aug_ren_conf = image_augment_helper::create_config{
+            .in_name = rgb_left, .out_name = aug_left,
+            .flip_image = true, .stream = &cam_left.stream
+    };
+    left_aug_ren_conf.render_func_list.emplace_back([this] { cam_left.aug_helper->render(); });
+    cam_left.aug_render = std::make_unique<image_augment_helper>(left_aug_ren_conf);
+
+    auto right_aug_ren_conf = image_augment_helper::create_config{
+            .in_name = rgb_right, .out_name = aug_right,
+            .flip_image = true, .stream = &cam_right.stream
+    };
+    right_aug_ren_conf.render_func_list.emplace_back([this] { cam_right.aug_helper->render(); });
+    cam_right.aug_render = std::make_unique<image_augment_helper>(right_aug_ren_conf);
+
+    auto output_size = cv::Size(
+            LOAD_NUMBER(int, "output_width"),
+            LOAD_NUMBER(int, "output_height"));
+    auto stereo_aug_conf = stereo_augment_helper::create_config{
+            .left_name = aug_left, .right_name = aug_right,
+            .out_name = img_out, .fbo_size = output_size,
+            .flip_image = true, .stream = default_cuda_stream
+    };
+    stereo_aug = std::make_unique<stereo_augment_helper>(stereo_aug_conf);
+
+    auto streamer_conf = image_streamer::create_config{
+            .img_name = img_out, .asio_ctx = asio_ctx,
+            .cuda_ctx = cuda_ctx, .stream = default_cuda_stream
+    };
+    out_streamer = std::make_unique<image_streamer>(streamer_conf);
+
     auto bg_viewer_conf = image_viewer::create_config{
-            .mode = VIEW_STEREO, .flip_y = true,
+            .mode = VIEW_STEREO, .flip_y = false,
             .stream = default_cuda_stream,
     };
-    auto &bg_extra_conf = bg_viewer_conf.extra.stereo;
-    bg_extra_conf.left_name = rgb_left;
-    bg_extra_conf.right_name = rgb_right;
+    auto &stereo_conf = bg_viewer_conf.extra.stereo;
+    stereo_conf.c_fmt = COLOR_RGB;
+    stereo_conf.left_name = aug_left;
+    stereo_conf.right_name = aug_right;
     bg_viewer = std::make_unique<image_viewer>(bg_viewer_conf);
 }
 
+void app_remote_ar::start_tracking() {
+    sophiar_conn->start_object(sophiar_start_var);
+}
+
+app_remote_ar::~app_remote_ar() {
+    // sophiar
+    sophiar::stop_sophiar();
+    sophiar_thread->join();
+}
+
 void app_remote_ar::show_ui() {
     if (ImGui::Begin("Remote AR Control")) {
         ImGui::PushItemWidth(200);
@@ -65,6 +133,33 @@ void app_remote_ar::show_ui() {
             cam_left.img_proc->show();
         }
 
+        if (ImGui::CollapsingHeader("Augment")) {
+            ImGui::SeparatorText("Tracker");
+            if (ImGui::Button("Start")) {
+                start_tracking();
+            }
+            {
+                ImGui::SeparatorText("Scene");
+                auto id_guard = imgui_id_guard("augment_scene");
+                aug_manager->show();
+            }
+            {
+                ImGui::SeparatorText("Camera");
+                auto id_guard = imgui_id_guard("augment_camera");
+                cam_left.aug_helper->show();
+            }
+        }
+
+        if (ImGui::CollapsingHeader("Stereo")) {
+            auto id_guard = imgui_id_guard("stereo");
+            stereo_aug->show();
+        }
+
+        if (ImGui::CollapsingHeader("Streamer")) {
+            auto id_guard = imgui_id_guard("streamer");
+            out_streamer->show();
+        }
+
         if (ImGui::CollapsingHeader("Debug")) {
             if (ImGui::TreeNode("Background")) {
                 bg_viewer->show();

src/impl/apps/remote_ar/remote_ar.h

@@ -4,16 +4,23 @@
 #include "core/event_timer.h"
 #include "core/object_manager.h"
 #include "device/mvs_camera_ui.h"
+#include "module/augment_manager.h"
+#include "module/camera_augment_helper.h"
+#include "module/image_augment_helper.h"
+#include "module/image_streamer.h"
 #include "module/image_viewer.h"
 #include "image_process/image_process_ui.h"
 #include "impl/app_base.h"
 
+// sophiar
+#include "core/local_connection.h"
+
 class app_remote_ar : public app_base {
 public:
 
     explicit app_remote_ar(const create_config &conf);
 
-    ~app_remote_ar() override = default;
+    ~app_remote_ar() override;
 
     const char *window_name() override { return "RemoteAR V4.-1"; }
 
@@ -23,14 +30,19 @@ public:
 
 private:
 
+    void start_tracking();
+
     enum obj_names : object_manager::name_type {
-        // raw images
         raw_left, raw_right,
         rgb_left, rgb_right,
+        aug_left, aug_right,
+        img_out,
     };
 
     struct camera_module {
         std::unique_ptr<image_process_ui> img_proc;
+        std::unique_ptr<camera_augment_helper> aug_helper;
+        std::unique_ptr<image_augment_helper> aug_render;
         smart_cuda_stream stream;
     };
 
@@ -38,9 +50,18 @@ private:
     io_context *asio_ctx;
     CUcontext *cuda_ctx;
 
+    // sophiar
+    using sophiar_conn_type = sophiar::local_connection;
+    std::unique_ptr<sophiar_conn_type> sophiar_conn;
+    std::unique_ptr<std::thread> sophiar_thread;
+    std::string sophiar_start_var;
+
     // modules
     std::unique_ptr<mvs_camera_ui> mvs_cam;
     std::unique_ptr<image_viewer> bg_viewer; // background viewer
+    std::unique_ptr<augment_manager> aug_manager;
+    std::unique_ptr<stereo_augment_helper> stereo_aug;
+    std::unique_ptr<image_streamer> out_streamer;
 
     camera_module cam_left;
     camera_module cam_right;

src/module/augment_manager.h

@@ -0,0 +1,65 @@
+#ifndef DEPTHGUIDE_AUGMENT_MANAGER_H
+#define DEPTHGUIDE_AUGMENT_MANAGER_H
+
+#include "core/yaml_utility.hpp"
+
+// sophiar
+#include "core/local_connection.h"
+
+#include <glm/glm.hpp>
+
+#include <optional>
+#include <memory>
+#include <vector>
+
+class augment_manager {
+public:
+
+    struct create_config {
+
+        struct item_type {
+
+            // basic information
+            std::string name;
+            std::string model_path;
+
+            // for depth-based alpha-blending
+            std::optional<std::string> bg_name;
+
+            // for sophiar
+            std::string transform_var;
+        };
+
+        using item_list_type = std::vector<item_type>;
+        item_list_type item_list;
+
+        // sophiar
+        using sophiar_conn_type = sophiar::local_connection;
+        sophiar_conn_type *sophiar_conn = nullptr;
+    };
+
+    using item_list_type = create_config::item_list_type;
+
+    static item_list_type item_list_from_yaml(const YAML::Node &conf);
+
+    explicit augment_manager(const create_config &conf);
+
+    ~augment_manager();
+
+    struct camera_info {
+        using transform_type = glm::mat4;
+        transform_type transform;
+        float fov; // field of view (in degree)
+        float clip_near, clip_far;
+    };
+
+    void render(const camera_info &info);
+
+    void show();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#endif //DEPTHGUIDE_AUGMENT_MANAGER_H

src/module/camera_augment_helper.h

@@ -0,0 +1,37 @@
+#ifndef DEPTHGUIDE_CAMERA_AUGMENT_HELPER_H
+#define DEPTHGUIDE_CAMERA_AUGMENT_HELPER_H
+
+#include "module/augment_manager.h"
+
+#include <memory>
+
+class camera_augment_helper {
+public:
+
+    struct create_config {
+        float fov = 0.0f; // field of view (in degree)
+        std::string transform_var; // for sophiar
+
+        // sophiar
+        using sophiar_conn_type = sophiar::local_connection;
+        sophiar_conn_type *sophiar_conn = nullptr;
+
+        augment_manager *manager = nullptr;
+    };
+
+    explicit camera_augment_helper(const create_config &conf);
+
+    ~camera_augment_helper();
+
+    void render();
+
+    void show();
+
+    void sync_with(camera_augment_helper *ui);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#endif //DEPTHGUIDE_CAMERA_AUGMENT_HELPER_H

src/module/image_augment_helper.h

@@ -0,0 +1,69 @@
+#ifndef DEPTHGUIDE_IMAGE_AUGMENT_HELPER_H
+#define DEPTHGUIDE_IMAGE_AUGMENT_HELPER_H
+
+#include "core/object_manager.h"
+#include "render/render_utility.h"
+
+#include <boost/container/small_vector.hpp>
+
+#include <memory>
+
+class image_augment_helper {
+public:
+
+    struct create_config {
+        obj_name_type in_name = invalid_obj_name;
+        obj_name_type out_name = invalid_obj_name;
+
+        static constexpr auto reserve_cnt = 4;
+        using render_func_type = std::function<void()>;
+        using render_func_list_type =
+                boost::container::small_vector<render_func_type, reserve_cnt>;
+        render_func_list_type render_func_list;
+
+        using fbo_config_type = smart_frame_buffer::create_config;
+        fbo_config_type fbo_conf;
+
+        bool flip_image = false;
+        bool follow_image_size = true;
+        smart_cuda_stream *stream = nullptr;
+    };
+
+    explicit image_augment_helper(const create_config &conf);
+
+    ~image_augment_helper();
+
+    void resize(cv::Size size);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+class stereo_augment_helper {
+public:
+
+    struct create_config {
+        obj_name_type left_name = invalid_obj_name;
+        obj_name_type right_name = invalid_obj_name;
+        obj_name_type out_name = invalid_obj_name;
+
+        cv::Size fbo_size;
+        bool flip_image = false;
+        smart_cuda_stream *stream = nullptr;
+    };
+
+    explicit stereo_augment_helper(const create_config &conf);
+
+    ~stereo_augment_helper();
+
+    void resize(cv::Size size);
+
+    void show();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#endif //DEPTHGUIDE_IMAGE_AUGMENT_HELPER_H

src/module/image_viewer.h

@@ -4,19 +4,24 @@
 #include "core/object_manager.h"
 #include "render/render_utility.h"
 
+#include <boost/container/static_vector.hpp>
+
 #include <memory>
 
 enum image_viewer_mode {
     VIEW_COLOR_ONLY,
     VIEW_DEPTH_ONLY,
     VIEW_COLOR_DEPTH,
-    VIEW_STEREO
+    VIEW_STEREO,
+    VIEW_CUSTOM
 };
 
 class image_viewer {
 public:
 
     struct create_config {
+        static constexpr auto max_custom_size = 2;
+
         image_viewer_mode mode = VIEW_COLOR_DEPTH;
         bool flip_y = false;
         smart_cuda_stream *stream = nullptr;
@@ -37,6 +42,17 @@ public:
                 obj_name_type left_name, right_name;
             } stereo;
         } extra = {};
+
+        struct {
+            struct item_type {
+                using render_func_type = std::function<void()>;
+                std::string name;
+                render_func_type render_func;
+            };
+            using item_list_type =
+                    boost::container::static_vector<item_type, max_custom_size>;
+            item_list_type item_list;
+        } custom;
     };
 
     explicit image_viewer(create_config conf);

src/module/impl/augment_manager.cpp

@@ -0,0 +1,133 @@
+#include "augment_manager_impl.h"
+#include "core/imgui_utility.hpp"
+#include "core/math_helper.hpp"
+
+#include <glm/gtc/type_ptr.hpp>
+
+#include <ranges>
+
+glm::mat4 augment_manager::impl::item_store_type::extra_transform() const {
+    return to_transform_mat(extra_offset, extra_rotation);
+}
+
+void augment_manager::impl::item_store_type::update_color() {
+    using sren = scene_render;
+    sren::set_actor_color(actor, glm::value_ptr(color), ambient_factor);
+    if (bg_actor != nullptr) {
+        sren::set_actor_alpha_factor(actor, alpha_factor);
+    }
+}
+
+augment_manager::impl::impl(const create_config &conf) {
+    sophiar_conn = conf.sophiar_conn;
+
+    // create items
+    for (auto &item_base: conf.item_list) {
+        auto item = item_store_type(item_base);
+
+        // find background if exists
+        using actor_tyep = item_store_type::actor_type;
+        if (item.bg_name.has_value()) {
+            auto &bg_name = item.bg_name.value();
+            assert(item_pool.contains(bg_name)); // background must appear before item
+            item.bg_actor = item_pool.at(bg_name).actor;
+        }
+
+        // create actor
+        item.actor = ren.create_actor(item.model_path.c_str(), item.bg_actor);
+        item.update_color();
+
+        item_pool.emplace(item.name, item);
+    }
+}
+
+void augment_manager::impl::update_scene() {
+    using sren = scene_render;
+    for (auto &item: item_pool | std::views::values) {
+        auto trans = sophiar_conn->query_transform_variable(item.transform_var);
+        if (!ignore_missing && !trans.has_value()) {
+            sren::set_actor_visibility(item.actor, false);
+        } else {
+            sren::set_actor_visibility(item.actor, item.visible);
+            if (!item.visible) continue;
+            sren::set_actor_transform(
+                    item.actor, to_mat4(trans.value()) * item.extra_transform());
+        }
+    }
+}
+
+void augment_manager::impl::render(const camera_info &info) {
+    update_scene();
+    ren.set_camera_transform(info.transform);
+    ren.set_camera_fov(info.fov);
+    ren.set_camera_clip_range(info.clip_near, info.clip_far);
+    ren.render();
+}
+
+void augment_manager::impl::show() {
+    ImGui::Checkbox("Enable", &enable);
+    if (!enable) return;
+    ImGui::SameLine();
+    ImGui::Checkbox("Ignore Missing", &ignore_missing);
+    for (auto &item: item_pool | std::views::values) {
+        if (ImGui::TreeNode(item.name.c_str())) {
+            ImGui::Checkbox("Visibility", &item.visible);
+            if (!item.visible) continue;
+            if (ImGui::ColorEdit3("Color", glm::value_ptr(item.color))) {
+                item.update_color();
+            }
+            if (ImGui::DragFloat("Ambient Factor", &item.ambient_factor, 0.005f, 0.0f, 1.0f)) {
+                item.update_color();
+            }
+            if (item.bg_actor != nullptr) {
+                if (ImGui::DragFloat("Alpha Factor", &item.alpha_factor, 0.005f, 0.0f, 1.0f)) {
+                    item.update_color();
+                }
+            }
+            ImGui::DragFloat3("Offset (mm)", glm::value_ptr(item.extra_offset),
+                              0.05f, 0.0f, 0.0f, "%.02f");
+            ImGui::DragFloat3("Offset (deg)", glm::value_ptr(item.extra_rotation),
+                              0.1f, -180.0f, 180.0f, "%.01f");
+            ImGui::TreePop();
+        }
+    }
+    if (ImGui::TreeNode("Light")) {
+        if (ImGui::Checkbox("Follow Camera", &enable_light_follow_camera)) {
+            ren.set_light_follow_camera(enable_light_follow_camera);
+        }
+        if (!enable_light_follow_camera) {
+            if (ImGui::gizmo3D("##direction", light_direction, ImGui::CalcItemWidth())) {
+                ren.set_light_direction(to_vec3(light_direction));
+            }
+        }
+        ImGui::TreePop();
+    }
+}
+
+augment_manager::item_list_type
+augment_manager::item_list_from_yaml(const YAML::Node &_conf) {
+    auto item_num = _conf.size();
+    auto ret = item_list_type(item_num);
+    for (auto k = 0; k < item_num; ++k) {
+        auto conf = _conf[k];
+        ret[k].name = LOAD_STR("name");
+        ret[k].transform_var = LOAD_STR("transform_var");
+        ret[k].model_path = LOAD_STR("model");
+        ret[k].bg_name = TRY_LOAD_STR("background");
+    }
+    return ret;
+}
+
+augment_manager::augment_manager(const create_config &conf)
+        : pimpl(std::make_unique<impl>(conf)) {
+}
+
+augment_manager::~augment_manager() = default;
+
+void augment_manager::render(const camera_info &info) {
+    pimpl->render(info);
+}
+
+void augment_manager::show() {
+    pimpl->show();
+}

src/module/impl/augment_manager_impl.h

@@ -0,0 +1,59 @@
+#ifndef DEPTHGUIDE_AUGMENT_MANAGER_IMPL_H
+#define DEPTHGUIDE_AUGMENT_MANAGER_IMPL_H
+
+#include "module/augment_manager.h"
+#include "render_v3/scene_render.h"
+
+#include <imGuIZMOquat.h>
+
+#include <Eigen/Geometry>
+
+struct augment_manager::impl {
+
+    using item_store_base_type = create_config::item_type;
+
+    struct item_store_type : public item_store_base_type {
+        using actor_type = scene_render::actor_token_type;
+        actor_type actor = nullptr;
+        actor_type bg_actor = nullptr;
+
+        bool visible = true;
+        glm::vec3 color = {1.0f, 0.0f, 0.0f};
+        float ambient_factor = 0.5f;
+        float alpha_factor = 0.25f;
+
+        glm::vec3 extra_offset;
+        glm::vec3 extra_rotation;
+
+        // combined from extra_offset and extra_rotation
+        glm::mat4 extra_transform() const;
+
+        void update_color();
+    };
+
+    using item_pool_type =
+            std::unordered_map<std::string, item_store_type>;
+    item_pool_type item_pool;
+
+    using sophiar_conn_type = create_config::sophiar_conn_type;
+    sophiar_conn_type *sophiar_conn = nullptr;
+
+    bool enable = true; // enable this module
+    bool enable_light_follow_camera = false;
+    bool ignore_missing = false; // show item even if missing from the tracker
+
+    vgm::Vec3 light_direction = {1.0f, 0.0f, 0.0f};
+
+    scene_render ren;
+
+    explicit impl(const create_config &conf);
+
+    void update_scene();
+
+    void render(const camera_info &info);
+
+    void show();
+
+};
+
+#endif //DEPTHGUIDE_AUGMENT_MANAGER_IMPL_H

src/module/impl/camera_augment_helper.cpp

@@ -0,0 +1,64 @@
+#include "camera_augment_helper_impl.h"
+#include "core/imgui_utility.hpp"
+#include "core/math_helper.hpp"
+
+#include <glm/gtc/type_ptr.hpp>
+
+glm::mat4 camera_augment_helper::impl::ui_config_type::extra_transform() const {
+    return to_transform_mat(extra_offset, extra_rotation);
+}
+
+camera_augment_helper::impl::impl(const create_config &conf) {
+    fov = conf.fov;
+    transform_var = conf.transform_var;
+    sophiar_conn = conf.sophiar_conn;
+    manager = conf.manager;
+}
+
+void camera_augment_helper::impl::render() {
+    // update camera transform
+    auto trans = sophiar_conn->query_transform_variable(transform_var);
+    if (trans.has_value()) {
+        transform = to_mat4(trans.value());
+    }
+
+    if (ui_conf->ignore_missing || trans.has_value()) {
+        auto conf = augment_manager::camera_info{
+                .transform = transform * ui_conf->extra_transform(),
+                .fov = fov,
+                .clip_near = ui_conf->clip.near,
+                .clip_far = ui_conf->clip.far
+        };
+        manager->render(conf);
+    }
+}
+
+void camera_augment_helper::impl::show() {
+    ImGui::Checkbox("Ignore Missing", &ui_conf->ignore_missing);
+    ImGui::DragFloat3("Offset (mm)", glm::value_ptr(ui_conf->extra_offset),
+                      0.05f, 0.0f, 0.0f, "%.02f");
+    ImGui::DragFloat3("Offset (deg)", glm::value_ptr(ui_conf->extra_rotation),
+                      0.1f, -180.0f, 180.0f, "%.01f");
+    ImGui::SliderFloat("Clip Near", &ui_conf->clip.near,
+                       1.0f, ui_conf->clip.far, "%.f", ImGuiSliderFlags_Logarithmic);
+    ImGui::SliderFloat("Clip Far", &ui_conf->clip.far,
+                       ui_conf->clip.near, 10000.0f, "%.f", ImGuiSliderFlags_Logarithmic);
+}
+
+camera_augment_helper::camera_augment_helper(const create_config &conf)
+        : pimpl(std::make_unique<impl>(conf)) {
+}
+
+camera_augment_helper::~camera_augment_helper() = default;
+
+void camera_augment_helper::render() {
+    pimpl->render();
+}
+
+void camera_augment_helper::show() {
+    pimpl->show();
+}
+
+void camera_augment_helper::sync_with(camera_augment_helper *ui) {
+    pimpl->ui_conf = ui->pimpl->ui_conf;
+}

src/module/impl/camera_augment_helper_impl.h

@@ -0,0 +1,42 @@
+#ifndef DEPTHGUIDE_CAMERA_AUGMENT_HELPER_IMPL_H
+#define DEPTHGUIDE_CAMERA_AUGMENT_HELPER_IMPL_H
+
+#include "module/camera_augment_helper.h"
+
+struct camera_augment_helper::impl {
+
+    struct ui_config_type {
+        glm::vec3 extra_offset;
+        glm::vec3 extra_rotation;
+        bool ignore_missing = false;
+
+        struct {
+            float near = 0.1f;
+            float far = 1000.0f;
+        } clip; // clip range
+
+        glm::mat4 extra_transform() const;
+    };
+
+    std::shared_ptr<ui_config_type> ui_conf =
+            std::make_shared<ui_config_type>();
+
+    float fov = 0.0f;
+
+    std::string transform_var; // for sophiar
+    glm::mat4 transform = glm::mat4(1.0f); // identity matrix, exclude extra_transform
+
+    using sophiar_conn_type = create_config::sophiar_conn_type;
+    sophiar_conn_type *sophiar_conn = nullptr;
+
+    augment_manager *manager = nullptr;
+
+    explicit impl(const create_config &conf);
+
+    void render();
+
+    void show();
+
+};
+
+#endif //DEPTHGUIDE_CAMERA_AUGMENT_HELPER_IMPL_H

src/module/impl/image_augment_helper.cpp

@@ -0,0 +1,153 @@
+#include "image_augment_helper_impl.h"
+#include "core/imgui_utility.hpp"
+
+image_augment_helper::impl::impl(const create_config &_conf) {
+    conf = _conf;
+
+    img_conn = OBJ_SIG(conf.in_name)->connect(
+            [this](auto name) { process(name); });
+
+    auto down_conf = viewport_downloader::create_config{
+            .stream = conf.stream};
+    img_downloader = std::make_unique<viewport_downloader>(down_conf);
+
+    if (!conf.follow_image_size) {
+        assert(!conf.fbo_conf.size.empty());
+        fbo.create(conf.fbo_conf);
+    }
+}
+
+image_augment_helper::impl::~impl() {
+    img_conn.disconnect();
+}
+
+void image_augment_helper::impl::process(obj_name_type name) {
+    assert(name == conf.in_name);
+
+    auto img_size = get_image_size(name);
+    if (img_size.empty()) return;
+    if (conf.follow_image_size) {
+        conf.fbo_conf.size = img_size;
+        fbo.create(conf.fbo_conf);
+    }
+
+    fbo.bind();
+
+    auto ren_conf = color_image_render::config_type{
+            .flip_y = conf.flip_image,
+            .stream = conf.stream,
+    };
+    image_ren.render(conf.in_name, ren_conf);
+    for (auto &func: conf.render_func_list) {
+        func();
+    }
+
+    auto out_img = img_downloader->download_rgb();
+    OBJ_SAVE(conf.out_name, out_img);
+
+    fbo.unbind();
+}
+
+void image_augment_helper::impl::resize(cv::Size size) {
+    assert(!conf.follow_image_size);
+    conf.fbo_conf.size = size;
+    fbo.create(conf.fbo_conf);
+}
+
+image_augment_helper::image_augment_helper(const create_config &conf)
+        : pimpl(std::make_unique<impl>(conf)) {
+}
+
+image_augment_helper::~image_augment_helper() = default;
+
+void image_augment_helper::resize(cv::Size size) {
+    pimpl->resize(size);
+}
+
+stereo_augment_helper::impl::impl(const create_config &_conf) {
+    conf = _conf;
+
+    auto trigger_conf = signal_group_and::create_config();
+    trigger_conf.name_list.emplace_back(conf.left_name);
+    trigger_conf.name_list.emplace_back(conf.right_name);
+    trigger_conf.cb_func = [this] { process(); };
+    trigger = std::make_unique<signal_group_and>(trigger_conf);
+
+    auto down_conf = viewport_downloader::create_config{
+            .stream = conf.stream};
+    img_downloader = std::make_unique<viewport_downloader>(down_conf);
+
+    fbo_conf.size = conf.fbo_size;
+    fbo_conf.color_fmts = {GL_RGB8};
+    fbo_conf.depth_fmt = GL_NONE;
+    fbo_conf.stencil_fmt = GL_NONE;
+    fbo.create(fbo_conf);
+}
+
+void stereo_augment_helper::impl::process() {
+    auto img_size = get_image_size(conf.left_name);
+    if (img_size.empty()) return;
+    if (follow_image_size) {
+        auto fbo_size = cv::Size(img_size.width * 2, img_size.height);
+        fbo_conf.size = fbo_size;
+        fbo.create(fbo_conf);
+    }
+
+    fbo.bind();
+
+    simple_rect left_rect, right_rect;
+    if (follow_image_size) {
+        left_rect = simple_rect{-1, -1, 1, 2};
+        right_rect = simple_rect{0, -1, 1, 2};
+    } else {
+        float width_normal = img_size.aspectRatio() /
+                             fbo.size.aspectRatio();
+        if (enable_halve_width) {
+            width_normal *= 0.5f;
+        }
+        left_rect = simple_rect{-1, -1, 1, 2}.fit_aspect(width_normal);
+        right_rect = simple_rect{0, -1, 1, 2}.fit_aspect(width_normal);
+    }
+
+    auto ren_conf = color_image_render::config_type{
+            .flip_y = conf.flip_image,
+            .stream = conf.stream,
+    };
+    ren_conf.range = left_rect;
+    image_ren.render(conf.left_name, ren_conf);
+    ren_conf.range = right_rect;
+    image_ren.render(conf.right_name, ren_conf);
+
+    auto out_img = img_downloader->download_argb(); // TODO: more elegant
+    OBJ_SAVE(conf.out_name, out_img);
+
+    fbo.unbind();
+}
+
+void stereo_augment_helper::impl::show() {
+    ImGui::Checkbox("Full Resolution", &follow_image_size);
+    if (!follow_image_size) {
+        ImGui::SameLine();
+        ImGui::Checkbox("Halve Width", &enable_halve_width);
+    }
+}
+
+void stereo_augment_helper::impl::resize(cv::Size size) {
+    assert(!follow_image_size);
+    fbo_conf.size = size;
+    fbo.create(fbo_conf);
+}
+
+stereo_augment_helper::stereo_augment_helper(const create_config &conf)
+        : pimpl(std::make_unique<impl>(conf)) {
+}
+
+stereo_augment_helper::~stereo_augment_helper() = default;
+
+void stereo_augment_helper::resize(cv::Size size) {
+    pimpl->resize(size);
+}
+
+void stereo_augment_helper::show() {
+    pimpl->show();
+}

src/module/impl/image_augment_helper_impl.h

@@ -0,0 +1,56 @@
+#ifndef DEPTHGUIDE_IMAGE_AUGMENT_HELPER_IMPL_H
+#define DEPTHGUIDE_IMAGE_AUGMENT_HELPER_IMPL_H
+
+#include "module/image_augment_helper.h"
+#include "module/signal_group.hpp"
+#include "module/viewport_downloader.hpp"
+#include "render/render_tools.h"
+
+struct image_augment_helper::impl {
+
+    create_config conf;
+    smart_frame_buffer fbo;
+    color_image_render image_ren;
+    std::unique_ptr<viewport_downloader> img_downloader;
+
+    using conn_type = boost::signals2::connection;
+    conn_type img_conn;
+
+    explicit impl(const create_config &conf);
+
+    ~impl();
+
+    void process(obj_name_type name);
+
+    void resize(cv::Size size);
+
+};
+
+struct stereo_augment_helper::impl {
+
+    using fbo_config_type = smart_frame_buffer::create_config;
+
+    create_config conf;
+    fbo_config_type fbo_conf;
+    smart_frame_buffer fbo;
+    color_image_render image_ren;
+    std::unique_ptr<viewport_downloader> img_downloader;
+
+    using conn_type = boost::signals2::connection;
+    conn_type img_conn;
+    std::unique_ptr<signal_group_and> trigger;
+
+    bool follow_image_size = false;
+    bool enable_halve_width = false;
+
+    explicit impl(const create_config &conf);
+
+    void process();
+
+    void resize(cv::Size size);
+
+    void show();
+
+};
+
+#endif //DEPTHGUIDE_IMAGE_AUGMENT_HELPER_IMPL_H

src/module/impl/image_viewer.cpp

@@ -37,6 +37,16 @@ void image_viewer::impl::show_stereo() {
     ImGui::RadioButton("Right", &chose_index, 1);
 }
 
+void image_viewer::impl::show_custom() {
+    auto &list = conf.custom.item_list;
+    for (auto k = 0; k < list.size(); ++k) {
+        ImGui::RadioButton(list[k].name.c_str(), &chose_index, k);
+        if (k + 1 != list.size()) {
+            ImGui::SameLine();
+        }
+    }
+}
+
 void image_viewer::impl::show() {
     ImGui::PushItemWidth(150);
     switch (conf.mode) {
@@ -55,6 +65,10 @@ void image_viewer::impl::show() {
             show_stereo();
             break;
         }
+        case VIEW_CUSTOM: {
+            show_custom();
+            break;
+        }
         default: {
             RET_ERROR;
         }
@@ -114,6 +128,10 @@ void image_viewer::impl::render_stereo() {
     }
 }
 
+void image_viewer::impl::render_custom() {
+    conf.custom.item_list[chose_index].render_func();
+}
+
 void image_viewer::impl::render() {
     switch (conf.mode) {
         case VIEW_COLOR_ONLY: {
@@ -133,6 +151,10 @@ void image_viewer::impl::render() {
             render_stereo();
             break;
         }
+        case VIEW_CUSTOM: {
+            render_custom();
+            break;
+        }
         default: {
             RET_ERROR;
         }

src/module/impl/image_viewer_impl.h

@@ -31,6 +31,8 @@ struct image_viewer::impl {
 
     void show_stereo();
 
+    void show_custom();
+
     void show();
 
     void render_color_obj(obj_name_type name);
@@ -42,6 +44,8 @@ struct image_viewer::impl {
 
     void render_stereo();
 
+    void render_custom();
+
     void render();
 
     explicit impl(create_config conf);

src/module/signal_group.hpp

@@ -0,0 +1,73 @@
+#ifndef DEPTHGUIDE_SIGNAL_GROUP_HPP
+#define DEPTHGUIDE_SIGNAL_GROUP_HPP
+
+#include "core/object_manager.h"
+
+#include <boost/container/flat_map.hpp>
+#include <boost/container/static_vector.hpp>
+
+#include <ranges>
+
+class signal_group_and {
+public:
+
+    static constexpr auto max_size = 4;
+
+    struct create_config {
+        using name_list_type =
+                boost::container::static_vector<obj_name_type, max_size>;
+        name_list_type name_list;
+
+        using cb_func_type = std::function<void()>;
+        cb_func_type cb_func;
+    };
+
+    explicit signal_group_and(const create_config &_conf) {
+        conf = _conf;
+        for (auto name: conf.name_list) {
+            ts_map.emplace(name, 0);
+            conn_list.emplace_back(
+                    OBJ_SIG(name)->connect([this](auto x) { notify(x); }));
+        }
+    }
+
+    ~signal_group_and() {
+        for (auto &conn: conn_list) {
+            conn.disconnect();
+        }
+    }
+
+private:
+
+    create_config conf;
+
+    using ts_map_type =
+            boost::container::flat_map<obj_name_type, bool, std::less<>,
+                    boost::container::static_vector<std::pair<obj_name_type, bool>, max_size>>;
+    ts_map_type ts_map;
+
+    using conn_list_type =
+            boost::container::static_vector<boost::signals2::connection, max_size>;
+    conn_list_type conn_list;
+
+    void notify(obj_name_type name) {
+        assert(ts_map.contains(name));
+        ts_map.at(name) = true;
+        check_status();
+    }
+
+    void check_status() {
+        if (std::ranges::all_of(
+                ts_map | std::views::values,
+                [](bool ok) { return ok; })) {
+            conf.cb_func();
+            // reset
+            std::ranges::for_each(
+                    ts_map,
+                    [](auto &x) { x.second = false; });
+        }
+    }
+
+};
+
+#endif //DEPTHGUIDE_SIGNAL_GROUP_HPP

src/module/viewport_downloader.hpp

@@ -7,17 +7,28 @@ class viewport_downloader {
 public:
 
     struct create_config {
-        image_pixel_type type = PIX_RGBA;
         smart_cuda_stream *stream = nullptr;
     };
 
     explicit viewport_downloader(create_config conf) {
-        type = conf.type;
         stream = conf.stream;
     }
 
-    image_u8c4 download_rgba() {
-        assert(type == PIX_RGBA);
+    image_u8c3 download_rgb() {
+        auto info = image_info_type<uchar3>();
+        info.loc = MEM_CUDA;
+        pbo.download_viewport(&info, GL_RGB, GL_UNSIGNED_BYTE, stream);
+        return create_image(info);
+    }
+
+    image_u8c3 download_rgba() {
+        auto info = image_info_type<uchar3>();
+        info.loc = MEM_CUDA;
+        pbo.download_viewport(&info, GL_RGBA, GL_UNSIGNED_BYTE, stream);
+        return create_image(info);
+    }
+
+    image_u8c4 download_argb() {
         auto info = image_info_type<uchar4>();
         info.loc = MEM_CUDA;
         pbo.download_viewport(&info, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, stream);
@@ -25,7 +36,6 @@ public:
     }
 
 private:
-    image_pixel_type type;
     smart_cuda_stream *stream;
     smart_pixel_buffer pbo;
 };

src/module_v3/registration.cpp

@@ -0,0 +1,464 @@
+#include "registration.h"
+#include "core/utility.hpp"
+#include "render_v3/vtk_viewer.h"
+
+#include <vtkActor.h>
+#include <vtkCellLocator.h>
+#include <vtkIterativeClosestPointTransform.h>
+#include <vtkLandmarkTransform.h>
+#include <vtkMatrix4x4.h>
+#include <vtkNamedColors.h>
+#include <vtkPolyData.h>
+#include <vtkProperty.h>
+#include <vtkSmartPointer.h>
+
+#include <queue>
+#include <vector>
+
+using namespace vtk_viewer_helper;
+
+struct registration::impl {
+
+    static constexpr auto MIN_REG_POINTS = 3;
+    static constexpr auto DIS_LIMIT_1 = 0.5;
+    static constexpr auto DIS_LIMIT_2 = 1.0;
+    static constexpr auto NAN_VALUE = std::numeric_limits<float>::quiet_NaN();
+
+    enum show_mode {
+        CONFIG,
+        COLLECTING,
+        FINISHED
+    };
+
+    struct target_store_type {
+        std::string name;
+        vtkSmartPointer<vtkPolyData> model;
+        vtkSmartPointer<vtkActor> model_actor;
+        vtkSmartPointer<vtkCellLocator> model_locator;
+        std::unique_ptr<smart_point_sets> all_points;
+        std::unique_ptr<smart_point_sets> pending_points;
+        std::unique_ptr<smart_point_sets> current_point;
+        std::unique_ptr<smart_point_sets> finished_points;
+        std::unique_ptr<smart_point_sets> level_points[3];
+
+        std::string target_var_name;
+        std::string point_var_name;
+        std::string collect_obj_name;
+        std::string probe_var_name;
+
+        bool is_finished = false;
+        float max_error = 0;
+    };
+
+    int cur_target_id = -1;
+    std::vector<target_store_type> targets;
+    target_store_type *cur_target = nullptr;
+
+    std::unique_ptr<vtk_viewer> viewer;
+    bool is_picking = false;
+    bool is_collecting = false;
+    std::vector<Eigen::Vector3d> collected_points;
+    std::queue<std::function<void()>> eq;
+
+    sophiar::local_connection *conn = nullptr;
+    vtkSmartPointer<vtkActor> probe_actor;
+    float probe_test_error = NAN_VALUE;
+
+    // colors
+    vtkColor4d target_color;
+    vtkColor4d idle_color;
+    vtkColor4d pending_color;
+    vtkColor4d current_color;
+    vtkColor4d finished_color;
+    vtkColor4d level_color[3]; // - 0.5mm - 1.0mm -
+
+    impl() {
+        viewer = std::make_unique<vtk_viewer>();
+
+        // preload colors
+        vtkNew<vtkNamedColors> colors;
+        target_color = colors->GetColor4d("silver");
+        idle_color = colors->GetColor4d("white");
+        pending_color = colors->GetColor4d("red");
+        current_color = colors->GetColor4d("yellow");
+        finished_color = colors->GetColor4d("lime");
+        level_color[0] = colors->GetColor4d("lime");
+        level_color[1] = colors->GetColor4d("yellow");
+        level_color[2] = colors->GetColor4d("red");
+    }
+
+    void switch_viewer_mode(show_mode mode) {
+        assert(cur_target != nullptr);
+        viewer->clear_actor();
+        viewer->add_actor(cur_target->model_actor);
+        viewer->add_actor(probe_actor);
+        switch (mode) {
+            case CONFIG: {
+                viewer->add_actor(cur_target->all_points->get_actor());
+                break;
+            }
+            case COLLECTING: {
+                viewer->add_actor(cur_target->pending_points->get_actor());
+                viewer->add_actor(cur_target->current_point->get_actor());
+                viewer->add_actor(cur_target->finished_points->get_actor());
+                break;
+            }
+            case FINISHED: {
+                for (auto &points: cur_target->level_points) {
+                    viewer->add_actor(points->get_actor());
+                }
+                break;
+            }
+        }
+    }
+
+    void add_target(const registration_target &conf) {
+        auto &target = targets.emplace_back();
+        target.name = conf.name;
+        target.model = load_any(conf.model_path);
+        target.model_actor = create_actor(target.model);
+        target.model_locator = vtkSmartPointer<vtkCellLocator>::New();
+        target.model_locator->SetDataSet(target.model);
+        target.model_locator->BuildLocator();
+
+        // copy information
+        target.target_var_name = conf.target_var_name;
+        target.collect_obj_name = conf.collect_obj_name;
+        target.point_var_name = conf.collect_var_name;
+        target.probe_var_name = conf.probe_var_name;
+
+        // create point sets
+        target.all_points = std::make_unique<smart_point_sets>();
+        target.pending_points = std::make_unique<smart_point_sets>();
+        target.current_point = std::make_unique<smart_point_sets>();
+        target.finished_points = std::make_unique<smart_point_sets>();
+        for (auto &points: target.level_points) {
+            points = std::make_unique<smart_point_sets>();
+        }
+
+        // set colors
+        target.model_actor->GetProperty()->SetColor(target_color.GetData());
+        target.all_points->get_actor()->GetProperty()->SetColor(idle_color.GetData());
+        target.pending_points->get_actor()->GetProperty()->SetColor(pending_color.GetData());
+        target.current_point->get_actor()->GetProperty()->SetColor(current_color.GetData());
+        target.finished_points->get_actor()->GetProperty()->SetColor(finished_color.GetData());
+        for (auto k = 0; k < 3; ++k) {
+            target.level_points[k]->get_actor()->GetProperty()->SetColor(level_color[k].GetData());
+        }
+    }
+
+    void change_target() {
+        assert(cur_target_id != -1);
+        cur_target = &targets[cur_target_id];
+        if (cur_target->is_finished) {
+            switch_viewer_mode(FINISHED);
+        } else {
+            switch_viewer_mode(CONFIG);
+        }
+        viewer->reset_camera();
+    }
+
+    bool progress_reg_point() {
+        if (!cur_target->current_point->empty()) {
+            cur_target->finished_points->add_point(
+                    cur_target->current_point->pop_front());
+        }
+        if (cur_target->pending_points->empty()) return false;
+        cur_target->current_point->add_point(
+                cur_target->pending_points->pop_front());
+        return true;
+    }
+
+    void start() {
+        assert(cur_target != nullptr);
+        assert(!is_collecting);
+        is_collecting = true;
+        cur_target->is_finished = false;
+        conn->mark_variable_disposal(cur_target->point_var_name);
+        CALL_CHECK(conn->start_object(cur_target->collect_obj_name));
+        collected_points.clear();
+
+        // copy points
+        cur_target->all_points->for_each([this](void *, const Eigen::Vector3d &point) {
+            cur_target->pending_points->add_point(point);
+        });
+        progress_reg_point();
+
+        // switch actor
+        switch_viewer_mode(COLLECTING);
+    }
+
+    void stop() {
+        assert(is_collecting);
+        is_collecting = false;
+        CALL_CHECK(conn->stop_object(cur_target->collect_obj_name));
+
+        // clear point set
+        cur_target->pending_points->clear();
+        cur_target->current_point->clear();
+        cur_target->finished_points->clear();
+
+        // switch actor
+        if (cur_target->is_finished) {
+            switch_viewer_mode(FINISHED);
+        } else {
+            switch_viewer_mode(CONFIG);
+        }
+    }
+
+    auto calc_closest_point(const Eigen::Vector3d &point) {
+        Eigen::Vector3d close_point;
+        vtkIdType cell_id;
+        int sub_id;
+        double dis2;
+        cur_target->model_locator->FindClosestPoint(
+                point.data(), close_point.data(), cell_id, sub_id, dis2);
+        return std::make_tuple(close_point, std::sqrt(dis2));
+    }
+
+    // return if it needs to continue
+    bool calc_result() {
+        // prepare landmark
+        auto num_points = collected_points.size();
+        if (num_points < MIN_REG_POINTS) return true;
+        auto source_points = Eigen::Matrix3Xd{3, num_points};
+        auto target_points = Eigen::Matrix3Xd{3, num_points};
+        for (auto k = 0; k < num_points; ++k) {
+            source_points.col(k) = (*cur_target->all_points)[k];
+        }
+        for (auto k = 0; k < num_points; ++k) {
+            target_points.col(k) = collected_points[k];
+        }
+
+        // calculate landmark
+        auto result = (Eigen::Isometry3d) Eigen::umeyama(source_points, target_points, false);
+
+        // prepare icp
+        vtkNew<vtkPoints> icp_points;
+        auto landmark_inv = result.inverse();
+        for (auto k = 0; k < num_points; ++k) {
+            Eigen::Vector3d point = landmark_inv * collected_points[k];
+            icp_points->InsertNextPoint(point.data());
+        }
+
+        // calculate icp
+        vtkNew<vtkIterativeClosestPointTransform> icp;
+        vtkNew<vtkPolyData> tmp_poly;
+        tmp_poly->SetPoints(icp_points);
+        icp->GetLandmarkTransform()->SetModeToRigidBody();
+        icp->SetSource(tmp_poly);
+        icp->SetTarget(cur_target->model);
+        icp->Modified();
+        icp->Update();
+
+        // refine result
+        Eigen::Isometry3d trans_delta;
+        for (auto i = 0; i < 4; ++i)
+            for (auto j = 0; j < 4; ++j) {
+                trans_delta(i, j) = icp->GetMatrix()->GetElement(i, j);
+            }
+        result = result * trans_delta.inverse();
+
+        // commit result
+        conn->update_transform_variable(cur_target->target_var_name, result);
+
+        // calculate error only when all points are collected
+        if (num_points != cur_target->all_points->size()) return true;
+
+        // calculate error
+        cur_target->max_error = 0;
+        for (auto &points: cur_target->level_points) {
+            points->clear();
+        }
+        auto result_inv = result.inverse();
+        for (auto k = 0; k < num_points; ++k) {
+            Eigen::Vector3d point = result_inv * collected_points[k];
+
+            // find the closest point
+            double dis;
+            std::tie(std::ignore, dis) = calc_closest_point(point);
+
+            // update results
+            if (dis > cur_target->max_error) {
+                cur_target->max_error = dis;
+            }
+            if (dis < DIS_LIMIT_1) {
+                cur_target->level_points[0]->add_point(point);
+            } else if (dis < DIS_LIMIT_2) {
+                cur_target->level_points[1]->add_point(point);
+            } else {
+                cur_target->level_points[2]->add_point(point);
+            }
+        }
+
+        cur_target->is_finished = true;
+        return false;
+    }
+
+    void try_picking() {
+        assert(is_picking);
+        auto val = viewer->get_picked_point();
+        if (!val.has_value()) return;
+        cur_target->all_points->add_point(val.value());
+    }
+
+    void try_collect() {
+        assert(is_collecting);
+        auto val = conn->query_scalarxyz_variable(cur_target->point_var_name);
+        if (!val.has_value()) return;
+        collected_points.emplace_back(val.value());
+        SPDLOG_INFO("Collected point ({}, {}, {}).", val->x(), val->y(), val->z());
+        calc_result();
+        if (!progress_reg_point()) {
+            stop();
+        }
+    }
+
+    void show() {
+        if (ImGui::Begin("Registration Control")) {
+            if (is_collecting || is_picking) {
+                ImGui::BeginDisabled();
+            }
+            const char *target_name = (cur_target == nullptr) ? nullptr : cur_target->name.c_str();
+            if (ImGui::BeginCombo("Target", target_name)) {
+                for (auto k = 0; k < targets.size(); ++k) {
+                    bool is_selected = (k == cur_target_id);
+                    if (ImGui::Selectable(targets[k].name.c_str(), is_selected)) {
+                        cur_target_id = k;
+                        eq.emplace([this] { change_target(); });
+                    }
+                    if (is_selected) {
+                        ImGui::SetItemDefaultFocus();
+                    }
+                }
+                ImGui::EndCombo();
+            }
+            if (is_collecting || is_picking) {
+                ImGui::EndDisabled();
+            }
+            if (cur_target != nullptr) {
+                auto point_set = cur_target->all_points.get();
+                if (ImGui::CollapsingHeader("Actions")) {
+                    if (is_collecting) {
+                        ImGui::BeginDisabled();
+                    }
+                    if (ImGui::Checkbox("Config Points", &is_picking)) {
+                        if (is_picking) {
+                            eq.emplace([this] {
+                                switch_viewer_mode(CONFIG);
+                                viewer->start_picking();
+                            });
+                        } else {
+                            eq.emplace([this] { viewer->stop_picking(); });
+                        }
+                    }
+                    if (is_collecting) {
+                        ImGui::EndDisabled();
+                    }
+                    if (!is_picking && point_set->size() >= MIN_REG_POINTS) {
+                        ImGui::SameLine();
+                        if (!is_collecting) {
+                            if (ImGui::Button("Start")) {
+                                eq.emplace([this] { start(); });
+                            }
+                        } else {
+                            if (ImGui::Button("Stop")) {
+                                eq.emplace([this] { stop(); });
+                            }
+                        }
+                    }
+
+                }
+                if (ImGui::CollapsingHeader("Infos")) {
+                    void *token_delete = nullptr;
+                    if (ImGui::TreeNode("Fiducial Points")) {
+                        ImGui::PushItemWidth(200);
+                        point_set->for_each([&](void *token, const Eigen::Vector3d &point) {
+                            Eigen::Vector3f point_f = point.cast<float>();
+                            ImGui::PushID(token);
+                            ImGui::Bullet();
+                            ImGui::BeginDisabled();
+                            ImGui::InputFloat3("", point_f.data());
+                            ImGui::EndDisabled();
+                            if (is_picking) {
+                                ImGui::SameLine();
+                                if (ImGui::SmallButton("Delete")) {
+                                    token_delete = token;
+                                }
+                            }
+                            ImGui::PopID();
+                        });
+                        ImGui::PopItemWidth();
+                        ImGui::TreePop();
+                    }
+                    if (token_delete != nullptr) {
+                        eq.emplace([=] { point_set->remove_point(token_delete); });
+                    }
+                    ImGui::PushItemWidth(100);
+                    ImGui::Bullet();
+                    ImGui::InputFloat("Probe Test Error (mm)", &probe_test_error);
+                    if (cur_target->is_finished) {
+                        ImGui::Bullet();
+                        ImGui::InputFloat("Max Fiducial Error (mm)", &cur_target->max_error);
+                    }
+                    ImGui::PopItemWidth();
+                }
+            }
+        }
+        ImGui::End();
+        if (cur_target == nullptr) return;
+
+        if (ImGui::Begin("Registration View", nullptr, vtk_viewer::no_scroll_flag)) {
+            viewer->show();
+        }
+        ImGui::End();
+    }
+
+    void process() {
+        while (!eq.empty()) {
+            eq.front()();
+            eq.pop();
+        }
+        if (is_picking) {
+            try_picking();
+        } else if (is_collecting) {
+            try_collect();
+        }
+
+        // update probe transform
+        if (cur_target != nullptr) {
+            auto trans = conn->query_transform_variable(cur_target->probe_var_name);
+            update_actor_pose(probe_actor, trans);
+            if (trans.has_value()) {
+                Eigen::Vector3d point = trans.value().translation();
+                std::tie(std::ignore, probe_test_error) = calc_closest_point(point);
+            } else {
+                probe_test_error = NAN_VALUE;
+            }
+        }
+    }
+
+};
+
+registration::~registration() = default;
+
+registration *registration::create(const registration_config &conf) {
+    auto pimpl = new impl{};
+    pimpl->conn = conf.conn;
+    pimpl->probe_actor = create_actor(conf.probe_model_path);
+    auto ret = new registration{};
+    ret->pimpl.reset(pimpl);
+    return ret;
+}
+
+void registration::add_target(const registration_target &item) {
+    pimpl->add_target(item);
+}
+
+void registration::show() {
+    pimpl->show();
+}
+
+void registration::process() {
+    pimpl->process();
+}

src/module_v3/registration.h

@@ -0,0 +1,48 @@
+#ifndef REMOTEAR3_REGISTRATION_H
+#define REMOTEAR3_REGISTRATION_H
+
+#include "core/local_connection.h"
+
+#include <vtkPolyData.h>
+
+#include <Eigen/Dense>
+
+#include <memory>
+#include <string_view>
+#include <vector>
+
+struct registration_config {
+    sophiar::local_connection *conn;
+    std::string probe_model_path;
+};
+
+struct registration_target {
+    std::string name;
+    std::string model_path;
+    std::vector<Eigen::Vector3d> pre_points;
+    std::string target_var_name;
+    std::string collect_var_name;
+    std::string collect_obj_name;
+    std::string probe_var_name;
+};
+
+class registration {
+public:
+
+    ~registration();
+
+    static registration *create(const registration_config &conf);
+
+    void add_target(const registration_target &item);
+
+    void show(); // show ImGui windows
+
+    void process();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+
+#endif //REMOTEAR3_REGISTRATION_H

src/network/binary_utility.hpp

@@ -116,6 +116,14 @@ struct data_type {
         *this = next;
     }
 
+    template<typename T>
+    T *at(size_t pos) {
+        static_assert(std::is_trivial_v<T>);
+        assert(size % sizeof(T) == 0);
+        assert(pos < size / sizeof(T));
+        return ((T *) start_ptr()) + pos;
+    }
+
     uint8_t *start_ptr() const {
         return ptr;
     }

src/render/impl/render_mesh.cpp

@@ -0,0 +1,168 @@
+#include "render_mesh_impl.h"
+
+#include <assimp/postprocess.h>
+
+namespace render_mesh_impl {
+
+    template<typename TReal>
+    inline glm::vec3 to_vec3(aiVector3t<TReal> vec) {
+        return {vec.x, vec.y, vec.z};
+    }
+
+    mesh_cache_type mesh_cache;
+
+}
+
+mesh_type::gl_info_type::~gl_info_type() {
+    glDeleteVertexArrays(1, &vao);
+    glDeleteBuffers(1, &vbo);
+    glDeleteBuffers(1, &ebo);
+}
+
+auto mesh_type::impl::create(const mesh_type::create_config &conf)
+-> std::unique_ptr<impl> {
+    auto ret = std::make_unique<impl>();
+    auto scene = ret->importer.ReadFile(
+            conf.path, aiProcessPreset_TargetRealtime_MaxQuality);
+    if (scene == nullptr) {
+        RET_ERROR_E;
+    }
+    assert(scene->mNumMeshes == 1);
+    auto mesh = scene->mMeshes[0];
+    assert(mesh->mPrimitiveTypes == aiPrimitiveType_TRIANGLE);
+    assert(mesh->mNormals != nullptr);
+    ret->mesh = mesh;
+    return ret;
+}
+
+void mesh_type::impl::upload_gl() {
+    assert(gl_info == nullptr);
+
+    // create vertex buffer data
+    auto num_vert = mesh->mNumVertices;
+    vbo_data = data_type(sizeof(v_type) * num_vert);
+    for (auto k = 0; k < num_vert; ++k) {
+        auto info = vbo_data.at<v_type>(k);
+        info->pos = to_vec3(mesh->mVertices[k]);
+        info->normal = to_vec3(mesh->mNormals[k]);
+    }
+
+    // create element buffer data
+    auto num_face = mesh->mNumFaces;
+    ebo_data = data_type(sizeof(e_type) * num_face);
+    for (auto k = 0; k < num_face; ++k) {
+        auto &info = *ebo_data.at<e_type>(k);
+        auto face = mesh->mFaces[k];
+        assert(face.mNumIndices == face_num_vert);
+        for (auto i = 0; i < face.mNumIndices; ++i) {
+            info[i] = face.mIndices[i];
+        }
+    }
+
+    // config vertex buffer
+    GLuint vbo;
+    glGenBuffers(1, &vbo);
+    glBindBuffer(GL_ARRAY_BUFFER, vbo);
+    glBufferData(GL_ARRAY_BUFFER, vbo_data.size, vbo_data.start_ptr(), GL_STATIC_DRAW);
+
+    // fill element buffer
+    GLuint ebo;
+    glGenBuffers(1, &ebo);
+    glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
+    glBufferData(GL_ELEMENT_ARRAY_BUFFER, ebo_data.size, ebo_data.start_ptr(), GL_STATIC_DRAW);
+
+    // config vertex array
+    GLuint vao;
+    glGenVertexArrays(1, &vao);
+    glBindVertexArray(vao);
+    glEnableVertexAttribArray(0);
+    glEnableVertexAttribArray(1);
+    glVertexAttribPointer(0, 3, GL_FLOAT, false, sizeof(v_type), (void *) offsetof(v_type, pos));
+    glVertexAttribPointer(1, 3, GL_FLOAT, false, sizeof(v_type), (void *) offsetof(v_type, normal));
+
+    gl_info = std::make_unique<gl_info_type>();
+    gl_info->vao = vao;
+    gl_info->vbo = vbo;
+    gl_info->ebo = ebo;
+    gl_info->num_triangle = num_face;
+}
+
+mesh_type::gl_info_type *mesh_type::impl::get_gl_info() {
+    if (gl_info == nullptr) [[unlikely]] {
+        upload_gl();
+    }
+    assert(gl_info != nullptr);
+    return gl_info.get();
+}
+
+mesh_type::~mesh_type() = default;
+
+mesh_type::pointer mesh_type::create(const create_config &conf) {
+    // prevent duplicated create
+    if (auto iter = mesh_cache.find(conf.path);
+            iter != mesh_cache.end()) [[likely]] {
+        return iter->second;
+    }
+
+    auto pimpl = impl::create(conf);
+    if (pimpl == nullptr) return nullptr;
+    auto ret = std::make_shared<this_type>();
+    ret->pimpl = std::move(pimpl);
+    mesh_cache.emplace(conf.path, ret);
+    return ret;
+}
+
+mesh_type::gl_info_type *mesh_type::get_gl_info() {
+    return pimpl->get_gl_info();
+}
+
+namespace render_mesh_impl {
+
+    using pg_type = std::unique_ptr<smart_program>;
+    pg_type pg_normal; // render mesh in normal ways
+
+    void draw_mesh(mesh_type *mesh) {
+        auto gl_info = mesh->get_gl_info();
+        glBindVertexArray(gl_info->vao);
+        glBindBuffer(GL_ARRAY_BUFFER, gl_info->vbo);
+        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, gl_info->ebo);
+        glDrawElements(GL_TRIANGLES, 3 * gl_info->num_triangle, GL_UNSIGNED_INT, nullptr);
+    }
+
+    void ren_mesh_normal(const mesh_render_info &info) {
+        if (pg_normal == nullptr) {
+            pg_normal = std::unique_ptr<smart_program>(
+                    smart_program::create("mesh_normal",
+                                          {{GL_VERTEX_SHADER,   "mesh_normal.vert"},
+                                           {GL_FRAGMENT_SHADER, "mesh_normal.frag"}}));
+        }
+        assert(pg_normal != nullptr);
+        pg_normal->use();
+
+        // mesh info
+        pg_normal->set_uniform_mat4("model_mat", info.model.transform);
+        pg_normal->set_uniform_vec3("material.ambient", info.model.material.ambient);
+        pg_normal->set_uniform_vec3("material.diffuse", info.model.material.diffuse);
+
+        // scene info
+        pg_normal->set_uniform_mat4("camera_mat", info.camera.project
+                                                  * info.camera.transform);
+        pg_normal->set_uniform_vec3("light.direction", info.light.direction);
+
+        glEnable(GL_DEPTH_TEST);
+        draw_mesh(info.model.mesh);
+    }
+
+}
+
+void render_mesh(const mesh_render_info &info) {
+    switch (info.mode) {
+        case MESH_NORMAL: {
+            ren_mesh_normal(info);
+            break;
+        }
+        default: {
+            RET_ERROR;
+        }
+    }
+}

src/render/impl/render_mesh_impl.h

@@ -0,0 +1,53 @@
+#ifndef DEPTHGUIDE_RENDER_MESH_IMPL_H
+#define DEPTHGUIDE_RENDER_MESH_IMPL_H
+
+#include "render/render_mesh.h"
+#include "network/binary_utility.hpp"
+
+#include <assimp/Importer.hpp>
+#include <assimp/scene.h>
+
+#include <unordered_map>
+
+namespace render_mesh_impl {
+
+    void draw_mesh(mesh_type *mesh);
+
+    void ren_mesh_normal(const mesh_render_info &info);
+
+    using mesh_cache_type =
+            std::unordered_map<std::string, std::shared_ptr<mesh_type>>;
+    extern mesh_cache_type mesh_cache;
+
+}
+
+using namespace render_mesh_impl;
+
+struct mesh_type::impl {
+
+    struct v_type { // vertex type
+        glm::vec3 pos;
+        glm::vec3 normal;
+    };
+
+    static constexpr auto face_num_vert = 3;
+    using e_type = std::array<GLuint, face_num_vert>;
+
+    data_type vbo_data;
+    data_type ebo_data;
+
+    Assimp::Importer importer;
+    aiMesh *mesh = nullptr;
+
+    std::unique_ptr<gl_info_type> gl_info;
+
+    static auto create(const create_config &conf)
+    -> std::unique_ptr<impl>;
+
+    void upload_gl();
+
+    gl_info_type *get_gl_info();
+
+};
+
+#endif //DEPTHGUIDE_RENDER_MESH_IMPL_H

src/render/impl/render_texture.cpp

@@ -14,6 +14,8 @@ namespace render_texture_impl {
 
     using pg_type = std::unique_ptr<smart_program>;
     pg_type pg_rgb; // render rgb texture
+    pg_type pg_rgba; // render rgba texture
+    pg_type pg_rgb_d; // render rgb and depth texture
     pg_type pg_nv12; // render nv12 textures
 //    pg_type pg_depth_only;
 //    pg_type pg_color_depth;
@@ -94,6 +96,51 @@ namespace render_texture_impl {
         draw();
     }
 
+    void ren_rgba_only(const tex_render_info &info) {
+        if (pg_rgba == nullptr) {
+            pg_rgba = std::unique_ptr<smart_program>(
+                    smart_program::create("tex_rgba",
+                                          {{GL_VERTEX_SHADER,   "tex.vert"},
+                                           {GL_FRAGMENT_SHADER, "tex_rgba.frag"}}));
+        }
+        assert(pg_rgba != nullptr);
+        pg_rgba->use();
+
+        pg_rgba->set_uniform_f("opacity", info.color.opacity);
+
+        glActiveTexture(GL_TEXTURE0 + 0);
+        glBindTexture(GL_TEXTURE_2D, info.color.id);
+        pg_rgba->set_uniform_i("c_tex", 0);
+
+        glDisable(GL_DEPTH_TEST);
+        config_buffers(info);
+        draw();
+    }
+
+    void ren_rgb_d(const tex_render_info &info) {
+        if (pg_rgb_d == nullptr) {
+            pg_rgb_d = std::unique_ptr<smart_program>(
+                    smart_program::create("tex_rgb_d",
+                                          {{GL_VERTEX_SHADER,   "tex.vert"},
+                                           {GL_FRAGMENT_SHADER, "tex_rgb_d.frag"}}));
+        }
+        assert(pg_rgb_d != nullptr);
+        pg_rgb_d->use();
+
+        pg_rgb_d->set_uniform_f("alpha", info.color.alpha);
+
+        glActiveTexture(GL_TEXTURE0 + 0);
+        glBindTexture(GL_TEXTURE_2D, info.color.id);
+        pg_rgb_d->set_uniform_i("c_tex", 0);
+        glActiveTexture(GL_TEXTURE0 + 1);
+        glBindTexture(GL_TEXTURE_2D, info.depth.id);
+        pg_rgb_d->set_uniform_i("d_tex", 1);
+
+        glEnable(GL_DEPTH_TEST);
+        config_buffers(info);
+        draw();
+    }
+
     void ren_nv12_only(const tex_render_info &info) {
         if (pg_nv12 == nullptr) {
             pg_nv12 = std::unique_ptr<smart_program>(
@@ -124,6 +171,10 @@ namespace render_texture_impl {
                 ren_rgb_only(info);
                 break;
             }
+            case COLOR_RGBA: {
+                ren_rgba_only(info);
+                break;
+            }
             case COLOR_NV12: {
                 ren_nv12_only(info);
                 break;
@@ -134,6 +185,19 @@ namespace render_texture_impl {
         }
     }
 
+    // render color and depth
+    void ren_c_d(const tex_render_info &info) {
+        switch (info.color.fmt) {
+            case COLOR_RGB: {
+                ren_rgb_d(info);
+                break;
+            }
+            default: {
+                RET_ERROR;
+            }
+        }
+    }
+
 }
 
 using namespace render_texture_impl;
@@ -144,6 +208,10 @@ void render_texture(const tex_render_info &info) {
             ren_c_only(info);
             break;
         }
+        case TEX_COLOR_DEPTH: {
+            ren_c_d(info);
+            break;
+        }
         default: {
             RET_ERROR;
         }

src/render/impl/render_texturer_impl.h

@@ -7,10 +7,16 @@ namespace render_texture_impl {
 
     void ren_rgb_only(const tex_render_info &info);
 
+    void ren_rgba_only(const tex_render_info &info);
+
+    void ren_rgb_d(const tex_render_info &info);
+
     void ren_nv12_only(const tex_render_info &info);
 
     void ren_c_only(const tex_render_info &info);
 
+    void ren_c_d(const tex_render_info &info);
+
 }
 
 #endif //DEPTHGUIDE_RENDER_TEXTURER_IMPL_H

src/render/impl/render_tools.cpp

@@ -13,7 +13,7 @@ void color_image_render::render_tex(cv::Size img_size, config_type conf) {
     auto info = tex_render_info();
     info.mode = TEX_COLOR_ONLY;
     info.flip_y = conf.flip_y;
-    info.range = calc_render_range(img_size);
+    info.range = conf.range.value_or(calc_render_range(img_size));
     info.color.fmt = conf.fmt;
     info.color.id = img_tex.id;
     if (conf.fmt == COLOR_NV12) { // chroma tex
@@ -122,4 +122,38 @@ void depth_image_render::render(obj_name_type name, config_type conf) {
     c_conf.alpha = conf.alpha;
     c_conf.stream = conf.stream;
     color_render.render_rgb(img_color, c_conf);
-}
+}
+
+//frame_builder::frame_builder(const create_config &_conf) {
+//    conf = _conf;
+//    fbo.create(conf.fbo_conf);
+//}
+//
+//void frame_builder::build() {
+//    fbo.bind();
+//    glClear(fbo.clear_flag);
+//    for (auto &func: conf.render_func_list) {
+//        func();
+//    }
+//    fbo.unbind();
+//}
+//
+//void frame_builder::render(render_config ren_conf) {
+//    if (fbo.id == 0) return; // framebuffer is not ready
+//    auto tex_info = tex_render_info{
+//            .mode = ren_conf.mode,
+//            .range = ren_conf.range,
+//            .flip_y =ren_conf.flip_y
+//    };
+//    tex_info.color.fmt = COLOR_RGBA;
+//    assert(!fbo.color_tex.empty());
+//    tex_info.color.id = fbo.color_tex[0].id;
+//    tex_info.color.opacity = ren_conf.opacity;
+//    tex_info.depth.id = fbo.depth_tex.id;
+//    render_texture(tex_info);
+//}
+//
+//void frame_builder::change_config(modifiable_config mod_conf) {
+//    conf.fbo_conf.size = mod_conf.size;
+//    fbo.create(conf.fbo_conf);
+//}

src/render/impl/render_utility.cpp

@@ -196,9 +196,24 @@ void smart_pixel_buffer::create(GLsizeiptr _size) {
 }
 
 void smart_pixel_buffer::download_viewport(GLenum format, GLenum type) {
+    size_t channel_num = 0;
+    switch (format) {
+        // @formatter:off
+        case GL_RED:
+        case GL_GREEN:
+        case GL_BLUE: { channel_num = 1; break; }
+        case GL_RGB:
+        case GL_BGR: { channel_num = 3; break; }
+        case GL_RGBA:
+        case GL_BGRA: { channel_num = 4; break; }
+        // @formatter:on
+    }
+    assert(channel_num != 0);
+
     size_t elem_size = 0;
     switch (type) {
         // @formatter:off
+        case GL_UNSIGNED_BYTE: { elem_size = sizeof(uchar) * channel_num; break; }
         case GL_UNSIGNED_INT_8_8_8_8_REV: { elem_size = sizeof(uchar4); break; }
         // @formatter:on
         default: {
@@ -266,31 +281,62 @@ void smart_frame_buffer::deallocate() {
     glDeleteFramebuffers(1, &id);
 }
 
-void smart_frame_buffer::create_impl(cv::Size _size, GLenum color_fmt, GLenum depth_fmt) {
-    if (_size == size) [[likely]] return;
+void smart_frame_buffer::create_impl(const create_config &conf) {
+    if (conf.size == size) [[likely]] return;
 
     deallocate();
 
-    size = _size;
-    color_tex.create(color_fmt, size);
-    depth_tex.create(depth_fmt, size);
-
-    // config
+    size = conf.size;
     glGenFramebuffers(1, &id);
-    glBindFramebuffer(GL_FRAMEBUFFER, id);
-    glBindTexture(GL_TEXTURE_2D, color_tex.id);
-    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, color_tex.id, 0);
-    glBindTexture(GL_TEXTURE_2D, depth_tex.id);
-    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_2D, depth_tex.id, 0);
+    bind();
+    clear_flag = GL_NONE;
+
+    // color textures
+    auto color_num = conf.color_fmts.size();
+    color_tex.resize(color_num);
+    for (auto k = 0; k < color_num; ++k) {
+        auto &tex = color_tex[k];
+        tex.create(conf.color_fmts[k], size);
+        glBindTexture(GL_TEXTURE_2D, tex.id);
+        glFramebufferTexture2D(
+                GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0 + k, GL_TEXTURE_2D, tex.id, 0);
+    }
+    assert(color_num >= 1);
+    if (color_num != 0) {
+        clear_flag |= GL_COLOR_BUFFER_BIT;
+    }
+
+    // depth texture
+    if (conf.depth_fmt != GL_NONE) {
+        depth_tex.create(conf.depth_fmt, size);
+        glBindTexture(GL_TEXTURE_2D, depth_tex.id);
+        glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_2D, depth_tex.id, 0);
+        clear_flag |= GL_DEPTH_BUFFER_BIT;
+    }
+
+    // stencil texture
+    if (conf.stencil_fmt != GL_NONE) {
+        stencil_tex.create(conf.stencil_fmt, size);
+        glBindTexture(GL_TEXTURE_2D, stencil_tex.id);
+        glFramebufferTexture2D(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT, GL_TEXTURE_2D, stencil_tex.id, 0);
+        clear_flag |= GL_STENCIL_BUFFER_BIT;
+    }
+
+    // cleanup
     check_framebuffer();
+    unbind();
 }
 
-void smart_frame_buffer::create(cv::Size _size, GLenum color_fmt, GLenum depth_fmt) {
-    create_impl(_size, color_fmt, depth_fmt);
+void smart_frame_buffer::create(const create_config &conf) {
+    if (conf.size.empty()) { // ignore zero-sized frame
+        assert(id == 0);
+        return;
+    }
+    create_impl(conf);
 
     // clear color and depth texture
     bind();
-    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+    glClear(clear_flag);
     unbind();
 }
 
@@ -298,15 +344,19 @@ void smart_frame_buffer::bind() {
     assert(id != 0);
 
     // save old framebuffer id
-    glGetIntegerv(GL_DRAW_FRAMEBUFFER_BINDING, (GLint *) &last_id);
+    glGetIntegerv(GL_FRAMEBUFFER_BINDING, (GLint *) &last_id);
+
+    // save old viewport
+    glGetIntegerv(GL_VIEWPORT, (GLint *) &last_vp);
 
     // bind and config framebuffer
-    glBindFramebuffer(GL_DRAW_FRAMEBUFFER, id);
+    glBindFramebuffer(GL_FRAMEBUFFER, id);
     glViewport(0, 0, size.width, size.height);
 }
 
 void smart_frame_buffer::unbind() {
-    glBindFramebuffer(GL_DRAW_FRAMEBUFFER, last_id);
+    glBindFramebuffer(GL_FRAMEBUFFER, last_id);
+    glViewport(last_vp.x, last_vp.y, last_vp.width, last_vp.height);
 }
 
 smart_program::~smart_program() {

src/render/impl/shader/mesh_normal.frag

@@ -0,0 +1,23 @@
+#version 460
+
+struct material_type {
+    vec3 ambient;
+    vec3 diffuse;
+};
+
+struct light_property {
+    vec3 direction;
+};
+
+uniform material_type material;
+uniform light_property light;
+
+in vec3 frag_normal;
+
+layout (location = 0) out vec4 frag_color;
+
+void main() {
+    float diffuse_weight = max(-dot(frag_normal, light.direction), 0.0);
+    vec3 color = material.ambient + diffuse_weight * material.diffuse;
+    frag_color = vec4(color, 1.0);
+}

src/render/impl/shader/mesh_normal.vert

@@ -0,0 +1,14 @@
+#version 460
+
+uniform mat4 model_mat;
+uniform mat4 camera_mat;
+
+layout (location = 0) in vec3 position;
+layout (location = 1) in vec3 normal;
+
+out vec3 frag_normal;
+
+void main() {
+    frag_normal = mat3(model_mat) * normal;
+    gl_Position = camera_mat * model_mat * vec4(position, 1.0);
+}

src/render/impl/shader/tex_rgb_d.frag

@@ -0,0 +1,17 @@
+#version 460
+
+uniform float alpha;
+
+uniform sampler2D c_tex; // color texture
+uniform sampler2D d_tex; // depth texture
+
+in vec2 frag_uv;
+
+layout (location = 0) out vec4 frag_color;
+layout (depth_less) out float gl_FragDepth;
+
+void main() {
+    frag_color.rgb = texture(c_tex, frag_uv).rgb;
+    frag_color.a = alpha;
+    gl_FragDepth = texture(d_tex, frag_uv).x;
+}

src/render/impl/shader/tex_rgba.frag

@@ -0,0 +1,14 @@
+#version 460
+
+uniform float opacity;
+
+uniform sampler2D c_tex;
+
+in vec2 frag_uv;
+
+layout (location = 0) out vec4 frag_color;
+
+void main() {
+    frag_color = texture(c_tex, frag_uv);
+    frag_color.a *= opacity;
+}

src/render/render_mesh.h

@@ -0,0 +1,80 @@
+#ifndef DEPTHGUIDE_RENDER_MESH_H
+#define DEPTHGUIDE_RENDER_MESH_H
+
+#include "render_utility.h"
+
+#include <memory>
+
+class mesh_type {
+public:
+
+    ~mesh_type();
+
+    struct create_config {
+        std::string path;
+    };
+
+    using this_type = mesh_type;
+    using pointer = std::shared_ptr<mesh_type>;
+
+    static pointer create(const create_config &conf);
+
+    struct gl_info_type : private boost::noncopyable {
+        GLuint vao = 0; // Vertex Array Object
+        GLuint vbo = 0; // Vertex Buffer Object
+        GLuint ebo = 0; // Element Buffer Object
+        GLuint num_triangle = 0; // number of triangles
+
+        ~gl_info_type();
+    };
+
+    gl_info_type *get_gl_info();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+enum mesh_render_mode {
+    MESH_NORMAL,
+    MESH_DEPTH_ALPHA
+};
+
+struct mesh_info_type {
+    mesh_type *mesh = nullptr;
+    glm::mat4 transform = {};
+    struct material_type {
+        glm::vec3 ambient;
+        glm::vec3 diffuse;
+    } material = {};
+};
+
+struct camera_info_type {
+    glm::mat4 transform;
+    glm::mat4 project;
+};
+
+struct light_info_type {
+    glm::vec3 direction;
+};
+
+struct mesh_render_info {
+    mesh_render_mode mode = MESH_NORMAL;
+    mesh_info_type model = {};
+    camera_info_type camera = {};
+    light_info_type light = {};
+
+    union {
+        struct {
+        } normal;
+        struct {
+            mesh_info_type bg;
+            float alpha_factor;
+            bool show_bg;
+        } depth_alpha;
+    } extra = {};
+};
+
+void render_mesh(const mesh_render_info &info);
+
+#endif //DEPTHGUIDE_RENDER_MESH_H

src/render/render_texture.h

@@ -6,9 +6,9 @@
 #include <glad/gl.h>
 
 enum tex_render_mode {
-    TEX_COLOR_ONLY,
-    TEX_DEPTH_ONLY,
-    TEX_COLOR_DEPTH,
+    TEX_COLOR_ONLY = 0x1,
+    TEX_DEPTH_ONLY = 0x2,
+    TEX_COLOR_DEPTH = TEX_COLOR_ONLY | TEX_DEPTH_ONLY,
     TEX_DEPTH_ALPHA
 };
 
@@ -22,7 +22,10 @@ struct tex_render_info {
         color_format fmt = COLOR_RGB;
         GLuint id = 0;
         GLuint id_ext[2] = {};
-        GLfloat alpha = 1.0;
+        union {
+            GLfloat alpha = 1.0; // for formats ``without'' alpha channel
+            GLfloat opacity; // for formats ``with'' alpha channel
+        };
     } color;
 
     // depth texture info

src/render/render_tools.h

@@ -3,6 +3,11 @@
 
 #include "core/object_manager.h"
 #include "render_utility.h"
+#include "render_texture.h"
+
+#include <boost/container/small_vector.hpp>
+
+#include <optional>
 
 simple_rect calc_render_range(cv::Size img_size);
 
@@ -13,6 +18,7 @@ public:
         color_format fmt = COLOR_RGB;
         bool flip_y = false;
         float alpha = 1.0;
+        std::optional<simple_rect> range;
         smart_cuda_stream *stream = nullptr;
     };
 
@@ -60,5 +66,43 @@ private:
     color_image_render color_render;
 };
 
+//class frame_builder {
+//public:
+//
+//    smart_frame_buffer fbo;
+//
+//    struct create_config {
+//        static constexpr auto reserve_cnt = 4;
+//        using render_func_type = std::function<void()>;
+//        using render_func_list_type =
+//                boost::container::small_vector<render_func_type, reserve_cnt>;
+//        render_func_list_type render_func_list;
+//
+//        using fbo_config_type = smart_frame_buffer::create_config;
+//        fbo_config_type fbo_conf;
+//    };
+//
+//    explicit frame_builder(const create_config &conf);
+//
+//    void build();
+//
+//    struct render_config {
+//        tex_render_mode mode = TEX_COLOR_ONLY;
+//        simple_rect range = {-1.0f, -1.0f, 2.0f, 2.0f};
+//        bool flip_y = false;
+//        GLfloat opacity = 1.0;
+//    };
+//
+//    void render(render_config conf);
+//
+//    struct modifiable_config {
+//        cv::Size size; // fbo size
+//    };
+//
+//    void change_config(modifiable_config conf);
+//
+//private:
+//    create_config conf;
+//};
 
 #endif //DEPTHGUIDE_RENDER_TOOLS_H

src/render/render_utility.h

@@ -7,6 +7,7 @@
 #include <glad/gl.h>
 #include <glm/glm.hpp>
 
+#include <boost/container/static_vector.hpp>
 #include <boost/core/noncopyable.hpp>
 
 #include <filesystem>
@@ -53,6 +54,7 @@ constexpr inline GLenum get_tex_internal_format() {
 
 enum color_format : uint8_t {
     COLOR_RGB,
+    COLOR_RGBA,
     COLOR_NV12
 };
 
@@ -165,16 +167,35 @@ public:
     GLuint id = 0;
     cv::Size size;
 
-    smart_texture color_tex;
+    static constexpr auto max_color_components = 1;
+    using color_textures_type =
+            boost::container::static_vector<smart_texture, max_color_components>;
+
+    color_textures_type color_tex;
     smart_texture depth_tex;
+    smart_texture stencil_tex;
+
+    GLbitfield clear_flag = GL_NONE; // flags for glClear()
 
     GLuint last_id = 0; // framebuffer before bind()
+    struct {
+        GLint x, y;
+        GLint width, height;
+    } last_vp;
 
     ~smart_frame_buffer();
 
-    void create(cv::Size size,
-                GLenum color_fmt = GL_RGBA8,
-                GLenum depth_fmt = GL_DEPTH_COMPONENT32);
+    struct create_config {
+        using color_formats_type =
+                boost::container::static_vector<GLenum, max_color_components>;
+
+        cv::Size size;
+        color_formats_type color_fmts = {GL_RGBA8};
+        GLenum depth_fmt = GL_DEPTH_COMPONENT32;
+        GLenum stencil_fmt = GL_NONE; // or GL_STENCIL_INDEX8
+    };
+
+    void create(const create_config &conf);
 
     void bind();
 
@@ -182,10 +203,9 @@ public:
 
 private:
 
-    void create_impl(cv::Size size, GLenum color_fmt, GLenum depth_fmt);
+    void create_impl(const create_config &conf);
 
     void deallocate();
-
 };
 
 class smart_program : private boost::noncopyable {

src/render_v3/scene_render.cpp

@@ -0,0 +1,146 @@
+#include "scene_render.h"
+#include "render/render_mesh.h"
+
+#include <glm/gtc/matrix_access.hpp>
+#include <glm/gtx/transform.hpp>
+
+#include <list>
+
+namespace scene_render_impl {
+
+    glm::vec3 vec3_from_ptr(const float *ptr) {
+        static_assert(sizeof(glm::vec3) == 3 * sizeof(float));
+        return *(glm::vec3 *) ptr;
+    }
+
+}
+
+using namespace scene_render_impl;
+
+struct scene_render::impl {
+
+    struct actor_store_type {
+        std::shared_ptr<mesh_type> model;
+
+        actor_store_type *background = nullptr;
+        float alpha_factor = 0.25f;
+
+        bool visible = true;
+        glm::mat4 transform = {};
+
+        using material_type = mesh_info_type::material_type;
+        material_type material = {};
+
+        mesh_info_type get_mesh_info() const {
+            return {.mesh = model.get(),
+                    .transform = transform, .material = material};
+        }
+    };
+
+    bool enable_light_follow_camera = false;
+    glm::vec3 light_direction = {1.0f, 0.0f, 0.0f};
+
+    float camera_fov = 60.0f; // in degree
+    float camera_near = 0.1f, camera_far = 1000.0f;
+    glm::mat4 camera_transform = {};
+
+    std::list<actor_store_type> actor_list;
+
+    void *create_actor(const char *model_path, void *bg_actor) {
+        auto &actor = actor_list.emplace_back();
+        actor.model = mesh_type::create({.path = model_path});
+        actor.background = (actor_store_type *) bg_actor;
+        return &actor;
+    }
+
+    void draw() {
+        auto info = mesh_render_info();
+        info.camera.transform = camera_transform;
+        float aspect = query_viewport_size().aspectRatio();
+        info.camera.project = glm::perspective(camera_fov, aspect, camera_near, camera_far);
+        if (enable_light_follow_camera) {
+            auto camera_rot = glm::inverse(glm::mat3{camera_transform});
+            light_direction = -glm::column(camera_rot, 2);
+        }
+        info.light.direction = light_direction;
+
+        for (const auto &actor: actor_list) {
+            if (!actor.visible) continue;
+            info.model = actor.get_mesh_info();
+            if (actor.background == nullptr) { // no background
+                info.mode = MESH_NORMAL;
+                render_mesh(info);
+            } else { // with background
+                info.mode = MESH_DEPTH_ALPHA;
+                auto &ext = info.extra.depth_alpha;
+                ext.bg = actor.background->get_mesh_info();
+                ext.alpha_factor = actor.alpha_factor;
+                ext.show_bg = false;
+                render_mesh(info);
+            }
+        }
+    }
+
+};
+
+scene_render::scene_render()
+        : pimpl(std::make_unique<impl>()) {}
+
+scene_render::~scene_render() = default;
+
+void *scene_render::create_actor(const char *model_path, void *bg_actor) {
+    return pimpl->create_actor(model_path, bg_actor);
+}
+
+void scene_render::set_actor_visibility(void *actor, bool visible) {
+    auto real_ptr = (impl::actor_store_type *) actor;
+    real_ptr->visible = visible;
+}
+
+void scene_render::set_actor_color(void *actor, const float *color, float ambient_factor) {
+    auto real_ptr = (impl::actor_store_type *) actor;
+    auto color_vec = vec3_from_ptr(color);
+    real_ptr->material.diffuse = color_vec;
+    real_ptr->material.ambient = ambient_factor * color_vec;
+}
+
+void scene_render::set_actor_alpha_factor(void *actor, float alpha_factor) {
+    auto real_ptr = (impl::actor_store_type *) actor;
+    real_ptr->alpha_factor = alpha_factor;
+}
+
+void scene_render::set_actor_transform(void *actor, glm::mat4 transform) {
+    auto real_ptr = (impl::actor_store_type *) actor;
+    real_ptr->transform = transform;
+}
+
+void scene_render::set_light_direction(glm::vec3 direction) {
+    pimpl->light_direction = direction;
+}
+
+void scene_render::set_light_follow_camera(bool enable) {
+    pimpl->enable_light_follow_camera = enable;
+}
+
+void scene_render::set_camera_fov(float fov) {
+    pimpl->camera_fov = fov;
+}
+
+void scene_render::set_camera_clip_range(float near, float far) {
+    pimpl->camera_near = near;
+    pimpl->camera_far = far;
+}
+
+void scene_render::set_camera_transform(glm::mat4 transform) {
+    static constexpr auto default_focal_length = 8.0f; // 8mm
+    auto trans_part = glm::vec3(transform[3]);
+    auto rot_part = glm::mat3(transform);
+    auto focal_point = trans_part + rot_part[2] * default_focal_length;
+    auto view_up = -rot_part[1];
+    auto view_mat = glm::lookAt(trans_part, focal_point, view_up);
+    pimpl->camera_transform = view_mat;
+}
+
+void scene_render::render() {
+    pimpl->draw();
+}

src/render_v3/scene_render.h

@@ -0,0 +1,48 @@
+#ifndef REMOTEAR3_SCENE_RENDER_H
+#define REMOTEAR3_SCENE_RENDER_H
+
+#include <glm/glm.hpp>
+
+#include <memory>
+
+class scene_render {
+public:
+
+    scene_render();
+
+    ~scene_render();
+
+    using actor_token_type = void *;
+
+    actor_token_type create_actor(const char *model_path,
+                                  actor_token_type bg_actor = nullptr);
+
+    static void set_actor_visibility(actor_token_type actor, bool visible);
+
+    static void set_actor_color(actor_token_type actor,
+                                const float color[3], float ambient_factor = 0.2f);
+
+    static void set_actor_alpha_factor(actor_token_type actor, float alpha_factor = 0.25f);
+
+    static void set_actor_transform(actor_token_type actor, glm::mat4 transform);
+
+    void set_light_direction(glm::vec3 direction);
+
+    void set_light_follow_camera(bool enable = true);
+
+    void set_camera_fov(float fov);
+
+    void set_camera_clip_range(float near = 0.1f, float far = 1000.0f);
+
+    // transform: standard camera coordinate (Z points inside image), like VTK
+    void set_camera_transform(glm::mat4 transform);
+
+    void render();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+
+#endif //REMOTEAR3_SCENE_RENDER_H

src/render_v3/vtk_viewer.cpp

@@ -0,0 +1,462 @@
+#include "vtk_viewer.h"
+#include "core/utility.hpp"
+
+#include <vtkCamera.h>
+#include <vtkCellArray.h>
+#include <vtkGenericOpenGLRenderWindow.h>
+#include <vtkGenericRenderWindowInteractor.h>
+#include <vtkInteractorStyleTrackballCamera.h>
+#include <vtkMatrix4x4.h>
+#include <vtkNew.h>
+#include <vtkOBJReader.h>
+#include <vtkOpenGLFramebufferObject.h>
+#include <vtkPointPicker.h>
+#include <vtkPoints.h>
+#include <vtkPolyDataMapper.h>
+#include <vtkProperty.h>
+#include <vtkRenderer.h>
+#include <vtkSTLReader.h>
+#include <vtkTextureObject.h>
+
+#include <spdlog/spdlog.h>
+
+#include <filesystem>
+#include <vector>
+
+namespace vtk_viewer_impl {
+
+    Eigen::Isometry3d to_eigen_transform(vtkMatrix4x4 *mat) {
+        Eigen::Isometry3d ret;
+        for (int i = 0; i < 4; ++i)
+            for (int j = 0; j < 4; ++j)
+                ret.matrix()(i, j) = mat->GetElement(i, j);
+        return ret;
+    }
+
+}
+
+using namespace vtk_viewer_impl;
+
+struct vtk_viewer::impl {
+
+    struct versatile_interaction_style;
+
+    static constexpr auto default_focal_length = 8; // 8mm
+
+    cv::Size last_size;
+    vtkSmartPointer<vtkGenericOpenGLRenderWindow> window;
+    vtkSmartPointer<vtkCamera> camera;
+    vtkSmartPointer<vtkRenderWindowInteractor> controller;
+    vtkSmartPointer<versatile_interaction_style> style;
+    vtkSmartPointer<vtkRenderer> renderer;
+
+    bool is_picking = false;
+    std::optional<Eigen::Vector3d> picked_point;
+
+    struct versatile_interaction_style
+            : public vtkInteractorStyleTrackballCamera {
+    vtkTypeMacro(versatile_interaction_style, vtkInteractorStyleTrackballCamera);
+
+        static versatile_interaction_style *New() {
+            auto ret = new versatile_interaction_style{};
+            ret->InitializeObjectBase();
+            return ret;
+        }
+
+        impl *pimpl = nullptr;
+        Eigen::Vector2i mouse_down_pose;
+
+        Eigen::Vector2i get_mouse_pos() {
+            return {Interactor->GetEventPosition()[0],
+                    Interactor->GetEventPosition()[1],
+            };
+        }
+
+        void OnLeftButtonDown() override {
+            mouse_down_pose = get_mouse_pos();
+            vtkInteractorStyleTrackballCamera::OnLeftButtonDown();
+        }
+
+        bool try_picking() {
+            assert(pimpl != nullptr);
+            if (!pimpl->is_picking) return false;
+            auto ret = Interactor->GetPicker()->Pick(mouse_down_pose.x(),
+                                                     mouse_down_pose.y(),
+                                                     0,
+                                                     pimpl->renderer);
+            if (ret == 0) return true; // does not pick anything
+            Eigen::Vector3d point;
+            Interactor->GetPicker()->GetPickPosition(point.data());
+            pimpl->picked_point = point;
+            SPDLOG_INFO("Picked point ({}, {}, {}).", point.x(), point.y(), point.z());
+            return true;
+        }
+
+        void OnLeftButtonUp() override {
+            if (get_mouse_pos() == mouse_down_pose) {
+                try_picking();
+            }
+            vtkInteractorStyleTrackballCamera::OnLeftButtonUp();
+        }
+    };
+
+    static cv::Size to_cv_size(ImVec2 size) {
+        return {(int) size.x, (int) size.y};
+    }
+
+    impl() {
+        window = vtkSmartPointer<vtkGenericOpenGLRenderWindow>::New();
+        window->InitializeFromCurrentContext();
+        window->SetOffScreenRendering(true);
+        window->SetAlphaBitPlanes(true);
+//        window->FramebufferFlipYOn();
+        window->SetIsCurrent(true);
+//        window->SwapBuffersOn();
+//        window->SetFrameBlitModeToNoBlit();
+
+        camera = vtkSmartPointer<vtkCamera>::New();
+        camera->SetClippingRange(default_focal_length, 2000); // 8mm to 2m
+
+        renderer = vtkSmartPointer<vtkRenderer>::New();
+        renderer->SetUseDepthPeeling(true);
+        renderer->SetActiveCamera(camera);
+        window->AddRenderer(renderer);
+
+        style = vtkSmartPointer<versatile_interaction_style>::New();
+        style->pimpl = this;
+        style->SetDefaultRenderer(renderer);
+        controller = vtkSmartPointer<vtkGenericRenderWindowInteractor>::New();
+        controller->SetInteractorStyle(style);
+        controller->EnableRenderOff();
+        window->SetInteractor(controller);
+    }
+
+    void set_camera_pose(const Eigen::Isometry3d &trans) {
+        auto trans_part = trans.translation();
+        auto rot_part = trans.rotation();
+        auto focal_point = trans_part + rot_part.col(2) * default_focal_length;
+        auto view_up = -rot_part.col(1);
+        camera->SetPosition(trans_part.x(), trans_part.y(), trans_part.z());
+        camera->SetFocalPoint(focal_point.x(), focal_point.y(), focal_point.z());
+        camera->SetViewUp(view_up.x(), view_up.y(), view_up.z());
+        camera->Modified();
+    }
+
+    void render(cv::Size size, bool interactive) {
+        assert(size.area() > 0);
+        if (size != last_size) [[unlikely]] {
+            controller->SetSize(size.width, size.height);
+            window->SetSize(size.width, size.height);
+            last_size = size;
+        }
+        if (interactive) {
+            process_event();
+        }
+        window->Render();
+    }
+
+    void process_event() {
+        if (!ImGui::IsWindowFocused() || !ImGui::IsWindowHovered()) return;
+
+        // set event position
+        auto &io = ImGui::GetIO();
+        io.ConfigWindowsMoveFromTitleBarOnly = true;
+        auto view_pos = ImGui::GetWindowPos();
+        auto x_pos = io.MousePos.x - view_pos.x;
+        auto y_pos = io.MousePos.y - view_pos.y;
+
+        // flip X axis
+        x_pos = controller->GetSize()[0] - x_pos;
+        controller->SetEventInformation(x_pos, y_pos, io.KeyCtrl, io.KeyShift);
+
+        // dispatch event
+        if (ImGui::IsWindowHovered()) {
+            if (io.MouseClicked[ImGuiMouseButton_Left]) {
+                controller->InvokeEvent(vtkCommand::LeftButtonPressEvent);
+            }
+            if (io.MouseClicked[ImGuiMouseButton_Middle]) {
+                controller->InvokeEvent(vtkCommand::MiddleButtonPressEvent);
+            }
+            if (io.MouseClicked[ImGuiMouseButton_Right]) {
+                controller->InvokeEvent(vtkCommand::RightButtonPressEvent);
+            }
+            if (io.MouseWheel > 0) {
+                controller->InvokeEvent(vtkCommand::MouseWheelForwardEvent);
+            } else if (io.MouseWheel < 0) {
+                controller->InvokeEvent(vtkCommand::MouseWheelBackwardEvent);
+            }
+        }
+        if (io.MouseReleased[ImGuiMouseButton_Left]) {
+            controller->InvokeEvent(vtkCommand::LeftButtonReleaseEvent);
+        }
+        if (io.MouseReleased[ImGuiMouseButton_Middle]) {
+            controller->InvokeEvent(vtkCommand::MiddleButtonReleaseEvent);
+        }
+        if (io.MouseReleased[ImGuiMouseButton_Right]) {
+            controller->InvokeEvent(vtkCommand::RightButtonReleaseEvent);
+        }
+        controller->InvokeEvent(vtkCommand::MouseMoveEvent);
+    }
+
+    GLuint get_tex() const {
+        return window->GetDisplayFramebuffer()
+                ->GetColorAttachmentAsTextureObject(0)->GetHandle();
+    }
+
+    void show_imgui_window(const char *name, ImVec2 req_size) {
+        ImGui::BeginChild(name, req_size, 0, no_scroll_flag);
+        auto render_size = ImGui::GetContentRegionAvail();
+        auto render_size_cv = to_cv_size(render_size);
+        if (render_size_cv.area() <= 0)return;
+        render(render_size_cv, true);
+        ImGui::Image(reinterpret_cast<void *>(get_tex()), render_size, {1, 0}, {0, 1});
+        ImGui::EndChild();
+    }
+
+};
+
+vtk_viewer::vtk_viewer()
+        : pimpl(std::make_unique<impl>()) {}
+
+vtk_viewer::~vtk_viewer() = default;
+
+void vtk_viewer::render(cv::Size size) {
+    return pimpl->render(size, false);
+}
+
+GLuint vtk_viewer::get_tex() const {
+    return pimpl->get_tex();
+}
+
+void vtk_viewer::add_actor(vtkActor *actor) {
+    pimpl->renderer->AddActor(actor);
+}
+
+void vtk_viewer::remove_actor(vtkActor *actor) {
+    pimpl->renderer->RemoveActor(actor);
+}
+
+void vtk_viewer::clear_actor() {
+    pimpl->renderer->RemoveAllViewProps();
+}
+
+void vtk_viewer::show(const std::string &name) {
+    pimpl->show_imgui_window(name.c_str(), ImGui::GetContentRegionAvail());
+}
+
+Eigen::Isometry3d vtk_viewer::get_camera_pose() {
+    auto ret = to_eigen_transform( // world in camera, I think
+            pimpl->camera->GetModelViewTransformMatrix());
+    ret = ret.inverse() * Eigen::AngleAxisd{std::numbers::pi, Eigen::Vector3d::UnitX()};
+    return ret;
+}
+
+void vtk_viewer::set_camera_pose(const Eigen::Isometry3d &trans) {
+    pimpl->set_camera_pose(trans);
+}
+
+void vtk_viewer::set_camera_view_angle(double angle) {
+    pimpl->camera->SetViewAngle(angle);
+    pimpl->camera->Modified();
+}
+
+void vtk_viewer::start_picking() {
+    pimpl->is_picking = true;
+}
+
+void vtk_viewer::stop_picking() {
+    pimpl->is_picking = false;
+}
+
+bool vtk_viewer::is_picking() {
+    return pimpl->is_picking;
+}
+
+std::optional<Eigen::Vector3d> vtk_viewer::get_picked_point() {
+    auto ret = pimpl->picked_point;
+    pimpl->picked_point = {};
+    return ret;
+}
+
+void vtk_viewer::reset_camera() {
+    pimpl->renderer->ResetCamera();
+}
+
+namespace vtk_viewer_helper {
+
+    vtkSmartPointer<vtkPolyData> load_stl(const std::string &path) {
+        vtkNew<vtkSTLReader> reader;
+        reader->SetFileName(path.c_str());
+        reader->Update();
+        return reader->GetOutput();
+    }
+
+    vtkSmartPointer<vtkPolyData> load_obj(const std::string &path) {
+        vtkNew<vtkOBJReader> reader;
+        reader->SetFileName(path.c_str());
+        reader->Update();
+        return reader->GetOutput();
+    }
+
+    vtkSmartPointer<vtkPolyData> load_any(const std::string &path) {
+        auto path_fs = std::filesystem::path{path};
+        auto ext = path_fs.extension();
+        if (ext == ".stl") {
+            return load_stl(path);
+        } else if (ext == ".obj") {
+            return load_obj(path);
+        }
+        RET_ERROR_P;
+    }
+
+    vtkSmartPointer<vtkActor> create_actor(vtkPolyData *data) {
+        vtkNew<vtkPolyDataMapper> mapper;
+        mapper->SetInputData(data);
+        vtkNew<vtkActor> actor;
+        actor->SetMapper(mapper);
+        vtkNew<vtkMatrix4x4> pose;
+        actor->SetUserMatrix(pose);
+        return actor;
+    }
+
+    vtkSmartPointer<vtkActor> create_actor(const std::string &path) {
+        return create_actor(load_stl(path));
+    }
+
+    void update_actor_pose(vtkActor *actor, const std::optional<Eigen::Isometry3d> &trans) {
+        if (!trans.has_value()) {
+            actor->VisibilityOff();
+            return;
+        }
+        actor->VisibilityOn();
+        auto &real_trans = trans.value();
+        auto matrix = actor->GetUserMatrix();
+        if (matrix == nullptr) {
+            actor->SetUserMatrix(vtkMatrix4x4::New());
+            matrix = actor->GetUserMatrix();
+        }
+        for (int i = 0; i < 4; ++i) {
+            for (int j = 0; j < 4; ++j) {
+                matrix->SetElement(i, j, real_trans(i, j));
+            }
+        }
+        matrix->Modified();
+        actor->Modified();
+    }
+
+}
+
+struct smart_point_sets::impl {
+
+    vtkSmartPointer<vtkPoints> points;
+    vtkSmartPointer<vtkCellArray> vertices;
+    vtkSmartPointer<vtkPolyData> poly;
+    vtkSmartPointer<vtkActor> actor;
+
+    std::vector<Eigen::Vector3d> points_store;
+
+    impl() {
+        actor = vtkSmartPointer<vtkActor>::New();
+        actor->GetProperty()->SetRenderPointsAsSpheres(true);
+        actor->GetProperty()->SetPointSize(10);
+        reconstruct();
+    }
+
+    void add_point_to_poly(const Eigen::Vector3d &point) {
+        vtkIdType pid[1];
+        pid[0] = points->InsertNextPoint(point.data());
+        points->Modified();
+        vertices->InsertNextCell(1, pid);
+        vertices->Modified();
+        poly->Modified();
+    }
+
+    auto add_point(const Eigen::Vector3d &point) {
+        points_store.emplace_back(point);
+        add_point_to_poly(point);
+    }
+
+    void reconstruct() {
+        points = vtkSmartPointer<vtkPoints>::New();
+        vertices = vtkSmartPointer<vtkCellArray>::New();
+        poly = vtkSmartPointer<vtkPolyData>::New();
+        poly->SetPoints(points);
+        poly->SetVerts(vertices);
+        vtkNew<vtkPolyDataMapper> mapper;
+        mapper->SetInputData(poly);
+        actor->SetMapper(mapper);
+        actor->Modified();
+
+        for (auto &p: points_store) {
+            add_point_to_poly(p);
+        }
+    }
+
+    void for_each(const for_each_func_type &func) {
+        for (auto iter = points_store.begin();
+             iter != points_store.end();
+             ++iter) {
+            func(*(void **) &iter, *iter);
+        }
+    }
+
+    void remove_point(void *token) {
+        using iter_type = decltype(points_store)::iterator;
+        points_store.erase(*(iter_type *) (&token));
+        reconstruct();
+    }
+
+    Eigen::Vector3d pop_front() {
+        assert(!points_store.empty());
+        auto ret = points_store.front();
+        points_store.erase(points_store.begin());
+        reconstruct();
+        return ret;
+    }
+};
+
+smart_point_sets::smart_point_sets()
+        : pimpl(std::make_unique<impl>()) {}
+
+smart_point_sets::~smart_point_sets() = default;
+
+void smart_point_sets::add_point(const Eigen::Vector3d &point) {
+    pimpl->add_point(point);
+}
+
+vtkActor *smart_point_sets::get_actor() {
+    return pimpl->actor;
+}
+
+void smart_point_sets::for_each(const for_each_func_type &func) {
+    pimpl->for_each(func);
+}
+
+void smart_point_sets::remove_point(void *token) {
+    pimpl->remove_point(token);
+}
+
+Eigen::Vector3d smart_point_sets::pop_front() {
+    return pimpl->pop_front();
+}
+
+bool smart_point_sets::empty() const {
+    return pimpl->points_store.empty();
+}
+
+size_t smart_point_sets::size() const {
+    return pimpl->points_store.size();
+}
+
+std::vector<Eigen::Vector3d> smart_point_sets::to_vector() const {
+    return pimpl->points_store;
+}
+
+Eigen::Vector3d smart_point_sets::operator[](size_t index) const {
+    return pimpl->points_store[index];
+}
+
+void smart_point_sets::clear() const {
+    pimpl->points_store.clear();
+    pimpl->reconstruct();
+}

src/render_v3/vtk_viewer.h

@@ -0,0 +1,115 @@
+#ifndef REMOTEAR3_VTK_VIEWER_H
+#define REMOTEAR3_VTK_VIEWER_H
+
+#include <glad/gl.h>
+
+#include <imgui.h>
+
+#include <opencv2/core/types.hpp>
+
+#include <Eigen/Geometry>
+
+#include <vtkActor.h>
+#include <vtkPolyData.h>
+#include <vtkSmartPointer.h>
+
+#include <functional>
+#include <memory>
+#include <optional>
+
+namespace vtk_viewer_helper {
+
+    vtkSmartPointer<vtkPolyData> load_stl(const std::string &path);
+
+    vtkSmartPointer<vtkPolyData> load_obj(const std::string &path);
+
+    vtkSmartPointer<vtkPolyData> load_any(const std::string &path);
+
+    vtkSmartPointer<vtkActor> create_actor(vtkPolyData *data);
+
+    vtkSmartPointer<vtkActor> create_actor(const std::string &path);
+
+    void update_actor_pose(vtkActor *actor, const std::optional<Eigen::Isometry3d> &trans);
+
+}
+
+class vtk_viewer {
+public:
+
+    static constexpr auto no_scroll_flag = ImGuiWindowFlags_NoScrollWithMouse |
+                                           ImGuiWindowFlags_NoScrollbar;
+
+    vtk_viewer();
+
+    ~vtk_viewer();
+
+    void add_actor(vtkActor *actor);
+
+    void remove_actor(vtkActor *actor);
+
+    void clear_actor();
+
+    void set_camera_view_angle(double angle);
+
+    Eigen::Isometry3d get_camera_pose();
+
+    // camera observed in world
+    void set_camera_pose(const Eigen::Isometry3d &trans);
+
+    void reset_camera();
+
+    void render(cv::Size size);
+
+    GLuint get_tex() const;
+
+    void show(const std::string &name = "VtkViewer"); // Show in ImGui as child
+
+    // picking related
+    void start_picking();
+
+    void stop_picking();
+
+    bool is_picking();
+
+    std::optional<Eigen::Vector3d> get_picked_point();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+class smart_point_sets {
+public:
+
+    smart_point_sets();
+
+    ~smart_point_sets();
+
+    vtkActor *get_actor();
+
+    void add_point(const Eigen::Vector3d &point);
+
+    void remove_point(void *token);
+
+    size_t size() const;
+
+    bool empty() const;
+
+    void clear() const;
+
+    Eigen::Vector3d pop_front();
+
+    using for_each_func_type = std::function<void(void *, Eigen::Vector3d)>; // void * means a token
+
+    void for_each(const for_each_func_type &func);
+
+    std::vector<Eigen::Vector3d> to_vector() const;
+
+    Eigen::Vector3d operator[](size_t index) const;
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#endif //REMOTEAR3_VTK_VIEWER_H