Fixed depth alpha effect.

data/config_depth_guide_v2.yaml

@@ -1 +1,30 @@
-app_name: depth_guide_v2
+app_name: depth_guide_v2
+
+sophiar_config: /home/tpx/project/DepthGuide/data/sophiar_config_depth_guide_v2.json
+sophiar_start_var: tracker_all
+
+camera_transform_var: camera_in_tracker_denoised
+
+augment_list:
+  - name: Femur
+    transform_var: femur_in_tracker_denoised
+    model: /home/tpx/project/DepthGuide/data/model/Femur_3.stl
+  - name: Tibia
+    transform_var: tibia_in_tracker_denoised
+    model: /home/tpx/project/DepthGuide/data/model/Tibia_3.stl
+
+probe_model: /home/tpx/project/RemoteAR3/data/models/Probe.stl
+
+registration_list:
+  - name: Femur
+    model_file: /home/tpx/project/DepthGuide/data/model/Femur_3.stl
+    collect_obj: point_picker_in_femur_ref
+    collect_var: picked_point_in_femur_ref
+    target_var: femur_in_femur_ref
+    probe_var: probe_in_femur
+  - name: Tibia
+    model_file: /home/tpx/project/DepthGuide/data/model/Tibia_3.stl
+    collect_obj: point_picker_in_tibia_ref
+    collect_var: picked_point_in_tibia_ref
+    target_var: tibia_in_tibia_ref
+    probe_var: probe_in_tibia

data/sophiar_config_depth_guide_v2.json

@@ -0,0 +1,436 @@
+{
+  "variable_list": [
+    {
+      "name": "probe_ref_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "probe_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "femur_ref_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "femur_in_femur_ref",
+      "type": "transform_obj",
+      "value": [
+        405.60876310826154,
+        -11.723419024851964,
+        -64.43931776227325,
+        -0.25881623739773235,
+        0.5976871267586126,
+        -0.3985157377272106,
+        0.6457317249059794
+      ]
+    },
+    {
+      "name": "tibia_ref_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "tibia_in_tibia_ref",
+      "type": "transform_obj",
+      "value": [
+        117.21224919149104,
+        38.32624851921699,
+        -72.90989200004259,
+        -0.19759914802407674,
+        0.6557689584549056,
+        -0.28931051779904626,
+        0.6687458965242118
+      ]
+    },
+    {
+      "name": "camera_ref_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "camera_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "camera_in_tracker_denoised",
+      "type": "transform_obj"
+    },
+    {
+      "name": "femur_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "tibia_in_tracker",
+      "type": "transform_obj"
+    },
+    {
+      "name": "probe_in_femur_ref",
+      "type": "transform_obj"
+    },
+    {
+      "name": "probe_in_tibia_ref",
+      "type": "transform_obj"
+    },
+    {
+      "name": "probe_in_femur",
+      "type": "transform_obj"
+    },
+    {
+      "name": "probe_in_tibia",
+      "type": "transform_obj"
+    },
+    {
+      "name": "probe_tip_in_femur_ref",
+      "type": "scalarxyz_obj"
+    },
+    {
+      "name": "probe_tip_in_tibia_ref",
+      "type": "scalarxyz_obj"
+    },
+    {
+      "name": "picked_point_in_femur_ref",
+      "type": "scalarxyz_obj"
+    },
+    {
+      "name": "picked_point_in_tibia_ref",
+      "type": "scalarxyz_obj"
+    },
+    {
+      "name": "probe_tip_in_femur",
+      "type": "scalarxyz_obj"
+    },
+    {
+      "name": "probe_tip_in_tibia",
+      "type": "scalarxyz_obj"
+    },
+    {
+      "name": "picked_point_in_femur",
+      "type": "scalarxyz_obj"
+    },
+    {
+      "name": "picked_point_in_tibia",
+      "type": "scalarxyz_obj"
+    },
+    {
+      "name": "femur_in_tracker_denoised",
+      "type": "transform_obj"
+    },
+    {
+      "name": "tibia_in_tracker_denoised",
+      "type": "transform_obj"
+    }
+  ],
+  "object_list": [
+    {
+      "type": "transform_tree",
+      "name": "transform_tree",
+      "init_config": {
+        "node_list": [
+          {
+            "name": "tracker"
+          },
+          {
+            "name": "probe_ref",
+            "parent": "tracker",
+            "transform_var_name": "probe_ref_in_tracker"
+          },
+          {
+            "name": "probe",
+            "parent": "probe_ref",
+            "transform": [
+              -0.23,
+              -13.98,
+              -119.65,
+              1,
+              0,
+              0,
+              0
+            ]
+          },
+          {
+            "name": "femur_ref",
+            "parent": "tracker",
+            "transform_var_name": "femur_ref_in_tracker"
+          },
+          {
+            "name": "femur",
+            "parent": "femur_ref",
+            "transform_var_name": "femur_in_femur_ref"
+          },
+          {
+            "name": "tibia_ref",
+            "parent": "tracker",
+            "transform_var_name": "tibia_ref_in_tracker"
+          },
+          {
+            "name": "tibia",
+            "parent": "tibia_ref",
+            "transform_var_name": "tibia_in_tibia_ref"
+          },
+          {
+            "name": "camera_ref",
+            "parent": "tracker",
+            "transform_var_name": "camera_ref_in_tracker"
+          },
+          {
+            "name": "camera",
+            "parent": "camera_ref",
+            "transform": [
+              19.3286001048916,
+              37.0595264763240,
+              -55.5602916285344,
+              0.505328802080145,
+              -0.493396136841779,
+              0.500875581191629,
+              0.500326599436610
+            ]
+          }
+        ]
+      },
+      "start_config": {
+        "watch_list": [
+          {
+            "target": "camera",
+            "observer": "tracker",
+            "transform_var_name": "camera_in_tracker"
+          },
+          {
+            "target": "femur",
+            "observer": "tracker",
+            "transform_var_name": "femur_in_tracker"
+          },
+          {
+            "target": "tibia",
+            "observer": "tracker",
+            "transform_var_name": "tibia_in_tracker"
+          },
+          {
+            "target": "probe",
+            "observer": "tracker",
+            "transform_var_name": "probe_in_tracker"
+          },
+          {
+            "target": "probe",
+            "observer": "femur_ref",
+            "transform_var_name": "probe_in_femur_ref"
+          },
+          {
+            "target": "probe",
+            "observer": "tibia_ref",
+            "transform_var_name": "probe_in_tibia_ref"
+          },
+          {
+            "target": "probe",
+            "observer": "femur",
+            "transform_var_name": "probe_in_femur"
+          },
+          {
+            "target": "probe",
+            "observer": "tibia",
+            "transform_var_name": "probe_in_tibia"
+          }
+        ]
+      }
+    },
+    {
+      "type": "ndi_interface",
+      "name": "ndi",
+      "init_config": {
+        "address_type": "ethernet",
+        "ip": "10.0.0.5",
+        "tcp_port": 8765,
+        "com_port": "/dev/ttyUSB0",
+        "tool_list": [
+          {
+            "rom_path": "/home/tpx/data/roms/GlassProbe_4Ball_4.rom",
+            "serial_number": "3DD50000",
+            "outputs": {
+              "transform": "probe_ref_in_tracker"
+            }
+          },
+          {
+            "rom_path": "/home/tpx/data/roms/Glass_4Ball_1.rom",
+            "outputs": {
+              "transform": "camera_ref_in_tracker"
+            }
+          },
+          {
+            "rom_path": "/home/tpx/data/roms/Glass_3Ball_5.rom",
+            "serial_number": "39B33001",
+            "outputs": {
+              "transform": "femur_ref_in_tracker"
+            }
+          },
+          {
+            "rom_path": "/home/tpx/data/roms/Glass_3Ball_6.rom",
+            "outputs": {
+              "transform": "tibia_ref_in_tracker"
+            }
+          }
+        ]
+      },
+      "start_config": {
+        "allow_unreliable": true,
+        "prefer_stream_tracking": false
+      }
+    },
+    {
+      "type": "variable_validity_watcher",
+      "name": "probe_visibility_watcher",
+      "start_config": {
+        "variable_name": "probe_in_tracker"
+      }
+    },
+    {
+      "type": "transform_obj_validity_watcher",
+      "name": "camera_visibility_watcher",
+      "start_config": {
+        "variable_name": "camera_ref_in_tracker"
+      }
+    },
+    {
+      "type": "transform_obj_validity_watcher",
+      "name": "femur_visibility_watcher",
+      "start_config": {
+        "variable_name": "femur_ref_in_tracker"
+      }
+    },
+    {
+      "type": "transform_obj_validity_watcher",
+      "name": "tibia_visibility_watcher",
+      "start_config": {
+        "variable_name": "tibia_ref_in_tracker"
+      }
+    },
+    {
+      "type": "scalarxyz_transformer",
+      "name": "probe_tip_in_femur_ref_transformer",
+      "start_config": {
+        "transform_type": "point",
+        "transform_var_name": "probe_in_femur_ref",
+        "target_value": [
+          0,
+          0,
+          0
+        ],
+        "output_var_name": "probe_tip_in_femur_ref"
+      },
+      "dependencies": [
+        "ndi",
+        "transform_tree"
+      ]
+    },
+    {
+      "type": "scalarxyz_transformer",
+      "name": "probe_tip_in_tibia_ref_transformer",
+      "start_config": {
+        "transform_type": "point",
+        "transform_var_name": "probe_in_tibia_ref",
+        "target_value": [
+          0,
+          0,
+          0
+        ],
+        "output_var_name": "probe_tip_in_tibia_ref"
+      },
+      "dependencies": [
+        "ndi",
+        "transform_tree"
+      ]
+    },
+    {
+      "type": "transform_stabilizer",
+      "name": "point_picker_in_femur_ref",
+      "start_config": {
+        "stable_type": "point",
+        "input_var_name": "probe_tip_in_femur_ref",
+        "output_var_name": "picked_point_in_femur_ref",
+        "linear_tolerance_mm": 0.05,
+        "temporal_interval_s": 3
+      },
+      "dependencies": [
+        "probe_tip_in_femur_ref_transformer"
+      ]
+    },
+    {
+      "type": "transform_stabilizer",
+      "name": "point_picker_in_tibia_ref",
+      "start_config": {
+        "stable_type": "point",
+        "input_var_name": "probe_tip_in_tibia_ref",
+        "output_var_name": "picked_point_in_tibia_ref",
+        "linear_tolerance_mm": 0.05,
+        "temporal_interval_s": 3
+      },
+      "dependencies": [
+        "probe_tip_in_tibia_ref_transformer"
+      ]
+    },
+    {
+      "type": "scalarxyz_obj_watcher",
+      "name": "picked_point_watcher_for_femur_ref",
+      "start_config": {
+        "variable_name": "picked_point_in_femur_ref"
+      }
+    },
+    {
+      "type": "scalarxyz_obj_watcher",
+      "name": "picked_point_watcher_for_tibia_ref",
+      "start_config": {
+        "variable_name": "picked_point_in_tibia_ref"
+      }
+    },
+    {
+      "type": "transform_obj_watcher",
+      "name": "femur_registration_result_watcher",
+      "start_config": {
+        "variable_name": "femur_in_femur_ref"
+      }
+    },
+    {
+      "type": "transform_obj_watcher",
+      "name": "tibia_registration_result_watcher",
+      "start_config": {
+        "variable_name": "tibia_in_tibia_ref"
+      }
+    },
+    {
+      "type": "kalman_denoiser",
+      "name": "camera_denoiser",
+      "init_config": {
+        "variable_in": "camera_in_tracker",
+        "variable_out": "camera_in_tracker_denoised"
+      }
+    },
+    {
+      "type": "kalman_denoiser",
+      "name": "femur_denoiser",
+      "init_config": {
+        "variable_in": "femur_in_tracker",
+        "variable_out": "femur_in_tracker_denoised"
+      }
+    },
+    {
+      "type": "kalman_denoiser",
+      "name": "tibia_denoiser",
+      "init_config": {
+        "variable_in": "tibia_in_tracker",
+        "variable_out": "tibia_in_tracker_denoised"
+      }
+    },
+    {
+      "type": "empty_object",
+      "name": "tracker_all",
+      "dependencies": [
+        "transform_tree",
+        "ndi",
+        "probe_visibility_watcher",
+        "camera_visibility_watcher",
+        "femur_visibility_watcher",
+        "tibia_visibility_watcher",
+        "camera_denoiser",
+        "femur_denoiser",
+        "tibia_denoiser"
+      ]
+    }
+  ]
+}

src/image_process/impl/process_funcs.cpp

@@ -158,6 +158,7 @@ namespace shrink_pc_zero_depth {
     }
 
     void call_direct(const config_direct &conf) {
+        if (conf.in == nullptr) return;
         auto pc_out = pc_ptr();
         switch (conf.in->format()) {
             case PC_XYZ: {

src/impl/apps/depth_guide/depth_guide.cpp

@@ -188,10 +188,10 @@ void app_depth_guide::show_ui() {
 }
 
 void app_depth_guide::render_background() {
-    aug_helper_left->render();
-//    if (enable_scene_viewer) {
-//        scene_viewer->render();
-//    } else {
-//        bg_viewer->render();
-//    }
+//    aug_helper_left->render();
+    if (enable_scene_viewer) {
+        scene_viewer->render();
+    } else {
+        bg_viewer->render();
+    }
 }

src/impl/apps/depth_guide_v2/depth_guide_v2.cpp

@@ -3,7 +3,8 @@
 #include "image_process/process_funcs.h"
 
 app_depth_guide_v2::app_depth_guide_v2(const create_config &_conf) {
-    conf = _conf;
+    main_conf = _conf;
+    auto conf = main_conf.ext_conf;
 
     // initialize object manager
     OBJ_SAVE(img_color, image_u8c3());
@@ -27,37 +28,62 @@ app_depth_guide_v2::app_depth_guide_v2(const create_config &_conf) {
         aug_manager->update(aug_helper->get_camera());
     });
 
+    // initialize sophiar
+    sophiar_thread = std::make_unique<std::thread>([conf_path = LOAD_STR("sophiar_config")] {
+        sophiar::run_sophiar(conf_path);
+    });
+    sophiar_conn = std::make_unique<sophiar_conn_type>();
+    sophiar_start_var = LOAD_STR("sophiar_start_var");
+
     // initialize modules
     auto orb_cam_conf = orb_camera_ui::create_config{
             .cf_name = img_color, .df_name = img_depth,
             .remap_name = img_remap,
-            .ctx = conf.asio_ctx, .stream = default_cuda_stream,
+            .ctx = main_conf.asio_ctx, .stream = default_cuda_stream,
     };
     orb_cam = std::make_unique<orb_camera_ui>(orb_cam_conf);
 
+    auto aug_list_v1 = augment_manager::item_list_from_yaml(LOAD_LIST("augment_list"));
+    auto aug_list = augment_manager_v2::item_list_from_v1(aug_list_v1, extra_name);
     auto aug_conf = augment_manager_v2::create_config{
             .scene_name = scene_out,
+            .sophiar_conn = sophiar_conn.get(),
             .stream = default_cuda_stream,
     };
     aug_conf.item_list.push_back(
             {.type = augment_manager_v2::ITEM_PC,
-                    .disp_name = "PC Raw", .name = pc_raw});
+                    .disp_name = "PC Raw", .name = pc_raw,
+                    .transform_var = LOAD_STR("camera_transform_var")});
+    std::copy(aug_list.begin(), aug_list.end(), std::back_inserter(aug_conf.item_list));
     aug_manager = std::make_unique<augment_manager_v2>(aug_conf);
 
     auto view_conf = camera_augment_helper_v2::create_config{
-            .camera = camera_augment_helper_v2::create_config::freedom_camera_config{},
+            .camera = camera_augment_helper_v2::create_config::fixed_camera_config{
+                    .fov = 54.75355, .transform_var = LOAD_STR("camera_transform_var")},
+//            .camera = camera_augment_helper_v2::create_config::freedom_camera_config{},
+            .sophiar_conn= sophiar_conn.get(),
             .manager = aug_manager.get(),
-            .ctx = conf.asio_ctx,
+            .ctx = main_conf.asio_ctx,
     };
     aug_helper = std::make_unique<camera_augment_helper_v2>(view_conf);
     aug_helper->set_interactive(true);
 
     auto out_conf = image_streamer::create_config{
             .img_name = scene_out, .encode_scene = true,
-            .asio_ctx = conf.asio_ctx,
-            .cuda_ctx = conf.cuda_ctx, .stream = default_cuda_stream
+            .asio_ctx = main_conf.asio_ctx,
+            .cuda_ctx = main_conf.cuda_ctx, .stream = default_cuda_stream
     };
     out_streamer = std::make_unique<image_streamer>(out_conf);
+
+    auto reg_conf = registration_config{
+            .conn = sophiar_conn.get(),
+            .probe_model_path = LOAD_STR("probe_model"),
+    };
+    reg.reset(registration::create(reg_conf));
+    for (auto reg_item: LOAD_LIST("registration_list")) {
+        auto item_conf = registration_target::from_yaml(reg_item);
+        reg->add_target(item_conf);
+    }
 }
 
 void app_depth_guide_v2::show_ui() {
@@ -69,12 +95,13 @@ void app_depth_guide_v2::show_ui() {
             orb_cam->show();
         }
 
-        if (ImGui::CollapsingHeader("Streamer")) {
-            auto id_guard = imgui_id_guard("streamer");
-            out_streamer->show();
-        }
-
         if (ImGui::CollapsingHeader("Augment")) {
+            ImGui::SeparatorText("Tracker");
+            if (ImGui::Button("Start")) {
+                start_tracking();
+            }
+            ImGui::SameLine();
+            ImGui::Checkbox("Registration", &enable_reg);
             {
                 ImGui::SeparatorText("Scene");
                 auto id_guard = imgui_id_guard("augment_scene");
@@ -87,11 +114,37 @@ void app_depth_guide_v2::show_ui() {
             }
         }
 
+        if (ImGui::CollapsingHeader("Streamer")) {
+            auto id_guard = imgui_id_guard("streamer");
+            out_streamer->show();
+        }
+
         ImGui::PopItemWidth();
     }
     ImGui::End();
+
+    if (enable_reg) {
+        reg->process();
+        reg->show();
+    }
 }
 
 void app_depth_guide_v2::render_background() {
     aug_helper->render();
 }
+
+void app_depth_guide_v2::start_tracking() {
+    // sophiar_conn->start_object(sophiar_start_var);
+    // work in another thread to prevent blocking
+    auto t = std::thread([this] {
+        auto conn = sophiar::local_connection();
+        conn.start_object(sophiar_start_var);
+    });
+    t.detach();
+}
+
+app_depth_guide_v2::~app_depth_guide_v2() {
+    // sophiar
+    sophiar::stop_sophiar();
+    sophiar_thread->join();
+}

src/impl/apps/depth_guide_v2/depth_guide_v2.h

@@ -3,6 +3,7 @@
 
 #include "core/object_manager.h"
 #include "device/orb_camera_ui.h"
+#include "module_v3/registration.h"
 #include "module/augment_manager_v2.h"
 #include "module/camera_augment_helper_v2.h"
 #include "module/image_streamer.h"
@@ -15,7 +16,7 @@ public:
 
     explicit app_depth_guide_v2(const create_config &conf);
 
-    ~app_depth_guide_v2() override = default;
+    ~app_depth_guide_v2() override;
 
     const char *window_name() override { return "DepthGuide V2.-1"; }
 
@@ -25,6 +26,8 @@ public:
 
 private:
 
+    void start_tracking();
+
     enum obj_names : object_manager::name_type {
 
         // images from device
@@ -35,9 +38,18 @@ private:
 
         // output image
         scene_out,
+
+        name_end,
     };
 
-    create_config conf;
+    create_config main_conf;
+    obj_name_type extra_name = name_end;
+
+    // sophiar
+    using sophiar_conn_type = sophiar::local_connection;
+    std::unique_ptr<sophiar_conn_type> sophiar_conn;
+    std::unique_ptr<std::thread> sophiar_thread;
+    std::string sophiar_start_var;
 
     // modules
     std::unique_ptr<orb_camera_ui> orb_cam;
@@ -46,6 +58,9 @@ private:
     std::unique_ptr<augment_manager_v2> aug_manager;
     std::unique_ptr<camera_augment_helper_v2> aug_helper;
 
+    bool enable_reg = false;
+    std::unique_ptr<registration> reg;
+
 };
 
 

src/render/impl/render_mesh.cpp

@@ -262,6 +262,7 @@ namespace render_mesh_impl {
         tex_info.depth.id = mesh_fbo.depth_tex.id;
         tex_info.depth.enable_alpha_effect = true;
         auto &tex_extra = tex_info.extra.color_depth;
+        tex_extra.bg_id = extra.bg_tex;
         tex_extra.proj_mat = info.camera.project;
         tex_extra.alpha_factor = extra.alpha_factor;
         render_texture(tex_info);

src/render/impl/render_scene.cpp

@@ -7,6 +7,11 @@ namespace render_scene_impl {
 
     color_image_render image_render;
 
+    using fbo_conf_type =
+            smart_frame_buffer::create_config;
+    fbo_conf_type fbo_conf = {};
+    smart_frame_buffer scene_fbo;
+
     void render_image(const render_request &req) {
         using info_type = scene_render_info::image_info;
         auto info = std::get<info_type>(req.item.info);
@@ -42,6 +47,7 @@ namespace render_scene_impl {
         if (info.enable_depth_alpha) {
             ren_conf.mode = MESH_DEPTH_ALPHA;
             auto &extra = ren_conf.extra.depth_alpha;
+            extra.bg_tex = scene_fbo.depth_tex.id;
             extra.alpha_factor = info.alpha_factor;
         }
 
@@ -80,60 +86,77 @@ namespace render_scene_impl {
         return glm::lookAt(trans_part, focal_point, view_up);
     }
 
+    void render_scene_impl(const scene_ptr &info_ptr, cv::Size vp_size) {
+        auto &info = *info_ptr;
+        auto req = render_request{
+                .camera_raw = info.camera,
+                .stream = info.stream,
+        };
+        req.camera.transform =
+                cvt_camera_transform(info.camera.transform);
+        req.camera.project = info.camera.projection(vp_size.aspectRatio());
+        if (info.light.follow_camera) {
+            req.light.direction = glm::column(info.camera.transform, 2);
+        } else {
+            req.light.direction = info.light.direction;
+        }
+
+        for (auto &item: info.items) {
+            req.item = item;
+            using info_type = std::remove_cvref_t<typeof(item.info)>;
+            switch (item.info.index()) {
+                case 1: {
+                    static_assert(std::is_same_v<std::variant_alternative_t<1, info_type>,
+                            scene_render_info::image_info>);
+                    render_image(req);
+                    break;
+                }
+                case 2: {
+                    static_assert(std::is_same_v<std::variant_alternative_t<2, info_type>,
+                            scene_render_info::mesh_info>);
+                    render_mesh(req);
+                    break;
+                }
+                case 3: {
+                    static_assert(std::is_same_v<std::variant_alternative_t<3, info_type>,
+                            scene_render_info::pc_info>);
+                    render_pc(req);
+                    break;
+                }
+                case 4: {
+                    static_assert(std::is_same_v<std::variant_alternative_t<4, info_type>,
+                            scene_render_info::custom_info>);
+                    std::get<4>(item.info).func();
+                    break;
+                }
+                default: {
+                    RET_ERROR;
+                }
+            }
+        }
+    }
+
 }
 
 glm::mat4 camera_info::projection(float aspect) const {
     return glm::perspective(glm::radians(fov), aspect, near, far);
 }
 
-void render_scene(const scene_ptr &info_ptr) {
+void render_scene(const scene_ptr &info) {
     auto vp_size = query_viewport_size();
-
-    auto &info = *info_ptr;
-    auto req = render_request{
-            .camera_raw = info.camera,
-            .stream = info.stream,
+    fbo_conf.size = vp_size;
+    scene_fbo.create(fbo_conf);
+    scene_fbo.bind();
+    render_scene_impl::render_scene_impl(info, vp_size);
+    scene_fbo.unbind();
+
+    // render frame buffer
+    auto t_conf = tex_render_info{
+            .mode = TEX_COLOR_DEPTH
     };
-    req.camera.transform =
-            cvt_camera_transform(info.camera.transform);
-    req.camera.project = info.camera.projection(vp_size.aspectRatio());
-    if (info.light.follow_camera) {
-        req.light.direction = glm::column(info.camera.transform, 2);
-    } else {
-        req.light.direction = info.light.direction;
-    }
-
-    for (auto &item: info.items) {
-        req.item = item;
-        using info_type = std::remove_cvref_t<typeof(item.info)>;
-        switch (item.info.index()) {
-            case 1: {
-                static_assert(std::is_same_v<std::variant_alternative_t<1, info_type>,
-                        scene_render_info::image_info>);
-                render_image(req);
-                break;
-            }
-            case 2: {
-                static_assert(std::is_same_v<std::variant_alternative_t<2, info_type>,
-                        scene_render_info::mesh_info>);
-                render_mesh(req);
-                break;
-            }
-            case 3: {
-                static_assert(std::is_same_v<std::variant_alternative_t<3, info_type>,
-                        scene_render_info::pc_info>);
-                render_pc(req);
-                break;
-            }
-            case 4: {
-                static_assert(std::is_same_v<std::variant_alternative_t<4, info_type>,
-                        scene_render_info::custom_info>);
-                std::get<4>(item.info).func();
-                break;
-            }
-            default: {
-                RET_ERROR;
-            }
-        }
-    }
+    t_conf.color.fmt = COLOR_RGB;
+    t_conf.color.id = scene_fbo.color_tex[0].id;
+    t_conf.depth.id = scene_fbo.depth_tex.id;
+    t_conf.depth.enable_alpha_effect = false;
+    render_texture(t_conf);
 }

src/render/impl/render_scene_impl.h

@@ -17,6 +17,7 @@ namespace render_scene_impl {
     };
 
     extern color_image_render image_render;
+    extern smart_frame_buffer scene_fbo;
 
     void render_image(const render_request &req);
 
@@ -25,6 +26,9 @@ namespace render_scene_impl {
     void render_pc(const render_request &req);
 
     glm::mat4 cvt_camera_transform(const glm::mat4 &transform);
+
+    void render_scene_impl(const scene_ptr &info_ptr);
+
 }
 
 using namespace render_scene_impl;

src/render/impl/render_texture.cpp

@@ -210,10 +210,19 @@ namespace render_texture_impl {
         glActiveTexture(GL_TEXTURE0 + 1);
         glBindTexture(GL_TEXTURE_2D, info.depth.id);
         pg->set_uniform_i("d_tex", 1);
+        glActiveTexture(GL_TEXTURE0 + 2);
+        glBindTexture(GL_TEXTURE_2D, extra.bg_id);
+        pg->set_uniform_i("bg_d_tex", 2);
 
         glDisable(GL_DEPTH_TEST);
         config_buffers(info);
         draw();
+
+        // draw depth part
+        auto d_info = info;
+        d_info.mode = TEX_DEPTH_ONLY;
+        d_info.depth.enable_alpha_effect = false;
+        ren_d_only(d_info);
     }
 
     void ren_nv12_only(const tex_render_info &info) {

src/render/impl/shader/tex_rgb_d_v2.frag

@@ -5,13 +5,13 @@ uniform float alpha_factor;
 
 uniform mat4 proj_inv;
 
-uniform sampler2D c_tex;// color texture
-uniform sampler2D d_tex;// depth texture
+uniform sampler2D c_tex; // color texture
+uniform sampler2D d_tex; // depth texture
+uniform sampler2D bg_d_tex; // background depth texture
 
 in vec2 frag_uv;
 
 layout (location = 0) out vec4 frag_color;
-layout (depth_less) out float gl_FragDepth;
 
 vec3 ndc_to_camera(in vec3 ndc) {
     vec4 position = proj_inv * vec4(ndc, 1.0);// NDC -> camera
@@ -31,13 +31,12 @@ void main() {
     if (cur_depth == 1.0) return;
 
     float alpha = opacity;
-    if (cur_depth > gl_FragDepth) {
+    float bg_depth = texture(bg_d_tex, frag_uv).x;
+    if (cur_depth > bg_depth) {
         vec3 cur_pos = tex_to_camera(vec3(frag_uv, cur_depth));
-        vec3 bg_pos  = tex_to_camera(vec3(frag_uv, gl_FragDepth));
+        vec3 bg_pos  = tex_to_camera(vec3(frag_uv, bg_depth));
         float dis = distance(cur_pos, bg_pos);
         alpha *= exp(-alpha_factor * dis);
-    } else {
-        gl_FragDepth = cur_depth;
     }
 
     frag_color.rgb = texture(c_tex, frag_uv).rgb;

src/render/render_mesh.h

@@ -92,7 +92,8 @@ struct mesh_render_info {
         struct {
         } normal;
         struct {
-            mesh_info_type bg; // if nullptr, depth buffer will be used.
+            mesh_info_type bg; // if nullptr, bg_tex should be provided.
+            GLuint bg_tex = 0; // background depth texture
             float alpha_factor;
             bool show_bg;
         } depth_alpha;

src/render/render_texture.h

@@ -35,6 +35,7 @@ struct tex_render_info {
 
     union {
         struct {
+            GLuint bg_id; // background depth texture
             glm::mat4 proj_mat;
             GLfloat alpha_factor;
         } color_depth;