Implemented depth mask remote guidance.

CMakeLists.txt

@@ -17,6 +17,7 @@ add_executable(${PROJECT_NAME} src/main.cpp
         src/module_v3/registration.cpp
         src/module/impl/augment_manager.cpp
         src/module/impl/camera_augment_helper.cpp
+        src/module/impl/depth_guide_controller.cpp
         src/module/impl/image_augment_helper.cpp
         src/module/impl/image_player.cpp
         src/module/impl/image_viewer.cpp

src/core/cuda_helper.hpp

@@ -58,4 +58,17 @@ struct smart_cuda_stream {
 
 extern smart_cuda_stream *default_cuda_stream;
 
+
+template<typename T>
+inline void extend_pointer_life(const std::shared_ptr<T> &ptr,
+                                smart_cuda_stream *stream) {
+    using ptr_type = std::shared_ptr<T>;
+    auto ptr_ptr = new ptr_type();
+    *ptr_ptr = ptr;
+    CUDA_API_CHECK(cudaLaunchHostFunc(stream->cuda, [](void *ptr) {
+        auto ptr_ptr = (ptr_type *) ptr;
+        delete ptr_ptr;
+    }, ptr_ptr));
+}
+
 #endif //DEPTHGUIDE_CUDA_HELPER_HPP

src/core/image_utility.hpp

@@ -284,6 +284,12 @@ public:
         }
     }
 
+    ~smart_image() {
+        if (event != nullptr) {
+            CUDA_API_CHECK(cudaEventDestroy(event));
+        }
+    }
+
     image_info_type<T> as_host_info(smart_cuda_stream *stream = nullptr) {
         if (host_info.ptr == nullptr) {
             assert(cuda_info.ptr != nullptr);
@@ -338,14 +344,34 @@ public:
         return host_info.size;
     }
 
+    void record_create(smart_cuda_stream *stream = nullptr) {
+        if (event == nullptr) {
+            CUDA_API_CHECK(cudaEventCreateWithFlags(&event, cudaEventDisableTiming));
+        }
+        assert(event != nullptr);
+        CUDA_API_CHECK(cudaEventRecord(event, stream->cuda));
+    }
+
+    void sync_create(smart_cuda_stream *stream = nullptr) {
+        if (event == nullptr) return;
+        CUDA_API_CHECK(cudaStreamWaitEvent(stream->cuda, event));
+    }
+
 private:
     image_info_type<T> host_info;
     image_info_type<T> cuda_info;
+
+    cudaEvent_t event = nullptr;
 };
 
 template<typename T>
-auto create_image(image_info_type<T> info) {
-    return std::make_shared<smart_image<T>>(info);
+auto create_image(image_info_type<T> info,
+                  smart_cuda_stream *stream = nullptr) {
+    auto ret = std::make_shared<smart_image<T>>(info);
+    if (stream != nullptr) {
+        ret->record_create(stream);
+    }
+    return ret;
 }
 
 using image_info_u8c1 = image_info_type<uchar1>;

src/image_process/cuda_impl/image_merge.cuh

@@ -8,6 +8,7 @@ struct depth_mask_config {
     float upper = 1;
 };
 
+// alpha = 0, if depth is not in the range
 void call_depth_mask(image_type_v2<uchar3> img, image_type_v2<float1> depth,
                      image_type_v2<uchar4> out,
                      depth_mask_config conf, cudaStream_t stream);

src/impl/apps/remote_ar/remote_ar.cpp

@@ -93,6 +93,11 @@ app_remote_ar::app_remote_ar(const create_config &_conf) {
     };
     guide_decode = std::make_unique<versatile_convertor>(guide_decode_conf);
 
+    auto guide_control_conf = depth_guide_controller::create_config{
+            .img_in = guide_img, .depth_in = guide_depth, .img_out = guide_final
+    };
+    guide_controller = std::make_unique<depth_guide_controller>(guide_control_conf);
+
     auto aug_conf = augment_manager::create_config{
             .item_list = augment_manager::item_list_from_yaml(LOAD_LIST("augment_list")),
             .sophiar_conn = sophiar_conn.get()
@@ -114,14 +119,24 @@ app_remote_ar::app_remote_ar(const create_config &_conf) {
             .in_name = rgb_left, .out_name = aug_left,
             .flip_image = true, .stream = &cam_left.stream
     };
-    left_aug_ren_conf.render_func_list.emplace_back([this] { cam_left.aug_helper->render(); });
+    left_aug_ren_conf.render_func_list.emplace_back([this] {
+        cam_left.aug_helper->render();
+    });
+    left_aug_ren_conf.render_func_list.emplace_back([this] {
+        guide_controller->render({.stream = &cam_left.stream}); // TODO: add camera specific options
+    });
     cam_left.aug_render = std::make_unique<image_augment_helper>(left_aug_ren_conf);
 
     auto right_aug_ren_conf = image_augment_helper::create_config{
             .in_name = rgb_right, .out_name = aug_right,
             .flip_image = true, .stream = &cam_right.stream
     };
-    right_aug_ren_conf.render_func_list.emplace_back([this] { cam_right.aug_helper->render(); });
+    right_aug_ren_conf.render_func_list.emplace_back([this] {
+        cam_right.aug_helper->render();
+    });
+    right_aug_ren_conf.render_func_list.emplace_back([this] {
+        guide_controller->render({.stream = &cam_right.stream});
+    });
     cam_right.aug_render = std::make_unique<image_augment_helper>(right_aug_ren_conf);
 
     auto output_size = cv::Size(
@@ -143,25 +158,25 @@ app_remote_ar::app_remote_ar(const create_config &_conf) {
         post(*asio_ctx, [=, this] { stereo_aug->resize(size); });
     });
 
-//    auto bg_viewer_conf = image_viewer::create_config{
-//            .mode = VIEW_STEREO, .flip_y = false,
-//            .stream = default_cuda_stream,
-//    };
-//    auto &stereo_conf = bg_viewer_conf.extra.stereo;
-//    stereo_conf.c_fmt = COLOR_RGB;
-//    stereo_conf.left_name = aug_left;
-//    stereo_conf.right_name = aug_right;
-//    bg_viewer = std::make_unique<image_viewer>(bg_viewer_conf);
-
     auto bg_viewer_conf = image_viewer::create_config{
-            .mode = VIEW_COLOR_DEPTH, .flip_y = true,
+            .mode = VIEW_STEREO, .flip_y = false,
             .stream = default_cuda_stream,
     };
-    auto &bg_extra_conf = bg_viewer_conf.extra.color_depth;
-    bg_extra_conf.c_fmt = COLOR_RGB;
-    bg_extra_conf.c_name = guide_img;
-    bg_extra_conf.d_name = guide_depth;
+    auto &stereo_conf = bg_viewer_conf.extra.stereo;
+    stereo_conf.c_fmt = COLOR_RGB;
+    stereo_conf.left_name = aug_left;
+    stereo_conf.right_name = aug_right;
     bg_viewer = std::make_unique<image_viewer>(bg_viewer_conf);
+
+//    auto bg_viewer_conf = image_viewer::create_config{
+//            .mode = VIEW_COLOR_DEPTH, .flip_y = true,
+//            .stream = default_cuda_stream,
+//    };
+//    auto &bg_extra_conf = bg_viewer_conf.extra.color_depth;
+//    bg_extra_conf.c_fmt = COLOR_RGB;
+//    bg_extra_conf.c_name = guide_img;
+//    bg_extra_conf.d_name = guide_depth;
+//    bg_viewer = std::make_unique<image_viewer>(bg_viewer_conf);
 }
 
 void app_remote_ar::start_tracking() {
@@ -205,6 +220,8 @@ void app_remote_ar::show_ui() {
         if (ImGui::CollapsingHeader("Depth Guide")) {
             auto id_guard = imgui_id_guard("depth_guide");
             guide_player->show();
+            ImGui::SeparatorText("Display");
+            guide_controller->show();
         }
 
         if (ImGui::CollapsingHeader("Stereo")) {

src/impl/apps/remote_ar/remote_ar.h

@@ -6,6 +6,7 @@
 #include "device/mvs_camera_ui.h"
 #include "module/augment_manager.h"
 #include "module/camera_augment_helper.h"
+#include "module/depth_guide_controller.h"
 #include "module/image_augment_helper.h"
 #include "module/image_player.h"
 #include "module/image_streamer.h"
@@ -44,6 +45,7 @@ private:
         guide_combine_rgb,
         guide_img, guide_depth_fake,
         guide_depth,
+        guide_final
     };
 
     struct camera_module {
@@ -74,6 +76,7 @@ private:
     std::unique_ptr<versatile_convertor> guide_cvt;
     std::unique_ptr<versatile_convertor> guide_split;
     std::unique_ptr<versatile_convertor> guide_decode;
+    std::unique_ptr<depth_guide_controller> guide_controller;
 
     camera_module cam_left;
     camera_module cam_right;

src/module/depth_guide_controller.h

@@ -0,0 +1,35 @@
+#ifndef DEPTHGUIDE_DEPTH_GUIDE_CONTROLLER_H
+#define DEPTHGUIDE_DEPTH_GUIDE_CONTROLLER_H
+
+#include "core/cuda_helper.hpp"
+#include "core/object_manager.h"
+
+#include <memory>
+
+class depth_guide_controller { // TODO: currently in passive mode
+public:
+
+    struct create_config {
+        obj_name_type img_in; // image_u8c3
+        obj_name_type depth_in; // image_f32c1
+        obj_name_type img_out = invalid_obj_name; // can be invalid
+    };
+
+    explicit depth_guide_controller(create_config conf);
+
+    ~depth_guide_controller();
+
+    struct render_config {
+        smart_cuda_stream *stream = nullptr;
+    };
+
+    void render(render_config conf);
+
+    void show();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#endif //DEPTHGUIDE_DEPTH_GUIDE_CONTROLLER_H

src/module/impl/depth_guide_controller.cpp

@@ -0,0 +1,61 @@
+#include "depth_guide_controller_impl.h"
+#include "core/imgui_utility.hpp"
+
+depth_guide_controller::impl::impl(create_config _conf) {
+    conf = _conf;
+}
+
+void depth_guide_controller::impl::build(render_config r_conf) {
+    auto obj_ts = OBJ_TS(conf.img_in);
+    if (obj_ts <= last_ts) return;
+
+    auto img_in = OBJ_QUERY(image_u8c3, conf.img_in);
+    auto depth_in = OBJ_QUERY(image_f32c1, conf.depth_in);
+    if (img_in == nullptr || depth_in == nullptr) return;
+    img_in->sync_create(r_conf.stream);
+    depth_in->sync_create(r_conf.stream);
+
+    auto out_info = create_image_info<uchar4>(img_in->size(), MEM_CUDA);
+    auto depth_conf = depth_mask_config{
+            .lower = depth_range.min * 1000.f, // mm -> m
+            .upper = depth_range.max * 1000.f,
+    };
+    call_depth_mask(
+            img_in->as_cuda(r_conf.stream), depth_in->as_cuda(r_conf.stream),
+            out_info.as_cuda(), depth_conf, r_conf.stream->cuda);
+    out_img = create_image(out_info, r_conf.stream);
+    if (conf.img_out != invalid_obj_name) {
+        OBJ_SAVE(conf.img_out, out_img);
+    }
+    last_ts = obj_ts;
+}
+
+void depth_guide_controller::impl::render(render_config r_conf) {
+    build(r_conf);
+    if (out_img == nullptr) return;
+    auto ren_conf = color_image_render::config_type{
+            .fmt = COLOR_RGBA, .flip_y = true, .stream = r_conf.stream,
+    };
+    ren.render_rgba(out_img->as_info(), ren_conf);
+}
+
+void depth_guide_controller::impl::show() {
+    static constexpr float dep_hard_min = 0.15; // TODO: config value
+    static constexpr float dep_hard_max = 1.00;
+    ImGui::DragFloat2("Depth Range (m)", (float *) &depth_range, 0.02f,
+                      dep_hard_min, dep_hard_max, "%.2f");
+}
+
+depth_guide_controller::depth_guide_controller(create_config conf)
+        : pimpl(std::make_unique<impl>(conf)) {
+}
+
+depth_guide_controller::~depth_guide_controller() = default;
+
+void depth_guide_controller::render(render_config conf) {
+    pimpl->render(conf);
+}
+
+void depth_guide_controller::show() {
+    pimpl->show();
+}

src/module/impl/depth_guide_controller_impl.h

@@ -0,0 +1,32 @@
+#ifndef DEPTHGUIDE_DEPTH_GUIDE_CONTROLLER_IMPL_H
+#define DEPTHGUIDE_DEPTH_GUIDE_CONTROLLER_IMPL_H
+
+#include "module/depth_guide_controller.h"
+#include "core/image_utility.hpp"
+#include "image_process/cuda_impl/image_merge.cuh"
+#include "render/render_tools.h"
+
+struct depth_guide_controller::impl {
+
+    create_config conf;
+
+    timestamp_type last_ts = 0;
+    image_u8c4 out_img;
+    color_image_render ren;
+
+    struct {
+        float min = 0.15; // unit in meter
+        float max = 10.0; // unit in meter
+    } depth_range;
+
+    explicit impl(create_config conf);
+
+    void build(render_config conf);
+
+    void render(render_config conf);
+
+    void show();
+
+};
+
+#endif //DEPTHGUIDE_DEPTH_GUIDE_CONTROLLER_IMPL_H

src/render/impl/render_tools.cpp

@@ -24,21 +24,19 @@ void color_image_render::render_tex(cv::Size img_size, config_type conf) {
 }
 
 template<typename T>
-void color_image_render::render_rgb(const image_info_type<T> &info, config_type conf) {
-    assert(conf.fmt == COLOR_RGB);
+void color_image_render::render_rgba(const image_info_type<T> &info, config_type conf) {
     img_tex.upload(info, conf.stream);
     render_tex(info.size, conf);
 }
 
 // @formatter:off
-template void color_image_render::render_rgb<uchar1>(const image_info_u8c1 &, config_type);
-template void color_image_render::render_rgb<uchar2>(const image_info_u8c2 &, config_type);
-template void color_image_render::render_rgb<uchar3>(const image_info_u8c3 &, config_type);
-template void color_image_render::render_rgb<uchar4>(const image_info_u8c4 &, config_type);
+template void color_image_render::render_rgba<uchar1>(const image_info_u8c1 &, config_type);
+template void color_image_render::render_rgba<uchar2>(const image_info_u8c2 &, config_type);
+template void color_image_render::render_rgba<uchar3>(const image_info_u8c3 &, config_type);
+template void color_image_render::render_rgba<uchar4>(const image_info_u8c4 &, config_type);
 // @formatter:on
 
-void color_image_render::render_rgb(obj_name_type name, config_type conf) {
-    assert(conf.fmt == COLOR_RGB);
+void color_image_render::render_rgba(obj_name_type name, config_type conf) {
     auto img_type = OBJ_TYPE(name);
 
     auto impl_func = [&](auto V) {
@@ -46,7 +44,8 @@ void color_image_render::render_rgb(obj_name_type name, config_type conf) {
         if (img_type == typeid(T)) {
             auto img = OBJ_QUERY(T, name);
             if (img == nullptr) return;
-            render_rgb(img->as_info(), conf);
+            img->sync_create(conf.stream);
+            render_rgba(img->as_info(), conf);
         }
     };
     impl_func(image_u8c1());
@@ -72,13 +71,15 @@ void color_image_render::render_nv12(obj_name_type name, config_type conf) {
     assert(OBJ_TYPE(name) == typeid(image_u8c1));
     auto img = OBJ_QUERY(image_u8c1, name);
     if (img == nullptr) return;
+    img->sync_create(conf.stream);
     render_nv12(img->as_info(), conf);
 }
 
 void color_image_render::render(obj_name_type name, config_type conf) {
     switch (conf.fmt) {
-        case COLOR_RGB: {
-            render_rgb(name, conf);
+        case COLOR_RGB:
+        case COLOR_RGBA: {
+            render_rgba(name, conf);
             break;
         }
         case COLOR_NV12: {
@@ -121,7 +122,7 @@ void depth_image_render::render(obj_name_type name, config_type conf) {
     c_conf.flip_y = conf.flip_y;
     c_conf.alpha = conf.alpha;
     c_conf.stream = conf.stream;
-    color_render.render_rgb(img_color, c_conf);
+    color_render.render_rgba(img_color, c_conf);
 }
 
 //frame_builder::frame_builder(const create_config &_conf) {

src/render/impl/render_utility.cpp

@@ -107,8 +107,6 @@ void smart_texture::deallocate() {
         CUDA_API_CHECK(cudaGraphicsUnregisterResource(cuda_res));
         cuda_res = nullptr;
     }
-
-    img_ptr = nullptr;
 }
 
 void smart_texture::set_filter(GLint min_filter, GLint max_filter) {
@@ -145,12 +143,12 @@ void smart_texture::upload_impl(const image_mem_info &img, smart_cuda_stream *st
     cudaArray_t tex_arr;
     CUDA_API_CHECK(cudaGraphicsMapResources(1, &cuda_res, stream->cuda));
     CUDA_API_CHECK(cudaGraphicsSubResourceGetMappedArray(&tex_arr, cuda_res, 0, 0));
-    img_ptr = img.ptr; // extend the lifetime of img.ptr
     CUDA_API_CHECK(cudaMemcpy2DToArrayAsync(
-            tex_arr, 0, 0, img_ptr.get(), img.pitch, img.width, img.height,
+            tex_arr, 0, 0, img.ptr.get(), img.pitch, img.width, img.height,
             img.loc == MEM_CUDA ? cudaMemcpyDeviceToDevice : cudaMemcpyHostToDevice,
             stream->cuda));
     CUDA_API_CHECK(cudaGraphicsUnmapResources(1, &cuda_res, stream->cuda));
+    extend_pointer_life(img.ptr, stream);
 }
 
 void smart_texture::upload_impl(GLuint pbo_id, GLenum _format, GLenum type) {

src/render/render_tools.h

@@ -26,10 +26,10 @@ public:
 
     // T = uchar{1-4}
     template<typename T>
-    void render_rgb(const image_info_type<T> &img, config_type conf);
+    void render_rgba(const image_info_type<T> &img, config_type conf);
 
     // supports image_u8c{1-4}
-    void render_rgb(obj_name_type name, config_type conf);
+    void render_rgba(obj_name_type name, config_type conf);
 
     // supports image_u8c1
     void render_nv12(const image_info_u8c1 &img, config_type conf);
@@ -66,43 +66,5 @@ private:
     color_image_render color_render;
 };
 
-//class frame_builder {
-//public:
-//
-//    smart_frame_buffer fbo;
-//
-//    struct create_config {
-//        static constexpr auto reserve_cnt = 4;
-//        using render_func_type = std::function<void()>;
-//        using render_func_list_type =
-//                boost::container::small_vector<render_func_type, reserve_cnt>;
-//        render_func_list_type render_func_list;
-//
-//        using fbo_config_type = smart_frame_buffer::create_config;
-//        fbo_config_type fbo_conf;
-//    };
-//
-//    explicit frame_builder(const create_config &conf);
-//
-//    void build();
-//
-//    struct render_config {
-//        tex_render_mode mode = TEX_COLOR_ONLY;
-//        simple_rect range = {-1.0f, -1.0f, 2.0f, 2.0f};
-//        bool flip_y = false;
-//        GLfloat opacity = 1.0;
-//    };
-//
-//    void render(render_config conf);
-//
-//    struct modifiable_config {
-//        cv::Size size; // fbo size
-//    };
-//
-//    void change_config(modifiable_config conf);
-//
-//private:
-//    create_config conf;
-//};
 
 #endif //DEPTHGUIDE_RENDER_TOOLS_H

src/render/render_utility.h

@@ -25,11 +25,12 @@ void check_program(const char *name, GLuint id);
 template<typename T>
 constexpr inline GLenum get_tex_format() {
     // @formatter:off
-    if constexpr (std::is_same_v<T, uchar1>) { return GL_RED; }
-    if constexpr (std::is_same_v<T, uchar2>) { return GL_RG;  }
-    if constexpr (std::is_same_v<T, uchar3>) { return GL_RGB; }
+    if constexpr (std::is_same_v<T, uchar1>) { return GL_RED;  }
+    if constexpr (std::is_same_v<T, uchar2>) { return GL_RG;   }
+    if constexpr (std::is_same_v<T, uchar3>) { return GL_RGB;  }
+    if constexpr (std::is_same_v<T, uchar4>) { return GL_RGBA; }
     // @formatter:on
-    return 0;
+    RET_ERROR_E;
 }
 
 template<typename T>
@@ -38,18 +39,20 @@ constexpr inline GLenum get_tex_type() {
     if constexpr (std::is_same_v<T, uchar1>) { return GL_UNSIGNED_BYTE; }
     if constexpr (std::is_same_v<T, uchar2>) { return GL_UNSIGNED_BYTE; }
     if constexpr (std::is_same_v<T, uchar3>) { return GL_UNSIGNED_BYTE; }
+    if constexpr (std::is_same_v<T, uchar4>) { return GL_UNSIGNED_BYTE; }
     // @formatter:on
-    return 0;
+    RET_ERROR_E;
 }
 
 template<typename T>
 constexpr inline GLenum get_tex_internal_format() {
     // @formatter:off
-    if constexpr (std::is_same_v<T, uchar1>) { return GL_R8;   }
-    if constexpr (std::is_same_v<T, uchar2>) { return GL_RG8;  }
-    if constexpr (std::is_same_v<T, uchar3>) { return GL_RGB8; }
+    if constexpr (std::is_same_v<T, uchar1>) { return GL_R8;    }
+    if constexpr (std::is_same_v<T, uchar2>) { return GL_RG8;   }
+    if constexpr (std::is_same_v<T, uchar3>) { return GL_RGB8;  }
+    if constexpr (std::is_same_v<T, uchar4>) { return GL_RGBA8; }
     // @formatter:on
-    return 0;
+    RET_ERROR_E;
 }
 
 enum color_format : uint8_t {
@@ -122,7 +125,6 @@ public:
 
     // used for CUDA inter-op
     cudaGraphicsResource_t cuda_res = nullptr;
-    std::shared_ptr<void> img_ptr;
 
     // for image upload
     smart_pixel_buffer img_pbo;