Implemented cursor guide.

CMakeLists.txt

@@ -167,6 +167,13 @@ target_link_libraries(${PROJECT_NAME} ${VTK_LIBRARIES})
 vtk_module_autoinit(TARGETS ${PROJECT_NAME} MODULES ${VTK_LIBRARIES})
 target_sources(${PROJECT_NAME} PRIVATE src/render_v3/vtk_viewer.cpp)
 
+# PCL config
+find_package(PCL REQUIRED)
+target_include_directories(${PROJECT_NAME} PRIVATE ${PCL_INCLUDE_DIRS})
+target_link_directories(${PROJECT_NAME} PRIVATE ${PCL_LIBRARY_DIRS})
+target_compile_definitions(${PROJECT_NAME} PRIVATE ${PCL_DEFINITIONS})
+target_link_libraries(${PROJECT_NAME} ${PCL_LIBRARIES})
+
 # Eigen3 config
 find_package(Eigen3 REQUIRED)
 target_link_libraries(${PROJECT_NAME} Eigen3::Eigen)

src/core/image_utility_v2.h

@@ -35,6 +35,7 @@ enum color_format : uint8_t {
     COLOR_RGBA,
     COLOR_ARGB,
     COLOR_NV12,
+    COLOR_DEPTH_32F,
     COLOR_ANY = 0xFF,
 };
 
@@ -117,6 +118,15 @@ public:
     pointer sub_image(int row = 0, int col = 0,
                       int width = -1, int height = -1) const;
 
+    // read certain pixel
+    template<typename T>
+    T pixel_at(int row, int col, int component = 0,
+               smart_cuda_stream *stream = nullptr) {
+        T ret;
+        pixel_at_impl(row, col, component, &ret, sizeof(T), stream);
+        return ret;
+    }
+
     pointer bit_cast(int type);
 
     template<typename T>
@@ -132,6 +142,9 @@ private:
 
     pointer shallow_clone() const;
 
+    void pixel_at_impl(int row, int col, int component,
+                       void *dst, size_t size, smart_cuda_stream *stream);
+
     friend class image_memory;
 
 public:

src/core/impl/image_utility_v2.cpp

@@ -243,6 +243,20 @@ std::shared_ptr<smart_image<T>> generic_image::impl::get_image_v1() const {
     return ret;
 }
 
+void generic_image::impl::pixel_at(int row, int col, int component,
+                                   void *dst, size_t _size, smart_cuda_stream *stream) {
+    if (store_host.ptr != nullptr) {
+        auto ptr = get_memory(MEM_HOST, stream).at(row, col, component);
+        memcpy(dst, ptr, _size);
+    } else {
+        assert(store_cuda.ptr != nullptr);
+        auto ptr = get_memory(MEM_CUDA, stream).at(row, col, component);
+        CUDA_API_CHECK(cudaMemcpyAsync(dst, ptr, _size,
+                                       cudaMemcpyDeviceToHost, stream->cuda));
+        CUDA_API_CHECK(cudaStreamSynchronize(stream->cuda));
+    }
+}
+
 template<typename T>
 void generic_image::impl::create_from_v1(const std::shared_ptr<smart_image<T>> &img) {
     if (img->host_info.ptr != nullptr) {
@@ -398,6 +412,11 @@ generic_image::pointer generic_image::shallow_clone() const {
     return ret;
 }
 
+void generic_image::pixel_at_impl(int row, int col, int component,
+                                  void *dst, size_t size, smart_cuda_stream *stream) {
+    return pimpl->pixel_at(row, col, component, dst, size, stream);
+}
+
 generic_image::pointer generic_image::sub_image(int row, int col, int width, int height) const {
     auto ret = shallow_clone();
     ret->pimpl->sub_image_inplace(row, col, width, height);

src/core/impl/image_utility_v2_impl.h

@@ -75,6 +75,9 @@ struct generic_image::impl : public meta_base::impl {
     template<typename T>
     std::shared_ptr<smart_image<T>> get_image_v1() const;
 
+    void pixel_at(int row, int col, int component,
+                  void *dst, size_t size, smart_cuda_stream *stream);
+
     template<typename T>
     void create_from_v1(const std::shared_ptr<smart_image<T>> &img);
 

src/core/math_helper.hpp

@@ -30,6 +30,17 @@ inline glm::mat4 to_mat4(const Eigen::Transform<Scalar, 3, Mode> &m) {
     return ret;
 }
 
+template<typename T>
+concept Vec2Type = requires(T t) {
+    { t.x } -> std::convertible_to<float>;
+    { t.y } -> std::convertible_to<float>;
+};
+
+template<Vec2Type T>
+inline auto to_vec2(const T &v) {
+    return glm::vec2(v.x, v.y);
+}
+
 template<typename T>
 concept Vec3Type = requires(T t) {
     { t.x } -> std::convertible_to<float>;
@@ -42,4 +53,12 @@ inline auto to_vec3(const T &v) {
     return glm::vec3(v.x, v.y, v.z);
 }
 
+inline auto to_homo(const glm::vec3 &v) {
+    return glm::vec4(v, 1.f);
+}
+
+inline auto from_homo(const glm::vec4 &v) {
+    return glm::vec3(v) / v.w;
+}
+
 #endif //DEPTHGUIDE_MATH_HELPER_HPP

src/image_process/cuda_impl/pc_filter.cu

@@ -1,9 +1,29 @@
 #include "pc_filter.cuh"
 
 #include <thrust/copy.h>
+#include <thrust/extrema.h>
 
 namespace pc_filter {
 
+    template<typename PointT>
+    __host__ __device__
+    constexpr auto to_vec3(const PointT &p) {
+        return glm::vec3(p.x, p.y, p.z);
+    }
+
+    template<typename PointT>
+    __host__ __device__
+    constexpr float proj_t_to_ray(const PointT &p, const ray_type &r) {
+        return glm::dot(to_vec3(p), r.d);
+    }
+
+    template<typename PointT>
+    __host__ __device__
+    constexpr float dis_to_ray(const PointT &p, const ray_type &r) {
+        auto t = proj_t_to_ray(p, r);
+        return glm::distance(to_vec3(p), r.o + r.d * t);
+    }
+
     template<typename PointT>
     struct non_zero_depth_op {
         __host__ __device__
@@ -31,4 +51,38 @@ namespace pc_filter {
     template void shrink_zero_depth<pc_xyz_rgb_type>(pc_type_rgb , pc_type_rgb, size_t *, cudaStream_t);
     // @formatter:on
 
+    template<typename PointT>
+    struct nearest_dis_op {
+        ray_type ray;
+        float radius;
+
+        __host__ __device__
+        bool operator()(const PointT &lhs, const PointT &rhs) { // TODO: optimize
+            auto dis_l = dis_to_ray(lhs, ray);
+            auto dis_r = dis_to_ray(rhs, ray);
+            auto flag_l = (dis_l <= radius), flag_r = (dis_r <= radius);
+            if (flag_l != flag_r) {
+                return flag_l > flag_r;
+            }
+
+            auto t_l = proj_t_to_ray(lhs, ray);
+            auto t_r = proj_t_to_ray(rhs, ray);
+            return t_l < t_r;
+        }
+    };
+
+    template<typename PointT>
+    void nearest_point_index(pc_type_v1<PointT> in, ray_type ray, float radius,
+                             size_t *out_index, cudaStream_t stream) {
+        auto op = nearest_dis_op<PointT>(ray, radius);
+        auto ret_iter = thrust::min_element(thrust::cuda::par.on(stream),
+                                            in.ptr, in.ptr + in.num, op);
+        *out_index = ret_iter - in.ptr;
+    }
+
+    // @formatter:off // TODO optimize before use
+//    template void nearest_point_index<pc_xyz_type>(pc_type_xyz, ray_type, float, size_t *, cudaStream_t);
+//    template void nearest_point_index<pc_xyz_rgb_type>(pc_type_rgb, ray_type, float, size_t *, cudaStream_t);
+    // @formatter:on
+
 }

src/image_process/cuda_impl/pc_filter.cuh

@@ -9,6 +9,10 @@ namespace pc_filter {
     void shrink_zero_depth(pc_type_v1<PointT> in, pc_type_v1<PointT> out,
                            size_t *out_num, cudaStream_t stream);
 
+    template<typename PointT>
+    void nearest_point_index(pc_type_v1<PointT> in, ray_type ray, float radius,
+                             size_t *out_index, cudaStream_t stream);
+
 }
 
 #endif //DEPTHGUIDE_PC_FILTER_CUH

src/image_process/cuda_impl/pc_utility.cuh

@@ -4,6 +4,8 @@
 #include <cassert>
 #include <cstdint>
 
+#include <glm/glm.hpp>
+
 template<typename PointT>
 struct pc_type_v1 {
     PointT *ptr = nullptr;
@@ -26,4 +28,9 @@ struct pc_xyz_rgb_type {
 using pc_type_xyz = pc_type_v1<pc_xyz_type>;
 using pc_type_rgb = pc_type_v1<pc_xyz_rgb_type>;
 
+struct ray_type {
+    glm::vec3 o; // origin
+    glm::vec3 d; // direction
+};
+
 #endif //DEPTHGUIDE_PC_UTILITY_CUH

src/impl/apps/depth_guide/depth_guide.cpp

@@ -103,8 +103,10 @@ app_depth_guide::app_depth_guide(const create_config &_conf) {
         auto aug_conf = camera_augment_helper_v2::create_config{
                 .camera = camera_augment_helper_v2::create_config::freedom_camera_config{},
                 .manager = aug_manager.get(),
+                .ctx = conf.asio_ctx,
         };
         aug_helper_left = std::make_unique<camera_augment_helper_v2>(aug_conf);
+        aug_helper_left->set_interactive(true);
     }
 }
 

src/module/augment_manager_v2.h

@@ -56,6 +56,11 @@ public:
 
     void render(const camera_info &info);
 
+    using render_sig_type =
+            boost::signals2::signal<void(scene_render_info &)>;
+    render_sig_type pre_render_sig;
+    render_sig_type post_render_sig;
+
 private:
     struct impl;
     std::unique_ptr<impl> pimpl;

src/module/camera_augment_helper_v2.h

@@ -39,6 +39,9 @@ public:
         sophiar_conn_type *sophiar_conn = nullptr;
 
         augment_manager_v2 *manager = nullptr;
+
+        using io_context = boost::asio::io_context;
+        io_context *ctx = nullptr;
     };
 
     explicit camera_augment_helper_v2(const create_config &conf);
@@ -50,12 +53,15 @@ public:
     struct output_config {
         cv::Size size;
         obj_name_type img_name;
+        smart_cuda_stream *stream;
     };
 
     void render_image(output_config conf);
 
     void show();
 
+    void set_interactive(bool flag);
+
 private:
     struct impl;
     std::unique_ptr<impl> pimpl;

src/module/impl/augment_manager_v2.cpp

@@ -94,7 +94,9 @@ void augment_manager_v2::impl::render(const camera_info &info) {
         ren_info.items.push_back(ren_item);
     }
 
+    q_this->pre_render_sig(ren_info);
     render_scene(ren_info);
+    q_this->post_render_sig(ren_info);
 }
 
 void augment_manager_v2::impl::show() {
@@ -140,6 +142,7 @@ void augment_manager_v2::impl::show() {
 
 augment_manager_v2::augment_manager_v2(const create_config &conf)
         : pimpl(std::make_unique<impl>(conf)) {
+    pimpl->q_this = this;
 }
 
 augment_manager_v2::~augment_manager_v2() = default;

src/module/impl/augment_manager_v2_impl.h

@@ -39,6 +39,8 @@ struct augment_manager_v2::impl {
         void update_transform(impl *pimpl);
     };
 
+    augment_manager_v2 *q_this = nullptr;
+
     using item_list_type =
             std::vector<item_store_type>;
     item_list_type item_list;

src/module/impl/camera_augment_helper_v2.cpp

@@ -6,6 +6,12 @@
 #include <glm/gtc/type_ptr.hpp>
 #include <glm/gtx/rotate_vector.hpp>
 
+#include <boost/asio/post.hpp>
+
+#include <vtkSphereSource.h>
+
+using boost::asio::post;
+
 glm::mat4 camera_augment_helper_v2::impl::freedom_info_type::to_transform() {
     // normalize view up
     view_up -= view_dir() * glm::dot(view_up, view_dir());
@@ -38,6 +44,7 @@ void camera_augment_helper_v2::impl::freedom_info_type::move_right(float dis) {
     translate(-axis * dis);
 }
 
+// TODO: using accelerometer determined world coordinate
 void camera_augment_helper_v2::impl::freedom_info_type::move_up(float dis) {
     translate(view_up * dis);
 }
@@ -57,7 +64,7 @@ void camera_augment_helper_v2::impl::freedom_info_type::yaw(float deg) {
 
 void camera_augment_helper_v2::impl::freedom_info_type::pitch(float deg) {
     auto rot_axis = glm::cross(view_up, view_dir());
-    view_up = glm::rotate(view_up, deg, rot_axis);
+    view_up = glm::rotate(view_up, glm::radians(deg), rot_axis);
     auto next_view_dir = glm::rotate(view_dir(), glm::radians(deg), rot_axis);
     center = eye + next_view_dir * view_dis();
 }
@@ -69,6 +76,10 @@ glm::mat4 camera_augment_helper_v2::impl::fixed_info_type::extra_transform() con
 camera_augment_helper_v2::impl::impl(const create_config &conf) {
     manager = conf.manager;
     sophiar_conn = conf.sophiar_conn;
+    ctx = conf.ctx;
+
+    pre_render_conn = manager->pre_render_sig.connect(
+            [this](auto &info) { pre_render_slot(info); });
 
     switch (conf.camera.index()) {
         case 1: {
@@ -96,6 +107,10 @@ camera_augment_helper_v2::impl::impl(const create_config &conf) {
     }
 }
 
+camera_augment_helper_v2::impl::~impl() {
+    pre_render_conn.disconnect();
+}
+
 void camera_augment_helper_v2::impl::update_freedom() {
     transform = freedom_info.to_transform();
     is_missing = false;
@@ -136,6 +151,8 @@ void camera_augment_helper_v2::impl::render() {
     };
     if (!is_missing || ignore_missing) {
         manager->render(ren_info);
+        last_camera = ren_info;
+        last_vp_size = query_viewport_size();
     }
 }
 
@@ -143,6 +160,11 @@ void camera_augment_helper_v2::impl::render_image(output_config conf) {
     fbo_conf.size = conf.size;
     fbo.create(fbo_conf);
 
+    if (img_downloader == nullptr) [[unlikely]] {
+        img_downloader = std::make_unique<viewport_downloader>(
+                viewport_downloader::create_config{.stream = conf.stream});
+    }
+
     fbo.bind();
     render();
     auto out_img = img_downloader->download_v2();
@@ -205,7 +227,7 @@ void camera_augment_helper_v2::impl::show_fixed() {
 }
 
 void camera_augment_helper_v2::impl::show() {
-    ImGui::Checkbox("Ignore Missing", &ignore_missing);
+    ImGui::Checkbox("Ignore Missing", &ignore_missing); // TODO: enable sync with another
     ImGui::SliderFloat("Clip Near", &near, 1.0f, far, "%.f", ImGuiSliderFlags_Logarithmic);
     ImGui::SliderFloat("Clip Far", &far, near, 10000.0f, "%.f", ImGuiSliderFlags_Logarithmic);
 
@@ -219,6 +241,76 @@ void camera_augment_helper_v2::impl::show() {
             RET_ERROR;
         }
     }
+
+    if (is_interactive) {
+        ImGui::Checkbox("Cursor Guide", &enable_cursor_guide);
+    }
+}
+
+void camera_augment_helper_v2::impl::update_cursor_coordinate() {
+    assert(enable_cursor_guide);
+    cursor_pos = {};
+
+    auto s_pos = ImGui::GetMousePos();
+    auto w = last_vp_size.width, h = last_vp_size.height;
+    if (s_pos.x < 0 || s_pos.x > w) return;
+    if (s_pos.y < 0 || s_pos.y > h) return;
+    s_pos.y = h - s_pos.y; // flip y to texture coordinate
+
+    float depth;
+    glReadPixels(s_pos.x, s_pos.y, 1, 1, GL_DEPTH_COMPONENT, GL_FLOAT, &depth);
+    if (depth == 1.f) return;
+
+    auto proj = last_camera.projection(last_vp_size.aspectRatio());
+    auto ndc = glm::vec3(s_pos.x / w, s_pos.y / h, depth);
+    auto c_pos_h = glm::inverse(proj) * to_homo(2.f * ndc - 1.f);
+    auto c_pos = from_homo(c_pos_h);
+
+    auto cam_mat = glm::rotate(last_camera.transform,
+                               glm::radians(180.0f), glm::vec3(1.0f, 0.0f, 0.0f)); // camera -> viewport
+    auto w_pos_h = glm::inverse(cam_mat) * to_homo(c_pos);
+    auto w_pos = from_homo(w_pos_h);
+    cursor_pos = w_pos;
+}
+
+void camera_augment_helper_v2::impl::cursor_guide_slot(scene_render_info &info) {
+    assert(enable_cursor_guide);
+
+    if (cursor_symbol == nullptr) [[unlikely]] {
+        vtkNew<vtkSphereSource> source;
+        source->SetRadius(5); // TODO: make this adjustable
+        source->SetCenter(0, 0, 0);
+        source->Update();
+        cursor_symbol = mesh_type::from_vtk(source->GetOutput());
+    }
+
+    // record cursor position
+    auto rec_info = scene_render_info::custom_info{
+            .func = [this] { update_cursor_coordinate(); },
+    };
+    info.items.push_back({.info = rec_info});
+
+    if (cursor_pos.has_value()) {
+        auto color = glm::vec3(0.0f, 1.0f, 0.0f); // TODO: make this adjustable
+        auto amb_factor = 0.9f;
+        auto item_info = scene_render_info::mesh_info{
+                .mesh = cursor_symbol,
+                .material = {.ambient = color * amb_factor, .diffuse = color,},
+        };
+        info.items.push_back(
+                {.info = item_info, .transform = glm::translate(*cursor_pos),});
+    }
+}
+
+void camera_augment_helper_v2::impl::pre_render_slot(scene_render_info &info) {
+    if (!is_interactive) return;
+    if (enable_cursor_guide) {
+        cursor_guide_slot(info);
+    }
+}
+
+void camera_augment_helper_v2::impl::set_interactive(bool flag) {
+    is_interactive = flag;
 }
 
 camera_augment_helper_v2::camera_augment_helper_v2(const create_config &conf)
@@ -237,4 +329,8 @@ void camera_augment_helper_v2::render_image(output_config conf) {
 
 void camera_augment_helper_v2::show() {
     pimpl->show();
+}
+
+void camera_augment_helper_v2::set_interactive(bool flag) {
+    pimpl->set_interactive(flag);
 }

src/module/impl/camera_augment_helper_v2_impl.h

@@ -12,6 +12,8 @@ struct camera_augment_helper_v2::impl {
     augment_manager_v2 *manager = nullptr;
     using sophiar_conn_type = create_config::sophiar_conn_type;
     sophiar_conn_type *sophiar_conn = nullptr;
+    using io_context = boost::asio::io_context;
+    io_context *ctx = nullptr;
 
     // for all
     float fov = 60.0f; // field of view
@@ -76,8 +78,23 @@ struct camera_augment_helper_v2::impl {
     smart_frame_buffer fbo;
     std::unique_ptr<viewport_downloader> img_downloader;
 
+    // for interaction
+    using conn_type = boost::signals2::connection;
+    conn_type pre_render_conn;
+
+    bool is_interactive = false;
+
+    camera_info last_camera;
+    cv::Size last_vp_size;
+
+    bool enable_cursor_guide = false;
+    std::optional<glm::vec3> cursor_pos; // in world coordinate
+    mesh_ptr cursor_symbol;
+
     explicit impl(const create_config &conf);
 
+    ~impl();
+
     void update_freedom();
 
     void update_fixed();
@@ -96,6 +113,14 @@ struct camera_augment_helper_v2::impl {
 
     void show();
 
+    void update_cursor_coordinate();
+
+    void cursor_guide_slot(scene_render_info &info);
+
+    void pre_render_slot(scene_render_info &info);
+
+    void set_interactive(bool flag);
+
 };
 
 #endif //DEPTHGUIDE_CAMERA_AUGMENT_HELPER_V2_IMPL_H

src/module/viewport_downloader.hpp

@@ -46,6 +46,9 @@ public:
             case COLOR_ARGB: {
                 return pbo.download_viewport(GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, stream);
             }
+            case COLOR_DEPTH_32F: {
+                return pbo.download_viewport(GL_DEPTH_COMPONENT, GL_FLOAT, stream);
+            }
             default: {
                 RET_ERROR_E;
             }

src/render/impl/render_mesh.cpp

@@ -2,6 +2,12 @@
 
 #include <assimp/postprocess.h>
 
+#include <vtkFloatArray.h>
+#include <vtkPolyDataNormals.h>
+#include <vtkPointData.h>
+
+#include <glm/gtc/type_ptr.hpp>
+
 namespace render_mesh_impl {
 
     template<typename TReal>
@@ -35,30 +41,9 @@ auto mesh_type::impl::create(const mesh_type::create_config &conf)
     return ret;
 }
 
-void mesh_type::impl::upload_gl() {
+void mesh_type::impl::upload_gl(size_t num_face) {
     assert(gl_info == nullptr);
 
-    // create vertex buffer data
-    auto num_vert = mesh->mNumVertices;
-    vbo_data = data_type(sizeof(v_type) * num_vert);
-    for (auto k = 0; k < num_vert; ++k) {
-        auto info = vbo_data.at<v_type>(k);
-        info->pos = to_vec3(mesh->mVertices[k]);
-        info->normal = to_vec3(mesh->mNormals[k]);
-    }
-
-    // create element buffer data
-    auto num_face = mesh->mNumFaces;
-    ebo_data = data_type(sizeof(e_type) * num_face);
-    for (auto k = 0; k < num_face; ++k) {
-        auto &info = *ebo_data.at<e_type>(k);
-        auto face = mesh->mFaces[k];
-        assert(face.mNumIndices == face_num_vert);
-        for (auto i = 0; i < face.mNumIndices; ++i) {
-            info[i] = face.mIndices[i];
-        }
-    }
-
     // config vertex buffer
     GLuint vbo;
     glGenBuffers(1, &vbo);
@@ -87,9 +72,81 @@ void mesh_type::impl::upload_gl() {
     gl_info->num_triangle = num_face;
 }
 
+void mesh_type::impl::upload_gl_mesh() {
+    assert(mesh != nullptr);
+
+    // create vertex buffer data
+    auto num_vert = mesh->mNumVertices;
+    vbo_data = data_type(sizeof(v_type) * num_vert);
+    for (auto k = 0; k < num_vert; ++k) {
+        auto info = vbo_data.at<v_type>(k);
+        info->pos = to_vec3(mesh->mVertices[k]);
+        info->normal = to_vec3(mesh->mNormals[k]);
+    }
+
+    // create element buffer data
+    auto num_face = mesh->mNumFaces;
+    ebo_data = data_type(sizeof(e_type) * num_face);
+    for (auto k = 0; k < num_face; ++k) {
+        auto &info = *ebo_data.at<e_type>(k);
+        auto face = mesh->mFaces[k];
+        assert(face.mNumIndices == face_num_vert);
+        for (auto i = 0; i < face.mNumIndices; ++i) {
+            info[i] = face.mIndices[i];
+        }
+    }
+
+    upload_gl(num_face);
+}
+
+void mesh_type::impl::upload_gl_vtk(vtkPolyData *poly) {
+    // calculate vertex normal if needed
+    vtkNew<vtkPolyDataNormals> filter;
+    if (poly->GetPointData()->GetNormals() == nullptr) {
+        filter->SetInputData(poly);
+        filter->Update();
+        poly = filter->GetOutput();
+    }
+
+    // create vertex buffer data
+    auto points = poly->GetPoints();
+    auto num_vert = points->GetNumberOfPoints();
+    auto vert_normal = (vtkFloatArray *) poly->GetPointData()->GetNormals();
+    assert(vert_normal != nullptr);
+    assert(vert_normal->GetNumberOfTuples() == num_vert);
+    vbo_data = data_type(sizeof(v_type) * num_vert);
+    for (auto k = 0; k < num_vert; ++k) {
+        glm::dvec3 position, normal;
+        points->GetPoint(k, glm::value_ptr(position));
+        vert_normal->GetTuple(k, glm::value_ptr(normal));
+
+        auto info = vbo_data.at<v_type>(k);
+        info->pos = position;
+        info->normal = normal;
+    }
+
+    // create element buffer data
+    auto cells = poly->GetPolys();
+    auto num_face = cells->GetNumberOfCells();
+    ebo_data = data_type(sizeof(e_type) * num_face);
+    vtkNew<vtkIdList> cell;
+    for (auto k = 0; k < num_face; ++k) {
+        auto &info = *ebo_data.at<e_type>(k);
+        cells->GetCellAtId(k, cell);
+        assert(cell->GetNumberOfIds() == 3);
+        for (auto i = 0; i < 3; ++i) {
+            auto vert_id = cell->GetId(i);
+            assert(vert_id < num_vert);
+            info[i] = vert_id;
+        }
+    }
+
+    upload_gl(num_face);
+}
+
 mesh_type::gl_info_type *mesh_type::impl::get_gl_info() {
     if (gl_info == nullptr) [[unlikely]] {
-        upload_gl();
+        upload_gl_mesh();
     }
     assert(gl_info != nullptr);
     return gl_info.get();
@@ -112,6 +169,17 @@ mesh_type::pointer mesh_type::create(const create_config &conf) {
     return ret;
 }
 
+mesh_type::pointer mesh_type::from_vtk(vtkPolyData *poly) {
+    auto ret = std::make_shared<this_type>();
+    ret->pimpl = std::make_unique<impl>();
+    ret->pimpl->upload_gl_vtk(poly);
+    return ret;
+}
+
+mesh_type::pointer mesh_type::from_file(const std::string &path) {
+    return create({.path = path});
+}
+
 mesh_type::gl_info_type *mesh_type::get_gl_info() {
     return pimpl->get_gl_info();
 }

src/render/impl/render_mesh_impl.h

@@ -44,7 +44,11 @@ struct mesh_type::impl {
     static auto create(const create_config &conf)
     -> std::unique_ptr<impl>;
 
-    void upload_gl();
+    void upload_gl(size_t num_face);
+
+    void upload_gl_mesh();
+
+    void upload_gl_vtk(vtkPolyData *poly);
 
     gl_info_type *get_gl_info();
 

src/render/impl/render_scene.cpp

@@ -74,6 +74,10 @@ namespace render_scene_impl {
 
 }
 
+glm::mat4 camera_info::projection(float aspect) const {
+    return glm::perspective(glm::radians(fov), aspect, near, far);
+}
+
 void render_scene(const scene_render_info &info) {
     auto vp_size = query_viewport_size();
 
@@ -83,9 +87,7 @@ void render_scene(const scene_render_info &info) {
     };
     req.camera.transform =
             cvt_camera_transform(info.camera.transform);
-    req.camera.project = glm::perspective(
-            glm::radians(info.camera.fov), (float) vp_size.aspectRatio(),
-            info.camera.near, info.camera.far);
+    req.camera.project = info.camera.projection(vp_size.aspectRatio());
     if (info.light.follow_camera) {
         req.light.direction = glm::column(info.camera.transform, 2);
     } else {
@@ -114,6 +116,12 @@ void render_scene(const scene_render_info &info) {
                 render_pc(req);
                 break;
             }
+            case 4: {
+                static_assert(std::is_same_v<std::variant_alternative_t<4, info_type>,
+                        scene_render_info::custom_info>);
+                std::get<4>(item.info).func();
+                break;
+            }
             default: {
                 RET_ERROR;
             }

src/render/impl/render_utility.cpp

@@ -208,6 +208,7 @@ void smart_pixel_buffer::download_viewport(GLenum format, GLenum type) {
     size_t channel_num = 0;
     switch (format) {
         // @formatter:off
+        case GL_DEPTH_COMPONENT:
         case GL_RED:
         case GL_GREEN:
         case GL_BLUE: { channel_num = 1; break; }
@@ -224,11 +225,13 @@ void smart_pixel_buffer::download_viewport(GLenum format, GLenum type) {
         // @formatter:off
         case GL_UNSIGNED_BYTE: { elem_size = sizeof(uchar) * channel_num; break; }
         case GL_UNSIGNED_INT_8_8_8_8_REV: { elem_size = sizeof(uchar4); break; }
+        case GL_FLOAT: { elem_size = sizeof(float); break; }
         // @formatter:on
         default: {
             RET_ERROR;
         }
     }
+    assert(elem_size != 0);
 
     struct {
         GLint x, y, width, height;
@@ -297,7 +300,9 @@ void smart_pixel_buffer::upload_impl(const std::shared_ptr<void> &data, size_t d
 
 void smart_pixel_buffer::download(const image_ptr &img, smart_cuda_stream *stream) {
     assert(size == img->size_in_bytes());
+    img->memory(MEM_CUDA, stream); // trigger memory allocation
     download_impl(img->memory_v1(stream), stream);
+    img->cuda_modified(stream);
 }
 
 image_ptr smart_pixel_buffer::download_viewport(GLenum format, GLenum type,

src/render/render_mesh.h

@@ -3,6 +3,8 @@
 
 #include "render_utility.h"
 
+#include <vtkPolyData.h>
+
 #include <memory>
 
 class mesh_type {
@@ -19,6 +21,10 @@ public:
 
     static pointer create(const create_config &conf);
 
+    static pointer from_file(const std::string &path);
+
+    static pointer from_vtk(vtkPolyData *poly);
+
     struct gl_info_type : private boost::noncopyable {
         GLuint vao = 0; // Vertex Array Object
         GLuint vbo = 0; // Vertex Buffer Object

src/render/render_scene.h

@@ -11,6 +11,8 @@ struct camera_info {
     GLfloat fov = 60.0f; // field of view (degree)
     GLfloat near = 10.0f;
     GLfloat far = 1000.0f; // clip range (mm)
+
+    glm::mat4 projection(float aspect) const;
 };
 
 struct scene_render_info {
@@ -32,9 +34,13 @@ struct scene_render_info {
         glm::vec3 color = glm::vec3(0.5f);
     };
 
+    struct custom_info {
+        std::function<void()> func;
+    };
+
     struct item_type {
         using item_store_type = std::variant<std::monostate,
-                image_info, mesh_info, pc_info>;
+                image_info, mesh_info, pc_info, custom_info>;
         item_store_type info;
 
         GLfloat alpha = 1.0f;

src/render/render_utility.h

@@ -52,6 +52,7 @@ constexpr inline GLenum get_tex_format(int type) {
 constexpr inline int get_format_channels(int fmt) {
     switch (fmt) {
         // @formatter:off
+        case GL_DEPTH_COMPONENT:
         case GL_RED: { return 1; }
         case GL_RG:  { return 2; }
         case GL_RGB: { return 3; }
@@ -69,6 +70,7 @@ constexpr inline int get_type_depth(int type) {
         // @formatter:off
         case GL_UNSIGNED_INT_8_8_8_8_REV:
         case GL_UNSIGNED_BYTE: { return CV_8U; }
+        case GL_FLOAT: { return CV_32F; }
         // @formatter:on
         default: {
             RET_ERROR;