Image render with custom shader.

data/config_remote_ar_v2.yaml

@@ -7,4 +7,13 @@ ndi_ip: 192.168.1.202
 ndi_port: 8765
 sophiar_config: ./sophiar_config_endo_guide.json
 sophiar_start_name: tracker_all
-camera_ref_transform_var: camera_ref_in_tracker
+camera_ref_transform_var: camera_ref_in_tracker
+
+scene_info:
+  - name: BoneTransform
+    transform: femur_in_tracker_denoised
+    children:
+      - name: Bone
+        model: /home/tpx/ext/data/models/old_head/bone.stl
+      - name: Nerve
+        model: /home/tpx/ext/data/models/old_head/nerve.stl

src/image_process_v5/osg_helper.cpp

@@ -1,5 +1,4 @@
 #include "osg_helper.h"
-#include "third_party/scope_guard.hpp"
 
 #include <glad/gl.h>
 
@@ -37,6 +36,7 @@ ogl_buffer_proxy::~ogl_buffer_proxy() {
 
 void ogl_buffer_proxy::upload(const sp_image &img) {
     create(img.byte_size());
+    modify_ts = current_timestamp();
     auto status = img.mem->status();
     if (status.cuda_available
         || (status.host_available && !img.is_dense())) {
@@ -62,6 +62,21 @@ void ogl_buffer_proxy::upload(const sp_image &img) {
     }
 }
 
+void ogl_buffer_proxy::download(sp_image &img) {
+    assert(used_size == img.byte_size());
+    if (!down_res) [[unlikely]] {
+        down_res.emplace(id, cudaGraphicsMapFlagsReadOnly);
+    }
+    const auto read_helper = write_access_helper(img.cuda());
+    const auto img_ptr = img.start_ptr(read_helper.ptr());
+    size_t res_size = 0;
+    const auto res_ptr = up_res->mapped_ptr(&res_size);
+    assert(res_size >= img.byte_size());
+    CUDA_API_CHECK(cudaMemcpy2DAsync(img_ptr, img.pitch(), res_ptr, img.byte_width(),
+        img.byte_width(), img.height(), cudaMemcpyDeviceToDevice, current_cuda_stream()));
+    down_res->unmap();
+}
+
 void Texture2DSP::setImageSP(const sp_image &img) {
     assert(getTextureWidth() == img.width());
     assert(getTextureHeight() == img.height());
@@ -71,45 +86,186 @@ void Texture2DSP::setImageSP(const sp_image &img) {
     setSourceType(get_tex_type(img.cv_type()));
 }
 
+namespace {
+    auto get_tex_format_channels(const GLenum format) {
+        switch (format) {
+            // @formatter:off
+            case GL_DEPTH_COMPONENT:
+            case GL_RED:
+            case GL_GREEN:
+            case GL_BLUE: { return 1; }
+            case GL_RGB:
+            case GL_BGR: { return 3; }
+            case GL_RGBA:
+            case GL_BGRA: { return 4; }
+            default: { assert(false); return 0; };
+            // @formatter:on
+        }
+    }
+
+    auto get_date_type_cv_depth(const GLenum type) {
+        switch (type) {
+            // @formatter:off
+            case GL_UNSIGNED_BYTE:
+            case GL_UNSIGNED_INT_8_8_8_8_REV: { return CV_8U; }
+            case GL_FLOAT: { return CV_32F; }
+            default: { assert(false); return 0; };
+            // @formatter:on
+        }
+    }
+}
+
+sp_image Texture2DSP::asImageSP(GLenum format, GLenum type) {
+    const auto channels = get_tex_format_channels(format);
+    const auto cv_depth = get_date_type_cv_depth(type);
+    const auto cv_type = CV_MAKE_TYPE(cv_depth, channels);
+    const auto img_size = cv::Size(getTextureWidth(), getTextureHeight());
+    auto img = sp_image::create(cv_type, img_size);
+
+    if (!pbo) [[unlikely]] { pbo.emplace(); }
+    if (pbo->modify_ts < draw_ts) {
+        pbo->create(img.byte_size());
+        glBindTexture(getTextureTarget(), getTextureObject(0)->id());
+        glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo->id);
+        glReadPixels(0, 0, img.width(), img.height(), format, type, (void *) 0);
+        glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
+        glBindTexture(getTextureTarget(), 0);
+        pbo->modify_ts = current_timestamp();
+    }
+    pbo->download(img);
+    return img;
+}
+
 void Texture2DSP::apply(osg::State &state) const {
     setNumMipmapLevels(1);
     Texture2D::apply(state);
     // texture has already been bind
 
     if (!pbo) return;
-    glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pbo->id);
-    glTexSubImage2D(getTextureTarget(), 0, 0, 0,
-                    getTextureWidth(), getTextureHeight(),
-                    getSourceFormat(), getSourceType(), nullptr);
-    glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
+    if (download_ts < pbo->modify_ts) {
+        glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pbo->id);
+        glTexSubImage2D(getTextureTarget(), 0, 0, 0,
+                        getTextureWidth(), getTextureHeight(),
+                        getSourceFormat(), getSourceType(), nullptr);
+        glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
+        download_ts = current_timestamp();
+    }
 }
 
-constexpr auto image_vertex_num = 4;
+#include "render_osg/osg_utility.h"
+
+#include <osg/Program>
+
+using namespace osg;
+
+namespace {
+    constexpr auto image_vertex_num = 4;
+
+    struct prog_image_default_type {
+        ref_ptr<Program> prog;
+        ref_ptr<Uniform> image_tex;
+
+        prog_image_default_type() {
+            prog = new Program();
+            prog->addShader(new Shader(
+                Shader::VERTEX, load_shader("image_2d.vert")));
+            prog->addShader(new Shader(
+                Shader::FRAGMENT, load_shader("image_default.frag")));
+            image_tex = new Uniform(Uniform::INT, "image_tex");
+            image_tex->set(0);
+        }
+
+        void apply(StateSet *state) const {
+            state->setAttributeAndModes(prog);
+            state->addUniform(image_tex);
+        }
+    };
+
+    struct prog_image_remap_type {
+        ref_ptr<Program> prog;
+        ref_ptr<Uniform> image_tex;
+        ref_ptr<Uniform> remap_tex;
+
+        prog_image_remap_type() {
+            prog = new Program();
+            prog->addShader(new Shader(
+                Shader::VERTEX, load_shader("image_2d.vert")));
+            prog->addShader(new Shader(
+                Shader::FRAGMENT, load_shader("image_remap.frag")));
+            image_tex = new Uniform("image_tex", 0);
+            remap_tex = new Uniform("remap_tex", 1);
+        }
+
+        void apply(StateSet *state) const {
+            state->setAttributeAndModes(prog);
+            state->addUniform(image_tex);
+            state->addUniform(remap_tex);
+        }
+    };
+
+    std::optional<prog_image_default_type> prog_image_default;
+    std::optional<prog_image_remap_type> prog_image_remap;
+
+    void setup_program(ImageGeomSP *img) {
+        assert(img != nullptr);
+        if (img->remap_tex) {
+            if (!prog_image_remap) [[unlikely]] {
+                prog_image_remap.emplace();
+            }
+            prog_image_remap->apply(img->getOrCreateStateSet());
+        } else {
+            if (!prog_image_default) [[unlikely]] {
+                prog_image_default.emplace();
+            }
+            prog_image_default->apply(img->getOrCreateStateSet());
+        }
+    }
+
+    void upload_image_to_texture(const sp_image &img, ref_ptr<Texture2DSP> &tex) {
+        if (tex == nullptr
+            || tex->getTextureWidth() != img.width()
+            || tex->getTextureHeight() != img.height()) {
+            const auto next_tex = new Texture2DSP();
+            next_tex->setTextureSize(img.width(), img.height());
+            next_tex->setInternalFormat(GL_RGBA);
+            tex = next_tex;
+        }
+        tex->setImageSP(img);
+    }
+}
 
 ImageGeomSP::ImageGeomSP() {
-    const osg::ref_ptr tex_uv = new osg::Vec2Array();
+    const ref_ptr vertex = new Vec3Array(image_vertex_num);
+    vertex->setName("vertex");
+    setVertexArray(vertex);
+    setViewportRange({-1, -1, 2, 2});
+
+    const ref_ptr tex_uv = new Vec2Array();
+    tex_uv->setName("vertex_uv");
     tex_uv->push_back({0, 0});
     tex_uv->push_back({1, 0});
     tex_uv->push_back({1, 1});
     tex_uv->push_back({0, 1});
-    setTexCoordArray(0, tex_uv);
+    setVertexAttribArray(1, tex_uv, Array::BIND_PER_VERTEX);
 
-    const osg::ref_ptr colors = new osg::Vec4Array();
+    const ref_ptr colors = new Vec4Array();
+    colors->setName("color");
     colors->push_back({1, 1, 1, 1});
-    setColorArray(colors, osg::Array::BIND_OVERALL);
+    setColorArray(colors, Array::BIND_OVERALL);
 
-    const osg::ref_ptr vertex = new osg::Vec3Array(image_vertex_num);
-    setVertexArray(vertex);
-    setViewportRange({-1, -1, 2, 2});
+    addPrimitiveSet(new DrawArrays(
+        PrimitiveSet::QUADS, 0, 4));
+    setup_program(this);
 
-    addPrimitiveSet(new osg::DrawArrays(
-        osg::PrimitiveSet::QUADS, 0, 4));
+    ArrayList array_list;
+    getArrayList(array_list);
+    array_list.clear();
 }
 
 void ImageGeomSP::setViewportRange(simple_rect rect) {
     if (rect == last_rect) [[likely]] return;
     last_rect = rect;
-    const auto vertex = (osg::Vec3Array *) getVertexArray();
+    const auto vertex = (Vec3Array *) getVertexArray();
     assert(vertex->size() == image_vertex_num);
     auto [x, y, w, h] = rect;
     vertex->at(0) = {x + 0, y + 0, 0};
@@ -119,20 +275,28 @@ void ImageGeomSP::setViewportRange(simple_rect rect) {
     vertex->dirty();
     dirtyBound();
     dirtyGLObjects();
+
+    // disable depth test
+    getOrCreateStateSet()->setMode(GL_DEPTH_TEST,
+                                   StateAttribute::OVERRIDE | StateAttribute::OFF);
 }
 
 void ImageGeomSP::setImageSP(const sp_image &img) {
-    if (tex == nullptr
-        || tex->getTextureWidth() != img.width()
-        || tex->getTextureHeight() != img.height()) {
-        const auto next_tex = new Texture2DSP();
-        next_tex->setTextureSize(img.width(), img.height());
-        next_tex->setInternalFormat(GL_RGBA);
-        tex = next_tex;
+    upload_image_to_texture(img, tex);
+    getOrCreateStateSet()->setTextureAttributeAndModes(
+        0, tex, StateAttribute::ON);
+}
+
+void ImageGeomSP::setRemapImage(const sp_image &img) {
+    upload_image_to_texture(img, remap_tex);
+    if (remap_tex) {
+        getOrCreateStateSet()->setTextureAttributeAndModes(
+            1, remap_tex, StateAttribute::ON);
+    } else {
         getOrCreateStateSet()->setTextureAttributeAndModes(
-            0, tex, osg::StateAttribute::ON);
+            1, 0, StateAttribute::OFF);
     }
-    tex->setImageSP(img);
+    setup_program(this);
 }
 
 void ImageGeomSP::setViewportRange(const float viewport_aspect,

src/image_process_v5/osg_helper.h

@@ -12,6 +12,7 @@ struct ogl_buffer_proxy : private boost::noncopyable {
     GLuint id = {};
     size_t allocated_size = {};
     size_t used_size = {};
+    timestamp_type modify_ts = {};
 
     std::optional<cuda_ogl_buffer_proxy> up_res; // CUDA -> OpenGL
     std::optional<cuda_ogl_buffer_proxy> down_res; // OpenGL -> CUDA
@@ -20,6 +21,7 @@ struct ogl_buffer_proxy : private boost::noncopyable {
     void create(size_t req_size);
     void deallocate();
     void upload(const sp_image& img);
+    void download(sp_image& img);
     ~ogl_buffer_proxy();
     //@formatter:on
 };
@@ -30,7 +32,10 @@ struct ogl_buffer_proxy : private boost::noncopyable {
 struct Texture2DSP final : osg::Texture2D {
     //@formatter:off
     std::optional<ogl_buffer_proxy> pbo;
+    mutable timestamp_type download_ts = {}; // PBO -> tex
+    timestamp_type draw_ts = {}; // OpenGL draw, modify in the outer
     void setImageSP(const sp_image& img);
+    sp_image asImageSP(GLenum format = GL_RGB, GLenum type = GL_UNSIGNED_BYTE);
     void apply(osg::State &state) const override;
     //@formatter:on
 };
@@ -38,9 +43,11 @@ struct Texture2DSP final : osg::Texture2D {
 struct ImageGeomSP final : osg::Geometry {
     //@formatter:off
     osg::ref_ptr<Texture2DSP> tex;
+    osg::ref_ptr<Texture2DSP> remap_tex;
     simple_rect last_rect = {};
     ImageGeomSP();
     void setImageSP(const sp_image& img);
+    void setRemapImage(const sp_image& img);
     void setViewportRange(simple_rect rect);
     void setViewportRange(float viewport_aspect, bool flip_y = false);
     //@formatter:on

src/image_process_v5/video_stabilization.cpp

@@ -301,12 +301,12 @@ struct video_stabilization_ui::impl {
     obj_conn_type conn;
 
     bool passthrough = false;
-    using stab_ptr_type = std::shared_ptr<video_stabilization>;
-    stab_ptr_type video_stab;
+    std::optional<BS::thread_pool> work_tp;
+    std::optional<video_stabilization> video_stab;
 
-    static void image_callback_impl(create_config conf, const stab_ptr_type &stab) {
+    void image_callback_impl() const {
         const auto img = OBJ_QUERY(sp_image, conf.in_name);
-        if (const auto ret = stab->process(img)) {
+        if (const auto ret = video_stab->process(img)) {
             OBJ_SAVE(conf.out_name, *ret);
         }
     }
@@ -317,27 +317,31 @@ struct video_stabilization_ui::impl {
             return;
         }
 
-        if (video_stab == nullptr) [[unlikely]] {
-            video_stab = std::make_shared<video_stabilization>(conf.opts);
+        if (!video_stab) [[unlikely]] {
+            video_stab.emplace(conf.opts);
         }
-        auto task = [conf = conf, stab = video_stab] {
+        auto task = [this] {
             try {
-                image_callback_impl(conf, stab);
+                image_callback_impl();
             } catch (...) { (void) 0; }
         };
-        TP_DETACH(task);
+        if (work_tp->get_tasks_queued() <= 1) {
+            work_tp->detach_task(task);
+        } else {
+            SPDLOG_WARN("Too many frames for stabilization, frame dropped.");
+        }
     }
 
     void show_ui() {
         if (ImGui::Checkbox("Passthrough", &passthrough)) {
             if (!passthrough) {
-                video_stab = nullptr;
+                video_stab.reset();
             }
         }
         if (!passthrough) {
             ImGui::SameLine();
             if (ImGui::Button("Reset")) {
-                video_stab = nullptr;
+                video_stab.reset();
             }
             ImGui::DragInt("Frame Delay",
                            &conf.opts.st_conf.future_frames, 1, 1);
@@ -345,6 +349,7 @@ struct video_stabilization_ui::impl {
     }
 
     explicit impl(const create_config &_conf) : conf(_conf) {
+        work_tp.emplace(1);
         conn = OBJ_SIG(conf.in_name)->connect(
             [this](auto _) { image_callback(); });
     }

src/impl/apps/debug/app_debug.cpp

@@ -1,34 +1,49 @@
 #include "app_debug.h"
-#include "core/math_helper.hpp"
-#include "image_process_v5/process_python.h"
-#include "image_process_v5/image_process.h"
+#include "render/render_utility.h"
+#include "render_osg/osg_scene.h"
 
-#include <GLFW/glfw3.h>
+#include <osgViewer/Viewer>
+#include <osgUtil/UpdateVisitor>
 
-#include <opencv2/core/ocl.hpp>
+osg::ref_ptr<SceneSP> root;
+osg::ref_ptr<osgViewer::Viewer> viewer;
 
 app_debug::app_debug(const create_config &conf) {
-    SPDLOG_DEBUG("OpenCL: {}", cv::ocl::useOpenCL());
-    // if (true) {
-    //     const auto img_fixed = sp_image::from_file(
-    //         "/home/tpx/ext/project/DepthGuide/cmake-build-debug/capture/rotation/Endoscope_1.jpg");
-    //     const auto img_move = sp_image::from_file(
-    //         "/home/tpx/ext/project/DepthGuide/cmake-build-debug/capture/rotation/Endoscope_51.jpg");
-    //     image_rotate_registration(img_fixed, img_move, [](rotate_registration_result_type ret) {
-    //         SPDLOG_DEBUG("angle: {}, center: ({}, {}), error: {}",
-    //                      ret.angle, ret.center.x, ret.center.y, ret.error);
-    //     });
-    // }
-    //
-    // if (true) {
-    //     const auto img_fixed = sp_image::from_file(
-    //         "/home/tpx/ext/project/DepthGuide/cmake-build-debug/capture/rotation/Endoscope_51.jpg");
-    //     const auto img_move = sp_image::from_file(
-    //         "/home/tpx/ext/project/DepthGuide/cmake-build-debug/capture/rotation/Endoscope_81.jpg");
-    //     image_rotate_registration(img_fixed, img_move, [](rotate_registration_result_type ret) {
-    //         SPDLOG_DEBUG("angle: {}, center: ({}, {}), error: {}",
-    //                      ret.angle, ret.center.x, ret.center.y, ret.error);
-    //         MAIN_DETACH([] { glfwSetWindowShouldClose(glfwGetCurrentContext(), true); });
-    //     });
-    // }
+    auto model_conf = MeshSP::create_config();
+    model_conf.name = "Probe";
+    model_conf.model_path =
+            "/media/tpx/590d7229-f129-4612-b7ff-6d2cb3fe49b2/home/tpx/project/RemoteAR3/data/models/femur.stl";
+    osg::ref_ptr model = new MeshSP(model_conf);
+
+    root = new SceneSP();
+    root->addChild(model);
+
+    viewer = new osgViewer::Viewer();
+    viewer->setSceneData(root);
+    viewer->setUpViewerAsEmbeddedInWindow(0, 0, 800, 600);
+
+    auto bs = root->getBound();
+    auto center = bs.center();
+    auto radius = bs.radius();
+
+    auto eye = center + osg::Vec3(0.0f, 0.0f, radius * 2.0f); // position camera along z-axis
+    auto up = osg::Vec3(0.0f, 1.0f, 0.0f); // up vector
+    auto lookAt = center;
+
+    osg::Matrixd viewMatrix;
+    viewMatrix.makeLookAt(eye, lookAt, up);
+    auto camera = viewer->getCamera();
+    camera->setViewMatrix(viewMatrix);
+    camera->setClearColor({1, 1, 1, 1});
+    root->setCamera(camera);
+}
+
+void app_debug::render_background() {
+    const auto vp = query_viewport_size();
+    auto camera = viewer->getCamera();
+    camera->setViewport(0, 0, vp.width, vp.height);
+    camera->setProjectionMatrixAsPerspective(60.0f, 1. * vp.width / vp.height, 0.1f, 10000.0f);
+    osg::ref_ptr visitor = new osgUtil::UpdateVisitor();
+    visitor->apply(*root);
+    viewer->frame();
 }

src/impl/apps/debug/app_debug.h

@@ -13,7 +13,7 @@ public:
 
     void show_ui() override {}
 
-    void render_background() override {}
+    void render_background() override;
 };
 
 

src/impl/main_impl.cpp

@@ -96,9 +96,9 @@ void init_window() {
     assert(version > 0);
     SPDLOG_INFO("Loaded OpenGL {}.{}", GLAD_VERSION_MAJOR(version), GLAD_VERSION_MINOR(version));
 
-    // enable color blending
-    glEnable(GL_BLEND);
-    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
+    // // enable color blending
+    // glEnable(GL_BLEND);
+    // glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
 
 #ifndef NDEBUG
     // log opengl error
@@ -148,7 +148,7 @@ void init_all() {
     main_ob = new object_manager_v2({.ctx = main_ctx});
     g_memory_manager = new memory_manager();
 
-    constexpr auto background_thread_count = 1; // TODO: load this in config file
+    constexpr auto background_thread_count = 10; // TODO: load this in config file
     g_thread_pool = new BS::thread_pool(background_thread_count);
 
     auto app_conf = app_selector::create_config();

src/module_v5/transform_provider.h

@@ -19,12 +19,14 @@ struct transform_provider {
     virtual ~transform_provider() = default;
 };
 
+using transform_key_type = transform_provider::key_type;
+
 void create_sophiar_transform_provider();
-auto query_transform(const transform_provider::key_type &name) -> transform_provider::result_type;
+auto query_transform(const transform_key_type &name) -> transform_provider::result_type;
 
 using custom_transform_provider_type = std::function<
-    transform_provider::result_type(const transform_provider::key_type &name)>;
-void register_custom_transform_provider(const transform_provider::key_type &name,
+    transform_provider::result_type(const transform_key_type &name)>;
+void register_custom_transform_provider(const transform_key_type &name,
                                         custom_transform_provider_type &&func);
 
 #endif //TRANSFORM_PROVIDER_H

src/render_osg/CMakeLists.txt

@@ -1,3 +1,11 @@
-find_package(OpenSceneGraph REQUIRED osgViewer)
+set(OpenSceneGraph_DEBUG TRUE)
+set(OSG_DIR /home/tpx/usr_dbg)
+find_package(OpenSceneGraph REQUIRED osgViewer osgUtil)
 target_include_directories(${PROJECT_NAME} PRIVATE ${OPENSCENEGRAPH_INCLUDE_DIRS})
 target_link_libraries(${PROJECT_NAME} ${OPENSCENEGRAPH_LIBRARIES})
+
+target_sources(${PROJECT_NAME} PRIVATE
+        osg_scene.cpp
+        osg_utility.cpp)
+
+file(COPY shaders DESTINATION ${CMAKE_BINARY_DIR})

src/render_osg/osg_scene.cpp

@@ -0,0 +1,410 @@
+#include "osg_scene.h"
+#include "osg_utility.h"
+
+#include <spdlog/spdlog.h>
+
+#include <utility>
+#include <osgViewer/Scene>
+
+using namespace osg;
+
+namespace {
+    Matrix to_osg_matrix(glm::mat4 mat) {
+        auto ret = Matrix();
+        for (auto i = 0; i < 4; i++)
+            for (auto j = 0; j < 4; j++) {
+                ret(i, j) = mat[i][j]; // TODO: The matrix in OSG is weird, figure out why
+            }
+        return ret;
+    }
+}
+
+struct TransformSP::impl {
+    TransformSP *q_this;
+    create_config conf;
+    bool is_tracked_once = false;
+    bool ignore_missing = false;
+
+    struct update_callback final : NodeCallback {
+        impl *pimpl;
+
+        explicit update_callback(impl *_pimpl)
+            : pimpl(_pimpl) { (void) 0; }
+
+        void operator()(Node *node, NodeVisitor *nv) override {
+            pimpl->update_transform();
+            traverse(node, nv);
+        }
+    };
+
+    impl(TransformSP *_q_this, create_config _conf)
+        : q_this(_q_this), conf(std::move(_conf)) {
+        q_this->setNodeMask(0); // disable
+        q_this->setUpdateCallback(new update_callback(this));
+    }
+
+    void update_transform() const {
+        const auto ret = query_transform(conf.key_name).matrix;
+        if (!ret) {
+            if (!ignore_missing || !is_tracked_once) {
+                q_this->setNodeMask(0); // disable
+            }
+        } else {
+            q_this->setNodeMask(-1); // enable
+            q_this->setMatrix(to_osg_matrix(*ret));
+        }
+    }
+
+    void show_ui() {
+        // TODO
+    }
+};
+
+TransformSP::TransformSP(const create_config &conf)
+    : pimpl(std::make_unique<impl>(this, conf)) { (void) 0; }
+
+TransformSP::~TransformSP() = default;
+
+void TransformSP::showUI() {
+    pimpl->show_ui();
+}
+
+#include <assimp/Importer.hpp>
+#include <assimp/scene.h>
+#include <assimp/postprocess.h>
+
+#include <osg/Geometry>
+#include <osg/Program>
+
+#include <vtkPolyDataNormals.h>
+#include <vtkPointData.h>
+
+#include <glm/gtc/type_ptr.hpp>
+
+namespace {
+    struct prog_mesh_type {
+        ref_ptr<Program> prog;
+
+        prog_mesh_type() {
+            prog = new Program();
+            prog->addShader(new Shader(
+                Shader::VERTEX, load_shader("mesh_default.vert")));
+            prog->addShader(new Shader(
+                Shader::FRAGMENT, load_shader("mesh_default.frag")));
+        }
+
+        void apply(StateSet *state) const {
+            state->setAttributeAndModes(prog);
+
+            struct model_mat_callback : UniformCallback {
+                void operator ()(Uniform *uni, NodeVisitor *nv) override {
+                    uni->set(computeLocalToWorld(nv->getNodePath()));
+                }
+            };
+            const ref_ptr model_mat = new Uniform(
+                Uniform::FLOAT_MAT4, "model_mat");
+            model_mat->setUpdateCallback(new model_mat_callback());
+            state->addUniform(model_mat);
+
+            const ref_ptr base_weight = new Uniform(
+                Uniform::FLOAT, "base_weight");
+            state->addUniform(base_weight);
+
+            // state->setMode(GL_BLEND, StateAttribute::ON); // TODO: modified
+            // state->setRenderingHint(StateSet::TRANSPARENT_BIN);
+        }
+    };
+
+    std::optional<prog_mesh_type> prog_mesh;
+}
+
+namespace {
+    auto default_mesh_color = Vec4f(1, 1, 1, 1);
+}
+
+struct MeshSP::impl {
+    MeshSP *q_this;
+    Assimp::Importer importer;
+
+    create_config conf;
+    float base_weight = 0.1f;
+
+    static ref_ptr<Geometry> create_geometry(const aiMesh *mesh) {
+        assert(mesh->mPrimitiveTypes == aiPrimitiveType_TRIANGLE);
+        assert(mesh->mNormals != nullptr);
+        const ref_ptr ret = new Geometry();
+
+        const ref_ptr vertices = new Vec3Array();
+        vertices->setName("vertex");
+        for (auto i = 0; i < mesh->mNumVertices; i++) {
+            const auto vertex = mesh->mVertices[i];
+            vertices->push_back({vertex.x, vertex.y, vertex.z});
+        }
+        ret->setVertexArray(vertices);
+
+        const ref_ptr normals = new Vec3Array();
+        normals->setName("normal");
+        for (auto i = 0; i < mesh->mNumVertices; i++) {
+            const auto normal = mesh->mNormals[i];
+            normals->push_back({normal.x, normal.y, normal.z});
+        }
+        ret->setNormalArray(normals, Array::BIND_PER_VERTEX);
+
+        const ref_ptr indices = new DrawElementsUInt(PrimitiveSet::TRIANGLES, 0);
+        for (auto i = 0; i < mesh->mNumFaces; i++) {
+            const auto face = mesh->mFaces[i];
+            indices->push_back(face.mIndices[0]);
+            indices->push_back(face.mIndices[1]);
+            indices->push_back(face.mIndices[2]);
+        }
+        ret->addPrimitiveSet(indices);
+
+        const ref_ptr colors = new Vec4Array();
+        colors->setName("color");
+        if (const auto mesh_color = mesh->mColors[0]; mesh_color == nullptr) {
+            colors->push_back(default_mesh_color);
+            ret->setColorArray(colors, Array::BIND_OVERALL);
+        } else {
+            for (auto i = 0; i < mesh->mNumFaces; i++) {
+                const auto color = mesh_color[i];
+                colors->push_back({color.r, color.g, color.b, color.a});
+            }
+            ret->setColorArray(colors, Array::BIND_PER_VERTEX);
+        }
+
+        return ret;
+    }
+
+    static ref_ptr<Geometry> create_geometry(vtkPolyData *mesh) {
+        assert(mesh != nullptr);
+        const ref_ptr ret = new Geometry();
+
+        const ref_ptr vertices = new Vec3Array();
+        vertices->setName("vertex");
+        const auto mesh_vertex = mesh->GetPoints();
+        const auto num_vertex = mesh_vertex->GetNumberOfPoints();
+        for (auto i = 0; i < num_vertex; i++) {
+            double vertex[3];
+            mesh_vertex->GetPoint(i, vertex);
+            vertices->push_back(Vec3f(vertex[0], vertex[1], vertex[2]));
+        }
+        ret->setVertexArray(vertices);
+
+        const ref_ptr normals = new Vec3Array();
+        normals->setName("normal");
+        if (mesh->GetPointData()->GetNormals() == nullptr) {
+            vtkNew<vtkPolyDataNormals> filter;
+            filter->SetInputData(mesh);
+            filter->Update();
+            mesh = filter->GetOutput();
+        }
+        const auto mesh_normal = mesh->GetPointData()->GetNormals();
+        for (auto i = 0; i < num_vertex; i++) {
+            double normal[3];
+            mesh_normal->GetTuple(i, normal);
+            normals->push_back(Vec3f(normal[0], normal[1], normal[2]));
+        }
+        ret->setNormalArray(normals, Array::BIND_PER_VERTEX);
+
+        const ref_ptr indices = new DrawElementsUInt(PrimitiveSet::TRIANGLES, 0);
+        const auto faces = mesh->GetPolys();
+        const auto num_face = faces->GetNumberOfCells();
+        vtkNew<vtkIdList> face_index;
+        for (auto i = 0; i < num_face; i++) {
+            faces->GetCellAtId(i, face_index);
+            assert(face_index->GetNumberOfIds() == 3);
+            for (auto j = 0; j < 3; ++j) {
+                indices->push_back(face_index->GetId(j));
+            }
+        }
+        ret->addPrimitiveSet(indices);
+
+        const ref_ptr colors = new Vec4Array();
+        colors->setName("color");
+        colors->push_back(default_mesh_color);
+        ret->setColorArray(colors, Array::BIND_OVERALL);
+
+        return ret;
+    }
+
+    void load_from_file() {
+        const auto scene = importer.ReadFile(
+            conf.model_path, aiProcessPreset_TargetRealtime_MaxQuality);
+        assert(scene != nullptr);
+        for (auto i = 0; i < scene->mNumMeshes; i++) {
+            q_this->addDrawable(create_geometry(scene->mMeshes[i]));
+        }
+        SPDLOG_INFO("Loaded model {} from file {} with {} components.",
+                    conf.name, conf.model_path, scene->mNumMeshes);
+    }
+
+    void load_from_vtk() const {
+        q_this->addDrawable(create_geometry(conf.vtk_model));
+    }
+
+    struct update_callback final : NodeCallback {
+        impl *pimpl;
+
+        explicit update_callback(impl *_pimpl)
+            : pimpl(_pimpl) { (void) 0; }
+
+        void operator()(Node *node, NodeVisitor *nv) override {
+            const auto states = pimpl->q_this->getOrCreateStateSet();
+            assert(states != nullptr);
+            const auto base_weight = states->getUniform("base_weight");
+            base_weight->set(pimpl->base_weight);
+            pimpl->q_this->onUpdate();
+            traverse(node, nv);
+        }
+    };
+
+    void re_create(const create_config &_conf) {
+        q_this->removeDrawables(0, q_this->getNumDrawables());
+
+        conf = _conf;
+        if (!conf.model_path.empty()) {
+            load_from_file();
+        } else if (conf.vtk_model != nullptr) {
+            load_from_vtk();
+        }
+    }
+
+    impl(MeshSP *_q_this, const create_config &_conf)
+        : q_this(_q_this) {
+        if (!prog_mesh) [[unlikely]] {
+            prog_mesh.emplace();
+        }
+        prog_mesh->apply(q_this->getOrCreateStateSet());
+        q_this->setUpdateCallback(new update_callback(this));
+
+        re_create(_conf);
+    }
+
+    void show_ui() {
+        // TODO
+    }
+};
+
+MeshSP::MeshSP()
+    : pimpl(std::make_unique<impl>(
+        this, create_config())) { (void) 0; }
+
+MeshSP::MeshSP(const create_config &conf)
+    : pimpl(std::make_unique<impl>(this, conf)) {
+}
+
+MeshSP::~MeshSP() = default;
+
+void MeshSP::showUI() {
+    pimpl->show_ui();
+}
+
+struct SceneSP::impl {
+    SceneSP *q_this;
+
+    ref_ptr<Camera> camera;
+    ref_ptr<Uniform> camera_mat;
+    ref_ptr<Uniform> light_direction;
+
+    void setup_uniforms() {
+        //@formatter:off
+        struct camera_mat_callback : UniformCallback {
+            impl *pimpl;
+            explicit camera_mat_callback(impl *_p_impl)
+                : pimpl(_p_impl) { (void) 0; }
+            void operator ()(Uniform *uni, NodeVisitor *) override {
+                const auto cam = pimpl->camera;
+                const auto proj_mat = cam->getProjectionMatrix();
+                const auto view_mat = cam->getViewMatrix();
+                uni->set(view_mat * proj_mat);
+            }
+        };
+        //@formatter:on
+
+        camera = new Camera();
+        camera_mat = new Uniform(
+            Uniform::FLOAT_MAT4, "camera_mat");
+        camera_mat->setUpdateCallback(new camera_mat_callback(this));
+
+        //@formatter:off
+        struct light_direction_callback : UniformCallback {
+            impl *pimpl;
+            explicit light_direction_callback(impl *_p_impl)
+                : pimpl(_p_impl) { (void) 0; }
+            void operator ()(Uniform *uni, NodeVisitor *) override {
+                const auto cam = pimpl->camera;
+                const auto view_mat = cam->getViewMatrix();
+                const auto z_dir = Vec3f(
+                    view_mat(2, 0),
+                    view_mat(2, 1),
+                    view_mat(2, 2));
+                uni->set(-z_dir);
+            }
+        };
+        //@formatter:on
+
+        light_direction = new Uniform(
+            Uniform::FLOAT_VEC3, "light_direction");
+        light_direction->setUpdateCallback(new light_direction_callback(this));
+
+        const auto state = q_this->getOrCreateStateSet();
+        state->addUniform(camera_mat);
+        state->addUniform(light_direction);
+    }
+
+    explicit impl(SceneSP *_q_this)
+        : q_this(_q_this) {
+        setup_uniforms();
+    }
+};
+
+SceneSP::SceneSP()
+    : pimpl(std::make_unique<impl>(this)) {
+}
+
+SceneSP::~SceneSP() = default;
+
+void SceneSP::setCamera(Camera *camera) const {
+    pimpl->camera = camera;
+}
+
+Camera *SceneSP::getCamera() const {
+    return pimpl->camera;
+}
+
+void SceneSP::showUI() {
+    // TODO
+}
+
+namespace {
+    ref_ptr<Node> create_node(const YAML::Node &conf) {
+        const auto name = LOAD_STR("name");
+        if (conf["model"]) {
+            auto model_conf = MeshSP::create_config();
+            model_conf.name = name;
+            model_conf.model_path = LOAD_STR("model");
+            return new MeshSP(model_conf);
+        } else if (conf["transform"]) {
+            auto transform_conf = TransformSP::create_config();
+            transform_conf.name = name;
+            transform_conf.key_name = LOAD_STR("transform");
+            ref_ptr ret = new TransformSP(transform_conf);
+            if (const auto children = conf["children"]; children) {
+                for (auto child: children) {
+                    ret->addChild(create_node(child));
+                }
+            }
+            return ret;
+        }
+        assert(false);
+        return {};
+    }
+}
+
+ref_ptr<SceneSP> SceneSP::createFromYAML(const YAML::Node &conf) {
+    ref_ptr ret = new SceneSP();
+    for (auto child: conf) {
+        ret->addChild(create_node(child));
+    }
+    return ret;
+}

src/render_osg/osg_scene.h

@@ -0,0 +1,90 @@
+#ifndef OSG_SCENE_H
+#define OSG_SCENE_H
+
+#include "module_v5/transform_provider.h"
+
+#include <osg/Geode>
+#include <osg/MatrixTransform>
+
+#include <memory>
+
+//@formatter:off
+struct osgHasUI {
+    virtual ~osgHasUI() = default;
+    virtual void showUI() = 0;
+};
+//@formatter:on
+
+class TransformSP final : public osg::MatrixTransform,
+                          public osgHasUI {
+public:
+    struct create_config {
+        std::string name;
+        transform_key_type key_name;
+    };
+
+    explicit TransformSP(const create_config &conf);
+
+    ~TransformSP() override;
+
+    void showUI() override;
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#include <vtkPolyData.h>
+
+class MeshSP : public osg::Geode,
+               public osgHasUI {
+public:
+    struct create_config {
+        std::string name;
+        std::string model_path;
+        vtkSmartPointer<vtkPolyData> vtk_model = nullptr;
+        // glm::vec3 color;
+        // float transparency = 0; // opaque
+    };
+
+    MeshSP();
+
+    explicit MeshSP(const create_config &conf);
+
+    ~MeshSP() override;
+
+    void showUI() override;
+
+    void reCreate(create_config conf);
+
+    virtual void onUpdate() { (void) 0; }
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#include "core/yaml_utility.hpp"
+
+class SceneSP final : public osg::Group,
+                      public osgHasUI {
+public:
+    SceneSP();
+
+    ~SceneSP() override;
+
+    static osg::ref_ptr<SceneSP> createFromYAML(const YAML::Node& conf);
+
+    void setCamera(osg::Camera *camera) const;
+
+    osg::Camera *getCamera() const;
+
+    void showUI() override;
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+
+#endif //OSG_SCENE_H

src/render_osg/osg_utility.cpp

@@ -0,0 +1,17 @@
+#include "osg_utility.h"
+
+#include <boost/iostreams/device/mapped_file.hpp>
+
+#include <filesystem>
+
+namespace fs = std::filesystem;
+
+std::string load_shader(const std::string &shader_name) {
+    using boost::iostreams::mapped_file;
+    // TODO: load from current binary directory
+    const fs::path shader_prefix = "/home/tpx/ext/project/DepthGuide/src/render_osg/shaders";
+    const auto shader_path = shader_prefix / shader_name;
+    const auto shader = mapped_file(shader_path);
+    assert(shader.is_open());
+    return {shader.const_data(), shader.size()};
+}

src/render_osg/osg_utility.h

@@ -0,0 +1,8 @@
+#ifndef OSG_UTILITY_H
+#define OSG_UTILITY_H
+
+#include <string>
+
+std::string load_shader(const std::string &shader_name);
+
+#endif //OSG_UTILITY_H

src/render_osg/shaders/image_2d.vert

@@ -0,0 +1,11 @@
+#version 460
+
+layout (location = 0) in vec3 vertex;
+layout (location = 1) in vec2 vertex_uv;
+
+out vec2 frag_uv;
+
+void main() {
+    gl_Position = vec4(vertex, 1.0);
+    frag_uv = vertex_uv;
+}

src/render_osg/shaders/image_default.frag

@@ -0,0 +1,11 @@
+#version 460
+
+uniform sampler2D image_tex;
+
+in vec2 frag_uv;
+
+layout (location = 0) out vec4 frag_color;
+
+void main() {
+    frag_color = texture(image_tex, frag_uv);
+}

src/render_osg/shaders/image_remap.frag

@@ -0,0 +1,13 @@
+#version 460
+
+uniform sampler2D remap_tex; // remap texture
+uniform sampler2D image_tex; // color texture
+
+in vec2 frag_uv;
+
+layout (location = 0) out vec4 frag_color;
+
+void main() {
+    vec2 image_uv = texture(remap_tex, frag_uv).xy; // remap coordinate
+    frag_color = texture(image_tex, image_uv);
+}

src/render_osg/shaders/mesh_default.frag

@@ -0,0 +1,15 @@
+#version 460
+
+uniform vec3 light_direction;
+uniform float base_weight;
+
+in vec3 vert_normal;
+in vec4 vert_color;
+
+layout (location = 0) out vec4 frag_color;
+
+void main() {
+    float diffuse_weight = max(-dot(vert_normal, light_direction), 0.0) + base_weight;
+    frag_color.rgb = diffuse_weight * vert_color.rgb;
+    frag_color.a = vert_color.a;
+}

src/render_osg/shaders/mesh_default.vert

@@ -0,0 +1,17 @@
+#version 460
+
+uniform mat4 model_mat;
+uniform mat4 camera_mat;
+
+layout (location = 0) in vec3 vertex;
+layout (location = 1) in vec3 normal;
+layout (location = 2) in vec4 color;
+
+out vec3 vert_normal;
+out vec4 vert_color;
+
+void main() {
+    vert_normal = mat3(model_mat) * normal;
+    vert_color = color;
+    gl_Position = camera_mat * model_mat * vec4(vertex, 1.0);
+}