Implemented scene encoder and decoder.

CMakeLists.txt

@@ -13,9 +13,20 @@ add_executable(${PROJECT_NAME} src/main.cpp
         src/impl/apps/app_selector/app_selector.cpp
         src/impl/apps/debug/app_debug.cpp
         src/impl/apps/depth_guide/depth_guide.cpp
+        src/impl/apps/depth_guide_v2/depth_guide_v2.cpp
         src/impl/apps/endo_guide/endo_guide.cpp
         src/impl/apps/remote_ar/remote_ar.cpp
+        src/impl/apps/scene_player/scene_player.cpp
         src/impl/apps/tiny_player/tiny_player.cpp
+        src/codec/image_codec.cpp
+        src/codec/image_decoder.cpp
+        src/codec/image_encoder.cpp
+        src/codec/mesh_decoder.cpp
+        src/codec/mesh_encoder.cpp
+        src/codec/pc_decoder.cpp
+        src/codec/pc_encoder.cpp
+        src/codec/scene_decoder.cpp
+        src/codec/scene_encoder.cpp
         src/core/impl/event_timer.cpp
         src/core/impl/image_utility_v2.cpp
         src/core/impl/memory_pool.cpp
@@ -71,6 +82,8 @@ target_link_libraries(${PROJECT_NAME} ImageProcessCudaV3)
 target_sources(${PROJECT_NAME} PRIVATE
         src/image_process_v3/image_process.cpp)
 
+find_package(PkgConfig REQUIRED)
+
 # CUDA config
 find_package(CUDAToolkit REQUIRED)
 target_link_libraries(${PROJECT_NAME} CUDA::cudart CUDA::cuda_driver)
@@ -221,6 +234,11 @@ set(CRYPTOPP_INCLUDE_DIR ${CRYPTOPP_DIR}/include)
 target_include_directories(${PROJECT_NAME} PRIVATE ${CRYPTOPP_INCLUDE_DIR})
 target_link_libraries(${PROJECT_NAME} ${CRYPTOPP_LIB})
 
+# xxHash config
+pkg_search_module(XXHASH REQUIRED libxxhash)
+target_include_directories(${PROJECT_NAME} PRIVATE ${XXHASH_INCLUDE_DIRS})
+target_link_libraries(${PROJECT_NAME} ${XXHASH_LIBRARIES})
+
 # NvEnc config
 if (WIN32)
     set(NVCODEC_DIR C:/BuildEssentials/CUDA/Video_Codec_SDK_12.0.16)

data/config_depth_guide_v2.yaml

@@ -0,0 +1 @@
+app_name: depth_guide_v2

data/config_scene_player.yaml

@@ -0,0 +1 @@
+app_name: scene_player

src/codec/codec_base.hpp

@@ -3,14 +3,16 @@
 
 #include "network/binary_utility.hpp"
 
-enum encoder_type {
+enum encoder_type : uint8_t {
     ENCODER_NVENC,
-    ENCODER_JPEG
+    ENCODER_JPEG,
+    ENCODER_SPECIAL,
 };
 
-enum decoder_type {
+enum decoder_type : uint8_t {
     DECODER_NVDEC,
-    DECODER_JPEG
+    DECODER_JPEG,
+    DECODER_SPECIAL,
 };
 
 struct frame_info {

src/codec/decoder_nvdec.cpp

@@ -106,24 +106,29 @@ struct decoder_nvdec::impl {
         CUDA_API_CHECK(cuvidGetDecodeStatus(decoder, pic->CurrPicIdx, &status));
         CALL_CHECK(status.decodeStatus == cuvidDecodeStatus_Success);
 
-        auto img_size = img_size_to_nv12(frame_size);
-        auto img_info = create_image_info<uchar1>(img_size, MEM_CUDA);
+        auto img = create_image(frame_size, CV_8UC1, PIX_NV12);
+        auto img_mem = img->memory(MEM_CUDA, conf.stream);
 
         // copy frame
         auto luma_in = (void *) ptr_in;
-        auto luma_out = img_info.start_ptr();
-        CUDA_API_CHECK(cudaMemcpy2DAsync(luma_out, img_info.pitch, luma_in, pitch_in,
+        auto luma_out = img_mem.start_ptr(0);
+        CUDA_API_CHECK(cudaMemcpy2DAsync(luma_out, img_mem.pitch, luma_in, pitch_in,
                                          frame_size.width, frame_size.height, cudaMemcpyDeviceToDevice));
         auto chroma_in = (char *) ptr_in + pitch_in * ((frame_size.height + 1) & ~1);
-        auto chroma_out = img_info.start_ptr(frame_size.height);
-        CUDA_API_CHECK(cudaMemcpy2D(chroma_out, img_info.pitch, chroma_in, pitch_in,
+        auto chroma_out = img_mem.start_ptr(1);
+        CUDA_API_CHECK(cudaMemcpy2D(chroma_out, img_mem.pitch, chroma_in, pitch_in,
                                     frame_size.width, frame_size.height >> 1, cudaMemcpyDeviceToDevice));
 
         // unmap frame
         CUDA_API_CHECK(cuvidUnmapVideoFrame(decoder, ptr_in));
+        img_mem.modified(conf.stream);
 
         // commit frame
-        OBJ_SAVE(conf.img_name, create_image(img_info));
+        if (conf.img_name != invalid_obj_name) {
+            OBJ_SAVE(conf.img_name, img->v1<uchar1>());
+        } else {
+            conf.cb_func(img);
+        }
 
         return 1; // success
     }

src/codec/decoder_nvdec.h

@@ -4,6 +4,7 @@
 #include "codec_base.hpp"
 #include "core/cuda_helper.hpp"
 #include "core/object_manager.h"
+#include "core/image_utility_v2.h"
 
 #include <memory>
 
@@ -13,6 +14,9 @@ public:
     struct create_config {
         obj_name_type img_name = invalid_obj_name; // image_u8c1 (nv12)
         smart_cuda_stream *stream = nullptr;
+
+        using cb_func_type = std::function<void(image_ptr)>;
+        cb_func_type cb_func;
     };
 
     explicit decoder_nvdec(create_config conf);

src/codec/encoder_nvenc.cpp

@@ -264,3 +264,7 @@ frame_info encoder_nvenc::encode(const image_u8c4 &img, bool force_idr) {
 void encoder_nvenc::change_config(modifiable_config conf) {
     pimpl->change_config(conf);
 }
+
+cv::Size encoder_nvenc::frame_size() const {
+    return pimpl->frame_size;
+}

src/codec/encoder_nvenc.h

@@ -39,6 +39,8 @@ public:
 
     frame_info encode(const image_u8c4 &img, bool force_idr = false);
 
+    cv::Size frame_size() const;
+
 private:
     struct impl;
     std::unique_ptr<impl> pimpl;

src/codec/image_codec.cpp

@@ -0,0 +1,37 @@
+#include "image_codec.h"
+
+#include <unordered_map>
+
+namespace image_codec {
+
+    struct codec_item_type {
+        image_encoder_func enc_func;
+        image_decoder_func dec_func;
+    };
+
+    using codec_map_type =
+            std::unordered_map<size_t, codec_item_type>;
+    codec_map_type codec_map;
+
+    image_encoder_func get_special_encoder(size_t id) {
+        assert(codec_map.contains(id));
+        return codec_map.at(id).enc_func;
+    }
+
+    image_decoder_func get_special_decoder(size_t id) {
+        assert(codec_map.contains(id));
+        return codec_map.at(id).dec_func;
+    }
+
+}
+
+using namespace image_codec;
+
+void register_image_special_codec(size_t id,
+                                  const image_encoder_func &enc,
+                                  const image_decoder_func &dec) {
+    assert(!codec_map.contains(id));
+    codec_map.emplace(std::piecewise_construct,
+                      std::forward_as_tuple(id),
+                      std::forward_as_tuple(enc, dec));
+}

src/codec/image_codec.h

@@ -0,0 +1,24 @@
+#ifndef DEPTHGUIDE_IMAGE_CODEC_H
+#define DEPTHGUIDE_IMAGE_CODEC_H
+
+#include "core/image_utility_v2.h"
+#include "network/binary_utility.hpp"
+
+static constexpr auto META_IMAGE_SPECIAL_CODEC = meta_hash("image_special_codec"); // size_t, use hash
+
+using image_encoder_func = std::function<data_type(image_ptr, bool)>; // bool: force_idr
+using image_decoder_func = std::function<image_ptr(data_type)>;
+
+void register_image_special_codec(size_t id,
+                                  const image_encoder_func &enc,
+                                  const image_decoder_func &dec);
+
+namespace image_codec {
+
+    image_encoder_func get_special_encoder(size_t id);
+
+    image_decoder_func get_special_decoder(size_t id);
+
+}
+
+#endif //DEPTHGUIDE_IMAGE_CODEC_H

src/codec/image_decoder.cpp

@@ -0,0 +1,106 @@
+#include "image_decoder.h"
+#include "image_codec.h"
+#include "codec/codec_base.hpp"
+#include "codec/decoder_nvdec.h"
+#include "image_process/cuda_impl/pixel_convert.cuh"
+
+using namespace nlohmann;
+using namespace image_codec;
+
+struct image_decoder::impl {
+
+    create_config conf;
+
+    struct decoder_store_type {
+        std::unique_ptr<decoder_nvdec> nvdec;
+        cb_func_type cb_func;
+    };
+
+    using decoder_map_type =
+            std::unordered_map<size_t, decoder_store_type>;
+    decoder_map_type dec_map;
+
+    void on_nvdec_image(image_ptr img, size_t series, const json &head) {
+        // nv12 -> rgb
+        auto img_rgb = create_image(img->size(), CV_8UC3);
+        call_nv12_to_rgb(img->cuda<uchar1>(conf.stream),
+                         img_rgb->cuda<uchar3>(conf.stream),
+                         conf.stream->cuda);
+        img_rgb->cuda_modified(conf.stream);
+        img = img_rgb;
+
+        // decoded image may become larger
+        auto width = head["width"].get<int>();
+        auto height = head["height"].get<int>();
+        img = img->sub_image(0, 0, width, height);
+
+        img->set_meta_any(META_SERIES_NAME, series);
+        assert(dec_map.contains(series));
+        dec_map.at(series).cb_func(img);
+    }
+
+    void decode_nvdec(const data_type &data, size_t series, const json &head) {
+        assert(dec_map.contains(series));
+        auto &decoder = dec_map.at(series).nvdec;
+        if (decoder == nullptr) [[unlikely]] {
+            auto dec_conf = decoder_nvdec::create_config{
+                    .img_name = invalid_obj_name,
+                    .stream = conf.stream,
+                    // convert head to lvalue
+                    .cb_func = [=, this, head = json(head)](const image_ptr &img) {
+                        on_nvdec_image(img, series, head);
+                    },
+            };
+            decoder = std::make_unique<decoder_nvdec>(dec_conf);
+        }
+        assert(decoder != nullptr);
+        auto frame = frame_info{.data = data};
+        decoder->decode(frame);
+    }
+
+    void decode(const data_type &data, const cb_func_type &cb) {
+        auto reader = network_reader(data);
+        auto head = reader.read_json_with_length();
+
+        auto series = head["series"].get<size_t>();
+        if (!dec_map.contains(series)) [[unlikely]] {
+            dec_map.emplace(std::piecewise_construct,
+                            std::forward_as_tuple(series),
+                            std::forward_as_tuple());
+        }
+        assert(dec_map.contains(series));
+        dec_map.at(series).cb_func = cb;
+
+        auto type = head["type"].get<encoder_type>();
+        auto ret = reader.read_data_with_length();
+        assert(reader.empty());
+        switch (type) {
+            case ENCODER_NVENC: {
+                decode_nvdec(ret, series, head);
+                break;
+            }
+            case ENCODER_SPECIAL: {
+                auto sp_id = head["special"].get<size_t>();
+                auto img = get_special_decoder(sp_id)(ret);
+                img->set_meta_any(META_SERIES_NAME, series);
+                cb(img);
+                break;
+            }
+            default: {
+                RET_ERROR;
+            }
+        }
+    }
+
+};
+
+image_decoder::image_decoder(const create_config &conf)
+        : pimpl(std::make_unique<impl>()) {
+    pimpl->conf = conf;
+}
+
+image_decoder::~image_decoder() = default;
+
+void image_decoder::decode(const data_type &data, const cb_func_type &cb) {
+    pimpl->decode(data, cb);
+}

src/codec/image_decoder.h

@@ -0,0 +1,31 @@
+#ifndef DEPTHGUIDE_IMAGE_DECODER_H
+#define DEPTHGUIDE_IMAGE_DECODER_H
+
+#include "core/image_utility_v2.h"
+#include "network/binary_utility.hpp"
+
+#include <memory>
+
+class image_decoder {
+public:
+
+    struct create_config {
+        smart_cuda_stream *stream = nullptr;
+    };
+
+    explicit image_decoder(const create_config &conf);
+
+    ~image_decoder();
+
+    using cb_func_type = std::function<void(image_ptr)>;
+    cb_func_type cb_func;
+
+    void decode(const data_type &data, const cb_func_type &cb);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+
+#endif //DEPTHGUIDE_IMAGE_DECODER_H

src/codec/image_encoder.cpp

@@ -0,0 +1,133 @@
+#include "image_encoder.h"
+#include "image_codec.h"
+#include "codec/encoder_nvenc.h"
+#include "image_process/cuda_impl/pixel_convert.cuh"
+
+#include <nlohmann/json.hpp>
+
+using namespace nlohmann;
+using namespace image_codec;
+
+struct image_encoder::impl {
+
+    create_config conf;
+
+    struct encoder_store_type {
+        impl *pimpl = nullptr;
+        std::unique_ptr<encoder_nvenc> nvenc;
+        timestamp_type last_clear_ts = 0;
+
+        bool handle_idr() {
+            if (auto req_ts = pimpl->last_idr_req_ts;
+                    last_clear_ts < pimpl->last_idr_req_ts) {
+                last_clear_ts = req_ts;
+                return true;
+            }
+            return false;
+        }
+    };
+
+    using encoder_map_type =
+            std::unordered_map<size_t, encoder_store_type>;
+    encoder_map_type enc_map;
+
+    timestamp_type last_idr_req_ts = current_timestamp();
+
+    data_type encode_nvenc(image_ptr img, size_t series) {
+        // create NvEnc if needed
+        assert(enc_map.contains(series));
+        auto &enc_st = enc_map.at(series);
+        auto &encoder = enc_st.nvenc;
+        auto img_size = img->size();
+        if (encoder == nullptr
+            || encoder->frame_size() != img_size) [[unlikely]] {
+            auto fps = img->get_meta_ext<size_t>(META_REFRESH_RATE);
+            auto enc_conf = encoder_nvenc::create_config{
+                    .frame_size = img_size,
+                    .frame_rate = (int) fps,
+                    .bitrate_mbps = conf.bitrate_mbps,
+                    .save_file = conf.save_file,
+                    .save_length = conf.save_length,
+                    .ctx = conf.ctx,
+                    .stream = conf.stream,
+            };
+            encoder = encoder_nvenc::create(enc_conf);
+        }
+        assert(encoder != nullptr);
+        assert(encoder->frame_size() == img_size);
+
+        // rgb -> bgra
+        if (img->cv_type() == CV_8UC3) {
+            auto img_bgra = create_image(img_size, CV_8UC4);
+            call_cvt_rgb_bgra_u8(img->cuda<uchar3>(conf.stream),
+                                 img_bgra->cuda<uchar4>(conf.stream),
+                                 conf.stream->cuda);
+            img_bgra->cuda_modified(conf.stream);
+            img = img_bgra;
+        }
+
+        assert(img->cv_type() == CV_8UC4);
+        auto frame = encoder->encode(img->v1<uchar4>(), enc_st.handle_idr());
+        return frame.data;
+    }
+
+    data_type encode(const image_ptr &img) {
+        auto series = img->get_meta_ext<size_t>(META_SERIES_NAME);
+        if (!enc_map.contains(series)) [[unlikely]] {
+            enc_map.emplace(std::piecewise_construct,
+                            std::forward_as_tuple(series),
+                            std::forward_as_tuple(this));
+        }
+
+        auto enc_type = conf.type;
+        size_t sp_id = 0;
+        if (auto sp = img->get_meta_any(META_IMAGE_SPECIAL_CODEC); !sp.empty()) {
+            enc_type = ENCODER_SPECIAL;
+            sp_id = boost::any_cast<size_t>(sp);
+        }
+
+        auto head = json();
+        head["series"] = series;
+        head["type"] = enc_type;
+        head["special"] = sp_id;
+        head["width"] = img->width();
+        head["height"] = img->height();
+
+        auto ret = data_type();
+        switch (enc_type) {
+            case ENCODER_NVENC: {
+                ret = encode_nvenc(img, series);
+                break;
+            }
+            case ENCODER_SPECIAL: {
+                auto &enc_st = enc_map.at(series);
+                ret = get_special_encoder(sp_id)(img, enc_st.handle_idr());
+                break;
+            }
+            default: {
+                RET_ERROR_E;
+            }
+        }
+
+        auto writer = network_writer();
+        writer.write_with_length(head);
+        writer.write_with_length(ret);
+        return writer.current_data();
+    }
+
+};
+
+image_encoder::image_encoder(create_config conf)
+        : pimpl(std::make_unique<impl>()) {
+    pimpl->conf = conf;
+}
+
+image_encoder::~image_encoder() = default;
+
+data_type image_encoder::encode(const image_ptr &img) {
+    return pimpl->encode(img);
+}
+
+void image_encoder::clear_cache() {
+    pimpl->last_idr_req_ts = current_timestamp();
+}

src/codec/image_encoder.h

@@ -0,0 +1,37 @@
+#ifndef DEPTHGUIDE_IMAGE_ENCODER_H
+#define DEPTHGUIDE_IMAGE_ENCODER_H
+
+#include "codec/codec_base.hpp"
+#include "core/image_utility_v2.h"
+#include "network/binary_utility.hpp"
+
+#include <memory>
+
+class image_encoder {
+public:
+
+    struct create_config {
+        encoder_type type = ENCODER_NVENC;
+        smart_cuda_stream *stream = nullptr;
+
+        // for NvEnc
+        float bitrate_mbps;
+        bool save_file;
+        bool save_length;
+        CUcontext *ctx = nullptr;
+    };
+
+    explicit image_encoder(create_config conf);
+
+    ~image_encoder();
+
+    data_type encode(const image_ptr &img);
+
+    void clear_cache();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#endif //DEPTHGUIDE_IMAGE_ENCODER_H

src/codec/mesh_codec.hpp

@@ -0,0 +1,22 @@
+#ifndef DEPTHGUIDE_MESH_CODEC_HPP
+#define DEPTHGUIDE_MESH_CODEC_HPP
+
+#include <xxhash.h>
+
+namespace mesh_codec {
+
+    using hash_type = XXH64_hash_t;
+
+    using size_type = uint32_t;
+
+    using flag_type = uint8_t;
+    static constexpr flag_type FLAG_NEW = 0x00;
+    static constexpr flag_type FLAG_CACHED = 0x01;
+    static constexpr flag_type FLAG_NO_CACHE = 0x02;
+
+    static constexpr ptrdiff_t pre_size =
+            sizeof(flag_type) + sizeof(hash_type);
+
+}
+
+#endif //DEPTHGUIDE_MESH_CODEC_HPP

src/codec/mesh_decoder.cpp

@@ -0,0 +1,55 @@
+#include "mesh_decoder.h"
+#include "mesh_codec.hpp"
+
+#include <unordered_map>
+
+using namespace mesh_codec;
+
+struct mesh_decoder::impl {
+
+    using cache_map_type =
+            std::unordered_map<hash_type, mesh_ptr>;
+    cache_map_type cache_map;
+
+    static mesh_ptr decode_mesh(const data_type &data) {
+        auto ret = mesh_type::raw_info_type();
+        auto reader = network_reader(data);
+        reader >> ret.num_triangles;
+        auto vbo_size = reader.read_value<size_t>();
+        ret.vbo_data = reader.read_data(vbo_size);
+        auto ebo_size = reader.read_value<size_t>();
+        ret.ebo_data = reader.read_data(ebo_size);
+        assert(reader.empty());
+        return mesh_type::from_raw(ret);
+    }
+
+    mesh_ptr decode(const data_type &data) {
+        auto reader = network_reader(data);
+        auto flag = reader.read_value<flag_type>();
+        auto id = reader.read_value<hash_type>();
+        if (cache_map.contains(id)) {
+            return cache_map.at(id);
+        }
+
+//        assert(!reader.empty());
+        if (reader.empty()) return nullptr; // TODO: mesh IDR may not be handled???
+        auto mesh = decode_mesh(reader.read_remain());
+        if (flag != FLAG_NO_CACHE) {
+            assert(!cache_map.contains(id));
+            cache_map.emplace(id, mesh);
+        }
+
+        return mesh;
+    }
+
+};
+
+mesh_decoder::mesh_decoder(create_config _)
+        : pimpl(std::make_unique<impl>()) {
+}
+
+mesh_decoder::~mesh_decoder() = default;
+
+mesh_ptr mesh_decoder::decode(const data_type &data) {
+    return pimpl->decode(data);
+}

src/codec/mesh_decoder.h

@@ -0,0 +1,26 @@
+#ifndef DEPTHGUIDE_MESH_DECODER_H
+#define DEPTHGUIDE_MESH_DECODER_H
+
+#include "render/render_mesh.h"
+
+#include <memory>
+
+class mesh_decoder {
+public:
+
+    struct create_config {
+    };
+
+    explicit mesh_decoder(create_config conf);
+
+    ~mesh_decoder();
+
+    mesh_ptr decode(const data_type &data);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+
+#endif //DEPTHGUIDE_MESH_DECODER_H

src/codec/mesh_encoder.cpp

@@ -0,0 +1,67 @@
+#include "mesh_encoder.h"
+#include "mesh_codec.hpp"
+
+#include <xxhash.h>
+
+#include <unordered_set>
+
+using namespace mesh_codec;
+
+struct mesh_encoder::impl {
+
+    using cache_set_type =
+            std::unordered_set<hash_type>;
+    cache_set_type cache_set;
+
+    static data_type encode_mesh(const mesh_ptr &mesh) {
+        auto raw = mesh->get_raw_info();
+        auto ret_size = sizeof(raw.num_triangles)
+                        + sizeof(raw.vbo_data.size) + raw.vbo_data.size
+                        + sizeof(raw.ebo_data.size) + raw.ebo_data.size;
+        auto ret = data_type(ret_size, pre_size);
+        auto writer = network_writer(ret);
+        writer << raw.num_triangles
+               << raw.vbo_data.size << raw.vbo_data
+               << raw.ebo_data.size << raw.ebo_data;
+        assert(writer.empty());
+        return ret;
+    }
+
+    data_type encode(const mesh_ptr &mesh) {
+        static constexpr auto hash_seed = 0;
+        auto ret = encode_mesh(mesh);
+        auto id = XXH64(ret.start_ptr(), ret.size, hash_seed);
+
+        auto head = ret.sub_data(-pre_size);
+        auto writer = network_writer(head);
+        if (cache_set.contains(id)) {
+            writer << FLAG_CACHED << id;
+            assert(writer.empty());
+            return head;
+        } else {
+            writer << FLAG_NEW << id;
+            cache_set.emplace(id);
+            assert(writer.empty());
+            head = head.extend(ret.size); // recover data
+            return head;
+        }
+    }
+
+    void clear_cache() {
+        cache_set.clear();
+    }
+
+};
+
+mesh_encoder::mesh_encoder()
+        : pimpl(std::make_unique<impl>()) {}
+
+mesh_encoder::~mesh_encoder() = default;
+
+data_type mesh_encoder::encode(const mesh_ptr &mesh) {
+    return pimpl->encode(mesh);
+}
+
+void mesh_encoder::clear_cache() {
+    pimpl->clear_cache();
+}

src/codec/mesh_encoder.h

@@ -0,0 +1,24 @@
+#ifndef DEPTHGUIDE_MESH_ENCODER_H
+#define DEPTHGUIDE_MESH_ENCODER_H
+
+#include "render/render_mesh.h"
+
+#include <memory>
+
+class mesh_encoder {
+public:
+
+    mesh_encoder();
+
+    ~mesh_encoder();
+
+    data_type encode(const mesh_ptr &mesh);
+
+    void clear_cache();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+#endif //DEPTHGUIDE_MESH_ENCODER_H

src/codec/pc_codec.hpp

@@ -0,0 +1,18 @@
+#ifndef DEPTHGUIDE_PC_CODEC_HPP
+#define DEPTHGUIDE_PC_CODEC_HPP
+
+#include <cstdint>
+#include <cstddef>
+
+namespace pc_codec {
+
+    enum encode_method_enum {
+        PC_DIRECT,
+        PC_RGB_DEPTH,
+    };
+
+    using size_type = uint32_t;
+
+}
+
+#endif //DEPTHGUIDE_PC_CODEC_HPP

src/codec/pc_decoder.cpp

@@ -0,0 +1,148 @@
+#include "pc_decoder.h"
+#include "pc_codec.hpp"
+#include "codec/image_decoder.h"
+#include "image_process/process_funcs.h"
+#include "image_process/cuda_impl/fake_color.cuh"
+
+using namespace nlohmann;
+using namespace pc_codec;
+
+struct pc_decoder::impl {
+
+    create_config conf;
+
+    struct decoder_store_type {
+        impl *pimpl = nullptr;
+        std::unique_ptr<image_decoder> dec;
+
+        size_t series;
+        json head;
+        cb_func_type cb_func;
+
+        image_ptr img_rgb = nullptr;
+        image_ptr remap = nullptr;
+
+        void on_image(const image_ptr &img) {
+            switch (img->cv_type()) {
+                // @formatter:off
+                case CV_8UC3: { img_rgb = img; break; }
+                case CV_32FC2: { remap = img; break; }
+                // @formatter:on
+                default: {
+                    RET_ERROR;
+                }
+            }
+
+            if (img_rgb != nullptr && remap != nullptr) {
+                pimpl->on_image(this);
+            }
+        }
+    };
+
+    using decoder_map_type =
+            std::unordered_map<size_t, decoder_store_type>;
+    decoder_map_type dec_map;
+
+    void on_image(decoder_store_type *info) {
+        // split to [rgb, depth]
+        auto img = info->img_rgb;
+        assert(img != nullptr);
+        assert(img->width() % 2 == 0);
+        auto img_width = img->width() >> 1;
+        auto img_rgb = img->sub_image(0, 0, img_width);
+        auto depth_fake = img->sub_image(0, img_width);
+        auto img_size = img_rgb->size();
+
+        // fake color -> depth
+        auto method = info->head["depth_method"].get<fake_color_method>();
+        auto depth_min = info->head["depth_min"].get<float>();
+        auto depth_max = info->head["depth_max"].get<float>();
+        auto img_depth = create_image(img_size, CV_32FC1);
+        auto fake_conf = fake_color_config{
+                .mode = method, .lower = depth_min, .upper = depth_max,
+        };
+        call_fake_color_decode(depth_fake->cuda<uchar3>(conf.stream),
+                               img_depth->cuda<float1>(conf.stream),
+                               fake_conf, conf.stream->cuda);
+        img_depth->cuda_modified(conf.stream);
+
+        // generate point cloud
+        auto pc = pc_ptr();
+        auto pc_conf = gen_pc_rgbd::config_direct{
+                .color_img = img_rgb, .depth_img =img_depth,
+                .remap_img = info->remap, .pc_out = &pc,
+                .stream = conf.stream,
+        };
+        gen_pc_rgbd::call_direct(pc_conf);
+
+        // commit result
+        pc->set_meta_any(META_SERIES_NAME, info->series);
+        pc->set_meta_any(META_DEPTH_MIN, depth_min);
+        pc->set_meta_any(META_DEPTH_MAX, depth_max);
+        info->cb_func(pc);
+    }
+
+    void decode_image(const data_type &data, size_t series, const json &head) {
+        assert(dec_map.contains(series));
+        auto info = &dec_map.at(series);
+        auto &decoder = info->dec;
+        if (decoder == nullptr) [[unlikely]] {
+            auto dec_conf = image_decoder::create_config{
+                    .stream = conf.stream,
+            };
+            decoder = std::make_unique<image_decoder>(dec_conf);
+        }
+        assert(decoder != nullptr);
+
+        info->head = head;
+        info->series = series;
+
+        auto img_cb = [=](const image_ptr &img) { info->on_image(img); };
+
+        auto reader = network_reader(data);
+        auto img_data = reader.read_data_with_length();
+        decoder->decode(img_data, img_cb);
+        auto remap_data = reader.read_data_with_length();
+        decoder->decode(remap_data, img_cb);
+        assert(reader.empty());
+    }
+
+    void decode(const data_type &data, const cb_func_type &cb) {
+        auto reader = network_reader(data);
+        auto head = reader.read_json_with_length();
+
+        auto series = head["series"].get<size_t>();
+        if (!dec_map.contains(series)) [[unlikely]] {
+            dec_map.emplace(std::piecewise_construct,
+                            std::forward_as_tuple(series),
+                            std::forward_as_tuple(this));
+        }
+        assert(dec_map.contains(series));
+        dec_map.at(series).cb_func = cb;
+
+        auto method = head["method"].get<encode_method_enum>();
+        auto ret = reader.read_data_with_length();
+        assert(reader.empty());
+        switch (method) {
+            case PC_RGB_DEPTH: {
+                decode_image(ret, series, head);
+                break;
+            }
+            default: {
+                RET_ERROR;
+            }
+        }
+    }
+
+};
+
+pc_decoder::pc_decoder(const create_config &conf)
+        : pimpl(std::make_unique<impl>()) {
+    pimpl->conf = conf;
+}
+
+pc_decoder::~pc_decoder() = default;
+
+void pc_decoder::decode(const data_type &data, const cb_func_type &cb) {
+    pimpl->decode(data, cb);
+}

src/codec/pc_decoder.h

@@ -0,0 +1,31 @@
+#ifndef DEPTHGUIDE_PC_DECODER_H
+#define DEPTHGUIDE_PC_DECODER_H
+
+#include "core/pc_utility.h"
+#include "network/binary_utility.hpp"
+
+#include <memory>
+
+class pc_decoder {
+public:
+
+    struct create_config {
+        smart_cuda_stream *stream = nullptr;
+    };
+
+    explicit pc_decoder(const create_config &conf);
+
+    ~pc_decoder();
+
+    using cb_func_type = std::function<void(pc_ptr)>;
+    cb_func_type cb_func;
+
+    void decode(const data_type &data, const cb_func_type &cb);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+
+#endif //DEPTHGUIDE_PC_DECODER_H

src/codec/pc_encoder.cpp

@@ -0,0 +1,109 @@
+#include "pc_encoder.h"
+#include "core/image_utility_v2.h"
+#include "image_process/process_funcs.h"
+
+#include <nlohmann/json.hpp>
+
+#include <unordered_map>
+
+using namespace pc_codec;
+using namespace nlohmann;
+
+struct pc_encoder::impl {
+
+    create_config conf;
+
+    data_type encode_image(const pc_ptr &pc, json &head, size_t series) {
+        assert(pc->format() == PC_XYZ_RGB);
+
+        // depth -> fake color
+        auto depth_img = pc->get_meta_ext<image_ptr>(META_SOURCE_DEPTH);
+        auto depth_fake = create_image(depth_img->size(), CV_8UC3);
+        head["depth_method"] = conf.depth_method;
+        auto depth_min = pc->get_meta_ext<float>(META_DEPTH_MIN);
+        auto depth_max = pc->get_meta_ext<float>(META_DEPTH_MAX);
+        head["depth_min"] = depth_min;
+        head["depth_max"] = depth_max;
+        auto fake_conf = fake_color_config{
+                .mode = conf.depth_method,
+                .lower = depth_min, .upper = depth_max,
+        };
+        call_fake_color_encode(depth_img->cuda<float1>(conf.stream),
+                               depth_fake->cuda<uchar3>(conf.stream),
+                               fake_conf, conf.stream->cuda);
+        depth_fake->cuda_modified(conf.stream);
+
+        // concatenate [rgb, depth]
+        auto color_img = pc->get_meta_ext<image_ptr>(META_SOURCE_RGB);
+        auto img_rgb = image_ptr();
+        auto con_conf = concatenate_image::config_direct{
+                .left_img = color_img, .right_img = depth_fake,
+                .out_img = &img_rgb, .stream = conf.stream,
+        };
+        concatenate_image::call_direct(con_conf);
+
+        img_rgb->set_meta_any(META_SERIES_NAME, series);
+        img_rgb->set_meta_any(META_REFRESH_RATE,
+                              pc->get_meta_ext<size_t>(META_REFRESH_RATE));
+
+        // encode rgb & depth image
+        auto writer = network_writer();
+        assert(img_rgb->cv_type() == CV_8UC3);
+        auto img_data = conf.img_enc->encode(img_rgb);
+        writer.write_with_length(img_data);
+        SPDLOG_DEBUG("Size of image is {}", img_data.size);
+
+        // encode remap image
+        auto remap_img = pc->get_meta_ext<image_ptr>(META_SOURCE_REMAP);
+        assert(remap_img->cv_type() == CV_32FC2);
+        auto remap_data = conf.img_enc->encode(remap_img);
+        writer.write_with_length(remap_data);
+        SPDLOG_DEBUG("Size of remap is {}", remap_data.size);
+
+        return writer.current_data();
+    }
+
+    data_type encode(const pc_ptr &pc) {
+        auto head = json();
+        auto series = pc->get_meta_ext<size_t>(META_SERIES_NAME);
+        head["series"] = series;
+        head["type"] = pc->format();
+        head["method"] = conf.method;
+
+        auto ret = data_type();
+        switch (conf.method) {
+            case PC_RGB_DEPTH: {
+                ret = encode_image(pc, head, series);
+                break;
+            }
+            default: {
+                RET_ERROR_E;
+            }
+        }
+
+        auto writer = network_writer();
+        writer.write_with_length(head);
+        writer.write_with_length(ret);
+        return writer.current_data();
+    }
+
+    void clear_cache() {
+        conf.img_enc->clear_cache();
+    }
+
+};
+
+pc_encoder::pc_encoder(const create_config &conf)
+        : pimpl(std::make_unique<impl>()) {
+    pimpl->conf = conf;
+}
+
+pc_encoder::~pc_encoder() = default;
+
+data_type pc_encoder::encode(const pc_ptr &pc) {
+    return pimpl->encode(pc);
+}
+
+void pc_encoder::clear_cache() {
+    pimpl->clear_cache();
+}

src/codec/pc_encoder.h

@@ -0,0 +1,39 @@
+#ifndef DEPTHGUIDE_PC_ENCODER_H
+#define DEPTHGUIDE_PC_ENCODER_H
+
+#include "pc_codec.hpp"
+#include "codec/image_encoder.h"
+#include "core/pc_utility.h"
+#include "image_process/cuda_impl/fake_color.cuh"
+#include "network/binary_utility.hpp"
+
+#include <memory>
+
+class pc_encoder {
+public:
+
+    struct create_config {
+        using method_type =
+                pc_codec::encode_method_enum;
+        method_type method = pc_codec::PC_RGB_DEPTH;
+        fake_color_method depth_method = FAKE_800P;
+        smart_cuda_stream *stream = nullptr;
+
+        image_encoder *img_enc = nullptr;
+    };
+
+    explicit pc_encoder(const create_config &conf);
+
+    ~pc_encoder();
+
+    data_type encode(const pc_ptr &pc);
+
+    void clear_cache();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+
+#endif //DEPTHGUIDE_PC_ENCODER_H

src/codec/scene_decoder.cpp

@@ -0,0 +1,159 @@
+#include "scene_decoder.h"
+#include "codec/mesh_decoder.h"
+#include "codec/image_decoder.h"
+#include "codec/pc_decoder.h"
+#include "core/math_helper.hpp"
+
+#include <nlohmann/json.hpp>
+
+#include <glm/gtc/type_ptr.hpp>
+
+using namespace nlohmann;
+
+namespace scene_decoder_impl {
+
+    template<typename MatT>
+    MatT to_glm(const json &j) {
+        static constexpr auto size = sizeof(MatT) /
+                                     sizeof(typename MatT::value_type);
+        assert(j.size() == size);
+        auto ret = MatT();
+        auto ptr = glm::value_ptr(ret);
+        for (auto k = 0; k < size; ++k) {
+            *(ptr + k) = j[k].get<float>();
+        }
+        return ret;
+    }
+
+    camera_info json_to_camera_info(const json &j) {
+        auto ret = camera_info();
+        ret.transform = to_glm<glm::mat4>(j["transform"]);
+        ret.fov = j["fov"].get<float>();
+        ret.near = j["near"].get<float>();
+        ret.far = j["far"].get<float>();
+        return ret;
+    }
+
+    using light_info = scene_render_info::light_info;
+
+    light_info json_to_light_info(const json &j) {
+        auto ret = light_info();
+        ret.follow_camera = j["follow_camera"].get<bool>();
+        ret.direction = to_glm<glm::vec3>(j["direction"]);
+        return ret;
+    }
+
+}
+
+using namespace scene_decoder_impl;
+
+struct scene_decoder::impl {
+
+    std::unique_ptr<mesh_decoder> mesh_dec;
+    std::unique_ptr<image_decoder> img_dec;
+    std::unique_ptr<pc_decoder> pc_dec;
+
+    using cb_func_type = create_config::cb_func_type;
+    cb_func_type cb_func;
+
+    obj_name_type scene_name = invalid_obj_name;
+
+    using item_type = scene_render_info::item_type;
+    using image_info = scene_render_info::image_info;
+    using pc_info = scene_render_info::pc_info;
+    using mesh_info = scene_render_info::mesh_info;
+
+    explicit impl(const create_config &conf) {
+        mesh_dec = std::make_unique<mesh_decoder>(
+                mesh_decoder::create_config{});
+        img_dec = std::make_unique<image_decoder>(
+                image_decoder::create_config{.stream = conf.stream});
+        pc_dec = std::make_unique<pc_decoder>(
+                pc_decoder::create_config{.stream = conf.stream});
+
+        scene_name = conf.scene_name;
+        cb_func = conf.cb_func;
+    }
+
+    image_info json_to_image_info(const json &j, const data_type &extra) {
+        auto ret = image_info();
+        ret.flip_y = j["flip_y"].get<bool>();
+        auto img_cb = [ptr = &ret.img](const image_ptr &img) { *ptr = img; };
+        img_dec->decode(extra, img_cb);
+        assert(ret.img != nullptr);
+        return ret;
+    }
+
+    pc_info json_to_pc_info(const json &j, const data_type &extra) {
+        auto ret = pc_info();
+        ret.color = to_glm<glm::vec3>(j["color"]);
+        ret.point_size = j["point_size"].get<float>();
+        auto pc_cb = [ptr = &ret.pc](const pc_ptr &pc) { *ptr = pc; };
+        pc_dec->decode(extra, pc_cb);
+        assert(ret.pc != nullptr);
+        return ret;
+    }
+
+    mesh_info json_to_mesh_info(const json &j, const data_type &extra) {
+        auto ret = mesh_info();
+        ret.material.ambient =
+                to_glm<glm::vec3>(j["material.ambient"]);
+        ret.material.diffuse =
+                to_glm<glm::vec3>(j["material.diffuse"]);
+        ret.mesh = mesh_dec->decode(extra);
+        return ret;
+    }
+
+    void decode(const data_type &data) {
+        auto reader = network_reader(data);
+        auto head = reader.read_json_with_length();
+
+        auto ret = std::make_shared<scene_render_info>();
+        ret->camera = json_to_camera_info(head["camera"]);
+        ret->light = json_to_light_info(head["light"]);
+
+        for (auto &item_j: head["items"]) {
+            auto item = item_type();
+            item.transform = to_glm<glm::mat4>(item_j["transform"]);
+            item.alpha = item_j["alpha"].get<float>();
+
+            auto extra = reader.read_data_with_length();
+            auto &info = item_j["info"];
+            switch (item_j["info_type"].get<int>()) {
+                case 1: { // image_info
+                    item.info = json_to_image_info(info, extra);
+                    break;
+                }
+                case 2: { // mesh_info
+                    item.info = json_to_mesh_info(info, extra);
+                    break;
+                }
+                case 3: { // pc_info
+                    item.info = json_to_pc_info(info, extra);
+                    break;
+                }
+                default: {
+                    RET_ERROR;
+                }
+            }
+            ret->items.emplace_back(item);
+        }
+
+        if (scene_name != invalid_obj_name) {
+            OBJ_SAVE(scene_name, ret);
+        } else {
+            cb_func(ret);
+        }
+    }
+
+};
+
+scene_decoder::scene_decoder(const create_config &conf)
+        : pimpl(std::make_unique<impl>(conf)) {
+}
+
+scene_decoder::~scene_decoder() = default;
+
+void scene_decoder::decode(const data_type &data) {
+    pimpl->decode(data);
+}

src/codec/scene_decoder.h

@@ -0,0 +1,33 @@
+#ifndef DEPTHGUIDE_SCENE_DECODER_H
+#define DEPTHGUIDE_SCENE_DECODER_H
+
+#include "core/object_manager.h"
+#include "network/binary_utility.hpp"
+#include "render/render_scene.h"
+
+#include <memory>
+
+class scene_decoder {
+public:
+
+    struct create_config {
+        obj_name_type scene_name = invalid_obj_name;
+        smart_cuda_stream *stream = nullptr;
+
+        using cb_func_type = std::function<void(scene_ptr)>;
+        cb_func_type cb_func;
+    };
+
+    explicit scene_decoder(const create_config &conf);
+
+    ~scene_decoder();
+
+    void decode(const data_type &data);
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+
+#endif //DEPTHGUIDE_SCENE_DECODER_H

src/codec/scene_encoder.cpp

@@ -0,0 +1,165 @@
+#include "scene_encoder.h"
+#include "codec/mesh_encoder.h"
+#include "core/math_helper.hpp"
+
+#include <nlohmann/json.hpp>
+
+#include <glm/gtc/type_ptr.hpp>
+
+using namespace nlohmann;
+
+namespace scene_encoder_impl {
+
+    template<typename MatT>
+    json glm_to_json(const MatT &mat) {
+        static constexpr auto size = sizeof(MatT) /
+                                     sizeof(typename MatT::value_type);
+        auto ret = json::array();
+        auto ptr = glm::value_ptr(mat);
+        for (auto k = 0; k < size; ++k) {
+            ret.emplace_back(*(ptr + k));
+        }
+        return ret;
+    }
+
+    using image_info = scene_render_info::image_info;
+    using mesh_info = scene_render_info::mesh_info;
+    using pc_info = scene_render_info::pc_info;
+    using custom_info = scene_render_info::custom_info;
+
+    json info_to_json(const std::monostate &) {
+        RET_ERROR_E;
+    }
+
+    json info_to_json(const custom_info &) {
+        RET_ERROR_E;
+    }
+
+    json info_to_json(const image_info &info) {
+        auto ret = json();
+        ret["series"] = info.img->get_meta_ext<size_t>(META_SERIES_NAME);
+        ret["flip_y"] = info.flip_y;
+        return ret;
+    }
+
+    json info_to_json(const mesh_info &info) {
+        auto ret = json();
+        ret["material.ambient"] = glm_to_json(info.material.ambient);
+        ret["material.diffuse"] = glm_to_json(info.material.diffuse);
+        return ret;
+    }
+
+    json info_to_json(const pc_info &info) {
+        auto ret = json();
+        ret["series"] = info.pc->get_meta_ext<size_t>(META_SERIES_NAME);
+        ret["point_size"] = info.point_size;
+        ret["color"] = glm_to_json(info.color);
+        return ret;
+    }
+
+    json info_to_json(const camera_info &info) {
+        auto ret = json();
+        ret["transform"] = glm_to_json(info.transform);
+        ret["fov"] = info.fov;
+        ret["near"] = info.near;
+        ret["far"] = info.far;
+        return ret;
+    }
+
+    using light_info = scene_render_info::light_info;
+
+    json info_to_json(const light_info &info) {
+        auto ret = json();
+        ret["follow_camera"] = info.follow_camera;
+        ret["direction"] = glm_to_json(info.direction);
+        return ret;
+    }
+
+}
+
+using namespace scene_encoder_impl;
+
+struct scene_encoder::impl {
+
+    create_config conf;
+    mesh_encoder mesh_enc;
+
+    data_type encode_info(const std::monostate &) {
+        RET_ERROR_E;
+    }
+
+    data_type encode_info(const custom_info &) {
+        RET_ERROR_E;
+    }
+
+    data_type encode_info(const image_info &info) {
+        assert(info.depth == nullptr);
+        return conf.img_enc->encode(info.img);
+    }
+
+    data_type encode_info(const mesh_info &info) {
+        return mesh_enc.encode(info.mesh);
+    }
+
+    data_type encode_info(const pc_info &info) {
+        return conf.pc_enc->encode(info.pc);
+    }
+
+    data_type encode(const scene_ptr &info) {
+        auto extra_writer = network_writer();
+
+        auto list_json = json::array();
+        for (auto &item: info->items) {
+            auto index = item.info.index();
+            if (index == 4) continue; // ignore custom_info
+
+            auto item_json = json();
+            item_json["alpha"] = item.alpha;
+            item_json["transform"] = glm_to_json(item.transform);
+            item_json["info_type"] = index;
+            item_json["info"] = std::visit([](auto &info) {
+                return info_to_json(info);
+            }, item.info);
+
+            auto sub_data = std::visit([=, this](auto &info) {
+                return encode_info(info);
+            }, item.info);
+            extra_writer.write_with_length(sub_data);
+            SPDLOG_DEBUG("Size of extra is {}.", sub_data.size);
+
+            list_json.emplace_back(item_json);
+        }
+
+        auto head = json();
+        head["items"] = list_json;
+        head["camera"] = info_to_json(info->camera);
+        head["light"] = info_to_json(info->light);
+
+        auto out_writer = network_writer();
+        out_writer.write_with_length(head);
+        out_writer << extra_writer.current_data();
+        return out_writer.current_data();
+    }
+
+    void clear_cache() {
+        mesh_enc.clear_cache();
+        conf.img_enc->clear_cache();
+        conf.pc_enc->clear_cache();
+    }
+
+};
+
+scene_encoder::scene_encoder(const create_config &conf)
+        : pimpl(std::make_unique<impl>()) {
+    pimpl->conf = conf;
+}
+
+scene_encoder::~scene_encoder() = default;
+
+data_type scene_encoder::encode(const scene_ptr &scene) {
+    return pimpl->encode(scene);
+}
+
+void scene_encoder::clear_cache() {
+    pimpl->clear_cache();
+}

src/codec/scene_encoder.h

@@ -0,0 +1,33 @@
+#ifndef DEPTHGUIDE_SCENE_ENCODER_H
+#define DEPTHGUIDE_SCENE_ENCODER_H
+
+#include "codec/image_encoder.h"
+#include "codec/pc_encoder.h"
+#include "network/binary_utility.hpp"
+#include "render/render_scene.h"
+
+#include <memory>
+
+class scene_encoder {
+public:
+
+    struct create_config {
+        image_encoder *img_enc;
+        pc_encoder *pc_enc;
+    };
+
+    explicit scene_encoder(const create_config &conf);
+
+    ~scene_encoder();
+
+    data_type encode(const scene_ptr &scene);
+
+    void clear_cache();
+
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
+};
+
+
+#endif //DEPTHGUIDE_SCENE_ENCODER_H

src/core/image_utility_v2.h

@@ -102,6 +102,16 @@ public:
 
     pixel_format_enum pixel_format() const;
 
+    struct basic_info_type {
+        cv::Size size;
+        int cv_type;
+        pixel_format_enum pixel_format;
+
+        bool operator==(const basic_info_type &o) const;
+    };
+
+    basic_info_type basic_info() const;
+
     // synchronization will be performed
     cv::Mat cv_mat(smart_cuda_stream *stream = nullptr);
 
@@ -153,6 +163,7 @@ public:
 };
 
 using image_ptr = generic_image::pointer;
+static_assert(PointerContainsMeta<image_ptr>);
 
 inline image_ptr create_image(cv::Size size, int type,
                               pixel_format_enum pixel = PIX_NORMAL) {

src/core/impl/image_utility_v2.cpp

@@ -282,6 +282,11 @@ void generic_image::impl::sub_image_inplace(int row, int col, int width, int hei
                 (uint8_t *) store_cuda.row_start(row) + col * elem_bytes(),
                 [p = store_cuda.ptr](void *) {});
     }
+
+    if (width == -1) { width = size.width - col; }
+    if (height == -1) { height = size.height - row; }
+    assert(width + col <= size.width);
+    assert(height + row <= size.height);
     size = cv::Size(width, height);
 }
 
@@ -306,6 +311,13 @@ void generic_image::impl::cuda_modified(smart_cuda_stream *stream) {
     REC_CREATE(store_cuda.ptr, stream);
 }
 
+bool generic_image::basic_info_type::operator==(const basic_info_type &o) const {
+    if (size != o.size) return false;
+    if (cv_type != o.cv_type) return false;
+    if (pixel_format != o.pixel_format) return false;
+    return true;
+}
+
 generic_image::generic_image(std::unique_ptr<impl> &&_pimpl)
         : meta_base(_pimpl.get()), pimpl(std::move(_pimpl)) {
     pimpl->q_this = this;
@@ -360,6 +372,14 @@ pixel_format_enum generic_image::pixel_format() const {
     return pimpl->pix_fmt;
 }
 
+generic_image::basic_info_type generic_image::basic_info() const {
+    auto ret = basic_info_type();
+    ret.size = size();
+    ret.cv_type = cv_type();
+    ret.pixel_format = pixel_format();
+    return ret;
+}
+
 image_mem_info generic_image::memory_v1(smart_cuda_stream *stream) const {
     return pimpl->get_memory_v1(stream);
 }
@@ -454,5 +474,7 @@ image_ptr to_image(obj_name_type name) {
         }
     };
     FORALL_IMG_TYPE;
+
+    OBJ_MERGE_META(name, ret.get());
     return ret;
 }

src/core/impl/object_manager.cpp

@@ -27,7 +27,7 @@ object_manager::impl::query_info(name_type obj_name) {
     if (iter == obj_pool.end()) [[unlikely]] return {};
     auto &st = iter->second;
     return obj_info{
-            .type = st.type, .pl_ptr = st.ptr,
+            .type = st.type, .pl_ptr = st.ptr, .meta = &st.meta,
             .sig = &st.sig, .last_save_ts = st.stats_timer.last_ts()};
 }
 

src/core/impl/object_manager_impl.h

@@ -16,6 +16,7 @@ struct object_manager::impl {
         std::type_index type;
         bool is_pending = false; // whether signal is queued.
         obj_sig_type sig;
+        meta_obj meta;
 
         // statistical information
         event_timer stats_timer;

src/core/meta_helper.hpp

@@ -1,10 +1,12 @@
 #ifndef DEPTHGUIDE_META_HELPER_HPP
 #define DEPTHGUIDE_META_HELPER_HPP
 
+#include <boost/any.hpp>
 #include <boost/container/flat_map.hpp>
 #include <boost/container/static_vector.hpp>
 
 #include <optional>
+#include <unordered_map>
 
 class meta_base {
 public:
@@ -14,25 +16,53 @@ public:
     using meta_key_type = size_t; // std::hash
     using meta_value_type = uint64_t;
 
+    auto get_meta_any(meta_key_type key) {
+        return pimpl->get_meta_any(key);
+    }
+
+    template<typename T>
+    T get_meta_ext(meta_key_type key) {
+        auto ret = get_meta_any(key);
+        assert(ret.type() == typeid(T));
+        return boost::any_cast<T>(ret);
+    }
+
     std::optional<meta_value_type> get_meta(meta_key_type key) {
         return pimpl->get_meta(key);
     }
 
+    template<typename T>
+    void set_meta_any(meta_key_type key, T &&val) {
+        pimpl->set_meta_any(key, std::forward<T>(val));
+    }
+
     void set_meta(meta_key_type key, meta_value_type val) {
-        return pimpl->set_meta(key, val);
+        set_meta_any(key, val);
+    }
+
+    void remove_meta(meta_key_type key) {
+        pimpl->remove_meta(key);
+    }
+
+    void merge_from(meta_base *other) {
+        pimpl->merge_from(other->pimpl);
     }
 
 protected:
 
+    using meta_any_type = boost::any;
+
     struct impl {
-        static constexpr auto max_meta_count = 16;
+//        static constexpr auto max_meta_count = 16;
+//        using meta_pool_type =
+//                boost::container::flat_map<meta_key_type, meta_any_type, std::less<>,
+//                        boost::container::static_vector<std::pair<meta_key_type, meta_any_type>, max_meta_count>>;
         using meta_pool_type =
-                boost::container::flat_map<meta_key_type, meta_value_type, std::less<>,
-                        boost::container::static_vector<std::pair<meta_key_type, meta_value_type>, max_meta_count>>;
+                std::unordered_map<meta_key_type, meta_any_type>;
         std::shared_ptr<meta_pool_type> meta_pool =
                 std::make_shared<meta_pool_type>();
 
-        std::optional<meta_value_type> get_meta(meta_key_type key) {
+        meta_any_type get_meta_any(meta_key_type key) {
             if (auto iter = meta_pool->find(key); iter != meta_pool->end()) {
                 return iter->second;
             } else {
@@ -40,14 +70,32 @@ protected:
             }
         }
 
-        void set_meta(meta_key_type key, meta_value_type val) {
+        std::optional<meta_value_type> get_meta(meta_key_type key) {
+            auto ret = get_meta_any(key);
+            if (ret.empty() ||
+                ret.type() != typeid(meta_value_type))
+                return {};
+            return boost::any_cast<meta_value_type>(ret);
+        }
+
+        template<typename T>
+        void set_meta_any(meta_key_type key, T &&val) {
             if (auto iter = meta_pool->find(key); iter != meta_pool->end()) {
-                iter->second = val;
+                iter->second = std::forward<T>(val);
             } else {
-                meta_pool->emplace(key, val);
+                meta_pool->emplace(key, std::forward<T>(val));
             }
         }
 
+        void remove_meta(meta_key_type key) {
+            meta_pool->erase(key);
+        }
+
+        void merge_from(impl *other) {
+            auto map_other = *other->meta_pool;
+            meta_pool->merge(map_other); // copy the map
+        }
+
     };
 
 private:
@@ -61,4 +109,40 @@ public:
 
 };
 
+class meta_obj : public meta_base {
+public:
+    meta_obj() : meta_base(&_impl) {}
+
+private:
+    impl _impl;
+};
+
+template<typename RT>
+concept PointerContainsMeta =
+std::is_base_of_v<meta_base, typeof(*std::declval<RT>())>;
+
+static_assert(PointerContainsMeta<meta_obj *>);
+
+using meta_key_type = meta_base::meta_key_type;
+static_assert(std::is_same_v<meta_key_type, size_t>);
+
+template<size_t N, typename KeyT = meta_key_type>
+inline constexpr KeyT meta_hash(const char (&str)[N]) {
+    constexpr KeyT init_val = 7;
+    constexpr KeyT multi_val = 31;
+
+    auto ret = init_val;
+    for (auto k = 0; k < N; ++k) {
+        ret = ret * multi_val + str[k];
+    }
+    return ret;
+}
+
+static constexpr auto META_SERIES_NAME = meta_hash("series_name"); // size_t, use hash
+static constexpr auto META_FRAME_ID = meta_hash("frame_id"); // size_t
+static constexpr auto META_REFRESH_RATE = meta_hash("refresh_rate"); // size_t, Hz
+
+static constexpr auto META_STEREO_LEFT = meta_hash("stereo_left");
+static constexpr auto META_STEREO_RIGHT = meta_hash("stereo_right");
+
 #endif //DEPTHGUIDE_META_HELPER_HPP

src/core/object_manager.h

@@ -1,6 +1,7 @@
 #ifndef DEPTHGUIDE_OBJECT_MANAGER_H
 #define DEPTHGUIDE_OBJECT_MANAGER_H
 
+#include "core/meta_helper.hpp"
 #include "core/utility.hpp"
 
 #include <boost/asio/post.hpp>
@@ -52,6 +53,7 @@ public:
 
         static_assert(std::is_copy_assignable_v<RT>);
         *(RT *) pl_ptr = val;
+        merge_meta<RT>(obj_name, pl_ptr);
         notify_signal(obj_name);
     }
 
@@ -91,6 +93,22 @@ public:
 
     obj_stats query_obj_stats(name_type obj_name);
 
+    template<typename T>
+    void pin_meta(name_type obj_name, meta_key_type key, T &&val) {
+        auto meta = query_meta_obj(obj_name);
+        meta->set_meta_any(key, std::forward<T>(val));
+    }
+
+    void unpin_meta(name_type obj_name, meta_key_type key) {
+        auto meta = query_meta_obj(obj_name);
+        meta->remove_meta(key);
+    }
+
+    void merge_meta(name_type obj_name, meta_base *meta) {
+        if (meta == nullptr) return;
+        meta->merge_from(query_meta_obj(obj_name));
+    }
+
 private:
 
     using del_func_type = void (*)(void *);
@@ -98,6 +116,7 @@ private:
     struct obj_info {
         std::type_index type = typeid(void);
         void *pl_ptr = nullptr;
+        meta_obj *meta = nullptr;
         obj_sig_type *sig = nullptr;
         timestamp_type last_save_ts = 0;
     };
@@ -120,6 +139,26 @@ private:
     // return target context or nullptr
     io_context *switch_ctx();
 
+    meta_obj *query_meta_obj(name_type obj_name) {
+        auto info_o = query_info(obj_name);
+        assert(info_o.has_value());
+        return info_o->meta;
+    }
+
+    template<typename RT>
+    void merge_meta(name_type obj_name, void *pl_ptr) {}
+
+    template<PointerContainsMeta RT>
+    void merge_meta(name_type obj_name, void *pl_ptr) {
+        auto rt_ptr = *(RT *) pl_ptr;
+        if (rt_ptr == nullptr) return;
+        auto meta_ptr =
+                dynamic_cast<meta_base *>(&(*rt_ptr));
+        assert(meta_ptr != nullptr);
+        auto meta_obj = query_meta_obj(obj_name);
+        meta_ptr->merge_from(meta_obj);
+    }
+
     struct impl;
     std::unique_ptr<impl> pimpl;
 };
@@ -149,4 +188,10 @@ extern object_manager *main_ob;
 #define OBJ_SIG(name) \
     main_ob->query_signal(name)
 
+#define OBJ_PIN_META(name, key, val) \
+    main_ob->pin_meta(name, key, val)
+
+#define OBJ_MERGE_META(name, meta) \
+    main_ob->merge_meta(name, meta)
+
 #endif //DEPTHGUIDE_OBJECT_MANAGER_H

src/core/pc_utility.h

@@ -15,6 +15,12 @@ enum pc_format_enum {
     PC_XYZ_RGB
 };
 
+static constexpr auto META_SOURCE_RGB = meta_hash("source_rgb"); // image_ptr
+static constexpr auto META_SOURCE_DEPTH = meta_hash("source_depth"); // image_ptr
+static constexpr auto META_SOURCE_REMAP = meta_hash("source_remap"); // image_ptr
+static constexpr auto META_DEPTH_MIN = meta_hash("depth_min"); // float
+static constexpr auto META_DEPTH_MAX = meta_hash("depth_max"); // float
+
 class generic_pc;
 
 struct pc_memory {
@@ -89,6 +95,7 @@ public:
 };
 
 using pc_ptr = generic_pc::pointer;
+static_assert(PointerContainsMeta<pc_ptr>);
 
 inline pc_ptr create_pc(size_t size, pc_format_enum fmt) {
     return generic_pc::create(size, fmt);

src/device/impl/orb_camera.cpp

@@ -1,8 +1,9 @@
 #include "orb_camera_impl.h"
-#include "core/image_utility.hpp"
-#include "core/image_utility_v2.h"
+#include "codec/image_codec.h"
 #include "core/object_manager.h"
 #include "core/utility.hpp"
+#include "image_process/process_funcs.h"
+#include "third_party/static_block.hpp"
 
 #include <boost/asio/post.hpp>
 
@@ -13,6 +14,8 @@
 #include <glm/glm.hpp>
 #include <glm/gtc/type_ptr.hpp>
 
+#include <xxhash.h>
+
 using boost::asio::post;
 
 namespace orb_camera_impl {
@@ -37,9 +40,29 @@ namespace orb_camera_impl {
         RET_ERROR_P;
     }
 
-    // convert uncompressed video frame to image_xxx
+    int ob_fmt_to_cv_type(OBFormat fmt) {
+        switch (fmt) {
+            // @formatter:off
+            case OB_FORMAT_RGB: return CV_8UC3;
+            case OB_FORMAT_Y16: return CV_16FC1;
+            // @formatter:on
+            default: {
+                RET_ERROR_E;
+            }
+        }
+    }
+
+    // convert uncompressed video frame to image_ptr
     template<typename T>
     auto video_uc_frame_to_image(const video_frame_type &frame) {
+//        auto img_size = cv::Size(frame->width(), frame->height());
+//        auto img = create_image(img_size, ob_fmt_to_cv_type(frame->format()));
+//        assert(img->size_in_bytes() == frame->dataSize());
+//        auto img_mem = img->memory(MEM_HOST);
+//        memcpy(img_mem.start_ptr(), frame->data(), img->size_in_bytes());
+//        img_mem.modified();
+//        return img;
+
         auto info = image_info_type<T>();
         info.ptr = std::shared_ptr<T>( // extend frame's lifetime
                 (T *) frame->data(), [pf = frame](void *) {});
@@ -51,6 +74,14 @@ namespace orb_camera_impl {
     }
 
     auto mjpeg_frame_to_image(const video_frame_type &frame) { // TODO: accelerate with CUDA
+//        assert(frame->format() == OB_FORMAT_MJPG);
+//        auto img_data = cv::_InputArray((uchar *) frame->data(), frame->dataSize());
+//        auto img_bgr = cv::imdecode(img_data, cv::IMREAD_UNCHANGED);
+//        auto img_rgb = create_image(img_bgr.size(), CV_8UC3);
+//        cv::cvtColor(img_bgr, img_rgb->cv_mat(), cv::COLOR_BGR2RGB);
+//        img_rgb->host_modified();
+//        return img_rgb;
+
         assert(frame->format() == OB_FORMAT_MJPG);
         auto img_data = cv::_InputArray((uchar *) frame->data(), frame->dataSize());
         auto img_bgr = cv::imdecode(img_data, cv::IMREAD_UNCHANGED);
@@ -60,33 +91,98 @@ namespace orb_camera_impl {
     }
 
     image_f32c1 depth_y16_to_mm(const image_u16c1 &y16, float scale) { // TODO: accelerate with CUDA
+//        auto f32_img = create_image(y16->size(), CV_32FC1);
+//        y16->cv_mat().convertTo(f32_img->cv_mat(), CV_32FC1, scale);
+//        f32_img->host_modified();
+//        return f32_img;
+
         auto y16_info = y16->as_host_info();
         auto f32_info = create_image_info<float1>(y16_info.size, MEM_HOST);
         y16->as_mat().convertTo(f32_info.as_mat(), CV_32FC1, scale);
         return create_image(f32_info);
     }
 
-    pc_ptr pc_frame_to_pc(const frame_type &frame) {
-        assert(frame->format() == OB_FORMAT_RGB_POINT);
-        auto size = frame->dataSize() / sizeof(OBColorPoint);
-        auto pc = create_pc(size, PC_XYZ_RGB);
-        auto pc_mem = pc->memory(MEM_HOST);
-        auto start_ptr = (pc_xyz_rgb_type *) pc_mem.start_ptr();
-        auto ptr = start_ptr;
-        for (auto k = 0; k < size; ++k) { // TODO: do conversion in CUDA
-            auto ob_ptr = (OBColorPoint *) frame->data() + k;
-            if (ob_ptr->z == 0) continue;
-            ptr->x = ob_ptr->x;
-            ptr->y = ob_ptr->y;
-            ptr->z = ob_ptr->z;
-            ptr->r = ob_ptr->r;
-            ptr->g = ob_ptr->g;
-            ptr->b = ob_ptr->b;
-            ++ptr;
+    data_type remap_codec::encode(const image_ptr &img, bool force_idr) {
+        if (force_idr) {
+            cache.clear();
         }
-        pc->shrink(ptr - start_ptr);
-        pc_mem.modified();
-        return pc;
+
+        auto writer = network_writer();
+        auto info = img->get_meta_ext<remap_info>(META_ORB_REMAP_INFO);
+        info.write_to(writer);
+        return writer.current_data();
+    }
+
+    image_ptr remap_codec::decode(const data_type &data) {
+        auto key_id = XXH64(data.start_ptr(), data.size, hash_seed);
+        if (auto iter = cache.find(key_id); iter != cache.end()) {
+            return iter->second;
+        }
+
+        auto info = remap_info();
+        auto reader = network_reader(data);
+        info.fill_from(reader);
+        assert(reader.empty());
+
+        // @formatter:off
+        cv::Mat cam_mat_cv = (cv::Mat_<float>(3, 3) <<
+                info.fx, 0.f, info.cx,
+                0, info.fy, info.cy,
+                0, 0, 1);
+        cv::Mat dist_cv = cv::Mat_<float>(info.k);
+        // @formatter:on
+
+        auto img_size = cv::Size(info.width, info.height);
+        auto points_size = cv::Size(img_size.area(), 1);
+        auto points_mat = cv::Mat(points_size, CV_32FC2);
+        for (auto iy = 0; iy < info.height; ++iy)
+            for (auto ix = 0; ix < info.width; ++ix) {
+                auto id = iy * info.width + ix;
+                points_mat.at<cv::Vec2f>(id) = cv::Vec2f(ix, iy);
+            }
+
+        auto undistort_img = create_image(img_size, CV_32FC2);
+        auto img_data = undistort_img->memory(MEM_HOST).start_ptr();
+        auto undistort_mat = cv::Mat(points_size, CV_32FC2, img_data);
+        cv::undistortPoints(points_mat, undistort_mat, cam_mat_cv, dist_cv);
+        undistort_img->host_modified();
+        undistort_img->set_meta_any(META_ORB_REMAP_INFO, info);
+        undistort_img->set_meta_any(META_IMAGE_SPECIAL_CODEC, REMAP_CODEC_NAME);
+
+        assert(!cache.contains(key_id));
+        cache.emplace(key_id, undistort_img);
+        return undistort_img;
+    }
+
+    image_ptr remap_codec::decode(const OBCameraParam &param) { // TODO: accelerate with CUDA
+        auto info = remap_info();
+
+        auto &cam_in = param.rgbIntrinsic;
+        // @formatter:off
+        info.fx = cam_in.fx; info.fy = cam_in.fy;
+        info.cx = cam_in.cx; info.cy = cam_in.cy;
+        info.width = cam_in.width; info.height = cam_in.height;
+        // @formatter:on
+
+        auto &dist = param.rgbDistortion;
+        info.k = {dist.k1, dist.k2, dist.p1, dist.p2,
+                  dist.k3, dist.k4, dist.k5, dist.k6,};
+
+        auto writer = network_writer();
+        info.write_to(writer);
+        return decode(writer.current_data());
+    }
+
+    remap_codec re_codec;
+
+    static_block {
+        auto enc_func = [](const image_ptr &img, bool force_idr) {
+            return re_codec.encode(img, force_idr);
+        };
+        auto dec_func = [](const data_type &data) {
+            return re_codec.decode(data);
+        };
+        register_image_special_codec(REMAP_CODEC_NAME, enc_func, dec_func);
     }
 
 }
@@ -116,8 +212,6 @@ orb_camera::impl::~impl() {
     if (is_capturing) {
         stop();
     }
-
-    aux_thread->join();
 }
 
 orb_camera::vi_list_ptr orb_camera::impl::create_video_info_list(const pf_list_type &pf_list) {
@@ -164,42 +258,11 @@ orb_camera::vi_list_ptr orb_camera::impl::query_d2c_info(uint32_t c_conf_index)
     return create_video_info_list(d_pf);
 }
 
-void orb_camera::impl::generate_undistort_map() { // TODO: accelerate with CUDA
+void orb_camera::impl::generate_undistort_map() {
     if (remap_name == invalid_obj_name) return;
-
     auto cam_params = pipe->getCameraParam();
-//    assert(cam_params.rgbIntrinsic == cam_params.depthIntrinsic);
-//    assert(cam_params.rgbDistortion == cam_params.depthDistortion);
-    auto cam_in = cam_params.rgbIntrinsic;
-    // @formatter:off
-    cv::Mat cam_mat_cv = (cv::Mat_<float>(3, 3) <<
-            cam_in.fx, 0.f, cam_in.cx,
-            0, cam_in.fy, cam_in.cy,
-            0, 0, 1);
-    // @formatter:on
-
-    auto cam_dist = cam_params.rgbDistortion;
-    // @formatter:off
-    cv::Mat dist_cv = (cv::Mat_<float>(8, 1) <<
-            cam_dist.k1, cam_dist.k2, cam_dist.p1, cam_dist.p2,
-            cam_dist.k3, cam_dist.k4, cam_dist.k5, cam_dist.k6);
-    // @formatter:on
-
-    auto img_size = cv::Size(cam_in.width, cam_in.height);
-    auto points_size = cv::Size(img_size.area(), 1);
-    auto points_mat = cv::Mat(points_size, CV_32FC2);
-    for (auto iy = 0; iy < cam_in.height; ++iy)
-        for (auto ix = 0; ix < cam_in.width; ++ix) {
-            auto id = iy * cam_in.width + ix;
-            points_mat.at<cv::Vec2f>(id) = cv::Vec2f(ix, iy);
-        }
-
-    auto undistort_img = create_image(img_size, CV_32FC2);
-    auto data = undistort_img->memory(MEM_HOST).start_ptr();
-    auto undistort_mat = cv::Mat(points_size, CV_32FC2, data);
-    cv::undistortPoints(points_mat, undistort_mat, cam_mat_cv, dist_cv);
-    undistort_img->host_modified();
-    OBJ_SAVE(remap_name, undistort_img);
+    remap_img = re_codec.decode(cam_params);
+    OBJ_SAVE(remap_name, remap_img);
 }
 
 bool orb_camera::impl::start(start_config conf) {
@@ -212,6 +275,10 @@ bool orb_camera::impl::start(start_config conf) {
         c_prof = c_pf_list->getProfile(conf.color.config_index);
         ob_conf->enableStream(c_prof);
         c_name = conf.color.name;
+
+        // notify frame rate
+        auto c_fmt = c_prof->as<ob::VideoStreamProfile>();
+        OBJ_PIN_META(c_name, META_REFRESH_RATE, (size_t) c_fmt->fps());
     }
 
     auto d_prof = std::shared_ptr<ob::StreamProfile>();
@@ -233,20 +300,7 @@ bool orb_camera::impl::start(start_config conf) {
 
     pipe->start(ob_conf, [this](auto frames) { frames_callback(frames); });
     is_capturing = true;
-
-    // Example says that the filter depends on the started pipeline.
-    if (conf.pc.enable) {
-        pc_filter = std::make_shared<ob::PointCloudFilter>();
-        auto params = pipe->getCameraParam();
-        pc_filter->setCameraParam(pipe->getCameraParam());
-        pc_filter->setCreatePointFormat(OB_FORMAT_RGB_POINT);
-        pc_filter->setCallBack([this](auto frame) { pc_callback(frame); });
-        pc_name = conf.pc.name;
-    }
-
-    // open a new thread to do the time-consuming calculation work
-    aux_thread = std::make_unique<std::jthread>(
-            [this] { generate_undistort_map(); });
+    generate_undistort_map();
 
     return true;
 }
@@ -272,28 +326,31 @@ void orb_camera::impl::frames_callback(const frames_type &frames) {
         d_img = depth_y16_to_mm(y16_img, d_frame->getValueScale());
     }
 
-    if (pc_filter != nullptr &&
+    auto pc_out = pc_ptr();
+    if (pc_name != invalid_obj_name &&
         c_img != nullptr && d_img != nullptr) { // depth images may not be ready
-        pc_filter->setPositionDataScaled(frames->depthFrame()->getValueScale());
-        pc_filter->pushFrame(frames);
+        auto d_conf = gen_pc_rgbd::config_direct{
+                .color_img = create_image(c_img),
+                .depth_img = create_image(d_img),
+                .remap_img = remap_img,
+                .pc_out = &pc_out,
+                .stream = stream,
+        };
+        gen_pc_rgbd::call_direct(d_conf);
+        assert(pc_out != nullptr);
     }
 
-    post(*ctx, [=, _c_name = c_name, _d_name = d_name] {
-        if (c_img != nullptr) { OBJ_SAVE(_c_name, c_img); }
-        if (d_img != nullptr) { OBJ_SAVE(_d_name, d_img); }
+    post(*ctx, [=, this] {
+        if (c_img != nullptr) { OBJ_SAVE(c_name, c_img); }
+        if (d_img != nullptr) { OBJ_SAVE(d_name, d_img); }
+        if (pc_out != nullptr) { OBJ_SAVE(pc_name, pc_out); }
     });
 }
 
-void orb_camera::impl::pc_callback(const frame_type &frame) {
-    auto pc = pc_frame_to_pc(frame);
-    OBJ_SAVE(pc_name, pc);
-}
-
 void orb_camera::impl::stop() {
     assert(is_capturing);
     pipe->stop();
     is_capturing = false;
-    pc_filter = nullptr;
 }
 
 orb_camera::pointer orb_camera::create(create_config conf) {

src/device/impl/orb_camera_impl.h

@@ -3,7 +3,9 @@
 
 #include "device/orb_camera.h"
 #include "core/image_utility.hpp"
+#include "core/image_utility_v2.h"
 #include "core/pc_utility.h"
+#include "network/binary_utility.hpp"
 
 #include <boost/asio/io_context.hpp>
 
@@ -23,7 +25,48 @@ namespace orb_camera_impl {
     // new_value (mm) = old_value * scale
     image_f32c1 depth_y16_to_mm(const image_u16c1 &y16, float scale);
 
-    pc_ptr pc_frame_to_pc(const frame_type &frame);
+    struct remap_codec {
+
+        using cache_map_type =
+                std::unordered_map<size_t, image_ptr>;
+        cache_map_type cache;
+
+        struct remap_info {
+            static constexpr auto num_distort = 8;
+            float fx, fy;
+            float cx, cy;
+            std::array<float, num_distort> k; // k[1..2], p[1..2], k[3..6]
+            int16_t width, height;
+
+            template<typename ReaderType>
+            void fill_from(ReaderType &reader) {
+                reader >> fx >> fy >> cx >> cy;
+                reader >> k;
+                reader >> width >> height;
+            }
+
+            template<typename WriterType>
+            void write_to(WriterType &writer) {
+                writer << fx << fy << cx << cy;
+                writer << k;
+                writer << width << height;
+            }
+        };
+
+        static constexpr auto hash_seed = 0;
+        static constexpr meta_key_type META_ORB_REMAP_INFO = meta_hash("orbbec_remap_info");
+
+        data_type encode(const image_ptr &img, bool force_idr);
+
+        image_ptr decode(const data_type &data);
+
+        image_ptr decode(const OBCameraParam &param);
+
+    };
+
+    static constexpr size_t REMAP_CODEC_NAME = meta_hash("orbbec_remap_codec");
+
+    extern remap_codec re_codec;
 
 }
 
@@ -33,7 +76,6 @@ using namespace orb_camera_impl;
 struct orb_camera::impl {
 
     std::shared_ptr<ob::Pipeline> pipe;
-    std::shared_ptr<ob::PointCloudFilter> pc_filter;
     smart_cuda_stream *stream = nullptr;
     io_context *ctx = nullptr;
 
@@ -49,7 +91,7 @@ struct orb_camera::impl {
     obj_name_type pc_name = invalid_obj_name;
 
     obj_name_type remap_name = invalid_obj_name;
-    std::unique_ptr<std::jthread> aux_thread;
+    image_ptr remap_img = nullptr;
 
     static constexpr auto align_mode = ALIGN_D2C_HW_MODE;
 
@@ -77,8 +119,6 @@ struct orb_camera::impl {
 
     using frame_type = std::shared_ptr<ob::Frame>;
 
-    void pc_callback(const frame_type &frame);
-
     void stop();
 
 };

src/device/impl/orb_camera_ui.cpp

@@ -132,10 +132,10 @@ void orb_camera_ui::impl::show_config() {
             ImGui::EndCombo();
         }
 
-        ImGui::SameLine();
-        if (ImGui::Checkbox("Point Cloud", &enable_pc)) {
-            refresh_d_conf_list();
-        }
+//        ImGui::SameLine();
+//        if (ImGui::Checkbox("Point Cloud", &enable_pc)) {
+//            refresh_d_conf_list();
+//        }
     }
 }
 
@@ -180,14 +180,14 @@ void orb_camera_ui::impl::show() {
         auto size = d_img->size();
         ImGui::Text("Depth Stream: %dx%d / %.2fms", size.width, size.height, d_interval);
     }
-    if (enable_pc) {
-        auto pc = OBJ_QUERY(pc_ptr, cam_s_conf.pc.name);
-        auto interval = OBJ_STATS(cam_s_conf.pc.name).save_interval;
-        if (pc != nullptr) {
-            auto num = pc->size();
-            ImGui::Text("Point Cloud: %ld / %.2fms", num, interval);
-        }
-    }
+//    if (enable_pc) { // TODO: remove this
+//        auto pc = OBJ_QUERY(pc_ptr, cam_s_conf.pc.name);
+//        auto interval = OBJ_STATS(cam_s_conf.pc.name).save_interval;
+//        if (pc != nullptr) {
+//            auto num = pc->size();
+//            ImGui::Text("Point Cloud: %ld / %.2fms", num, interval);
+//        }
+//    }
 }
 
 orb_camera_ui::orb_camera_ui(create_config conf)

src/image_process/cuda_impl/pc_generate.cu

@@ -1,6 +1,7 @@
 #include "pc_generate.cuh"
 #include "kernel_utility.cuh"
 
+template<bool UseMask = true>
 __global__ void gen_rgb_with_mask(image_type_v2<uchar3> color,
                                   image_type_v2<float1> depth,
                                   image_type_v2<uchar1> mask,
@@ -22,7 +23,9 @@ __global__ void gen_rgb_with_mask(image_type_v2<uchar3> color,
             out.at(id)->z = 0; \
             continue
 
-            if (mask.at(idy, idx)->x == 0) { IGNORE_PIX; }
+            if constexpr (UseMask) {
+                if (mask.at(idy, idx)->x == 0) { IGNORE_PIX; }
+            }
 
             auto pix_depth = depth.at(idy, idx)->x;
             if (pix_depth == 0) { IGNORE_PIX; }
@@ -63,5 +66,19 @@ namespace gen_pc {
                 color, depth, mask, remap, out);
     }
 
+    void call_rgbd(image_type_v2<uchar3> color, image_type_v2<float1> depth,
+                   image_type_v2<float2> remap, pc_type_rgb out, cudaStream_t stream) {
+        assert(depth.width == color.width);
+        assert(remap.width == color.width);
+        assert(depth.height == color.height);
+        assert(remap.height == color.height);
+        assert(out.num == color.width * color.height);
+
+        auto [grid_dim, block_dim] = get_kernel_size(color.width, color.height);
+        static constexpr auto func_type = gen_rgb_with_mask<false>;
+        func_type<<<grid_dim, block_dim, 0, stream>>>(
+                color, depth, {}, remap, out);
+    }
+
 }
 

src/image_process/cuda_impl/pc_generate.cuh

@@ -10,6 +10,9 @@ namespace gen_pc {
                             image_type_v2<uchar1> mask, image_type_v2<float2> remap,
                             pc_type_rgb out, cudaStream_t stream);
 
+    void call_rgbd(image_type_v2<uchar3> color, image_type_v2<float1> depth,
+                   image_type_v2<float2> remap, pc_type_rgb out, cudaStream_t stream);
+
 }
 
 #endif //DEPTHGUIDE_PC_GENERATE_CUH

src/image_process/impl/process_funcs.cpp

@@ -7,8 +7,8 @@
 namespace gen_pc_rgb_with_mask {
 
     void call(config conf) {
-        auto color_img = OBJ_QUERY(image_u8c3, conf.color_name);
-        auto depth_img = OBJ_QUERY(image_f32c1, conf.depth_name);
+        auto color_img = to_image(conf.color_name);
+        auto depth_img = to_image(conf.depth_name);
         auto mask_img = OBJ_QUERY(image_ptr, conf.mask_name);
         auto remap_img = OBJ_QUERY(image_ptr, conf.remap_name);
 
@@ -16,8 +16,8 @@ namespace gen_pc_rgb_with_mask {
         assert(color_img != nullptr && depth_img != nullptr);
 
         auto stream = conf.stream;
-        auto color = color_img->as_cuda(stream);
-        auto depth = depth_img->as_cuda(stream);
+        auto color = color_img->cuda<uchar3>(stream);
+        auto depth = depth_img->cuda<float1>(stream);
         auto mask = mask_img->cuda<uchar1>(stream);
         auto remap = remap_img->cuda<float2>(stream);
         auto out_pc = create_pc(color_img->size().area(), PC_XYZ_RGB);
@@ -26,39 +26,157 @@ namespace gen_pc_rgb_with_mask {
         gen_pc::call_rgb_with_mask(
                 color, depth, mask, remap, pc_info, stream->cuda);
         out_pc->cuda_modified(stream);
+
+        // shrink invisible points
+        auto shrink_conf = shrink_pc_zero_depth::config_direct{
+                .in = out_pc, .out = &out_pc, .stream = conf.stream,
+        };
+        shrink_pc_zero_depth::call_direct(shrink_conf);
+
+        out_pc->set_meta_any(META_SOURCE_RGB, color_img);
+        out_pc->set_meta_any(META_SOURCE_DEPTH, depth_img);
+        out_pc->set_meta_any(META_SOURCE_REMAP, remap_img);
+        OBJ_SAVE(conf.pc_out_name, out_pc);
+    }
+
+}
+
+namespace gen_pc_rgbd {
+
+    void call(config conf) {
+        auto out_pc = pc_ptr();
+        auto d_conf = config_direct();
+        d_conf.color_img = to_image(conf.color_name);
+        d_conf.depth_img = to_image(conf.depth_name);
+        d_conf.remap_img = to_image(conf.remap_name);
+        d_conf.pc_out = &out_pc;
+        d_conf.stream = conf.stream;
+        call_direct(d_conf);
         OBJ_SAVE(conf.pc_out_name, out_pc);
     }
 
+    void call_direct(const config_direct &conf) {
+        if (conf.remap_img == nullptr) return;
+        if (conf.color_img == nullptr) return;
+        if (conf.depth_img == nullptr) return;
+
+        auto stream = conf.stream;
+        auto color = conf.color_img->cuda<uchar3>(stream);
+        auto depth = conf.depth_img->cuda<float1>(stream);
+        auto remap = conf.remap_img->cuda<float2>(stream);
+        auto out_pc = create_pc(conf.color_img->size().area(), PC_XYZ_RGB);
+        auto pc_info = out_pc->cuda<pc_xyz_rgb_type>(stream);
+
+        gen_pc::call_rgbd(color, depth, remap, pc_info, stream->cuda);
+        out_pc->cuda_modified(stream);
+
+        // shrink invisible points
+        auto shrink_conf = shrink_pc_zero_depth::config_direct{
+                .in = out_pc, .out = &out_pc, .stream = conf.stream,
+        };
+        shrink_pc_zero_depth::call_direct(shrink_conf);
+
+        if (auto val = conf.color_img->get_meta_any(META_REFRESH_RATE); !val.empty()) {
+            out_pc->set_meta_any(META_REFRESH_RATE, boost::any_cast<size_t>(val));
+        }
+        out_pc->set_meta_any(META_SOURCE_RGB, conf.color_img);
+        out_pc->set_meta_any(META_SOURCE_DEPTH, conf.depth_img);
+        out_pc->set_meta_any(META_SOURCE_REMAP, conf.remap_img);
+        *conf.pc_out = out_pc;
+    }
+
+}
+
+namespace concatenate_image {
+
+    void call(config conf) {
+        auto ret = image_ptr();
+        auto d_conf = config_direct();
+        d_conf.left_img = to_image(conf.left_img);
+        d_conf.right_img = to_image(conf.right_img);
+        d_conf.out_img = &ret;
+        d_conf.stream = conf.stream;
+        call_direct(d_conf);
+        OBJ_SAVE(conf.out_name, ret);
+    }
+
+    void call_direct(const config_direct &conf) {
+        auto img_left = conf.left_img;
+        auto img_right = conf.right_img;
+        assert(img_left->basic_info() == img_right->basic_info());
+
+        auto ret_width = img_left->width() + img_right->width();
+        auto ret_height = std::max(img_left->height(), img_right->height());
+        auto ret_size = cv::Size(ret_width, ret_height);
+        auto ret = create_image(
+                ret_size, img_left->cv_type(), img_left->pixel_format());
+
+        auto ret_mem = ret->memory(MEM_CUDA, conf.stream);
+        auto left_mem = img_left->memory(MEM_CUDA, conf.stream);
+        CUDA_API_CHECK(cudaMemcpy2DAsync(ret_mem.start_ptr(), ret_mem.pitch,
+                                         left_mem.start_ptr(), left_mem.pitch,
+                                         left_mem.width, left_mem.height,
+                                         cudaMemcpyDeviceToDevice, conf.stream->cuda));
+
+        auto right_mem = img_right->memory(MEM_CUDA, conf.stream);
+        auto right_ptr = (uint8_t *) ret_mem.start_ptr() + left_mem.width;
+        CUDA_API_CHECK(cudaMemcpy2DAsync(right_ptr, ret_mem.pitch,
+                                         right_mem.start_ptr(), right_mem.pitch,
+                                         right_mem.width, right_mem.height,
+                                         cudaMemcpyDeviceToDevice, conf.stream->cuda));
+
+        ret->cuda_modified(conf.stream);
+        ret->set_meta_any(META_STEREO_LEFT, img_left);
+        ret->set_meta_any(META_STEREO_RIGHT, img_right);
+        *conf.out_img = ret;
+    }
+
 }
 
 namespace shrink_pc_zero_depth {
 
     template<typename PointT>
-    void call_impl(config conf, const pc_ptr &pc_in) {
-        auto stream = conf.stream;
+    pc_ptr call_impl(const pc_ptr &pc_in, smart_cuda_stream *stream) {
         auto in_info = pc_in->cuda<PointT>(stream);
         auto pc_out = create_pc(pc_in->size(), pc_in->format());
         size_t out_num = 0;
         pc_filter::shrink_zero_depth(in_info, pc_out->cuda<PointT>(stream),
-                                     &out_num, conf.stream->cuda);
+                                     &out_num, stream->cuda);
         pc_out->shrink(out_num);
         pc_out->cuda_modified(stream);
-        OBJ_SAVE(conf.out_name, pc_out);
+        return pc_out;
     }
 
     void call(config conf) {
-        auto pc_in = OBJ_QUERY(pc_ptr, conf.in_name);
-        if (pc_in == nullptr) return;
-
-        switch (pc_in->format()) {
-            // @formatter:off
-            case PC_XYZ: { call_impl<pc_xyz_type>(conf, pc_in); break; }
-            case PC_XYZ_RGB: { call_impl<pc_xyz_rgb_type>(conf, pc_in); break; }
-            // @formatter:off
+        auto pc_out = pc_ptr();
+        auto d_conf = config_direct();
+        d_conf.in = OBJ_QUERY(pc_ptr, conf.in_name);
+        d_conf.out = &pc_out;
+        d_conf.stream = conf.stream;
+        call_direct(d_conf);
+        OBJ_SAVE(conf.out_name, pc_out);
+    }
+
+    void call_direct(const config_direct &conf) {
+        auto pc_out = pc_ptr();
+        switch (conf.in->format()) {
+            case PC_XYZ: {
+                pc_out = call_impl<pc_xyz_type>(conf.in, conf.stream);
+                break;
+            }
+            case PC_XYZ_RGB: {
+                pc_out = call_impl<pc_xyz_rgb_type>(conf.in, conf.stream);
+                break;
+            }
             default: {
                 RET_ERROR;
             }
         }
+
+        // copy meta info
+        pc_out->merge_from(conf.in.get());
+
+        *conf.out = pc_out;
     }
 
 }

src/image_process/process_funcs.h

@@ -2,6 +2,8 @@
 #define DEPTHGUIDE_PROCESS_FUNCS_H
 
 #include "core/cuda_helper.hpp"
+#include "core/image_utility_v2.h"
+#include "core/pc_utility.h"
 #include "core/object_manager.h"
 
 namespace gen_pc_rgb_with_mask {
@@ -19,6 +21,52 @@ namespace gen_pc_rgb_with_mask {
 
 }
 
+namespace gen_pc_rgbd {
+
+    struct config {
+        obj_name_type color_name;
+        obj_name_type depth_name;
+        obj_name_type remap_name;
+        obj_name_type pc_out_name;
+        smart_cuda_stream *stream;
+    };
+
+    void call(config conf);
+
+    struct config_direct {
+        image_ptr color_img;
+        image_ptr depth_img;
+        image_ptr remap_img;
+        pc_ptr *pc_out;
+        smart_cuda_stream *stream;
+    };
+
+    void call_direct(const config_direct &conf);
+
+}
+
+namespace concatenate_image {
+
+    struct config {
+        obj_name_type left_img;
+        obj_name_type right_img;
+        obj_name_type out_name;
+        smart_cuda_stream *stream;
+    };
+
+    void call(config conf);
+
+    struct config_direct {
+        image_ptr left_img;
+        image_ptr right_img;
+        image_ptr *out_img;
+        smart_cuda_stream *stream;
+    };
+
+    void call_direct(const config_direct &conf);
+
+}
+
 namespace shrink_pc_zero_depth {
 
     struct config {
@@ -29,6 +77,14 @@ namespace shrink_pc_zero_depth {
 
     void call(config conf);
 
+    struct config_direct {
+        pc_ptr in;
+        pc_ptr *out;
+        smart_cuda_stream *stream;
+    };
+
+    void call_direct(const config_direct &conf);
+
 }
 
 

src/impl/apps/app_selector/app_selector.cpp

@@ -2,7 +2,9 @@
 #include "impl/apps/debug/app_debug.h"
 #include "impl/apps/endo_guide/endo_guide.h"
 #include "impl/apps/depth_guide/depth_guide.h"
+#include "impl/apps/depth_guide_v2/depth_guide_v2.h"
 #include "impl/apps/remote_ar/remote_ar.h"
+#include "impl/apps/scene_player/scene_player.h"
 #include "impl/apps/tiny_player/tiny_player.h"
 
 #include <GLFW/glfw3.h>
@@ -50,10 +52,14 @@ void app_selector::load_app(const std::string &conf_path) {
     auto app = std::unique_ptr<app_base>();
     if (app_name == "depth_guide") {
         app = std::make_unique<app_depth_guide>(create_conf);
+    } else if (app_name == "depth_guide_v2") {
+        app = std::make_unique<app_depth_guide_v2>(create_conf);
     } else if (app_name == "remote_ar") {
         app = std::make_unique<app_remote_ar>(create_conf);
     } else if (app_name == "tiny_player") {
         app = std::make_unique<app_tiny_player>(create_conf);
+    } else if (app_name == "scene_player") {
+        app = std::make_unique<app_scene_player>(create_conf);
     } else if (app_name == "endo_guide") {
         app = std::make_unique<app_endo_guide>(create_conf);
     } else if (app_name == "debug") {

src/impl/apps/depth_guide/depth_guide.cpp

@@ -21,6 +21,10 @@ app_depth_guide::app_depth_guide(const create_config &_conf) {
     OBJ_SAVE(pc_obj_raw, pc_ptr());
     OBJ_SAVE(pc_obj, pc_ptr());
 
+    // initialize meta
+    OBJ_PIN_META(img_out, META_SERIES_NAME, meta_hash("img_out"));
+    OBJ_PIN_META(pc_raw, META_SERIES_NAME, meta_hash("pc_raw"));
+
     OBJ_SIG(img_obj_mask)->connect([](auto _) {
         gen_pc_rgb_with_mask::call(
                 {img_color, img_depth, img_obj_mask,

src/impl/apps/depth_guide_v2/depth_guide_v2.cpp

@@ -0,0 +1,97 @@
+#include "depth_guide_v2.h"
+#include "core/imgui_utility.hpp"
+#include "image_process/process_funcs.h"
+
+app_depth_guide_v2::app_depth_guide_v2(const create_config &_conf) {
+    conf = _conf;
+
+    // initialize object manager
+    OBJ_SAVE(img_color, image_u8c3());
+    OBJ_SAVE(img_depth, image_f32c1());
+    OBJ_SAVE(img_remap, image_ptr());
+    OBJ_SAVE(pc_raw, pc_ptr());
+    OBJ_SAVE(scene_out, scene_ptr());
+
+    // initialize meta
+    OBJ_PIN_META(pc_raw, META_SERIES_NAME, meta_hash("pc_raw"));
+    OBJ_PIN_META(img_remap, META_SERIES_NAME, meta_hash("img_remap"));
+    OBJ_PIN_META(pc_raw, META_DEPTH_MIN, (float) 0.0f); // TODO: make adjustable
+    OBJ_PIN_META(pc_raw, META_DEPTH_MAX, (float) 3000.0f);
+
+    OBJ_SIG(img_color)->connect([](auto _) {
+        gen_pc_rgbd::call(
+                {img_color, img_depth, img_remap,
+                 pc_raw, default_cuda_stream});
+    });
+    OBJ_SIG(pc_raw)->connect([this](auto _) {
+        aug_manager->update(aug_helper->get_camera());
+    });
+
+    // initialize modules
+    auto orb_cam_conf = orb_camera_ui::create_config{
+            .cf_name = img_color, .df_name = img_depth,
+            .remap_name = img_remap,
+            .ctx = conf.asio_ctx, .stream = default_cuda_stream,
+    };
+    orb_cam = std::make_unique<orb_camera_ui>(orb_cam_conf);
+
+    auto aug_conf = augment_manager_v2::create_config{
+            .scene_name = scene_out,
+            .stream = default_cuda_stream,
+    };
+    aug_conf.item_list.push_back(
+            {.type = augment_manager_v2::ITEM_PC,
+                    .disp_name = "PC Raw", .name = pc_raw});
+    aug_manager = std::make_unique<augment_manager_v2>(aug_conf);
+
+    auto view_conf = camera_augment_helper_v2::create_config{
+            .camera = camera_augment_helper_v2::create_config::freedom_camera_config{},
+            .manager = aug_manager.get(),
+            .ctx = conf.asio_ctx,
+    };
+    aug_helper = std::make_unique<camera_augment_helper_v2>(view_conf);
+    aug_helper->set_interactive(true);
+
+    auto out_conf = image_streamer::create_config{
+            .img_name = scene_out, .encode_scene = true,
+            .asio_ctx = conf.asio_ctx,
+            .cuda_ctx = conf.cuda_ctx, .stream = default_cuda_stream
+    };
+    out_streamer = std::make_unique<image_streamer>(out_conf);
+}
+
+void app_depth_guide_v2::show_ui() {
+    if (ImGui::Begin("Depth Guide Control")) {
+        ImGui::PushItemWidth(200);
+
+        if (ImGui::CollapsingHeader("Camera")) {
+            auto id_guard = imgui_id_guard("camera");
+            orb_cam->show();
+        }
+
+        if (ImGui::CollapsingHeader("Streamer")) {
+            auto id_guard = imgui_id_guard("streamer");
+            out_streamer->show();
+        }
+
+        if (ImGui::CollapsingHeader("Augment")) {
+            {
+                ImGui::SeparatorText("Scene");
+                auto id_guard = imgui_id_guard("augment_scene");
+                aug_manager->show();
+            }
+            {
+                ImGui::SeparatorText("Camera");
+                auto id_guard = imgui_id_guard("augment_camera");
+                aug_helper->show();
+            }
+        }
+
+        ImGui::PopItemWidth();
+    }
+    ImGui::End();
+}
+
+void app_depth_guide_v2::render_background() {
+    aug_helper->render();
+}

src/impl/apps/depth_guide_v2/depth_guide_v2.h

@@ -0,0 +1,52 @@
+#ifndef DEPTHGUIDE_DEPTH_GUIDE_V2_H
+#define DEPTHGUIDE_DEPTH_GUIDE_V2_H
+
+#include "core/object_manager.h"
+#include "device/orb_camera_ui.h"
+#include "module/augment_manager_v2.h"
+#include "module/camera_augment_helper_v2.h"
+#include "module/image_streamer.h"
+#include "impl/app_base.h"
+
+#include <boost/asio/io_context.hpp>
+
+class app_depth_guide_v2 : public app_base {
+public:
+
+    explicit app_depth_guide_v2(const create_config &conf);
+
+    ~app_depth_guide_v2() override = default;
+
+    const char *window_name() override { return "DepthGuide V2.-1"; }
+
+    void show_ui() override;
+
+    void render_background() override;
+
+private:
+
+    enum obj_names : object_manager::name_type {
+
+        // images from device
+        img_color, img_depth, img_remap,
+
+        // point cloud from device
+        pc_raw,
+
+        // output image
+        scene_out,
+    };
+
+    create_config conf;
+
+    // modules
+    std::unique_ptr<orb_camera_ui> orb_cam;
+    std::unique_ptr<image_streamer> out_streamer;
+
+    std::unique_ptr<augment_manager_v2> aug_manager;
+    std::unique_ptr<camera_augment_helper_v2> aug_helper;
+
+};
+
+
+#endif //DEPTHGUIDE_DEPTH_GUIDE_V2_H

src/impl/apps/scene_player/scene_player.cpp

@@ -0,0 +1,61 @@
+#include "scene_player.h"
+#include "core/imgui_utility.hpp"
+
+app_scene_player::app_scene_player(const create_config &_conf) {
+    conf = _conf;
+
+    // initialize object manager
+    OBJ_SAVE(scene_in, scene_ptr());
+
+    // initialize modules
+    auto in_conf = image_player::create_config{
+            .img_name = scene_in, .ctx = conf.asio_ctx,
+            .decode_scene = true, .stream = default_cuda_stream,
+    };
+    in_streamer = std::make_unique<image_player>(in_conf);
+
+    auto aug_conf = augment_manager_v2::create_config{
+            .scene_name = scene_in, .player_mode = true,
+            .stream = default_cuda_stream,
+    };
+    aug_manager = std::make_unique<augment_manager_v2>(aug_conf);
+
+    auto view_conf = camera_augment_helper_v2::create_config{
+            .camera = camera_augment_helper_v2::create_config::freedom_camera_config{},
+            .manager = aug_manager.get(),
+            .ctx = conf.asio_ctx,
+    };
+    aug_helper = std::make_unique<camera_augment_helper_v2>(view_conf);
+    aug_helper->set_interactive(true);
+}
+
+void app_scene_player::show_ui() {
+    if (ImGui::Begin("Depth Guide Control")) {
+        ImGui::PushItemWidth(200);
+
+        if (ImGui::CollapsingHeader("Player")) {
+            auto id_guard = imgui_id_guard("Player");
+            in_streamer->show();
+        }
+
+        if (ImGui::CollapsingHeader("Augment")) {
+            {
+                ImGui::SeparatorText("Scene");
+                auto id_guard = imgui_id_guard("augment_scene");
+                aug_manager->show();
+            }
+            {
+                ImGui::SeparatorText("Camera");
+                auto id_guard = imgui_id_guard("augment_camera");
+                aug_helper->show();
+            }
+        }
+
+        ImGui::PopItemWidth();
+    }
+    ImGui::End();
+}
+
+void app_scene_player::render_background() {
+    aug_helper->render();
+}

src/impl/apps/scene_player/scene_player.h

@@ -0,0 +1,41 @@
+#ifndef DEPTHGUIDE_SCENE_PLAYER_H
+#define DEPTHGUIDE_SCENE_PLAYER_H
+
+#include "core/object_manager.h"
+#include "module/augment_manager_v2.h"
+#include "module/camera_augment_helper_v2.h"
+#include "module/image_player.h"
+#include "impl/app_base.h"
+
+#include <boost/asio/io_context.hpp>
+
+class app_scene_player : public app_base {
+public:
+
+    explicit app_scene_player(const create_config &conf);
+
+    ~app_scene_player() override = default;
+
+    const char *window_name() override { return "ScenePlayer V1.-1"; }
+
+    void show_ui() override;
+
+    void render_background() override;
+
+private:
+
+    enum obj_names : object_manager::name_type {
+        // input scene
+        scene_in,
+    };
+
+    create_config conf;
+
+    // modules
+    std::unique_ptr<image_player> in_streamer;
+    std::unique_ptr<augment_manager_v2> aug_manager;
+    std::unique_ptr<camera_augment_helper_v2> aug_helper;
+};
+
+
+#endif //DEPTHGUIDE_SCENE_PLAYER_H

src/module/augment_manager_v2.h

@@ -35,6 +35,11 @@ public:
 
         using item_list_type = std::vector<item_type>;
         item_list_type item_list;
+        obj_name_type scene_name = invalid_obj_name;
+
+        // if player mode is on,
+        // scene name will be used as input rather than output
+        bool player_mode = false;
 
         using sophiar_conn_type = sophiar::local_connection;
         sophiar_conn_type *sophiar_conn = nullptr;
@@ -54,10 +59,12 @@ public:
 
     void show();
 
+    void update(const camera_info &info);
+
     void render(const camera_info &info);
 
     using render_sig_type =
-            boost::signals2::signal<void(scene_render_info &)>;
+            boost::signals2::signal<void(scene_ptr)>;
     render_sig_type pre_render_sig;
     render_sig_type post_render_sig;
 

src/module/camera_augment_helper_v2.h

@@ -62,6 +62,8 @@ public:
 
     void set_interactive(bool flag);
 
+    camera_info get_camera();
+
 private:
     struct impl;
     std::unique_ptr<impl> pimpl;

src/module/image_player.h

@@ -19,6 +19,8 @@ public:
         obj_name_type ext_name = invalid_obj_name; // data_type, see image_streamer
         io_context *ctx = nullptr;
 
+        bool decode_scene = false;
+
         // for decoder
         smart_cuda_stream *stream = nullptr;
     };

src/module/image_streamer.h

@@ -19,6 +19,8 @@ public:
         // receiver must do the same parsing work for proper work
         obj_name_type ext_name = invalid_obj_name; // data_type
 
+        bool encode_scene = false;
+
         std::optional<int> frame_rate;
         io_context *asio_ctx = nullptr;
 

src/module/impl/augment_manager_v2.cpp

@@ -25,7 +25,8 @@ void augment_manager_v2::impl::item_store_type::update_transform(impl *_pimpl) {
     }
 }
 
-augment_manager_v2::impl::impl(const create_config &conf) {
+augment_manager_v2::impl::impl(const create_config &_conf) {
+    conf = _conf;
     sophiar_conn = conf.sophiar_conn;
     stream = conf.stream;
     for (auto &item: conf.item_list) {
@@ -42,13 +43,26 @@ augment_manager_v2::impl::impl(const create_config &conf) {
     glGetFloatv(GL_POINT_SIZE_GRANULARITY, &ps_interval);
 }
 
-void augment_manager_v2::impl::render(const camera_info &info) {
-    ren_info.camera = info;
-    ren_info.light.follow_camera = enable_light_follow_camera;
-    ren_info.light.direction = to_vec3(light_direction);
-    ren_info.stream = stream;
+void augment_manager_v2::impl::update(const camera_info &info, bool no_commit) {
+    auto ren_info = std::make_shared<scene_render_info>();
+
+    // load from scene_name in player mode
+    if (conf.player_mode) {
+        auto scene = OBJ_QUERY(scene_ptr, conf.scene_name);
+        if (scene != nullptr) {
+            *ren_info = *scene;
+        }
+        no_commit = true;
+    }
+
+//    if (!conf.player_mode) {
+        ren_info->camera = info;
+        ren_info->light.follow_camera = enable_light_follow_camera;
+        ren_info->light.direction = to_vec3(light_direction);
+//    }
+
+    ren_info->stream = stream;
 
-    ren_info.items.clear();
     for (auto &item: item_list) {
         if (!item.visible) continue;
 
@@ -91,12 +105,19 @@ void augment_manager_v2::impl::render(const camera_info &info) {
                 RET_ERROR;
             }
         }
-        ren_info.items.push_back(ren_item);
+        ren_info->items.push_back(ren_item);
     }
 
     q_this->pre_render_sig(ren_info);
-    render_scene(ren_info);
-    q_this->post_render_sig(ren_info);
+    last_scene_info = ren_info;
+    if (!no_commit && conf.scene_name != invalid_obj_name) {
+        OBJ_SAVE(conf.scene_name, ren_info);
+    }
+}
+
+void augment_manager_v2::impl::render() {
+    render_scene(last_scene_info);
+    q_this->post_render_sig(last_scene_info);
 }
 
 void augment_manager_v2::impl::show() {
@@ -147,8 +168,13 @@ augment_manager_v2::augment_manager_v2(const create_config &conf)
 
 augment_manager_v2::~augment_manager_v2() = default;
 
+void augment_manager_v2::update(const camera_info &info) {
+    pimpl->update(info);
+}
+
 void augment_manager_v2::render(const camera_info &info) {
-    pimpl->render(info);
+    pimpl->update(info, true); // no commit
+    pimpl->render();
 }
 
 void augment_manager_v2::show() {

src/module/impl/augment_manager_v2_impl.h

@@ -40,11 +40,14 @@ struct augment_manager_v2::impl {
     };
 
     augment_manager_v2 *q_this = nullptr;
+    create_config conf;
 
     using item_list_type =
             std::vector<item_store_type>;
     item_list_type item_list;
 
+    scene_ptr last_scene_info;
+
     using sophiar_conn_type = create_config::sophiar_conn_type;
     sophiar_conn_type *sophiar_conn = nullptr;
 
@@ -56,9 +59,6 @@ struct augment_manager_v2::impl {
 
     vgm::Vec3 light_direction = {0.0f, 0.0f, -1.0f};
 
-    // cached to prevent frequent memory operation
-    scene_render_info ren_info;
-
     // low-level GL_POINT info
     struct {
         GLfloat min, max;
@@ -67,7 +67,9 @@ struct augment_manager_v2::impl {
 
     explicit impl(const create_config &conf);
 
-    void render(const camera_info &info);
+    void update(const camera_info &info, bool no_commit = false);
+
+    void render();
 
     void show();
 };

src/module/impl/camera_augment_helper_v2.cpp

@@ -79,7 +79,7 @@ camera_augment_helper_v2::impl::impl(const create_config &conf) {
     ctx = conf.ctx;
 
     pre_render_conn = manager->pre_render_sig.connect(
-            [this](auto &info) { pre_render_slot(info); });
+            [this](auto info) { pre_render_slot(info); });
 
     switch (conf.camera.index()) {
         case 1: {
@@ -143,15 +143,18 @@ void camera_augment_helper_v2::impl::update_transform() {
     }
 }
 
-void camera_augment_helper_v2::impl::render() {
+void camera_augment_helper_v2::impl::update_camera() {
     update_transform();
-    auto ren_info = camera_info{
+    last_camera = camera_info{
             .transform = transform, .fov = fov,
             .near = near, .far = far
     };
+}
+
+void camera_augment_helper_v2::impl::render() {
     if (!is_missing || ignore_missing) {
-        manager->render(ren_info);
-        last_camera = ren_info;
+        update_camera();
+        manager->render(last_camera);
         last_vp_size = query_viewport_size();
     }
 }
@@ -273,7 +276,7 @@ void camera_augment_helper_v2::impl::update_cursor_coordinate() {
     cursor_pos = w_pos;
 }
 
-void camera_augment_helper_v2::impl::cursor_guide_slot(scene_render_info &info) {
+void camera_augment_helper_v2::impl::cursor_guide_slot(const scene_ptr &info) {
     assert(enable_cursor_guide);
 
     if (cursor_symbol == nullptr) [[unlikely]] {
@@ -288,7 +291,7 @@ void camera_augment_helper_v2::impl::cursor_guide_slot(scene_render_info &info)
     auto rec_info = scene_render_info::custom_info{
             .func = [this] { update_cursor_coordinate(); },
     };
-    info.items.push_back({.info = rec_info});
+    info->items.push_back({.info = rec_info});
 
     if (cursor_pos.has_value()) {
         auto color = glm::vec3(0.0f, 1.0f, 0.0f); // TODO: make this adjustable
@@ -297,12 +300,12 @@ void camera_augment_helper_v2::impl::cursor_guide_slot(scene_render_info &info)
                 .mesh = cursor_symbol,
                 .material = {.ambient = color * amb_factor, .diffuse = color,},
         };
-        info.items.push_back(
+        info->items.push_back(
                 {.info = item_info, .transform = glm::translate(*cursor_pos),});
     }
 }
 
-void camera_augment_helper_v2::impl::pre_render_slot(scene_render_info &info) {
+void camera_augment_helper_v2::impl::pre_render_slot(const scene_ptr &info) {
     if (!is_interactive) return;
     if (enable_cursor_guide) {
         cursor_guide_slot(info);
@@ -333,4 +336,9 @@ void camera_augment_helper_v2::show() {
 
 void camera_augment_helper_v2::set_interactive(bool flag) {
     pimpl->set_interactive(flag);
+}
+
+camera_info camera_augment_helper_v2::get_camera() {
+    pimpl->update_camera();
+    return pimpl->last_camera;
 }

src/module/impl/camera_augment_helper_v2_impl.h

@@ -103,6 +103,8 @@ struct camera_augment_helper_v2::impl {
 
     void update_transform();
 
+    void update_camera();
+
     void render();
 
     void render_image(output_config conf);
@@ -115,9 +117,9 @@ struct camera_augment_helper_v2::impl {
 
     void update_cursor_coordinate();
 
-    void cursor_guide_slot(scene_render_info &info);
+    void cursor_guide_slot(const scene_ptr &info);
 
-    void pre_render_slot(scene_render_info &info);
+    void pre_render_slot(const scene_ptr &info);
 
     void set_interactive(bool flag);
 

src/module/impl/image_player.cpp

@@ -1,6 +1,13 @@
 #include "image_player_impl.h"
 #include "core/imgui_utility.hpp"
 
+void image_player::impl::create_scene_decoder() {
+    auto scene_dec_conf = scene_decoder::create_config{
+            .scene_name = conf.img_name, .stream = conf.stream,
+    };
+    s_dec = std::make_unique<scene_decoder>(scene_dec_conf);
+}
+
 void image_player::impl::create_decoder() {
     switch (chose_decoder_type) {
         case DECODER_NVDEC: {
@@ -63,6 +70,12 @@ void image_player::impl::frame_callback(frame_info frame) {
 }
 
 void image_player::impl::decode_image(const frame_info &frame) {
+    SPDLOG_DEBUG("Decode image {}.", frame.frame_id);
+    if (conf.decode_scene) { // TODO: ugly hacked
+        s_dec->decode(frame.data);
+        return;
+    }
+
     switch (chose_decoder_type) {
         case DECODER_NVDEC: {
             dec_nvdec->decode(frame);
@@ -87,7 +100,11 @@ void image_player::impl::start() {
         frame_queue = frame_queue_type::create(queue_conf);
     }
 
-    create_decoder();
+    if (conf.decode_scene) {
+        create_scene_decoder();
+    } else {
+        create_decoder();
+    }
     create_receiver();
 
     if (enable_aux_thread) {

src/module/impl/image_player_impl.h

@@ -3,6 +3,7 @@
 
 #include "module/image_player.h"
 #include "codec/decoder_nvdec.h"
+#include "codec/scene_decoder.h"
 #include "core/async_queue.hpp"
 #include "network_v3/receiver_udp_fec.h"
 
@@ -37,6 +38,9 @@ struct image_player::impl {
     std::shared_ptr<frame_queue_type> frame_queue;
     std::unique_ptr<std::thread> aux_thread;
 
+    // for scene decoder
+    std::unique_ptr<scene_decoder> s_dec;
+
     explicit impl(create_config _conf) {
         conf = _conf;
         ctx = conf.ctx;
@@ -56,6 +60,8 @@ struct image_player::impl {
 
     void stop();
 
+    void create_scene_decoder();
+
     void create_decoder();
 
     void create_receiver();

src/module/impl/image_streamer.cpp

@@ -1,6 +1,30 @@
 #include "image_streamer_impl.h"
 #include "core/imgui_utility.hpp"
 
+void image_streamer::impl::create_scene_encoder() {
+    auto img_enc_conf = image_encoder::create_config{
+            .type = ENCODER_NVENC, .stream = conf.stream,
+            .bitrate_mbps = enc_bitrate_mbps,
+            .save_file = enc_save_file,
+            .save_length = enc_save_length,
+            .ctx = conf.cuda_ctx,
+    };
+    i_enc = std::make_unique<image_encoder>(img_enc_conf);
+
+    auto pc_enc_conf = pc_encoder::create_config{
+            .method = pc_codec::PC_RGB_DEPTH,
+            .depth_method = FAKE_800P,
+            .stream = conf.stream,
+            .img_enc = i_enc.get(),
+    };
+    p_enc = std::make_unique<pc_encoder>(pc_enc_conf);
+
+    auto scene_enc_conf = scene_encoder::create_config{
+            .img_enc = i_enc.get(), .pc_enc = p_enc.get(),
+    };
+    s_enc = std::make_unique<scene_encoder>(scene_enc_conf);
+}
+
 void image_streamer::impl::create_encoder() {
     switch (chose_encoder_type) {
         case ENCODER_NVENC: {
@@ -83,7 +107,11 @@ void image_streamer::impl::start() {
         aux_ctx = std::make_unique<io_context>();
     }
 
-    create_encoder();
+    if (conf.encode_scene) {
+        create_scene_encoder();
+    } else {
+        create_encoder();
+    }
     create_sender();
 
     if (enable_aux_thread) {
@@ -115,6 +143,8 @@ void image_streamer::impl::stop() {
 }
 
 void image_streamer::impl::change_frame_rate(int fps) {
+    assert(!conf.encode_scene);
+
     conf.frame_rate = fps;
     if (!is_running) return;
 
@@ -164,7 +194,25 @@ void image_streamer::impl::image_callback(obj_name_type name) {
     }
 }
 
+frame_info image_streamer::impl::encode_scene() {
+    auto scene = OBJ_QUERY(scene_ptr, conf.img_name);
+    if (enc_idr_requested) {
+        s_enc->clear_cache();
+    }
+    auto ret = frame_info{
+            .data = s_enc->encode(scene),
+            .idr = enc_idr_requested,
+            .frame_id = ++last_scene_id,
+    };
+    enc_idr_requested = false;
+    return ret;
+}
+
 frame_info image_streamer::impl::encode_image() {
+    if (conf.encode_scene) { // TODO: ugly hacked
+        return encode_scene();
+    }
+
     switch (chose_encoder_type) {
         case ENCODER_NVENC: {
             auto img = OBJ_QUERY(image_u8c4, conf.img_name);

src/module/impl/image_streamer_impl.h

@@ -3,6 +3,7 @@
 
 #include "module/image_streamer.h"
 #include "codec/encoder_nvenc.h"
+#include "codec/scene_encoder.h"
 #include "core/async_queue.hpp"
 #include "network_v3/sender_tcp.h"
 #include "network_v3/sender_udp_fec.h"
@@ -57,6 +58,12 @@ struct image_streamer::impl {
 
     obj_conn_type img_cb_conn;
 
+    // for scene encoder
+    uint64_t last_scene_id = 0;
+    std::unique_ptr<image_encoder> i_enc;
+    std::unique_ptr<pc_encoder> p_enc;
+    std::unique_ptr<scene_encoder> s_enc;
+
     explicit impl(create_config _conf) {
         conf = _conf;
         ctx = conf.asio_ctx;
@@ -78,12 +85,16 @@ struct image_streamer::impl {
 
     void stop();
 
+    void create_scene_encoder();
+
     void create_encoder();
 
     void create_sender();
 
     void image_callback(obj_name_type name);
 
+    frame_info encode_scene();
+
     frame_info encode_image();
 
     /* sender may run in the auxiliary thread,

src/network/binary_utility.hpp

@@ -3,9 +3,12 @@
 
 #include "core/memory_pool.h"
 
+#include <nlohmann/json.hpp>
+
 #include <boost/core/noncopyable.hpp>
 #include <boost/endian.hpp>
 
+#include <bit>
 #include <cassert>
 #include <cstdint>
 #include <memory>
@@ -108,7 +111,11 @@ struct data_type {
             size = _size;
             return;
         }
-        auto next = data_type(_size);
+
+        auto next_size = std::bit_ceil(_size);
+        auto next = data_type(next_size);
+        next.shrink(_size);
+
         if (keep_data) {
             assert(next.size >= size);
             next.replace(0, size, start_ptr());
@@ -162,9 +169,15 @@ inline swap_net_loc_endian(T &val) {
 class versatile_io : public boost::noncopyable {
 public:
 
-    explicit versatile_io(data_type _data)
+    versatile_io() // enable dynamic memory if no data provided
+            : versatile_io({}, true) {
+    }
+
+    explicit versatile_io(data_type _data,
+                          bool _extendable = false)
             : data(std::move(_data)) {
         cur_ptr = start_ptr();
+        extendable = _extendable;
     }
 
     // 从当前位置开始调整 offset
@@ -186,9 +199,14 @@ public:
         return cur_ptr == end_ptr();
     }
 
+    data_type current_data() const {
+        return data;
+    }
+
 protected:
     data_type data;
     uint8_t *cur_ptr = nullptr;
+    bool extendable = false;
 
     uint8_t *start_ptr() const {
         return data.start_ptr();
@@ -197,6 +215,17 @@ protected:
     uint8_t *end_ptr() const {
         return data.end_ptr();
     }
+
+    void ensure_remaining(size_t size, bool may_extend = true) {
+        if (remaining_bytes() >= size) [[likely]] return;
+        if (extendable && may_extend) {
+            auto offset = current_offset(); // underlying data may be changed after reserve
+            data.reserve(offset + size, true);
+            cur_ptr = start_ptr() + offset;
+        } else {
+            RET_ERROR;
+        }
+    }
 };
 
 // 分多次读取数据
@@ -210,10 +239,10 @@ public:
     std::enable_if_t<std::is_arithmetic_v<T>, T>
     read_value() {
         T tmp_val;
+        ensure_remaining(sizeof(T), false);
         std::copy_n(cur_ptr, sizeof(T), (uint8_t *) &tmp_val);
         swap_net_loc_endian<net_order>(tmp_val);
         cur_ptr += sizeof(T);
-        assert(cur_ptr <= end_ptr());
         return tmp_val;
     }
 
@@ -231,30 +260,43 @@ public:
     }
 
     data_type read_data(size_t size) {
+        ensure_remaining(size, false);
         auto offset = cur_ptr - start_ptr();
         auto ret = data.sub_data(offset, size);
         cur_ptr += size;
-        assert(cur_ptr <= end_ptr());
         return ret;
     }
 
+    template<typename SizeType = uint32_t>
+    data_type read_data_with_length() {
+        auto size = read_value<SizeType>();
+        return read_data(size);
+    }
+
     void read_data(void *data, size_t size) {
+        ensure_remaining(size, false);
         std::copy_n(cur_ptr, size, (uint8_t *) data);
         cur_ptr += size;
-        assert(cur_ptr <= end_ptr());
     }
 
-    void read_data(const data_type &out) {
-        read_data(data.start_ptr(), data.size);
+    void read_data(const data_type &_data) {
+        read_data(_data.start_ptr(), _data.size);
     }
 
     std::string read_std_string(size_t size) {
+        ensure_remaining(size, false);
         auto ret = std::string((char *) cur_ptr, size);
         cur_ptr += size;
-        assert(cur_ptr <= end_ptr());
         return ret;
     }
 
+    template<typename SizeType = uint32_t>
+    nlohmann::json read_json_with_length() {
+        auto j_size = read_value<SizeType>();
+        auto j_str = read_std_string(j_size);
+        return nlohmann::json::parse(j_str);
+    }
+
     data_type read_remain() {
         auto offset = cur_ptr - start_ptr();
         auto size = end_ptr() - cur_ptr;
@@ -281,9 +323,9 @@ public:
     std::enable_if_t<std::is_arithmetic_v<T>>
     write_value(T val) {
         swap_net_loc_endian<net_order>(val);
+        ensure_remaining(sizeof(T));
         std::copy_n((uint8_t *) &val, sizeof(T), cur_ptr);
         cur_ptr += sizeof(T);
-        assert(cur_ptr <= end_ptr());
     }
 
     template<typename T, size_t Length>
@@ -294,15 +336,22 @@ public:
     }
 
     void write_data(const void *data, size_t size) {
+        ensure_remaining(size);
         std::copy_n((uint8_t *) data, size, cur_ptr);
         cur_ptr += size;
-        assert(cur_ptr <= end_ptr());
     }
 
     void write_data(const data_type &_data) {
         write_data(_data.start_ptr(), _data.size);
     }
 
+    template<typename SizeType = uint32_t>
+    void write_with_length(const data_type &_data) {
+        auto size = (SizeType) _data.size;
+        write_value(size);
+        write_value(_data);
+    }
+
     void write_value(const std::string &str) {
         write_data(str.data(), str.length());
     }
@@ -311,6 +360,14 @@ public:
         write_data(_data);
     }
 
+    template<typename SizeType = uint32_t>
+    void write_with_length(const nlohmann::json &j) {
+        auto j_str = j.dump();
+        auto j_size = (SizeType) j_str.length();
+        write_value(j_size);
+        write_value(j_str);
+    }
+
     template<typename T>
     auto &operator<<(const T &val) {
         write_value(val);

src/network_v3/receiver_udp_fec.cpp

@@ -314,7 +314,7 @@ struct receiver_udp_fec::impl {
         // decode frame
         frame_cache.status = READY;
         auto frame = ::frame_info(); // not frame_info in this impl
-        frame.data = frame_cache.data;
+        frame.data = frame_cache.data.clone(); // TODO: find out if deepcopy is really needed.
         frame.idr = header.frame_type == 'I';
         frame.frame_id = header.frame_id;
         q_this->commit_frame(frame);

src/network_v3/sender_udp_fec.cpp

@@ -208,7 +208,8 @@ struct sender_udp_fec::impl {
         }
 
         q_this->log_frame_sent(frame.frame_id);
-        SPDLOG_TRACE("Frame {} sent.", frame.frame_id);
+        SPDLOG_TRACE("Frame {} sent with length {}.",
+                     frame.frame_id, frame.size());
     }
 
     void hole_punch() { // UDP hole punch

src/render/impl/render_mesh.cpp

@@ -38,11 +38,13 @@ auto mesh_type::impl::create(const mesh_type::create_config &conf)
     assert(mesh->mPrimitiveTypes == aiPrimitiveType_TRIANGLE);
     assert(mesh->mNormals != nullptr);
     ret->mesh = mesh;
+    ret->upload_data_mesh();
     return ret;
 }
 
-void mesh_type::impl::upload_gl(size_t num_face) {
+void mesh_type::impl::upload_gl() {
     assert(gl_info == nullptr);
+    assert(!vbo_data.empty() && !ebo_data.empty());
 
     // config vertex buffer
     GLuint vbo;
@@ -69,10 +71,10 @@ void mesh_type::impl::upload_gl(size_t num_face) {
     gl_info->vao = vao;
     gl_info->vbo = vbo;
     gl_info->ebo = ebo;
-    gl_info->num_triangle = num_face;
+    gl_info->num_triangle = num_triangles;
 }
 
-void mesh_type::impl::upload_gl_mesh() {
+void mesh_type::impl::upload_data_mesh() {
     assert(mesh != nullptr);
 
     // create vertex buffer data
@@ -95,11 +97,10 @@ void mesh_type::impl::upload_gl_mesh() {
             info[i] = face.mIndices[i];
         }
     }
-
-    upload_gl(num_face);
+    num_triangles = num_face;
 }
 
-void mesh_type::impl::upload_gl_vtk(vtkPolyData *poly) {
+void mesh_type::impl::upload_data_vtk(vtkPolyData *poly) {
     // calculate vertex normal if needed
     vtkNew<vtkPolyDataNormals> filter;
     if (poly->GetPointData()->GetNormals() == nullptr) {
@@ -140,18 +141,21 @@ void mesh_type::impl::upload_gl_vtk(vtkPolyData *poly) {
             info[i] = vert_id;
         }
     }
-
-    upload_gl(num_face);
+    num_triangles = num_face;
 }
 
 mesh_type::gl_info_type *mesh_type::impl::get_gl_info() {
     if (gl_info == nullptr) [[unlikely]] {
-        upload_gl_mesh();
+        upload_gl();
     }
     assert(gl_info != nullptr);
     return gl_info.get();
 }
 
+mesh_type::raw_info_type mesh_type::impl::get_raw_info() {
+    return {vbo_data, ebo_data, num_triangles};
+}
+
 mesh_type::~mesh_type() = default;
 
 mesh_type::pointer mesh_type::create(const create_config &conf) {
@@ -172,7 +176,7 @@ mesh_type::pointer mesh_type::create(const create_config &conf) {
 mesh_type::pointer mesh_type::from_vtk(vtkPolyData *poly) {
     auto ret = std::make_shared<this_type>();
     ret->pimpl = std::make_unique<impl>();
-    ret->pimpl->upload_gl_vtk(poly);
+    ret->pimpl->upload_data_vtk(poly);
     return ret;
 }
 
@@ -180,16 +184,31 @@ mesh_type::pointer mesh_type::from_file(const std::string &path) {
     return create({.path = path});
 }
 
+mesh_type::pointer mesh_type::from_raw(const raw_info_type &raw) {
+    auto ret = std::make_shared<this_type>();
+    ret->pimpl = std::make_unique<impl>();
+    auto &pimpl = ret->pimpl;
+    pimpl->vbo_data = raw.vbo_data;
+    pimpl->ebo_data = raw.ebo_data;
+    pimpl->num_triangles = raw.num_triangles;
+    return ret;
+}
+
 mesh_type::gl_info_type *mesh_type::get_gl_info() {
     return pimpl->get_gl_info();
 }
 
+mesh_type::raw_info_type mesh_type::get_raw_info() {
+    return pimpl->get_raw_info();
+}
+
 namespace render_mesh_impl {
 
     using pg_type = std::unique_ptr<smart_program>;
     pg_type pg_normal; // render mesh in normal ways
 
     void draw_mesh(mesh_type *mesh) {
+        assert(mesh != nullptr);
         auto gl_info = mesh->get_gl_info();
         glBindVertexArray(gl_info->vao);
         glBindBuffer(GL_ARRAY_BUFFER, gl_info->vbo);

src/render/impl/render_mesh_impl.h

@@ -2,7 +2,6 @@
 #define DEPTHGUIDE_RENDER_MESH_IMPL_H
 
 #include "render/render_mesh.h"
-#include "network/binary_utility.hpp"
 
 #include <assimp/Importer.hpp>
 #include <assimp/scene.h>
@@ -35,6 +34,7 @@ struct mesh_type::impl {
 
     data_type vbo_data;
     data_type ebo_data;
+    size_t num_triangles;
 
     Assimp::Importer importer;
     aiMesh *mesh = nullptr;
@@ -44,14 +44,16 @@ struct mesh_type::impl {
     static auto create(const create_config &conf)
     -> std::unique_ptr<impl>;
 
-    void upload_gl(size_t num_face);
+    void upload_gl();
 
-    void upload_gl_mesh();
+    void upload_data_mesh();
 
-    void upload_gl_vtk(vtkPolyData *poly);
+    void upload_data_vtk(vtkPolyData *poly);
 
     gl_info_type *get_gl_info();
 
+    raw_info_type get_raw_info();
+
 };
 
 #endif //DEPTHGUIDE_RENDER_MESH_IMPL_H

src/render/impl/render_scene.cpp

@@ -78,9 +78,10 @@ glm::mat4 camera_info::projection(float aspect) const {
     return glm::perspective(glm::radians(fov), aspect, near, far);
 }
 
-void render_scene(const scene_render_info &info) {
+void render_scene(const scene_ptr &info_ptr) {
     auto vp_size = query_viewport_size();
 
+    auto &info = *info_ptr;
     auto req = render_request{
             .camera_raw = info.camera,
             .stream = info.stream,

src/render/render_mesh.h

@@ -2,6 +2,7 @@
 #define DEPTHGUIDE_RENDER_MESH_H
 
 #include "render_utility.h"
+#include "network/binary_utility.hpp"
 
 #include <vtkPolyData.h>
 
@@ -25,6 +26,14 @@ public:
 
     static pointer from_vtk(vtkPolyData *poly);
 
+    struct raw_info_type {
+        data_type vbo_data;
+        data_type ebo_data;
+        size_t num_triangles = 0;
+    };
+
+    static pointer from_raw(const raw_info_type &raw);
+
     struct gl_info_type : private boost::noncopyable {
         GLuint vao = 0; // Vertex Array Object
         GLuint vbo = 0; // Vertex Buffer Object
@@ -36,6 +45,8 @@ public:
 
     gl_info_type *get_gl_info();
 
+    raw_info_type get_raw_info();
+
 private:
     struct impl;
     std::unique_ptr<impl> pimpl;

src/render/render_scene.h

@@ -61,6 +61,8 @@ struct scene_render_info {
     smart_cuda_stream *stream = nullptr;
 };
 
-void render_scene(const scene_render_info &info);
+using scene_ptr = std::shared_ptr<scene_render_info>;
+
+void render_scene(const scene_ptr &info);
 
 #endif //DEPTHGUIDE_RENDER_SCENE_H

src/third_party/static_block.hpp

@@ -0,0 +1,80 @@
+/**
+ * @file static_block.hpp
+ *
+ * @brief An implementation of a Java-style static block, in C++ (and potentially a
+ * GCC/clang extension to avoid warnings).
+ *
+ * @note Partially inspired by Andrei Alexandrescu's Scope Guard and
+ * discussions on stackoverflow.com
+ *
+ * By Eyal Rozenberg <eyalroz1@gmx.com>
+ *
+ * Licensed under the Apache License v2.0:
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ */
+#pragma once
+#ifndef STATIC_BLOCK_HPP_
+#define STATIC_BLOCK_HPP_
+
+#ifndef CONCATENATE
+#define CONCATENATE(s1, s2)     s1##s2
+#define EXPAND_THEN_CONCATENATE(s1, s2) CONCATENATE(s1, s2)
+#endif /* CONCATENATE */
+
+#ifndef UNIQUE_IDENTIFIER
+/**
+ * This macro expands into a different identifier in every expansion.
+ * Note that you _can_ clash with an invocation of UNIQUE_IDENTIFIER
+ * by manually using the same identifier elsewhere; or by carefully
+ * choosing another prefix etc.
+ */
+#ifdef __COUNTER__
+#define UNIQUE_IDENTIFIER(prefix) EXPAND_THEN_CONCATENATE(prefix, __COUNTER__)
+#else
+#define UNIQUE_IDENTIFIER(prefix) EXPAND_THEN_CONCATENATE(prefix, __LINE__)
+#endif /* COUNTER */
+#else
+#endif /* UNIQUE_IDENTIFIER */
+
+/**
+ * Following is a mechanism for executing code statically.
+ *
+ * @note Caveats:
+ * - Your static block must be surround by curly braces.
+ * - No need for a semicolon after the block (but it won't hurt).
+ * - Do not put static blocks in files, as it might get compiled multiple
+ *   times ane execute multiple times.
+ * - A static_block can only be used in file scope - not within any other block etc.
+ * - Templated static blocks will probably not work. Avoid them.
+ * - No other funny business, this is fragile.
+ * - This does not having any threading issues (AFAICT) - as it has no static
+ *   initialization order issue. Of course, you have to _keep_ it safe with
+ *   your static code.
+ * - Execution of the code is guaranteed to occur before main() executes,
+ *   but the relative order of statics being initialized is unknown/unclear. So,
+ *   do not call any method of an instance of a class which you expect to have been
+ *   constructed; it may not have been. Instead, you can use a static getInstance() method
+ *   (look this idiom up on the web, it's safe).
+ * - Variables defined within the static block are not global; they will
+ *   go out of scope as soon as its execution concludes.
+ *
+ * Usage example:
+ *
+ *   static_block {
+ *         do_stuff();
+ *         std::cout << "in the static block!\n";
+ *   }
+ *
+ */
+#define static_block STATIC_BLOCK_IMPL1(UNIQUE_IDENTIFIER(_static_block_))
+
+#define STATIC_BLOCK_IMPL1(prefix) \
+	STATIC_BLOCK_IMPL2(CONCATENATE(prefix,_fn),CONCATENATE(prefix,_var))
+
+#define STATIC_BLOCK_IMPL2(function_name,var_name) \
+static void function_name(); \
+static int var_name = (function_name(), 0) ; \
+static void function_name()
+
+#endif // STATIC_BLOCK_HPP_