Calling python script directly using Boost.Process

src/ai/impl/fast_sam.cpp

@@ -3,7 +3,10 @@
 
 #include <boost/asio/post.hpp>
 
+#include <filesystem>
+
 using boost::asio::post;
+namespace fs = std::filesystem;
 
 fast_sam::impl::impl(create_config _conf) {
     conf = _conf;
@@ -41,7 +44,13 @@ void fast_sam::impl::reply_callback(const zmq_client::reply_type &rep) {
 }
 
 void fast_sam::impl::start() {
+    static constexpr auto interpreter_path = "/home/tpx/anaconda3/envs/FastSAM/bin/python";
+    fs::path script_dir = "/home/tpx/src/FastSAM";
+    auto script_path = script_dir / "fastsam_server.py";
     auto client_conf = zmq_client::create_config{
+            .python_interpreter = interpreter_path,
+            .server_working_dir = script_dir,
+            .server_script_path = script_path,
             .serv_addr = "ipc:///tmp/fast_sam_v1",
 //            .serv_addr = "tcp://127.0.0.1:7899",
             .reply_cb = [this](auto rep) { reply_callback(rep); },

src/ai/impl/python/zmq_server.py

@@ -56,9 +56,15 @@ class ZmqServer(object):
 
     def start(self, func):
         self.socket.bind(self.addr)
+        print(f"ØMQ server started at {self.addr}")
         while True:
             req = self.socket.recv()
             imgs, opts = self.decode_request(req)
+
+            # exit condition
+            if 'exit' in opts and opts['exit']:
+                break
+
             imgs, opts = func(imgs, opts)
             rep = self.encode_reply(imgs, opts)
             self.socket.send(rep)

src/ai/impl/zmq_client.cpp

@@ -9,8 +9,6 @@ using boost::system::error_code;
 
 namespace zmq_client_impl {
 
-//    zmq::context_t zmq_ctx;
-
     const char *cv_type_to_dtype(int type) {
         static constexpr bool le =
                 boost::endian::order::native == boost::endian::order::little;
@@ -134,6 +132,16 @@ namespace zmq_client_impl {
 }
 
 zmq_client::impl::impl(const create_config &conf) {
+    // create python process
+    static constexpr auto zmq_server_path =
+            "/home/tpx/project/DepthGuide/src/ai/impl/python";
+    bp::environment aux_env =
+            boost::this_process::environment();
+    aux_env["PYTHONPATH"] += zmq_server_path;
+    aux_proc = std::make_unique<bp::child>(
+            conf.python_interpreter, conf.server_script_path,
+            aux_env, bp::start_dir(conf.server_working_dir));
+
     aux_ctx = std::make_unique<io_context>();
     aux_thread = std::make_unique<std::thread>([=, this]() {
         aux_thread_work(conf);
@@ -143,6 +151,7 @@ zmq_client::impl::impl(const create_config &conf) {
 zmq_client::impl::~impl() {
     aux_ctx->stop();
     aux_thread->join();
+    aux_proc->terminate();
 }
 
 void zmq_client::impl::aux_thread_work(const create_config &conf) {
@@ -188,8 +197,12 @@ void zmq_client::impl::on_reply(const data_type &data) {
     socket_busy.clear();
 }
 
+bool zmq_client::impl::is_running() const {
+    return aux_proc->running();
+}
+
 void zmq_client::impl::on_request(const request_type &req) {
-    if (socket_busy.test()) return;
+    if (socket_busy.test() || !is_running()) return;
     post(*aux_ctx, [req = req, this]() { aux_on_request(req); });
 }
 

src/ai/impl/zmq_client_impl.h

@@ -4,6 +4,8 @@
 #include "ai/zmq_client.h"
 #include "network/binary_utility.hpp"
 
+#include <boost/process.hpp>
+
 #include <azmq/socket.hpp>
 
 #include <atomic>
@@ -11,8 +13,6 @@
 
 namespace zmq_client_impl {
 
-//    extern zmq::context_t zmq_ctx;
-
     enum image_transfer_mode {
         DIRECT = 0,
         HOST_SHARED = 1,
@@ -36,9 +36,11 @@ namespace zmq_client_impl {
 }
 
 using namespace zmq_client_impl;
+namespace bp = boost::process;
 
 struct zmq_client::impl {
 
+    std::unique_ptr<bp::child> aux_proc;
     std::unique_ptr<std::thread> aux_thread;
 //    image_transfer_mode trans_mode = DIRECT;
 
@@ -56,6 +58,8 @@ struct zmq_client::impl {
 
     ~impl();
 
+    bool is_running() const;
+
     void on_request(const request_type &req);
 
     // for auxiliary thread

src/ai/zmq_client.h

@@ -32,6 +32,9 @@ public:
     using reply_cb_type = std::function<void(reply_type)>;
 
     struct create_config {
+        std::string python_interpreter;
+        std::string server_working_dir;
+        std::string server_script_path;
         std::string serv_addr; // like "ipc://fast_sam_v1"
         reply_cb_type reply_cb;
         CUcontext *cuda_ctx = nullptr;
@@ -46,6 +49,8 @@ public:
 
     bool is_ideal() const;
 
+    bool is_running() const;
+
 private:
     struct impl;
     std::unique_ptr<impl> pimpl;

src/core/impl/image_utility_v2.cpp

@@ -314,6 +314,15 @@ generic_image::pointer generic_image::create(const std::shared_ptr<smart_image<T
     return ret;
 }
 
+// @formatter:off
+template generic_image::pointer generic_image::create(const image_u8c1 &);
+template generic_image::pointer generic_image::create(const image_u8c2 &);
+template generic_image::pointer generic_image::create(const image_u8c3 &);
+template generic_image::pointer generic_image::create(const image_u8c4 &);
+template generic_image::pointer generic_image::create(const image_u16c1 &);
+template generic_image::pointer generic_image::create(const image_f32c1 &);
+// @formatter:on
+
 cv::Size generic_image::size() const {
     return pimpl->display_size();
 }
@@ -403,13 +412,4 @@ void generic_image::host_modified(smart_cuda_stream *stream) {
 
 void generic_image::cuda_modified(smart_cuda_stream *stream) {
     pimpl->cuda_modified(stream);
-}
-
-// @formatter:off
-template image_ptr create_image(const image_u8c1 &img);
-template image_ptr create_image(const image_u8c2 &img);
-template image_ptr create_image(const image_u8c3 &img);
-template image_ptr create_image(const image_u8c4 &img);
-template image_ptr create_image(const image_u16c1 &img);
-template image_ptr create_image(const image_f32c1 &img);
-// @formatter:on
+}

src/image_process/cuda_impl/pixel_convert.cu

@@ -3,6 +3,7 @@
 
 #include <glm/glm.hpp>
 #include <cuda/std/limits>
+#include <cuda/std/tuple>
 
 template<typename PixIn, typename PixOut>
 struct cvt_rgb_bgra {
@@ -68,15 +69,18 @@ __global__ void nv12_to_rgb(image_type_v2<uchar1> luma_img, image_type_v2<uchar2
             auto chroma = *chroma_img.at(idy, idx);
 
 #pragma unroll
-            for (auto dy = 0; dy < 2; ++dy)
-#pragma unroll
-                    for (auto dx = 0; dx < 2; ++dx) {
-                        auto iy = 2 * idy + dy, ix = 2 * idx + dx;
-
-                        auto luma = *luma_img.at(iy, ix);
-                        auto yuv = uchar3(luma.x, chroma.x, chroma.y);
-                        *rgb_img.at(iy, ix) = yuv_to_rgb::cvt::Op(yuv);
-                    }
+            for (auto dy = 0; dy < 2; ++dy) {
+                auto iy = 2 * idy + dy, ix = 2 * idx;
+                auto luma_pack = *(uchar2 *) luma_img.at(iy, ix);
+                auto yuv_1 = uchar3(luma_pack.x, chroma.x, chroma.y);
+                auto yuv_2 = uchar3(luma_pack.y, chroma.x, chroma.y);
+                auto rgb_1 = yuv_to_rgb::cvt::Op(yuv_1);
+                auto rgb_2 = yuv_to_rgb::cvt::Op(yuv_2);
+
+                using rgb_pack_type = cuda::std::tuple<uchar3, uchar3>;
+                *(rgb_pack_type *) rgb_img.at(iy, ix) =
+                        cuda::std::make_tuple(rgb_1, rgb_2);
+            }
         }
     }
 }

src/module/impl/pc_viewer.cpp

@@ -17,7 +17,8 @@ void pc_viewer::impl::render() {
     r_conf.pc.point_size = point_size;
     // camera coordinate -> view space coordinate
     auto real_rot = rotation + glm::vec3(180.f, 0.f, 0.f);
-    r_conf.camera.transform = to_transform_mat(offset, glm::radians(real_rot));
+    r_conf.camera.transform = to_transform_mat(1000.f * offset, // m -> mm
+                                               glm::radians(real_rot));
     r_conf.camera.fov = fov;
     r_conf.camera.near = clip.near;
     r_conf.camera.far = clip.far;
@@ -26,8 +27,8 @@ void pc_viewer::impl::render() {
 }
 
 void pc_viewer::impl::show() {
-    ImGui::DragFloat3("Offset (mm)", glm::value_ptr(offset),
-                      0.05f, 0.0f, 0.0f, "%.02f");
+    ImGui::DragFloat3("Offset (m)", glm::value_ptr(offset),
+                      0.01f, 0.0f, 0.0f, "%.02f");
     ImGui::DragFloat3("Offset (deg)", glm::value_ptr(rotation),
                       0.1f, -180.0f, 180.0f, "%.01f");
     ImGui::DragFloat("FOV (deg)", &fov, 0.5f, 10.0f, 178.0f, "%.01f");