Implemented video stabilization.

src/image_process_v5/sp_image.cpp

@@ -177,6 +177,7 @@ void copy_ndarray(const image_ndarray_proxy &src, image_ndarray_proxy &dst, cuda
 void copy_sp_image(const sp_image &src, sp_image &dst, const cudaMemcpyKind kind) {
     assert(src.type == dst.type);
     copy_ndarray(src, dst, kind);
+    dst.merge_meta(src);
 }
 
 sp_image create_dense(const sp_image &src) {

src/image_process_v5/video_stabilization.cpp

@@ -4,7 +4,6 @@
 
 #include <opencv2/calib3d.hpp>
 #include <opencv2/cudaoptflow.hpp>
-#include <opencv2/cudafeatures2d.hpp>
 #include <opencv2/cudaimgproc.hpp>
 
 #include <Eigen/Dense>
@@ -98,46 +97,11 @@ namespace {
             return ret;
         }
 
-        [[nodiscard]] record_type optimize(
-            const record_type &traj,
-            const record_type &prev_traj = {}) const {
-            const auto num = traj.rows();
-            Eigen::MatrixXf mat_Q = Eigen::MatrixXf::Identity(num, num);
-            Eigen::VectorXf vec_b = -2 * traj;
-
-            auto gaussian = [=, this](const float x) {
-                return expf(-9 * powf(x, 2)) / powf(conf.smooth_radius, 2);
-            };
-            if (true) {
-                auto mat_D = Eigen::MatrixXf(num, num);
-                for (auto i = 0; i < num; ++i)
-                    for (auto j = 0; j < num; ++j)
-                        mat_D(i, j) = gaussian(i - j);
-                for (auto i = 0; i < num; ++i)
-                    mat_D(i, i) -= mat_D.col(i).sum();
-                mat_Q += conf.smooth_factor * (mat_D * mat_D.transpose());
-            }
-
-            if (prev_traj.rows() > 0) {
-                assert(prev_traj.rows() == num);
-                Eigen::MatrixXf mat_D = Eigen::MatrixXf::Identity(num, num);
-                mat_D(num - 1, num - 1) = 0;
-                mat_Q += conf.beta * mat_D;
-                vec_b += conf.beta * (-2 * prev_traj);
-            }
-
-            mat_Q *= 2;
-            record_type ret = mat_Q.partialPivLu().solve(-vec_b);
-            return ret;
-        }
-
-        [[nodiscard]] trajectory_type optimize_trajectory(
-            const trajectory_type &traj,
-            const trajectory_type &prev_traj = {}) const {
+        static trajectory_type optimize_trajectory(const trajectory_type &traj) {
             const auto num_dims = traj.rows();
             auto ret = trajectory_type(num_dims, traj.cols());
             for (auto i = 0; i < num_dims; ++i) {
-                ret.row(i) = optimize(traj.row(i), prev_traj.row(i));
+                ret.row(i) = optimize_simple(traj.row(i));
             }
             return ret;
         }
@@ -165,11 +129,10 @@ namespace {
 
         size_t buffer_length = 0;
         std::deque<matrix_param_type> transform_buf;
-        trajectory_type last_traj;
 
         explicit transform_stabilization(const create_config &_conf)
             : conf(_conf) {
-            buffer_length = conf.future_frames + conf.history_frames + 1;
+            buffer_length = 2 * conf.future_frames + 1;
         }
 
         std::optional<matrix_type> process(const matrix_type &mat) {
@@ -180,16 +143,6 @@ namespace {
                 transform_buf.pop_front();
             }
 
-            if (last_traj.cols() > 0) {
-                last_traj.conservativeResize(4, last_traj.cols() + 1);
-                last_traj.col(last_traj.cols() - 1).setZero();
-                if (last_traj.cols() > buffer_length) {
-                    last_traj.block(0, 0, 4, buffer_length)
-                            = last_traj.block(0, 1, 4, buffer_length);
-                    last_traj.conservativeResize(4, buffer_length);
-                }
-            }
-
             auto curr_traj = trajectory_type(4, transform_buf.size());
             for (int i = 0; i < transform_buf.size(); i++) {
                 curr_traj.col(i) = transform_buf[i];
@@ -198,15 +151,10 @@ namespace {
                 }
             }
 
-            const auto traj_opt = optimize_trajectory(curr_traj, last_traj);
-            last_traj = traj_opt;
-
+            const auto traj_opt = optimize_trajectory(curr_traj);
             const auto curr_pos = conf.future_frames;
             const auto diff = traj_opt.col(curr_pos) - curr_traj.col(curr_pos);
             const auto curr = transform_buf[curr_pos] + diff;
-            // const Eigen::Vector4f diff = traj_opt.col(curr_pos) - curr_traj.col(curr_pos);
-            // const Eigen::Vector4f curr = transform_buf[curr_pos] + diff;
-            // SPDLOG_DEBUG("Transform: {}, {}, {}, {}", curr.x(), curr.y(), curr.z(), curr.w());
             return compose_transform(curr);
         }
     };
@@ -369,6 +317,9 @@ struct video_stabilization_ui::impl {
             return;
         }
 
+        if (video_stab == nullptr) [[unlikely]] {
+            video_stab = std::make_shared<video_stabilization>(conf.opts);
+        }
         auto task = [conf = conf, stab = video_stab] {
             try {
                 image_callback_impl(conf, stab);
@@ -378,11 +329,22 @@ struct video_stabilization_ui::impl {
     }
 
     void show_ui() {
-        ImGui::Checkbox("Passthrough", &passthrough);
+        if (ImGui::Checkbox("Passthrough", &passthrough)) {
+            if (!passthrough) {
+                video_stab = nullptr;
+            }
+        }
+        if (!passthrough) {
+            ImGui::SameLine();
+            if (ImGui::Button("Reset")) {
+                video_stab = nullptr;
+            }
+            ImGui::DragInt("Frame Delay",
+                           &conf.opts.st_conf.future_frames, 1, 1);
+        }
     }
 
     explicit impl(const create_config &_conf) : conf(_conf) {
-        video_stab = std::make_shared<video_stabilization>(conf.opts);
         conn = OBJ_SIG(conf.in_name)->connect(
             [this](auto _) { image_callback(); });
     }

src/image_process_v5/video_stabilization.h

@@ -10,11 +10,7 @@ public:
     };
 
     struct transform_stabilization_config {
-        size_t future_frames = 20;
-        size_t history_frames = 20;
-        float smooth_radius = 50;
-        float smooth_factor = 1e5;
-        float beta = 1; // for prev_traj
+        int future_frames = 50;
     };
 
     struct create_config {