From 978dbac4cc6159ead9a8c5a7b24efa93309c8d90 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Tue, 30 Jul 2024 09:00:04 +0200 Subject: [PATCH 1/5] residual function can disable distortion --- src/aliceVision/camera/IntrinsicBase.hpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/aliceVision/camera/IntrinsicBase.hpp b/src/aliceVision/camera/IntrinsicBase.hpp index a9137f1a49..0ddcc12377 100644 --- a/src/aliceVision/camera/IntrinsicBase.hpp +++ b/src/aliceVision/camera/IntrinsicBase.hpp @@ -125,7 +125,6 @@ class IntrinsicBase * @brief Get the derivative of a projection of a 3D point into the camera plane * @param[in] pose The pose * @param[in] pt3D The 3D point - * @param[in] applyDistortion If true, apply the distortion if there is any * @return The projection jacobian with respect to the pose */ virtual Eigen::Matrix getDerivativeProjectWrtPose(const Eigen::Matrix4d& pose, const Vec4& pt3D) const = 0; @@ -134,7 +133,6 @@ class IntrinsicBase * @brief Get the derivative of a projection of a 3D point into the camera plane * @param[in] pose The pose * @param[in] pt3D The 3D point - * @param[in] applyDistortion If true, apply the distortion if there is any * @return The projection jacobian with respect to the pose */ virtual Eigen::Matrix getDerivativeProjectWrtPoseLeft(const Eigen::Matrix4d& pose, const Vec4& pt3D) const = 0; @@ -143,7 +141,6 @@ class IntrinsicBase * @brief Get the derivative of a projection of a 3D point into the camera plane * @param[in] pose The pose * @param[in] pt3D The 3D point - * @param[in] applyDistortion If true, apply the distortion if there is any * @return The projection jacobian with respect to the point */ virtual Eigen::Matrix getDerivativeProjectWrtPoint(const Eigen::Matrix4d& pose, const Vec4& pt3D) const = 0; @@ -152,7 +149,6 @@ class IntrinsicBase * @brief Get the derivative of a projection of a 3D point into the camera plane * @param[in] pose The pose * @param[in] pt3D The 3D point - * @param[in] applyDistortion If true, apply the distortion if there is any * @return The projection jacobian with respect to the point */ virtual Eigen::Matrix getDerivativeProjectWrtPoint3(const Eigen::Matrix4d& pose, const Vec4& pt3D) const = 0; @@ -161,7 +157,6 @@ class IntrinsicBase * @brief Get the derivative of a projection of a 3D point into the camera plane * @param[in] pose The pose * @param[in] pt3D The 3D point - * @param[in] applyDistortion If true, apply the distortion if there is any * @return The projection jacobian with respect to the params */ virtual Eigen::Matrix getDerivativeProjectWrtParams(const Eigen::Matrix4d& pos, const Vec4& pt3D) const = 0; @@ -171,11 +166,12 @@ class IntrinsicBase * @param[in] pose The pose * @param[in] X The 3D projected point * @param[in] x The image observation + * @param[in] applyDistortion If true, apply the distortion if there is any * @return residual between the 3D projected point and the image observation */ - inline Vec2 residual(const geometry::Pose3& pose, const Vec4& X, const Vec2& x) const + inline Vec2 residual(const geometry::Pose3& pose, const Vec4& X, const Vec2& x, bool applyDistortion = true) const { - const Vec2 proj = this->project(pose, X); + const Vec2 proj = this->project(pose, X, applyDistortion); return x - proj; } From 34a784e85747bb9e0d67c6a1206e435e2e9937b6 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Tue, 30 Jul 2024 09:17:54 +0200 Subject: [PATCH 2/5] Undistort observations in residual --- src/aliceVision/camera/IntrinsicBase.hpp | 6 ++++-- .../multiview/resection/ResectionSphericalKernel.hpp | 8 ++++---- .../triangulation/TriangulationSphericalKernel.hpp | 11 ++++++----- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/aliceVision/camera/IntrinsicBase.hpp b/src/aliceVision/camera/IntrinsicBase.hpp index 0ddcc12377..e1a146bd85 100644 --- a/src/aliceVision/camera/IntrinsicBase.hpp +++ b/src/aliceVision/camera/IntrinsicBase.hpp @@ -171,8 +171,10 @@ class IntrinsicBase */ inline Vec2 residual(const geometry::Pose3& pose, const Vec4& X, const Vec2& x, bool applyDistortion = true) const { - const Vec2 proj = this->project(pose, X, applyDistortion); - return x - proj; + // We will compare to an undistorted point, so always ignore the distortion when computing coordinates + const Vec2 proj = this->project(pose, X, false); + + return ((applyDistortion)?this->getUndistortedPixel(x):x) - proj; } /** diff --git a/src/aliceVision/multiview/resection/ResectionSphericalKernel.hpp b/src/aliceVision/multiview/resection/ResectionSphericalKernel.hpp index 15d422905b..7534755191 100644 --- a/src/aliceVision/multiview/resection/ResectionSphericalKernel.hpp +++ b/src/aliceVision/multiview/resection/ResectionSphericalKernel.hpp @@ -25,11 +25,11 @@ class ResectionSphericalKernel const std::vector & structure, const std::vector & observations) : _camera(camera), - _structure(structure), - _observations(observations) + _structure(structure) { - for (const auto & pt : _observations) + for (const auto & pt : observations) { + _observations.push_back(_camera->getUndistortedPixel(pt)); _liftedObservations.push_back(_camera->toUnitSphere(_camera->removeDistortion(_camera->ima2cam(pt)))); } @@ -120,7 +120,7 @@ class ResectionSphericalKernel const Vec4 X = _structure[idx].homogeneous(); const Vec2 x = _observations[idx]; - const Vec2 residual = _camera->residual(geometry::Pose3(model), X, x); + const Vec2 residual = _camera->residual(geometry::Pose3(model), X, x, false); errors[idx] = residual.norm(); } diff --git a/src/aliceVision/multiview/triangulation/TriangulationSphericalKernel.hpp b/src/aliceVision/multiview/triangulation/TriangulationSphericalKernel.hpp index 153dfdd0ce..8dcc61422e 100644 --- a/src/aliceVision/multiview/triangulation/TriangulationSphericalKernel.hpp +++ b/src/aliceVision/multiview/triangulation/TriangulationSphericalKernel.hpp @@ -29,15 +29,16 @@ class TriangulationSphericalKernel : public robustEstimation::IRansacKernel& poses, std::vector> & intrinsics ) - : _observations(observations) - , _poses(poses) + : _poses(poses) , _intrinsics(intrinsics) { for (int id = 0; id < observations.size(); id++) { //Lift all points onto the metric unit sphere - const Vec2 & obs = _observations[id]; + const Vec2 & obs = observations[id]; std::shared_ptr camera = _intrinsics[id]; + + _observations.push_back(camera->getUndistortedPixel(obs)); _lifted.push_back(camera->toUnitSphere(camera->removeDistortion(camera->ima2cam(obs)))); } } @@ -146,7 +147,7 @@ class TriangulationSphericalKernel : public robustEstimation::IRansacKernelresidual(_poses[sample], X, _observations[sample]); + Vec2 residual = _intrinsics[sample]->residual(_poses[sample], X, _observations[sample], false); return residual.norm(); } @@ -215,7 +216,7 @@ class TriangulationSphericalKernel : public robustEstimation::IRansacKernel _lifted; - const std::vector _observations; + std::vector _observations; const std::vector _poses; const std::vector> _intrinsics; multiview::TriangulateNViewsSphericalSolver _solver; From 311e53966659c0936d4381bb0f67405b551af8af Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Tue, 30 Jul 2024 09:18:30 +0200 Subject: [PATCH 3/5] handle edge cases in bootstrapping --- .../sfm/pipeline/expanding/SfmTriangulation.cpp | 1 + src/software/pipeline/main_sfmBootstraping.cpp | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/aliceVision/sfm/pipeline/expanding/SfmTriangulation.cpp b/src/aliceVision/sfm/pipeline/expanding/SfmTriangulation.cpp index 1c76d74750..d7ab3746d3 100644 --- a/src/aliceVision/sfm/pipeline/expanding/SfmTriangulation.cpp +++ b/src/aliceVision/sfm/pipeline/expanding/SfmTriangulation.cpp @@ -139,6 +139,7 @@ bool SfmTriangulation::processTrack( if (observations.size() <= 0) { + return false; } else { diff --git a/src/software/pipeline/main_sfmBootstraping.cpp b/src/software/pipeline/main_sfmBootstraping.cpp index f5e429549e..2823eeb5a4 100644 --- a/src/software/pipeline/main_sfmBootstraping.cpp +++ b/src/software/pipeline/main_sfmBootstraping.cpp @@ -294,8 +294,9 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_INFO("Give a score to all pairs"); int count = 0; - double bestScore = 0.0; + double bestScore = std::numeric_limits::lowest(); sfm::ReconstructedPair bestPair; + bestPair.reference = UndefinedIndexT; std::vector bestUsedTracks; for (const sfm::ReconstructedPair & pair: reconstructedPairs) @@ -308,9 +309,10 @@ int aliceVision_main(int argc, char** argv) continue; } + //If the angle is too small, then dramatically reduce its chances if (radianToDegree(angle) < minAngle) { - continue; + angle = -1.0 / angle; } const sfmData::View & vref = sfmData.getView(pair.reference); @@ -332,6 +334,12 @@ int aliceVision_main(int argc, char** argv) } } + if (bestPair.reference == UndefinedIndexT) + { + ALICEVISION_LOG_INFO("No valid pair"); + return EXIT_FAILURE; + } + if (!buildSfmData(sfmData, bestPair.reference, bestPair.next, bestPair.pose, tracksHandler.getAllTracks(), bestUsedTracks)) { return EXIT_FAILURE; From 47456ceb9342d597a0e4fdceb88374784916500c Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Tue, 30 Jul 2024 09:18:38 +0200 Subject: [PATCH 4/5] Legacy code compatibility --- .../sfm/pipeline/expanding/ExpansionPolicyLegacy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aliceVision/sfm/pipeline/expanding/ExpansionPolicyLegacy.cpp b/src/aliceVision/sfm/pipeline/expanding/ExpansionPolicyLegacy.cpp index 6a502f52c9..069e4355bb 100644 --- a/src/aliceVision/sfm/pipeline/expanding/ExpansionPolicyLegacy.cpp +++ b/src/aliceVision/sfm/pipeline/expanding/ExpansionPolicyLegacy.cpp @@ -188,7 +188,7 @@ double ExpansionPolicyLegacy::computeScore(const track::TracksMap & tracksMap, } //The higher the level, the higher the weight per cell - double w = pow(2.0, shiftLevel); + double w = pow(2.0, countLevels - (shiftLevel + 1)); sum += w * double(size); } From b1afcdb9c6f011ad65bf60f1ea7787ec2fa26ac5 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Tue, 30 Jul 2024 13:07:10 +0200 Subject: [PATCH 5/5] Increase fisheye undistortion precision --- src/aliceVision/camera/DistortionFisheye.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aliceVision/camera/DistortionFisheye.cpp b/src/aliceVision/camera/DistortionFisheye.cpp index fbe6443a38..2aaa9ac8b6 100644 --- a/src/aliceVision/camera/DistortionFisheye.cpp +++ b/src/aliceVision/camera/DistortionFisheye.cpp @@ -119,7 +119,7 @@ Vec2 DistortionFisheye::removeDistortion(const Vec2& p) const if (theta_dist > eps) { double theta = theta_dist; - for (int j = 0; j < 10; ++j) + for (int j = 0; j < 20; ++j) { const double theta2 = theta * theta; const double theta4 = theta2 * theta2;