From 6bd043897f753c2add1280e259211aec983f942c Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Sun, 8 Sep 2024 17:32:28 +0200 Subject: [PATCH] Add lock for scale and offset and distortion --- src/aliceVision/camera/Distortion.hpp | 6 ++ .../camera/IntrinsicScaleOffset.hpp | 18 ++++++ .../sfm/bundle/BundleAdjustmentCeres.cpp | 56 ++++++++++++------- .../bundle/BundleAdjustmentSymbolicCeres.cpp | 54 ++++++++++++------ src/aliceVision/sfmDataIO/AlembicExporter.cpp | 3 + src/aliceVision/sfmDataIO/AlembicImporter.cpp | 24 +++++++- src/aliceVision/sfmDataIO/jsonIO.cpp | 17 ++++++ src/aliceVision/sfmDataIO/sfmDataIO.hpp | 2 +- 8 files changed, 140 insertions(+), 40 deletions(-) diff --git a/src/aliceVision/camera/Distortion.hpp b/src/aliceVision/camera/Distortion.hpp index 9002b37b17..2cc9839e75 100644 --- a/src/aliceVision/camera/Distortion.hpp +++ b/src/aliceVision/camera/Distortion.hpp @@ -67,10 +67,16 @@ class Distortion virtual Eigen::MatrixXd getDerivativeRemoveDistoWrtDisto(const Vec2& p) const { return Eigen::MatrixXd(0, 0); } + inline void setLocked(bool lock) { _isLocked = lock; } + + inline bool isLocked() const { return _isLocked; } + virtual ~Distortion() = default; + protected: std::vector _distortionParams{}; + bool _isLocked{false}; }; } // namespace camera diff --git a/src/aliceVision/camera/IntrinsicScaleOffset.hpp b/src/aliceVision/camera/IntrinsicScaleOffset.hpp index 817cbd8573..416cc944f3 100644 --- a/src/aliceVision/camera/IntrinsicScaleOffset.hpp +++ b/src/aliceVision/camera/IntrinsicScaleOffset.hpp @@ -105,6 +105,22 @@ class IntrinsicScaleOffset : public IntrinsicBase inline bool isRatioLocked() const { return _ratioLocked; } + /** + * @brief Lock the offset + * @param lock True if the offset is locked, false otherwise + */ + inline void setOffsetLocked(bool lock) { _offsetLocked = lock; } + + inline bool isOffsetLocked() const { return _offsetLocked; } + + /** + * @brief Lock the scale + * @param lock True if the scale is locked, false otherwise + */ + inline void setScaleLocked(bool lock) { _scaleLocked = lock; } + + inline bool isScaleLocked() const { return _scaleLocked; } + /** * @brief get focal length in mm * @return focal length in mm @@ -123,6 +139,8 @@ class IntrinsicScaleOffset : public IntrinsicBase Vec2 _offset{0.0, 0.0}; Vec2 _initialScale{-1.0, -1.0}; bool _ratioLocked{true}; + bool _offsetLocked{false}; + bool _scaleLocked{false}; }; } // namespace camera diff --git a/src/aliceVision/sfm/bundle/BundleAdjustmentCeres.cpp b/src/aliceVision/sfm/bundle/BundleAdjustmentCeres.cpp index 59b45fd03b..f7c6cf466a 100644 --- a/src/aliceVision/sfm/bundle/BundleAdjustmentCeres.cpp +++ b/src/aliceVision/sfm/bundle/BundleAdjustmentCeres.cpp @@ -531,6 +531,13 @@ void BundleAdjustmentCeres::addIntrinsicsToProblem(const sfmData::SfMData& sfmDa continue; } + std::shared_ptr intrinsicScaleOffset = + std::dynamic_pointer_cast(intrinsicPtr); + if (!intrinsicScaleOffset) + { + continue; + } + assert(isValid(intrinsicPtr->getType())); std::vector& intrinsicBlock = _intrinsicsBlocks[intrinsicId]; @@ -559,11 +566,11 @@ void BundleAdjustmentCeres::addIntrinsicsToProblem(const sfmData::SfMData& sfmDa bool lockDistortion = false; double focalRatio = 1.0; + lockFocal = (!refineIntrinsicsFocalLength) || intrinsicScaleOffset->isScaleLocked(); + // refine the focal length - if (refineIntrinsicsFocalLength) + if (lockFocal) { - std::shared_ptr intrinsicScaleOffset = - std::dynamic_pointer_cast(intrinsicPtr); if (intrinsicScaleOffset->getInitialScale().x() > 0 && intrinsicScaleOffset->getInitialScale().y() > 0) { @@ -593,22 +600,22 @@ void BundleAdjustmentCeres::addIntrinsicsToProblem(const sfmData::SfMData& sfmDa } focalRatio = intrinsicBlockPtr[0] / intrinsicBlockPtr[1]; - std::shared_ptr castedcam_iso = std::dynamic_pointer_cast(intrinsicPtr); - if (castedcam_iso) - { - lockRatio = castedcam_iso->isRatioLocked(); - } + lockRatio = intrinsicScaleOffset->isRatioLocked(); } - else + + // optical center + lockCenter = intrinsicScaleOffset->isOffsetLocked(); + + bool validRefineCenter = (refineOptions & REFINE_INTRINSICS_OPTICALOFFSET_ALWAYS) || + ((refineOptions & REFINE_INTRINSICS_OPTICALOFFSET_IF_ENOUGH_DATA) + && _minNbImagesToRefineOpticalCenter > 0 + && usageCount >= _minNbImagesToRefineOpticalCenter); + if (!validRefineCenter) { - // set focal length as constant - lockFocal = true; + lockCenter = true; } - // optical center - if ((refineOptions & REFINE_INTRINSICS_OPTICALOFFSET_ALWAYS) || - ((refineOptions & REFINE_INTRINSICS_OPTICALOFFSET_IF_ENOUGH_DATA) && _minNbImagesToRefineOpticalCenter > 0 && - usageCount >= _minNbImagesToRefineOpticalCenter)) + if (!lockCenter) { // refine optical center within 10% of the image size. assert(intrinsicBlock.size() >= 3); @@ -622,11 +629,6 @@ void BundleAdjustmentCeres::addIntrinsicsToProblem(const sfmData::SfMData& sfmDa problem.SetParameterLowerBound(intrinsicBlockPtr, 3, opticalCenterMinPercent * intrinsicPtr->h()); problem.SetParameterUpperBound(intrinsicBlockPtr, 3, opticalCenterMaxPercent * intrinsicPtr->h()); } - else - { - // don't refine the optical center - lockCenter = true; - } // lens distortion if (!refineIntrinsicsDistortion || intrinsicPtr->getDistortionInitializationMode() == camera::EInitMode::CALIBRATED) @@ -634,6 +636,20 @@ void BundleAdjustmentCeres::addIntrinsicsToProblem(const sfmData::SfMData& sfmDa lockDistortion = true; } + //Check if this particular distortion is locked + std::shared_ptr intrinsicScaleOffsetDisto = + std::dynamic_pointer_cast(intrinsicPtr); + if (intrinsicScaleOffsetDisto) + { + if (intrinsicScaleOffsetDisto->getDistortion()) + { + if (intrinsicScaleOffsetDisto->getDistortion()->isLocked()) + { + lockDistortion = true; + } + } + } + IntrinsicsManifold* subsetManifold = new IntrinsicsManifold(intrinsicBlock.size(), focalRatio, lockFocal, lockRatio, lockCenter, lockDistortion); problem.SetManifold(intrinsicBlockPtr, subsetManifold); diff --git a/src/aliceVision/sfm/bundle/BundleAdjustmentSymbolicCeres.cpp b/src/aliceVision/sfm/bundle/BundleAdjustmentSymbolicCeres.cpp index c151c77a05..9c33b4047b 100644 --- a/src/aliceVision/sfm/bundle/BundleAdjustmentSymbolicCeres.cpp +++ b/src/aliceVision/sfm/bundle/BundleAdjustmentSymbolicCeres.cpp @@ -348,6 +348,13 @@ void BundleAdjustmentSymbolicCeres::addIntrinsicsToProblem(const sfmData::SfMDat continue; } + std::shared_ptr intrinsicScaleOffset = + std::dynamic_pointer_cast(intrinsicPtr); + if (!intrinsicScaleOffset) + { + continue; + } + assert(isValid(intrinsicPtr->getType())); _intrinsicObjects[intrinsicId].reset(intrinsicPtr->clone()); @@ -377,8 +384,10 @@ void BundleAdjustmentSymbolicCeres::addIntrinsicsToProblem(const sfmData::SfMDat bool lockDistortion = false; double focalRatio = 1.0; + lockFocal = (!refineIntrinsicsFocalLength) || intrinsicScaleOffset->isScaleLocked(); + // refine the focal length - if (refineIntrinsicsFocalLength) + if (!lockFocal) { std::shared_ptr intrinsicScaleOffset = std::dynamic_pointer_cast(intrinsicPtr); @@ -410,22 +419,22 @@ void BundleAdjustmentSymbolicCeres::addIntrinsicsToProblem(const sfmData::SfMDat } focalRatio = intrinsicBlockPtr[0] / intrinsicBlockPtr[1]; - - std::shared_ptr castedcam_iso = std::dynamic_pointer_cast(intrinsicPtr); - if (castedcam_iso) - { - lockRatio = castedcam_iso->isRatioLocked(); - } + lockRatio = intrinsicScaleOffset->isRatioLocked(); } - else + + // optical center + lockCenter = intrinsicScaleOffset->isOffsetLocked(); + + bool validRefineCenter = (refineOptions & REFINE_INTRINSICS_OPTICALOFFSET_ALWAYS) || + ((refineOptions & REFINE_INTRINSICS_OPTICALOFFSET_IF_ENOUGH_DATA) + && _minNbImagesToRefineOpticalCenter > 0 + && usageCount >= _minNbImagesToRefineOpticalCenter); + if (!validRefineCenter) { - // set focal length as constant - lockFocal = true; + lockCenter = true; } - const bool optional_center = - ((refineOptions & REFINE_INTRINSICS_OPTICALOFFSET_IF_ENOUGH_DATA) && (usageCount > _minNbImagesToRefineOpticalCenter)); - if ((refineOptions & REFINE_INTRINSICS_OPTICALOFFSET_ALWAYS) || optional_center) + if (!lockCenter) { // refine optical center within 10% of the image size. assert(intrinsicBlock.size() >= 4); @@ -439,11 +448,6 @@ void BundleAdjustmentSymbolicCeres::addIntrinsicsToProblem(const sfmData::SfMDat problem.SetParameterLowerBound(intrinsicBlockPtr, 3, opticalCenterMinPercent * intrinsicPtr->h()); problem.SetParameterUpperBound(intrinsicBlockPtr, 3, opticalCenterMaxPercent * intrinsicPtr->h()); } - else - { - // don't refine the optical center - lockCenter = true; - } // lens distortion if (!refineIntrinsicsDistortion) @@ -451,6 +455,20 @@ void BundleAdjustmentSymbolicCeres::addIntrinsicsToProblem(const sfmData::SfMDat lockDistortion = true; } + //Check if this particular distortion is locked + std::shared_ptr intrinsicScaleOffsetDisto = + std::dynamic_pointer_cast(intrinsicPtr); + if (intrinsicScaleOffsetDisto) + { + if (intrinsicScaleOffsetDisto->getDistortion()) + { + if (intrinsicScaleOffsetDisto->getDistortion()->isLocked()) + { + lockDistortion = true; + } + } + } + IntrinsicsManifold* subsetManifold = new IntrinsicsManifold(intrinsicBlock.size(), focalRatio, lockFocal, lockRatio, lockCenter, lockDistortion); problem.SetManifold(intrinsicBlockPtr, subsetManifold); diff --git a/src/aliceVision/sfmDataIO/AlembicExporter.cpp b/src/aliceVision/sfmDataIO/AlembicExporter.cpp index 65e04ece4a..7f96ae920c 100644 --- a/src/aliceVision/sfmDataIO/AlembicExporter.cpp +++ b/src/aliceVision/sfmDataIO/AlembicExporter.cpp @@ -228,6 +228,8 @@ void AlembicExporter::DataImpl::addCamera(const std::string& name, ODoubleProperty(userProps, "mvg_initialFocalLength").set(initialFocalLength); OBoolProperty(userProps, "mvg_intrinsicLocked").set(intrinsicCasted->isLocked()); OBoolProperty(userProps, "mvg_intrinsicPixelRatioLocked").set(intrinsicCasted->isRatioLocked()); + OBoolProperty(userProps, "mvg_intrinsicOffsetLocked").set(intrinsicCasted->isOffsetLocked()); + OBoolProperty(userProps, "mvg_intrinsicScaleLocked").set(intrinsicCasted->isOffsetLocked()); OStringProperty(userProps, "mvg_intrinsicDistortionInitializationMode") .set(camera::EInitMode_enumToString(intrinsicCasted->getDistortionInitializationMode())); @@ -246,6 +248,7 @@ void AlembicExporter::DataImpl::addCamera(const std::string& name, { distortionType = distortion->getType(); ODoubleArrayProperty(userProps, "mvg_distortionParams").set(distortion->getParameters()); + OBoolProperty(userProps, "mvg_distortionLocked").set(distortion->isLocked()); } // Undistortion parameters and offset diff --git a/src/aliceVision/sfmDataIO/AlembicImporter.cpp b/src/aliceVision/sfmDataIO/AlembicImporter.cpp index b66876eb73..9c3b33899a 100644 --- a/src/aliceVision/sfmDataIO/AlembicImporter.cpp +++ b/src/aliceVision/sfmDataIO/AlembicImporter.cpp @@ -419,6 +419,9 @@ bool readCamera(const Version& abcVersion, bool poseLocked = false; bool poseIndependant = true; bool lockRatio = true; + bool lockOffset = false; + bool lockScale = false; + bool lockDistortion = false; std::vector distortionParams; std::vector undistortionParams; Vec2 undistortionOffset = {0, 0}; @@ -469,6 +472,14 @@ bool readCamera(const Version& abcVersion, { lockRatio = getAbcProp(userProps, *propHeader, "mvg_intrinsicPixelRatioLocked", sampleFrame); } + if (const Alembic::Abc::PropertyHeader* propHeader = userProps.getPropertyHeader("mvg_intrinsicOffsetLocked")) + { + lockOffset = getAbcProp(userProps, *propHeader, "mvg_intrinsicOffsetLocked", sampleFrame); + } + if (const Alembic::Abc::PropertyHeader* propHeader = userProps.getPropertyHeader("mvg_intrinsicScaleLocked")) + { + lockScale = getAbcProp(userProps, *propHeader, "mvg_intrinsicScaleLocked", sampleFrame); + } if (const Alembic::Abc::PropertyHeader* propHeader = userProps.getPropertyHeader("mvg_poseLocked")) { poseLocked = getAbcProp(userProps, *propHeader, "mvg_poseLocked", sampleFrame); @@ -577,6 +588,10 @@ bool readCamera(const Version& abcVersion, prop.get(sample, ISampleSelector(sampleFrame)); distortionParams.assign(sample->get(), sample->get() + sample->size()); } + if (const Alembic::Abc::PropertyHeader* propHeader = userProps.getPropertyHeader("mvg_distortionLocked")) + { + lockDistortion = getAbcProp(userProps, *propHeader, "mvg_distortionLocked", sampleFrame); + } if (userProps.getPropertyHeader("mvg_undistortionParams")) { // Undistortion parameters @@ -656,11 +671,18 @@ bool readCamera(const Version& abcVersion, (initialFocalLengthPix(0) > 0) ? initialFocalLengthPix(0) * mvg_intrinsicParams[1] / mvg_intrinsicParams[0] : -1; intrinsicCasted->setInitialScale(initialFocalLengthPix); intrinsicCasted->setRatioLocked(lockRatio); + intrinsicCasted->setOffsetLocked(lockOffset); + intrinsicCasted->setScaleLocked(lockScale); std::shared_ptr distortion = intrinsicCasted->getDistortion(); if (distortion) { - distortion->setParameters(distortionParams); + distortion->setParameters(distortionParams); + + if (abcVersion >= Version(1, 2, 10)) + { + distortion->setLocked(lockDistortion); + } } std::shared_ptr undistortion = intrinsicCasted->getUndistortion(); diff --git a/src/aliceVision/sfmDataIO/jsonIO.cpp b/src/aliceVision/sfmDataIO/jsonIO.cpp index af20bdcf52..1ebb7c68c5 100644 --- a/src/aliceVision/sfmDataIO/jsonIO.cpp +++ b/src/aliceVision/sfmDataIO/jsonIO.cpp @@ -174,6 +174,9 @@ void saveIntrinsic(const std::string& name, IndexT intrinsicId, const std::share intrinsicTree.put("pixelRatio", pixelRatio); intrinsicTree.put("pixelRatioLocked", intrinsicScaleOffset->isRatioLocked()); + intrinsicTree.put("offsetLocked", intrinsicScaleOffset->isOffsetLocked()); + intrinsicTree.put("scaleLocked", intrinsicScaleOffset->isScaleLocked()); + saveMatrix("principalPoint", intrinsicScaleOffset->getOffset(), intrinsicTree); } @@ -197,7 +200,10 @@ void saveIntrinsic(const std::string& name, IndexT intrinsicId, const std::share paramTree.put("", param); distParamsTree.push_back(std::make_pair("", paramTree)); } + + intrinsicTree.put("distortionLocked", distortionObject->isLocked()); } + intrinsicTree.put("distortionInitializationMode", camera::EInitMode_enumToString(intrinsicScaleOffsetDisto->getDistortionInitializationMode())); @@ -360,6 +366,12 @@ void loadIntrinsic(const Version& version, IndexT& intrinsicId, std::shared_ptr< intrinsicWithScale->setInitialScale(initialFocalLengthPx); intrinsicWithScale->setRatioLocked(intrinsicTree.get("pixelRatioLocked")); } + + if (version >= Version(1, 2, 10)) + { + intrinsicWithScale->setOffsetLocked(intrinsicTree.get("offsetLocked")); + intrinsicWithScale->setScaleLocked(intrinsicTree.get("scaleLocked")); + } } // Load distortion @@ -416,6 +428,11 @@ void loadIntrinsic(const Version& version, IndexT& intrinsicId, std::shared_ptr< { distortionObject->setParameters(distortionParams); } + + if (version >= Version(1, 2, 10)) + { + distortionObject->setLocked(intrinsicTree.get("distortionLocked")); + } } } diff --git a/src/aliceVision/sfmDataIO/sfmDataIO.hpp b/src/aliceVision/sfmDataIO/sfmDataIO.hpp index 61a4736947..51fafe7f38 100644 --- a/src/aliceVision/sfmDataIO/sfmDataIO.hpp +++ b/src/aliceVision/sfmDataIO/sfmDataIO.hpp @@ -12,7 +12,7 @@ #define ALICEVISION_SFMDATAIO_VERSION_MAJOR 1 #define ALICEVISION_SFMDATAIO_VERSION_MINOR 2 -#define ALICEVISION_SFMDATAIO_VERSION_REVISION 9 +#define ALICEVISION_SFMDATAIO_VERSION_REVISION 10 // AliceVision version as a string; for example "0.9.0". #define ALICEVISION_SFMDATAIO_VERSION_STRING \