From c3c4b12328eabecfa0eaa05ba4f3078fb41dd73a Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Mon, 20 Nov 2023 16:09:48 +0100 Subject: [PATCH 01/24] Optimized implementation of Split grouping --- .../SEFramework/Pipeline/SourceGrouping.h | 11 ++- .../Grouping/GroupingFactory.h | 45 +++--------- .../Grouping/SplitSourcesGrouping.h | 47 ++++++++++++ .../src/lib/Grouping/GroupingFactory.cpp | 66 +++++++++++++++++ .../src/lib/Grouping/SplitSourcesGrouping.cpp | 71 +++++++++++++++++++ 5 files changed, 204 insertions(+), 36 deletions(-) create mode 100644 SEImplementation/SEImplementation/Grouping/SplitSourcesGrouping.h create mode 100644 SEImplementation/src/lib/Grouping/GroupingFactory.cpp create mode 100644 SEImplementation/src/lib/Grouping/SplitSourcesGrouping.cpp diff --git a/SEFramework/SEFramework/Pipeline/SourceGrouping.h b/SEFramework/SEFramework/Pipeline/SourceGrouping.h index 6e028634b..fc744f9f2 100644 --- a/SEFramework/SEFramework/Pipeline/SourceGrouping.h +++ b/SEFramework/SEFramework/Pipeline/SourceGrouping.h @@ -27,6 +27,7 @@ #include #include +#include "SEFramework/Property/PropertyId.h" #include "SEFramework/Pipeline/PipelineStage.h" #include "SEFramework/Source/SourceGroupFactory.h" #include "SEFramework/Source/SourceGroupInterface.h" @@ -81,6 +82,12 @@ class GroupingCriteria { virtual std::set requiredProperties() const { return {}; } }; +class SourceGroupingInterface : public PipelineEmitter, public PipelineReceiver { +public: + /// Returns the set of required properties to compute the grouping + virtual std::set requiredProperties() const = 0; +}; + /** * @class SourceGrouping * @brief SourceGrouping takes Source, groups them together according to its GroupingCriteria and stores them. @@ -88,7 +95,7 @@ class GroupingCriteria { * sources they are grouped with as a SourceGroup. * */ -class SourceGrouping : public PipelineEmitter, public PipelineReceiver { +class SourceGrouping : public SourceGroupingInterface { public: /** @@ -101,7 +108,7 @@ class SourceGrouping : public PipelineEmitter, public Pipe unsigned int hard_limit); /// Returns the set of required properties to compute the grouping - std::set requiredProperties() const; + std::set requiredProperties() const override; /// Handles a new Source void receiveSource(std::unique_ptr source) override; diff --git a/SEImplementation/SEImplementation/Grouping/GroupingFactory.h b/SEImplementation/SEImplementation/Grouping/GroupingFactory.h index 56bd3d6f3..cfaed6162 100644 --- a/SEImplementation/SEImplementation/Grouping/GroupingFactory.h +++ b/SEImplementation/SEImplementation/Grouping/GroupingFactory.h @@ -34,6 +34,8 @@ #include "SEImplementation/Grouping/MoffatCriteria.h" #include "SEImplementation/Grouping/AssocCriteria.h" +#include "SEImplementation/Grouping/SplitSourcesGrouping.h" + #include "SEImplementation/Configuration/GroupingConfig.h" namespace SourceXtractor { @@ -42,45 +44,20 @@ class GroupingFactory : public Configurable { public: - explicit GroupingFactory(std::shared_ptr source_group_factory) - : m_source_group_factory(source_group_factory), m_hard_limit(0) {} + explicit GroupingFactory(std::shared_ptr source_group_factory); virtual ~GroupingFactory() = default; - void reportConfigDependencies(Euclid::Configuration::ConfigManager& manager) const override { - manager.registerConfiguration(); - } - - void configure(Euclid::Configuration::ConfigManager& manager) override { - auto grouping_config = manager.getConfiguration(); - switch (grouping_config.getAlgorithmOption()) { - case GroupingConfig::Algorithm::NO_GROUPING: - m_grouping_criteria = std::make_shared(); - break; - case GroupingConfig::Algorithm::OVERLAPPING: - m_grouping_criteria = std::make_shared(); - break; - case GroupingConfig::Algorithm::SPLIT_SOURCES: - m_grouping_criteria = std::make_shared(); - break; - case GroupingConfig::Algorithm::MOFFAT: - m_grouping_criteria = std::make_shared(grouping_config.getMoffatThreshold(), grouping_config.getMoffatMaxDistance()); - break; - case GroupingConfig::Algorithm::ASSOC: - m_grouping_criteria = std::make_shared(); - break; - } - m_hard_limit = grouping_config.getHardLimit(); - } - - std::shared_ptr createGrouping() const { - assert(m_grouping_criteria != nullptr); - assert(m_source_group_factory != nullptr); - - return std::make_shared(m_grouping_criteria, m_source_group_factory, m_hard_limit); - } + void reportConfigDependencies(Euclid::Configuration::ConfigManager& manager) const override; + + void configure(Euclid::Configuration::ConfigManager& manager) override; + + std::shared_ptr createGrouping() const; private: + std::shared_ptr getCriteria() const; + + GroupingConfig::Algorithm m_algorithm; std::shared_ptr m_grouping_criteria; std::shared_ptr m_source_group_factory; unsigned int m_hard_limit; diff --git a/SEImplementation/SEImplementation/Grouping/SplitSourcesGrouping.h b/SEImplementation/SEImplementation/Grouping/SplitSourcesGrouping.h new file mode 100644 index 000000000..097688108 --- /dev/null +++ b/SEImplementation/SEImplementation/Grouping/SplitSourcesGrouping.h @@ -0,0 +1,47 @@ +/** Copyright © 2019 - 2023 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef _SEIMPLEMENTATION_GROUPING_SPLITSOURCESGROUPING_H_ +#define _SEIMPLEMENTATION_GROUPING_SPLITSOURCESGROUPING_H_ + +#include "SEFramework/Pipeline/SourceGrouping.h" + +namespace SourceXtractor { + +class SplitSourcesGrouping : public SourceGroupingInterface { +public: + + SplitSourcesGrouping(std::shared_ptr group_factory, unsigned int hard_limit); + + std::set requiredProperties() const override; + + /// Handles a new Source + void receiveSource(std::unique_ptr source) override; + + /// Handles a ProcessSourcesEvent to trigger the processing of some of the Sources stored in SourceGrouping + void receiveProcessSignal(const ProcessSourcesEvent& event) override; + +private: + std::shared_ptr m_group_factory; + std::map> m_source_groups; + unsigned int m_hard_limit; + +}; + +} + +#endif /* SEIMPLEMENTATION_SEIMPLEMENTATION_GROUPING_SPLITSOURCESGROUPING_H_ */ diff --git a/SEImplementation/src/lib/Grouping/GroupingFactory.cpp b/SEImplementation/src/lib/Grouping/GroupingFactory.cpp new file mode 100644 index 000000000..1e0042428 --- /dev/null +++ b/SEImplementation/src/lib/Grouping/GroupingFactory.cpp @@ -0,0 +1,66 @@ +/** Copyright © 2019-2023 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEImplementation/Grouping/GroupingFactory.h" + +namespace SourceXtractor { + +GroupingFactory::GroupingFactory(std::shared_ptr source_group_factory) + : m_source_group_factory(source_group_factory), m_hard_limit(0) {} + +void GroupingFactory::reportConfigDependencies(Euclid::Configuration::ConfigManager& manager) const { + manager.registerConfiguration(); +} + +void GroupingFactory::configure(Euclid::Configuration::ConfigManager& manager) { + auto grouping_config = manager.getConfiguration(); + m_algorithm = grouping_config.getAlgorithmOption(); + switch (m_algorithm) { + case GroupingConfig::Algorithm::NO_GROUPING: + m_grouping_criteria = std::make_shared(); + break; + case GroupingConfig::Algorithm::OVERLAPPING: + m_grouping_criteria = std::make_shared(); + break; + case GroupingConfig::Algorithm::SPLIT_SOURCES: + m_grouping_criteria = std::make_shared(); + break; + case GroupingConfig::Algorithm::MOFFAT: + m_grouping_criteria = std::make_shared(grouping_config.getMoffatThreshold(), grouping_config.getMoffatMaxDistance()); + break; + case GroupingConfig::Algorithm::ASSOC: + m_grouping_criteria = std::make_shared(); + break; + } + m_hard_limit = grouping_config.getHardLimit(); +} + +std::shared_ptr GroupingFactory::createGrouping() const { + assert(m_grouping_criteria != nullptr); + assert(m_source_group_factory != nullptr); + + // return optimized grouping if available, if not uses general grouping with criteria + switch (m_algorithm) { + case GroupingConfig::Algorithm::SPLIT_SOURCES: + return std::make_shared(m_source_group_factory, m_hard_limit); + default: + return std::make_shared(m_grouping_criteria, m_source_group_factory, m_hard_limit); + } +} + +} // SourceXtractor namespace + diff --git a/SEImplementation/src/lib/Grouping/SplitSourcesGrouping.cpp b/SEImplementation/src/lib/Grouping/SplitSourcesGrouping.cpp new file mode 100644 index 000000000..2f163140b --- /dev/null +++ b/SEImplementation/src/lib/Grouping/SplitSourcesGrouping.cpp @@ -0,0 +1,71 @@ +/** Copyright © 2019 - 2023 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEImplementation/Grouping/SplitSourcesGrouping.h" +#include "SEImplementation/Property/SourceId.h" + +namespace SourceXtractor { + +SplitSourcesGrouping::SplitSourcesGrouping(std::shared_ptr group_factory, unsigned int hard_limit) + : m_group_factory(group_factory), m_hard_limit(hard_limit) +{ +} + +std::set SplitSourcesGrouping::requiredProperties() const { + return {}; +} + +/// Handles a new Source +void SplitSourcesGrouping::receiveSource(std::unique_ptr source) { + auto source_id = source->getProperty().getDetectionId(); + + if (m_source_groups.find(source_id) == m_source_groups.end()) { + auto new_group = m_group_factory->createSourceGroup(); + m_source_groups[source_id] = std::move(new_group); + } + + m_source_groups.at(source_id)->addSource(std::move(source)); + + // TODO handle hard limit + // if (m_hard_limit > 0) { +} + +/// Handles a ProcessSourcesEvent to trigger the processing of some of the Sources stored in SourceGrouping +void SplitSourcesGrouping::receiveProcessSignal(const ProcessSourcesEvent& event) { + std::vector groups_to_process; + + // We iterate through all the SourceGroups we have + for (auto const& it : m_source_groups) { + // We look at its Sources and if we find at least one that needs to be processed + for (auto& source : *it.second) { + if (event.m_selection_criteria->mustBeProcessed(source)) { + groups_to_process.push_back(it.first); + break; + } + } + } + + // For each SourceGroup that we put in groups_to_process, + for (auto group_id : groups_to_process) { + // we remove it from our list of stored SourceGroups and notify our observers + sendSource(std::move(m_source_groups[group_id])); + m_source_groups.erase(group_id); + } +} + +} // SourceXtractor namespace + From 8248923236d73b015fee151c9860440501315c1a Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Mon, 20 Nov 2023 17:48:57 +0100 Subject: [PATCH 02/24] Optimized Assoc grouping --- .../SEImplementation/Grouping/AssocGrouping.h | 50 +++++++++++++ .../Grouping/GroupingFactory.h | 2 - .../Grouping/SplitSourcesGrouping.h | 2 +- .../src/lib/Grouping/AssocGrouping.cpp | 71 +++++++++++++++++++ .../src/lib/Grouping/GroupingFactory.cpp | 5 ++ 5 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 SEImplementation/SEImplementation/Grouping/AssocGrouping.h create mode 100644 SEImplementation/src/lib/Grouping/AssocGrouping.cpp diff --git a/SEImplementation/SEImplementation/Grouping/AssocGrouping.h b/SEImplementation/SEImplementation/Grouping/AssocGrouping.h new file mode 100644 index 000000000..e441a353a --- /dev/null +++ b/SEImplementation/SEImplementation/Grouping/AssocGrouping.h @@ -0,0 +1,50 @@ +/** Copyright © 2019 - 2023 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef _SEIMPLEMENTATION_GROUPING_ASSOCGROUPING_H_ +#define _SEIMPLEMENTATION_GROUPING_ASSOCGROUPING_H_ + +#include "SEFramework/Pipeline/SourceGrouping.h" + +namespace SourceXtractor { + +class AssocGrouping : public SourceGroupingInterface { +public: + + AssocGrouping(std::shared_ptr group_factory, unsigned int hard_limit); + + std::set requiredProperties() const override; + + /// Handles a new Source + void receiveSource(std::unique_ptr source) override; + + /// Handles a ProcessSourcesEvent to trigger the processing of some of the Sources stored in SourceGrouping + void receiveProcessSignal(const ProcessSourcesEvent& event) override; + +private: + std::shared_ptr m_group_factory; + std::map> m_source_groups; + unsigned int m_hard_limit; + +}; + +} + + + + +#endif /* _SEIMPLEMENTATION_GROUPING_ASSOCGROUPING_H_ */ diff --git a/SEImplementation/SEImplementation/Grouping/GroupingFactory.h b/SEImplementation/SEImplementation/Grouping/GroupingFactory.h index cfaed6162..43939eaf2 100644 --- a/SEImplementation/SEImplementation/Grouping/GroupingFactory.h +++ b/SEImplementation/SEImplementation/Grouping/GroupingFactory.h @@ -34,8 +34,6 @@ #include "SEImplementation/Grouping/MoffatCriteria.h" #include "SEImplementation/Grouping/AssocCriteria.h" -#include "SEImplementation/Grouping/SplitSourcesGrouping.h" - #include "SEImplementation/Configuration/GroupingConfig.h" namespace SourceXtractor { diff --git a/SEImplementation/SEImplementation/Grouping/SplitSourcesGrouping.h b/SEImplementation/SEImplementation/Grouping/SplitSourcesGrouping.h index 097688108..908397590 100644 --- a/SEImplementation/SEImplementation/Grouping/SplitSourcesGrouping.h +++ b/SEImplementation/SEImplementation/Grouping/SplitSourcesGrouping.h @@ -44,4 +44,4 @@ class SplitSourcesGrouping : public SourceGroupingInterface { } -#endif /* SEIMPLEMENTATION_SEIMPLEMENTATION_GROUPING_SPLITSOURCESGROUPING_H_ */ +#endif /* _SEIMPLEMENTATION_GROUPING_SPLITSOURCESGROUPING_H_ */ diff --git a/SEImplementation/src/lib/Grouping/AssocGrouping.cpp b/SEImplementation/src/lib/Grouping/AssocGrouping.cpp new file mode 100644 index 000000000..8c860d36e --- /dev/null +++ b/SEImplementation/src/lib/Grouping/AssocGrouping.cpp @@ -0,0 +1,71 @@ +/** Copyright © 2019 - 2023 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEImplementation/Grouping/AssocGrouping.h" +#include "SEImplementation/Plugin/AssocMode/AssocMode.h" + +namespace SourceXtractor { + +AssocGrouping::AssocGrouping(std::shared_ptr group_factory, unsigned int hard_limit) + : m_group_factory(group_factory), m_hard_limit(hard_limit) +{ +} + +std::set AssocGrouping::requiredProperties() const { + return { PropertyId::create(), }; +} + +/// Handles a new Source +void AssocGrouping::receiveSource(std::unique_ptr source) { + auto source_id = source->getProperty().getGroupId(); + + if (m_source_groups.find(source_id) == m_source_groups.end()) { + auto new_group = m_group_factory->createSourceGroup(); + m_source_groups[source_id] = std::move(new_group); + } + + m_source_groups.at(source_id)->addSource(std::move(source)); + + // TODO handle hard limit + // if (m_hard_limit > 0) { +} + +/// Handles a ProcessSourcesEvent to trigger the processing of some of the Sources stored in SourceGrouping +void AssocGrouping::receiveProcessSignal(const ProcessSourcesEvent& event) { + std::vector groups_to_process; + + // We iterate through all the SourceGroups we have + for (auto const& it : m_source_groups) { + // We look at its Sources and if we find at least one that needs to be processed + for (auto& source : *it.second) { + if (event.m_selection_criteria->mustBeProcessed(source)) { + groups_to_process.push_back(it.first); + break; + } + } + } + + // For each SourceGroup that we put in groups_to_process, + for (auto group_id : groups_to_process) { + // we remove it from our list of stored SourceGroups and notify our observers + sendSource(std::move(m_source_groups[group_id])); + m_source_groups.erase(group_id); + } +} + +} // SourceXtractor namespace + diff --git a/SEImplementation/src/lib/Grouping/GroupingFactory.cpp b/SEImplementation/src/lib/Grouping/GroupingFactory.cpp index 1e0042428..459f8b861 100644 --- a/SEImplementation/src/lib/Grouping/GroupingFactory.cpp +++ b/SEImplementation/src/lib/Grouping/GroupingFactory.cpp @@ -17,6 +17,9 @@ #include "SEImplementation/Grouping/GroupingFactory.h" +#include "SEImplementation/Grouping/SplitSourcesGrouping.h" +#include "SEImplementation/Grouping/AssocGrouping.h" + namespace SourceXtractor { GroupingFactory::GroupingFactory(std::shared_ptr source_group_factory) @@ -57,6 +60,8 @@ std::shared_ptr GroupingFactory::createGrouping() const switch (m_algorithm) { case GroupingConfig::Algorithm::SPLIT_SOURCES: return std::make_shared(m_source_group_factory, m_hard_limit); + case GroupingConfig::Algorithm::ASSOC: + return std::make_shared(m_source_group_factory, m_hard_limit); default: return std::make_shared(m_grouping_criteria, m_source_group_factory, m_hard_limit); } From 6040b5ae4d74c8ded5601b53447e428baa9e10e5 Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Tue, 21 Nov 2023 12:40:22 +0100 Subject: [PATCH 03/24] missing include on f39 --- SEImplementation/SEImplementation/Grouping/AssocGrouping.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/SEImplementation/SEImplementation/Grouping/AssocGrouping.h b/SEImplementation/SEImplementation/Grouping/AssocGrouping.h index e441a353a..9916b85d1 100644 --- a/SEImplementation/SEImplementation/Grouping/AssocGrouping.h +++ b/SEImplementation/SEImplementation/Grouping/AssocGrouping.h @@ -18,6 +18,8 @@ #ifndef _SEIMPLEMENTATION_GROUPING_ASSOCGROUPING_H_ #define _SEIMPLEMENTATION_GROUPING_ASSOCGROUPING_H_ +#include + #include "SEFramework/Pipeline/SourceGrouping.h" namespace SourceXtractor { @@ -44,7 +46,4 @@ class AssocGrouping : public SourceGroupingInterface { } - - - #endif /* _SEIMPLEMENTATION_GROUPING_ASSOCGROUPING_H_ */ From f50fc30aff4f320966135612ff6fcb0f24d3ee1a Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Tue, 21 Nov 2023 13:35:59 +0100 Subject: [PATCH 04/24] missing include on f39 (2) --- .../SEImplementation/Grouping/SplitSourcesGrouping.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SEImplementation/SEImplementation/Grouping/SplitSourcesGrouping.h b/SEImplementation/SEImplementation/Grouping/SplitSourcesGrouping.h index 908397590..7b60486d5 100644 --- a/SEImplementation/SEImplementation/Grouping/SplitSourcesGrouping.h +++ b/SEImplementation/SEImplementation/Grouping/SplitSourcesGrouping.h @@ -20,6 +20,8 @@ #include "SEFramework/Pipeline/SourceGrouping.h" +#include + namespace SourceXtractor { class SplitSourcesGrouping : public SourceGroupingInterface { From 042d032377e18c72d4e6cd9d5f746169d5501007 Mon Sep 17 00:00:00 2001 From: Marc Schefer <31961290+marcschefer@users.noreply.github.com> Date: Tue, 19 Dec 2023 14:10:31 +0100 Subject: [PATCH 05/24] Update CMakeLists.txt Bump version --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b5a20d1d1..d0d48383d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,4 +6,4 @@ find_package(ElementsProject) #--------------------------------------------------------------- # Declare project name and version -elements_project(SourceXtractorPlusPlus 0.21 USE Alexandria 2.31.0 DESCRIPTION "SourceXtractor++, the next generation SExtractor") +elements_project(SourceXtractorPlusPlus 0.22 USE Alexandria 2.31.0 DESCRIPTION "SourceXtractor++, the next generation SExtractor") From f18369f93e4d43b308f920ed1eb1d2a757464972 Mon Sep 17 00:00:00 2001 From: Marc Schefer <31961290+marcschefer@users.noreply.github.com> Date: Wed, 10 Jan 2024 17:08:17 +0100 Subject: [PATCH 06/24] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bfacc104d..8cbd84e92 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,13 @@ Documentation: https://astrorama.github.io/SourceXtractorPlusPlus/ SourceXtractor++ is available on [Anaconda Cloud for Linux and MacOSX](https://anaconda.org/astrorama/sourcextractor) ```bash -conda install -c conda-forge -c astrorama sourcextractor==0.19.2 +conda install -c conda-forge -c astrorama sourcextractor==0.21 ``` We would recommend, however, to install it into its own environment. ```bash -conda create -n sourcex -c astrorama -c conda-forge sourcextractor==0.19.2 +conda create -n sourcex -c astrorama -c conda-forge sourcextractor==0.21 conda activate sourcex ``` From 70c694ecd86449e6d21a55a67318cf588f72694d Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Tue, 16 Jan 2024 16:51:22 +0100 Subject: [PATCH 07/24] fix filtered check images regression --- .../src/lib/Configuration/DetectionFrameConfig.cpp | 3 --- SEMain/src/program/SourceXtractor.cpp | 7 +++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/SEImplementation/src/lib/Configuration/DetectionFrameConfig.cpp b/SEImplementation/src/lib/Configuration/DetectionFrameConfig.cpp index 3e5f244e4..c1dcba3a8 100644 --- a/SEImplementation/src/lib/Configuration/DetectionFrameConfig.cpp +++ b/SEImplementation/src/lib/Configuration/DetectionFrameConfig.cpp @@ -108,9 +108,6 @@ void DetectionFrameConfig::initialize(const UserValues& ) { detection_frame->setHduIndex(i); CheckImages::getInstance().addVarianceCheckImage(detection_frame->getImage(FrameImageLayer::LayerVarianceMap)); - CheckImages::getInstance().addFilteredCheckImage(detection_frame->getImage(FrameImageLayer::LayerFilteredImage)); - CheckImages::getInstance().addThresholdedCheckImage(detection_frame->getImage(FrameImageLayer::LayerThresholdedImage)); - CheckImages::getInstance().addSnrCheckImage(detection_frame->getImage(FrameImageLayer::LayerSignalToNoiseMap)); m_frames.emplace_back(detection_frame); } diff --git a/SEMain/src/program/SourceXtractor.cpp b/SEMain/src/program/SourceXtractor.cpp index 579dcb179..bb9507175 100644 --- a/SEMain/src/program/SourceXtractor.cpp +++ b/SEMain/src/program/SourceXtractor.cpp @@ -483,6 +483,13 @@ class SEMain : public Elements::Program { } measurement->stopThreads(); + // Those check images can only be added AFTER the processing of the detection frames + for (auto& detection_frame : detection_frames) { + CheckImages::getInstance().addFilteredCheckImage(detection_frame->getFilteredImage()); + CheckImages::getInstance().addThresholdedCheckImage(detection_frame->getThresholdedImage()); + CheckImages::getInstance().addSnrCheckImage(detection_frame->getSnrImage()); + } + CheckImages::getInstance().saveImages(); TileManager::getInstance()->flush(); progress_mediator->done(); From e4a996c9475234b5da0dd1b0eda6f36e43dc146a Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Wed, 13 Dec 2023 16:56:58 +0100 Subject: [PATCH 08/24] wip --- .../Grouping/MoffatGrouping.h | 45 ++++++ .../lib/Configuration/CheckImagesConfig.cpp | 2 +- .../src/lib/Grouping/MoffatCriteria.cpp | 2 +- .../src/lib/Grouping/MoffatGrouping.cpp | 52 +++++++ SEUtils/SEUtils/QuadTree.h | 49 ++++++ SEUtils/SEUtils/_impl/QuadTree.icpp | 146 ++++++++++++++++++ 6 files changed, 294 insertions(+), 2 deletions(-) create mode 100644 SEImplementation/SEImplementation/Grouping/MoffatGrouping.h create mode 100644 SEImplementation/src/lib/Grouping/MoffatGrouping.cpp create mode 100644 SEUtils/SEUtils/QuadTree.h create mode 100644 SEUtils/SEUtils/_impl/QuadTree.icpp diff --git a/SEImplementation/SEImplementation/Grouping/MoffatGrouping.h b/SEImplementation/SEImplementation/Grouping/MoffatGrouping.h new file mode 100644 index 000000000..5382fd1c1 --- /dev/null +++ b/SEImplementation/SEImplementation/Grouping/MoffatGrouping.h @@ -0,0 +1,45 @@ +/* + * MoffatGrouping.h + * + * Created on: Nov 22, 2023 + * Author: mschefer + */ + +#ifndef _SEIMPLEMENTATION_GROUPING_MOFFATGROUPING_H_ +#define _SEIMPLEMENTATION_GROUPING_MOFFATGROUPING_H_ + +#include "SEFramework/Pipeline/SourceGrouping.h" + +#include + +namespace SourceXtractor { + +class MoffatGrouping : public SourceGroupingInterface { +public: + + MoffatGrouping(std::shared_ptr group_factory, unsigned int hard_limit, float max_range); + + std::set requiredProperties() const override; + + /// Handles a new Source + void receiveSource(std::unique_ptr source) override; + + /// Handles a ProcessSourcesEvent to trigger the processing of some of the Sources stored in SourceGrouping + void receiveProcessSignal(const ProcessSourcesEvent& event) override; + +private: + std::shared_ptr m_group_factory; + std::map> m_source_groups; + unsigned int m_hard_limit; + float m_max_range; + + struct GridCoord { + int x, y; + }; + + std::map>> m_groups; +}; + +} + +#endif /* _SEIMPLEMENTATION_GROUPING_MOFFATGROUPING_H_ */ diff --git a/SEImplementation/src/lib/Configuration/CheckImagesConfig.cpp b/SEImplementation/src/lib/Configuration/CheckImagesConfig.cpp index 25ec51893..c2ba343ca 100644 --- a/SEImplementation/src/lib/Configuration/CheckImagesConfig.cpp +++ b/SEImplementation/src/lib/Configuration/CheckImagesConfig.cpp @@ -87,7 +87,7 @@ std::map CheckImagesConfig::g "Path to save the PSF check image"}, {CHECK_ML_DETECTION.c_str(), po::value()->default_value(""), "Path to save the ML detection check images"} - }}, {"Debug options (Use with caution!)", { + }}, {"Debug options (Use with caution!)", { {CHECK_MOFFAT.c_str(), po::value()->default_value(""), "Path to save the moffat debug image (VERY SLOW)"} }}}; diff --git a/SEImplementation/src/lib/Grouping/MoffatCriteria.cpp b/SEImplementation/src/lib/Grouping/MoffatCriteria.cpp index 002d8ef3b..5c716b9dc 100644 --- a/SEImplementation/src/lib/Grouping/MoffatCriteria.cpp +++ b/SEImplementation/src/lib/Grouping/MoffatCriteria.cpp @@ -43,7 +43,7 @@ bool MoffatCriteria::doesImpact(const SourceInterface& impactor, const SourceInt auto dx = centroid.getCentroidX() - other_centroid.getCentroidX(); auto dy = centroid.getCentroidY() - other_centroid.getCentroidY(); - if (dx*dx + dy*dy > m_max_distance * m_max_distance ) { + if (dx*dx + dy*dy > m_max_distance * m_max_distance) { return false; } diff --git a/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp b/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp new file mode 100644 index 000000000..72729d982 --- /dev/null +++ b/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp @@ -0,0 +1,52 @@ +/** Copyright © 2019 - 2023 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEUtils/QuadTree.h" + +#include "SEImplementation/Grouping/MoffatGrouping.h" + +#include "SEImplementation/Plugin/MoffatModelFitting/MoffatModelFitting.h" +#include "SEImplementation/Plugin/MoffatModelFitting/MoffatModelEvaluator.h" +#include "SEImplementation/Plugin/PixelCentroid/PixelCentroid.h" +#include "SEImplementation/Plugin/PeakValue/PeakValue.h" + +namespace SourceXtractor { + +MoffatGrouping::MoffatGrouping( + std::shared_ptr group_factory, unsigned int hard_limit, float max_range) + : m_group_factory(group_factory), m_hard_limit(hard_limit), m_max_range(max_range) +{ +} + +std::set MoffatGrouping::requiredProperties() const { + return { + PropertyId::create(), + PropertyId::create(), + PropertyId::create(), + }; +} + +/// Handles a new Source +void MoffatGrouping::receiveSource(std::unique_ptr source) { +} + +/// Handles a ProcessSourcesEvent to trigger the processing of some of the Sources stored in SourceGrouping +void MoffatGrouping::receiveProcessSignal(const ProcessSourcesEvent& event) { +} + +} // SourceXtractor namespace + diff --git a/SEUtils/SEUtils/QuadTree.h b/SEUtils/SEUtils/QuadTree.h new file mode 100644 index 000000000..69ecdc165 --- /dev/null +++ b/SEUtils/SEUtils/QuadTree.h @@ -0,0 +1,49 @@ +/* + * QuadTree.h + * + * Created on: Nov 28, 2023 + * Author: mschefer + */ + +#ifndef _SEUTILS_QUADTREE_H_ +#define _SEUTILS_QUADTREE_H_ + + +#include +#include +#include + +namespace SourceXtractor { + +template +struct QuadTreeTraits; + +template +class QuadTree { +public: + using Traits = QuadTreeTraits; + + struct Coord { + double x, y; + }; + + void add(T&& data); + +private: + class Node; + class Branch; + class Leaf; + + std::shared_ptr m_root; +}; + +template +struct QuadTreeTraits { + static typename QuadTree::Coord getCoord(const T& t); +}; + +} + +#include "_impl/QuadTree.icpp" + +#endif /* _SEUTILS_QUADTREE_H_ */ diff --git a/SEUtils/SEUtils/_impl/QuadTree.icpp b/SEUtils/SEUtils/_impl/QuadTree.icpp new file mode 100644 index 000000000..69b41fd0d --- /dev/null +++ b/SEUtils/SEUtils/_impl/QuadTree.icpp @@ -0,0 +1,146 @@ +/** Copyright © 2021-2023 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +namespace SourceXtractor { + +template +class QuadTree::Node { +public: + //virtual std::vector findPointsWithinRadius(Coord coord, double radius) const = 0; + virtual ~Node() = default; + + virtual void add(T&& data) = 0; + +}; + +template +class QuadTree::Branch : public Node +{ +public: + //virtual std::vector findPointsWithinRadius(Coord coord, double radius) const = 0; + virtual ~Branch() = default; + + Branch(Coord min_coord, Coord max_coord) + : m_min(min_coord), m_max(max_coord) {} + + void add(T&& data) override { + auto c = Traits::getCoords(data); + + auto quad = getQuadrant(c); + if (m_sub_trees[quad] == nullptr) { + m_sub_trees[quad] = std::make_shared(); + } + m_sub_trees[quad].add(std::move(data)); + } + + size_t getQuadrant(Coord c) const { + size_t quadrant = 0; + if (c.x < (m_min.x + m_max.x) / 2.0) { + quadrant += 1; + } + if (c.y < (m_min.y + m_max.y) / 2.0) { + quadrant += 2; + } + return quadrant; + } + + bool isFull() const override { + return false; + } + + Coord getMin() const { + return m_min; + } + + Coord getMax() const { + return m_min; + } + +private: + Coord m_min, m_max; + std::shared_ptr m_sub_trees[4]; +}; + +template +class QuadTree::Leaf : public Node { +public: + //virtual std::vector findPointsWithinRadius(Coord coord, double radius) const = 0; + + Leaf() { + } + virtual ~Leaf() = default; + + void add(T&& data) override { + auto c = Traits::getCoord(data); + + if (m_data.empty()) { + m_min.x = m_max.x = c.x; + m_min.y = m_max.y = c.y; + } else { + m_min.x = std::min(m_min.x, c.x); + m_min.y = std::min(m_min.y, c.y); + m_max.x = std::max(m_max.x, c.x); + m_max.y = std::max(m_max.y, c.y); + } + + m_data.emplace_back(std::move(data)); + } + + std::shared_ptr split() { + auto branch = std::make_shared(); + } + + bool isFull() const override { + return m_data.size() < N; + } + + size_t size() const { + return m_data.size(); + } + + Coord getMin() const { + return m_min; + } + + Coord getMax() const { + return m_min; + } + +private: + Coord m_min, m_max; + std::vector m_data; +}; + +template +void QuadTree::add(T&& data) { + if (m_root == nullptr) { + m_root = std::make_shared(); + } + + auto leaf = std::dynamic_pointer_cast(m_root); + if (leaf != nullptr && leaf.isFull()) { + auto leaf_min = leaf->getMin(); + auto leaf_max = leaf->getMax(); + + leaf_max.x = leaf_max.y = std::max(leaf_max.x, leaf_max.y); + m_root = std::make_shared(Coord {0,0}, leaf_max); + } else { + m_root.add(std::move(data)); + } +} + +} From 2b3860033a96dc89e1c396b9c4f355f57961eb89 Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Wed, 24 Jan 2024 17:37:05 +0100 Subject: [PATCH 09/24] added hard-limit to new grouping --- SEImplementation/src/lib/Grouping/AssocGrouping.cpp | 13 ++++++++++--- .../src/lib/Grouping/SplitSourcesGrouping.cpp | 13 ++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/SEImplementation/src/lib/Grouping/AssocGrouping.cpp b/SEImplementation/src/lib/Grouping/AssocGrouping.cpp index 8c860d36e..75f1b6d80 100644 --- a/SEImplementation/src/lib/Grouping/AssocGrouping.cpp +++ b/SEImplementation/src/lib/Grouping/AssocGrouping.cpp @@ -38,10 +38,17 @@ void AssocGrouping::receiveSource(std::unique_ptr source) { m_source_groups[source_id] = std::move(new_group); } - m_source_groups.at(source_id)->addSource(std::move(source)); + if (m_source_groups.at(source_id)->size() >= m_hard_limit) { + // the stored group has reached the hard limit + // send the current group to processing + sendSource(std::move(m_source_groups.at(source_id))); + + // and replace it with a new empty one + auto new_group = m_group_factory->createSourceGroup(); + m_source_groups[source_id] = std::move(new_group); + } - // TODO handle hard limit - // if (m_hard_limit > 0) { + m_source_groups.at(source_id)->addSource(std::move(source)); } /// Handles a ProcessSourcesEvent to trigger the processing of some of the Sources stored in SourceGrouping diff --git a/SEImplementation/src/lib/Grouping/SplitSourcesGrouping.cpp b/SEImplementation/src/lib/Grouping/SplitSourcesGrouping.cpp index 2f163140b..d92923d8d 100644 --- a/SEImplementation/src/lib/Grouping/SplitSourcesGrouping.cpp +++ b/SEImplementation/src/lib/Grouping/SplitSourcesGrouping.cpp @@ -38,10 +38,17 @@ void SplitSourcesGrouping::receiveSource(std::unique_ptr source m_source_groups[source_id] = std::move(new_group); } - m_source_groups.at(source_id)->addSource(std::move(source)); + if (m_source_groups.at(source_id)->size() >= m_hard_limit) { + // the stored group has reached the hard limit + // send the current group to processing + sendSource(std::move(m_source_groups.at(source_id))); + + // and replace it with a new empty one + auto new_group = m_group_factory->createSourceGroup(); + m_source_groups[source_id] = std::move(new_group); + } - // TODO handle hard limit - // if (m_hard_limit > 0) { + m_source_groups.at(source_id)->addSource(std::move(source)); } /// Handles a ProcessSourcesEvent to trigger the processing of some of the Sources stored in SourceGrouping From 95cf53a36854e7e3c0bc8ac07be73d1b19fd6901 Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Fri, 9 Feb 2024 17:57:56 +0100 Subject: [PATCH 10/24] MoffatGrouping optimization, reimplemented QuadTree --- .../SEImplementation/Grouping/AssocGrouping.h | 1 + .../Grouping/MoffatGrouping.h | 24 +- .../Grouping/SplitSourcesGrouping.h | 1 + .../src/lib/Grouping/GroupingFactory.cpp | 5 + .../src/lib/Grouping/MoffatGrouping.cpp | 79 ++++++- SEUtils/SEUtils/QuadTree.h | 38 +-- SEUtils/SEUtils/_impl/QuadTree.icpp | 221 ++++++++++-------- 7 files changed, 254 insertions(+), 115 deletions(-) diff --git a/SEImplementation/SEImplementation/Grouping/AssocGrouping.h b/SEImplementation/SEImplementation/Grouping/AssocGrouping.h index 9916b85d1..7e5e67a79 100644 --- a/SEImplementation/SEImplementation/Grouping/AssocGrouping.h +++ b/SEImplementation/SEImplementation/Grouping/AssocGrouping.h @@ -28,6 +28,7 @@ class AssocGrouping : public SourceGroupingInterface { public: AssocGrouping(std::shared_ptr group_factory, unsigned int hard_limit); + virtual ~AssocGrouping() = default; std::set requiredProperties() const override; diff --git a/SEImplementation/SEImplementation/Grouping/MoffatGrouping.h b/SEImplementation/SEImplementation/Grouping/MoffatGrouping.h index 5382fd1c1..8e896e8bf 100644 --- a/SEImplementation/SEImplementation/Grouping/MoffatGrouping.h +++ b/SEImplementation/SEImplementation/Grouping/MoffatGrouping.h @@ -9,15 +9,27 @@ #define _SEIMPLEMENTATION_GROUPING_MOFFATGROUPING_H_ #include "SEFramework/Pipeline/SourceGrouping.h" +#include "SEUtils/QuadTree.h" #include +#include +#include namespace SourceXtractor { class MoffatGrouping : public SourceGroupingInterface { public: + struct SourceInfo { + std::unique_ptr m_source; + double m_x, m_y; + size_t m_group_id; + }; + + using Group = std::vector>; - MoffatGrouping(std::shared_ptr group_factory, unsigned int hard_limit, float max_range); + MoffatGrouping(std::shared_ptr grouping_criteria, + std::shared_ptr group_factory, unsigned int hard_limit, float max_range); + virtual ~MoffatGrouping() = default; std::set requiredProperties() const override; @@ -28,16 +40,14 @@ class MoffatGrouping : public SourceGroupingInterface { void receiveProcessSignal(const ProcessSourcesEvent& event) override; private: + std::shared_ptr m_grouping_criteria; std::shared_ptr m_group_factory; - std::map> m_source_groups; unsigned int m_hard_limit; float m_max_range; - struct GridCoord { - int x, y; - }; - - std::map>> m_groups; + size_t m_group_counter; + std::map> m_groups; + QuadTree> m_tree; }; } diff --git a/SEImplementation/SEImplementation/Grouping/SplitSourcesGrouping.h b/SEImplementation/SEImplementation/Grouping/SplitSourcesGrouping.h index 7b60486d5..81db92714 100644 --- a/SEImplementation/SEImplementation/Grouping/SplitSourcesGrouping.h +++ b/SEImplementation/SEImplementation/Grouping/SplitSourcesGrouping.h @@ -28,6 +28,7 @@ class SplitSourcesGrouping : public SourceGroupingInterface { public: SplitSourcesGrouping(std::shared_ptr group_factory, unsigned int hard_limit); + virtual ~SplitSourcesGrouping() = default; std::set requiredProperties() const override; diff --git a/SEImplementation/src/lib/Grouping/GroupingFactory.cpp b/SEImplementation/src/lib/Grouping/GroupingFactory.cpp index 459f8b861..9bf4c938c 100644 --- a/SEImplementation/src/lib/Grouping/GroupingFactory.cpp +++ b/SEImplementation/src/lib/Grouping/GroupingFactory.cpp @@ -19,6 +19,7 @@ #include "SEImplementation/Grouping/SplitSourcesGrouping.h" #include "SEImplementation/Grouping/AssocGrouping.h" +#include "SEImplementation/Grouping/MoffatGrouping.h" namespace SourceXtractor { @@ -62,6 +63,10 @@ std::shared_ptr GroupingFactory::createGrouping() const return std::make_shared(m_source_group_factory, m_hard_limit); case GroupingConfig::Algorithm::ASSOC: return std::make_shared(m_source_group_factory, m_hard_limit); + case GroupingConfig::Algorithm::MOFFAT: + //m_grouping_criteria = std::make_shared(grouping_config.getMoffatThreshold(), grouping_config.getMoffatMaxDistance()); + return std::make_shared(m_grouping_criteria, m_source_group_factory, m_hard_limit, 500); // FIXME!!!!!!!!!!!!! hardcoded + break; default: return std::make_shared(m_grouping_criteria, m_source_group_factory, m_hard_limit); } diff --git a/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp b/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp index 72729d982..4607537d2 100644 --- a/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp +++ b/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp @@ -17,7 +17,10 @@ #include "SEUtils/QuadTree.h" +#include + #include "SEImplementation/Grouping/MoffatGrouping.h" +#include "SEImplementation/Grouping/MoffatCriteria.h" #include "SEImplementation/Plugin/MoffatModelFitting/MoffatModelFitting.h" #include "SEImplementation/Plugin/MoffatModelFitting/MoffatModelEvaluator.h" @@ -26,9 +29,22 @@ namespace SourceXtractor { +template <> +struct QuadTreeTraits> { + static double getCoord(std::shared_ptr t, size_t index) { + if (index == 0) { + return t->m_x; + } else { + return t->m_y; + } + } +}; + + MoffatGrouping::MoffatGrouping( - std::shared_ptr group_factory, unsigned int hard_limit, float max_range) - : m_group_factory(group_factory), m_hard_limit(hard_limit), m_max_range(max_range) + std::shared_ptr grouping_criteria, std::shared_ptr group_factory, unsigned int hard_limit, float max_range) + : m_grouping_criteria(grouping_criteria), m_group_factory(group_factory), m_hard_limit(hard_limit), m_max_range(max_range), m_group_counter(0) + { } @@ -42,10 +58,69 @@ std::set MoffatGrouping::requiredProperties() const { /// Handles a new Source void MoffatGrouping::receiveSource(std::unique_ptr source) { + + // Encapsulates the source unique_ptr + auto& centroid = source->getProperty(); + auto source_info = std::make_shared(); + source_info->m_source = std::move(source); + source_info->m_x = centroid.getCentroidX(); + source_info->m_y = centroid.getCentroidY(); + source_info->m_group_id = m_group_counter++; + + auto group = std::make_shared(); + m_groups[source_info->m_group_id] = group; + + // Find sources within range + auto sources = m_tree.getPointsWithinRange({source_info->m_x, source_info->m_y}, m_max_range); + + std::set matching_groups; + for (auto& s : sources) { + // check only sources that belong to a group we don't match with + if (matching_groups.find(s->m_group_id) == matching_groups.end() && + m_grouping_criteria->shouldGroup(*source_info->m_source, *s->m_source)) { + matching_groups.insert(s->m_group_id); + } + } + + // merge groups and keep the new group + for (auto group_id : matching_groups) { + // merge group + group->insert(group->end(), m_groups.at(group_id)->begin(), m_groups.at(group_id)->end()); + m_groups.erase(group_id); + } + + // Add source to the Quad Tree + m_tree.add(source_info); } /// Handles a ProcessSourcesEvent to trigger the processing of some of the Sources stored in SourceGrouping void MoffatGrouping::receiveProcessSignal(const ProcessSourcesEvent& event) { + std::vector groups_to_process; + + // We iterate through all the SourceGroups we have + for (auto const& it : m_groups) { + // We look at its Sources and if we find at least one that needs to be processed + for (auto& source : *it.second) { + if (event.m_selection_criteria->mustBeProcessed(*source->m_source)) { + groups_to_process.push_back(it.first); + break; + } + } + } + + // For each SourceGroup that we put in groups_to_process, + for (auto group_id : groups_to_process) { + // we remove it from our list of stored SourceGroups and notify our observers + auto new_group = m_group_factory->createSourceGroup(); + + for (auto& source_info : *m_groups.at(group_id)) { + new_group->addSource(std::move(source_info->m_source)); + m_tree.remove(source_info); + } + + //sendSource(std::move(m_source_groups[group_id])); + m_groups.erase(group_id); + } } } // SourceXtractor namespace diff --git a/SEUtils/SEUtils/QuadTree.h b/SEUtils/SEUtils/QuadTree.h index 69ecdc165..804238745 100644 --- a/SEUtils/SEUtils/QuadTree.h +++ b/SEUtils/SEUtils/QuadTree.h @@ -8,17 +8,17 @@ #ifndef _SEUTILS_QUADTREE_H_ #define _SEUTILS_QUADTREE_H_ - #include #include -#include namespace SourceXtractor { -template -struct QuadTreeTraits; +template +struct QuadTreeTraits { + static double getCoord(const T& t, size_t index); +}; -template +template class QuadTree { public: using Traits = QuadTreeTraits; @@ -27,19 +27,29 @@ class QuadTree { double x, y; }; - void add(T&& data); + QuadTree(size_t capacity=100); + QuadTree(const QuadTree& tree); + + void add(const T& data); + void remove(const T& data); + std::vector getPointsWithinRange(Coord c, double range) const; private: - class Node; - class Branch; - class Leaf; + void addLocally(const T& data); + void split(); + void expand(Coord c); + size_t getQuadrant(Coord c) const; + bool isContained(Coord c) const; - std::shared_ptr m_root; -}; + Coord getQuadrantMin(size_t quadrant) const; + Coord getQuadrantMax(size_t quadrant) const; -template -struct QuadTreeTraits { - static typename QuadTree::Coord getCoord(const T& t); + size_t m_capacity; + + bool m_is_divided; + Coord m_min, m_max; + std::vector m_data; + std::shared_ptr> m_sub_trees[4]; }; } diff --git a/SEUtils/SEUtils/_impl/QuadTree.icpp b/SEUtils/SEUtils/_impl/QuadTree.icpp index 69b41fd0d..e1d44ccd3 100644 --- a/SEUtils/SEUtils/_impl/QuadTree.icpp +++ b/SEUtils/SEUtils/_impl/QuadTree.icpp @@ -15,132 +15,169 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -namespace SourceXtractor { - -template -class QuadTree::Node { -public: - //virtual std::vector findPointsWithinRadius(Coord coord, double radius) const = 0; - virtual ~Node() = default; +#include +#include // For std::find - virtual void add(T&& data) = 0; +namespace SourceXtractor { -}; +template +QuadTree::QuadTree(size_t capacity) : m_capacity(capacity), m_is_divided(false) { +} -template -class QuadTree::Branch : public Node -{ -public: - //virtual std::vector findPointsWithinRadius(Coord coord, double radius) const = 0; - virtual ~Branch() = default; +template +QuadTree::QuadTree(const QuadTree& tree) { + m_capacity = tree.m_capacity; + m_is_divided = tree.m_is_divided; + m_min = tree.m_min; + m_max = tree.m_max; + m_data = tree.m_data; + for (int i = 0; i<4; i++) { + m_sub_trees[i] = tree.m_sub_trees[i]; + } +} - Branch(Coord min_coord, Coord max_coord) - : m_min(min_coord), m_max(max_coord) {} +template +void QuadTree::add(const T& data) { + if (m_is_divided) { + auto c = Coord { Traits::getCoord(data, 0), Traits::getCoord(data, 1) }; - void add(T&& data) override { - auto c = Traits::getCoords(data); + while (!isContained(c)) { + expand(c); + } auto quad = getQuadrant(c); if (m_sub_trees[quad] == nullptr) { - m_sub_trees[quad] = std::make_shared(); + m_sub_trees[quad] = std::make_shared(m_capacity); + m_sub_trees[quad]->m_min = getQuadrantMin(quad); + m_sub_trees[quad]->m_max = getQuadrantMax(quad); + } + m_sub_trees[quad]->add(data); + } else { + addLocally(data); + if (m_data.size() >= m_capacity) { + split(); } - m_sub_trees[quad].add(std::move(data)); } +} - size_t getQuadrant(Coord c) const { - size_t quadrant = 0; - if (c.x < (m_min.x + m_max.x) / 2.0) { - quadrant += 1; +template +void QuadTree::remove(const T& data) { + if (m_is_divided) { + auto c = Coord { Traits::getCoord(data, 0), Traits::getCoord(data, 1) }; + auto quad = getQuadrant(c); + if (m_sub_trees[quad] != nullptr) { + m_sub_trees[quad]->remove(data); } - if (c.y < (m_min.y + m_max.y) / 2.0) { - quadrant += 2; + } else { + auto it = std::find(m_data.begin(), m_data.end(), data); + if (it != m_data.end()) { + m_data.erase(it); } - return quadrant; } +} - bool isFull() const override { - return false; +template +std::vector QuadTree::getPointsWithinRange(Coord c, double range) const { + if (m_is_divided) { + std::vector points; + for (int i=0; i<4; i++) { + // FIXME only if overlap + auto subtree_points = m_sub_trees[i]->getPointsWithinRange(c, range); + points.insert(points.end(), subtree_points.begin(), subtree_points.end()); + } + return points; + } else { + return m_data; } +} - Coord getMin() const { - return m_min; - } +template +void QuadTree::addLocally(const T& data) { + assert(!m_is_divided); - Coord getMax() const { - return m_min; - } + auto c = Coord { Traits::getCoord(data, 0), Traits::getCoord(data, 1) }; -private: - Coord m_min, m_max; - std::shared_ptr m_sub_trees[4]; -}; + if (m_data.empty()) { + m_min.x = m_max.x = c.x; + m_min.y = m_max.y = c.y; + } else { + m_min.x = std::min(m_min.x, c.x); + m_min.y = std::min(m_min.y, c.y); + m_max.x = std::max(m_max.x, c.x); + m_max.y = std::max(m_max.y, c.y); + } -template -class QuadTree::Leaf : public Node { -public: - //virtual std::vector findPointsWithinRadius(Coord coord, double radius) const = 0; + m_data.push_back(data); +} - Leaf() { - } - virtual ~Leaf() = default; - - void add(T&& data) override { - auto c = Traits::getCoord(data); - - if (m_data.empty()) { - m_min.x = m_max.x = c.x; - m_min.y = m_max.y = c.y; - } else { - m_min.x = std::min(m_min.x, c.x); - m_min.y = std::min(m_min.y, c.y); - m_max.x = std::max(m_max.x, c.x); - m_max.y = std::max(m_max.y, c.y); - } +template +void QuadTree::split() { + assert(!m_is_divided); - m_data.emplace_back(std::move(data)); - } + m_is_divided = true; + // expands to a square + m_max.x = std::max(m_max.x, m_min.x + (m_max.y - m_min.y)); + m_max.y = std::max(m_max.y, m_min.y + (m_max.x - m_min.x)); - std::shared_ptr split() { - auto branch = std::make_shared(); + for (auto& d : m_data) { + add(d); } + m_data.clear(); +} - bool isFull() const override { - return m_data.size() < N; - } +template +void QuadTree::expand(Coord c) { + assert(m_is_divided && !isContained(c)); - size_t size() const { - return m_data.size(); - } + auto clone = std::make_shared>(*this); - Coord getMin() const { - return m_min; + if (c.x < m_min.x) { + m_min.x -= (m_max.x - m_min.x); + } else { + m_max.x += (m_max.x - m_min.x); } - - Coord getMax() const { - return m_min; + if (c.y < m_min.y) { + m_min.y -= (m_max.y - m_min.y); + } else { + m_max.y += (m_max.y - m_min.y); } -private: - Coord m_min, m_max; - std::vector m_data; -}; + auto quad = getQuadrant({ (clone->m_min.x + clone->m_max.x) / 2.0, (clone->m_min.y + clone->m_max.y) / 2.0 }); + m_sub_trees[quad] = clone; +} -template -void QuadTree::add(T&& data) { - if (m_root == nullptr) { - m_root = std::make_shared(); +template +size_t QuadTree::getQuadrant(Coord c) const { + size_t quadrant = 0; + if (c.x >= (m_min.x + m_max.x) / 2.0) { + quadrant += 1; + } + if (c.y >= (m_min.y + m_max.y) / 2.0) { + quadrant += 2; } + return quadrant; +} - auto leaf = std::dynamic_pointer_cast(m_root); - if (leaf != nullptr && leaf.isFull()) { - auto leaf_min = leaf->getMin(); - auto leaf_max = leaf->getMax(); +template +bool QuadTree::isContained(Coord c) const { + return c.x >= m_min.x && c.x <= m_max.x && c.y >= m_min.y && c.y <= m_max.y; +} - leaf_max.x = leaf_max.y = std::max(leaf_max.x, leaf_max.y); - m_root = std::make_shared(Coord {0,0}, leaf_max); - } else { - m_root.add(std::move(data)); - } +template +typename QuadTree::Coord QuadTree::getQuadrantMin(size_t quadrant) const { + return { + quadrant & 1 ? (m_max.x + m_min.x) / 2.0 : m_min.x, + quadrant & 2 ? (m_max.y + m_min.y) / 2.0 : m_min.y + }; +} + +template +typename QuadTree::Coord QuadTree::getQuadrantMax(size_t quadrant) const { + return { + quadrant & 1 ? m_max.x : (m_max.x + m_min.x) / 2.0, + quadrant & 2 ? m_max.y : (m_max.y + m_min.y) / 2.0 + }; } + } From 668ba0b6a6539f569ae64d20e0071f4dfd333771 Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Fri, 16 Feb 2024 14:50:07 +0100 Subject: [PATCH 11/24] model fitting outputs the fitting area dimensions --- .../FlexibleModelFitting.h | 19 +++++++++++++++++-- .../FlexibleModelFittingIterativeTask.h | 2 ++ .../FlexibleModelFittingIterativeTask.cpp | 17 ++++++++++++++++- .../FlexibleModelFittingPlugin.cpp | 18 ++++++++++++++++++ .../FlexibleModelFittingTask.cpp | 6 ++++-- 5 files changed, 57 insertions(+), 5 deletions(-) diff --git a/SEImplementation/SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFitting.h b/SEImplementation/SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFitting.h index e85e0ce76..a39fc4bea 100644 --- a/SEImplementation/SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFitting.h +++ b/SEImplementation/SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFitting.h @@ -47,7 +47,9 @@ class FlexibleModelFitting : public Property { std::unordered_map parameter_sigmas, std::vector chi_squared_per_meta, std::vector iterations_per_meta, - int meta_iterations) : + int meta_iterations, + std::vector fitting_areas_x, + std::vector fitting_areas_y) : m_iterations(iterations), m_stop_reason(stop_reason), m_chi_squared(chi_squared), @@ -57,7 +59,9 @@ class FlexibleModelFitting : public Property { m_parameter_sigmas(parameter_sigmas), m_chi_squared_per_meta(chi_squared_per_meta), m_iterations_per_meta(iterations_per_meta), - m_meta_iterations(meta_iterations) + m_meta_iterations(meta_iterations), + m_fitting_areas_x(fitting_areas_x), + m_fitting_areas_y(fitting_areas_y) {} unsigned int getIterations() const { @@ -100,6 +104,14 @@ class FlexibleModelFitting : public Property { return m_meta_iterations; } + std::vector getFittingAreasX() const { + return m_fitting_areas_x; + } + + std::vector getFittingAreasY() const { + return m_fitting_areas_y; + } + private: unsigned int m_iterations, m_stop_reason; SeFloat m_chi_squared, m_duration; @@ -110,6 +122,9 @@ class FlexibleModelFitting : public Property { std::vector m_chi_squared_per_meta; std::vector m_iterations_per_meta; int m_meta_iterations; + + std::vector m_fitting_areas_x; + std::vector m_fitting_areas_y; }; } diff --git a/SEImplementation/SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFittingIterativeTask.h b/SEImplementation/SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFittingIterativeTask.h index 6cee8a9e7..e84f075ac 100644 --- a/SEImplementation/SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFittingIterativeTask.h +++ b/SEImplementation/SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFittingIterativeTask.h @@ -67,6 +67,8 @@ class FlexibleModelFittingIterativeTask : public GroupTask { unsigned int stop_reason; std::vector chi_squared_per_meta; std::vector iterations_per_meta; + std::vector fitting_aeras_x; + std::vector fitting_aeras_y; }; struct FittingState { diff --git a/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingIterativeTask.cpp b/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingIterativeTask.cpp index b0a13f6f2..ae5860944 100644 --- a/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingIterativeTask.cpp +++ b/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingIterativeTask.cpp @@ -204,6 +204,21 @@ void FlexibleModelFittingIterativeTask::computeProperties(SourceGroupInterface& // Make sure we have a default value for sigmas in case we cannot do the fit initial_state.parameters_sigmas[parameter->getId()] = std::numeric_limits::quiet_NaN(); } + + // Store fitting areas for later output + for (auto frame : m_frames) { + int frame_index = frame->getFrameNb(); + // Validate that each frame covers the model fitting region + if (isFrameValid(source, frame_index)) { + auto stamp_rect = getFittingRect(source, frame_index); + initial_state.fitting_aeras_x.push_back(stamp_rect.getWidth()); + initial_state.fitting_aeras_y.push_back(stamp_rect.getHeight()); + } else { + initial_state.fitting_aeras_x.push_back(-1.f); + initial_state.fitting_aeras_y.push_back(-1.f); + } + } + fitting_state.source_states.emplace_back(std::move(initial_state)); } @@ -263,7 +278,7 @@ void FlexibleModelFittingIterativeTask::computeProperties(SourceGroupInterface& source_state.reduced_chi_squared, source_state.duration, source_state.flags, source_state.parameters_values, source_state.parameters_sigmas, source_state.chi_squared_per_meta, source_state.iterations_per_meta, - meta_iterations); + meta_iterations, source_state.fitting_aeras_x, source_state.fitting_aeras_y); index++; } diff --git a/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingPlugin.cpp b/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingPlugin.cpp index 7a1128342..012e7829c 100644 --- a/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingPlugin.cpp +++ b/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingPlugin.cpp @@ -109,6 +109,24 @@ void FlexibleModelFittingPlugin::registerPlugin(PluginAPI& plugin_api) { "Meta-iterations" ); + plugin_api.getOutputRegistry().registerColumnConverter>( + "fmf_fitting_areas_x", + [](const FlexibleModelFitting& prop) { + return prop.getFittingAreasX(); + }, + "", + "Fitting areas X" + ); + + plugin_api.getOutputRegistry().registerColumnConverter>( + "fmf_fitting_areas_y", + [](const FlexibleModelFitting& prop) { + return prop.getFittingAreasY(); + }, + "", + "Fitting areas Y" + ); + plugin_api.getOutputRegistry().enableOutput("FlexibleModelFitting"); } diff --git a/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingTask.cpp b/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingTask.cpp index f6122a04d..4c6555c5e 100644 --- a/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingTask.cpp +++ b/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingTask.cpp @@ -308,7 +308,8 @@ void FlexibleModelFittingTask::computeProperties(SourceGroupInterface& group) co avg_reduced_chi_squared, solution.duration, source_flags, parameter_values, parameter_sigmas, std::vector({avg_reduced_chi_squared}), - std::vector({(int) iterations}), (int) 1); + std::vector({(int) iterations}), (int) 1, + std::vector({99.f}), std::vector({99.f})); } updateCheckImages(group, pixel_scale, parameter_manager); @@ -335,7 +336,8 @@ void FlexibleModelFittingTask::setDummyProperty(SourceGroupInterface& group, } source.setProperty(0, 0, std::numeric_limits::quiet_NaN(), 0., flags, dummy_values, dummy_values, - std::vector(1), std::vector(1), 0); + std::vector(1), std::vector(1), 0, + std::vector({99.f}), std::vector({99.f})); } } From 77cbb540d8c204d9ab6554de6a176d207d19e290 Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Fri, 16 Feb 2024 17:04:16 +0100 Subject: [PATCH 12/24] fix typo --- .../FlexibleModelFittingIterativeTask.h | 4 ++-- .../FlexibleModelFittingIterativeTask.cpp | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/SEImplementation/SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFittingIterativeTask.h b/SEImplementation/SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFittingIterativeTask.h index e84f075ac..7525afe62 100644 --- a/SEImplementation/SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFittingIterativeTask.h +++ b/SEImplementation/SEImplementation/Plugin/FlexibleModelFitting/FlexibleModelFittingIterativeTask.h @@ -67,8 +67,8 @@ class FlexibleModelFittingIterativeTask : public GroupTask { unsigned int stop_reason; std::vector chi_squared_per_meta; std::vector iterations_per_meta; - std::vector fitting_aeras_x; - std::vector fitting_aeras_y; + std::vector fitting_areas_x; + std::vector fitting_areas_y; }; struct FittingState { diff --git a/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingIterativeTask.cpp b/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingIterativeTask.cpp index ae5860944..3a2822de4 100644 --- a/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingIterativeTask.cpp +++ b/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingIterativeTask.cpp @@ -211,11 +211,11 @@ void FlexibleModelFittingIterativeTask::computeProperties(SourceGroupInterface& // Validate that each frame covers the model fitting region if (isFrameValid(source, frame_index)) { auto stamp_rect = getFittingRect(source, frame_index); - initial_state.fitting_aeras_x.push_back(stamp_rect.getWidth()); - initial_state.fitting_aeras_y.push_back(stamp_rect.getHeight()); + initial_state.fitting_areas_x.push_back(stamp_rect.getWidth()); + initial_state.fitting_areas_y.push_back(stamp_rect.getHeight()); } else { - initial_state.fitting_aeras_x.push_back(-1.f); - initial_state.fitting_aeras_y.push_back(-1.f); + initial_state.fitting_areas_x.push_back(-1.f); + initial_state.fitting_areas_y.push_back(-1.f); } } @@ -278,7 +278,7 @@ void FlexibleModelFittingIterativeTask::computeProperties(SourceGroupInterface& source_state.reduced_chi_squared, source_state.duration, source_state.flags, source_state.parameters_values, source_state.parameters_sigmas, source_state.chi_squared_per_meta, source_state.iterations_per_meta, - meta_iterations, source_state.fitting_aeras_x, source_state.fitting_aeras_y); + meta_iterations, source_state.fitting_areas_x, source_state.fitting_areas_y); index++; } From 02b2e558ca9a5bd3221eeb4a3eaf5a986c98920c Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Wed, 21 Feb 2024 16:15:48 +0100 Subject: [PATCH 13/24] QuadTree completed and unit test --- .../Grouping/GroupingFactory.h | 1 + .../Grouping/MoffatGrouping.h | 18 ++- .../src/lib/Grouping/GroupingFactory.cpp | 10 +- .../src/lib/Grouping/MoffatGrouping.cpp | 2 +- SEUtils/CMakeLists.txt | 3 + SEUtils/SEUtils/QuadTree.h | 18 ++- SEUtils/SEUtils/_impl/QuadTree.icpp | 30 ++++- SEUtils/tests/src/QuadTree_test.cpp | 108 ++++++++++++++++++ 8 files changed, 172 insertions(+), 18 deletions(-) create mode 100644 SEUtils/tests/src/QuadTree_test.cpp diff --git a/SEImplementation/SEImplementation/Grouping/GroupingFactory.h b/SEImplementation/SEImplementation/Grouping/GroupingFactory.h index 43939eaf2..ac64b4b4b 100644 --- a/SEImplementation/SEImplementation/Grouping/GroupingFactory.h +++ b/SEImplementation/SEImplementation/Grouping/GroupingFactory.h @@ -59,6 +59,7 @@ class GroupingFactory : public Configurable { std::shared_ptr m_grouping_criteria; std::shared_ptr m_source_group_factory; unsigned int m_hard_limit; + double m_moffat_max_distance; }; } /* namespace SourceXtractor */ diff --git a/SEImplementation/SEImplementation/Grouping/MoffatGrouping.h b/SEImplementation/SEImplementation/Grouping/MoffatGrouping.h index 8e896e8bf..11ce5b8ea 100644 --- a/SEImplementation/SEImplementation/Grouping/MoffatGrouping.h +++ b/SEImplementation/SEImplementation/Grouping/MoffatGrouping.h @@ -1,8 +1,18 @@ -/* - * MoffatGrouping.h +/** Copyright © 2019-2024 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * - * Created on: Nov 22, 2023 - * Author: mschefer + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #ifndef _SEIMPLEMENTATION_GROUPING_MOFFATGROUPING_H_ diff --git a/SEImplementation/src/lib/Grouping/GroupingFactory.cpp b/SEImplementation/src/lib/Grouping/GroupingFactory.cpp index 9bf4c938c..582bf6285 100644 --- a/SEImplementation/src/lib/Grouping/GroupingFactory.cpp +++ b/SEImplementation/src/lib/Grouping/GroupingFactory.cpp @@ -24,7 +24,8 @@ namespace SourceXtractor { GroupingFactory::GroupingFactory(std::shared_ptr source_group_factory) - : m_source_group_factory(source_group_factory), m_hard_limit(0) {} + : m_algorithm(GroupingConfig::Algorithm::NO_GROUPING), m_source_group_factory(source_group_factory), + m_hard_limit(0), m_moffat_max_distance(1000.0) {} void GroupingFactory::reportConfigDependencies(Euclid::Configuration::ConfigManager& manager) const { manager.registerConfiguration(); @@ -33,6 +34,7 @@ void GroupingFactory::reportConfigDependencies(Euclid::Configuration::ConfigMana void GroupingFactory::configure(Euclid::Configuration::ConfigManager& manager) { auto grouping_config = manager.getConfiguration(); m_algorithm = grouping_config.getAlgorithmOption(); + m_moffat_max_distance = grouping_config.getMoffatMaxDistance(); switch (m_algorithm) { case GroupingConfig::Algorithm::NO_GROUPING: m_grouping_criteria = std::make_shared(); @@ -44,7 +46,7 @@ void GroupingFactory::configure(Euclid::Configuration::ConfigManager& manager) m_grouping_criteria = std::make_shared(); break; case GroupingConfig::Algorithm::MOFFAT: - m_grouping_criteria = std::make_shared(grouping_config.getMoffatThreshold(), grouping_config.getMoffatMaxDistance()); + m_grouping_criteria = std::make_shared(grouping_config.getMoffatThreshold(), m_moffat_max_distance); break; case GroupingConfig::Algorithm::ASSOC: m_grouping_criteria = std::make_shared(); @@ -64,8 +66,8 @@ std::shared_ptr GroupingFactory::createGrouping() const case GroupingConfig::Algorithm::ASSOC: return std::make_shared(m_source_group_factory, m_hard_limit); case GroupingConfig::Algorithm::MOFFAT: - //m_grouping_criteria = std::make_shared(grouping_config.getMoffatThreshold(), grouping_config.getMoffatMaxDistance()); - return std::make_shared(m_grouping_criteria, m_source_group_factory, m_hard_limit, 500); // FIXME!!!!!!!!!!!!! hardcoded + return std::make_shared( + m_grouping_criteria, m_source_group_factory, m_hard_limit, m_moffat_max_distance); break; default: return std::make_shared(m_grouping_criteria, m_source_group_factory, m_hard_limit); diff --git a/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp b/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp index 4607537d2..8867422c2 100644 --- a/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp +++ b/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp @@ -1,4 +1,4 @@ -/** Copyright © 2019 - 2023 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** Copyright © 2019-2024 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free diff --git a/SEUtils/CMakeLists.txt b/SEUtils/CMakeLists.txt index 1557e4b5d..9f3c453aa 100644 --- a/SEUtils/CMakeLists.txt +++ b/SEUtils/CMakeLists.txt @@ -63,6 +63,9 @@ elements_add_unit_test(NumericalDerivative_test tests/src/NumericalDerivative_te elements_add_unit_test(Misc_test tests/src/Misc_test.cpp LINK_LIBRARIES SEUtils TYPE Boost) +elements_add_unit_test(QuadTree_test tests/src/QuadTree_test.cpp + LINK_LIBRARIES SEUtils + TYPE Boost) if(GMOCK_FOUND) elements_add_unit_test(Observable_test tests/src/Observable_test.cpp diff --git a/SEUtils/SEUtils/QuadTree.h b/SEUtils/SEUtils/QuadTree.h index 804238745..4d2479064 100644 --- a/SEUtils/SEUtils/QuadTree.h +++ b/SEUtils/SEUtils/QuadTree.h @@ -1,8 +1,18 @@ -/* - * QuadTree.h +/** Copyright © 2019-2024 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * - * Created on: Nov 28, 2023 - * Author: mschefer + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #ifndef _SEUTILS_QUADTREE_H_ diff --git a/SEUtils/SEUtils/_impl/QuadTree.icpp b/SEUtils/SEUtils/_impl/QuadTree.icpp index e1d44ccd3..082994684 100644 --- a/SEUtils/SEUtils/_impl/QuadTree.icpp +++ b/SEUtils/SEUtils/_impl/QuadTree.icpp @@ -1,4 +1,4 @@ -/** Copyright © 2021-2023 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** Copyright © 2019-2024 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -78,16 +78,31 @@ void QuadTree::remove(const T& data) { template std::vector QuadTree::getPointsWithinRange(Coord c, double range) const { + auto range_sq = range * range; if (m_is_divided) { std::vector points; for (int i=0; i<4; i++) { - // FIXME only if overlap - auto subtree_points = m_sub_trees[i]->getPointsWithinRange(c, range); - points.insert(points.end(), subtree_points.begin(), subtree_points.end()); + if (m_sub_trees[i] != nullptr) { + // compare range with distance to closest corner of subtree + auto dx_sq = std::min((c.x - m_sub_trees[i]->m_min.x) * (c.x - m_sub_trees[i]->m_min.x), + (c.x - m_sub_trees[i]->m_max.x) * (c.x - m_sub_trees[i]->m_max.x)); + auto dy_sq = std::min((c.y - m_sub_trees[i]->m_min.y) * (c.y - m_sub_trees[i]->m_min.y), + (c.y - m_sub_trees[i]->m_max.y) * (c.y - m_sub_trees[i]->m_max.y)); + if (dx_sq + dy_sq <= range_sq) { + auto subtree_points = m_sub_trees[i]->getPointsWithinRange(c, range); + points.insert(points.end(), subtree_points.begin(), subtree_points.end()); + } + } } return points; } else { - return m_data; + std::vector points; + std::copy_if(m_data.begin(), m_data.end(), std::back_inserter(points), + [c, range_sq](const T& point) { + auto pc = Coord { Traits::getCoord(point, 0), Traits::getCoord(point, 1) }; + return (pc.x - c.x) * (pc.x - c.x) + (pc.y - c.y) * (pc.y - c.y) <= range_sq; + }); + return points; } } @@ -144,6 +159,11 @@ void QuadTree::expand(Coord c) { auto quad = getQuadrant({ (clone->m_min.x + clone->m_max.x) / 2.0, (clone->m_min.y + clone->m_max.y) / 2.0 }); m_sub_trees[quad] = clone; + for (size_t i=0; i<4; i++) { + if (i != quad) { + m_sub_trees[i] = nullptr; + } + } } template diff --git a/SEUtils/tests/src/QuadTree_test.cpp b/SEUtils/tests/src/QuadTree_test.cpp new file mode 100644 index 000000000..9d513cf42 --- /dev/null +++ b/SEUtils/tests/src/QuadTree_test.cpp @@ -0,0 +1,108 @@ +/** Copyright © 2019-2024 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + + +#include + +#include "SEUtils/QuadTree.h" + +namespace SourceXtractor { + +struct Test { + double m_x, m_y; +}; + +bool operator==(const Test&a, const Test&b) { + return a.m_x == b.m_x && a.m_y == b.m_y; +}; + +template <> +struct QuadTreeTraits { + static double getCoord(Test t, size_t index) { + if (index == 0) { + return t.m_x; + } else { + return t.m_y; + } + } +}; + + +BOOST_AUTO_TEST_SUITE (QuadTree_test) + +BOOST_AUTO_TEST_CASE( smoke_test ) { + QuadTree tree(100); +} + +BOOST_AUTO_TEST_CASE( add_test ) { + QuadTree tree(100); + + tree.add({1,2}); + tree.add({3,4}); + tree.add({5,6}); + + auto result = tree.getPointsWithinRange({0,0}, 100); + BOOST_CHECK_EQUAL(result.size(), 3); + + for (int i = 0; i<1000; i++) { + tree.add({5+i/1000.0,2+i/500.0}); + } + + // add some well outside the area + for (int i = 0; i<1000; i++) { + tree.add({500+i/1000.0,2+i/500.0}); + } + + result = tree.getPointsWithinRange({0,0}, 100); + BOOST_CHECK_EQUAL(result.size(), 1003); +} + +BOOST_AUTO_TEST_CASE( simple_remove_test ) { + QuadTree tree(100); + + tree.add({1,2}); + tree.add({3,4}); + tree.add({5,6}); + + tree.remove({3,4}); + + auto result = tree.getPointsWithinRange({0,0}, 100); + BOOST_CHECK_EQUAL(result.size(), 2); +} + +BOOST_AUTO_TEST_CASE( remove_in_tree_test ) { + QuadTree tree(100); + + for (int i = 0; i<1000; i++) { + tree.add({5+i/1000.0,2+i/500.0}); + } + + tree.add({1,2}); + tree.add({3,4}); + tree.add({5,6}); + + tree.remove({3,4}); + + auto result = tree.getPointsWithinRange({0,0}, 100); + BOOST_CHECK_EQUAL(result.size(), 1002); +} + + + +BOOST_AUTO_TEST_SUITE_END () + +} From 78168108c5377c051c1317c311da795c46adbb79 Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Fri, 23 Feb 2024 15:45:34 +0100 Subject: [PATCH 14/24] bug fixes --- SEImplementation/src/lib/Grouping/MoffatGrouping.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp b/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp index 8867422c2..05cfc5925 100644 --- a/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp +++ b/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp @@ -61,6 +61,7 @@ void MoffatGrouping::receiveSource(std::unique_ptr source) { // Encapsulates the source unique_ptr auto& centroid = source->getProperty(); + auto source_info = std::make_shared(); source_info->m_source = std::move(source); source_info->m_x = centroid.getCentroidX(); @@ -68,6 +69,7 @@ void MoffatGrouping::receiveSource(std::unique_ptr source) { source_info->m_group_id = m_group_counter++; auto group = std::make_shared(); + group->push_back(source_info); m_groups[source_info->m_group_id] = group; // Find sources within range @@ -89,6 +91,10 @@ void MoffatGrouping::receiveSource(std::unique_ptr source) { m_groups.erase(group_id); } + for (auto& s : *group) { + s->m_group_id = source_info->m_group_id; + } + // Add source to the Quad Tree m_tree.add(source_info); } @@ -118,7 +124,7 @@ void MoffatGrouping::receiveProcessSignal(const ProcessSourcesEvent& event) { m_tree.remove(source_info); } - //sendSource(std::move(m_source_groups[group_id])); + sendSource(std::move(new_group)); m_groups.erase(group_id); } } From f18110b92fed595ab282b2ab98f5a92665e51f9d Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Thu, 14 Mar 2024 17:41:25 +0100 Subject: [PATCH 15/24] D'oh! --- SEImplementation/src/lib/Grouping/AssocGrouping.cpp | 2 +- SEImplementation/src/lib/Grouping/SplitSourcesGrouping.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SEImplementation/src/lib/Grouping/AssocGrouping.cpp b/SEImplementation/src/lib/Grouping/AssocGrouping.cpp index 75f1b6d80..ce80b812c 100644 --- a/SEImplementation/src/lib/Grouping/AssocGrouping.cpp +++ b/SEImplementation/src/lib/Grouping/AssocGrouping.cpp @@ -38,7 +38,7 @@ void AssocGrouping::receiveSource(std::unique_ptr source) { m_source_groups[source_id] = std::move(new_group); } - if (m_source_groups.at(source_id)->size() >= m_hard_limit) { + if (m_hard_limit > 0 && m_source_groups.at(source_id)->size() >= m_hard_limit) { // the stored group has reached the hard limit // send the current group to processing sendSource(std::move(m_source_groups.at(source_id))); diff --git a/SEImplementation/src/lib/Grouping/SplitSourcesGrouping.cpp b/SEImplementation/src/lib/Grouping/SplitSourcesGrouping.cpp index d92923d8d..5cd807296 100644 --- a/SEImplementation/src/lib/Grouping/SplitSourcesGrouping.cpp +++ b/SEImplementation/src/lib/Grouping/SplitSourcesGrouping.cpp @@ -38,7 +38,7 @@ void SplitSourcesGrouping::receiveSource(std::unique_ptr source m_source_groups[source_id] = std::move(new_group); } - if (m_source_groups.at(source_id)->size() >= m_hard_limit) { + if (m_hard_limit > 0 && m_source_groups.at(source_id)->size() >= m_hard_limit) { // the stored group has reached the hard limit // send the current group to processing sendSource(std::move(m_source_groups.at(source_id))); From 68494415087dabbf2910f6add9c68120e2337af2 Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Fri, 15 Mar 2024 13:10:50 +0100 Subject: [PATCH 16/24] disable partition steps that require a detection image when none is available --- .../AttractorsPartitionConfig.cpp | 19 ++++++---- .../MultiThresholdPartitionConfig.cpp | 36 +++++++++++-------- .../lib/Plugin/AssocMode/AssocModeConfig.cpp | 2 ++ .../CoreThresholdPartitionConfig.cpp | 15 +++++--- 4 files changed, 45 insertions(+), 27 deletions(-) diff --git a/SEImplementation/src/lib/Configuration/AttractorsPartitionConfig.cpp b/SEImplementation/src/lib/Configuration/AttractorsPartitionConfig.cpp index 1337f05dc..adc241373 100644 --- a/SEImplementation/src/lib/Configuration/AttractorsPartitionConfig.cpp +++ b/SEImplementation/src/lib/Configuration/AttractorsPartitionConfig.cpp @@ -22,6 +22,7 @@ #include "SEImplementation/Configuration/AttractorsPartitionConfig.h" #include "SEImplementation/Configuration/PartitionStepConfig.h" #include "SEImplementation/Configuration/MinAreaPartitionConfig.h" +#include "SEImplementation/Configuration/DetectionFrameConfig.h" #include "SEImplementation/Partition/AttractorsPartitionStep.h" @@ -34,8 +35,10 @@ static const std::string USE_ATTRACTORS_PARTITION {"use-attractors-partition"}; AttractorsPartitionConfig::AttractorsPartitionConfig(long manager_id) : Configuration(manager_id) { declareDependency(); - ConfigManager::getInstance(manager_id).registerDependency(); + declareDependency(); + // this is used to enforce the order the PartitionSteps are added and performed + ConfigManager::getInstance(manager_id).registerDependency(); } auto AttractorsPartitionConfig::getProgramOptions() -> std::map { @@ -46,12 +49,14 @@ auto AttractorsPartitionConfig::getProgramOptions() -> std::map()) { - getDependency().addPartitionStepCreator( - [](std::shared_ptr source_factory) { - return std::make_shared(source_factory); - } - ); + if (getDependency().getDetectionFrames().size() > 0) { + if (args.at(USE_ATTRACTORS_PARTITION).as()) { + getDependency().addPartitionStepCreator( + [](std::shared_ptr source_factory) { + return std::make_shared(source_factory); + } + ); + } } } diff --git a/SEImplementation/src/lib/Configuration/MultiThresholdPartitionConfig.cpp b/SEImplementation/src/lib/Configuration/MultiThresholdPartitionConfig.cpp index 0b34371e9..f4ca3d100 100644 --- a/SEImplementation/src/lib/Configuration/MultiThresholdPartitionConfig.cpp +++ b/SEImplementation/src/lib/Configuration/MultiThresholdPartitionConfig.cpp @@ -24,6 +24,7 @@ #include "SEImplementation/Configuration/MultiThresholdPartitionConfig.h" #include "SEImplementation/Configuration/MinAreaPartitionConfig.h" #include "SEImplementation/Configuration/PartitionStepConfig.h" +#include "SEImplementation/Configuration/DetectionFrameConfig.h" #include "SEImplementation/Partition/MultiThresholdPartitionStep.h" @@ -39,7 +40,9 @@ static const std::string MTHRESH_MIN_CONTRAST {"partition-minimum-contrast"}; MultiThresholdPartitionConfig::MultiThresholdPartitionConfig(long manager_id) : Configuration(manager_id) { declareDependency(); + declareDependency(); + // this is used to enforce the order the PartitionSteps are added and performed ConfigManager::getInstance(manager_id).registerDependency(); } @@ -53,23 +56,26 @@ auto MultiThresholdPartitionConfig::getProgramOptions() -> std::map()) { - auto threshold_nb = args.at(MTHRESH_THRESHOLDS_NB).as(); - auto min_area = args.at(MTHRESH_MIN_AREA).as(); - auto min_contrast = args.at(MTHRESH_MIN_CONTRAST).as(); + // We can only use this if we have a detection image and it was on by default + if (getDependency().getDetectionFrames().size() > 0) { + if (args.at(MTHRESH_USE).as()) { + auto threshold_nb = args.at(MTHRESH_THRESHOLDS_NB).as(); + auto min_area = args.at(MTHRESH_MIN_AREA).as(); + auto min_contrast = args.at(MTHRESH_MIN_CONTRAST).as(); - if (min_area <= 0) { - throw Elements::Exception() << "Invalid " << MTHRESH_MIN_AREA << " value: " << min_area; - } - if (threshold_nb <= 0) { - throw Elements::Exception() << "Invalid " << MTHRESH_THRESHOLDS_NB << " value: " << threshold_nb; - } - - getDependency().addPartitionStepCreator( - [=](std::shared_ptr source_factory) { - return std::make_shared(source_factory, min_contrast, threshold_nb, min_area); + if (min_area <= 0) { + throw Elements::Exception() << "Invalid " << MTHRESH_MIN_AREA << " value: " << min_area; } - ); + if (threshold_nb <= 0) { + throw Elements::Exception() << "Invalid " << MTHRESH_THRESHOLDS_NB << " value: " << threshold_nb; + } + + getDependency().addPartitionStepCreator( + [=](std::shared_ptr source_factory) { + return std::make_shared(source_factory, min_contrast, threshold_nb, min_area); + } + ); + } } } diff --git a/SEImplementation/src/lib/Plugin/AssocMode/AssocModeConfig.cpp b/SEImplementation/src/lib/Plugin/AssocMode/AssocModeConfig.cpp index 70acc1b36..f7a0cd411 100644 --- a/SEImplementation/src/lib/Plugin/AssocMode/AssocModeConfig.cpp +++ b/SEImplementation/src/lib/Plugin/AssocMode/AssocModeConfig.cpp @@ -106,6 +106,8 @@ AssocModeConfig::AssocModeConfig(long manager_id) : Configuration(manager_id), m m_assoc_radius(0.), m_default_pixel_size(10), m_pixel_size_column(-1), m_group_id_column(-1) { declareDependency(); declareDependency(); + + // this is used to enforce the order the PartitionSteps are added and performed ConfigManager::getInstance(manager_id).registerDependency(); } diff --git a/SEImplementation/src/lib/Plugin/CoreThresholdPartition/CoreThresholdPartitionConfig.cpp b/SEImplementation/src/lib/Plugin/CoreThresholdPartition/CoreThresholdPartitionConfig.cpp index 2bbbe4ea4..246dfb203 100644 --- a/SEImplementation/src/lib/Plugin/CoreThresholdPartition/CoreThresholdPartitionConfig.cpp +++ b/SEImplementation/src/lib/Plugin/CoreThresholdPartition/CoreThresholdPartitionConfig.cpp @@ -28,6 +28,7 @@ #include "SEImplementation/Plugin/CoreThresholdPartition/CoreThresholdPartitionStep.h" #include "SEImplementation/Configuration/PartitionStepConfig.h" #include "SEImplementation/Configuration/MultiThresholdPartitionConfig.h" +#include "SEImplementation/Configuration/DetectionFrameConfig.h" namespace po = boost::program_options; @@ -42,7 +43,9 @@ static const std::string CORE_THRESH_USE {"partition-corethreshold" }; CoreThresholdPartitionConfig::CoreThresholdPartitionConfig(long manager_id) : Configuration(manager_id), m_core_threshold(0.), m_core_minarea(0) { declareDependency(); + declareDependency(); + // this is used to enforce the order the PartitionSteps are added and performed ConfigManager::getInstance(manager_id).registerDependency(); } @@ -65,11 +68,13 @@ void CoreThresholdPartitionConfig::initialize(const UserValues &args) { throw Elements::Exception() << "Invalid " << CORE_MINAREA << " value: " << m_core_minarea; } - if (m_core_threshold > 0.0 && m_core_minarea > 0 && args.at(CORE_THRESH_USE).as()) { - double core_threshold = m_core_threshold; - int core_minarea = m_core_minarea; - getDependency().addPartitionStepCreator([core_threshold, core_minarea](std::shared_ptr) - { return std::make_shared(core_threshold, core_minarea); } ); + if (getDependency().getDetectionFrames().size() > 0) { + if (m_core_threshold > 0.0 && m_core_minarea > 0 && args.at(CORE_THRESH_USE).as()) { + double core_threshold = m_core_threshold; + int core_minarea = m_core_minarea; + getDependency().addPartitionStepCreator([core_threshold, core_minarea](std::shared_ptr) + { return std::make_shared(core_threshold, core_minarea); } ); + } } } From 02bbfd7316f2c4cf12a1ad7d45c2362f0b6e0b1f Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Wed, 20 Mar 2024 17:14:52 +0100 Subject: [PATCH 17/24] add hard limit to moffat grouping --- .../Grouping/MoffatGrouping.h | 2 ++ .../src/lib/Grouping/MoffatGrouping.cpp | 31 ++++++++++++------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/SEImplementation/SEImplementation/Grouping/MoffatGrouping.h b/SEImplementation/SEImplementation/Grouping/MoffatGrouping.h index 11ce5b8ea..6220b157a 100644 --- a/SEImplementation/SEImplementation/Grouping/MoffatGrouping.h +++ b/SEImplementation/SEImplementation/Grouping/MoffatGrouping.h @@ -50,6 +50,8 @@ class MoffatGrouping : public SourceGroupingInterface { void receiveProcessSignal(const ProcessSourcesEvent& event) override; private: + void processGroup(unsigned int group_id); + std::shared_ptr m_grouping_criteria; std::shared_ptr m_group_factory; unsigned int m_hard_limit; diff --git a/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp b/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp index 05cfc5925..1e162c567 100644 --- a/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp +++ b/SEImplementation/src/lib/Grouping/MoffatGrouping.cpp @@ -86,9 +86,14 @@ void MoffatGrouping::receiveSource(std::unique_ptr source) { // merge groups and keep the new group for (auto group_id : matching_groups) { - // merge group - group->insert(group->end(), m_groups.at(group_id)->begin(), m_groups.at(group_id)->end()); - m_groups.erase(group_id); + if (m_hard_limit > 0 && m_groups.at(group_id)->size() + group->size() > m_hard_limit) { + // if we have a hard limit and the group is too large to merge, just process it + processGroup(group_id); + } else { + // merge group + group->insert(group->end(), m_groups.at(group_id)->begin(), m_groups.at(group_id)->end()); + m_groups.erase(group_id); + } } for (auto& s : *group) { @@ -116,17 +121,21 @@ void MoffatGrouping::receiveProcessSignal(const ProcessSourcesEvent& event) { // For each SourceGroup that we put in groups_to_process, for (auto group_id : groups_to_process) { - // we remove it from our list of stored SourceGroups and notify our observers - auto new_group = m_group_factory->createSourceGroup(); + processGroup(group_id); + } +} - for (auto& source_info : *m_groups.at(group_id)) { - new_group->addSource(std::move(source_info->m_source)); - m_tree.remove(source_info); - } +void MoffatGrouping::processGroup(unsigned int group_id) { + // we remove it from our list of stored SourceGroups and notify our observers + auto new_group = m_group_factory->createSourceGroup(); - sendSource(std::move(new_group)); - m_groups.erase(group_id); + for (auto& source_info : *m_groups.at(group_id)) { + new_group->addSource(std::move(source_info->m_source)); + m_tree.remove(source_info); } + + sendSource(std::move(new_group)); + m_groups.erase(group_id); } } // SourceXtractor namespace From d3fedc93334ba1c97d74e977c3cbc1d99e08fd13 Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Fri, 22 Mar 2024 15:39:40 +0100 Subject: [PATCH 18/24] use rng seed instead of time() --- .../Configuration/RngConfig.h | 38 +++++++++++++++++ .../Partition/MultiThresholdPartitionStep.h | 6 ++- .../MultiThresholdPartitionConfig.cpp | 6 ++- .../src/lib/Configuration/RngConfig.cpp | 42 +++++++++++++++++++ .../Partition/MultiThresholdPartitionStep.cpp | 10 ++--- .../MultiThresholdPartitionStep_test.cpp | 3 +- 6 files changed, 96 insertions(+), 9 deletions(-) create mode 100644 SEImplementation/SEImplementation/Configuration/RngConfig.h create mode 100644 SEImplementation/src/lib/Configuration/RngConfig.cpp diff --git a/SEImplementation/SEImplementation/Configuration/RngConfig.h b/SEImplementation/SEImplementation/Configuration/RngConfig.h new file mode 100644 index 000000000..e3baec7fb --- /dev/null +++ b/SEImplementation/SEImplementation/Configuration/RngConfig.h @@ -0,0 +1,38 @@ +/* + * RngConfig.h + * + * Created on: Mar 21, 2024 + * Author: mschefer + */ + +#ifndef _SEIMPLEMENTATION_CONFIGURATION_RNGCONFIG_H_ +#define _SEIMPLEMENTATION_CONFIGURATION_RNGCONFIG_H_ + +#include "Configuration/Configuration.h" +#include "AlexandriaKernel/ThreadPool.h" + +namespace SourceXtractor { + +class RngConfig : public Euclid::Configuration::Configuration { +public: + explicit RngConfig(long manager_id); + + virtual ~RngConfig() = default; + + std::map getProgramOptions() override; + + void initialize(const UserValues& args) override; + + unsigned int getSeed() const { + return m_seed; + } + +private: + unsigned int m_seed; +}; + + +} + + +#endif /* _SEIMPLEMENTATION_CONFIGURATION_RNGCONFIG_H_ */ diff --git a/SEImplementation/SEImplementation/Partition/MultiThresholdPartitionStep.h b/SEImplementation/SEImplementation/Partition/MultiThresholdPartitionStep.h index 8f8ab174a..79c9d9668 100644 --- a/SEImplementation/SEImplementation/Partition/MultiThresholdPartitionStep.h +++ b/SEImplementation/SEImplementation/Partition/MultiThresholdPartitionStep.h @@ -24,6 +24,8 @@ #ifndef _SEIMPLEMENTATION_PARTITION_MULTITHRESHOLDPARTITIONSTEP_H_ #define _SEIMPLEMENTATION_PARTITION_MULTITHRESHOLDPARTITIONSTEP_H_ +#include + #include "SEUtils/Types.h" #include "SEImplementation/Property/PixelCoordinateList.h" @@ -48,7 +50,7 @@ class MultiThresholdPartitionStep : public PartitionStep { public: MultiThresholdPartitionStep(std::shared_ptr source_factory, SeFloat contrast, - unsigned int thresholds_nb, unsigned int min_deblend_area); + unsigned int thresholds_nb, unsigned int min_deblend_area, unsigned int seed); virtual ~MultiThresholdPartitionStep() = default; @@ -67,6 +69,8 @@ class MultiThresholdPartitionStep : public PartitionStep { SeFloat m_contrast; unsigned int m_thresholds_nb; unsigned int m_min_deblend_area; + unsigned int m_seed; + boost::random::mt19937 m_rng; }; diff --git a/SEImplementation/src/lib/Configuration/MultiThresholdPartitionConfig.cpp b/SEImplementation/src/lib/Configuration/MultiThresholdPartitionConfig.cpp index 0b34371e9..0dd2c8d24 100644 --- a/SEImplementation/src/lib/Configuration/MultiThresholdPartitionConfig.cpp +++ b/SEImplementation/src/lib/Configuration/MultiThresholdPartitionConfig.cpp @@ -24,6 +24,7 @@ #include "SEImplementation/Configuration/MultiThresholdPartitionConfig.h" #include "SEImplementation/Configuration/MinAreaPartitionConfig.h" #include "SEImplementation/Configuration/PartitionStepConfig.h" +#include "SEImplementation/Configuration/RngConfig.h" #include "SEImplementation/Partition/MultiThresholdPartitionStep.h" @@ -39,6 +40,7 @@ static const std::string MTHRESH_MIN_CONTRAST {"partition-minimum-contrast"}; MultiThresholdPartitionConfig::MultiThresholdPartitionConfig(long manager_id) : Configuration(manager_id) { declareDependency(); + declareDependency(); ConfigManager::getInstance(manager_id).registerDependency(); } @@ -57,6 +59,7 @@ void MultiThresholdPartitionConfig::initialize(const UserValues& args) { auto threshold_nb = args.at(MTHRESH_THRESHOLDS_NB).as(); auto min_area = args.at(MTHRESH_MIN_AREA).as(); auto min_contrast = args.at(MTHRESH_MIN_CONTRAST).as(); + auto seed = getDependency().getSeed(); if (min_area <= 0) { throw Elements::Exception() << "Invalid " << MTHRESH_MIN_AREA << " value: " << min_area; @@ -67,7 +70,8 @@ void MultiThresholdPartitionConfig::initialize(const UserValues& args) { getDependency().addPartitionStepCreator( [=](std::shared_ptr source_factory) { - return std::make_shared(source_factory, min_contrast, threshold_nb, min_area); + return std::make_shared( + source_factory, min_contrast, threshold_nb, min_area, seed); } ); } diff --git a/SEImplementation/src/lib/Configuration/RngConfig.cpp b/SEImplementation/src/lib/Configuration/RngConfig.cpp new file mode 100644 index 000000000..396ac5010 --- /dev/null +++ b/SEImplementation/src/lib/Configuration/RngConfig.cpp @@ -0,0 +1,42 @@ +/** Copyright © 2019-2024 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include + +#include "SEImplementation/Configuration/RngConfig.h" + +using namespace Euclid::Configuration; +namespace po = boost::program_options; + +namespace SourceXtractor { + +static const std::string RNG_SEED {"rng-seed"}; + +RngConfig::RngConfig(long manager_id) : Configuration(manager_id), m_seed(42) {} + +auto RngConfig::getProgramOptions() -> std::map { + return { {"Random number generation", { + {RNG_SEED.c_str(), po::value()->default_value(42), "Random number generator seed"}, + }}}; +} + +void RngConfig::initialize(const UserValues& args) { + m_seed = args.at(RNG_SEED).as(); +} + +} // SourceXtractor namespace + diff --git a/SEImplementation/src/lib/Partition/MultiThresholdPartitionStep.cpp b/SEImplementation/src/lib/Partition/MultiThresholdPartitionStep.cpp index 57ae88599..afeb33eb3 100644 --- a/SEImplementation/src/lib/Partition/MultiThresholdPartitionStep.cpp +++ b/SEImplementation/src/lib/Partition/MultiThresholdPartitionStep.cpp @@ -21,8 +21,6 @@ * Author: mschefer */ -#include - #include "SEImplementation/Measurement/MultithreadedMeasurement.h" #include "SEImplementation/Partition/MultiThresholdPartitionStep.h" @@ -43,7 +41,6 @@ namespace SourceXtractor { namespace { - boost::random::mt19937 rng { ((unsigned int) time(NULL)) }; } class MultiThresholdNode : public std::enable_shared_from_this { @@ -313,7 +310,8 @@ std::vector> MultiThresholdPartitionStep::reass } if (probabilities.back() > 1.0e-31) { - auto drand = double(probabilities.back()) * boost::random::uniform_01()(rng); + auto drand = double(probabilities.back()) * boost::random::uniform_01() + (const_cast(this)->m_rng); unsigned int i=0; for (; i= probabilities[i]; i++); @@ -353,8 +351,8 @@ std::vector> MultiThresholdPartitionStep::reass } MultiThresholdPartitionStep::MultiThresholdPartitionStep(std::shared_ptr source_factory, SeFloat contrast, - unsigned int thresholds_nb, unsigned int min_deblend_area) : + unsigned int thresholds_nb, unsigned int min_deblend_area, unsigned int seed) : m_source_factory(source_factory), m_contrast(contrast), m_thresholds_nb(thresholds_nb), - m_min_deblend_area(min_deblend_area) {} + m_min_deblend_area(min_deblend_area), m_seed(seed), m_rng(seed) {} } // namespace diff --git a/SEImplementation/tests/src/Partition/MultiThresholdPartitionStep_test.cpp b/SEImplementation/tests/src/Partition/MultiThresholdPartitionStep_test.cpp index b1d79b07f..cf374ef8b 100644 --- a/SEImplementation/tests/src/Partition/MultiThresholdPartitionStep_test.cpp +++ b/SEImplementation/tests/src/Partition/MultiThresholdPartitionStep_test.cpp @@ -72,7 +72,8 @@ struct MultiThresholdPartitionFixture { std::shared_ptr task_provider {new TaskProvider(task_factory_registry)}; std::unique_ptr source {new SourceWithOnDemandProperties(task_provider)}; std::shared_ptr multithreshold_step { - new MultiThresholdPartitionStep(std::make_shared(task_provider), 0.005, 32, 1) + new MultiThresholdPartitionStep( + std::make_shared(task_provider), 0.005, 32, 1, 42) }; MultiThresholdPartitionFixture() { From 1f6bf7f5209a7e2ff271e941a3a2fab8a1e3d885 Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Wed, 27 Mar 2024 16:16:02 +0100 Subject: [PATCH 19/24] fix cryptic errors when running sx++ without parameters --- .../src/lib/Configuration/MeasurementImageConfig.cpp | 7 +++++-- .../src/lib/Plugin/AssocMode/AssocModeConfig.cpp | 10 +++++----- SEMain/src/program/SourceXtractor.cpp | 8 ++++++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/SEImplementation/src/lib/Configuration/MeasurementImageConfig.cpp b/SEImplementation/src/lib/Configuration/MeasurementImageConfig.cpp index 5b973f675..4f09fc686 100644 --- a/SEImplementation/src/lib/Configuration/MeasurementImageConfig.cpp +++ b/SEImplementation/src/lib/Configuration/MeasurementImageConfig.cpp @@ -223,11 +223,14 @@ void MeasurementImageConfig::initialize(const UserValues&) { m_image_infos.emplace_back(std::move(info)); } } else { - logger.debug() << "No measurement image provided, using the detection image for measurements"; - auto detection_image = getDependency(); auto weight_image = getDependency(); + if (detection_image.getExtensionsNb() < 1) { + throw Elements::Exception() << "No measurement or detection image"; + } + + logger.debug() << "No measurement image provided, using the detection image for measurements"; // note: flux scale was already applied m_image_infos.emplace_back(MeasurementImageInfo { diff --git a/SEImplementation/src/lib/Plugin/AssocMode/AssocModeConfig.cpp b/SEImplementation/src/lib/Plugin/AssocMode/AssocModeConfig.cpp index f7a0cd411..ab5b89d00 100644 --- a/SEImplementation/src/lib/Plugin/AssocMode/AssocModeConfig.cpp +++ b/SEImplementation/src/lib/Plugin/AssocMode/AssocModeConfig.cpp @@ -153,16 +153,16 @@ void AssocModeConfig::initialize(const UserValues& args) { } // sanity check that the configuration is coherent - checkConfig(); // read the catalogs if (m_filename != "") { + checkConfig(); readCatalogs(m_filename, m_columns, m_assoc_coord_type); - } - if (args.at(ASSOC_TEST).as()) { - printConfig(); - throw Elements::Exception() << "Exiting by user request"; + if (args.at(ASSOC_TEST).as()) { + printConfig(); + throw Elements::Exception() << "Exiting by user request"; + } } } diff --git a/SEMain/src/program/SourceXtractor.cpp b/SEMain/src/program/SourceXtractor.cpp index bb9507175..49b2d4937 100644 --- a/SEMain/src/program/SourceXtractor.cpp +++ b/SEMain/src/program/SourceXtractor.cpp @@ -454,6 +454,14 @@ class SEMain : public Elements::Program { } } else { // Running detection-less + + auto assoc_mode_config = config_manager.getConfiguration(); + if (assoc_mode_config.getCatalogs().size() < 1) { + logger.error() << "No detection image and no assoc catalog"; + measurement->stopThreads(); + return Elements::ExitCode::NOT_OK; + } + try { // Process the catalog logger.info() << "Processing assoc catalog (no detection image)\n"; From e79e7e8cdb013ada3a11d3660927f60f4ae05813 Mon Sep 17 00:00:00 2001 From: Marc Schefer <31961290+marcschefer@users.noreply.github.com> Date: Tue, 28 May 2024 15:16:29 +0200 Subject: [PATCH 20/24] Update dependencies.txt --- .github/workflows/dependencies.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dependencies.txt b/.github/workflows/dependencies.txt index cfb67146d..8c7a68404 100644 --- a/.github/workflows/dependencies.txt +++ b/.github/workflows/dependencies.txt @@ -2,6 +2,7 @@ $PYTHON-devel $PYTHON-numpy $PYTHON-pytest $PYTHON-sphinx +$PYTHON-setuptools CCfits-devel blas-devel $BOOST-$PYTHON-devel From 8b67017872eb31f7a87cc5bd450c34c8e7b4f596 Mon Sep 17 00:00:00 2001 From: Marc Schefer <31961290+marcschefer@users.noreply.github.com> Date: Tue, 28 May 2024 16:00:42 +0200 Subject: [PATCH 21/24] Fix git lfs --- .github/workflows/run-litmus.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/run-litmus.sh b/.github/workflows/run-litmus.sh index 64eddd0f2..c5af94ed3 100755 --- a/.github/workflows/run-litmus.sh +++ b/.github/workflows/run-litmus.sh @@ -6,6 +6,8 @@ TARGET_BRANCH=${GITHUB_BASE_REF:-$GITHUB_REF} # In the case of pull requests, this is the origin branch ORIGIN_BRANCH=${GITHUB_HEAD_REF:-$GITHUB_REF} +GIT_CLONE_PROTECTION_ACTIVE=false + # Platform-specific configuration source /etc/os-release From 5024855ac0254f6a9c04ee29f51257230609de4e Mon Sep 17 00:00:00 2001 From: Marc Schefer <31961290+marcschefer@users.noreply.github.com> Date: Wed, 29 May 2024 15:34:37 +0200 Subject: [PATCH 22/24] Update run-litmus.sh --- .github/workflows/run-litmus.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-litmus.sh b/.github/workflows/run-litmus.sh index c5af94ed3..1e34998ad 100755 --- a/.github/workflows/run-litmus.sh +++ b/.github/workflows/run-litmus.sh @@ -6,7 +6,7 @@ TARGET_BRANCH=${GITHUB_BASE_REF:-$GITHUB_REF} # In the case of pull requests, this is the origin branch ORIGIN_BRANCH=${GITHUB_HEAD_REF:-$GITHUB_REF} -GIT_CLONE_PROTECTION_ACTIVE=false +export GIT_CLONE_PROTECTION_ACTIVE=false # Platform-specific configuration source /etc/os-release From 42c37bdc019ab487c622ad22018e42ebad43ef4b Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Fri, 7 Jun 2024 15:17:34 +0200 Subject: [PATCH 23/24] AperturePhotometry ignore flags when in no detection image mode --- .../AperturePhotometry/AperturePhotometryTask.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/SEImplementation/src/lib/Plugin/AperturePhotometry/AperturePhotometryTask.cpp b/SEImplementation/src/lib/Plugin/AperturePhotometry/AperturePhotometryTask.cpp index 44a1e96d6..42d838f9d 100644 --- a/SEImplementation/src/lib/Plugin/AperturePhotometry/AperturePhotometryTask.cpp +++ b/SEImplementation/src/lib/Plugin/AperturePhotometry/AperturePhotometryTask.cpp @@ -97,10 +97,14 @@ void AperturePhotometryTask::computeProperties(SourceInterface &source) const { additional_flags |= Flags::SATURATED * source.getProperty(m_instance).getSaturateFlag(); additional_flags |= Flags::BLENDED * source.getProperty().getBlendedFlag(); - auto aperture_flags = source.getProperty().getFlags(); - for (size_t i = 0; i < m_apertures.size(); ++i) { - auto det_flag = aperture_flags.at(m_apertures[i]); - flags[i] |= additional_flags | det_flag; + try { + auto aperture_flags = source.getProperty().getFlags(); + for (size_t i = 0; i < m_apertures.size(); ++i) { + auto det_flag = aperture_flags.at(m_apertures[i]); + flags[i] |= additional_flags | det_flag; + } + } catch (PropertyNotFoundException& e) { + // In no detection image mode we can't get flags from the detection image } // set the source properties From 70996527b04b0ed729f59100544199edc4f6c36b Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Fri, 7 Jun 2024 15:31:40 +0200 Subject: [PATCH 24/24] still use flags that don't require the detection image --- .../lib/Plugin/AperturePhotometry/AperturePhotometryTask.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/SEImplementation/src/lib/Plugin/AperturePhotometry/AperturePhotometryTask.cpp b/SEImplementation/src/lib/Plugin/AperturePhotometry/AperturePhotometryTask.cpp index 42d838f9d..05a786fb4 100644 --- a/SEImplementation/src/lib/Plugin/AperturePhotometry/AperturePhotometryTask.cpp +++ b/SEImplementation/src/lib/Plugin/AperturePhotometry/AperturePhotometryTask.cpp @@ -105,6 +105,9 @@ void AperturePhotometryTask::computeProperties(SourceInterface &source) const { } } catch (PropertyNotFoundException& e) { // In no detection image mode we can't get flags from the detection image + for (size_t i = 0; i < m_apertures.size(); ++i) { + flags[i] |= additional_flags; + } } // set the source properties