From 06fec65aa2b0882729b7518a57c89c4bf0a3f3d2 Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Wed, 24 Apr 2024 18:29:27 +1200 Subject: [PATCH 1/3] Add option to not reate events at detection time --- .../core/match_filter/helpers/processes.py | 5 ++++- eqcorrscan/core/match_filter/helpers/tribe.py | 12 +++++++++--- eqcorrscan/core/match_filter/tribe.py | 17 ++++++++++------- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/eqcorrscan/core/match_filter/helpers/processes.py b/eqcorrscan/core/match_filter/helpers/processes.py index 7a85945c0..89323e96f 100644 --- a/eqcorrscan/core/match_filter/helpers/processes.py +++ b/eqcorrscan/core/match_filter/helpers/processes.py @@ -638,6 +638,7 @@ def _make_detections( threshold: float, threshold_type: str, save_progress: bool, + make_events: bool, output_queue: Queue, poison_queue: Queue, ): @@ -661,6 +662,8 @@ def _make_detections( :param save_progress: Whether to save progress or not: If true, individual Party files will be written each time this is run. + :param make_events: + Whether to make events for all detections or not. :param output_queue: Queue of output Party filenames. :param poison_queue: @@ -690,7 +693,7 @@ def _make_detections( detections=detections, threshold=threshold, threshold_type=threshold_type, templates=templates, chunk_start=starttime, chunk_id=chunk_id, - save_progress=save_progress) + save_progress=save_progress, make_events=make_events) chunk_id += 1 output_queue.put_nowait(chunk_file) except Exception as e: diff --git a/eqcorrscan/core/match_filter/helpers/tribe.py b/eqcorrscan/core/match_filter/helpers/tribe.py index 9786a68fc..09086e239 100644 --- a/eqcorrscan/core/match_filter/helpers/tribe.py +++ b/eqcorrscan/core/match_filter/helpers/tribe.py @@ -606,7 +606,8 @@ def _make_party( templates: List[Template], chunk_start: UTCDateTime, chunk_id: int, - save_progress: bool + save_progress: bool, + make_events: bool, ) -> str: """ Construct a Party from Detections. @@ -618,6 +619,7 @@ def _make_party( :param chunk_start: Starttime of party epoch :param chunk_id: Internal index for party epoch :param save_progress: Whether to save progress or not + :param make_events: Whether to make events for all detections or not :return: The filename the party has been pickled to. """ @@ -646,7 +648,10 @@ def _make_party( detection_idx_dict[detection.template_name].append(n) # Convert to Families and build party. - Logger.info("Converting to party and making events") + if not make_events: + Logger.info("Converting to party") + else: + Logger.info("Converting to party and making events") chunk_party = Party() # Make a dictionary of templates keyed by name - we could be passed a dict @@ -665,7 +670,8 @@ def _make_party( with open(template, "rb") as f: template = pickle.load(f) for d in family_detections: - d._calculate_event(template=template) + if make_events: + d._calculate_event(template=template) family = Family( template=template, detections=family_detections) chunk_party += family diff --git a/eqcorrscan/core/match_filter/tribe.py b/eqcorrscan/core/match_filter/tribe.py index e415d1897..3ad1a644c 100644 --- a/eqcorrscan/core/match_filter/tribe.py +++ b/eqcorrscan/core/match_filter/tribe.py @@ -657,7 +657,7 @@ def detect(self, stream, threshold, threshold_type, trig_int, plot=False, concurrent_processing=False, ignore_length=False, ignore_bad_data=False, group_size=None, overlap="calculate", full_peaks=False, save_progress=False, process_cores=None, - pre_processed=False, check_processing=True, + pre_processed=False, check_processing=True, make_events=True, **kwargs): """ Detect using a Tribe of templates within a continuous stream. @@ -871,7 +871,7 @@ def detect(self, stream, threshold, threshold_type, trig_int, plot=False, ignore_bad_data, group_size, groups, sampling_rate, threshold, threshold_type, save_progress, xcorr_func, concurrency, cores, export_cccsums, parallel, peak_cores, trig_int, full_peaks, - plot, plotdir, plot_format,) + plot, plotdir, plot_format, make_events,) if concurrent_processing: party = self._detect_concurrent(*args, **inner_kwargs) @@ -899,7 +899,7 @@ def _detect_serial( group_size, groups, sampling_rate, threshold, threshold_type, save_progress, xcorr_func, concurrency, cores, export_cccsums, parallel, peak_cores, trig_int, full_peaks, plot, plotdir, plot_format, - **kwargs + make_events,**kwargs ): """ Internal serial detect workflow. """ from eqcorrscan.core.match_filter.helpers.tribe import ( @@ -961,7 +961,8 @@ def _detect_serial( detections=detections, threshold=threshold, threshold_type=threshold_type, templates=self.templates, chunk_start=starttime, - chunk_id=i, save_progress=save_progress) + chunk_id=i, save_progress=save_progress, + make_events=make_events) chunk_files.append(chunk_file) # Rebuild for _chunk_file in chunk_files: @@ -986,7 +987,7 @@ def _detect_concurrent( group_size, groups, sampling_rate, threshold, threshold_type, save_progress, xcorr_func, concurrency, cores, export_cccsums, parallel, peak_cores, trig_int, full_peaks, plot, plotdir, plot_format, - **kwargs + make_events, **kwargs ): """ Internal concurrent detect workflow. """ from eqcorrscan.core.match_filter.helpers.processes import ( @@ -1074,6 +1075,7 @@ def _detect_concurrent( threshold=threshold, threshold_type=threshold_type, save_progress=save_progress, + make_events=make_events, output_queue=party_file_queue, poison_queue=poison_queue, ), @@ -1224,7 +1226,7 @@ def client_detect(self, client, starttime, endtime, threshold, ignore_bad_data=False, group_size=None, return_stream=False, full_peaks=False, save_progress=False, process_cores=None, retries=3, - check_processing=True, **kwargs): + check_processing=True, make_events, **kwargs): """ Detect using a Tribe of templates within a continuous stream. @@ -1425,7 +1427,8 @@ def client_detect(self, client, starttime, endtime, threshold, process_cores=process_cores, save_progress=save_progress, return_stream=return_stream, check_processing=False, poison_queue=poison_queue, shutdown=False, - concurrent_processing=concurrent_processing, groups=groups) + concurrent_processing=concurrent_processing, groups=groups, + make_events=make_events) if not concurrent_processing: Logger.warning("Using concurrent_processing=True can be faster if" From 9e109ebce536e53909e844e94909f6d478cf8f7e Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Wed, 24 Apr 2024 19:37:41 +1200 Subject: [PATCH 2/3] add default --- eqcorrscan/core/match_filter/tribe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eqcorrscan/core/match_filter/tribe.py b/eqcorrscan/core/match_filter/tribe.py index 3ad1a644c..186090096 100644 --- a/eqcorrscan/core/match_filter/tribe.py +++ b/eqcorrscan/core/match_filter/tribe.py @@ -1226,7 +1226,7 @@ def client_detect(self, client, starttime, endtime, threshold, ignore_bad_data=False, group_size=None, return_stream=False, full_peaks=False, save_progress=False, process_cores=None, retries=3, - check_processing=True, make_events, **kwargs): + check_processing=True, make_events=True, **kwargs): """ Detect using a Tribe of templates within a continuous stream. From b6d6faa70e962a03156515d45d7b089dc10340fc Mon Sep 17 00:00:00 2001 From: Calum Chamberlain Date: Wed, 26 Jun 2024 14:39:11 +1200 Subject: [PATCH 3/3] linter --- eqcorrscan/core/match_filter/tribe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eqcorrscan/core/match_filter/tribe.py b/eqcorrscan/core/match_filter/tribe.py index d71382740..d7db6b1ba 100644 --- a/eqcorrscan/core/match_filter/tribe.py +++ b/eqcorrscan/core/match_filter/tribe.py @@ -968,7 +968,7 @@ def _detect_serial( detections=detections, threshold=threshold, threshold_type=threshold_type, templates=self.templates, chunk_start=starttime, - chunk_id=i, save_progress=save_progress, + chunk_id=i, save_progress=save_progress, make_events=make_events) chunk_files.append(chunk_file) # Rebuild @@ -1234,7 +1234,7 @@ def client_detect(self, client, starttime, endtime, threshold, ignore_bad_data=False, group_size=None, return_stream=False, full_peaks=False, save_progress=False, process_cores=None, retries=3, - check_processing=True, make_events=True, + check_processing=True, make_events=True, min_stations=0, **kwargs): """ Detect using a Tribe of templates within a continuous stream.