Elasticc 2.1 (#520)

* Deploy Elasticc v2.1 training set * Update path to elasticc data v2.1
astrolabsoftware · Nov 6, 2023 · 2f4ecf2 · 2f4ecf2
1 parent 575a276
commit 2f4ecf2
Show file tree

Hide file tree

Showing 4 changed files with 945 additions and 5 deletions.
diff --git a/apps/query_cluster.py b/apps/query_cluster.py
@@ -47,6 +47,10 @@
 elasticc_v2_dates = pd.read_csv('assets/elasticc_v2_dates.csv')
 elasticc_v2_dates['date'] = elasticc_v2_dates['date'].astype('str')
 
+elasticc_v2p1_classes = pd.read_csv('assets/elasticc_v2p1_classes.csv')
+elasticc_v2p1_dates = pd.read_csv('assets/elasticc_v2p1_dates.csv')
+elasticc_v2p1_dates['date'] = elasticc_v2p1_dates['date'].astype('str')
+
 coeffs_per_class = pd.read_parquet('assets/fclass_2022_060708_coeffs.parquet')
 
 @app.callback(
@@ -259,7 +263,7 @@ def display_filter_tab(trans_datasource):
                 dmc.Radio(label=l, value=k, size='sm', color='orange')
                 for l, k in zip(labels, values)
             ]
-        elif trans_datasource == 'ELASTiCC (v2)':
+        elif trans_datasource == 'ELASTiCC (v2.0)':
             minDate = date(2023, 11, 27)
             maxDate = date(2026, 12, 5)
             data_class_select = [
@@ -278,6 +282,25 @@ def display_filter_tab(trans_datasource):
                 dmc.Radio(label=l, value=k, size='sm', color='orange')
                 for l, k in zip(labels, values)
             ]
+        elif trans_datasource == 'ELASTiCC (v2.1)':
+            minDate = date(2023, 11, 27)
+            maxDate = date(2026, 12, 5)
+            data_class_select = [
+                {'label': 'All classes', 'value': 'allclasses'},
+                *[{'label': str(simtype), 'value': simtype} for simtype in sorted(elasticc_v2p1_classes['classId'].values)],
+            ]
+            description = [
+                "One condition per line (SQL syntax), ending with semi-colon. See ",
+                dmc.Anchor("here", href="https://portal.nersc.gov/cfs/lsst/DESC_TD_PUBLIC/ELASTICC/#alertschema", size="xs", target="_blank"),
+                " for field description.",
+            ]
+            placeholder = "e.g. diaSource.psFlux > 0.0;"
+            labels = ["Full packet (~1.4 KB/alert)"]
+            values = ['Full packet']
+            data_content = [
+                dmc.Radio(label=l, value=k, size='sm', color='orange')
+                for l, k in zip(labels, values)
+            ]
 
         return {}, minDate, maxDate, data_class_select, description, placeholder, data_content
 
@@ -457,9 +480,12 @@ def summary_tab(trans_content, trans_datasource, date_range_picker, class_select
         elif trans_datasource == 'ELASTiCC (v1)':
             total, count = estimate_alert_number_elasticc(date_range_picker, class_select, elasticc_v1_dates, elasticc_v1_classes)
             sizeGb = estimate_size_gb_elasticc(trans_content)
-        elif trans_datasource == 'ELASTiCC (v2)':
+        elif trans_datasource == 'ELASTiCC (v2.0)':
             total, count = estimate_alert_number_elasticc(date_range_picker, class_select, elasticc_v2_dates, elasticc_v2_classes)
             sizeGb = estimate_size_gb_elasticc(trans_content)
+        elif trans_datasource == 'ELASTiCC (v2.1)':
+            total, count = estimate_alert_number_elasticc(date_range_picker, class_select, elasticc_v2p1_dates, elasticc_v2p1_classes)
+            sizeGb = estimate_size_gb_elasticc(trans_content)
 
         if count == 0:
             msg_title = 'No alerts found. Try to update your criteria.'
@@ -653,10 +679,14 @@ def submit_job(n_clicks, trans_content, trans_datasource, date_range_picker, cla
             topic_name = 'ftransfer_elasticc_v1_{}_{}'.format(d.date().isoformat(), d.microsecond)
             fn = 'assets/spark_elasticc_transfer.py'
             basepath = '/user/julien.peloton/elasticc_curated_truth_int'
-        elif trans_datasource == 'ELASTiCC (v2)':
+        elif trans_datasource == 'ELASTiCC (v2.0)':
             topic_name = 'ftransfer_elasticc_v2_{}_{}'.format(d.date().isoformat(), d.microsecond)
             fn = 'assets/spark_elasticc_transfer.py'
             basepath = '/user/julien.peloton/elasticc-2023-training_v2'
+        elif trans_datasource == 'ELASTiCC (v2.1)':
+            topic_name = 'ftransfer_elasticc_v2p1_{}_{}'.format(d.date().isoformat(), d.microsecond)
+            fn = 'assets/spark_elasticc_transfer.py'
+            basepath = '/user/julien.peloton/elasticc_training_v2p1_partitioned'
         filename = 'stream_{}.py'.format(topic_name)
 
         with open(fn, 'r') as f:
@@ -732,7 +762,7 @@ def query_builder():
         [
             dmc.Divider(variant="solid", label='Data Source'),
             dmc.RadioGroup(
-                [dmc.Radio(k, value=k, size='sm', color='orange') for k in ['ZTF', 'ELASTiCC (v1)', 'ELASTiCC (v2)']],
+                [dmc.Radio(k, value=k, size='sm', color='orange') for k in ['ZTF', 'ELASTiCC (v1)', 'ELASTiCC (v2.0)', 'ELASTiCC (v2.1)']],
                 id="trans_datasource",
                 value=None,
                 label="Choose the type of alerts you want to retrieve",

diff --git a/assets/elasticc_v2p1_classes.csv b/assets/elasticc_v2p1_classes.csv
@@ -0,0 +1,19 @@
+,classId,count
+0,2235,1636505
+1,2225,271616
+2,2322,3201048
+3,2323,4277723
+4,2232,14739
+5,2223,2130375
+6,2245,93055
+7,2224,3187533
+8,2246,3077338
+9,2244,205653
+10,2324,5840976
+11,2226,289524
+12,2234,84419
+13,2233,3612
+14,2332,7670059
+15,2222,1310502
+16,2243,1053183
+17,2325,4472195