Improvements for the search page

app.py

@@ -30,6 +30,7 @@
 args = yaml.load(open('config.yml'), yaml.Loader)
 
 APIURL = args['APIURL']
+LOCALAPI = args.get('LOCALAPI', False)
 
 # bootstrap theme
 external_stylesheets = [

apps/api/api.py

@@ -266,28 +266,28 @@ def layout():
         'description': 'Conesearch radius in arcsec. Maximum is 36,000 arcseconds (10 degrees).'
     },
     {
-        'name': 'startdate_conesearch',
+        'name': 'startdate',
         'required': False,
-        'group': 1,
-        'description': '[Optional] Starting date in UTC for the conesearch query.'
+        'group': None,
+        'description': '[Optional] Starting date in UTC, as either ISO string, JD or MJD.'
     },
     {
-        'name': 'window_days_conesearch',
+        'name': 'stopdate',
         'required': False,
-        'group': 1,
-        'description': '[Optional] Time window in days for the conesearch query.'
+        'group': None,
+        'description': '[Optional] Stopping date in UTC, as either ISO string, JD or MJD.'
     },
     {
-        'name': 'startdate',
-        'required': True,
-        'group': 2,
-        'description': 'Starting date in UTC'
+        'name': 'window',
+        'required': False,
+        'group': None,
+        'description': '[Optional] Time window in days, may be used instead of stopdate'
     },
     {
-        'name': 'window',
-        'required': True,
-        'group': 2,
-        'description': 'Time window in minutes. Maximum is 180 minutes.'
+        'name': 'n',
+        'required': False,
+        'group': None,
+        'description': 'Maximal number of alerts to return. Default is 1000.'
     },
     {
         'name': 'output-format',
@@ -789,24 +789,27 @@ def query_db(payload=None):
 
     # Check the user specifies only one group
     all_groups = [i['group'] for i in args_explorer if i['group'] is not None and i['name'] in payload]
-    if len(np.unique(all_groups)) != 1:
+    if len(np.unique(all_groups)) > 1:
         rep = {
             'status': 'error',
             'text': "You need to set parameters from the same group\n"
         }
         return Response(str(rep), 400)
 
     # Check the user specifies all parameters within a group
-    user_group = np.unique(all_groups)[0]
-    required_args = [i['name'] for i in args_explorer if i['group'] == user_group]
-    required = [i['required'] for i in args_explorer if i['group'] == user_group]
-    for required_arg, required_ in zip(required_args, required):
-        if (required_arg not in payload) and required_:
-            rep = {
-                'status': 'error',
-                'text': "A value for `{}` is required for group {}. Use GET to check arguments.\n".format(required_arg, user_group)
-            }
-            return Response(str(rep), 400)
+    if len(np.unique(all_groups)) == 1:
+        user_group = np.unique(all_groups)[0]
+        required_args = [i['name'] for i in args_explorer if i['group'] == user_group]
+        required = [i['required'] for i in args_explorer if i['group'] == user_group]
+        for required_arg, required_ in zip(required_args, required):
+            if (required_arg not in payload) and required_:
+                rep = {
+                    'status': 'error',
+                    'text': "A value for `{}` is required for group {}. Use GET to check arguments.\n".format(required_arg, user_group)
+                }
+                return Response(str(rep), 400)
+    else:
+        user_group = None
 
     pdfs = return_explorer_pdf(payload, user_group)
 

apps/api/doc.py

@@ -266,7 +266,7 @@
 you should use the `Retrieve single object data` service instead.
 
 Currently, you cannot query using several conditions.
-You must choose among `Search by Object ID` (group 0), `Conesearch` (group 1), or `Search by Date` (group 2).
+You must choose among `Search by Object ID` and `Conesearch` / `Search by Date`.
 In a future release, you will be able to combine searches.
 The list of arguments for querying the Fink alert database can be found at https://fink-portal.org/api/v1/explorer.
 
@@ -365,8 +365,8 @@
     'ra': '193.822',
     'dec': '2.89732',
     'radius': '5',
-    'startdate_conesearch': '2021-06-25 05:59:37.000',
-    'window_days_conesearch': 7
+    'startdate': '2021-06-25 05:59:37.000',
+    'window': 7
   }
 )
 
@@ -386,8 +386,8 @@
 
 ### Search by Date
 
-Choose a starting date and a time window to see all alerts in this period.
-Dates are in UTC, and the time window in minutes.
+Choose a starting date and stopping date or a time window to see all alerts in this period.
+Dates are in UTC, and the time window in days (but should not exceed 3 hours).
 Example of valid search:
 
 * 2021-07-01 05:59:37.000
@@ -396,10 +396,10 @@
 
 ```bash
 # Get all objects between 2021-07-01 05:59:37.000 and 2021-07-01 06:09:37.000 UTC
-curl -H "Content-Type: application/json" -X POST -d '{"startdate":"2021-07-01 05:59:37.000", "window":"10"}' https://fink-portal.org/api/v1/explorer -o datesearch.json
+curl -H "Content-Type: application/json" -X POST -d '{"startdate":"2021-07-01 05:59:37.000", "window":"0.007"}' https://fink-portal.org/api/v1/explorer -o datesearch.json
 
 # you can also specify parameters in the URL, e.g. with wget:
-wget "https://fink-portal.org/api/v1/explorer?startdate=2021-07-01 05:59:37.000&window=10&output-format=json" -O datesearch.json
+wget "https://fink-portal.org/api/v1/explorer?startdate=2021-07-01 05:59:37.000&window=0.007&output-format=json" -O datesearch.json
 ```
 
 In python, you would use
@@ -413,7 +413,7 @@
   'https://fink-portal.org/api/v1/explorer',
   json={
     'startdate': '2021-07-01 05:59:37.000',
-    'window': '10'
+    'window': '0.007'
   }
 )
 
@@ -1795,4 +1795,4 @@
 2    number     624188
 3  ssnamenr     624188
 ```
-"""
+"""

apps/api/utils.py

@@ -39,6 +39,7 @@
 from apps.utils import format_hbase_output
 from apps.utils import extract_cutouts
 from apps.utils import hbase_type_converter
+from apps.utils import isoify_time
 
 from apps.euclid.utils import load_euclid_header
 from apps.euclid.utils import add_columns
@@ -186,7 +187,24 @@ def return_explorer_pdf(payload: dict, user_group: int) -> pd.DataFrame:
     out: pandas dataframe
     """
     truncated = False
+
+    if 'startdate' in payload:
+        jd_start = Time(isoify_time(payload['startdate'])).jd
+    else:
+        jd_start = Time('2019-11-01 00:00:00').jd
+
+    if 'stopdate' in payload:
+        jd_stop = Time(isoify_time(payload['stopdate'])).jd
+    elif 'window' in payload and 'startdate' in payload:
+        window = float(payload['window'])
+        jd_stop = jd_start + window
+    else:
+        jd_stop = Time.now().jd
+
+    n = int(payload.get('n', 1000))
+
     if user_group == 0:
+        # objectId search
         client = connect_to_hbase_table('ztf')
         results = {}
         for oid in payload['objectId'].split(','):
@@ -201,21 +219,15 @@ def return_explorer_pdf(payload: dict, user_group: int) -> pd.DataFrame:
             results.update(result)
 
         schema_client = client.schema()
-    if user_group == 1:
+    elif user_group == 1:
+        # Conesearch with optional date range
         client = connect_to_hbase_table('ztf.pixel128')
+        client.setLimit(n)
+
         # Interpret user input
         ra, dec = payload['ra'], payload['dec']
         radius = payload['radius']
 
-        if 'startdate_conesearch' in payload:
-            startdate = payload['startdate_conesearch']
-        else:
-            startdate = None
-        if 'window_days_conesearch' in payload and payload['window_days_conesearch'] is not None:
-            window_days = float(payload['window_days_conesearch'])
-        else:
-            window_days = 1.0
-
         if float(radius) > 18000.:
             rep = {
                 'status': 'error',
@@ -254,24 +266,12 @@ def return_explorer_pdf(payload: dict, user_group: int) -> pd.DataFrame:
             inclusive=True
         )
 
-        # For the future: we could set client.setRangeScan(True)
-        # and pass directly the time boundaries here instead of
-        # grouping by later.
-
-        # Filter by time - logic to be improved...
-        if startdate is not None:
-            if ':' in str(startdate):
-                jdstart = Time(startdate).jd
-            elif str(startdate).startswith('24'):
-                jdstart = Time(startdate, format='jd').jd
-            else:
-                jdstart = Time(startdate, format='mjd').jd
-            jdend = jdstart + window_days
-
+        # Filter by time
+        if 'startdate' in payload:
             client.setRangeScan(True)
             results = {}
             for pix in pixs:
-                to_search = "key:key:{}_{},key:key:{}_{}".format(pix, jdstart, pix, jdend)
+                to_search = "key:key:{}_{},key:key:{}_{}".format(pix, jd_start, pix, jd_stop)
                 result = client.scan(
                     "",
                     to_search,
@@ -281,7 +281,7 @@ def return_explorer_pdf(payload: dict, user_group: int) -> pd.DataFrame:
                 results.update(result)
             client.setRangeScan(False)
         else:
-            client = connect_to_hbase_table('ztf.pixel128')
+            # Do we really need this separate branch?.. Is it much faster than range scan?..
             results = {}
             for pix in pixs:
                 to_search = "key:key:{}".format(pix)
@@ -295,21 +295,18 @@ def return_explorer_pdf(payload: dict, user_group: int) -> pd.DataFrame:
 
         schema_client = client.schema()
         truncated = True
-    elif user_group == 2:
+    else:
+        # Plain date search
         client = connect_to_hbase_table('ztf.jd')
-        if int(payload['window']) > 180:
-            rep = {
-                'status': 'error',
-                'text': "`window` cannot be bigger than 180 minutes.\n"
-            }
-            return Response(str(rep), 400)
-        # Time to jd
-        jd_start = Time(payload['startdate']).jd
-        jd_end = jd_start + TimeDelta(int(payload['window']) * 60, format='sec').jd
+
+        # Limit the time window to 3 hours days
+        if jd_stop - jd_start > 3/24:
+            jd_stop = jd_start + 3/24
 
         # Send the request. RangeScan.
         client.setRangeScan(True)
-        to_evaluate = "key:key:{},key:key:{}".format(jd_start, jd_end)
+        client.setLimit(n)
+        to_evaluate = "key:key:{},key:key:{}".format(jd_start, jd_stop)
         results = client.scan(
             "",
             to_evaluate,