FilterTestHandler - minor bugfixes (#431)

* add a cutoff on the prv_candidates we add to each alert, to not include data points after the obs time of the alert that we are filtering

* loosen numpy version requirement, lint, update actions versions

.github/workflows/lint.yaml

@@ -13,8 +13,8 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
       - name: Install dependencies

.github/workflows/test.yaml

@@ -13,15 +13,15 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install "wheel>=0.36.0"
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/pip
@@ -47,12 +47,12 @@ jobs:
       - name: Run tests
         run: make docker_test
       - name: Collect Docker Logs
-        uses: jwalton/[email protected].1
+        uses: jwalton/[email protected].2
         with:
           images: 'skyportal/kowalski-ingester,skyportal/kowalski-api,kowalski-mongo'
           dest: './logs'
       - name: Upload logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: logs
           path: ./logs

kowalski/api/handlers/filter.py

@@ -1278,6 +1278,20 @@ async def post(self, request: web.Request) -> web.Response:
                         "$and"
                     ][0]["$in"][1] = filter_existing.permissions
 
+                # since we are running after the fact, we need to also have a cut on jd
+                # to only keep the data points from prv_candidates and fp_hists
+                # that are older than the alert we are merging it with, like:
+                # {
+                #   $lt: ["$$item.jd", "$candidate.jd"]
+                # }
+                filter_pipeline[3]["$project"]["prv_candidates"]["$filter"]["cond"][
+                    "$and"
+                ].append({"$lt": ["$$item.jd", "$candidate.jd"]})
+                if "fp_hists" in filter_pipeline[3]["$project"]:
+                    filter_pipeline[3]["$project"]["fp_hists"]["$filter"]["cond"][
+                        "$and"
+                    ].append({"$lt": ["$$item.jd", "$candidate.jd"]})
+
             if objects is not None:
                 # match objects
                 filter_pipeline[0]["$match"]["objectId"] = {"$in": objects}
@@ -1288,7 +1302,7 @@ async def post(self, request: web.Request) -> web.Response:
                 }
 
             cursor = request.app["mongo"][filter_existing.catalog].aggregate(
-                filter_pipeline, allowDiskUse=False, maxTimeMS=30000
+                filter_pipeline, allowDiskUse=False, maxTimeMS=max_time_ms
             )
             alerts = await cursor.to_list(length=None)
 

requirements/requirements_api.txt

@@ -10,7 +10,7 @@ matplotlib==3.9.2
 motor==3.1.1
 multidict==6.0.5
 numba==0.60.0
-numpy==1.23.5
+numpy>=1.23.5, <2.0.0
 odmantic==0.9.2
 pandas==2.2.2
 pydantic==1.10.13 # bump to v2 once odmantic supports it

requirements/requirements_ingester.txt

@@ -16,7 +16,7 @@ jinja2>=2.11.2
 matplotlib==3.9.2
 motor==3.1.1
 numba==0.60.0
-numpy==1.23.5
+numpy>=1.23.5, <2.0.0
 pandas==2.2.2
 protobuf==4.22.1
 pyarrow==17.0.0