diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 472c9fe5..67c757be 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -13,8 +13,8 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install dependencies diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0b65672c..08b32261 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -13,15 +13,15 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.10" - name: Install dependencies run: | python -m pip install --upgrade pip pip install "wheel>=0.36.0" - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: | ~/.cache/pip @@ -47,12 +47,12 @@ jobs: - name: Run tests run: make docker_test - name: Collect Docker Logs - uses: jwalton/gh-docker-logs@v2.2.1 + uses: jwalton/gh-docker-logs@v2.2.2 with: images: 'skyportal/kowalski-ingester,skyportal/kowalski-api,kowalski-mongo' dest: './logs' - name: Upload logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: logs path: ./logs diff --git a/kowalski/api/handlers/filter.py b/kowalski/api/handlers/filter.py index 78a8a8b4..c0971a0e 100644 --- a/kowalski/api/handlers/filter.py +++ b/kowalski/api/handlers/filter.py @@ -1278,6 +1278,20 @@ async def post(self, request: web.Request) -> web.Response: "$and" ][0]["$in"][1] = filter_existing.permissions + # since we are running after the fact, we need to also have a cut on jd + # to only keep the data points from prv_candidates and fp_hists + # that are older than the alert we are merging it with, like: + # { + # $lt: ["$$item.jd", "$candidate.jd"] + # } + filter_pipeline[3]["$project"]["prv_candidates"]["$filter"]["cond"][ + "$and" + ].append({"$lt": ["$$item.jd", "$candidate.jd"]}) + if "fp_hists" in filter_pipeline[3]["$project"]: + filter_pipeline[3]["$project"]["fp_hists"]["$filter"]["cond"][ + "$and" + ].append({"$lt": ["$$item.jd", "$candidate.jd"]}) + if objects is not None: # match objects filter_pipeline[0]["$match"]["objectId"] = {"$in": objects} @@ -1288,7 +1302,7 @@ async def post(self, request: web.Request) -> web.Response: } cursor = request.app["mongo"][filter_existing.catalog].aggregate( - filter_pipeline, allowDiskUse=False, maxTimeMS=30000 + filter_pipeline, allowDiskUse=False, maxTimeMS=max_time_ms ) alerts = await cursor.to_list(length=None) diff --git a/requirements/requirements_api.txt b/requirements/requirements_api.txt index bfae060a..6360b129 100644 --- a/requirements/requirements_api.txt +++ b/requirements/requirements_api.txt @@ -10,7 +10,7 @@ matplotlib==3.9.2 motor==3.1.1 multidict==6.0.5 numba==0.60.0 -numpy==1.23.5 +numpy>=1.23.5, <2.0.0 odmantic==0.9.2 pandas==2.2.2 pydantic==1.10.13 # bump to v2 once odmantic supports it diff --git a/requirements/requirements_ingester.txt b/requirements/requirements_ingester.txt index ef5ad92d..7c4934f0 100644 --- a/requirements/requirements_ingester.txt +++ b/requirements/requirements_ingester.txt @@ -16,7 +16,7 @@ jinja2>=2.11.2 matplotlib==3.9.2 motor==3.1.1 numba==0.60.0 -numpy==1.23.5 +numpy>=1.23.5, <2.0.0 pandas==2.2.2 protobuf==4.22.1 pyarrow==17.0.0