Harvard IHP Harvest fails (local docker)

[aaron-collier]

[2025-01-23, 17:54:11 UTC] {taskinstance.py:2905} ERROR - Task failed with exception
Traceback (most recent call last):
  File "/home/airflow/.local/lib/python3.12/site-packages/airflow/models/taskinstance.py", line 465, in _execute_task
    result = _execute_callable(context=context, **execute_callable_kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/airflow/.local/lib/python3.12/site-packages/airflow/models/taskinstance.py", line 432, in _execute_callable
    return execute_callable(context=context, **execute_callable_kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/airflow/.local/lib/python3.12/site-packages/airflow/models/baseoperator.py", line 401, in wrapper
    return func(self, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/airflow/.local/lib/python3.12/site-packages/airflow/operators/python.py", line 235, in execute
    return_value = self.execute_callable()
                   ^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/airflow/.local/lib/python3.12/site-packages/airflow/operators/python.py", line 252, in execute_callable
    return self.python_callable(*self.op_args, **self.op_kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/airflow/dlme_airflow/harvester/source_harvester.py", line 24, in data_source_harvester
    df_and_csv = dataframe_to_file(collection, last_harvest_start_date)
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/airflow/dlme_airflow/utils/dataframe.py", line 40, in dataframe_to_file
    source_df = collection.catalog.read()
                ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/airflow/dlme_airflow/drivers/oai_xml.py", line 248, in read
    self._load_metadata()
  File "/home/airflow/.local/lib/python3.12/site-packages/intake/source/base.py", line 84, in _load_metadata
    self._schema = self._get_schema()
                   ^^^^^^^^^^^^^^^^^^
  File "/opt/airflow/dlme_airflow/drivers/oai_xml.py", line 200, in _get_schema
    self._open_set()
  File "/opt/airflow/dlme_airflow/drivers/oai_xml.py", line 82, in _open_set
    oai_records = self._collection.ListRecords(**opts)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/airflow/.local/lib/python3.12/site-packages/sickle/app.py", line 150, in ListRecords
    return self.iterator(self, params, ignore_deleted=ignore_deleted)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/airflow/.local/lib/python3.12/site-packages/sickle/iterator.py", line 135, in __init__
    super(OAIItemIterator, self).__init__(sickle, params, ignore_deleted)
  File "/home/airflow/.local/lib/python3.12/site-packages/sickle/iterator.py", line 46, in __init__
    self._next_response()
  File "/home/airflow/.local/lib/python3.12/site-packages/sickle/iterator.py", line 138, in _next_response
    super(OAIItemIterator, self)._next_response()
  File "/home/airflow/.local/lib/python3.12/site-packages/sickle/iterator.py", line 84, in _next_response
    self.oai_response = self.sickle.harvest(**params)
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/airflow/.local/lib/python3.12/site-packages/sickle/app.py", line 130, in harvest
    http_response.raise_for_status()
  File "/home/airflow/.local/lib/python3.12/site-packages/requests/models.py", line 1024, in raise_for_status
    raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 404 Client Error: Not Found for url: https://api.lib.harvard.edu/oai?set=ihp&metadataPrefix=mods&verb=ListRecords