tag transformer for justice data (#205)

* tag transformer for justice data

* rename of transformer script

* up duration for assuming role

.github/workflows/ingest-cadet-metadata.yml

@@ -41,6 +41,7 @@ jobs:
       - uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ secrets.CADET_METADATA_ROLE_TO_ASSUME }}
+          role-duration-seconds: 14400
           aws-region: ${{ inputs.ECR_REGION }}
 
       - name: cache poetry install

ingestion/justice_data_ingest.yaml

@@ -2,3 +2,8 @@ source:
   type: ingestion.justice_data_source.source.JusticeDataAPISource
   config:
     base_url: "https://data.justice.gov.uk/api"
+
+transformers:
+  - type: "add_dataset_tags"
+    config:
+      get_tags_to_add: "ingestion.taggers.display_in_catalogue_tagger.add_display_in_catalogue_tag"

ingestion/taggers/cadet_display_in_catalogue_tagger.py → ingestion/taggers/display_in_catalogue_tagger.py

@@ -6,7 +6,10 @@
 
 
 def add_display_in_catalogue_tag(entity_urn: str) -> List[TagAssociationClass]:
-    """Compute the tags to associate to a given dataset."""
+    """
+    Transformer to add dc_display_in_catalogue tag to all ingested entities
+    other than the athena entities created via a dbt cadet ingestion
+    """
     if "athena_cadet" not in entity_urn:
         tag_urn = builder.make_tag_urn(tag="dc_display_in_catalogue")
         tags = [TagAssociationClass(tag=tag_urn)]