Merge branch 'main' into fmd-808-nginx-rate-limiting-limit-whitelist

ingestion/cadet.yaml

@@ -27,7 +27,7 @@ source:
         # These tables cause a `java.lang.NullPointerException` and are staging tables anyway
         - ".*avature_stg\\.stg_people_fields_pivoted_wide.*"
         - ".*avature_stg\\.stg_job_fields_pivoted_wide.*"
-        
+
 
         # This table has a huge number of columns, which breaks things.
         - "source.mojap_derived_tables.data_eng_uploader_prod_calc_release_dates.survey_data"
@@ -63,3 +63,8 @@ transformers:
   - type: "ingestion.transformers.assign_cadet_databases.AssignCadetDatabases"
     config:
       manifest_s3_uri: "s3://mojap-derived-tables/prod/run_artefacts/latest/target/manifest.json"
+  - type: "simple_add_dataset_properties"
+    config:
+      properties:
+        Audience: Internal
+        Provider: Analytical Platform

ingestion/create_cadet_databases_source/source.py

@@ -32,6 +32,11 @@
 
 logging.basicConfig(level=logging.DEBUG)
 
+properties_to_add = {
+    "Audience": "Internal",
+    "Provider": "Analytical Platform"
+}
+
 
 @config_class(CreateCadetDatabasesConfig)
 class CreateCadetDatabases(Source):
@@ -117,6 +122,7 @@ def create_database_mcps(
                 backcompat_env_as_instance=True,
             )
             db_meta_dict = dict(database_metadata)
+            db_meta_dict.update(properties_to_add)
             domain_name = format_domain_name(db_meta_dict["domain"])
             domain_urn = mce_builder.make_domain_urn(domain=domain_name)
             display_tag = display_tags.get(database_name, ["dc_cadet"])