Merge branch 'feature/record-user-during-import'

devops/manage-statbus.sh

@@ -265,7 +265,7 @@ case "$action" in
         ./devops/manage-statbus.sh create-users
       ;;
     'create-db' )
-        ./devops/manage-statbus.sh start not_app
+        ./devops/manage-statbus.sh start required_not_app
         ./devops/manage-statbus.sh activate_sql_saga
         ./devops/manage-statbus.sh create-db-structure
         ./devops/manage-statbus.sh create-users
@@ -291,14 +291,6 @@ case "$action" in
       ;;
     'create-users' )
         export $(awk -F= '/^[^#]/{output=output" "$1"="$2} END {print output}' .env)
-        echo "Wait for admin api (gotrue) to start"
-        starting=true
-        while $starting; do
-            sleep 1
-            curl "http://$SUPABASE_BIND_ADDRESS/auth/v1/health" \
-            -H 'accept: application/json' \
-            -H "apikey: $SERVICE_ROLE_KEY" && starting=false
-        done
 
         echo Create users for the developers
         echo 'Creating users defined in .users.yml'
@@ -309,21 +301,12 @@ case "$action" in
           email=$(echo "${user_details}" | awk '{print $1}')
           password=$(echo "${user_details}" | awk '{print $2}')
 
-          # Use the official API, since there isn't an SQL route for this! :-(
-          # Run the curl command for each user
-          curl "http://$SUPABASE_BIND_ADDRESS/auth/v1/admin/users" \
-            -H 'accept: application/json' \
-            -H "apikey: $SERVICE_ROLE_KEY" \
-            -H "authorization: Bearer $SERVICE_ROLE_KEY" \
-            -H 'content-type: application/json' \
-            --data-raw "{\"email\":\"$email\", \"password\":\"$password\", \"email_confirm\":true}"
           ./devops/manage-statbus.sh psql <<EOS
-            INSERT INTO public.statbus_user (uuid, role_id)
-            SELECT id, (SELECT id FROM public.statbus_role WHERE type = 'super_user')
-            FROM auth.users
-            WHERE email like '$email'
-            ON CONFLICT (uuid)
-            DO UPDATE SET role_id = EXCLUDED.role_id;
+            SELECT * FROM public.statbus_user_create(
+              p_email := '$email',
+              p_role_type := 'super_user',
+              p_password := '$password'
+            );
 EOS
         done
       ;;

doc/db/function/admin_apply_rls_and_policies(regclass).md

@@ -23,25 +23,62 @@ BEGIN
         HAVING COUNT(*) = 2
     ) INTO has_custom_and_active;
 
-    RAISE NOTICE '%s.%s: Enabling Row Level Security', schema_name_str, table_name_str;
-    EXECUTE format('ALTER TABLE %I.%I ENABLE ROW LEVEL SECURITY', schema_name_str, table_name_str);
+    -- Check if RLS is already enabled
+    IF NOT EXISTS (
+        SELECT 1 FROM pg_tables 
+        WHERE schemaname = schema_name_str 
+        AND tablename = table_name_str 
+        AND rowsecurity = true
+    ) THEN
+        RAISE NOTICE '%s.%s: Enabling Row Level Security', schema_name_str, table_name_str;
+        EXECUTE format('ALTER TABLE %I.%I ENABLE ROW LEVEL SECURITY', schema_name_str, table_name_str);
+    END IF;
 
-    RAISE NOTICE '%s.%s: Authenticated users can read', schema_name_str, table_name_str;
-    EXECUTE format('CREATE POLICY %s_authenticated_read ON %I.%I FOR SELECT TO authenticated USING (true)', table_name_str, schema_name_str, table_name_str);
+    -- Check if authenticated read policy exists before creating
+    IF NOT EXISTS (
+        SELECT 1 FROM pg_policies 
+        WHERE schemaname = schema_name_str 
+        AND tablename = table_name_str 
+        AND policyname = table_name_str || '_authenticated_read'
+    ) THEN
+        RAISE NOTICE '%s.%s: Creating authenticated users read policy', schema_name_str, table_name_str;
+        EXECUTE format('CREATE POLICY %s_authenticated_read ON %I.%I FOR SELECT TO authenticated USING (true)', table_name_str, schema_name_str, table_name_str);
+    END IF;
 
     -- The tables with custom and active are managed through views,
     -- where one _system view is used for system updates, and the
     -- _custom view is used for managing custom rows by the super_user.
     IF has_custom_and_active THEN
-        RAISE NOTICE '%s.%s: regular_user(s) can read', schema_name_str, table_name_str;
-        EXECUTE format('CREATE POLICY %s_regular_user_read ON %I.%I FOR SELECT TO authenticated USING (auth.has_statbus_role(auth.uid(), ''regular_user''::public.statbus_role_type))', table_name_str, schema_name_str, table_name_str);
+        IF NOT EXISTS (
+            SELECT 1 FROM pg_policies 
+            WHERE schemaname = schema_name_str 
+            AND tablename = table_name_str 
+            AND policyname = table_name_str || '_regular_user_read'
+        ) THEN
+            RAISE NOTICE '%s.%s: Creating regular_user read policy', schema_name_str, table_name_str;
+            EXECUTE format('CREATE POLICY %s_regular_user_read ON %I.%I FOR SELECT TO authenticated USING (auth.has_statbus_role(auth.uid(), ''regular_user''::public.statbus_role_type))', table_name_str, schema_name_str, table_name_str);
+        END IF;
     ELSE
-        RAISE NOTICE '%s.%s: regular_user(s) can manage', schema_name_str, table_name_str;
-        EXECUTE format('CREATE POLICY %s_regular_user_manage ON %I.%I FOR ALL TO authenticated USING (auth.has_statbus_role(auth.uid(), ''regular_user''::public.statbus_role_type))', table_name_str, schema_name_str, table_name_str);
+        IF NOT EXISTS (
+            SELECT 1 FROM pg_policies 
+            WHERE schemaname = schema_name_str 
+            AND tablename = table_name_str 
+            AND policyname = table_name_str || '_regular_user_manage'
+        ) THEN
+            RAISE NOTICE '%s.%s: Creating regular_user manage policy', schema_name_str, table_name_str;
+            EXECUTE format('CREATE POLICY %s_regular_user_manage ON %I.%I FOR ALL TO authenticated USING (auth.has_statbus_role(auth.uid(), ''regular_user''::public.statbus_role_type)) WITH CHECK (auth.has_statbus_role(auth.uid(), ''regular_user''::public.statbus_role_type))', table_name_str, schema_name_str, table_name_str);
+        END IF;
     END IF;
 
-    RAISE NOTICE '%s.%s: super_user(s) can manage', schema_name_str, table_name_str;
-    EXECUTE format('CREATE POLICY %s_super_user_manage ON %I.%I FOR ALL TO authenticated USING (auth.has_statbus_role(auth.uid(), ''super_user''::public.statbus_role_type))', table_name_str, schema_name_str, table_name_str);
+    IF NOT EXISTS (
+        SELECT 1 FROM pg_policies 
+        WHERE schemaname = schema_name_str 
+        AND tablename = table_name_str 
+        AND policyname = table_name_str || '_super_user_manage'
+    ) THEN
+        RAISE NOTICE '%s.%s: Creating super_user manage policy', schema_name_str, table_name_str;
+        EXECUTE format('CREATE POLICY %s_super_user_manage ON %I.%I FOR ALL TO authenticated USING (auth.has_statbus_role(auth.uid(), ''super_user''::public.statbus_role_type)) WITH CHECK (auth.has_statbus_role(auth.uid(), ''super_user''::public.statbus_role_type))', table_name_str, schema_name_str, table_name_str);
+    END IF;
 END;
 $function$
 ```

doc/db/function/admin_import_establishment_era_upsert().md

@@ -60,11 +60,14 @@ BEGIN
     SELECT NULL::int AS employees
          , NULL::int AS turnover
         INTO stats;
+    SELECT NULL::int AS id INTO inserted_establishment;
+    SELECT NULL::int AS id INTO inserted_location;
+    SELECT NULL::int AS id INTO inserted_activity;
+    SELECT NULL::int AS id INTO inserted_stat_for_unit;
 
     SELECT * INTO edited_by_user
     FROM public.statbus_user
-    -- TODO: Uncomment when going into production
-    -- WHERE uuid = auth.uid()
+    WHERE uuid = auth.uid()
     LIMIT 1;
 
     SELECT tag_id INTO tag.id FROM admin.import_lookup_tag(new_jsonb);

doc/db/function/admin_import_legal_unit_era_upsert().md

@@ -55,8 +55,7 @@ BEGIN
 
     SELECT * INTO edited_by_user
     FROM public.statbus_user
-    -- TODO: Uncomment when going into production
-    -- WHERE uuid = auth.uid()
+    WHERE uuid = auth.uid()
     LIMIT 1;
 
     SELECT tag_id INTO tag.id FROM admin.import_lookup_tag(new_jsonb);

doc/db/function/admin_insert_external_idents(jsonb, external_ident[], integer, integer, integer).md

@@ -10,87 +10,97 @@ BEGIN
   IF (p_legal_unit_id IS NOT NULL AND p_establishment_id IS NOT NULL) OR
       (p_legal_unit_id IS NULL AND p_establishment_id IS NULL) THEN
       RAISE EXCEPTION 'Must provide either a p_legal_unit_id or an p_establishment_id, but not both.';
+  ELSIF p_legal_unit_id IS NOT NULL THEN
+    unit_type := 'legal_unit';
+  ELSIF p_establishment_id IS NOT NULL THEN
+    unit_type := 'establishment';
   END IF;
 
   IF array_length(external_idents_to_add, 1) > 0 THEN
     BEGIN
-      IF p_legal_unit_id IS NOT NULL THEN
-        unit_type := 'legal_unit';
-        -- Insert for legal units if legal_unit_id is provided.
-        INSERT INTO public.external_ident
-            ( type_id
+      INSERT INTO public.external_ident
+          ( type_id
+          , ident
+          , legal_unit_id
+          , establishment_id
+          , updated_by_user_id
+          )
+      SELECT type_id
             , ident
-            , legal_unit_id
-            , updated_by_user_id
-            )
-        SELECT type_id
-              , ident
-              , p_legal_unit_id
-              , p_updated_by_user_id
-        FROM unnest(external_idents_to_add);
-      ELSIF p_establishment_id IS NOT NULL THEN
-        unit_type := 'establishment';
-        -- Insert for establishments if establishment_id is provided.
-        INSERT INTO public.external_ident
-            ( type_id
-            , ident
-            , establishment_id
-            , updated_by_user_id
-            )
-        SELECT type_id
-              , ident
-              , p_establishment_id
-              , p_updated_by_user_id
-        FROM unnest(external_idents_to_add);
-      END IF;
+            , p_legal_unit_id
+            , p_establishment_id
+            , p_updated_by_user_id
+      FROM unnest(external_idents_to_add);
     EXCEPTION WHEN unique_violation THEN
-      IF SQLERRM LIKE '%external_ident_type_for_%' THEN
-        DECLARE
-          pg_exception_detail	TEXT;
-
-          extracted_values TEXT[];
-
-          extracted_conflict_unit_type TEXT;
-          extracted_foreign_column TEXT;
-          extracted_type_id INTEGER;
-          extracted_unit_id INTEGER;
-          offending_rows JSONB;
+      DECLARE
+        identifier_problems_jsonb JSONB;
+      BEGIN
+        RAISE DEBUG 'External identifiers to add: %', to_jsonb(external_idents_to_add);
+        -- There are two scenarios to consider.
+        -- 1. Conflicting Ident: If there is an existing identifier conflict on (type,ident) for another entry
+        -- 2. Unstable Ident: If there is an existing identifier conflict on (type, legal_unit_id) or (type, establishment_id) where the ident is different.
+        -- This can happen if there are multiple unique identifiers, and one is changed or used inconcistently.
+        -- {"tax_ident": "1234", stat_ident: "2345"} - First entry
+        -- {"tax_ident": "1234", stat_ident: "3456"} - Second entry
+        -- It is not clear at this point if they are supposed to be the same entry, and the stat_ident was changed,
+        -- or if it is an error with duplicate tax_ident.
+        -- In this case there will not be a match for {stat_ident: "3456"}, the conflict is with (type_id, legal_unit_id, establishment_id).
 
-        BEGIN
-          GET STACKED DIAGNOSTICS pg_exception_detail = PG_EXCEPTION_DETAIL;
-
-          -- pg_exception_detail='Key (type_id, establishment_id)=(1, 1) already exists.'
-          extracted_values := regexp_matches(
-              pg_exception_detail,
-              'Key \(type_id, ((.*?)_id)\)=\((\d+), (\d+)\)'
-          );
-
-          IF array_length(extracted_values, 1) = 4 THEN
-            extracted_foreign_column := extracted_values[1];
-            extracted_conflict_unit_type := extracted_values[2];
-            extracted_type_id := extracted_values[3]::INT;
-            extracted_unit_id := extracted_values[4]::INT;
-
-            EXECUTE format($$
-              SELECT jsonb_object_agg(eit.code,ei.ident)
-              FROM public.external_ident AS ei
-              JOIN public.external_ident_type AS eit
-              ON ei.type_id = eit.id
-              WHERE ei.%s = %L
-            $$, extracted_foreign_column, extracted_unit_id)
-            INTO offending_rows;
+        -- Example from the raise above:
+        -- DEBUG:  External identifiers to add:
+        -- [
+        --   {
+        --     "id": null,
+        --     "ident": "82212760144",
+        --     "type_id": 1,
+        --     "enterprise_id": null,
+        --     "legal_unit_id": null,
+        --     "establishment_id": null,
+        --     "updated_by_user_id": null,
+        --     "enterprise_group_id": null
+        --   }
+        -- ]
+        --
+        WITH identifier_problems AS (
+          SELECT DISTINCT
+                 CASE
+                   WHEN ei.legal_unit_id IS NOT NULL THEN 'legal_unit'
+                   WHEN ei.establishment_id IS NOT NULL THEN 'establishment'
+                 END AS unit_type
+               , eit.code AS code
+               , ei.ident AS current_ident
+               , new_ei.ident AS new_ident
+               , CASE
+                   WHEN ei.ident = new_ei.ident THEN 'conflicting_identifier'
+                   ELSE 'unstable_identifier'
+                 END AS problem
+          FROM unnest(external_idents_to_add) AS new_ei
+          JOIN public.external_ident AS ei
+            ON (ei.type_id = new_ei.type_id AND ei.ident = new_ei.ident)  -- conflicting case
+            OR (ei.type_id = new_ei.type_id  -- unstable case
+                AND ei.ident <> new_ei.ident
+                AND ei.legal_unit_id IS NOT DISTINCT FROM p_legal_unit_id
+                AND ei.establishment_id IS NOT DISTINCT FROM p_establishment_id
+                )
+          JOIN public.external_ident_type AS eit
+            ON ei.type_id = eit.id
+        )
+        SELECT jsonb_agg(
+            jsonb_build_object(
+              'unit_type', ic.unit_type,
+              'code', ic.code,
+              'current_ident', ic.current_ident,
+              'new_ident', ic.new_ident,
+              'problem', ic.problem
+            )
+        ) INTO identifier_problems_jsonb
+        FROM identifier_problems AS ic;
 
-            RAISE EXCEPTION 'Another % % already uses the same identier(s) as the % in row %', extracted_conflict_unit_type, offending_rows, unit_type, new_jsonb
-            USING ERRCODE = 'unique_violation',
-              HINT = 'Check for other units already using the same identifier',
-              DETAIL = 'Key constraint (type_id, '||unit_type||'_id) is violated.';
-          ELSE
-              RAISE EXCEPTION 'Another unit already uses the same identier(s) as the % in row %', unit_type, new_jsonb;
-          END IF;
-        END;
-      ELSE
-        RAISE EXCEPTION 'Another unit already uses the same identier(s) as the % in row %', unit_type, new_jsonb;
-      END IF;
+        RAISE EXCEPTION 'Identifier conflicts % for row %', identifier_problems_jsonb, new_jsonb
+        USING ERRCODE = 'unique_violation',
+          HINT = 'Check for other units already using the same identifier',
+          DETAIL = 'Key constraint (type_id, '||unit_type||'_id) is violated.';
+      END;
     END;
   END IF;
 END;

doc/db/function/auth_assert_is_super_user().md

@@ -0,0 +1,17 @@
+```sql
+CREATE OR REPLACE FUNCTION auth.assert_is_super_user()
+ RETURNS void
+ LANGUAGE plpgsql
+ SECURITY DEFINER
+AS $function$
+BEGIN
+  IF auth.uid() IS NULL THEN
+    RAISE EXCEPTION 'No authenticated user found';
+  END IF;
+
+  IF NOT auth.has_statbus_role(auth.uid(), 'super_user'::statbus_role_type) THEN
+    RAISE EXCEPTION 'Only super users can update user roles';
+  END IF;
+END;
+$function$
+```

doc/db/function/auth_assert_is_super_user_or_system_account().md

@@ -0,0 +1,12 @@
+```sql
+CREATE OR REPLACE FUNCTION auth.assert_is_super_user_or_system_account()
+ RETURNS void
+ LANGUAGE plpgsql
+AS $function$
+BEGIN
+  IF NOT (auth.check_is_system_account() OR auth.check_is_super_user()) THEN
+    RAISE EXCEPTION 'Only super users or system accounts can perform this action';
+  END IF;
+END;
+$function$
+```

doc/db/function/auth_check_is_super_user().md

@@ -0,0 +1,12 @@
+```sql
+CREATE OR REPLACE FUNCTION auth.check_is_super_user()
+ RETURNS boolean
+ LANGUAGE plpgsql
+ SECURITY DEFINER
+AS $function$
+BEGIN
+  RETURN (auth.uid() IS NOT NULL)
+    AND auth.has_statbus_role(auth.uid(), 'super_user'::statbus_role_type);
+END;
+$function$
+```

doc/db/function/auth_check_is_system_account().md

@@ -0,0 +1,14 @@
+```sql
+CREATE OR REPLACE FUNCTION auth.check_is_system_account()
+ RETURNS boolean
+ LANGUAGE plpgsql
+AS $function$
+BEGIN
+  RETURN EXISTS (
+    SELECT 1 FROM pg_roles
+    WHERE rolname = current_user
+    AND rolbypassrls = true
+  );
+END;
+$function$
+```