fix(workflow): execute sql files with python script & move files to `…

…test` folder

.github/workflows/database_ws.yml

@@ -37,7 +37,7 @@ jobs:
     - name: Install Python dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r .github/workflows/requirements.txt
+        pip install -r test/requirements.txt
 
     # - name: Wait for PostgreSQL to be ready
     #   run: |
@@ -60,85 +60,12 @@ jobs:
         psql -h localhost -U postgres -d giswater_test_db -c 'CREATE EXTENSION postgis_raster;'
 
     - name: Replace variables in SQL files
-      run: python .github/workflows/replace_vars.py
+      run: python test/replace_vars.py
 
     - name: Create sample schema
       env:
         PGPASSWORD: postgres
-      run: |
-        # set -e  # Exit immediately if a command exits with a non-zero status
-
-        # Define the root directories to process
-        root_directories=("utils" "ws" "i18n/en_US")
-
-        # Function to execute SQL files and handle errors
-        execute_sql() {
-          local file=$1
-          echo "Executing $file..."
-            # Check if the file path contains "fct", "ftrg", or "trg"
-            if [[ "$file" != *"fct"* && "$file" != *"ftrg"* && "$file" != *"trg"* ]]; then
-              # If not, include the -v ON_ERROR_STOP=1 option
-              output=$(psql -v ON_ERROR_STOP=1 -h localhost -U postgres -d giswater_test_db -f "$file" 2>&1)
-            else
-              # Otherwise, do not include the option
-              output=$(psql -h localhost -U postgres -d giswater_test_db -f "$file" 2>&1)
-            fi
-          echo "Output: $output"
-          # Check the exit status of the psql command
-          if [ $? -ne 0 ]; then
-              # Print the captured output
-              echo "Error executing $file"
-              echo "Output:"
-              echo "$output"
-              exit 1
-          fi
-        }
-
-        # Process each root directory and its subdirectories
-        for root_dir in "${root_directories[@]}"; do
-          echo "Processing root directory: $root_dir"
-          find "$root_dir" -type f -name "*.sql" | sort | grep -v "ud_" | while read -r file; do
-            execute_sql "$file"
-          done
-        done
-
-        # Define the base updates directory
-        updates_dir="updates/36"
-
-        # Check if the updates directory exists
-        if [ -d "$updates_dir" ]; then
-          # Process "utils" and "ws" subdirectories within updates_dir
-          find "$updates_dir" -type d -name "utils" -o -name "ws" | sort | while read -r subdir; do
-            echo "Processing directory: $subdir"
-            find "$subdir" -type f -name "*.sql" | sort | grep -v "ud_" | while read -r file; do
-              execute_sql "$file"
-            done
-          done
-        else
-          echo "Directory $updates_dir does not exist"
-        fi
-
-        # child views
-        execute_sql "childviews/en_US/ws_schema_model.sql"
-
-        # execute lastprocess
-        psql -h localhost -U postgres -d giswater_test_db -c "SELECT ws_36.gw_fct_admin_schema_lastprocess('{\"client\":{\"device\":4, \"lang\":\"en_US\"}, \"data\":{\"isNewProject\":\"TRUE\", \"gwVersion\":\"3.6.012\", \"projectType\":\"WS\", \"epsg\":25831, \"descript\":\"ws_36\", \"name\":\"ws_36\", \"author\":\"postgres\", \"date\":\"29-07-2024\"}}');"
-
-        # Define the example directory
-        updates_dir="example/user/ws"
-
-        # Check if the updates directory exists
-        if [ -d "$updates_dir" ]; then
-          # Process "utils" and "ws" subdirectories within updates_dir
-          find "$updates_dir" -type d | sort | while read -r subdir; do
-            echo "Processing directory: $subdir"
-            find "$subdir" -type f -name "*.sql" | sort | grep -v "ud_" | while read -r file; do
-              execute_sql "$file"
-            done
-          done
-        else
-          echo "Directory $updates_dir does not exist"
-        fi
+      run: python test/execute_sql_files.py
 
     - name: Verify Database
       env:

test/execute_sql_files.py

@@ -0,0 +1,100 @@
+import os
+import argparse
+import psycopg2
+from psycopg2 import sql
+
+
+def execute_sql_file(conn, file_path):
+    with open(file_path, 'r') as file:
+        sql_content = file.read()
+    with conn.cursor() as cursor:
+        try:
+            cursor.execute(sql_content)
+            conn.commit()
+            print(f"Executed {file_path} successfully.")
+        except Exception as e:
+            conn.rollback()
+            print(f"Error executing {file_path}: {e}")
+            raise
+
+
+def connect_to_db():
+    # Database connection parameters
+    db_params = {
+        'dbname': 'giswater_test_db',
+        'user': 'postgres',
+        'password': os.getenv('PGPASSWORD', 'postgres'),
+        'host': 'localhost',
+        'port': 5432
+    }
+
+    # Connect to the PostgreSQL database
+    conn = psycopg2.connect(**db_params)
+    return conn
+
+
+def main(project_type):
+    print(f"Project type: {project_type}")
+
+    conn = connect_to_db()
+
+    # Define the root directories to process
+    root_directories = ["utils", f"{project_type}", "i18n/en_US"]
+    exclude_prefix = "ud_" if project_type == "ws" else "ws_"
+
+    # Execute SQL files in the root directories
+    for root_dir in root_directories:
+        print(f"Processing root directory: {root_dir}")
+        for root, _, files in os.walk(root_dir):
+            for file in sorted(files):
+                if file.endswith(".sql") and exclude_prefix not in file:
+                    file_path = os.path.join(root, file)
+                    execute_sql_file(conn, file_path)
+
+    # Define the base updates directory
+    updates_dir = "updates/36"
+
+    # Check if the updates directory exists and process it
+    if os.path.isdir(updates_dir):
+        for root, _, files in os.walk(updates_dir):
+            for file in sorted(files):
+                if file.endswith(".sql") and exclude_prefix not in file:
+                    file_path = os.path.join(root, file)
+                    execute_sql_file(conn, file_path)
+    else:
+        print(f"Directory {updates_dir} does not exist")
+
+    # Execute child views
+    execute_sql_file(conn, f"childviews/en_US/{project_type}_schema_model.sql")
+
+    # Execute last process command
+    with conn.cursor() as cursor:
+        lastprocess_command = f"""
+            SELECT {project_type}_36.gw_fct_admin_schema_lastprocess(
+                '{{"client":{{"device":4, "lang":"en_US"}}, "data":{{"isNewProject":"TRUE", "gwVersion":"3.6.012", "projectType":"{project_type.upper()}", "epsg":25831, "descript":"{project_type}_36", "name":"{project_type}_36", "author":"postgres", "date":"29-07-2024"}}}}'
+            );
+        """
+        cursor.execute(lastprocess_command)
+        conn.commit()
+
+    # Define the example directory
+    example_dir = f"example/user/{project_type}"
+
+    # Check if the example directory exists and process it
+    if os.path.isdir(example_dir):
+        for root, _, files in os.walk(example_dir):
+            for file in sorted(files):
+                if file.endswith(".sql") and exclude_prefix not in file:
+                    file_path = os.path.join(root, file)
+                    execute_sql_file(conn, file_path)
+    else:
+        print(f"Directory {example_dir} does not exist")
+
+    # Close the database connection
+    conn.close()
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Execute SQL files of a certain project type.')
+    parser.add_argument('project_type', type=str, help='Project type. Must be "ws" or "ud"')
+    args = parser.parse_args()
+    main(args.project_type)