Merge pull request #3187 from centerofci/0.1.3

0.1.3 release PR

.github/workflows/run-pytest.yml

@@ -2,10 +2,14 @@ name: Run Python tests
 on: 
   push:
     paths:
-      - '**.py'
+      - '*.py'
+      - 'mathesar/**'
+      - 'db/**'
   pull_request:
     paths:
-      - '**.py'
+      - '*.py'
+      - 'mathesar/**'
+      - 'db/**'
 
 
 jobs:
@@ -28,17 +32,5 @@ jobs:
       - name: Build the stack
         run: docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build -d test-service
 
-      - name: Create coverage directory
-        run: docker exec mathesar_service_test mkdir coverage_report
-
       - name: Run tests with pytest
-        run: docker exec mathesar_service_test pytest --exitfirst --ignore=mathesar/tests/integration/ --cov-report=xml:coverage_report/coverage.xml
-
-      - name: Save the coverage report
-        uses: codecov/codecov-action@v2
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./coverage_report
-          flags: pytest-backend
-          name: codecov-mathesar
-          verbose: true
+        run: docker exec mathesar_service_test ./run_pytest.sh

.github/workflows/run-sql-tests.yml

@@ -0,0 +1,32 @@
+name: Run SQL tests
+on: 
+  push:
+    paths:
+      - '**.sql'
+  pull_request:
+    paths:
+      - '**.sql'
+
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    # We only want to run on external PRs, since internal PRs are covered by "push"
+    # This prevents this from running twice on internal PRs
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Copy env file
+        run: cp .env.example .env
+
+      # The code is checked out under uid 1001 - reset this to 1000 for the
+      # container to run tests successfully
+      - name: Fix permissions
+        run: sudo chown -R 1000:1000 .
+
+      - name: Build the test DB
+        run: docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build -d dev-db
+
+      - name: Run tests with pg_prove
+        run: docker exec mathesar_dev_db /bin/bash /sql/run_tests.sh

.gitignore

@@ -193,3 +193,6 @@ test-results/
 
 # core dumps generated sometimes when something goes wrong with Playwright
 core
+
+# non tracked settings
+config/settings/local.py

DEVELOPER_GUIDE.md

@@ -15,12 +15,11 @@ Mathesar is built using:
 
 > **Note:** If you are developing on Windows, first install [WSL](https://learn.microsoft.com/en-us/windows/wsl/install). Then do all of your development work from within the WSL shell, including running commands like `git clone` and `docker compose`.
 
-1. Ensure ensure that you have [Docker](https://docs.docker.com/get-docker/) installed.
+1. Ensure that you have [Docker](https://docs.docker.com/get-docker/) installed.
 
 1. Clone the repository and `cd` into it.
 
-1. Copy the `.env.example` file to `.env` like so:
-
+1. Copy the .env file by running the following command in the repository's root directory:
     ```
     cp .env.example .env
     ```
@@ -65,19 +64,25 @@ See our [API guide](./mathesar/api/README.md) for more information on API usage
 
 We use [pytest](https://docs.pytest.org) for our backend tests.
 
-- Run all backend tests:
+- Run all python backend tests:
 
     ```
     docker exec mathesar_service_dev pytest mathesar/ db/
     ```
 
-- Run a specific test, by name:
+- Run a specific python test, by name:
 
     ```
     docker exec mathesar_service_dev pytest -k "test_name"
     ```
+
+- See the [pytest documentation](https://docs.pytest.org/en/latest/how-to/usage.html), or run pytest with the `--help` flag to learn about more options for running tests.
+
+- Run all SQL tests:
 
-- See the [pytest documentation](https://docs.pytest.org/en/latest/how-to/usage.html), or run pytest with the `--help` flag to lear about more options for running tests.
+    ```
+    docker exec mathesar_dev_db /bin/bash sql/run_tests.sh
+    ```
 
 ## Front end development
 
@@ -165,7 +170,7 @@ Hot module replacement for front end code does not work when the project is pres
 
 If you you see the following error after attempting to start Docker, then the port used by Postgres is already in use on your host machine.
 
-> ERROR: for db Cannot start server db: driver failed programming external connectivity on endpoint mathesar_db (70c521f468cf2bd54014f089f0051ba28e2514667): Error starting userland proxy: listen tcp4 0.0.0.0:5432: bind: address already in use.
+> ERROR: for db Cannot start server db: driver failed programming external connectivity on endpoint mathesar_dev_db (70c521f468cf2bd54014f089f0051ba28e2514667): Error starting userland proxy: listen tcp4 0.0.0.0:5432: bind: address already in use.
 
 1. First stop Postgres on your host machine.
 

Dockerfile.devdb

@@ -0,0 +1,4 @@
+FROM postgres:13
+
+RUN apt update
+RUN apt install -y postgresql-13-pgtap postgresql-13-pldebugger && rm -rf /var/lib/apt/lists/*

README.md

@@ -33,6 +33,7 @@ You can use Mathesar to build **data models**, **enter data**, and even **build
 - [Self-hosting](#self-hosting)
 - [Our motivation](#our-motivation)
 - [Contributing](#contributing)
+- [Bugs and troubleshooting](#bugs-and-troubleshooting)
 - [License](#license)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
@@ -106,5 +107,8 @@ We want to make existing database functionality more accessible, for users of al
 ## Contributing
 We actively encourage contribution! Get started by reading our [Contributor Guide](./CONTRIBUTING.md).
 
+## Bugs and troubleshooting
+If you run into problems, refer to our [troubleshooting guide](./TROUBLESHOOTING.md).
+
 ## License
 Mathesar is open source under the GPLv3 license - see [LICENSE](LICENSE). It also contains derivatives of third-party open source modules licensed under the MIT license. See the list and respective licenses in [THIRDPARTY](THIRDPARTY).

TROUBLESHOOTING.md

@@ -0,0 +1,7 @@
+# Bugs and troubleshooting
+
+If you run into problems, here's what you can try:
+1. Narrow down the problem you're having to a minimal search phrase
+2. Search on [our issue tracker](https://github.com/centerofci/mathesar/issues/)
+3. If you think this is a previously unreported problem, report on [our issue tracker](https://github.com/centerofci/mathesar/issues/)
+4. In case you're having trouble formulating a report, reach out [on Matrix](https://wiki.mathesar.org/en/community/matrix)

config/settings/common_settings.py

@@ -42,6 +42,7 @@ def pipe_delim(pipe_string):
     "rest_framework",
     "django_filters",
     "django_property_filter",
+    "drf_spectacular",
     "mathesar",
 ]
 
@@ -183,7 +184,19 @@ def pipe_delim(pipe_string):
     ),
     'TEST_REQUEST_DEFAULT_FORMAT': 'json',
     'EXCEPTION_HANDLER':
-        'mathesar.exception_handlers.mathesar_exception_handler'
+        'mathesar.exception_handlers.mathesar_exception_handler',
+    'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema'
+}
+SPECTACULAR_SETTINGS = {
+    'TITLE': 'Mathesar API',
+    'DESCRIPTION': '',
+    'VERSION': '1.0.0',
+    'SERVE_INCLUDE_SCHEMA': False,
+    'PREPROCESSING_HOOKS': ['config.settings.openapi.custom_preprocessing_hook'],
+    'POSTPROCESSING_HOOKS': [
+        'config.settings.openapi.remove_url_prefix_hook',
+    ],
+    # OTHER SETTINGS
 }
 FRIENDLY_ERRORS = {
     'FIELD_ERRORS': {

config/settings/development.py

@@ -1,3 +1,10 @@
 from config.settings.common_settings import * # noqa
 
 # Override default settings
+
+
+# Use a local.py module for settings that shouldn't be version tracked
+try:
+    from .local import * # noqa
+except ImportError:
+    pass

config/settings/openapi.py

@@ -0,0 +1,21 @@
+def custom_preprocessing_hook(endpoints):
+    filtered = []
+    for (path, path_regex, method, callback) in endpoints:
+        # Remove all but DRF API endpoints
+        if path.startswith("/api/db/v0/databases/") or path.startswith("/api/db/v0/data_files/") or path.startswith("/api/db/v0/schemas/"):
+            filtered.append((path, path_regex, method, callback))
+    return filtered
+
+
+def remove_url_prefix_hook(result, **kwargs):
+    # Remove namespace and version URL prefix from the operation Id of the generated API schema
+    for path, path_info in result['paths'].items():
+        for method, operation in path_info.items():
+            operation_id = operation.get('operationId')
+            if operation_id:
+                if path.startswith('/api/db/v0/'):
+                    operation['operationId'] = operation_id.replace('db_v0_', '')
+                elif path.startswith('/api/ui/v0/'):
+                    operation['operationId'] = operation_id.replace('ui_v0_', '')
+
+    return result

config/settings/production.py

@@ -1,3 +1,10 @@
 from config.settings.common_settings import * # noqa
 
 # Override default settings
+
+
+# Use a local.py module for settings that shouldn't be version tracked
+try:
+    from .local import * # noqa 
+except ImportError:
+    pass

conftest.py

@@ -260,6 +260,9 @@ def _get_connection_string(username, password, hostname, database):
 ACADEMICS_SQL = os.path.join(RESOURCES, "academics_create.sql")
 LIBRARY_SQL = os.path.join(RESOURCES, "library_without_checkouts.sql")
 LIBRARY_CHECKOUTS_SQL = os.path.join(RESOURCES, "library_add_checkouts.sql")
+FRAUDULENT_PAYMENTS_SQL = os.path.join(RESOURCES, "fraudulent_payments.sql")
+PLAYER_PROFILES_SQL = os.path.join(RESOURCES, "player_profiles.sql")
+MARATHON_ATHLETES_SQL = os.path.join(RESOURCES, "marathon_athletes.sql")
 
 
 @pytest.fixture
@@ -331,3 +334,54 @@ def make_table(table_name):
         in table_names
     }
     return tables
+
+
+@pytest.fixture
+def engine_with_fraudulent_payment(engine_with_schema):
+    engine, schema = engine_with_schema
+    with engine.begin() as conn, open(FRAUDULENT_PAYMENTS_SQL) as f:
+        conn.execute(text(f"SET search_path={schema}"))
+        conn.execute(text(f.read()))
+    yield engine, schema
+
+
+@pytest.fixture
+def payments_db_table(engine_with_fraudulent_payment):
+    engine, schema = engine_with_fraudulent_payment
+    metadata = MetaData(bind=engine)
+    table = Table("Payments", metadata, schema=schema, autoload_with=engine)
+    return table
+
+
+@pytest.fixture
+def engine_with_player_profiles(engine_with_schema):
+    engine, schema = engine_with_schema
+    with engine.begin() as conn, open(PLAYER_PROFILES_SQL) as f:
+        conn.execute(text(f"SET search_path={schema}"))
+        conn.execute(text(f.read()))
+    yield engine, schema
+
+
+@pytest.fixture
+def players_db_table(engine_with_player_profiles):
+    engine, schema = engine_with_player_profiles
+    metadata = MetaData(bind=engine)
+    table = Table("Players", metadata, schema=schema, autoload_with=engine)
+    return table
+
+
+@pytest.fixture
+def engine_with_marathon_athletes(engine_with_schema):
+    engine, schema = engine_with_schema
+    with engine.begin() as conn, open(MARATHON_ATHLETES_SQL) as f:
+        conn.execute(text(f"SET search_path={schema}"))
+        conn.execute(text(f.read()))
+    yield engine, schema
+
+
+@pytest.fixture
+def athletes_db_table(engine_with_marathon_athletes):
+    engine, schema = engine_with_marathon_athletes
+    metadata = MetaData(bind=engine)
+    table = Table("Marathon", metadata, schema=schema, autoload_with=engine)
+    return table

db/columns/base.py

@@ -172,7 +172,7 @@ def valid_target_types(self):
                 ),
                 key=lambda db_type: db_type.id
             )
-            return valid_target_types if valid_target_types else None
+            return valid_target_types if valid_target_types else []
 
     @property
     def column_attnum(self):