diff --git a/airflow/migrations/versions/0152_2_10_3_fix_dag_schedule_dataset_alias_reference_naming.py b/airflow/migrations/versions/0152_2_10_3_fix_dag_schedule_dataset_alias_reference_naming.py new file mode 100644 index 000000000000..8fb02d3dcf19 --- /dev/null +++ b/airflow/migrations/versions/0152_2_10_3_fix_dag_schedule_dataset_alias_reference_naming.py @@ -0,0 +1,129 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Rename dag_schedule_dataset_alias_reference constraint names. + +Revision ID: 5f2621c13b39 +Revises: 22ed7efa9da2 +Create Date: 2024-10-25 04:03:33.002701 + +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from alembic import op +from sqlalchemy import inspect + +# revision identifiers, used by Alembic. +revision = "5f2621c13b39" +down_revision = "22ed7efa9da2" +branch_labels = None +depends_on = None +airflow_version = "2.10.3" + +if TYPE_CHECKING: + from alembic.operations.base import BatchOperations + from sqlalchemy.sql.elements import conv + + +def _rename_fk_constraint( + *, + batch_op: BatchOperations, + original_name: str | conv, + new_name: str | conv, + referent_table: str, + local_cols: list[str], + remote_cols: list[str], + ondelete: str, +) -> None: + batch_op.drop_constraint(original_name, type_="foreignkey") + batch_op.create_foreign_key( + constraint_name=new_name, + referent_table=referent_table, + local_cols=local_cols, + remote_cols=remote_cols, + ondelete=ondelete, + ) + + +def upgrade(): + """Rename dag_schedule_dataset_alias_reference constraint.""" + with op.batch_alter_table("dag_schedule_dataset_alias_reference", schema=None) as batch_op: + bind = op.get_context().bind + insp = inspect(bind) + fk_constraints = [fk["name"] for fk in insp.get_foreign_keys("dag_schedule_dataset_alias_reference")] + + # "dsdar_dataset_alias_fkey" was the constraint name defined in the model while "dsdar_dataset_fkey" is the one + # defined in the previous migration. + # Rename this constraint name if user is using the name "dsdar_dataset_fkey". + if "dsdar_dataset_fkey" in fk_constraints: + _rename_fk_constraint( + batch_op=batch_op, + original_name="dsdar_dataset_fkey", + new_name="dsdar_dataset_alias_fkey", + referent_table="dataset_alias", + local_cols=["alias_id"], + remote_cols=["id"], + ondelete="CASCADE", + ) + + # "dsdar_dag_fkey" was the constraint name defined in the model while "dsdar_dag_id_fkey" is the one + # defined in the previous migration. + # Rename this constraint name if user is using the name "dsdar_dag_fkey". + if "dsdar_dag_fkey" in fk_constraints: + _rename_fk_constraint( + batch_op=batch_op, + original_name="dsdar_dag_fkey", + new_name="dsdar_dag_id_fkey", + referent_table="dataset_alias", + local_cols=["alias_id"], + remote_cols=["id"], + ondelete="CASCADE", + ) + + +def downgrade(): + """Undo dag_schedule_dataset_alias_reference constraint rename.""" + with op.batch_alter_table("dag_schedule_dataset_alias_reference", schema=None) as batch_op: + bind = op.get_context().bind + insp = inspect(bind) + fk_constraints = [fk["name"] for fk in insp.get_foreign_keys("dag_schedule_dataset_alias_reference")] + if "dsdar_dataset_alias_fkey" in fk_constraints: + _rename_fk_constraint( + batch_op=batch_op, + original_name="dsdar_dataset_alias_fkey", + new_name="dsdar_dataset_fkey", + referent_table="dataset_alias", + local_cols=["alias_id"], + remote_cols=["id"], + ondelete="CASCADE", + ) + + if "dsdar_dag_id_fkey" in fk_constraints: + _rename_fk_constraint( + batch_op=batch_op, + original_name="dsdar_dag_id_fkey", + new_name="dsdar_dag_fkey", + referent_table="dataset_alias", + local_cols=["alias_id"], + remote_cols=["id"], + ondelete="CASCADE", + ) diff --git a/airflow/models/dataset.py b/airflow/models/dataset.py index 5033da48a305..81991498fbd7 100644 --- a/airflow/models/dataset.py +++ b/airflow/models/dataset.py @@ -224,7 +224,7 @@ class DagScheduleDatasetAliasReference(Base): ForeignKeyConstraint( columns=(dag_id,), refcolumns=["dag.dag_id"], - name="dsdar_dag_fkey", + name="dsdar_dag_id_fkey", ondelete="CASCADE", ), Index("idx_dag_schedule_dataset_alias_reference_dag_id", dag_id), diff --git a/airflow/utils/db.py b/airflow/utils/db.py index aa18204f0f6e..c92ab223b43b 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -119,6 +119,7 @@ class MappedClassProtocol(Protocol): "2.9.0": "1949afb29106", "2.9.2": "686269002441", "2.10.0": "22ed7efa9da2", + "2.10.3": "5f2621c13b39", } diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 269ee5d26910..96c9ea1c16fd 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1 @@ -85015e607fda61f3f0a00e8f3ef327c54ccc7ccfd0185d0d862b3e01556fe60c \ No newline at end of file +3009a8ae271c281d59ada514c7181f1be4f0e2c6ba37148d4d5bf57502edf1d1 \ No newline at end of file diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index 1b43a3877a6a..a2fcb3a27eb5 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -64,131 +64,131 @@ [INTEGER] - + +dag_priority_parsing_request + +dag_priority_parsing_request + +id + + [VARCHAR(32)] + NOT NULL + +fileloc + + [VARCHAR(2000)] + NOT NULL + + + job - -job - -id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - -end_date - - [TIMESTAMP] - -executor_class - - [VARCHAR(500)] - -hostname - - [VARCHAR(500)] - -job_type - - [VARCHAR(30)] - -latest_heartbeat - - [TIMESTAMP] - -start_date - - [TIMESTAMP] - -state - - [VARCHAR(20)] - -unixname - - [VARCHAR(1000)] + +job + +id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + +end_date + + [TIMESTAMP] + +executor_class + + [VARCHAR(500)] + +hostname + + [VARCHAR(500)] + +job_type + + [VARCHAR(30)] + +latest_heartbeat + + [TIMESTAMP] + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(20)] + +unixname + + [VARCHAR(1000)] - + slot_pool - -slot_pool - -id - - [INTEGER] - NOT NULL - -description - - [TEXT] - -include_deferred - - [BOOLEAN] - NOT NULL - -pool - - [VARCHAR(256)] - -slots - - [INTEGER] + +slot_pool + +id + + [INTEGER] + NOT NULL + +description + + [TEXT] + +include_deferred + + [BOOLEAN] + NOT NULL + +pool + + [VARCHAR(256)] + +slots + + [INTEGER] - -callback_request - -callback_request - -id - - [INTEGER] - NOT NULL - -callback_data - - [JSON] - NOT NULL - -callback_type - - [VARCHAR(20)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -priority_weight - - [INTEGER] - NOT NULL - -processor_subdir - - [VARCHAR(2000)] - - -dag_priority_parsing_request - -dag_priority_parsing_request - -id - - [VARCHAR(32)] - NOT NULL - -fileloc - - [VARCHAR(2000)] - NOT NULL +callback_request + +callback_request + +id + + [INTEGER] + NOT NULL + +callback_data + + [JSON] + NOT NULL + +callback_type + + [VARCHAR(20)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +priority_weight + + [INTEGER] + NOT NULL + +processor_subdir + + [VARCHAR(2000)] diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index ca69dd0d7388..3bec04a622aa 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -39,7 +39,10 @@ Here's the list of all the Database Migrations that are executed via when you ru +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=================================+===================+===================+==============================================================+ -| ``22ed7efa9da2`` (head) | ``8684e37832e6`` | ``2.10.0`` | Add dag_schedule_dataset_alias_reference table. | +| ``5f2621c13b39`` (head) | ``22ed7efa9da2`` | ``2.10.3`` | Rename dag_schedule_dataset_alias_reference constraint | +| | | | names. | ++---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ +| ``22ed7efa9da2`` | ``8684e37832e6`` | ``2.10.0`` | Add dag_schedule_dataset_alias_reference table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``8684e37832e6`` | ``41b3bc7c0272`` | ``2.10.0`` | Add dataset_alias_dataset association table. | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+