remove redundant scheduler_lock column from dag table (#43023)

* remove scheduler_lock column from dag table * add migration script * add migration script * add migration script * add migration script * delete duplicate file * remove scheduler_lock column * remove scheduler_lock column * remove scheduler_lock column * remove scheduler_lock column * remove scheduler_lock column * resolve conflicts --------- Co-authored-by: Sneha Prabhu <[email protected]>
apache · Oct 28, 2024 · b5b00ef · b5b00ef
1 parent ce0c1c0
commit b5b00ef
Show file tree

Hide file tree

Showing 19 changed files with 551 additions and 625 deletions.
diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml
@@ -3069,14 +3069,6 @@ components:
             Time when the DAG last received a refresh signal
             (e.g. the DAG's "refresh" button was clicked in the web UI)
 
-            *New in version 2.3.0*
-        scheduler_lock:
-          type: boolean
-          readOnly: true
-          nullable: true
-          description: |
-            Whether (one of) the scheduler is scheduling this DAG at the moment
-
             *New in version 2.3.0*
         pickle_id:
           type: string

diff --git a/airflow/api_connexion/schemas/dag_schema.py b/airflow/api_connexion/schemas/dag_schema.py
@@ -56,7 +56,6 @@ class Meta:
     last_parsed_time = auto_field(dump_only=True)
     last_pickled = auto_field(dump_only=True)
     last_expired = auto_field(dump_only=True)
-    scheduler_lock = auto_field(dump_only=True)
     pickle_id = auto_field(dump_only=True)
     default_view = auto_field(dump_only=True)
     fileloc = auto_field(dump_only=True)

diff --git a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml
@@ -1681,12 +1681,6 @@ components:
             format: date-time
           - type: 'null'
           title: Last Expired
-        scheduler_lock:
-          anyOf:
-          - type: string
-            format: date-time
-          - type: 'null'
-          title: Scheduler Lock
         pickle_id:
           anyOf:
           - type: string
@@ -1851,7 +1845,6 @@ components:
       - last_parsed_time
       - last_pickled
       - last_expired
-      - scheduler_lock
       - pickle_id
       - default_view
       - fileloc
@@ -1928,12 +1921,6 @@ components:
             format: date-time
           - type: 'null'
           title: Last Expired
-        scheduler_lock:
-          anyOf:
-          - type: string
-            format: date-time
-          - type: 'null'
-          title: Scheduler Lock
         pickle_id:
           anyOf:
           - type: string
@@ -2028,7 +2015,6 @@ components:
       - last_parsed_time
       - last_pickled
       - last_expired
-      - scheduler_lock
       - pickle_id
       - default_view
       - fileloc
@@ -2238,12 +2224,6 @@ components:
             format: date-time
           - type: 'null'
           title: Last Expired
-        scheduler_lock:
-          anyOf:
-          - type: string
-            format: date-time
-          - type: 'null'
-          title: Scheduler Lock
         pickle_id:
           anyOf:
           - type: string
@@ -2343,7 +2323,6 @@ components:
       - last_parsed_time
       - last_pickled
       - last_expired
-      - scheduler_lock
       - pickle_id
       - default_view
       - fileloc

diff --git a/airflow/api_fastapi/core_api/serializers/dags.py b/airflow/api_fastapi/core_api/serializers/dags.py
@@ -45,7 +45,6 @@ class DAGResponse(BaseModel):
     last_parsed_time: datetime | None
     last_pickled: datetime | None
     last_expired: datetime | None
-    scheduler_lock: datetime | None
     pickle_id: datetime | None
     default_view: str | None
     fileloc: str

diff --git a/airflow/cli/commands/dag_command.py b/airflow/cli/commands/dag_command.py
@@ -227,7 +227,6 @@ def _get_dagbag_dag_details(dag: DAG) -> dict:
         "last_parsed_time": None,
         "last_pickled": None,
         "last_expired": None,
-        "scheduler_lock": None,
         "pickle_id": dag.pickle_id,
         "default_view": dag.default_view,
         "fileloc": dag.fileloc,

diff --git a/airflow/migrations/versions/0043_3_0_0_remove_scheduler_lock_column.py b/airflow/migrations/versions/0043_3_0_0_remove_scheduler_lock_column.py
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+remove scheduler_lock column.
+
+Revision ID: 486ac7936b78
+Revises: d59cbbef95eb
+Create Date: 2024-10-23 07:48:52.494396
+
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "486ac7936b78"
+down_revision = "d59cbbef95eb"
+branch_labels = None
+depends_on = None
+airflow_version = "3.0.0"
+
+
+def upgrade():
+    """Apply remove scheduler_lock column."""
+    with op.batch_alter_table("dag", schema=None) as batch_op:
+        batch_op.drop_column("scheduler_lock")
+
+
+def downgrade():
+    """Unapply remove scheduler_lock column."""
+    with op.batch_alter_table("dag", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("scheduler_lock", sa.BOOLEAN(), autoincrement=False, nullable=True))
diff --git a/airflow/models/dag.py b/airflow/models/dag.py
@@ -2818,8 +2818,6 @@ class DagModel(Base):
     # Time when the DAG last received a refresh signal
     # (e.g. the DAG's "refresh" button was clicked in the web UI)
     last_expired = Column(UtcDateTime)
-    # Whether (one  of) the scheduler is scheduling this DAG at the moment
-    scheduler_lock = Column(Boolean)
     # Foreign key to the latest pickle_id
     pickle_id = Column(Integer)
     # The location of the file containing the DAG object

diff --git a/airflow/serialization/pydantic/dag.py b/airflow/serialization/pydantic/dag.py
@@ -82,7 +82,6 @@ class DagModelPydantic(BaseModelPydantic):
     last_parsed_time: Optional[datetime]
     last_pickled: Optional[datetime]
     last_expired: Optional[datetime]
-    scheduler_lock: Optional[bool]
     pickle_id: Optional[int]
     fileloc: str
     processor_subdir: Optional[str]

diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow/ui/openapi-gen/requests/schemas.gen.ts
@@ -295,18 +295,6 @@ export const $DAGDetailsResponse = {
       ],
       title: "Last Expired",
     },
-    scheduler_lock: {
-      anyOf: [
-        {
-          type: "string",
-          format: "date-time",
-        },
-        {
-          type: "null",
-        },
-      ],
-      title: "Scheduler Lock",
-    },
     pickle_id: {
       anyOf: [
         {
@@ -607,7 +595,6 @@ export const $DAGDetailsResponse = {
     "last_parsed_time",
     "last_pickled",
     "last_expired",
-    "scheduler_lock",
     "pickle_id",
     "default_view",
     "fileloc",
@@ -712,18 +699,6 @@ export const $DAGResponse = {
       ],
       title: "Last Expired",
     },
-    scheduler_lock: {
-      anyOf: [
-        {
-          type: "string",
-          format: "date-time",
-        },
-        {
-          type: "null",
-        },
-      ],
-      title: "Scheduler Lock",
-    },
     pickle_id: {
       anyOf: [
         {
@@ -889,7 +864,6 @@ export const $DAGResponse = {
     "last_parsed_time",
     "last_pickled",
     "last_expired",
-    "scheduler_lock",
     "pickle_id",
     "default_view",
     "fileloc",
@@ -1196,18 +1170,6 @@ export const $DAGWithLatestDagRunsResponse = {
       ],
       title: "Last Expired",
     },
-    scheduler_lock: {
-      anyOf: [
-        {
-          type: "string",
-          format: "date-time",
-        },
-        {
-          type: "null",
-        },
-      ],
-      title: "Scheduler Lock",
-    },
     pickle_id: {
       anyOf: [
         {
@@ -1380,7 +1342,6 @@ export const $DAGWithLatestDagRunsResponse = {
     "last_parsed_time",
     "last_pickled",
     "last_expired",
-    "scheduler_lock",
     "pickle_id",
     "default_view",
     "fileloc",

diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts
@@ -69,7 +69,6 @@ export type DAGDetailsResponse = {
   last_parsed_time: string | null;
   last_pickled: string | null;
   last_expired: string | null;
-  scheduler_lock: string | null;
   pickle_id: string | null;
   default_view: string | null;
   fileloc: string;
@@ -132,7 +131,6 @@ export type DAGResponse = {
   last_parsed_time: string | null;
   last_pickled: string | null;
   last_expired: string | null;
-  scheduler_lock: string | null;
   pickle_id: string | null;
   default_view: string | null;
   fileloc: string;
@@ -225,7 +223,6 @@ export type DAGWithLatestDagRunsResponse = {
   last_parsed_time: string | null;
   last_pickled: string | null;
   last_expired: string | null;
-  scheduler_lock: string | null;
   pickle_id: string | null;
   default_view: string | null;
   fileloc: string;

diff --git a/airflow/ui/src/pages/DagsList/DagCard.test.tsx b/airflow/ui/src/pages/DagsList/DagCard.test.tsx
@@ -53,7 +53,6 @@ const mockDag = {
   next_dagrun_data_interval_start: "2024-08-22T00:00:00+00:00",
   owners: ["airflow"],
   pickle_id: null,
-  scheduler_lock: null,
   tags: [],
   timetable_description: "",
   timetable_summary: "",

diff --git a/airflow/utils/db.py b/airflow/utils/db.py
@@ -97,7 +97,7 @@ class MappedClassProtocol(Protocol):
     "2.9.2": "686269002441",
     "2.10.0": "22ed7efa9da2",
     "2.10.3": "5f2621c13b39",
-    "3.0.0": "d59cbbef95eb",
+    "3.0.0": "486ac7936b78",
 }
 
 

diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts
@@ -1058,12 +1058,6 @@ export interface components {
        * *New in version 2.3.0*
        */
       last_expired?: string | null;
-      /**
-       * @description Whether (one of) the scheduler is scheduling this DAG at the moment
-       *
-       * *New in version 2.3.0*
-       */
-      scheduler_lock?: boolean | null;
       /**
        * @description Foreign key to the latest pickle_id
        *

diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256
@@ -1 +1 @@
-621811bf2476a947cf52d8b75149e2be7e0ec9d383eda551b1c6496c44f2a22d
+247d6705552b669793ce92a4e15c9b225b1f852d5cbc066f204ec7eb21e77ca7