diff --git a/airflow/api/client/local_client.py b/airflow/api/client/local_client.py index 7d828792c426..800afa283021 100644 --- a/airflow/api/client/local_client.py +++ b/airflow/api/client/local_client.py @@ -36,14 +36,14 @@ def __init__(self, auth=None, session: httpx.Client | None = None): self._session.auth = auth def trigger_dag( - self, dag_id, run_id=None, conf=None, execution_date=None, replace_microseconds=True + self, dag_id, run_id=None, conf=None, logical_date=None, replace_microseconds=True ) -> dict | None: dag_run = trigger_dag.trigger_dag( dag_id=dag_id, triggered_by=DagRunTriggeredByType.CLI, run_id=run_id, conf=conf, - execution_date=execution_date, + logical_date=logical_date, replace_microseconds=replace_microseconds, ) if dag_run: diff --git a/airflow/api/common/mark_tasks.py b/airflow/api/common/mark_tasks.py index 957e82e7de49..a170e6901a50 100644 --- a/airflow/api/common/mark_tasks.py +++ b/airflow/api/common/mark_tasks.py @@ -59,19 +59,19 @@ def _create_dagruns( :param dag: The DAG to create runs for. :param infos: List of logical dates and data intervals to evaluate. :param state: The state to set the dag run to - :param run_type: The prefix will be used to construct dag run id: ``{run_id_prefix}__{execution_date}``. - :return: Newly created and existing dag runs for the execution dates supplied. + :param run_type: The prefix will be used to construct dag run id: ``{run_id_prefix}__{logical_date}``. + :return: Newly created and existing dag runs for the logical dates supplied. """ # Find out existing DAG runs that we don't need to create. dag_runs = { run.logical_date: run - for run in DagRun.find(dag_id=dag.dag_id, execution_date=[info.logical_date for info in infos]) + for run in DagRun.find(dag_id=dag.dag_id, logical_date=[info.logical_date for info in infos]) } for info in infos: if info.logical_date not in dag_runs: dag_runs[info.logical_date] = dag.create_dagrun( - execution_date=info.logical_date, + logical_date=info.logical_date, data_interval=info.data_interval, start_date=timezone.utcnow(), external_trigger=False, @@ -87,7 +87,7 @@ def set_state( *, tasks: Collection[Operator | tuple[Operator, int]], run_id: str | None = None, - execution_date: datetime | None = None, + logical_date: datetime | None = None, upstream: bool = False, downstream: bool = False, future: bool = False, @@ -107,11 +107,11 @@ def set_state( :param tasks: the iterable of tasks or (task, map_index) tuples from which to work. ``task.dag`` needs to be set :param run_id: the run_id of the dagrun to start looking from - :param execution_date: the execution date from which to start looking (deprecated) + :param logical_date: the logical date from which to start looking (deprecated) :param upstream: Mark all parents (upstream tasks) :param downstream: Mark all siblings (downstream tasks) of task_id :param future: Mark all future tasks on the interval of the dag up until - last execution date. + last logical date. :param past: Retroactively mark all tasks starting from start_date of the DAG :param state: State to which the tasks need to be set :param commit: Commit tasks to be altered to the database @@ -121,11 +121,11 @@ def set_state( if not tasks: return [] - if not exactly_one(execution_date, run_id): - raise ValueError("Exactly one of dag_run_id and execution_date must be set") + if not exactly_one(logical_date, run_id): + raise ValueError("Exactly one of dag_run_id and logical_date must be set") - if execution_date and not timezone.is_localized(execution_date): - raise ValueError(f"Received non-localized date {execution_date}") + if logical_date and not timezone.is_localized(logical_date): + raise ValueError(f"Received non-localized date {logical_date}") task_dags = {task[0].dag if isinstance(task, tuple) else task.dag for task in tasks} if len(task_dags) > 1: @@ -134,8 +134,8 @@ def set_state( if dag is None: raise ValueError("Received tasks with no DAG") - if execution_date: - run_id = dag.get_dagrun(execution_date=execution_date, session=session).run_id + if logical_date: + run_id = dag.get_dagrun(logical_date=logical_date, session=session).run_id if not run_id: raise ValueError("Received tasks with no run_id") @@ -200,26 +200,26 @@ def find_task_relatives(tasks, downstream, upstream): @provide_session -def get_execution_dates( - dag: DAG, execution_date: datetime, future: bool, past: bool, *, session: SASession = NEW_SESSION +def get_logical_dates( + dag: DAG, logical_date: datetime, future: bool, past: bool, *, session: SASession = NEW_SESSION ) -> list[datetime]: - """Return DAG execution dates.""" - latest_execution_date = dag.get_latest_execution_date(session=session) - if latest_execution_date is None: - raise ValueError(f"Received non-localized date {execution_date}") - execution_date = timezone.coerce_datetime(execution_date) + """Return DAG logical dates.""" + latest_logical_date = dag.get_latest_logical_date(session=session) + if latest_logical_date is None: + raise ValueError(f"Received non-localized date {logical_date}") + logical_date = timezone.coerce_datetime(logical_date) # determine date range of dag runs and tasks to consider - end_date = latest_execution_date if future else execution_date + end_date = latest_logical_date if future else logical_date if dag.start_date: start_date = dag.start_date else: - start_date = execution_date - start_date = execution_date if not past else start_date + start_date = logical_date + start_date = logical_date if not past else start_date if not dag.timetable.can_be_scheduled: # If the DAG never schedules, need to look at existing DagRun if the user wants future or # past runs. dag_runs = dag.get_dagruns_between(start_date=start_date, end_date=end_date) - dates = sorted({d.execution_date for d in dag_runs}) + dates = sorted({d.logical_date for d in dag_runs}) elif not dag.timetable.periodic: dates = [start_date] else: @@ -235,7 +235,7 @@ def get_run_ids(dag: DAG, run_id: str, future: bool, past: bool, session: SASess last_dagrun = dag.get_last_dagrun(include_externally_triggered=True, session=session) current_dagrun = dag.get_dagrun(run_id=run_id, session=session) first_dagrun = session.scalar( - select(DagRun).filter(DagRun.dag_id == dag.dag_id).order_by(DagRun.execution_date.asc()).limit(1) + select(DagRun).filter(DagRun.dag_id == dag.dag_id).order_by(DagRun.logical_date.asc()).limit(1) ) if last_dagrun is None: @@ -255,7 +255,7 @@ def get_run_ids(dag: DAG, run_id: str, future: bool, past: bool, session: SASess dates = [ info.logical_date for info in dag.iter_dagrun_infos_between(start_date, end_date, align=False) ] - run_ids = [dr.run_id for dr in DagRun.find(dag_id=dag.dag_id, execution_date=dates, session=session)] + run_ids = [dr.run_id for dr in DagRun.find(dag_id=dag.dag_id, logical_date=dates, session=session)] return run_ids @@ -279,7 +279,7 @@ def _set_dag_run_state(dag_id: str, run_id: str, state: DagRunState, session: SA def set_dag_run_state_to_success( *, dag: DAG, - execution_date: datetime | None = None, + logical_date: datetime | None = None, run_id: str | None = None, commit: bool = False, session: SASession = NEW_SESSION, @@ -287,29 +287,29 @@ def set_dag_run_state_to_success( """ Set the dag run's state to success. - Set for a specific execution date and its task instances to success. + Set for a specific logical date and its task instances to success. :param dag: the DAG of which to alter state - :param execution_date: the execution date from which to start looking(deprecated) + :param logical_date: the logical date from which to start looking(deprecated) :param run_id: the run_id to start looking from :param commit: commit DAG and tasks to be altered to the database :param session: database session :return: If commit is true, list of tasks that have been updated, otherwise list of tasks that will be updated - :raises: ValueError if dag or execution_date is invalid + :raises: ValueError if dag or logical_date is invalid """ - if not exactly_one(execution_date, run_id): + if not exactly_one(logical_date, run_id): return [] if not dag: return [] - if execution_date: - if not timezone.is_localized(execution_date): - raise ValueError(f"Received non-localized date {execution_date}") - dag_run = dag.get_dagrun(execution_date=execution_date) + if logical_date: + if not timezone.is_localized(logical_date): + raise ValueError(f"Received non-localized date {logical_date}") + dag_run = dag.get_dagrun(logical_date=logical_date) if not dag_run: - raise ValueError(f"DagRun with execution_date: {execution_date} not found") + raise ValueError(f"DagRun with logical_date: {logical_date} not found") run_id = dag_run.run_id if not run_id: raise ValueError(f"Invalid dag_run_id: {run_id}") @@ -333,7 +333,7 @@ def set_dag_run_state_to_success( def set_dag_run_state_to_failed( *, dag: DAG, - execution_date: datetime | None = None, + logical_date: datetime | None = None, run_id: str | None = None, commit: bool = False, session: SASession = NEW_SESSION, @@ -341,28 +341,28 @@ def set_dag_run_state_to_failed( """ Set the dag run's state to failed. - Set for a specific execution date and its task instances to failed. + Set for a specific logical date and its task instances to failed. :param dag: the DAG of which to alter state - :param execution_date: the execution date from which to start looking(deprecated) + :param logical_date: the logical date from which to start looking(deprecated) :param run_id: the DAG run_id to start looking from :param commit: commit DAG and tasks to be altered to the database :param session: database session :return: If commit is true, list of tasks that have been updated, otherwise list of tasks that will be updated - :raises: AssertionError if dag or execution_date is invalid + :raises: AssertionError if dag or logical_date is invalid """ - if not exactly_one(execution_date, run_id): + if not exactly_one(logical_date, run_id): return [] if not dag: return [] - if execution_date: - if not timezone.is_localized(execution_date): - raise ValueError(f"Received non-localized date {execution_date}") - dag_run = dag.get_dagrun(execution_date=execution_date) + if logical_date: + if not timezone.is_localized(logical_date): + raise ValueError(f"Received non-localized date {logical_date}") + dag_run = dag.get_dagrun(logical_date=logical_date) if not dag_run: - raise ValueError(f"DagRun with execution_date: {execution_date} not found") + raise ValueError(f"DagRun with logical_date: {logical_date} not found") run_id = dag_run.run_id if not run_id: @@ -429,16 +429,16 @@ def __set_dag_run_state_to_running_or_queued( *, new_state: DagRunState, dag: DAG, - execution_date: datetime | None = None, + logical_date: datetime | None = None, run_id: str | None = None, commit: bool = False, session: SASession, ) -> list[TaskInstance]: """ - Set the dag run for a specific execution date to running. + Set the dag run for a specific logical date to running. :param dag: the DAG of which to alter state - :param execution_date: the execution date from which to start looking + :param logical_date: the logical date from which to start looking :param run_id: the id of the DagRun :param commit: commit DAG and tasks to be altered to the database :param session: database session @@ -447,18 +447,18 @@ def __set_dag_run_state_to_running_or_queued( """ res: list[TaskInstance] = [] - if not exactly_one(execution_date, run_id): + if not exactly_one(logical_date, run_id): return res if not dag: return res - if execution_date: - if not timezone.is_localized(execution_date): - raise ValueError(f"Received non-localized date {execution_date}") - dag_run = dag.get_dagrun(execution_date=execution_date) + if logical_date: + if not timezone.is_localized(logical_date): + raise ValueError(f"Received non-localized date {logical_date}") + dag_run = dag.get_dagrun(logical_date=logical_date) if not dag_run: - raise ValueError(f"DagRun with execution_date: {execution_date} not found") + raise ValueError(f"DagRun with logical_date: {logical_date} not found") run_id = dag_run.run_id if not run_id: raise ValueError(f"DagRun with run_id: {run_id} not found") @@ -474,7 +474,7 @@ def __set_dag_run_state_to_running_or_queued( def set_dag_run_state_to_running( *, dag: DAG, - execution_date: datetime | None = None, + logical_date: datetime | None = None, run_id: str | None = None, commit: bool = False, session: SASession = NEW_SESSION, @@ -482,12 +482,12 @@ def set_dag_run_state_to_running( """ Set the dag run's state to running. - Set for a specific execution date and its task instances to running. + Set for a specific logical date and its task instances to running. """ return __set_dag_run_state_to_running_or_queued( new_state=DagRunState.RUNNING, dag=dag, - execution_date=execution_date, + logical_date=logical_date, run_id=run_id, commit=commit, session=session, @@ -498,7 +498,7 @@ def set_dag_run_state_to_running( def set_dag_run_state_to_queued( *, dag: DAG, - execution_date: datetime | None = None, + logical_date: datetime | None = None, run_id: str | None = None, commit: bool = False, session: SASession = NEW_SESSION, @@ -506,12 +506,12 @@ def set_dag_run_state_to_queued( """ Set the dag run's state to queued. - Set for a specific execution date and its task instances to queued. + Set for a specific logical date and its task instances to queued. """ return __set_dag_run_state_to_running_or_queued( new_state=DagRunState.QUEUED, dag=dag, - execution_date=execution_date, + logical_date=logical_date, run_id=run_id, commit=commit, session=session, diff --git a/airflow/api/common/trigger_dag.py b/airflow/api/common/trigger_dag.py index 44beae3f1f78..4a94f990191c 100644 --- a/airflow/api/common/trigger_dag.py +++ b/airflow/api/common/trigger_dag.py @@ -44,7 +44,7 @@ def _trigger_dag( triggered_by: DagRunTriggeredByType, run_id: str | None = None, conf: dict | str | None = None, - execution_date: datetime | None = None, + logical_date: datetime | None = None, replace_microseconds: bool = True, ) -> DagRun | None: """ @@ -53,9 +53,9 @@ def _trigger_dag( :param dag_id: DAG ID :param dag_bag: DAG Bag model :param triggered_by: the entity which triggers the dag_run - :param run_id: ID of the dag_run + :param run_id: ID of the run :param conf: configuration - :param execution_date: date of execution + :param logical_date: logical date of the run :param replace_microseconds: whether microseconds should be zeroed :return: list of triggered dags """ @@ -64,31 +64,31 @@ def _trigger_dag( if dag is None or dag_id not in dag_bag.dags: raise DagNotFound(f"Dag id {dag_id} not found") - execution_date = execution_date or timezone.utcnow() + logical_date = logical_date or timezone.utcnow() - if not timezone.is_localized(execution_date): - raise ValueError("The execution_date should be localized") + if not timezone.is_localized(logical_date): + raise ValueError("The logical date should be localized") if replace_microseconds: - execution_date = execution_date.replace(microsecond=0) + logical_date = logical_date.replace(microsecond=0) if dag.default_args and "start_date" in dag.default_args: min_dag_start_date = dag.default_args["start_date"] - if min_dag_start_date and execution_date < min_dag_start_date: + if min_dag_start_date and logical_date < min_dag_start_date: raise ValueError( - f"The execution_date [{execution_date.isoformat()}] should be >= start_date " + f"Logical date [{logical_date.isoformat()}] should be >= start_date " f"[{min_dag_start_date.isoformat()}] from DAG's default_args" ) - logical_date = timezone.coerce_datetime(execution_date) + coerced_logical_date = timezone.coerce_datetime(logical_date) - data_interval = dag.timetable.infer_manual_data_interval(run_after=logical_date) + data_interval = dag.timetable.infer_manual_data_interval(run_after=coerced_logical_date) run_id = run_id or dag.timetable.generate_run_id( - run_type=DagRunType.MANUAL, logical_date=logical_date, data_interval=data_interval + run_type=DagRunType.MANUAL, logical_date=coerced_logical_date, data_interval=data_interval ) - dag_run = DagRun.find_duplicate(dag_id=dag_id, execution_date=execution_date, run_id=run_id) + dag_run = DagRun.find_duplicate(dag_id=dag_id, run_id=run_id, logical_date=logical_date) if dag_run: - raise DagRunAlreadyExists(dag_run=dag_run, execution_date=execution_date, run_id=run_id) + raise DagRunAlreadyExists(dag_run, logical_date=logical_date, run_id=run_id) run_conf = None if conf: @@ -96,7 +96,7 @@ def _trigger_dag( dag_version = DagVersion.get_latest_version(dag.dag_id) dag_run = dag.create_dagrun( run_id=run_id, - execution_date=execution_date, + logical_date=logical_date, state=DagRunState.QUEUED, conf=run_conf, external_trigger=True, @@ -116,7 +116,7 @@ def trigger_dag( triggered_by: DagRunTriggeredByType, run_id: str | None = None, conf: dict | str | None = None, - execution_date: datetime | None = None, + logical_date: datetime | None = None, replace_microseconds: bool = True, session: Session = NEW_SESSION, ) -> DagRun | None: @@ -126,7 +126,7 @@ def trigger_dag( :param dag_id: DAG ID :param run_id: ID of the dag_run :param conf: configuration - :param execution_date: date of execution + :param logical_date: date of execution :param replace_microseconds: whether microseconds should be zeroed :param session: Unused. Only added in compatibility with database isolation mode :param triggered_by: the entity which triggers the dag_run @@ -142,7 +142,7 @@ def trigger_dag( dag_bag=dagbag, run_id=run_id, conf=conf, - execution_date=execution_date, + logical_date=logical_date, replace_microseconds=replace_microseconds, triggered_by=triggered_by, ) diff --git a/airflow/api_connexion/endpoints/dag_run_endpoint.py b/airflow/api_connexion/endpoints/dag_run_endpoint.py index 6a38eb27ff45..4ee621158b66 100644 --- a/airflow/api_connexion/endpoints/dag_run_endpoint.py +++ b/airflow/api_connexion/endpoints/dag_run_endpoint.py @@ -157,11 +157,11 @@ def _fetch_dag_runs( query = query.where(DagRun.start_date >= start_date_gte) if start_date_lte: query = query.where(DagRun.start_date <= start_date_lte) - # filter execution date + # filter logical date if execution_date_gte: - query = query.where(DagRun.execution_date >= execution_date_gte) + query = query.where(DagRun.logical_date >= execution_date_gte) if execution_date_lte: - query = query.where(DagRun.execution_date <= execution_date_lte) + query = query.where(DagRun.logical_date <= execution_date_lte) # filter end date if end_date_gte: query = query.where(DagRun.end_date >= end_date_gte) @@ -174,12 +174,12 @@ def _fetch_dag_runs( query = query.where(DagRun.updated_at <= updated_at_lte) total_entries = get_query_count(query, session=session) - to_replace = {"dag_run_id": "run_id", "execution_date": "logical_date"} + to_replace = {"dag_run_id": "run_id", "logical_date": "logical_date"} allowed_sort_attrs = [ "id", "state", "dag_id", - "execution_date", + "logical_date", "dag_run_id", "start_date", "end_date", @@ -319,13 +319,13 @@ def post_dag_run(*, dag_id: str, session: Session = NEW_SESSION) -> APIResponse: except ValidationError as err: raise BadRequest(detail=str(err)) - logical_date = pendulum.instance(post_body["execution_date"]) + logical_date = pendulum.instance(post_body["logical_date"]) run_id = post_body["run_id"] dagrun_instance = session.scalar( select(DagRun) .where( DagRun.dag_id == dag_id, - or_(DagRun.run_id == run_id, DagRun.execution_date == logical_date), + or_(DagRun.run_id == run_id, DagRun.logical_date == logical_date), ) .limit(1) ) @@ -346,7 +346,7 @@ def post_dag_run(*, dag_id: str, session: Session = NEW_SESSION) -> APIResponse: dag_run = dag.create_dagrun( run_type=DagRunType.MANUAL, run_id=run_id, - execution_date=logical_date, + logical_date=logical_date, data_interval=data_interval, state=DagRunState.QUEUED, conf=post_body.get("conf"), @@ -363,7 +363,7 @@ def post_dag_run(*, dag_id: str, session: Session = NEW_SESSION) -> APIResponse: except (ValueError, ParamValidationError) as ve: raise BadRequest(detail=str(ve)) - if dagrun_instance.execution_date == logical_date: + if dagrun_instance.logical_date == logical_date: raise AlreadyExists( detail=( f"DAGRun with DAG ID: '{dag_id}' and " diff --git a/airflow/api_connexion/endpoints/task_instance_endpoint.py b/airflow/api_connexion/endpoints/task_instance_endpoint.py index 9f9461910082..68267317af3b 100644 --- a/airflow/api_connexion/endpoints/task_instance_endpoint.py +++ b/airflow/api_connexion/endpoints/task_instance_endpoint.py @@ -198,7 +198,7 @@ def get_mapped_task_instances( # Other search criteria base_query = _apply_range_filter( base_query, - key=DR.execution_date, + key=DR.logical_date, value_range=(execution_date_gte, execution_date_lte), ) base_query = _apply_range_filter( @@ -336,7 +336,7 @@ def get_task_instances( base_query = base_query.where(TI.run_id == dag_run_id) base_query = _apply_range_filter( base_query, - key=DR.execution_date, + key=DR.logical_date, value_range=(execution_date_gte, execution_date_lte), ) base_query = _apply_range_filter( @@ -398,7 +398,7 @@ def get_task_instances_batch(session: Session = NEW_SESSION) -> APIResponse: base_query = _apply_array_filter(base_query, key=TI.task_id, values=data["task_ids"]) base_query = _apply_range_filter( base_query, - key=DR.execution_date, + key=DR.logical_date, value_range=(data["execution_date_gte"], data["execution_date_lte"]), ) base_query = _apply_range_filter( @@ -522,22 +522,18 @@ def post_set_task_instances_state(*, dag_id: str, session: Session = NEW_SESSION error_message = f"Task ID {task_id} not found" raise NotFound(error_message) - execution_date = data.get("execution_date") + logical_date = data.get("logical_date") run_id = data.get("dag_run_id") if ( - execution_date + logical_date and ( session.scalars( - select(TI).where( - TI.task_id == task_id, TI.dag_id == dag_id, TI.execution_date == execution_date - ) + select(TI).where(TI.task_id == task_id, TI.dag_id == dag_id, TI.logical_date == logical_date) ).one_or_none() ) is None ): - raise NotFound( - detail=f"Task instance not found for task {task_id!r} on execution_date {execution_date}" - ) + raise NotFound(detail=f"Task instance not found for task {task_id!r} on logical_date {logical_date}") select_stmt = select(TI).where( TI.dag_id == dag_id, TI.task_id == task_id, TI.run_id == run_id, TI.map_index == -1 @@ -550,7 +546,7 @@ def post_set_task_instances_state(*, dag_id: str, session: Session = NEW_SESSION tis = dag.set_task_instance_state( task_id=task_id, run_id=run_id, - execution_date=execution_date, + logical_date=logical_date, state=data["new_state"], upstream=data["include_upstream"], downstream=data["include_downstream"], diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml index af6a3055670d..10cee57d95fc 100644 --- a/airflow/api_connexion/openapi/v1.yaml +++ b/airflow/api_connexion/openapi/v1.yaml @@ -3080,7 +3080,7 @@ components: The value of this field can be set only when creating the object. If you try to modify the field of an existing object, the request fails with an BAD_REQUEST error. - If not provided, a value will be generated based on execution_date. + If not provided, a value will be generated based on logical_date. If the specified dag_run_id is in use, the creation request fails with an ALREADY_EXISTS error. @@ -3102,19 +3102,6 @@ components: *New in version 2.2.0* format: date-time - execution_date: - type: string - nullable: true - description: | - The execution date. This is the same as logical_date, kept for backwards compatibility. - If both this field and logical_date are provided but with different values, the request - will fail with an BAD_REQUEST error. - - *Changed in version 2.2.0*: Field becomes nullable. - - *Deprecated since version 2.2.0*: Use 'logical_date' instead. - format: date-time - deprecated: true start_date: type: string format: date-time @@ -3677,7 +3664,7 @@ components: The DagRun ID for this task instance *New in version 2.3.0* - execution_date: + logical_date: type: string format: datetime start_date: @@ -3788,7 +3775,7 @@ components: type: string readOnly: true description: The DAG ID. - execution_date: + logical_date: type: string format: datetime readOnly: true @@ -3860,7 +3847,7 @@ components: timestamp: type: string format: datetime - execution_date: + logical_date: type: string format: datetime map_index: @@ -4734,14 +4721,14 @@ components: description: The task ID. type: string - execution_date: - description: The execution date. Either set this or dag_run_id but not both. + logical_date: + description: The logical date. Either set this or dag_run_id but not both. type: string format: datetime dag_run_id: description: | - The task instance's DAG run ID. Either set this or execution_date but not both. + The task instance's DAG run ID. Either set this or logical_date but not both. *New in version 2.3.0* type: string diff --git a/airflow/api_connexion/schemas/asset_schema.py b/airflow/api_connexion/schemas/asset_schema.py index 7f84b799d1a7..e83c4f1b4279 100644 --- a/airflow/api_connexion/schemas/asset_schema.py +++ b/airflow/api_connexion/schemas/asset_schema.py @@ -119,7 +119,7 @@ class Meta: run_id = auto_field(data_key="dag_run_id") dag_id = auto_field(dump_only=True) - execution_date = auto_field(data_key="logical_date", dump_only=True) + logical_date = auto_field(data_key="logical_date", dump_only=True) start_date = auto_field(dump_only=True) end_date = auto_field(dump_only=True) state = auto_field(dump_only=True) diff --git a/airflow/api_connexion/schemas/dag_run_schema.py b/airflow/api_connexion/schemas/dag_run_schema.py index e829dd956a4c..c2560613def7 100644 --- a/airflow/api_connexion/schemas/dag_run_schema.py +++ b/airflow/api_connexion/schemas/dag_run_schema.py @@ -63,7 +63,7 @@ class Meta: run_id = auto_field(data_key="dag_run_id") dag_id = auto_field(dump_only=True) - execution_date = auto_field(data_key="logical_date", validate=validate_istimezone) + logical_date = auto_field(data_key="logical_date", validate=validate_istimezone) start_date = auto_field(dump_only=True) end_date = auto_field(dump_only=True) state = DagStateField(dump_only=True) @@ -78,25 +78,12 @@ class Meta: @pre_load def autogenerate(self, data, **kwargs): - """ - Auto generate run_id and logical_date if they are not provided. - - For compatibility, if `execution_date` is submitted, it is converted - to `logical_date`. - """ + """Auto generate run_id and logical_date if they are not provided.""" logical_date = data.get("logical_date", _MISSING) - execution_date = data.pop("execution_date", _MISSING) - if logical_date is execution_date is _MISSING: # Both missing. + + # Auto-generate logical_date if missing + if logical_date is _MISSING: data["logical_date"] = str(timezone.utcnow()) - elif logical_date is _MISSING: # Only logical_date missing. - data["logical_date"] = execution_date - elif execution_date is _MISSING: # Only execution_date missing. - pass - elif logical_date != execution_date: # Both provided but don't match. - raise BadRequest( - "logical_date conflicts with execution_date", - detail=f"{logical_date!r} != {execution_date!r}", - ) if "dag_run_id" not in data: try: @@ -109,9 +96,8 @@ def autogenerate(self, data, **kwargs): @post_dump def autofill(self, data, **kwargs): - """Populate execution_date from logical_date for compatibility.""" + """Ensure that only requested fields are returned if 'fields' context is set.""" ret_data = {} - data["execution_date"] = data["logical_date"] if self.context.get("fields"): ret_fields = self.context.get("fields") for ret_field in ret_fields: diff --git a/airflow/api_connexion/schemas/event_log_schema.py b/airflow/api_connexion/schemas/event_log_schema.py index bf88d1a9ce88..33ecf4ca947a 100644 --- a/airflow/api_connexion/schemas/event_log_schema.py +++ b/airflow/api_connexion/schemas/event_log_schema.py @@ -40,7 +40,7 @@ class Meta: map_index = auto_field(dump_only=True) try_number = auto_field(dump_only=True) event = auto_field(dump_only=True) - execution_date = auto_field(dump_only=True) + logical_date = auto_field(dump_only=True) owner = auto_field(dump_only=True) extra = auto_field(dump_only=True) diff --git a/airflow/api_connexion/schemas/task_instance_schema.py b/airflow/api_connexion/schemas/task_instance_schema.py index f0b8285fdfa0..3e864f18652c 100644 --- a/airflow/api_connexion/schemas/task_instance_schema.py +++ b/airflow/api_connexion/schemas/task_instance_schema.py @@ -45,7 +45,7 @@ class Meta: dag_id = auto_field() run_id = auto_field(data_key="dag_run_id") map_index = auto_field() - execution_date = auto_field() + logical_date = auto_field() start_date = auto_field() end_date = auto_field() duration = auto_field() @@ -196,7 +196,7 @@ class SetTaskInstanceStateFormSchema(Schema): dry_run = fields.Boolean(load_default=True) task_id = fields.Str(required=True) - execution_date = fields.DateTime(validate=validate_istimezone) + logical_date = fields.DateTime(validate=validate_istimezone) dag_run_id = fields.Str() include_upstream = fields.Boolean(required=True) include_downstream = fields.Boolean(required=True) @@ -212,8 +212,8 @@ class SetTaskInstanceStateFormSchema(Schema): @validates_schema def validate_form(self, data, **kwargs): """Validate set task instance state form.""" - if not exactly_one(data.get("execution_date"), data.get("dag_run_id")): - raise ValidationError("Exactly one of execution_date or dag_run_id must be provided") + if not exactly_one(data.get("logical_date"), data.get("dag_run_id")): + raise ValidationError("Exactly one of logical_date or dag_run_id must be provided") class SetSingleTaskInstanceStateFormSchema(Schema): @@ -234,7 +234,7 @@ class TaskInstanceReferenceSchema(Schema): task_id = fields.Str() run_id = fields.Str(data_key="dag_run_id") dag_id = fields.Str() - execution_date = fields.DateTime() + logical_date = fields.DateTime() class TaskInstanceReferenceCollection(NamedTuple): diff --git a/airflow/api_connexion/schemas/xcom_schema.py b/airflow/api_connexion/schemas/xcom_schema.py index 625f05bd1459..a56adf7551d7 100644 --- a/airflow/api_connexion/schemas/xcom_schema.py +++ b/airflow/api_connexion/schemas/xcom_schema.py @@ -34,7 +34,7 @@ class Meta: key = auto_field() timestamp = auto_field() - execution_date = auto_field() + logical_date = auto_field() map_index = auto_field() task_id = auto_field() dag_id = auto_field() diff --git a/airflow/api_fastapi/core_api/datamodels/event_logs.py b/airflow/api_fastapi/core_api/datamodels/event_logs.py index f70e5bd15834..5b65ec85ba7b 100644 --- a/airflow/api_fastapi/core_api/datamodels/event_logs.py +++ b/airflow/api_fastapi/core_api/datamodels/event_logs.py @@ -33,7 +33,7 @@ class EventLogResponse(BaseModel): map_index: int | None try_number: int | None event: str - execution_date: datetime | None = Field(alias="logical_date") + logical_date: datetime | None owner: str | None extra: str | None diff --git a/airflow/api_fastapi/core_api/datamodels/task_instances.py b/airflow/api_fastapi/core_api/datamodels/task_instances.py index 00910bce1c8f..a54b85e58ddf 100644 --- a/airflow/api_fastapi/core_api/datamodels/task_instances.py +++ b/airflow/api_fastapi/core_api/datamodels/task_instances.py @@ -36,7 +36,7 @@ class TaskInstanceResponse(BaseModel): dag_id: str run_id: str = Field(alias="dag_run_id") map_index: int - execution_date: datetime = Field(alias="logical_date") + logical_date: datetime start_date: datetime | None end_date: datetime | None duration: float | None diff --git a/airflow/api_fastapi/core_api/datamodels/xcom.py b/airflow/api_fastapi/core_api/datamodels/xcom.py index dadbc51290e0..186b5aad77f0 100644 --- a/airflow/api_fastapi/core_api/datamodels/xcom.py +++ b/airflow/api_fastapi/core_api/datamodels/xcom.py @@ -27,7 +27,7 @@ class XComResponse(BaseModel): key: str timestamp: datetime - execution_date: datetime + logical_date: datetime map_index: int task_id: str dag_id: str diff --git a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml index 890fb6b6c8d0..741b6d360532 100644 --- a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml @@ -6119,10 +6119,10 @@ components: type: string format: date-time title: Timestamp - execution_date: + logical_date: type: string format: date-time - title: Execution Date + title: Logical Date map_index: type: integer title: Map Index @@ -6138,7 +6138,7 @@ components: required: - key - timestamp - - execution_date + - logical_date - map_index - task_id - dag_id @@ -6154,10 +6154,10 @@ components: type: string format: date-time title: Timestamp - execution_date: + logical_date: type: string format: date-time - title: Execution Date + title: Logical Date map_index: type: integer title: Map Index @@ -6176,7 +6176,7 @@ components: required: - key - timestamp - - execution_date + - logical_date - map_index - task_id - dag_id diff --git a/airflow/api_fastapi/core_api/routes/public/event_logs.py b/airflow/api_fastapi/core_api/routes/public/event_logs.py index 4047bc5a2749..6405c177a390 100644 --- a/airflow/api_fastapi/core_api/routes/public/event_logs.py +++ b/airflow/api_fastapi/core_api/routes/public/event_logs.py @@ -79,7 +79,7 @@ def get_event_logs( "task_id", "run_id", "event", - "execution_date", # logical_date + "logical_date", "owner", "extra", ], diff --git a/airflow/api_fastapi/core_api/routes/public/task_instances.py b/airflow/api_fastapi/core_api/routes/public/task_instances.py index 56fa57d3ebeb..516e113fc24f 100644 --- a/airflow/api_fastapi/core_api/routes/public/task_instances.py +++ b/airflow/api_fastapi/core_api/routes/public/task_instances.py @@ -95,9 +95,7 @@ def get_mapped_task_instances( dag_run_id: str, task_id: str, request: Request, - logical_date_range: Annotated[ - RangeFilter, Depends(datetime_range_filter_factory("logical_date", TI, "execution_date")) - ], + logical_date_range: Annotated[RangeFilter, Depends(datetime_range_filter_factory("logical_date", TI))], start_date_range: Annotated[RangeFilter, Depends(datetime_range_filter_factory("start_date", TI))], end_date_range: Annotated[RangeFilter, Depends(datetime_range_filter_factory("end_date", TI))], update_at_range: Annotated[RangeFilter, Depends(datetime_range_filter_factory("updated_at", TI))], @@ -270,9 +268,7 @@ def get_task_instances( dag_id: str, dag_run_id: str, request: Request, - logical_date: Annotated[ - RangeFilter, Depends(datetime_range_filter_factory("logical_date", TI, "execution_date")) - ], + logical_date: Annotated[RangeFilter, Depends(datetime_range_filter_factory("logical_date", TI))], start_date_range: Annotated[RangeFilter, Depends(datetime_range_filter_factory("start_date", TI))], end_date_range: Annotated[RangeFilter, Depends(datetime_range_filter_factory("end_date", TI))], update_at_range: Annotated[RangeFilter, Depends(datetime_range_filter_factory("updated_at", TI))], diff --git a/airflow/api_fastapi/core_api/routes/ui/assets.py b/airflow/api_fastapi/core_api/routes/ui/assets.py index 1615f97c2bd2..ca2aae1bb6c0 100644 --- a/airflow/api_fastapi/core_api/routes/ui/assets.py +++ b/airflow/api_fastapi/core_api/routes/ui/assets.py @@ -71,8 +71,8 @@ def next_run_assets( and_( AssetEvent.asset_id == AssetModel.id, ( - AssetEvent.timestamp >= latest_run.execution_date - if latest_run and latest_run.execution_date + AssetEvent.timestamp >= latest_run.logical_date + if latest_run and latest_run.logical_date else True ), ), diff --git a/airflow/api_fastapi/core_api/routes/ui/dags.py b/airflow/api_fastapi/core_api/routes/ui/dags.py index da906bbbd3c9..fad736ced379 100644 --- a/airflow/api_fastapi/core_api/routes/ui/dags.py +++ b/airflow/api_fastapi/core_api/routes/ui/dags.py @@ -68,39 +68,39 @@ def recent_dag_runs( recent_runs_subquery = ( select( DagRun.dag_id, - DagRun.execution_date, + DagRun.logical_date, func.rank() .over( partition_by=DagRun.dag_id, - order_by=DagRun.execution_date.desc(), + order_by=DagRun.logical_date.desc(), ) .label("rank"), ) - .order_by(DagRun.execution_date.desc()) + .order_by(DagRun.logical_date.desc()) .subquery() ) dags_with_recent_dag_runs_select = ( select( DagRun, DagModel, - recent_runs_subquery.c.execution_date, + recent_runs_subquery.c.logical_date, ) .join(DagModel, DagModel.dag_id == recent_runs_subquery.c.dag_id) .join( DagRun, and_( DagRun.dag_id == DagModel.dag_id, - DagRun.execution_date == recent_runs_subquery.c.execution_date, + DagRun.logical_date == recent_runs_subquery.c.logical_date, ), ) .where(recent_runs_subquery.c.rank <= dag_runs_limit) .group_by( DagModel.dag_id, - recent_runs_subquery.c.execution_date, - DagRun.execution_date, + recent_runs_subquery.c.logical_date, + DagRun.logical_date, DagRun.id, ) - .order_by(recent_runs_subquery.c.execution_date.desc()) + .order_by(recent_runs_subquery.c.logical_date.desc()) ) dags_with_recent_dag_runs_select_filter, _ = paginated_select( dags_with_recent_dag_runs_select, diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py index e93d5e25c631..00dd68041cbd 100644 --- a/airflow/cli/cli_config.py +++ b/airflow/cli/cli_config.py @@ -144,17 +144,17 @@ def string_lower_type(val): # Shared ARG_DAG_ID = Arg(("dag_id",), help="The id of the dag") ARG_TASK_ID = Arg(("task_id",), help="The id of the task") -ARG_EXECUTION_DATE = Arg(("execution_date",), help="The execution date of the DAG", type=parsedate) -ARG_EXECUTION_DATE_OPTIONAL = Arg( - ("execution_date",), nargs="?", help="The execution date of the DAG (optional)", type=parsedate +ARG_LOGICAL_DATE = Arg(("logical_date",), help="The logical date of the DAG", type=parsedate) +ARG_LOGICAL_DATE_OPTIONAL = Arg( + ("logical_date",), nargs="?", help="The logical date of the DAG (optional)", type=parsedate ) -ARG_EXECUTION_DATE_OR_RUN_ID = Arg( - ("execution_date_or_run_id",), help="The execution_date of the DAG or run_id of the DAGRun" +ARG_LOGICAL_DATE_OR_RUN_ID = Arg( + ("logical_date_or_run_id",), help="The logical date of the DAG or run_id of the DAGRun" ) -ARG_EXECUTION_DATE_OR_RUN_ID_OPTIONAL = Arg( - ("execution_date_or_run_id",), +ARG_LOGICAL_DATE_OR_RUN_ID_OPTIONAL = Arg( + ("logical_date_or_run_id",), nargs="?", - help="The execution_date of the DAG or run_id of the DAGRun (optional)", + help="The logical date of the DAG or run_id of the DAGRun (optional)", ) ARG_TASK_REGEX = Arg(("-t", "--task-regex"), help="The regex to filter specific task_ids (optional)") ARG_SUBDIR = Arg( @@ -261,7 +261,7 @@ def string_lower_type(val): ("-n", "--num-executions"), default=1, type=positive_int(allow_zero=False), - help="The number of next execution datetimes to show", + help="The number of next logical date times to show", ) # misc @@ -410,7 +410,7 @@ def string_lower_type(val): # trigger_dag ARG_RUN_ID = Arg(("-r", "--run-id"), help="Helps to identify this run") ARG_CONF = Arg(("-c", "--conf"), help="JSON string that gets pickled into the DagRun's conf attribute") -ARG_EXEC_DATE = Arg(("-e", "--exec-date"), help="The execution date of the DAG", type=parsedate) +ARG_EXEC_DATE = Arg(("-e", "--exec-date"), help="The logical date of the DAG", type=parsedate) ARG_REPLACE_MICRO = Arg( ("--no-replace-microseconds",), help="whether microseconds should be zeroed", @@ -1089,13 +1089,13 @@ class GroupCommand(NamedTuple): name="state", help="Get the status of a dag run", func=lazy_load_command("airflow.cli.commands.dag_command.dag_state"), - args=(ARG_DAG_ID, ARG_EXECUTION_DATE, ARG_SUBDIR, ARG_VERBOSE), + args=(ARG_DAG_ID, ARG_LOGICAL_DATE, ARG_SUBDIR, ARG_VERBOSE), ), ActionCommand( name="next-execution", - help="Get the next execution datetimes of a DAG", + help="Get the next logical datetimes of a DAG", description=( - "Get the next execution datetimes of a DAG. It returns one execution unless the " + "Get the next logical datetimes of a DAG. It returns one execution unless the " "num-executions option is given" ), func=lazy_load_command("airflow.cli.commands.dag_command.dag_next_execution"), @@ -1208,7 +1208,7 @@ class GroupCommand(NamedTuple): name="test", help="Execute one single DagRun", description=( - "Execute one single DagRun for a given DAG and execution date.\n" + "Execute one single DagRun for a given DAG and logical date.\n" "\n" "The --imgcat-dagrun option only works in iTerm.\n" "\n" @@ -1221,15 +1221,15 @@ class GroupCommand(NamedTuple): "see: https://www.graphviz.org/doc/info/output.html\n" "\n" "If you want to create a PNG file then you should execute the following command:\n" - "airflow dags test --save-dagrun output.png\n" + "airflow dags test --save-dagrun output.png\n" "\n" "If you want to create a DOT file then you should execute the following command:\n" - "airflow dags test --save-dagrun output.dot\n" + "airflow dags test --save-dagrun output.dot\n" ), func=lazy_load_command("airflow.cli.commands.dag_command.dag_test"), args=( ARG_DAG_ID, - ARG_EXECUTION_DATE_OPTIONAL, + ARG_LOGICAL_DATE_OPTIONAL, ARG_CONF, ARG_SUBDIR, ARG_SHOW_DAGRUN, @@ -1289,7 +1289,7 @@ class GroupCommand(NamedTuple): args=( ARG_DAG_ID, ARG_TASK_ID, - ARG_EXECUTION_DATE_OR_RUN_ID, + ARG_LOGICAL_DATE_OR_RUN_ID, ARG_SUBDIR, ARG_VERBOSE, ARG_MAP_INDEX, @@ -1304,7 +1304,7 @@ class GroupCommand(NamedTuple): "and then run by an executor." ), func=lazy_load_command("airflow.cli.commands.task_command.task_failed_deps"), - args=(ARG_DAG_ID, ARG_TASK_ID, ARG_EXECUTION_DATE_OR_RUN_ID, ARG_SUBDIR, ARG_MAP_INDEX, ARG_VERBOSE), + args=(ARG_DAG_ID, ARG_TASK_ID, ARG_LOGICAL_DATE_OR_RUN_ID, ARG_SUBDIR, ARG_MAP_INDEX, ARG_VERBOSE), ), ActionCommand( name="render", @@ -1313,7 +1313,7 @@ class GroupCommand(NamedTuple): args=( ARG_DAG_ID, ARG_TASK_ID, - ARG_EXECUTION_DATE_OR_RUN_ID, + ARG_LOGICAL_DATE_OR_RUN_ID, ARG_SUBDIR, ARG_VERBOSE, ARG_MAP_INDEX, @@ -1326,7 +1326,7 @@ class GroupCommand(NamedTuple): args=( ARG_DAG_ID, ARG_TASK_ID, - ARG_EXECUTION_DATE_OR_RUN_ID, + ARG_LOGICAL_DATE_OR_RUN_ID, ARG_SUBDIR, ARG_MARK_SUCCESS, ARG_FORCE, @@ -1355,7 +1355,7 @@ class GroupCommand(NamedTuple): args=( ARG_DAG_ID, ARG_TASK_ID, - ARG_EXECUTION_DATE_OR_RUN_ID_OPTIONAL, + ARG_LOGICAL_DATE_OR_RUN_ID_OPTIONAL, ARG_SUBDIR, ARG_DRY_RUN, ARG_TASK_PARAMS, @@ -1369,7 +1369,7 @@ class GroupCommand(NamedTuple): name="states-for-dag-run", help="Get the status of all task instances in a dag run", func=lazy_load_command("airflow.cli.commands.task_command.task_states_for_dag_run"), - args=(ARG_DAG_ID, ARG_EXECUTION_DATE_OR_RUN_ID, ARG_OUTPUT, ARG_VERBOSE), + args=(ARG_DAG_ID, ARG_LOGICAL_DATE_OR_RUN_ID, ARG_OUTPUT, ARG_VERBOSE), ), ) POOLS_COMMANDS = ( @@ -1817,7 +1817,7 @@ class GroupCommand(NamedTuple): help="Generate YAML files for all tasks in DAG. Useful for debugging tasks without " "launching into a cluster", func=lazy_load_command("airflow.providers.cncf.kubernetes.cli.kubernetes_command.generate_pod_yaml"), - args=(ARG_DAG_ID, ARG_EXECUTION_DATE, ARG_SUBDIR, ARG_OUTPUT_PATH, ARG_VERBOSE), + args=(ARG_DAG_ID, ARG_LOGICAL_DATE, ARG_SUBDIR, ARG_OUTPUT_PATH, ARG_VERBOSE), ), ) diff --git a/airflow/cli/commands/dag_command.py b/airflow/cli/commands/dag_command.py index dfff75ee2d6c..0b4d3192d6ed 100644 --- a/airflow/cli/commands/dag_command.py +++ b/airflow/cli/commands/dag_command.py @@ -65,7 +65,7 @@ def dag_trigger(args) -> None: dag_id=args.dag_id, run_id=args.run_id, conf=args.conf, - execution_date=args.exec_date, + logical_date=args.exec_date, replace_microseconds=args.replace_microseconds, ) AirflowConsole().print_as( @@ -264,7 +264,7 @@ def dag_state(args, session: Session = NEW_SESSION) -> None: if not dag: raise SystemExit(f"DAG: {args.dag_id} does not exist in 'dag' table") - dr = session.scalar(select(DagRun).filter_by(dag_id=args.dag_id, execution_date=args.execution_date)) + dr = session.scalar(select(DagRun).filter_by(dag_id=args.dag_id, logical_date=args.logical_date)) out = dr.state if dr else None conf_out = "" if out and dr.conf: @@ -276,7 +276,7 @@ def dag_state(args, session: Session = NEW_SESSION) -> None: @providers_configuration_loaded def dag_next_execution(args) -> None: """ - Return the next execution datetime of a DAG at the command line. + Return the next logical datetime of a DAG at the command line. >>> airflow dags next-execution tutorial 2018-08-31 10:38:00 @@ -461,12 +461,12 @@ def dag_list_dag_runs(args, dag: DAG | None = None, session: Session = NEW_SESSI dag_id=args.dag_id, state=state, no_backfills=args.no_backfill, - execution_start_date=args.start_date, - execution_end_date=args.end_date, + logical_start_date=args.start_date, + logical_end_date=args.end_date, session=session, ) - dag_runs.sort(key=lambda x: x.execution_date, reverse=True) + dag_runs.sort(key=lambda x: x.logical_date, reverse=True) AirflowConsole().print_as( data=dag_runs, output=args.output, @@ -474,7 +474,7 @@ def dag_list_dag_runs(args, dag: DAG | None = None, session: Session = NEW_SESSI "dag_id": dr.dag_id, "run_id": dr.run_id, "state": dr.state, - "execution_date": dr.execution_date.isoformat(), + "logical_date": dr.logical_date.isoformat(), "start_date": dr.start_date.isoformat() if dr.start_date else "", "end_date": dr.end_date.isoformat() if dr.end_date else "", }, @@ -485,14 +485,14 @@ def dag_list_dag_runs(args, dag: DAG | None = None, session: Session = NEW_SESSI @providers_configuration_loaded @provide_session def dag_test(args, dag: DAG | None = None, session: Session = NEW_SESSION) -> None: - """Execute one single DagRun for a given DAG and execution date.""" + """Execute one single DagRun for a given DAG and logical date.""" run_conf = None if args.conf: try: run_conf = json.loads(args.conf) except ValueError as e: raise SystemExit(f"Configuration {args.conf!r} is not valid JSON. Error: {e}") - execution_date = args.execution_date or timezone.utcnow() + logical_date = args.logical_date or timezone.utcnow() use_executor = args.use_executor mark_success_pattern = ( @@ -502,7 +502,7 @@ def dag_test(args, dag: DAG | None = None, session: Session = NEW_SESSION) -> No with _airflow_parsing_context_manager(dag_id=args.dag_id): dag = dag or get_dag(subdir=args.subdir, dag_id=args.dag_id) dr: DagRun = dag.test( - execution_date=execution_date, + logical_date=logical_date, run_conf=run_conf, use_executor=use_executor, mark_success_pattern=mark_success_pattern, @@ -515,7 +515,7 @@ def dag_test(args, dag: DAG | None = None, session: Session = NEW_SESSION) -> No tis = session.scalars( select(TaskInstance).where( TaskInstance.dag_id == args.dag_id, - TaskInstance.execution_date == execution_date, + TaskInstance.logical_date == logical_date, ) ).all() diff --git a/airflow/cli/commands/kubernetes_command.py b/airflow/cli/commands/kubernetes_command.py index eab11133c9dd..8f5f7333b7f4 100644 --- a/airflow/cli/commands/kubernetes_command.py +++ b/airflow/cli/commands/kubernetes_command.py @@ -48,10 +48,10 @@ @providers_configuration_loaded def generate_pod_yaml(args): """Generate yaml files for each task in the DAG. Used for testing output of KubernetesExecutor.""" - execution_date = args.execution_date + logical_date = args.logical_date dag = get_dag(subdir=args.subdir, dag_id=args.dag_id) yaml_output_path = args.output_path - dr = DagRun(dag.dag_id, execution_date=execution_date) + dr = DagRun(dag.dag_id, logical_date=logical_date) kube_config = KubeConfig() for task in dag.tasks: ti = TaskInstance(task, None) @@ -62,7 +62,7 @@ def generate_pod_yaml(args): pod_id=create_unique_id(args.dag_id, ti.task_id), try_number=ti.try_number, kube_image=kube_config.kube_image, - date=ti.execution_date, + date=ti.logical_date, args=ti.command_as_list(), pod_override_object=PodGenerator.from_obj(ti.executor_config), scheduler_job_id="worker-config", @@ -71,7 +71,7 @@ def generate_pod_yaml(args): with_mutation_hook=True, ) api_client = ApiClient() - date_string = pod_generator.datetime_to_label_safe_datestring(execution_date) + date_string = pod_generator.datetime_to_label_safe_datestring(logical_date) yaml_file_name = f"{args.dag_id}_{ti.task_id}_{date_string}.yml" os.makedirs(os.path.dirname(yaml_output_path + "/airflow_yaml_output/"), exist_ok=True) with open(yaml_output_path + "/airflow_yaml_output/" + yaml_file_name, "w") as output: diff --git a/airflow/cli/commands/task_command.py b/airflow/cli/commands/task_command.py index e14c18399555..396186bf14d2 100644 --- a/airflow/cli/commands/task_command.py +++ b/airflow/cli/commands/task_command.py @@ -114,43 +114,43 @@ def _get_dag_run( """ if not exec_date_or_run_id and not create_if_necessary: raise ValueError("Must provide `exec_date_or_run_id` if not `create_if_necessary`.") - execution_date: pendulum.DateTime | None = None + logical_date: pendulum.DateTime | None = None if exec_date_or_run_id: dag_run = DAG.fetch_dagrun(dag_id=dag.dag_id, run_id=exec_date_or_run_id, session=session) if dag_run: return dag_run, False with suppress(ParserError, TypeError): - execution_date = timezone.parse(exec_date_or_run_id) - if execution_date: - dag_run = DAG.fetch_dagrun(dag_id=dag.dag_id, execution_date=execution_date, session=session) + logical_date = timezone.parse(exec_date_or_run_id) + if logical_date: + dag_run = DAG.fetch_dagrun(dag_id=dag.dag_id, logical_date=logical_date, session=session) if dag_run: return dag_run, False elif not create_if_necessary: raise DagRunNotFound( - f"DagRun for {dag.dag_id} with run_id or execution_date " + f"DagRun for {dag.dag_id} with run_id or logical_date " f"of {exec_date_or_run_id!r} not found" ) - if execution_date is not None: - dag_run_execution_date = execution_date + if logical_date is not None: + dag_run_logical_date = logical_date else: - dag_run_execution_date = pendulum.instance(timezone.utcnow()) + dag_run_logical_date = pendulum.instance(timezone.utcnow()) if create_if_necessary == "memory": dag_run = DagRun( dag_id=dag.dag_id, run_id=exec_date_or_run_id, - execution_date=dag_run_execution_date, - data_interval=dag.timetable.infer_manual_data_interval(run_after=dag_run_execution_date), + logical_date=dag_run_logical_date, + data_interval=dag.timetable.infer_manual_data_interval(run_after=dag_run_logical_date), triggered_by=DagRunTriggeredByType.CLI, ) return dag_run, True elif create_if_necessary == "db": dag_run = dag.create_dagrun( state=DagRunState.QUEUED, - execution_date=dag_run_execution_date, + logical_date=dag_run_logical_date, run_id=_generate_temporary_run_id(), - data_interval=dag.timetable.infer_manual_data_interval(run_after=dag_run_execution_date), + data_interval=dag.timetable.infer_manual_data_interval(run_after=dag_run_logical_date), session=session, triggered_by=DagRunTriggeredByType.CLI, ) @@ -165,7 +165,7 @@ def _get_ti_db_access( task: Operator, map_index: int, *, - exec_date_or_run_id: str | None = None, + logical_date_or_run_id: str | None = None, pool: str | None = None, create_if_necessary: CreateIfNecessary = False, session: Session = NEW_SESSION, @@ -177,7 +177,7 @@ def _get_ti_db_access( if task.task_id not in dag.task_dict: raise ValueError(f"Provided task {task.task_id} is not in dag '{dag.dag_id}.") - if not exec_date_or_run_id and not create_if_necessary: + if not logical_date_or_run_id and not create_if_necessary: raise ValueError("Must provide `exec_date_or_run_id` if not `create_if_necessary`.") if task.get_needs_expansion(): if map_index < 0: @@ -186,7 +186,7 @@ def _get_ti_db_access( raise RuntimeError("map_index passed to non-mapped task") dag_run, dr_created = _get_dag_run( dag=dag, - exec_date_or_run_id=exec_date_or_run_id, + exec_date_or_run_id=logical_date_or_run_id, create_if_necessary=create_if_necessary, session=session, ) @@ -197,7 +197,7 @@ def _get_ti_db_access( if not create_if_necessary: raise TaskInstanceNotFound( f"TaskInstance for {dag.dag_id}, {task.task_id}, map={map_index} with " - f"run_id or execution_date of {exec_date_or_run_id!r} not found" + f"run_id or logical_date of {logical_date_or_run_id!r} not found" ) # TODO: Validate map_index is in range? ti = TaskInstance(task, run_id=dag_run.run_id, map_index=map_index) @@ -214,7 +214,7 @@ def _get_ti( task: Operator, map_index: int, *, - exec_date_or_run_id: str | None = None, + logical_date_or_run_id: str | None = None, pool: str | None = None, create_if_necessary: CreateIfNecessary = False, ): @@ -226,7 +226,7 @@ def _get_ti( dag=dag, task=task, map_index=map_index, - exec_date_or_run_id=exec_date_or_run_id, + logical_date_or_run_id=logical_date_or_run_id, pool=pool, create_if_necessary=create_if_necessary, ) @@ -438,7 +438,7 @@ def task_run(args, dag: DAG | None = None) -> TaskReturnCode | None: else: _dag = dag task = _dag.get_task(task_id=args.task_id) - ti, _ = _get_ti(task, args.map_index, exec_date_or_run_id=args.execution_date_or_run_id, pool=args.pool) + ti, _ = _get_ti(task, args.map_index, logical_date_or_run_id=args.logical_date_or_run_id, pool=args.pool) ti.init_run_context(raw=args.raw) hostname = get_hostname() @@ -487,7 +487,7 @@ def task_failed_deps(args) -> None: """ dag = get_dag(args.subdir, args.dag_id) task = dag.get_task(task_id=args.task_id) - ti, _ = _get_ti(task, args.map_index, exec_date_or_run_id=args.execution_date_or_run_id) + ti, _ = _get_ti(task, args.map_index, logical_date_or_run_id=args.logical_date_or_run_id) # tasks_failed-deps is executed with access to the database. if isinstance(ti, TaskInstancePydantic): raise ValueError("not a TaskInstance") @@ -514,7 +514,7 @@ def task_state(args) -> None: """ dag = get_dag(args.subdir, args.dag_id) task = dag.get_task(task_id=args.task_id) - ti, _ = _get_ti(task, args.map_index, exec_date_or_run_id=args.execution_date_or_run_id) + ti, _ = _get_ti(task, args.map_index, logical_date_or_run_id=args.logical_date_or_run_id) # task_state is executed with access to the database. if isinstance(ti, TaskInstancePydantic): raise ValueError("not a TaskInstance") @@ -572,20 +572,20 @@ def _guess_debugger() -> _SupportedDebugger: def task_states_for_dag_run(args, session: Session = NEW_SESSION) -> None: """Get the status of all task instances in a DagRun.""" dag_run = session.scalar( - select(DagRun).where(DagRun.run_id == args.execution_date_or_run_id, DagRun.dag_id == args.dag_id) + select(DagRun).where(DagRun.run_id == args.logical_date_or_run_id, DagRun.dag_id == args.dag_id) ) if not dag_run: try: - execution_date = timezone.parse(args.execution_date_or_run_id) + logical_date = timezone.parse(args.logical_date_or_run_id) dag_run = session.scalar( - select(DagRun).where(DagRun.execution_date == execution_date, DagRun.dag_id == args.dag_id) + select(DagRun).where(DagRun.logical_date == logical_date, DagRun.dag_id == args.dag_id) ) except (ParserError, TypeError) as err: - raise AirflowException(f"Error parsing the supplied execution_date. Error: {err}") + raise AirflowException(f"Error parsing the supplied logical_date. Error: {err}") if dag_run is None: raise DagRunNotFound( - f"DagRun for {args.dag_id} with run_id or execution_date of {args.execution_date_or_run_id!r} " + f"DagRun for {args.dag_id} with run_id or logical_date of {args.logical_date_or_run_id!r} " "not found" ) @@ -594,7 +594,7 @@ def task_states_for_dag_run(args, session: Session = NEW_SESSION) -> None: def format_task_instance(ti: TaskInstance) -> dict[str, str]: data = { "dag_id": ti.dag_id, - "execution_date": dag_run.execution_date.isoformat(), + "logical_date": dag_run.logical_date.isoformat(), "task_id": ti.task_id, "state": ti.state, "start_date": ti.start_date.isoformat() if ti.start_date else "", @@ -643,7 +643,7 @@ def task_test(args, dag: DAG | None = None, session: Session = NEW_SESSION) -> N task.params.validate() ti, dr_created = _get_ti( - task, args.map_index, exec_date_or_run_id=args.execution_date_or_run_id, create_if_necessary="db" + task, args.map_index, logical_date_or_run_id=args.logical_date_or_run_id, create_if_necessary="db" ) # task_test is executed with access to the database. if isinstance(ti, TaskInstancePydantic): @@ -694,7 +694,7 @@ def task_render(args, dag: DAG | None = None) -> None: dag = get_dag(args.subdir, args.dag_id) task = dag.get_task(task_id=args.task_id) ti, _ = _get_ti( - task, args.map_index, exec_date_or_run_id=args.execution_date_or_run_id, create_if_necessary="memory" + task, args.map_index, logical_date_or_run_id=args.logical_date_or_run_id, create_if_necessary="memory" ) # task_render is executed with access to the database. if isinstance(ti, TaskInstancePydantic): diff --git a/airflow/configuration.py b/airflow/configuration.py index 82718325865f..cd218657133a 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -376,7 +376,7 @@ def inversed_deprecated_sections(self): }, "elasticsearch": { "log_id_template": ( - re2.compile("^" + re2.escape("{dag_id}-{task_id}-{execution_date}-{try_number}") + "$"), + re2.compile("^" + re2.escape("{dag_id}-{task_id}-{logical_date}-{try_number}") + "$"), "{dag_id}-{task_id}-{run_id}-{map_index}-{try_number}", "3.0", ) diff --git a/airflow/dag_processing/collection.py b/airflow/dag_processing/collection.py index f608900ee76e..0ca121c56185 100644 --- a/airflow/dag_processing/collection.py +++ b/airflow/dag_processing/collection.py @@ -89,7 +89,7 @@ def _get_latest_runs_stmt(dag_ids: Collection[str]) -> Select: if len(dag_ids) == 1: # Index optimized fast path to avoid more complicated & slower groupby queryplan. (dag_id,) = dag_ids last_automated_runs_subq = ( - select(func.max(DagRun.execution_date).label("max_execution_date")) + select(func.max(DagRun.logical_date).label("max_execution_date")) .where( DagRun.dag_id == dag_id, DagRun.run_type.in_((DagRunType.BACKFILL_JOB, DagRunType.SCHEDULED)), @@ -98,11 +98,11 @@ def _get_latest_runs_stmt(dag_ids: Collection[str]) -> Select: ) query = select(DagRun).where( DagRun.dag_id == dag_id, - DagRun.execution_date == last_automated_runs_subq, + DagRun.logical_date == last_automated_runs_subq, ) else: last_automated_runs_subq = ( - select(DagRun.dag_id, func.max(DagRun.execution_date).label("max_execution_date")) + select(DagRun.dag_id, func.max(DagRun.logical_date).label("max_execution_date")) .where( DagRun.dag_id.in_(dag_ids), DagRun.run_type.in_((DagRunType.BACKFILL_JOB, DagRunType.SCHEDULED)), @@ -112,12 +112,12 @@ def _get_latest_runs_stmt(dag_ids: Collection[str]) -> Select: ) query = select(DagRun).where( DagRun.dag_id == last_automated_runs_subq.c.dag_id, - DagRun.execution_date == last_automated_runs_subq.c.max_execution_date, + DagRun.logical_date == last_automated_runs_subq.c.max_execution_date, ) return query.options( load_only( DagRun.dag_id, - DagRun.execution_date, + DagRun.logical_date, DagRun.data_interval_start, DagRun.data_interval_end, ) diff --git a/airflow/dag_processing/processor.py b/airflow/dag_processing/processor.py index 394e09245127..219c7aa9776a 100644 --- a/airflow/dag_processing/processor.py +++ b/airflow/dag_processing/processor.py @@ -608,7 +608,7 @@ def _execute_dag_callbacks(cls, dagbag: DagBag, request: DagCallbackRequest, ses dag = dagbag.dags[request.dag_id] callbacks, context = DAG.fetch_callback( dag=dag, - dag_run_id=request.run_id, + run_id=request.run_id, success=not request.is_failure_callback, reason=request.msg, session=session, diff --git a/airflow/example_dags/example_branch_python_dop_operator_3.py b/airflow/example_dags/example_branch_python_dop_operator_3.py index a7031961ab55..4bb940ca36dd 100644 --- a/airflow/example_dags/example_branch_python_dop_operator_3.py +++ b/airflow/example_dags/example_branch_python_dop_operator_3.py @@ -32,15 +32,13 @@ @task.branch() def should_run(**kwargs) -> str: """ - Determine which empty_task should be run based on if the execution date minute is even or odd. + Determine which empty_task should be run based on if the logical date minute is even or odd. :param dict kwargs: Context :return: Id of the task to run """ - print( - f"------------- exec dttm = {kwargs['execution_date']} and minute = {kwargs['execution_date'].minute}" - ) - if kwargs["execution_date"].minute % 2 == 0: + print(f"------------- exec dttm = {kwargs['logical_date']} and minute = {kwargs['logical_date'].minute}") + if kwargs["logical_date"].minute % 2 == 0: return "empty_task_1" else: return "empty_task_2" diff --git a/airflow/example_dags/tutorial_objectstorage.py b/airflow/example_dags/tutorial_objectstorage.py index 4660aa3c8e8c..d03ec6ec8003 100644 --- a/airflow/example_dags/tutorial_objectstorage.py +++ b/airflow/example_dags/tutorial_objectstorage.py @@ -72,7 +72,7 @@ def get_air_quality_data(**kwargs) -> ObjectStoragePath: """ import pandas as pd - execution_date = kwargs["logical_date"] + logical_date = kwargs["logical_date"] start_time = kwargs["data_interval_start"] params = { @@ -83,7 +83,7 @@ def get_air_quality_data(**kwargs) -> ObjectStoragePath: "area": "Uusimaa", "param": ",".join(aq_fields.keys()), "starttime": start_time.isoformat(timespec="seconds"), - "endtime": execution_date.isoformat(timespec="seconds"), + "endtime": logical_date.isoformat(timespec="seconds"), "tz": "UTC", } @@ -93,7 +93,7 @@ def get_air_quality_data(**kwargs) -> ObjectStoragePath: # ensure the bucket exists base.mkdir(exist_ok=True) - formatted_date = execution_date.format("YYYYMMDD") + formatted_date = logical_date.format("YYYYMMDD") path = base / f"air_quality_{formatted_date}.parquet" df = pd.DataFrame(response.json()).astype(aq_fields) diff --git a/airflow/exceptions.py b/airflow/exceptions.py index 316fe880b66b..3b07b9a6fda9 100644 --- a/airflow/exceptions.py +++ b/airflow/exceptions.py @@ -230,12 +230,12 @@ class DagRunNotFound(AirflowNotFoundException): class DagRunAlreadyExists(AirflowBadRequest): """Raise when creating a DAG run for DAG which already has DAG run entry.""" - def __init__(self, dag_run: DagRun, execution_date: datetime.datetime, run_id: str) -> None: + def __init__(self, dag_run: DagRun, logical_date: datetime.datetime, run_id: str) -> None: super().__init__( - f"A DAG Run already exists for DAG {dag_run.dag_id} at {execution_date} with run id {run_id}" + f"A DAG Run already exists for DAG {dag_run.dag_id} at {logical_date} with run id {run_id}" ) self.dag_run = dag_run - self.execution_date = execution_date + self.logical_date = logical_date self.run_id = run_id def serialize(self): @@ -249,13 +249,13 @@ def serialize(self): run_id=self.dag_run.run_id, external_trigger=self.dag_run.external_trigger, run_type=self.dag_run.run_type, - execution_date=self.dag_run.execution_date, + logical_date=self.dag_run.logical_date, ) dag_run.id = self.dag_run.id return ( f"{cls.__module__}.{cls.__name__}", (), - {"dag_run": dag_run, "execution_date": self.execution_date, "run_id": self.run_id}, + {"dag_run": dag_run, "logical_date": self.logical_date, "run_id": self.run_id}, ) diff --git a/airflow/jobs/scheduler_job_runner.py b/airflow/jobs/scheduler_job_runner.py index 5369f425c9c4..6c7887c643cb 100644 --- a/airflow/jobs/scheduler_job_runner.py +++ b/airflow/jobs/scheduler_job_runner.py @@ -340,7 +340,7 @@ def _executable_task_instances_to_queued(self, max_tis: int, session: Session) - .where(not_(DM.is_paused)) .where(TI.state == TaskInstanceState.SCHEDULED) .options(selectinload(TI.dag_model)) - .order_by(-TI.priority_weight, DR.execution_date, TI.map_index) + .order_by(-TI.priority_weight, DR.logical_date, TI.map_index) ) if starved_pools: @@ -735,7 +735,7 @@ def process_executor_events( # Report execution for ti_key, (state, _) in event_buffer.items(): - # We create map (dag_id, task_id, execution_date) -> in-memory try_number + # We create map (dag_id, task_id, logical_date) -> in-memory try_number ti_primary_key_to_try_number_map[ti_key.primary] = ti_key.try_number cls.logger().info("Received executor event with state %s for task instance %s", state, ti_key) @@ -871,7 +871,7 @@ def _set_span_attrs__process_executor_events(cls, span, state, ti): "end_date": str(ti.end_date), "duration": ti.duration, "executor_config": str(ti.executor_config), - "execution_date": str(ti.execution_date), + "logical_date": str(ti.logical_date), "hostname": ti.hostname, "log_url": ti.log_url, "operator": str(ti.operator), @@ -1288,15 +1288,15 @@ def _mark_backfills_complete(self, session: Session = NEW_SESSION) -> None: @add_span def _create_dag_runs(self, dag_models: Collection[DagModel], session: Session) -> None: """Create a DAG run and update the dag_model to control if/when the next DAGRun should be created.""" - # Bulk Fetch DagRuns with dag_id and execution_date same + # Bulk Fetch DagRuns with dag_id and logical_date same # as DagModel.dag_id and DagModel.next_dagrun # This list is used to verify if the DagRun already exist so that we don't attempt to create # duplicate dag runs existing_dagruns = ( session.execute( - select(DagRun.dag_id, DagRun.execution_date).where( + select(DagRun.dag_id, DagRun.logical_date).where( tuple_in_condition( - (DagRun.dag_id, DagRun.execution_date), + (DagRun.dag_id, DagRun.logical_date), ((dm.dag_id, dm.next_dagrun) for dm in dag_models), ), ) @@ -1337,7 +1337,7 @@ def _create_dag_runs(self, dag_models: Collection[DagModel], session: Session) - try: dag.create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=dag_model.next_dagrun, + logical_date=dag_model.next_dagrun, state=DagRunState.QUEUED, data_interval=data_interval, external_trigger=False, @@ -1372,18 +1372,18 @@ def _create_dag_runs_asset_triggered( session: Session, ) -> None: """For DAGs that are triggered by assets, create dag runs.""" - # Bulk Fetch DagRuns with dag_id and execution_date same + # Bulk Fetch DagRuns with dag_id and logical_date same # as DagModel.dag_id and DagModel.next_dagrun # This list is used to verify if the DagRun already exist so that we don't attempt to create # duplicate dag runs - exec_dates = { + logical_dates = { dag_id: timezone.coerce_datetime(last_time) for dag_id, (_, last_time) in asset_triggered_dag_info.items() } existing_dagruns: set[tuple[str, timezone.DateTime]] = set( session.execute( - select(DagRun.dag_id, DagRun.execution_date).where( - tuple_in_condition((DagRun.dag_id, DagRun.execution_date), exec_dates.items()) + select(DagRun.dag_id, DagRun.logical_date).where( + tuple_in_condition((DagRun.dag_id, DagRun.logical_date), logical_dates.items()) ) ) ) @@ -1411,24 +1411,24 @@ def _create_dag_runs_asset_triggered( # we need to set dag.next_dagrun_info if the Dag Run already exists or if we # create a new one. This is so that in the next Scheduling loop we try to create new runs # instead of falling in a loop of Integrity Error. - exec_date = exec_dates[dag.dag_id] - if (dag.dag_id, exec_date) not in existing_dagruns: + logical_date = logical_dates[dag.dag_id] + if (dag.dag_id, logical_date) not in existing_dagruns: previous_dag_run = session.scalar( select(DagRun) .where( DagRun.dag_id == dag.dag_id, - DagRun.execution_date < exec_date, + DagRun.logical_date < logical_date, DagRun.run_type == DagRunType.ASSET_TRIGGERED, ) - .order_by(DagRun.execution_date.desc()) + .order_by(DagRun.logical_date.desc()) .limit(1) ) asset_event_filters = [ DagScheduleAssetReference.dag_id == dag.dag_id, - AssetEvent.timestamp <= exec_date, + AssetEvent.timestamp <= logical_date, ] if previous_dag_run: - asset_event_filters.append(AssetEvent.timestamp > previous_dag_run.execution_date) + asset_event_filters.append(AssetEvent.timestamp > previous_dag_run.logical_date) asset_events = session.scalars( select(AssetEvent) @@ -1439,10 +1439,10 @@ def _create_dag_runs_asset_triggered( .where(*asset_event_filters) ).all() - data_interval = dag.timetable.data_interval_for_events(exec_date, asset_events) + data_interval = dag.timetable.data_interval_for_events(logical_date, asset_events) run_id = dag.timetable.generate_run_id( run_type=DagRunType.ASSET_TRIGGERED, - logical_date=exec_date, + logical_date=logical_date, data_interval=data_interval, session=session, events=asset_events, @@ -1451,7 +1451,7 @@ def _create_dag_runs_asset_triggered( dag_run = dag.create_dagrun( run_id=run_id, run_type=DagRunType.ASSET_TRIGGERED, - execution_date=exec_date, + logical_date=logical_date, data_interval=data_interval, state=DagRunState.QUEUED, external_trigger=False, @@ -1706,8 +1706,8 @@ def _schedule_dag_run( ) return callback_to_execute - if dag_run.execution_date > timezone.utcnow() and not dag.allow_future_exec_dates: - self.log.error("Execution date is in future: %s", dag_run.execution_date) + if dag_run.logical_date > timezone.utcnow() and not dag.allow_future_exec_dates: + self.log.error("Logical date is in future: %s", dag_run.logical_date) return callback if not self._verify_integrity_if_dag_changed(dag_run=dag_run, session=session): @@ -1721,7 +1721,7 @@ def _schedule_dag_run( if self._should_update_dag_next_dagruns(dag, dag_model, last_dag_run=dag_run, session=session): dag_model.calculate_dagrun_date_fields(dag, dag.get_run_data_interval(dag_run)) # This will do one query per dag run. We "could" build up a complex - # query to update all the TIs across all the execution dates and dag + # query to update all the TIs across all the logical dates and dag # IDs in a single query, but it turns out that can be _very very slow_ # see #11147/commit ee90807ac for more details if span.is_recording(): diff --git a/airflow/models/backfill.py b/airflow/models/backfill.py index 11d677542fc0..0e88fa15bb04 100644 --- a/airflow/models/backfill.py +++ b/airflow/models/backfill.py @@ -171,7 +171,7 @@ def _create_backfill_dag_run( dr = session.scalar( with_row_locks( select(DagRun) - .where(DagRun.execution_date == info.logical_date) + .where(DagRun.logical_date == info.logical_date) .order_by(nulls_first(desc(DagRun.start_date), session=session)) .limit(1), session=session, @@ -204,7 +204,7 @@ def _create_backfill_dag_run( dag_version = DagVersion.get_latest_version(dag.dag_id, session=session) dr = dag.create_dagrun( triggered_by=DagRunTriggeredByType.BACKFILL, - execution_date=info.logical_date, + logical_date=info.logical_date, data_interval=info.data_interval, start_date=timezone.utcnow(), state=DagRunState.QUEUED, diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py index fa26a2026f5d..520060b8b8f7 100644 --- a/airflow/models/baseoperator.py +++ b/airflow/models/baseoperator.py @@ -438,16 +438,16 @@ class derived from this one results in the creation of a task object, :param max_retry_delay: maximum delay interval between retries, can be set as ``timedelta`` or ``float`` seconds, which will be converted into ``timedelta``. :param start_date: The ``start_date`` for the task, determines - the ``execution_date`` for the first task instance. The best practice + the ``logical_date`` for the first task instance. The best practice is to have the start_date rounded to your DAG's schedule. Daily jobs have their start_date some day at 00:00:00, hourly jobs have their start_date at 00:00 of a specific hour. Note that Airflow simply looks at the latest - ``execution_date`` and adds the schedule to determine - the next ``execution_date``. It is also very important + ``logical_date`` and adds the schedule to determine + the next ``logical_date``. It is also very important to note that different tasks' dependencies need to line up in time. If task A depends on task B and their - start_date are offset in a way that their execution_date don't line + start_date are offset in a way that their logical_date don't line up, A's dependencies will never be met. If you are looking to delay a task, for example running a daily task at 2AM, look into the ``TimeSensor`` and ``TimeDeltaSensor``. We advise against using @@ -552,7 +552,7 @@ class derived from this one results in the creation of a task object, Resources constructor) to their values. :param run_as_user: unix username to impersonate while running the task :param max_active_tis_per_dag: When set, a task will be able to limit the concurrent - runs across execution_dates. + runs across logical_dates. :param max_active_tis_per_dagrun: When set, a task will be able to limit the concurrent task instances per DAG run. :param executor: Which executor to target when running this task. NOT YET SUPPORTED @@ -772,9 +772,9 @@ def clear( qry = select(TaskInstance).where(TaskInstance.dag_id == self.dag_id) if start_date: - qry = qry.where(TaskInstance.execution_date >= start_date) + qry = qry.where(TaskInstance.logical_date >= start_date) if end_date: - qry = qry.where(TaskInstance.execution_date <= end_date) + qry = qry.where(TaskInstance.logical_date <= end_date) tasks = [self.task_id] @@ -814,10 +814,10 @@ def get_task_instances( .where(TaskInstance.task_id == self.task_id) ) if start_date: - query = query.where(DagRun.execution_date >= start_date) + query = query.where(DagRun.logical_date >= start_date) if end_date: - query = query.where(DagRun.execution_date <= end_date) - return session.scalars(query.order_by(DagRun.execution_date)).all() + query = query.where(DagRun.logical_date <= end_date) + return session.scalars(query.order_by(DagRun.logical_date)).all() @provide_session def run( @@ -853,7 +853,7 @@ def run( dag_run = session.scalars( select(DagRun).where( DagRun.dag_id == self.dag_id, - DagRun.execution_date == info.logical_date, + DagRun.logical_date == info.logical_date, ) ).one() ti = TaskInstance(self, run_id=dag_run.run_id) @@ -863,7 +863,7 @@ def run( dag_id=self.dag_id, run_id=DagRun.generate_run_id(DagRunType.MANUAL, info.logical_date), run_type=DagRunType.MANUAL, - execution_date=info.logical_date, + logical_date=info.logical_date, data_interval=info.data_interval, triggered_by=DagRunTriggeredByType.TEST, ) @@ -953,7 +953,7 @@ def xcom_pull( :param dag_id: If provided, only pulls XComs from this DAG. If None (default), the DAG of the calling task is used. :param include_prior_dates: If False, only XComs from the current - execution_date are returned. If True, XComs from previous dates + logical_date are returned. If True, XComs from previous dates are returned as well. """ return context["ti"].xcom_pull( diff --git a/airflow/models/dag.py b/airflow/models/dag.py index e48ec0a9a9c5..7ab252b88b02 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -194,7 +194,7 @@ def get_last_dagrun(dag_id, session, include_externally_triggered=False): query = select(DR).where(DR.dag_id == dag_id) if not include_externally_triggered: query = query.where(DR.external_trigger == expression.false()) - query = query.order_by(DR.execution_date.desc()) + query = query.order_by(DR.logical_date.desc()) return session.scalar(query.limit(1)) @@ -267,7 +267,7 @@ def _create_orm_dagrun( run = DagRun( dag_id=dag_id, run_id=run_id, - execution_date=logical_date, + logical_date=logical_date, start_date=start_date, external_trigger=external_trigger, conf=conf, @@ -538,7 +538,7 @@ def get_run_data_interval(self, run: DagRun | DagRunPydantic) -> DataInterval: return data_interval # Compatibility: runs created before AIP-39 implementation don't have an # explicit data interval. Try to infer from the logical date. - return self.infer_automated_data_interval(run.execution_date) + return self.infer_automated_data_interval(run.logical_date) def infer_automated_data_interval(self, logical_date: datetime) -> DataInterval: """ @@ -581,13 +581,13 @@ def next_dagrun_info( Get information about the next DagRun of this dag after ``date_last_automated_dagrun``. This calculates what time interval the next DagRun should operate on - (its execution date) and when it can be scheduled, according to the + (its logical date) and when it can be scheduled, according to the dag's timetable, start_date, end_date, etc. This doesn't check max active run or any other "max_active_tasks" type limits, but only performs calculations based on the various date and interval fields of this dag and its tasks. - :param last_automated_dagrun: The ``max(execution_date)`` of + :param last_automated_dagrun: The ``max(logical_date)`` of existing "automated" DagRuns for this dag (scheduled or backfill, but not manual). :param restricted: If set to *False* (default is *True*), ignore @@ -784,7 +784,7 @@ def get_serialized_fields(cls): @provide_session def fetch_callback( dag: DAG, - dag_run_id: str, + run_id: str, success: bool = True, reason: str | None = None, *, @@ -797,14 +797,14 @@ def fetch_callback( the list of callbacks. :param dag: DAG object - :param dag_run_id: The DAG run ID + :param run_id: The DAG run ID :param success: Flag to specify if failure or success callback should be called :param reason: Completion reason :param session: Database session """ callbacks = dag.on_success_callback if success else dag.on_failure_callback if callbacks: - dagrun = DAG.fetch_dagrun(dag_id=dag.dag_id, run_id=dag_run_id, session=session) + dagrun = DAG.fetch_dagrun(dag_id=dag.dag_id, run_id=run_id, session=session) callbacks = callbacks if isinstance(callbacks, list) else [callbacks] tis = dagrun.get_task_instances(session=session) # tis from a dagrun may not be a part of dag.partial_subset, @@ -840,7 +840,7 @@ def handle_callback(self, dagrun: DagRun, success=True, reason=None, session=NEW :param session: Database session """ callbacks, context = DAG.fetch_callback( - dag=self, dag_run_id=dagrun.run_id, success=success, reason=reason, session=session + dag=self, run_id=dagrun.run_id, success=success, reason=reason, session=session ) or (None, None) DAG.execute_callback(callbacks, context, self.dag_id) @@ -865,15 +865,15 @@ def execute_callback(cls, callbacks: list[Callable] | None, context: Context | N def get_active_runs(self): """ - Return a list of dag run execution dates currently running. + Return a list of dag run logical dates currently running. - :return: List of execution dates + :return: List of logical dates """ runs = DagRun.find(dag_id=self.dag_id, state=DagRunState.RUNNING) active_dates = [] for run in runs: - active_dates.append(run.execution_date) + active_dates.append(run.logical_date) return active_dates @@ -882,24 +882,24 @@ def get_active_runs(self): @provide_session def fetch_dagrun( dag_id: str, - execution_date: datetime | None = None, + logical_date: datetime | None = None, run_id: str | None = None, session: Session = NEW_SESSION, ) -> DagRun | DagRunPydantic: """ - Return the dag run for a given execution date or run_id if it exists, otherwise none. + Return the dag run for a given logical date or run_id if it exists, otherwise none. :param dag_id: The dag_id of the DAG to find. - :param execution_date: The execution date of the DagRun to find. + :param logical_date: The logical date of the DagRun to find. :param run_id: The run_id of the DagRun to find. :param session: :return: The DagRun if found, otherwise None. """ - if not (execution_date or run_id): - raise TypeError("You must provide either the execution_date or the run_id") + if not (logical_date or run_id): + raise TypeError("You must provide either the logical_date or the run_id") query = select(DagRun) - if execution_date: - query = query.where(DagRun.dag_id == dag_id, DagRun.execution_date == execution_date) + if logical_date: + query = query.where(DagRun.dag_id == dag_id, DagRun.logical_date == logical_date) if run_id: query = query.where(DagRun.dag_id == dag_id, DagRun.run_id == run_id) return session.scalar(query) @@ -907,38 +907,36 @@ def fetch_dagrun( @provide_session def get_dagrun( self, - execution_date: datetime | None = None, + logical_date: datetime | None = None, run_id: str | None = None, session: Session = NEW_SESSION, ) -> DagRun | DagRunPydantic: - return DAG.fetch_dagrun( - dag_id=self.dag_id, execution_date=execution_date, run_id=run_id, session=session - ) + return DAG.fetch_dagrun(dag_id=self.dag_id, logical_date=logical_date, run_id=run_id, session=session) @provide_session def get_dagruns_between(self, start_date, end_date, session=NEW_SESSION): """ Return the list of dag runs between start_date (inclusive) and end_date (inclusive). - :param start_date: The starting execution date of the DagRun to find. - :param end_date: The ending execution date of the DagRun to find. + :param start_date: The starting logical date of the DagRun to find. + :param end_date: The ending logical date of the DagRun to find. :param session: :return: The list of DagRuns found. """ dagruns = session.scalars( select(DagRun).where( DagRun.dag_id == self.dag_id, - DagRun.execution_date >= start_date, - DagRun.execution_date <= end_date, + DagRun.logical_date >= start_date, + DagRun.logical_date <= end_date, ) ).all() return dagruns @provide_session - def get_latest_execution_date(self, session: Session = NEW_SESSION) -> pendulum.DateTime | None: + def get_latest_logical_date(self, session: Session = NEW_SESSION) -> pendulum.DateTime | None: """Return the latest date for which at least one dag run exists.""" - return session.scalar(select(func.max(DagRun.execution_date)).where(DagRun.dag_id == self.dag_id)) + return session.scalar(select(func.max(DagRun.logical_date)).where(DagRun.dag_id == self.dag_id)) @provide_session def get_task_instances_before( @@ -955,21 +953,21 @@ def get_task_instances_before( corresponding to any DagRunType. It can have less if there are less than ``num`` scheduled DAG runs before ``base_date``. """ - execution_dates: list[Any] = session.execute( - select(DagRun.execution_date) + logical_dates: list[Any] = session.execute( + select(DagRun.logical_date) .where( DagRun.dag_id == self.dag_id, - DagRun.execution_date <= base_date, + DagRun.logical_date <= base_date, ) - .order_by(DagRun.execution_date.desc()) + .order_by(DagRun.logical_date.desc()) .limit(num) ).all() - if not execution_dates: + if not logical_dates: return self.get_task_instances(start_date=base_date, end_date=base_date, session=session) - min_date: datetime | None = execution_dates[-1]._mapping.get( - "execution_date" + min_date: datetime | None = logical_dates[-1]._mapping.get( + "logical_date" ) # getting the last value from the list return self.get_task_instances(start_date=min_date, end_date=base_date, session=session) @@ -997,7 +995,7 @@ def get_task_instances( exclude_task_ids=(), session=session, ) - return session.scalars(cast(Select, query).order_by(DagRun.execution_date)).all() + return session.scalars(cast(Select, query).order_by(DagRun.logical_date)).all() @overload def _get_task_instances( @@ -1074,14 +1072,14 @@ def _get_task_instances( if run_id: tis = tis.where(TaskInstance.run_id == run_id) if start_date: - tis = tis.where(DagRun.execution_date >= start_date) + tis = tis.where(DagRun.logical_date >= start_date) if task_ids is not None: tis = tis.where(TaskInstance.ti_selector_condition(task_ids)) # This allows allow_trigger_in_future config to take affect, rather than mandating exec_date <= UTC if end_date or not self.allow_future_exec_dates: end_date = end_date or timezone.utcnow() - tis = tis.where(DagRun.execution_date <= end_date) + tis = tis.where(DagRun.logical_date <= end_date) if state: if isinstance(state, (str, TaskInstanceState)): @@ -1146,7 +1144,7 @@ def _get_task_instances( .where( TI.dag_id == task.external_dag_id, TI.task_id == task.external_task_id, - DagRun.execution_date == pendulum.parse(task.execution_date), + DagRun.logical_date == pendulum.parse(task.logical_date), ) ) @@ -1219,7 +1217,7 @@ def set_task_instance_state( *, task_id: str, map_indexes: Collection[int] | None = None, - execution_date: datetime | None = None, + logical_date: datetime | None = None, run_id: str | None = None, state: TaskInstanceState, upstream: bool = False, @@ -1235,7 +1233,7 @@ def set_task_instance_state( :param task_id: Task ID of the TaskInstance :param map_indexes: Only set TaskInstance if its map_index matches. If None (default), all mapped TaskInstances of the task are set. - :param execution_date: Execution date of the TaskInstance + :param logical_date: Logical date of the TaskInstance :param run_id: The run_id of the TaskInstance :param state: State to set the TaskInstance to :param upstream: Include all upstream tasks of the given task_id @@ -1246,8 +1244,8 @@ def set_task_instance_state( """ from airflow.api.common.mark_tasks import set_state - if not exactly_one(execution_date, run_id): - raise ValueError("Exactly one of execution_date or run_id must be provided") + if not exactly_one(logical_date, run_id): + raise ValueError("Exactly one of logical_date or run_id must be provided") task = self.get_task(task_id) task.dag = self @@ -1260,7 +1258,7 @@ def set_task_instance_state( altered = set_state( tasks=tasks_to_set_state, - execution_date=execution_date, + logical_date=logical_date, run_id=run_id, upstream=upstream, downstream=downstream, @@ -1283,16 +1281,16 @@ def set_task_instance_state( include_upstream=False, ) - if execution_date is None: + if logical_date is None: dag_run = session.scalars( select(DagRun).where(DagRun.run_id == run_id, DagRun.dag_id == self.dag_id) ).one() # Raises an error if not found - resolve_execution_date = dag_run.execution_date + resolve_logical_date = dag_run.logical_date else: - resolve_execution_date = execution_date + resolve_logical_date = logical_date - end_date = resolve_execution_date if not future else None - start_date = resolve_execution_date if not past else None + end_date = resolve_logical_date if not future else None + start_date = resolve_logical_date if not past else None subdag.clear( start_date=start_date, @@ -1310,7 +1308,7 @@ def set_task_group_state( self, *, group_id: str, - execution_date: datetime | None = None, + logical_date: datetime | None = None, run_id: str | None = None, state: TaskInstanceState, upstream: bool = False, @@ -1324,7 +1322,7 @@ def set_task_group_state( Set TaskGroup to the given state and clear downstream tasks in failed or upstream_failed state. :param group_id: The group_id of the TaskGroup - :param execution_date: Execution date of the TaskInstance + :param logical_date: Logical date of the TaskInstance :param run_id: The run_id of the TaskInstance :param state: State to set the TaskInstance to :param upstream: Include all upstream tasks of the given task_id @@ -1336,22 +1334,22 @@ def set_task_group_state( """ from airflow.api.common.mark_tasks import set_state - if not exactly_one(execution_date, run_id): - raise ValueError("Exactly one of execution_date or run_id must be provided") + if not exactly_one(logical_date, run_id): + raise ValueError("Exactly one of logical_date or run_id must be provided") tasks_to_set_state: list[BaseOperator | tuple[BaseOperator, int]] = [] task_ids: list[str] = [] - if execution_date is None: + if logical_date is None: dag_run = session.scalars( select(DagRun).where(DagRun.run_id == run_id, DagRun.dag_id == self.dag_id) ).one() # Raises an error if not found - resolve_execution_date = dag_run.execution_date + resolve_logical_date = dag_run.logical_date else: - resolve_execution_date = execution_date + resolve_logical_date = logical_date - end_date = resolve_execution_date if not future else None - start_date = resolve_execution_date if not past else None + end_date = resolve_logical_date if not future else None + start_date = resolve_logical_date if not past else None task_group_dict = self.task_group.get_task_group_dict() task_group = task_group_dict.get(group_id) @@ -1361,17 +1359,17 @@ def set_task_group_state( task_ids = [task.task_id for task in task_group.iter_tasks()] dag_runs_query = select(DagRun.id).where(DagRun.dag_id == self.dag_id) if start_date is None and end_date is None: - dag_runs_query = dag_runs_query.where(DagRun.execution_date == start_date) + dag_runs_query = dag_runs_query.where(DagRun.logical_date == start_date) else: if start_date is not None: - dag_runs_query = dag_runs_query.where(DagRun.execution_date >= start_date) + dag_runs_query = dag_runs_query.where(DagRun.logical_date >= start_date) if end_date is not None: - dag_runs_query = dag_runs_query.where(DagRun.execution_date <= end_date) + dag_runs_query = dag_runs_query.where(DagRun.logical_date <= end_date) with lock_rows(dag_runs_query, session): altered = set_state( tasks=tasks_to_set_state, - execution_date=execution_date, + logical_date=logical_date, run_id=run_id, upstream=upstream, downstream=downstream, @@ -1423,8 +1421,8 @@ def clear( Clear a set of task instances associated with the current dag for a specified date range. :param task_ids: List of task ids or (``task_id``, ``map_index``) tuples to clear - :param start_date: The minimum execution_date to clear - :param end_date: The maximum execution_date to clear + :param start_date: The minimum logical_date to clear + :param end_date: The maximum logical_date to clear :param only_failed: Only clear failed tasks :param only_running: Only clear running tasks. :param confirm_prompt: Ask for confirmation @@ -1550,7 +1548,7 @@ def cli(self): @provide_session def test( self, - execution_date: datetime | None = None, + logical_date: datetime | None = None, run_conf: dict[str, Any] | None = None, conn_file_path: str | None = None, variable_file_path: str | None = None, @@ -1559,9 +1557,9 @@ def test( session: Session = NEW_SESSION, ) -> DagRun: """ - Execute one single DagRun for a given DAG and execution date. + Execute one single DagRun for a given DAG and logical date. - :param execution_date: execution date for the DAG run + :param logical_date: logical date for the DAG run :param run_conf: configuration to pass to newly created dagrun :param conn_file_path: file path to a connection file in either yaml or json :param variable_file_path: file path to a variable file in either yaml or json @@ -1599,23 +1597,23 @@ def add_logger_if_needed(ti: TaskInstance): exit_stack.callback(lambda: secrets_backend_list.pop(0)) with exit_stack: - execution_date = execution_date or timezone.utcnow() + logical_date = logical_date or timezone.utcnow() self.validate() - self.log.debug("Clearing existing task instances for execution date %s", execution_date) + self.log.debug("Clearing existing task instances for logical date %s", logical_date) self.clear( - start_date=execution_date, - end_date=execution_date, + start_date=logical_date, + end_date=logical_date, dag_run_state=False, # type: ignore session=session, ) self.log.debug("Getting dagrun for dag %s", self.dag_id) - logical_date = timezone.coerce_datetime(execution_date) + logical_date = timezone.coerce_datetime(logical_date) data_interval = self.timetable.infer_manual_data_interval(run_after=logical_date) dr: DagRun = _get_or_create_dagrun( dag=self, - start_date=execution_date, - execution_date=execution_date, - run_id=DagRun.generate_run_id(DagRunType.MANUAL, execution_date), + start_date=logical_date, + logical_date=logical_date, + run_id=DagRun.generate_run_id(DagRunType.MANUAL, logical_date), session=session, conf=run_conf, triggered_by=DagRunTriggeredByType.TEST, @@ -1699,8 +1697,8 @@ def create_dagrun( self, state: DagRunState, *, - triggered_by: DagRunTriggeredByType, - execution_date: datetime | None = None, + triggered_by: DagRunTriggeredByType | None, + logical_date: datetime | None = None, run_id: str | None = None, start_date: datetime | None = None, external_trigger: bool | None = False, @@ -1721,7 +1719,7 @@ def create_dagrun( :param triggered_by: The entity which triggers the DagRun :param run_id: defines the run id for this dag run :param run_type: type of DagRun - :param execution_date: the execution date of this dag run + :param logical_date: the logical date of this dag run :param start_date: the date this dag run should be evaluated :param external_trigger: whether this dag run is externally triggered :param conf: Dict containing configuration/parameters to pass to the DAG @@ -1731,7 +1729,7 @@ def create_dagrun( :param data_interval: Data interval of the DagRun :param backfill_id: id of the backfill run if one exists """ - logical_date = timezone.coerce_datetime(execution_date) + logical_date = timezone.coerce_datetime(logical_date) if data_interval and not isinstance(data_interval, DataInterval): data_interval = DataInterval(*map(timezone.coerce_datetime, data_interval)) @@ -1761,13 +1759,13 @@ def create_dagrun( f"A {run_type.value} DAG run cannot use ID {run_id!r} since it " f"is reserved for {inferred_run_type.value} runs" ) - elif run_type and logical_date is not None: # Generate run_id from run_type and execution_date. + elif run_type and logical_date is not None: # Generate run_id from run_type and logical_date. run_id = self.timetable.generate_run_id( run_type=run_type, logical_date=logical_date, data_interval=data_interval ) else: raise AirflowException( - "Creating DagRun needs either `run_id` or both `run_type` and `execution_date`" + "Creating DagRun needs either `run_id` or both `run_type` and `logical_date`" ) regex = airflow_conf.get("scheduler", "allowed_run_id_pattern") @@ -2427,7 +2425,7 @@ def _get_or_create_dagrun( dag: DAG, conf: dict[Any, Any] | None, start_date: datetime, - execution_date: datetime, + logical_date: datetime, run_id: str, session: Session, triggered_by: DagRunTriggeredByType, @@ -2441,7 +2439,7 @@ def _get_or_create_dagrun( :param dag: DAG to be used to find run. :param conf: Configuration to pass to newly created run. :param start_date: Start date of new run. - :param execution_date: Logical date for finding an existing run. + :param logical_date: Logical date for finding an existing run. :param run_id: Run ID for the new DAG run. :param triggered_by: the entity which triggers the dag_run @@ -2449,7 +2447,7 @@ def _get_or_create_dagrun( """ log.info("dagrun id: %s", dag.dag_id) dr: DagRun = session.scalar( - select(DagRun).where(DagRun.dag_id == dag.dag_id, DagRun.execution_date == execution_date) + select(DagRun).where(DagRun.dag_id == dag.dag_id, DagRun.logical_date == logical_date) ) if dr: session.delete(dr) @@ -2457,9 +2455,9 @@ def _get_or_create_dagrun( dag_version = DagVersion.get_latest_version(dag.dag_id, session=session) dr = dag.create_dagrun( state=DagRunState.RUNNING, - execution_date=execution_date, + logical_date=logical_date, run_id=run_id, - start_date=start_date or execution_date, + start_date=start_date or logical_date, session=session, conf=conf, data_interval=data_interval, diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py index ccc4832f5fb6..a2327221ad5d 100644 --- a/airflow/models/dagrun.py +++ b/airflow/models/dagrun.py @@ -129,7 +129,7 @@ class DagRun(Base, LoggingMixin): id = Column(Integer, primary_key=True) dag_id = Column(StringID(), nullable=False) queued_at = Column(UtcDateTime) - execution_date = Column("logical_date", UtcDateTime, default=timezone.utcnow, nullable=False) + logical_date = Column(UtcDateTime, default=timezone.utcnow, nullable=False) start_date = Column(UtcDateTime) end_date = Column(UtcDateTime) _state = Column("state", String(50), default=DagRunState.QUEUED) @@ -228,7 +228,7 @@ def __init__( dag_id: str | None = None, run_id: str | None = None, queued_at: datetime | None | ArgNotSet = NOTSET, - execution_date: datetime | None = None, + logical_date: datetime | None = None, start_date: datetime | None = None, external_trigger: bool | None = None, conf: Any | None = None, @@ -248,7 +248,7 @@ def __init__( self.dag_id = dag_id self.run_id = run_id - self.execution_date = execution_date + self.logical_date = logical_date self.start_date = start_date self.external_trigger = external_trigger self.conf = conf or {} @@ -268,7 +268,7 @@ def __init__( def __repr__(self): return ( - f"" ) @@ -287,10 +287,6 @@ def validate_run_id(self, key: str, run_id: str) -> str | None: def stats_tags(self) -> dict[str, str]: return prune_dict({"dag_id": self.dag_id, "run_type": self.run_type}) - @property - def logical_date(self) -> datetime: - return self.execution_date - def get_state(self): return self._state @@ -437,13 +433,13 @@ def get_running_dag_runs_to_examine(cls, session: Session) -> Query: .order_by( nulls_first(BackfillDagRun.sort_ordinal, session=session), nulls_first(cls.last_scheduling_decision, session=session), - cls.execution_date, + cls.logical_date, ) .limit(cls.DEFAULT_DAGRUNS_TO_EXAMINE) ) - if not settings.ALLOW_FUTURE_EXEC_DATES: - query = query.where(DagRun.execution_date <= func.now()) + if not settings.ALLOW_FUTURE_LOGICAL_DATES: + query = query.where(DagRun.logical_date <= func.now()) return session.scalars(with_row_locks(query, of=cls, session=session, skip_locked=True)) @@ -523,13 +519,13 @@ def get_queued_dag_runs_to_set_running(cls, session: Session) -> Query: nulls_first(BackfillDagRun.sort_ordinal, session=session), nulls_first(cls.last_scheduling_decision, session=session), nulls_first(running_drs.c.num_running, session=session), # many running -> lower priority - cls.execution_date, + cls.logical_date, ) .limit(cls.DEFAULT_DAGRUNS_TO_EXAMINE) ) - if not settings.ALLOW_FUTURE_EXEC_DATES: - query = query.where(DagRun.execution_date <= func.now()) + if not settings.ALLOW_FUTURE_LOGICAL_DATES: + query = query.where(DagRun.logical_date <= func.now()) return session.scalars(with_row_locks(query, of=cls, session=session, skip_locked=True)) @@ -539,14 +535,14 @@ def find( cls, dag_id: str | list[str] | None = None, run_id: Iterable[str] | None = None, - execution_date: datetime | Iterable[datetime] | None = None, + logical_date: datetime | Iterable[datetime] | None = None, state: DagRunState | None = None, external_trigger: bool | None = None, no_backfills: bool = False, run_type: DagRunType | None = None, session: Session = NEW_SESSION, - execution_start_date: datetime | None = None, - execution_end_date: datetime | None = None, + logical_start_date: datetime | None = None, + logical_end_date: datetime | None = None, ) -> list[DagRun]: """ Return a set of dag runs for the given search criteria. @@ -554,14 +550,14 @@ def find( :param dag_id: the dag_id or list of dag_id to find dag runs for :param run_id: defines the run id for this dag run :param run_type: type of DagRun - :param execution_date: the execution date + :param logical_date: the logical date :param state: the state of the dag run :param external_trigger: whether this dag run is externally triggered :param no_backfills: return no backfills (True), return all (False). Defaults to False :param session: database session - :param execution_start_date: dag run that was executed from this date - :param execution_end_date: dag run that was executed until this date + :param logical_start_date: dag run that was executed from this date + :param logical_end_date: dag run that was executed until this date """ qry = select(cls) dag_ids = [dag_id] if isinstance(dag_id, str) else dag_id @@ -572,16 +568,16 @@ def find( qry = qry.where(cls.run_id.in_(run_id)) elif run_id is not None: qry = qry.where(cls.run_id == run_id) - if is_container(execution_date): - qry = qry.where(cls.execution_date.in_(execution_date)) - elif execution_date is not None: - qry = qry.where(cls.execution_date == execution_date) - if execution_start_date and execution_end_date: - qry = qry.where(cls.execution_date.between(execution_start_date, execution_end_date)) - elif execution_start_date: - qry = qry.where(cls.execution_date >= execution_start_date) - elif execution_end_date: - qry = qry.where(cls.execution_date <= execution_end_date) + if is_container(logical_date): + qry = qry.where(cls.logical_date.in_(logical_date)) + elif logical_date is not None: + qry = qry.where(cls.logical_date == logical_date) + if logical_start_date and logical_end_date: + qry = qry.where(cls.logical_date.between(logical_start_date, logical_end_date)) + elif logical_start_date: + qry = qry.where(cls.logical_date >= logical_start_date) + elif logical_end_date: + qry = qry.where(cls.logical_date <= logical_end_date) if state: qry = qry.where(cls.state == state) if external_trigger is not None: @@ -591,7 +587,7 @@ def find( if no_backfills: qry = qry.where(cls.run_type != DagRunType.BACKFILL_JOB) - return session.scalars(qry.order_by(cls.execution_date)).all() + return session.scalars(qry.order_by(cls.logical_date)).all() @classmethod @provide_session @@ -599,31 +595,29 @@ def find_duplicate( cls, dag_id: str, run_id: str, - execution_date: datetime, + logical_date: datetime, session: Session = NEW_SESSION, ) -> DagRun | None: """ - Return an existing run for the DAG with a specific run_id or execution_date. - - *None* is returned if no such DAG run is found. + Return an existing run for the DAG with a specific run_id or logical date. :param dag_id: the dag_id to find duplicates for :param run_id: defines the run id for this dag run - :param execution_date: the execution date + :param logical_date: the logical date :param session: database session """ return session.scalars( select(cls).where( cls.dag_id == dag_id, - or_(cls.run_id == run_id, cls.execution_date == execution_date), + or_(cls.run_id == run_id, cls.logical_date == logical_date), ) ).one_or_none() @staticmethod - def generate_run_id(run_type: DagRunType, execution_date: datetime) -> str: - """Generate Run ID based on Run Type and Execution Date.""" + def generate_run_id(run_type: DagRunType, logical_date: datetime) -> str: + """Generate Run ID based on Run Type and Logical Date.""" # _Ensure_ run_type is a DagRunType, not just a string from user code - return DagRunType(run_type).generate_run_id(execution_date) + return DagRunType(run_type).generate_run_id(logical_date) @staticmethod @internal_api_call @@ -669,7 +663,7 @@ def _check_last_n_dagruns_failed(self, dag_id, max_consecutive_failed_dag_runs, dag_runs = ( session.query(DagRun) .filter(DagRun.dag_id == dag_id) - .order_by(DagRun.execution_date.desc()) + .order_by(DagRun.logical_date.desc()) .limit(max_consecutive_failed_dag_runs) .all() ) @@ -797,11 +791,11 @@ def get_previous_dagrun( """ filters = [ DagRun.dag_id == dag_run.dag_id, - DagRun.execution_date < dag_run.execution_date, + DagRun.logical_date < dag_run.logical_date, ] if state is not None: filters.append(DagRun.state == state) - return session.scalar(select(DagRun).where(*filters).order_by(DagRun.execution_date.desc()).limit(1)) + return session.scalar(select(DagRun).where(*filters).order_by(DagRun.logical_date.desc()).limit(1)) @staticmethod @internal_api_call @@ -821,10 +815,10 @@ def get_previous_scheduled_dagrun( select(DagRun) .where( DagRun.dag_id == dag_run.dag_id, - DagRun.execution_date < dag_run.execution_date, + DagRun.logical_date < dag_run.logical_date, DagRun.run_type != DagRunType.MANUAL, ) - .order_by(DagRun.execution_date.desc()) + .order_by(DagRun.logical_date.desc()) .limit(1) ) @@ -994,7 +988,7 @@ def recalculate(self) -> _UnfinishedStates: if self._state == DagRunState.FAILED or self._state == DagRunState.SUCCESS: msg = ( - "DagRun Finished: dag_id=%s, execution_date=%s, run_id=%s, " + "DagRun Finished: dag_id=%s, logical_date=%s, run_id=%s, " "run_start_date=%s, run_end_date=%s, run_duration=%s, " "state=%s, external_trigger=%s, run_type=%s, " "data_interval_start=%s, data_interval_end=%s, dag_version_name=%s" @@ -1003,7 +997,7 @@ def recalculate(self) -> _UnfinishedStates: self.log.info( msg, self.dag_id, - self.execution_date, + self.logical_date, self.run_id, self.start_date, self.end_date, @@ -1039,7 +1033,7 @@ def _trace_dagrun(self, dagv) -> None: attributes = { "category": "DAG runs", "dag_id": self.dag_id, - "execution_date": str(self.execution_date), + "logical_date": str(self.logical_date), "run_id": self.run_id, "queued_at": str(self.queued_at), "run_start_date": str(self.start_date), @@ -1330,8 +1324,8 @@ def verify_integrity(self, *, session: Session = NEW_SESSION) -> None: def task_filter(task: Operator) -> bool: return task.task_id not in task_ids and ( self.run_type == DagRunType.BACKFILL_JOB - or (task.start_date is None or task.start_date <= self.execution_date) - and (task.end_date is None or self.execution_date <= task.end_date) + or (task.start_date is None or task.start_date <= self.logical_date) + and (task.end_date is None or self.logical_date <= task.end_date) ) created_counts: dict[str, int] = defaultdict(int) @@ -1607,14 +1601,14 @@ def _revise_map_indexes_if_mapped(self, task: Operator, *, session: Session) -> def get_latest_runs(cls, session: Session = NEW_SESSION) -> list[DagRun]: """Return the latest DagRun for each DAG.""" subquery = ( - select(cls.dag_id, func.max(cls.execution_date).label("execution_date")) + select(cls.dag_id, func.max(cls.logical_date).label("logical_date")) .group_by(cls.dag_id) .subquery() ) return session.scalars( select(cls).join( subquery, - and_(cls.dag_id == subquery.c.dag_id, cls.execution_date == subquery.c.execution_date), + and_(cls.dag_id == subquery.c.dag_id, cls.logical_date == subquery.c.logical_date), ) ).all() diff --git a/airflow/models/log.py b/airflow/models/log.py index e01cc29b45e0..8669d228b250 100644 --- a/airflow/models/log.py +++ b/airflow/models/log.py @@ -41,7 +41,7 @@ class Log(Base): task_id = Column(StringID()) map_index = Column(Integer) event = Column(String(60)) - execution_date = Column(UtcDateTime) + logical_date = Column(UtcDateTime) run_id = Column(StringID()) owner = Column(String(500)) owner_display_name = Column(String(500)) @@ -69,13 +69,13 @@ def __init__( task_owner = None - self.execution_date = None + self.logical_date = None if task_instance: self.dag_id = task_instance.dag_id self.task_id = task_instance.task_id - if execution_date := getattr(task_instance, "execution_date", None): - self.execution_date = execution_date self.run_id = task_instance.run_id + if logical_date := getattr(task_instance, "logical_date", None): + self.logical_date = logical_date self.try_number = task_instance.try_number self.map_index = task_instance.map_index if task := getattr(task_instance, "task", None): @@ -85,8 +85,8 @@ def __init__( self.task_id = kwargs["task_id"] if "dag_id" in kwargs: self.dag_id = kwargs["dag_id"] - if kwargs.get("execution_date"): - self.execution_date = kwargs["execution_date"] + if kwargs.get("logical_date"): + self.logical_date = kwargs["logical_date"] if kwargs.get("run_id"): self.run_id = kwargs["run_id"] if "map_index" in kwargs: diff --git a/airflow/models/renderedtifields.py b/airflow/models/renderedtifields.py index 5b0b2bef52d3..b8ab93cf41a7 100644 --- a/airflow/models/renderedtifields.py +++ b/airflow/models/renderedtifields.py @@ -104,7 +104,7 @@ class RenderedTaskInstanceFields(TaskInstanceDependencies): ) # We don't need a DB level FK here, as we already have that to TI (which has one to DR) but by defining - # the relationship we can more easily find the execution date for these rows + # the relationship we can more easily find the logical date for these rows dag_run = relationship( "DagRun", primaryjoin="""and_( @@ -114,7 +114,7 @@ class RenderedTaskInstanceFields(TaskInstanceDependencies): viewonly=True, ) - execution_date = association_proxy("dag_run", "execution_date") + logical_date = association_proxy("dag_run", "logical_date") def __init__(self, ti: TaskInstance, render_templates=True, rendered_fields=None): self.dag_id = ti.dag_id @@ -254,11 +254,11 @@ def delete_old_records( from airflow.models.dagrun import DagRun tis_to_keep_query = ( - select(cls.dag_id, cls.task_id, cls.run_id, DagRun.execution_date) + select(cls.dag_id, cls.task_id, cls.run_id, DagRun.logical_date) .where(cls.dag_id == dag_id, cls.task_id == task_id) .join(cls.dag_run) .distinct() - .order_by(DagRun.execution_date.desc()) + .order_by(DagRun.logical_date.desc()) .limit(num_to_keep) ) diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index 30ef941ceea5..7a2f451727a7 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -292,11 +292,11 @@ def _run_raw_task( raise ti.defer_task(exception=defer, session=session) ti.log.info( - "Pausing task as DEFERRED. dag_id=%s, task_id=%s, run_id=%s, execution_date=%s, start_date=%s", + "Pausing task as DEFERRED. dag_id=%s, task_id=%s, run_id=%s, logical_date=%s, start_date=%s", ti.dag_id, ti.task_id, ti.run_id, - _date_or_empty(task_instance=ti, attr="execution_date"), + _date_or_empty(task_instance=ti, attr="logical_date"), _date_or_empty(task_instance=ti, attr="start_date"), ) return TaskReturnCode.DEFERRED @@ -577,7 +577,7 @@ def _xcom_pull( If *None* (default), this is inferred from the task(s) being pulled (see below for details). :param include_prior_dates: If False, only XComs from the current - execution_date are returned. If *True*, XComs from previous dates + logical_date are returned. If *True*, XComs from previous dates are returned as well. :param run_id: If provided, only pulls XComs from a DagRun w/a matching run_id. If *None* (default), the run_id of the calling task is used. @@ -821,11 +821,11 @@ def _set_ti_attrs(target, source, include_dag_run=False): target.dag_version_id = source.dag_version_id if include_dag_run: - target.execution_date = source.execution_date + target.logical_date = source.logical_date target.dag_run.id = source.dag_run.id target.dag_run.dag_id = source.dag_run.dag_id target.dag_run.queued_at = source.dag_run.queued_at - target.dag_run.execution_date = source.dag_run.execution_date + target.dag_run.logical_date = source.dag_run.logical_date target.dag_run.start_date = source.dag_run.start_date target.dag_run.end_date = source.dag_run.end_date target.dag_run.state = source.dag_run.state @@ -967,7 +967,7 @@ def _get_template_context( validated_params = process_params(dag, task, dag_run, suppress_exception=ignore_param_exceptions) - logical_date: DateTime = timezone.coerce_datetime(task_instance.execution_date) + logical_date: DateTime = timezone.coerce_datetime(task_instance.logical_date) ds = logical_date.strftime("%Y-%m-%d") ds_nodash = ds.replace("-", "") ts = logical_date.isoformat() @@ -1008,76 +1008,6 @@ def get_prev_end_date_success() -> pendulum.DateTime | None: return None return timezone.coerce_datetime(dagrun.end_date) - @cache - def get_yesterday_ds() -> str: - return (logical_date - timedelta(1)).strftime("%Y-%m-%d") - - def get_yesterday_ds_nodash() -> str: - return get_yesterday_ds().replace("-", "") - - @cache - def get_tomorrow_ds() -> str: - return (logical_date + timedelta(1)).strftime("%Y-%m-%d") - - def get_tomorrow_ds_nodash() -> str: - return get_tomorrow_ds().replace("-", "") - - @cache - def get_next_execution_date() -> pendulum.DateTime | None: - # For manually triggered dagruns that aren't run on a schedule, - # the "next" execution date doesn't make sense, and should be set - # to execution date for consistency with how execution_date is set - # for manually triggered tasks, i.e. triggered_date == execution_date. - if dag_run.external_trigger: - return logical_date - if dag is None: - return None - next_info = dag.next_dagrun_info(data_interval, restricted=False) - if next_info is None: - return None - return timezone.coerce_datetime(next_info.logical_date) - - def get_next_ds() -> str | None: - execution_date = get_next_execution_date() - if execution_date is None: - return None - return execution_date.strftime("%Y-%m-%d") - - def get_next_ds_nodash() -> str | None: - ds = get_next_ds() - if ds is None: - return ds - return ds.replace("-", "") - - @cache - def get_prev_execution_date(): - # For manually triggered dagruns that aren't run on a schedule, - # the "previous" execution date doesn't make sense, and should be set - # to execution date for consistency with how execution_date is set - # for manually triggered tasks, i.e. triggered_date == execution_date. - if dag_run.external_trigger: - return logical_date - - # Workaround code copy until deprecated context fields are removed in Airflow 3 - from airflow.timetables.interval import _DataIntervalTimetable - - if not isinstance(dag.timetable, _DataIntervalTimetable): - return None - return dag.timetable._get_prev(timezone.coerce_datetime(logical_date)) - - @cache - def get_prev_ds() -> str | None: - execution_date = get_prev_execution_date() - if execution_date is None: - return None - return execution_date.strftime("%Y-%m-%d") - - def get_prev_ds_nodash() -> str | None: - prev_ds = get_prev_ds() - if prev_ds is None: - return None - return prev_ds.replace("-", "") - def get_triggering_events() -> dict[str, list[AssetEvent | AssetEventPydantic]]: if TYPE_CHECKING: assert session is not None @@ -1114,27 +1044,16 @@ def get_triggering_events() -> dict[str, list[AssetEvent | AssetEventPydantic]]: "outlet_events": OutletEventAccessors(), "ds": ds, "ds_nodash": ds_nodash, - "execution_date": logical_date, "expanded_ti_count": expanded_ti_count, "inlets": task.inlets, "inlet_events": InletEventsAccessors(task.inlets, session=session), "logical_date": logical_date, "macros": macros, "map_index_template": task.map_index_template, - "next_ds": get_next_ds(), - "next_ds_nodash": get_next_ds_nodash(), - "next_execution_date": get_next_execution_date(), "outlets": task.outlets, "params": validated_params, "prev_data_interval_start_success": get_prev_data_interval_start_success(), "prev_data_interval_end_success": get_prev_data_interval_end_success(), - "prev_ds": get_prev_ds(), - "prev_ds_nodash": get_prev_ds_nodash(), - "prev_execution_date": get_prev_execution_date(), - "prev_execution_date_success": task_instance.get_previous_execution_date( - state=DagRunState.SUCCESS, - session=session, - ), "prev_start_date_success": get_prev_start_date_success(), "prev_end_date_success": get_prev_end_date_success(), "run_id": task_instance.run_id, @@ -1143,8 +1062,6 @@ def get_triggering_events() -> dict[str, list[AssetEvent | AssetEventPydantic]]: "task_instance_key_str": f"{task.dag_id}__{task.task_id}__{ds_nodash}", "test_mode": task_instance.test_mode, "ti": task_instance, - "tomorrow_ds": get_tomorrow_ds(), - "tomorrow_ds_nodash": get_tomorrow_ds_nodash(), "triggering_asset_events": lazy_object_proxy.Proxy(get_triggering_events), "ts": ts, "ts_nodash": ts_nodash, @@ -1154,8 +1071,6 @@ def get_triggering_events() -> dict[str, list[AssetEvent | AssetEventPydantic]]: "value": VariableAccessor(deserialize_json=False), }, "conn": ConnectionAccessor(), - "yesterday_ds": get_yesterday_ds(), - "yesterday_ds_nodash": get_yesterday_ds_nodash(), } # Mypy doesn't like turning existing dicts in to a TypeDict -- and we "lie" in the type stub to say it # is one, but in practice it isn't. See https://github.com/python/mypy/issues/8890 @@ -1257,12 +1172,11 @@ def _handle_failure( "end_date": str(task_instance.end_date), "duration": task_instance.duration, "executor_config": str(task_instance.executor_config), - "execution_date": str(task_instance.execution_date), + "logical_date": str(task_instance.logical_date), "hostname": task_instance.hostname, "operator": str(task_instance.operator), } ) - if isinstance(task_instance, TaskInstance): span.set_attribute("log_url", task_instance.log_url) @@ -1389,14 +1303,14 @@ def _get_previous_dagrun( return None -def _get_previous_execution_date( +def _get_previous_logical_date( *, task_instance: TaskInstance | TaskInstancePydantic, state: DagRunState | None, session: Session, ) -> pendulum.DateTime | None: """ - Get execution date from property previous_ti_success. + Get logical date from property previous_ti_success. :param task_instance: the task instance :param session: SQLAlchemy ORM Session @@ -1404,9 +1318,9 @@ def _get_previous_execution_date( :meta private: """ - log.debug("previous_execution_date was called") + log.debug("previous_logical_date was called") prev_ti = task_instance.get_previous_ti(state=state, session=session) - return pendulum.instance(prev_ti.execution_date) if prev_ti and prev_ti.execution_date else None + return pendulum.instance(prev_ti.logical_date) if prev_ti and prev_ti.logical_date else None def _get_previous_start_date( @@ -1592,11 +1506,11 @@ def _log_state(*, task_instance: TaskInstance | TaskInstancePydantic, lead_msg: if task_instance.map_index >= 0: params.append(task_instance.map_index) message += "map_index=%d, " - message += "execution_date=%s, start_date=%s, end_date=%s" + message += "logical_date=%s, start_date=%s, end_date=%s" log.info( message, *params, - _date_or_empty(task_instance=task_instance, attr="execution_date"), + _date_or_empty(task_instance=task_instance, attr="logical_date"), _date_or_empty(task_instance=task_instance, attr="start_date"), _date_or_empty(task_instance=task_instance, attr="end_date"), stacklevel=2, @@ -1920,7 +1834,7 @@ class TaskInstance(Base, LoggingMixin): triggerer_job = association_proxy("trigger", "triggerer_job") dag_run = relationship("DagRun", back_populates="task_instances", lazy="joined", innerjoin=True) rendered_task_instance_fields = relationship("RenderedTaskInstanceFields", lazy="noload", uselist=False) - execution_date = association_proxy("dag_run", "execution_date") + logical_date = association_proxy("dag_run", "logical_date") task_instance_note = relationship( "TaskInstanceNote", back_populates="task_instance", @@ -2183,7 +2097,7 @@ def generate_command( def log_url(self) -> str: """Log URL for TaskInstance.""" run_id = quote(self.run_id) - base_date = quote(self.execution_date.strftime("%Y-%m-%dT%H:%M:%S%z")) + base_date = quote(self.logical_date.strftime("%Y-%m-%dT%H:%M:%S%z")) base_url = conf.get_mandatory_value("webserver", "BASE_URL") map_index = f"&map_index={self.map_index}" if self.map_index >= 0 else "" return ( @@ -2432,18 +2346,18 @@ def get_previous_ti( return _get_previous_ti(task_instance=self, state=state, session=session) @provide_session - def get_previous_execution_date( + def get_previous_logical_date( self, state: DagRunState | None = None, session: Session = NEW_SESSION, ) -> pendulum.DateTime | None: """ - Return the execution date from property previous_ti_success. + Return the logical date from property previous_ti_success. :param state: If passed, it only take into account instances of a specific state. :param session: SQLAlchemy ORM Session """ - return _get_previous_execution_date(task_instance=self, state=state, session=session) + return _get_previous_logical_date(task_instance=self, state=state, session=session) @provide_session def get_previous_start_date( @@ -2542,7 +2456,7 @@ def next_retry_datetime(self): # deterministic per task instance ti_hash = int( hashlib.sha1( - f"{self.dag_id}#{self.task_id}#{self.execution_date}#{self.try_number}".encode() + f"{self.dag_id}#{self.task_id}#{self.logical_date}#{self.try_number}".encode() ).hexdigest(), 16, ) @@ -2592,7 +2506,7 @@ def get_dagrun(self, session: Session = NEW_SESSION) -> DagRun: if TYPE_CHECKING: assert self.task dr.dag = self.task.dag - # Record it in the instance for next time. This means that `self.execution_date` will work correctly + # Record it in the instance for next time. This means that `self.logical_date` will work correctly set_committed_value(self, "dag_run", dr) return dr @@ -2757,9 +2671,9 @@ def _check_and_change_state_before_execution( settings.engine.dispose() # type: ignore if verbose: if mark_success: - cls.logger().info("Marking success for %s on %s", ti.task, ti.execution_date) + cls.logger().info("Marking success for %s on %s", ti.task, ti.logical_date) else: - cls.logger().info("Executing %s on %s", ti.task, ti.execution_date) + cls.logger().info("Executing %s on %s", ti.task, ti.logical_date) return True @provide_session @@ -3483,7 +3397,7 @@ def xcom_pull( If *None* (default), this is inferred from the task(s) being pulled (see below for details). :param include_prior_dates: If False, only XComs from the current - execution_date are returned. If *True*, XComs from previous dates + logical_date are returned. If *True*, XComs from previous dates are returned as well. :param run_id: If provided, only pulls XComs from a DagRun w/a matching run_id. If *None* (default), the run_id of the calling task is used. diff --git a/airflow/models/taskreschedule.py b/airflow/models/taskreschedule.py index 2fc06b798a67..aa987294e8ee 100644 --- a/airflow/models/taskreschedule.py +++ b/airflow/models/taskreschedule.py @@ -75,7 +75,7 @@ class TaskReschedule(TaskInstanceDependencies): ), ) dag_run = relationship("DagRun") - execution_date = association_proxy("dag_run", "execution_date") + logical_date = association_proxy("dag_run", "logical_date") def __init__( self, diff --git a/airflow/models/xcom.py b/airflow/models/xcom.py index 87c72d5bf7f5..69843db5ab5e 100644 --- a/airflow/models/xcom.py +++ b/airflow/models/xcom.py @@ -111,7 +111,7 @@ class BaseXCom(TaskInstanceDependencies, LoggingMixin): lazy="joined", passive_deletes="all", ) - execution_date = association_proxy("dag_run", "execution_date") + logical_date = association_proxy("dag_run", "logical_date") @reconstructor def init_on_load(self): @@ -374,12 +374,12 @@ def get_many( query = query.filter(BaseXCom.map_index == map_indexes) if include_prior_dates: - dr = session.query(DagRun.execution_date).filter(DagRun.run_id == run_id).subquery() - query = query.filter(BaseXCom.execution_date <= dr.c.execution_date) + dr = session.query(DagRun.logical_date).filter(DagRun.run_id == run_id).subquery() + query = query.filter(BaseXCom.logical_date <= dr.c.logical_date) else: query = query.filter(BaseXCom.run_id == run_id) - query = query.order_by(DagRun.execution_date.desc(), BaseXCom.timestamp.desc()) + query = query.order_by(DagRun.logical_date.desc(), BaseXCom.timestamp.desc()) if limit: return query.limit(limit) return query @@ -424,7 +424,7 @@ def clear( :param session: Database session. If not given, a new session will be created for this function. """ - # Given the historic order of this function (execution_date was first argument) to add a new optional + # Given the historic order of this function (logical_date was first argument) to add a new optional # param we need to add default values for everything :( if dag_id is None: raise TypeError("clear() missing required argument: dag_id") diff --git a/airflow/operators/trigger_dagrun.py b/airflow/operators/trigger_dagrun.py index bb1eac7c6963..e65a74ef2b9a 100644 --- a/airflow/operators/trigger_dagrun.py +++ b/airflow/operators/trigger_dagrun.py @@ -195,7 +195,7 @@ def execute(self, context: Context): dag_id=self.trigger_dag_id, run_id=run_id, conf=self.conf, - execution_date=parsed_logical_date, + logical_date=parsed_logical_date, replace_microseconds=False, triggered_by=DagRunTriggeredByType.OPERATOR, ) @@ -234,7 +234,7 @@ def execute(self, context: Context): trigger=DagStateTrigger( dag_id=self.trigger_dag_id, states=self.allowed_states + self.failed_states, - execution_dates=[dag_run.logical_date], + logical_dates=[dag_run.logical_date], poll_interval=self.poke_interval, ), method_name="execute_complete", @@ -261,12 +261,12 @@ def execute(self, context: Context): @provide_session def execute_complete(self, context: Context, session: Session, event: tuple[str, dict[str, Any]]): # This logical_date is parsed from the return trigger event - provided_logical_date = event[1]["execution_dates"][0] + provided_logical_date = event[1]["logical_dates"][0] try: # Note: here execution fails on database isolation mode. Needs structural changes for AIP-72 dag_run = session.execute( select(DagRun).where( - DagRun.dag_id == self.trigger_dag_id, DagRun.execution_date == provided_logical_date + DagRun.dag_id == self.trigger_dag_id, DagRun.logical_date == provided_logical_date ) ).scalar_one() except NoResultFound: diff --git a/airflow/policies.py b/airflow/policies.py index 49691b393866..e0c48931a286 100644 --- a/airflow/policies.py +++ b/airflow/policies.py @@ -99,7 +99,7 @@ def get_airflow_context_vars(context) -> dict[str, str]: # type: ignore[empty-b This setting allows getting the airflow context vars, which are key value pairs. They are then injected to default airflow context vars, which in the end are available as environment variables when running - tasks dag_id, task_id, execution_date, dag_run_id, try_number are reserved keys. + tasks dag_id, task_id, logical_date, dag_run_id, try_number are reserved keys. :param context: The context for the task_instance of interest. """ diff --git a/airflow/sensors/external_task.py b/airflow/sensors/external_task.py index 331e17168bab..d5504ab2e9fb 100644 --- a/airflow/sensors/external_task.py +++ b/airflow/sensors/external_task.py @@ -70,7 +70,7 @@ def get_link(self, operator: BaseOperator, *, ti_key: TaskInstanceKey) -> str: ) query = { "dag_id": external_dag_id, - "execution_date": ti.execution_date.isoformat(), # type: ignore[union-attr] + "logical_date": ti.logical_date.isoformat(), # type: ignore[union-attr] } return build_airflow_url_with_query(query) @@ -351,7 +351,7 @@ def execute(self, context: Context) -> None: external_dag_id=self.external_dag_id, external_task_group_id=self.external_task_group_id, external_task_ids=self.external_task_ids, - execution_dates=self._get_dttm_filter(context), + logical_dates=self._get_dttm_filter(context), allowed_states=self.allowed_states, poke_interval=self.poll_interval, soft_fail=self.soft_fail, @@ -406,7 +406,7 @@ def get_count(self, dttm_filter, session, states) -> int: """ Get the count of records against dttm filter and states. - :param dttm_filter: date time filter for execution date + :param dttm_filter: date time filter for logical date :param session: airflow session object :param states: task or dag states :return: count of record against the filters @@ -436,7 +436,7 @@ def _handle_execution_date_fn(self, context) -> Any: Handle backward compatibility. This function is to handle backwards compatibility with how this operator was - previously where it only passes the execution date, but also allow for the newer + previously where it only passes the logical date, but also allow for the newer implementation to pass all context variables as keyword arguments, to allow for more sophisticated returns of dates to return. """ @@ -464,14 +464,14 @@ class ExternalTaskMarker(EmptyOperator): :param external_dag_id: The dag_id that contains the dependent task that needs to be cleared. :param external_task_id: The task_id of the dependent task that needs to be cleared. - :param execution_date: The logical date of the dependent task execution that needs to be cleared. + :param logical_date: The logical date of the dependent task execution that needs to be cleared. :param recursion_depth: The maximum level of transitive dependencies allowed. Default is 10. This is mostly used for preventing cyclic dependencies. It is fine to increase this number if necessary. However, too many levels of transitive dependencies will make it slower to clear tasks in the web UI. """ - template_fields = ["external_dag_id", "external_task_id", "execution_date"] + template_fields = ["external_dag_id", "external_task_id", "logical_date"] ui_color = "#4db7db" operator_extra_links = [ExternalDagLink()] @@ -483,20 +483,20 @@ def __init__( *, external_dag_id: str, external_task_id: str, - execution_date: str | datetime.datetime | None = "{{ logical_date.isoformat() }}", + logical_date: str | datetime.datetime | None = "{{ logical_date.isoformat() }}", recursion_depth: int = 10, **kwargs, ): super().__init__(**kwargs) self.external_dag_id = external_dag_id self.external_task_id = external_task_id - if isinstance(execution_date, datetime.datetime): - self.execution_date = execution_date.isoformat() - elif isinstance(execution_date, str): - self.execution_date = execution_date + if isinstance(logical_date, datetime.datetime): + self.logical_date = logical_date.isoformat() + elif isinstance(logical_date, str): + self.logical_date = logical_date else: raise TypeError( - f"Expected str or datetime.datetime type for execution_date. Got {type(execution_date)}" + f"Expected str or datetime.datetime type for logical_date. Got {type(logical_date)}" ) if recursion_depth <= 0: diff --git a/airflow/sentry.py b/airflow/sentry.py index d5fbf3c04d21..22261bd99fd2 100644 --- a/airflow/sentry.py +++ b/airflow/sentry.py @@ -62,7 +62,7 @@ def flush(self): class ConfiguredSentry(DummySentry): """Configure Sentry SDK.""" - SCOPE_DAG_RUN_TAGS = frozenset(("data_interval_end", "data_interval_start", "execution_date")) + SCOPE_DAG_RUN_TAGS = frozenset(("data_interval_end", "data_interval_start", "logical_date")) SCOPE_TASK_INSTANCE_TAGS = frozenset(("task_id", "dag_id", "try_number")) SCOPE_CRUMBS = frozenset(("task_id", "state", "operator", "duration")) diff --git a/airflow/serialization/pydantic/dag_run.py b/airflow/serialization/pydantic/dag_run.py index a0175e3749d9..b3b5ab6d70d3 100644 --- a/airflow/serialization/pydantic/dag_run.py +++ b/airflow/serialization/pydantic/dag_run.py @@ -40,7 +40,7 @@ class DagRunPydantic(BaseModelPydantic): id: int dag_id: str queued_at: Optional[datetime] - execution_date: datetime + logical_date: datetime start_date: Optional[datetime] end_date: Optional[datetime] state: str @@ -61,10 +61,6 @@ class DagRunPydantic(BaseModelPydantic): model_config = ConfigDict(from_attributes=True, arbitrary_types_allowed=True) - @property - def logical_date(self) -> datetime: - return self.execution_date - def get_task_instances( self, state: Iterable[TaskInstanceState | None] | None = None, diff --git a/airflow/serialization/pydantic/taskinstance.py b/airflow/serialization/pydantic/taskinstance.py index e8a1cfaf79a7..eea493758d17 100644 --- a/airflow/serialization/pydantic/taskinstance.py +++ b/airflow/serialization/pydantic/taskinstance.py @@ -90,7 +90,7 @@ class TaskInstancePydantic(BaseModelPydantic, LoggingMixin): map_index: int start_date: Optional[datetime] end_date: Optional[datetime] - execution_date: Optional[datetime] + logical_date: Optional[datetime] duration: Optional[float] state: Optional[str] try_number: int @@ -180,7 +180,7 @@ def xcom_pull( :param task_ids: task id or list of task ids, if None, the task_id of the current task is used :param dag_id: dag id, if None, the dag_id of the current task is used :param key: the key to identify the XCom value - :param include_prior_dates: whether to include prior execution dates + :param include_prior_dates: whether to include prior logical dates :param session: the sqlalchemy session :param map_indexes: map index or list of map indexes, if None, the map_index of the current task is used @@ -367,20 +367,20 @@ def get_previous_dagrun( return _get_previous_dagrun(task_instance=self, state=state, session=session) - def get_previous_execution_date( + def get_previous_logical_date( self, state: DagRunState | None = None, session: Session | None = None, ) -> pendulum.DateTime | None: """ - Return the execution date from property previous_ti_success. + Return the logical date from property previous_ti_success. :param state: If passed, it only take into account instances of a specific state. :param session: SQLAlchemy ORM Session """ - from airflow.models.taskinstance import _get_previous_execution_date + from airflow.models.taskinstance import _get_previous_logical_date - return _get_previous_execution_date(task_instance=self, state=state, session=session) + return _get_previous_logical_date(task_instance=self, state=state, session=session) def get_previous_start_date( self, @@ -388,7 +388,7 @@ def get_previous_start_date( session: Session | None = None, ) -> pendulum.DateTime | None: """ - Return the execution date from property previous_ti_success. + Return the logical date from property previous_ti_success. :param state: If passed, it only take into account instances of a specific state. :param session: SQLAlchemy ORM Session diff --git a/airflow/settings.py b/airflow/settings.py index c3f32fa59d98..db8ee4f41117 100644 --- a/airflow/settings.py +++ b/airflow/settings.py @@ -829,7 +829,7 @@ def is_usage_data_collection_enabled() -> bool: fallback=False, ) -ALLOW_FUTURE_EXEC_DATES = conf.getboolean("scheduler", "allow_trigger_in_future", fallback=False) +ALLOW_FUTURE_LOGICAL_DATES = conf.getboolean("scheduler", "allow_trigger_in_future", fallback=False) USE_JOB_SCHEDULE = conf.getboolean("scheduler", "use_job_schedule", fallback=True) diff --git a/airflow/task/standard_task_runner.py b/airflow/task/standard_task_runner.py index bc846574f024..c446bccac960 100644 --- a/airflow/task/standard_task_runner.py +++ b/airflow/task/standard_task_runner.py @@ -160,7 +160,7 @@ def _start_by_fork(self): self.log.info("Running: %s", self._command) self.log.info("Subtask %s", self._task_instance.task_id) - proc_title = "airflow task runner: {0.dag_id} {0.task_id} {0.execution_date_or_run_id}" + proc_title = "airflow task runner: {0.dag_id} {0.task_id} {0.logical_date_or_run_id}" setproctitle(proc_title.format(args)) return_code = 0 try: diff --git a/airflow/ti_deps/dep_context.py b/airflow/ti_deps/dep_context.py index 988cefdfcf94..bc2d476d8018 100644 --- a/airflow/ti_deps/dep_context.py +++ b/airflow/ti_deps/dep_context.py @@ -89,7 +89,7 @@ def ensure_finished_tis(self, dag_run: DagRun, session: Session) -> list[TaskIns Ensure finished_tis is populated if it's currently None, which allows running tasks without dag_run. :param dag_run: The DagRun for which to find finished tasks - :return: A list of all the finished tasks of this DAG and execution_date + :return: A list of all the finished tasks of this DAG and logical_date """ if self.finished_tis is None: finished_tis = dag_run.get_task_instances(state=State.finished, session=session) diff --git a/airflow/ti_deps/deps/exec_date_after_start_date_dep.py b/airflow/ti_deps/deps/exec_date_after_start_date_dep.py index 09e0d8c22961..4832ea2806be 100644 --- a/airflow/ti_deps/deps/exec_date_after_start_date_dep.py +++ b/airflow/ti_deps/deps/exec_date_after_start_date_dep.py @@ -22,25 +22,25 @@ class ExecDateAfterStartDateDep(BaseTIDep): - """Determines whether a task's execution date is after start date.""" + """Determines whether a task's logical date is after start date.""" - NAME = "Execution Date" + NAME = "Logical Date" IGNORABLE = True @provide_session def _get_dep_statuses(self, ti, session, dep_context): - if ti.task.start_date and ti.execution_date < ti.task.start_date: + if ti.task.start_date and ti.logical_date < ti.task.start_date: yield self._failing_status( reason=( - f"The execution date is {ti.execution_date.isoformat()} but this is before " + f"The logical date is {ti.logical_date.isoformat()} but this is before " f"the task's start date {ti.task.start_date.isoformat()}." ) ) - if ti.task.dag and ti.task.dag.start_date and ti.execution_date < ti.task.dag.start_date: + if ti.task.dag and ti.task.dag.start_date and ti.logical_date < ti.task.dag.start_date: yield self._failing_status( reason=( - f"The execution date is {ti.execution_date.isoformat()} but this is " + f"The logical date is {ti.logical_date.isoformat()} but this is " f"before the task's DAG's start date {ti.task.dag.start_date.isoformat()}." ) ) diff --git a/airflow/ti_deps/deps/prev_dagrun_dep.py b/airflow/ti_deps/deps/prev_dagrun_dep.py index 7265727f6d53..c756e6ec1c64 100644 --- a/airflow/ti_deps/deps/prev_dagrun_dep.py +++ b/airflow/ti_deps/deps/prev_dagrun_dep.py @@ -78,7 +78,7 @@ def _has_any_prior_tis(ti: TI, *, session: Session) -> bool: query = exists_query( TI.dag_id == ti.dag_id, TI.task_id == ti.task_id, - TI.execution_date < ti.execution_date, + TI.logical_date < ti.logical_date, session=session, ) return query @@ -169,7 +169,7 @@ def _get_dep_statuses(self, ti: TI, session: Session, dep_context): return # There was a DAG run, but the task wasn't active back then. - if catchup and last_dagrun.execution_date < ti.task.start_date: + if catchup and last_dagrun.logical_date < ti.task.start_date: self._push_past_deps_met_xcom_if_needed(ti, dep_context) yield self._passing_status(reason="This task instance was the first task instance for its task.") return diff --git a/airflow/ti_deps/deps/runnable_exec_date_dep.py b/airflow/ti_deps/deps/runnable_exec_date_dep.py index da9d88a2ec06..990264563a2f 100644 --- a/airflow/ti_deps/deps/runnable_exec_date_dep.py +++ b/airflow/ti_deps/deps/runnable_exec_date_dep.py @@ -23,7 +23,7 @@ class RunnableExecDateDep(BaseTIDep): - """Determines whether a task's execution date is valid.""" + """Determines whether a task's logical date is valid.""" NAME = "Execution Date" IGNORABLE = True @@ -34,11 +34,11 @@ def _get_dep_statuses(self, ti, session, dep_context): # don't consider runs that are executed in the future unless # specified by config and schedule is None - logical_date = ti.get_dagrun(session).execution_date + logical_date = ti.get_dagrun(session).logical_date if logical_date > cur_date and not ti.task.dag.allow_future_exec_dates: yield self._failing_status( reason=( - f"Execution date {logical_date.isoformat()} is in the future " + f"Logical date {logical_date.isoformat()} is in the future " f"(the current date is {cur_date.isoformat()})." ) ) @@ -46,7 +46,7 @@ def _get_dep_statuses(self, ti, session, dep_context): if ti.task.end_date and logical_date > ti.task.end_date: yield self._failing_status( reason=( - f"The execution date is {logical_date.isoformat()} but this is " + f"The logical date is {logical_date.isoformat()} but this is " f"after the task's end date {ti.task.end_date.isoformat()}." ) ) @@ -54,7 +54,7 @@ def _get_dep_statuses(self, ti, session, dep_context): if ti.task.dag and ti.task.dag.end_date and logical_date > ti.task.dag.end_date: yield self._failing_status( reason=( - f"The execution date is {logical_date.isoformat()} but this is after " + f"The logical date is {logical_date.isoformat()} but this is after " f"the task's DAG's end date {ti.task.dag.end_date.isoformat()}." ) ) diff --git a/airflow/timetables/base.py b/airflow/timetables/base.py index 64a261202651..f8aa4279ebba 100644 --- a/airflow/timetables/base.py +++ b/airflow/timetables/base.py @@ -174,7 +174,7 @@ class Timetable(Protocol): ``NullTimetable`` sets this to *False*. """ - run_ordering: Sequence[str] = ("data_interval_end", "execution_date") + run_ordering: Sequence[str] = ("data_interval_end", "logical_date") """How runs triggered from this timetable should be ordered in UI. This should be a list of field names on the DAG run object. diff --git a/airflow/timetables/simple.py b/airflow/timetables/simple.py index 5a931b40dd11..3457c52a08aa 100644 --- a/airflow/timetables/simple.py +++ b/airflow/timetables/simple.py @@ -36,7 +36,7 @@ class _TrivialTimetable(Timetable): """Some code reuse for "trivial" timetables that has nothing complex.""" periodic = False - run_ordering: Sequence[str] = ("execution_date",) + run_ordering: Sequence[str] = ("logical_date",) @classmethod def deserialize(cls, data: dict[str, Any]) -> Timetable: diff --git a/airflow/triggers/external_task.py b/airflow/triggers/external_task.py index cd43d59876e9..159a6df90950 100644 --- a/airflow/triggers/external_task.py +++ b/airflow/triggers/external_task.py @@ -41,7 +41,7 @@ class WorkflowTrigger(BaseTrigger): A trigger to monitor tasks, task group and dag execution in Apache Airflow. :param external_dag_id: The ID of the external DAG. - :param execution_dates: A list of execution dates for the external DAG. + :param logical_dates: A list of logical dates for the external DAG. :param external_task_ids: A collection of external task IDs to wait for. :param external_task_group_id: The ID of the external task group to wait for. :param failed_states: States considered as failed for external tasks. @@ -54,7 +54,7 @@ class WorkflowTrigger(BaseTrigger): def __init__( self, external_dag_id: str, - execution_dates: list, + logical_dates: list, external_task_ids: typing.Collection[str] | None = None, external_task_group_id: str | None = None, failed_states: typing.Iterable[str] | None = None, @@ -70,7 +70,7 @@ def __init__( self.failed_states = failed_states self.skipped_states = skipped_states self.allowed_states = allowed_states - self.execution_dates = execution_dates + self.logical_dates = logical_dates self.poke_interval = poke_interval self.soft_fail = soft_fail super().__init__(**kwargs) @@ -86,7 +86,7 @@ def serialize(self) -> tuple[str, dict[str, Any]]: "failed_states": self.failed_states, "skipped_states": self.skipped_states, "allowed_states": self.allowed_states, - "execution_dates": self.execution_dates, + "logical_dates": self.logical_dates, "poke_interval": self.poke_interval, "soft_fail": self.soft_fail, }, @@ -109,7 +109,7 @@ async def run(self) -> typing.AsyncIterator[TriggerEvent]: yield TriggerEvent({"status": "skipped"}) return allowed_count = await self._get_count(self.allowed_states) - if allowed_count == len(self.execution_dates): + if allowed_count == len(self.logical_dates): yield TriggerEvent({"status": "success"}) return self.log.info("Sleeping for %s seconds", self.poke_interval) @@ -124,7 +124,7 @@ def _get_count(self, states: typing.Iterable[str] | None) -> int: :return The count of records. """ return _get_count( - dttm_filter=self.execution_dates, + dttm_filter=self.logical_dates, external_task_ids=self.external_task_ids, external_task_group_id=self.external_task_group_id, external_dag_id=self.external_dag_id, @@ -138,7 +138,7 @@ class DagStateTrigger(BaseTrigger): :param dag_id: The dag_id that contains the task you want to wait for :param states: allowed states, default is ``['success']`` - :param execution_dates: The logical date at which DAG run. + :param logical_dates: The logical date at which DAG run. :param poll_interval: The time interval in seconds to check the state. The default value is 5.0 sec. """ @@ -147,13 +147,13 @@ def __init__( self, dag_id: str, states: list[DagRunState], - execution_dates: list[datetime], + logical_dates: list[datetime], poll_interval: float = 5.0, ): super().__init__() self.dag_id = dag_id self.states = states - self.execution_dates = execution_dates + self.logical_dates = logical_dates self.poll_interval = poll_interval def serialize(self) -> tuple[str, dict[str, typing.Any]]: @@ -163,7 +163,7 @@ def serialize(self) -> tuple[str, dict[str, typing.Any]]: { "dag_id": self.dag_id, "states": self.states, - "execution_dates": self.execution_dates, + "logical_dates": self.logical_dates, "poll_interval": self.poll_interval, }, ) @@ -173,7 +173,7 @@ async def run(self) -> typing.AsyncIterator[TriggerEvent]: while True: # mypy confuses typing here num_dags = await self.count_dags() # type: ignore[call-arg] - if num_dags == len(self.execution_dates): + if num_dags == len(self.logical_dates): yield TriggerEvent(self.serialize()) return await asyncio.sleep(self.poll_interval) @@ -187,7 +187,7 @@ def count_dags(self, *, session: Session = NEW_SESSION) -> int | None: .filter( DagRun.dag_id == self.dag_id, DagRun.state.in_(self.states), - DagRun.execution_date.in_(self.execution_dates), + DagRun.logical_date.in_(self.logical_dates), ) .scalar() ) diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow/ui/openapi-gen/requests/schemas.gen.ts index 8a8a50bb7437..d6a5a299ec40 100644 --- a/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -3895,10 +3895,10 @@ export const $XComResponseNative = { format: "date-time", title: "Timestamp", }, - execution_date: { + logical_date: { type: "string", format: "date-time", - title: "Execution Date", + title: "Logical Date", }, map_index: { type: "integer", @@ -3920,7 +3920,7 @@ export const $XComResponseNative = { required: [ "key", "timestamp", - "execution_date", + "logical_date", "map_index", "task_id", "dag_id", @@ -3941,10 +3941,10 @@ export const $XComResponseString = { format: "date-time", title: "Timestamp", }, - execution_date: { + logical_date: { type: "string", format: "date-time", - title: "Execution Date", + title: "Logical Date", }, map_index: { type: "integer", @@ -3974,7 +3974,7 @@ export const $XComResponseString = { required: [ "key", "timestamp", - "execution_date", + "logical_date", "map_index", "task_id", "dag_id", diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 35848f3a88f9..80fa416445c2 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -933,7 +933,7 @@ export type VersionInfo = { export type XComResponseNative = { key: string; timestamp: string; - execution_date: string; + logical_date: string; map_index: number; task_id: string; dag_id: string; @@ -946,7 +946,7 @@ export type XComResponseNative = { export type XComResponseString = { key: string; timestamp: string; - execution_date: string; + logical_date: string; map_index: number; task_id: string; dag_id: string; diff --git a/airflow/utils/cli.py b/airflow/utils/cli.py index 81b09f9d1104..c3cd1faf6b26 100644 --- a/airflow/utils/cli.py +++ b/airflow/utils/cli.py @@ -75,7 +75,7 @@ def action_logging(f: T) -> T: log : airflow.models.log.Log ORM instance dag_id : dag id (optional) task_id : task_id (optional) - execution_date : execution date (optional) + logical_date : logical date (optional) error : exception instance if there's an exception :param f: function instance @@ -130,7 +130,7 @@ def _build_metrics(func_name, namespace): It assumes that function arguments is from airflow.bin.cli module's function and has Namespace instance where it optionally contains "dag_id", "task_id", - and "execution_date". + and "logical_date". :param func_name: name of function :param namespace: Namespace instance from argparse @@ -172,7 +172,7 @@ def _build_metrics(func_name, namespace): tmp_dic = vars(namespace) metrics["dag_id"] = tmp_dic.get("dag_id") metrics["task_id"] = tmp_dic.get("task_id") - metrics["execution_date"] = tmp_dic.get("execution_date") + metrics["logical_date"] = tmp_dic.get("logical_date") metrics["host_name"] = socket.gethostname() return metrics diff --git a/airflow/utils/cli_action_loggers.py b/airflow/utils/cli_action_loggers.py index dfb8e41de856..568286c11179 100644 --- a/airflow/utils/cli_action_loggers.py +++ b/airflow/utils/cli_action_loggers.py @@ -103,7 +103,7 @@ def on_post_execution(**kwargs): logger.exception("Failed on post-execution callback using %s", callback) -def default_action_log(sub_command, user, task_id, dag_id, execution_date, host_name, full_command, **_): +def default_action_log(sub_command, user, task_id, dag_id, logical_date, host_name, full_command, **_): """ Behave similar to ``action_logging``; default action logger callback. @@ -115,7 +115,7 @@ def default_action_log(sub_command, user, task_id, dag_id, execution_date, host_ user=user, task_id=task_id, dag_id=dag_id, - execution_date=execution_date, + logical_date=logical_date, host_name=host_name, full_command=full_command, ) @@ -129,7 +129,7 @@ def _default_action_log_internal( user, task_id, dag_id, - execution_date, + logical_date, host_name, full_command, session: Session = NEW_SESSION, @@ -160,7 +160,7 @@ def _default_action_log_internal( "extra": json.dumps({"host_name": host_name, "full_command": full_command}), "task_id": task_id, "dag_id": dag_id, - "execution_date": execution_date, + "logical_date": logical_date, "dttm": timezone.utcnow(), } ], diff --git a/airflow/utils/context.py b/airflow/utils/context.py index b28559999e50..3e217e748d02 100644 --- a/airflow/utils/context.py +++ b/airflow/utils/context.py @@ -71,7 +71,6 @@ "data_interval_start", "ds", "ds_nodash", - "execution_date", "expanded_ti_count", "exception", "inlets", @@ -79,18 +78,11 @@ "logical_date", "macros", "map_index_template", - "next_ds", - "next_ds_nodash", - "next_execution_date", "outlets", "outlet_events", "params", "prev_data_interval_start_success", "prev_data_interval_end_success", - "prev_ds", - "prev_ds_nodash", - "prev_execution_date", - "prev_execution_date_success", "prev_start_date_success", "prev_end_date_success", "reason", @@ -101,16 +93,12 @@ "test_mode", "templates_dict", "ti", - "tomorrow_ds", - "tomorrow_ds_nodash", "triggering_asset_events", "ts", "ts_nodash", "ts_nodash_with_tz", "try_number", "var", - "yesterday_ds", - "yesterday_ds_nodash", } @@ -330,20 +318,7 @@ class Context(MutableMapping[str, Any]): (and only when) deprecated context keys are accessed. """ - _DEPRECATION_REPLACEMENTS: dict[str, list[str]] = { - "execution_date": ["data_interval_start", "logical_date"], - "next_ds": ["{{ data_interval_end | ds }}"], - "next_ds_nodash": ["{{ data_interval_end | ds_nodash }}"], - "next_execution_date": ["data_interval_end"], - "prev_ds": [], - "prev_ds_nodash": [], - "prev_execution_date": [], - "prev_execution_date_success": ["prev_data_interval_start_success"], - "tomorrow_ds": [], - "tomorrow_ds_nodash": [], - "yesterday_ds": [], - "yesterday_ds_nodash": [], - } + _DEPRECATION_REPLACEMENTS: dict[str, list[str]] = {} def __init__(self, context: MutableMapping[str, Any] | None = None, **kwargs: Any) -> None: self._context: MutableMapping[str, Any] = context or {} diff --git a/airflow/utils/context.pyi b/airflow/utils/context.pyi index 4dc4659548ac..f4ed77537ff1 100644 --- a/airflow/utils/context.pyi +++ b/airflow/utils/context.pyi @@ -102,24 +102,16 @@ class Context(TypedDict, total=False): ds: str ds_nodash: str exception: BaseException | str | None - execution_date: DateTime expanded_ti_count: int | None inlets: list inlet_events: InletEventsAccessors logical_date: DateTime macros: Any map_index_template: str - next_ds: str | None - next_ds_nodash: str | None - next_execution_date: DateTime | None outlets: list params: ParamsDict prev_data_interval_start_success: DateTime | None prev_data_interval_end_success: DateTime | None - prev_ds: str | None - prev_ds_nodash: str | None - prev_execution_date: DateTime | None - prev_execution_date_success: DateTime | None prev_start_date_success: DateTime | None prev_end_date_success: DateTime | None reason: str | None @@ -130,16 +122,12 @@ class Context(TypedDict, total=False): test_mode: bool templates_dict: Mapping[str, Any] | None ti: TaskInstance | TaskInstancePydantic - tomorrow_ds: str - tomorrow_ds_nodash: str triggering_asset_events: Mapping[str, Collection[AssetEvent | AssetEventPydantic]] ts: str ts_nodash: str ts_nodash_with_tz: str try_number: int | None var: _VariableAccessors - yesterday_ds: str - yesterday_ds_nodash: str class AirflowContextDeprecationWarning(DeprecationWarning): ... diff --git a/airflow/utils/db.py b/airflow/utils/db.py index a45b14ae666e..7995c1a802d6 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -973,7 +973,7 @@ def synchronize_log_template(*, session: Session = NEW_SESSION) -> None: session.add( LogTemplate( filename="{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log", - elasticsearch_id="{dag_id}-{task_id}-{execution_date}-{try_number}", + elasticsearch_id="{dag_id}-{task_id}-{logical_date}-{try_number}", ) ) diff --git a/airflow/utils/helpers.py b/airflow/utils/helpers.py index 8b83e3b13496..1a6b7396e322 100644 --- a/airflow/utils/helpers.py +++ b/airflow/utils/helpers.py @@ -198,7 +198,7 @@ def render_log_filename(ti: TaskInstance, try_number, filename_template) -> str: return filename_template.format( dag_id=ti.dag_id, task_id=ti.task_id, - execution_date=ti.execution_date.isoformat(), + logical_date=ti.logical_date.isoformat(), try_number=try_number, ) @@ -234,7 +234,7 @@ def build_airflow_url_with_query(query: dict[str, Any]) -> str: Build airflow url using base_url and default_view and provided query. For example: - http://0.0.0.0:8000/base/graph?dag_id=my-task&root=&execution_date=2020-10-27T10%3A59%3A25.615587 + http://0.0.0.0:8000/base/graph?dag_id=my-task&root=&logical_date=2020-10-27T10%3A59%3A25.615587 """ import flask diff --git a/airflow/utils/log/file_task_handler.py b/airflow/utils/log/file_task_handler.py index 0192892603f2..622effe60ee3 100644 --- a/airflow/utils/log/file_task_handler.py +++ b/airflow/utils/log/file_task_handler.py @@ -315,7 +315,7 @@ def _render_filename(self, ti: TaskInstance | TaskInstancePydantic, try_number: run_id=ti.run_id, data_interval_start=data_interval_start, data_interval_end=data_interval_end, - execution_date=ti.get_dagrun().logical_date.isoformat(), + logical_date=ti.get_dagrun().logical_date.isoformat(), try_number=try_number, ) else: diff --git a/airflow/utils/operator_helpers.py b/airflow/utils/operator_helpers.py index a259b455776b..a06cdf42b50c 100644 --- a/airflow/utils/operator_helpers.py +++ b/airflow/utils/operator_helpers.py @@ -46,9 +46,9 @@ "default": f"{DEFAULT_FORMAT_PREFIX}task_id", "env_var_format": f"{ENV_VAR_FORMAT_PREFIX}TASK_ID", }, - "AIRFLOW_CONTEXT_EXECUTION_DATE": { - "default": f"{DEFAULT_FORMAT_PREFIX}execution_date", - "env_var_format": f"{ENV_VAR_FORMAT_PREFIX}EXECUTION_DATE", + "AIRFLOW_CONTEXT_LOGICAL_DATE": { + "default": f"{DEFAULT_FORMAT_PREFIX}logical_date", + "env_var_format": f"{ENV_VAR_FORMAT_PREFIX}LOGICAL_DATE", }, "AIRFLOW_CONTEXT_TRY_NUMBER": { "default": f"{DEFAULT_FORMAT_PREFIX}try_number", @@ -97,7 +97,7 @@ def context_to_airflow_vars(context: Mapping[str, Any], in_env_var_format: bool (task, "owner", "AIRFLOW_CONTEXT_DAG_OWNER"), (task_instance, "dag_id", "AIRFLOW_CONTEXT_DAG_ID"), (task_instance, "task_id", "AIRFLOW_CONTEXT_TASK_ID"), - (task_instance, "execution_date", "AIRFLOW_CONTEXT_EXECUTION_DATE"), + (task_instance, "logical_date", "AIRFLOW_CONTEXT_LOGICAL_DATE"), (task_instance, "try_number", "AIRFLOW_CONTEXT_TRY_NUMBER"), (dag_run, "run_id", "AIRFLOW_CONTEXT_DAG_RUN_ID"), ] diff --git a/airflow/utils/sensor_helper.py b/airflow/utils/sensor_helper.py index fe72a7004180..57d906da671c 100644 --- a/airflow/utils/sensor_helper.py +++ b/airflow/utils/sensor_helper.py @@ -40,7 +40,7 @@ def _get_count( """ Get the count of records against dttm filter and states. - :param dttm_filter: date time filter for execution date + :param dttm_filter: date time filter for logical date :param external_task_ids: The list of task_ids :param external_task_group_id: The ID of the external task group :param external_dag_id: The ID of the external DAG. @@ -85,12 +85,12 @@ def _count_query(model, states, dttm_filter, external_dag_id, session: Session) :param model: The SQLAlchemy model representing the relevant table. :param states: task or dag states - :param dttm_filter: date time filter for execution date + :param dttm_filter: date time filter for logical date :param external_dag_id: The ID of the external DAG. :param session: airflow session object """ query = select(func.count()).filter( - model.dag_id == external_dag_id, model.state.in_(states), model.execution_date.in_(dttm_filter) + model.dag_id == external_dag_id, model.state.in_(states), model.logical_date.in_(dttm_filter) ) return query @@ -99,7 +99,7 @@ def _get_external_task_group_task_ids(dttm_filter, external_task_group_id, exter """ Get the count of records against dttm filter and states. - :param dttm_filter: date time filter for execution date + :param dttm_filter: date time filter for logical date :param external_task_group_id: The ID of the external task group :param external_dag_id: The ID of the external DAG. :param session: airflow session object @@ -112,7 +112,7 @@ def _get_external_task_group_task_ids(dttm_filter, external_task_group_id, exter select(TaskInstance).filter( TaskInstance.dag_id == external_dag_id, TaskInstance.task_id.in_(task.task_id for task in task_group), - TaskInstance.execution_date.in_(dttm_filter), + TaskInstance.logical_date.in_(dttm_filter), ) ) diff --git a/airflow/utils/timezone.py b/airflow/utils/timezone.py index 476cfe2df28b..aca3810c548d 100644 --- a/airflow/utils/timezone.py +++ b/airflow/utils/timezone.py @@ -38,7 +38,7 @@ utc = pendulum.UTC -def is_localized(value): +def is_localized(value: dt.datetime) -> bool: """ Determine if a given datetime.datetime is aware. diff --git a/airflow/www/decorators.py b/airflow/www/decorators.py index 3eae5f623918..fc0ad893369f 100644 --- a/airflow/www/decorators.py +++ b/airflow/www/decorators.py @@ -105,7 +105,7 @@ def wrapper(*args, **kwargs): "task_id", "dag_run_id", "run_id", - "execution_date", + "logical_date", } extra_fields = { k: secrets_masker.redact(v, k) @@ -145,13 +145,13 @@ def wrapper(*args, **kwargs): run_id=params.get("run_id") or params.get("dag_run_id"), ) - if "execution_date" in request.values: - execution_date_value = request.values.get("execution_date") + if "logical_date" in request.values: + logical_date_value = request.values.get("logical_date") try: - log.execution_date = pendulum.parse(execution_date_value, strict=False) + log.logical_date = pendulum.parse(logical_date_value, strict=False) except ParserError: logger.exception( - "Failed to parse execution_date from the request: %s", execution_date_value + "Failed to parse logical_date from the request: %s", logical_date_value ) session.add(log) diff --git a/airflow/www/forms.py b/airflow/www/forms.py index a3b1d5262db2..57b4fde3a6e6 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -95,7 +95,7 @@ def _get_default_timezone(self): class DateTimeForm(FlaskForm): """Date filter form needed for task views.""" - execution_date = DateTimeWithTimezoneField("Logical date", widget=AirflowDateTimePickerWidget()) + logical_date = DateTimeWithTimezoneField("Logical date", widget=AirflowDateTimePickerWidget()) class DagRunEditForm(DynamicForm): @@ -114,7 +114,7 @@ class DagRunEditForm(DynamicForm): ) run_id = StringField(lazy_gettext("Run Id"), validators=[ReadOnly()], widget=BS3TextFieldROWidget()) state = StringField(lazy_gettext("State"), validators=[ReadOnly()], widget=BS3TextFieldROWidget()) - execution_date = DateTimeWithTimezoneField( + logical_date = DateTimeWithTimezoneField( lazy_gettext("Logical Date"), validators=[ReadOnly()], widget=AirflowDateTimePickerROWidget(), @@ -162,7 +162,7 @@ class TaskInstanceEditForm(DynamicForm): widget=Select2Widget(), validators=[InputRequired()], ) - execution_date = DateTimeWithTimezoneField( + logical_date = DateTimeWithTimezoneField( lazy_gettext("Logical Date"), widget=AirflowDateTimePickerROWidget(), validators=[InputRequired(), ReadOnly()], diff --git a/airflow/www/static/js/api/useClearTask.ts b/airflow/www/static/js/api/useClearTask.ts index b4f80e5a7bf2..56f7d7b8601f 100644 --- a/airflow/www/static/js/api/useClearTask.ts +++ b/airflow/www/static/js/api/useClearTask.ts @@ -31,13 +31,13 @@ export default function useClearTask({ dagId, runId, taskId, - executionDate, + logicalDate, isGroup, }: { dagId: string; runId: string; taskId: string; - executionDate: string; + logicalDate: string; isGroup: boolean; }) { const queryClient = useQueryClient(); @@ -70,7 +70,7 @@ export default function useClearTask({ dag_id: dagId, dag_run_id: runId, confirmed, - execution_date: executionDate, + logical_date: logicalDate, past, future, upstream, diff --git a/airflow/www/static/js/api/useClearTaskDryRun.ts b/airflow/www/static/js/api/useClearTaskDryRun.ts index 33986a9b4868..ba0cbf6bb163 100644 --- a/airflow/www/static/js/api/useClearTaskDryRun.ts +++ b/airflow/www/static/js/api/useClearTaskDryRun.ts @@ -30,7 +30,7 @@ const useClearTaskDryRun = ({ dagId, runId, taskId, - executionDate, + logicalDate, isGroup, past, future, @@ -44,7 +44,7 @@ const useClearTaskDryRun = ({ dagId: string; runId: string; taskId: string; - executionDate: string; + logicalDate: string; isGroup: boolean; past: boolean; future: boolean; @@ -75,7 +75,7 @@ const useClearTaskDryRun = ({ dag_id: dagId, dag_run_id: runId, confirmed: false, - execution_date: executionDate, + logical_date: logicalDate, past, future, upstream, diff --git a/airflow/www/static/js/api/useExtraLinks.ts b/airflow/www/static/js/api/useExtraLinks.ts index e4140d7d4c88..a61f0f3aa69d 100644 --- a/airflow/www/static/js/api/useExtraLinks.ts +++ b/airflow/www/static/js/api/useExtraLinks.ts @@ -31,20 +31,20 @@ interface LinkData { export default function useExtraLinks({ dagId, taskId, - executionDate, + logicalDate, mapIndex, extraLinks, tryNumber, }: { dagId: string; taskId: string; - executionDate: string; + logicalDate: string; mapIndex?: number | undefined; extraLinks: string[]; tryNumber?: number | undefined; }) { return useQuery( - ["extraLinks", dagId, taskId, executionDate, mapIndex, tryNumber], + ["extraLinks", dagId, taskId, logicalDate, mapIndex, tryNumber], async () => { const data = await Promise.all( extraLinks.map(async (link) => { @@ -55,8 +55,8 @@ export default function useExtraLinks({ taskId )}&dag_id=${encodeURIComponent( dagId - )}&execution_date=${encodeURIComponent( - executionDate + )}&logical_date=${encodeURIComponent( + logicalDate )}&link_name=${encodeURIComponent( link )}&map_index=${definedMapIndex}${tryNumberParam}`; diff --git a/airflow/www/static/js/api/useGridData.test.ts b/airflow/www/static/js/api/useGridData.test.ts index 3e83e29ee56a..efbfe5d9820b 100644 --- a/airflow/www/static/js/api/useGridData.test.ts +++ b/airflow/www/static/js/api/useGridData.test.ts @@ -24,7 +24,7 @@ import { areActiveRuns } from "./useGridData"; const commonDagRunParams = { runId: "runId", - executionDate: "2022-01-01T10:00+00:00", + logicalDate: "2022-01-01T10:00+00:00", dataIntervalStart: "2022-01-01T05:00+00:00", dataIntervalEnd: "2022-01-01T10:00+00:00", runType: "scheduled" as DagRun["runType"], diff --git a/airflow/www/static/js/api/useGridData.ts b/airflow/www/static/js/api/useGridData.ts index 6bed9dfd71c7..795a5db1d5b8 100644 --- a/airflow/www/static/js/api/useGridData.ts +++ b/airflow/www/static/js/api/useGridData.ts @@ -124,8 +124,8 @@ const useGridData = () => { const selectedRun = await axios.get( dagRunUrl ); - if (selectedRun?.executionDate) { - onBaseDateChange(selectedRun.executionDate); + if (selectedRun?.logicalDate) { + onBaseDateChange(selectedRun.logicalDate); } // otherwise the run_id isn't valid and we should unselect it } catch (e) { diff --git a/airflow/www/static/js/dag/details/index.tsx b/airflow/www/static/js/dag/details/index.tsx index 0046d339972a..b26311c464eb 100644 --- a/airflow/www/static/js/dag/details/index.tsx +++ b/airflow/www/static/js/dag/details/index.tsx @@ -272,7 +272,7 @@ const Details = ({ { tryNumber={tryNumber} dagId="dummyDagId" taskId="dummyTaskId" - executionDate="2020:01:01T01:00+00:00" + logicalDate="2020:01:01T01:00+00:00" isInternal /> ); @@ -42,7 +42,7 @@ describe("Test LogLink Component.", () => { expect(linkElement).not.toHaveAttribute("target"); expect( linkElement?.href.includes( - `?dag_id=dummyDagId&task_id=dummyTaskId&execution_date=2020%3A01%3A01T01%3A00%2B00%3A00&map_index=-1&try_number=${tryNumber}&format=file` + `?dag_id=dummyDagId&task_id=dummyTaskId&logical_date=2020%3A01%3A01T01%3A00%2B00%3A00&map_index=-1&try_number=${tryNumber}&format=file` ) ).toBeTruthy(); }); @@ -55,7 +55,7 @@ describe("Test LogLink Component.", () => { tryNumber={tryNumber} dagId="dummyDagId" taskId="dummyTaskId" - executionDate="2020:01:01T01:00+00:00" + logicalDate="2020:01:01T01:00+00:00" mapIndex={mapIndex} /> ); @@ -66,7 +66,7 @@ describe("Test LogLink Component.", () => { expect(linkElement).toHaveAttribute("target", "_blank"); expect( linkElement?.href.includes( - `?dag_id=dummyDagId&task_id=dummyTaskId&execution_date=2020%3A01%3A01T01%3A00%2B00%3A00&map_index=${mapIndex}&try_number=${tryNumber}` + `?dag_id=dummyDagId&task_id=dummyTaskId&logical_date=2020%3A01%3A01T01%3A00%2B00%3A00&map_index=${mapIndex}&try_number=${tryNumber}` ) ).toBeTruthy(); }); diff --git a/airflow/www/static/js/dag/details/taskInstance/Logs/LogLink.tsx b/airflow/www/static/js/dag/details/taskInstance/Logs/LogLink.tsx index 72a203c7bf84..828f09236176 100644 --- a/airflow/www/static/js/dag/details/taskInstance/Logs/LogLink.tsx +++ b/airflow/www/static/js/dag/details/taskInstance/Logs/LogLink.tsx @@ -29,7 +29,7 @@ const externalLogUrl = getMetaValue("external_log_url"); interface Props { dagId: Dag["id"]; taskId: TaskInstance["taskId"]; - executionDate: DagRun["executionDate"]; + logicalDate: DagRun["logicalDate"]; isInternal?: boolean; tryNumber: TaskInstance["tryNumber"]; mapIndex?: TaskInstance["mapIndex"]; @@ -38,7 +38,7 @@ interface Props { const LogLink = ({ dagId, taskId, - executionDate, + logicalDate, isInternal, tryNumber, mapIndex, @@ -47,8 +47,8 @@ const LogLink = ({ isInternal ? logsWithMetadataUrl : externalLogUrl }?dag_id=${encodeURIComponent(dagId)}&task_id=${encodeURIComponent( taskId - )}&execution_date=${encodeURIComponent( - executionDate + )}&logical_date=${encodeURIComponent( + logicalDate )}&map_index=${encodeURIComponent( mapIndex?.toString() ?? "-1" )}&try_number=${tryNumber}`; diff --git a/airflow/www/static/js/dag/details/taskInstance/Logs/index.test.tsx b/airflow/www/static/js/dag/details/taskInstance/Logs/index.test.tsx index 74db45a7d2f3..31bfbe36c738 100644 --- a/airflow/www/static/js/dag/details/taskInstance/Logs/index.test.tsx +++ b/airflow/www/static/js/dag/details/taskInstance/Logs/index.test.tsx @@ -108,7 +108,7 @@ describe("Test Logs Component.", () => { dagId="dummyDagId" dagRunId="dummyDagRunId" taskId="dummyTaskId" - executionDate="2020:01:01T01:00+00:00" + logicalDate="2020:01:01T01:00+00:00" tryNumber={tryNumber} /> ); @@ -153,7 +153,7 @@ describe("Test Logs Component.", () => { dagId="dummyDagId" dagRunId="dummyDagRunId" taskId="dummyTaskId" - executionDate="2020:01:01T01:00+00:00" + logicalDate="2020:01:01T01:00+00:00" mapIndex={1} tryNumber={tryNumber} /> @@ -175,7 +175,7 @@ describe("Test Logs Component.", () => { dagId="dummyDagId" dagRunId="dummyDagRunId" taskId="dummyTaskId" - executionDate="2020:01:01T01:00+00:00" + logicalDate="2020:01:01T01:00+00:00" mapIndex={1} tryNumber={tryNumber} /> @@ -211,7 +211,7 @@ describe("Test Logs Component.", () => { dagId="dummyDagId" dagRunId="dummyDagRunId" taskId="dummyTaskId" - executionDate="2020:01:01T01:00+00:00" + logicalDate="2020:01:01T01:00+00:00" tryNumber={tryNumber} /> ); diff --git a/airflow/www/static/js/dag/details/taskInstance/Logs/index.tsx b/airflow/www/static/js/dag/details/taskInstance/Logs/index.tsx index 2647e590d84d..3c861f396b41 100644 --- a/airflow/www/static/js/dag/details/taskInstance/Logs/index.tsx +++ b/airflow/www/static/js/dag/details/taskInstance/Logs/index.tsx @@ -64,7 +64,7 @@ interface Props { dagRunId: DagRun["runId"]; taskId: TaskInstance["taskId"]; mapIndex?: TaskInstance["mapIndex"]; - executionDate: DagRun["executionDate"]; + logicalDate: DagRun["logicalDate"]; tryNumber: TaskInstance["tryNumber"]; state?: TaskInstance["state"]; } @@ -74,7 +74,7 @@ const Logs = ({ dagRunId, taskId, mapIndex, - executionDate, + logicalDate, tryNumber: finalTryNumber, state, }: Props) => { @@ -109,7 +109,7 @@ const Logs = ({ const params = new URLSearchParamsWrapper({ task_id: taskId, - execution_date: executionDate, + logical_date: logicalDate, }); if (mapIndex !== undefined) { @@ -164,7 +164,7 @@ const Logs = ({ key={tryNumber} dagId={dagId} taskId={taskId} - executionDate={executionDate} + logicalDate={logicalDate} tryNumber={tryNumber} mapIndex={mapIndex} /> @@ -231,7 +231,7 @@ const Logs = ({ ( - ({ taskId, executionDate, isMapped = false, mapIndex }, ref) => { + ({ taskId, logicalDate, isMapped = false, mapIndex }, ref) => { if (!taskId) return null; const params = new URLSearchParamsWrapper({ task_id: taskId, - execution_date: executionDate, + logical_date: logicalDate, map_index: mapIndex ?? -1, }); const detailsLink = `${taskUrl}&${params}`; const listParams = new URLSearchParamsWrapper({ _flt_3_dag_id: dagId, _flt_3_task_id: taskId, - _oc_TaskInstanceModelView: "dag_run.execution_date", + _oc_TaskInstanceModelView: "dag_run.logical_date", }); if (mapIndex !== undefined && mapIndex >= 0) diff --git a/airflow/www/static/js/dag/details/taskInstance/index.tsx b/airflow/www/static/js/dag/details/taskInstance/index.tsx index 22617dcbd65a..dda550b5bd4b 100644 --- a/airflow/www/static/js/dag/details/taskInstance/index.tsx +++ b/airflow/www/static/js/dag/details/taskInstance/index.tsx @@ -83,12 +83,12 @@ const TaskInstance = ({ taskId, runId, mapIndex }: Props) => { ref={taskInstanceRef} overflowY="auto" > - {!isGroup && run?.executionDate && ( + {!isGroup && run?.logicalDate && ( )} {!isGroupOrMappedTaskSummary && } @@ -105,12 +105,12 @@ const TaskInstance = ({ taskId, runId, mapIndex }: Props) => { )} {!!group?.extraLinks?.length && !isGroupOrMappedTaskSummary && - run?.executionDate && ( + run?.logicalDate && ( diff --git a/airflow/www/static/js/dag/details/taskInstance/taskActions/ClearInstance.tsx b/airflow/www/static/js/dag/details/taskInstance/taskActions/ClearInstance.tsx index f2a63fcb8927..e4853cecdc69 100644 --- a/airflow/www/static/js/dag/details/taskInstance/taskActions/ClearInstance.tsx +++ b/airflow/www/static/js/dag/details/taskInstance/taskActions/ClearInstance.tsx @@ -45,7 +45,7 @@ const dagId = getMetaValue("dag_id"); interface Props { runId: string; taskId: string; - executionDate: string; + logicalDate: string; isGroup?: boolean; isMapped?: boolean; mapIndex?: number; @@ -60,7 +60,7 @@ const ClearModal = ({ runId, taskId, mapIndex, - executionDate, + logicalDate, isGroup, isMapped, isOpen, @@ -94,7 +94,7 @@ const ClearModal = ({ dagId, runId, taskId, - executionDate, + logicalDate, isGroup: !!isGroup, past, future, @@ -110,7 +110,7 @@ const ClearModal = ({ dagId, runId, taskId, - executionDate, + logicalDate, isGroup: !!isGroup, }); @@ -233,7 +233,7 @@ const ClearInstance = ({ runId, taskId, mapIndex, - executionDate, + logicalDate, isGroup, isMapped, ...otherProps @@ -262,7 +262,7 @@ const ClearInstance = ({ runId={runId} taskId={taskId} mapIndex={mapIndex} - executionDate={executionDate} + logicalDate={logicalDate} isGroup={isGroup} isMapped={isMapped} isOpen={isOpen} diff --git a/airflow/www/static/js/dag/grid/dagRuns/index.test.tsx b/airflow/www/static/js/dag/grid/dagRuns/index.test.tsx index ebbb1f86e0cb..10cb7aee50e9 100644 --- a/airflow/www/static/js/dag/grid/dagRuns/index.test.tsx +++ b/airflow/www/static/js/dag/grid/dagRuns/index.test.tsx @@ -40,7 +40,7 @@ const generateRuns = (length: number): DagRun[] => startDate: "2021-11-08T21:14:19.704433+00:00", endDate: "2021-11-08T21:17:13.206426+00:00", lastSchedulingDecision: datestring, - executionDate: datestring, + logicalDate: datestring, externalTrigger: false, conf: null, note: "someRandomValue", @@ -59,7 +59,7 @@ describe("Test DagRuns", () => { endDate: "2021-11-08T21:17:13.206426+00:00", state: "failed", runType: "scheduled", - executionDate: "2021-11-08T21:14:19.704433+00:00", + logicalDate: "2021-11-08T21:14:19.704433+00:00", lastSchedulingDecision: datestring, externalTrigger: false, conf: null, @@ -74,7 +74,7 @@ describe("Test DagRuns", () => { queuedAt: "2021-11-08T21:14:18.21521+00:00", startDate: "2021-11-09T00:19:43.023200+00:00", endDate: "2021-11-09T00:22:18.607167+00:00", - executionDate: "2021-11-08T21:14:19.704433+00:00", + logicalDate: "2021-11-08T21:14:19.704433+00:00", lastSchedulingDecision: datestring, externalTrigger: false, conf: null, @@ -102,7 +102,7 @@ describe("Test DagRuns", () => { expect(getByText("00:01:26")).toBeInTheDocument(); expect( // @ts-ignore - queryByText(moment.utc(dagRuns[0].executionDate).format("MMM DD, HH:mm")) + queryByText(moment.utc(dagRuns[0].logicalDate).format("MMM DD, HH:mm")) ).toBeNull(); spy.mockRestore(); diff --git a/airflow/www/static/js/dag/grid/index.test.tsx b/airflow/www/static/js/dag/grid/index.test.tsx index b6eb000b768c..7daf444a94b7 100644 --- a/airflow/www/static/js/dag/grid/index.test.tsx +++ b/airflow/www/static/js/dag/grid/index.test.tsx @@ -148,7 +148,7 @@ const mockGridData = { endDate: "2021-11-08T21:17:13.206426+00:00", state: "failed", runType: "scheduled", - executionDate: "2021-11-08T21:14:19.704433+00:00", + logicalDate: "2021-11-08T21:14:19.704433+00:00", lastSchedulingDecision: "2021-11-08T21:14:19.704433+00:00", note: "myCoolDagRun", externalTrigger: false, diff --git a/airflow/www/static/js/dag/useFilters.tsx b/airflow/www/static/js/dag/useFilters.tsx index 5a1a49b68655..8f82598230de 100644 --- a/airflow/www/static/js/dag/useFilters.tsx +++ b/airflow/www/static/js/dag/useFilters.tsx @@ -69,7 +69,7 @@ export interface FilterHookReturn extends UtilFunctions { // Params names export const BASE_DATE_PARAM = "base_date"; -export const EXECUTION_DATE_PARAM = "execution_date"; +export const LOGICAL_DATE_PARAM = "logical_date"; export const NUM_RUNS_PARAM = "num_runs"; export const RUN_TYPE_PARAM = "run_type"; export const RUN_STATE_PARAM = "run_state"; @@ -96,7 +96,7 @@ const useFilters = (): FilterHookReturn => { const baseDate = searchParams.get(BASE_DATE_PARAM) || - searchParams.get(EXECUTION_DATE_PARAM) || + searchParams.get(LOGICAL_DATE_PARAM) || now; const numRuns = searchParams.get(NUM_RUNS_PARAM) || defaultDagRunDisplayNumber.toString(); @@ -182,7 +182,7 @@ const useFilters = (): FilterHookReturn => { searchParams.delete(BASE_DATE_PARAM); searchParams.delete(RUN_TYPE_PARAM); searchParams.delete(RUN_STATE_PARAM); - searchParams.delete(EXECUTION_DATE_PARAM); + searchParams.delete(LOGICAL_DATE_PARAM); setSearchParams(searchParams); }; diff --git a/airflow/www/static/js/dags.js b/airflow/www/static/js/dags.js index 66ea1244879c..9c69edd80618 100644 --- a/airflow/www/static/js/dags.js +++ b/airflow/www/static/js/dags.js @@ -198,7 +198,7 @@ function lastDagRunsHandler(error, json) { $(".js-loading-last-run").remove(); Object.keys(json).forEach((safeDagId) => { const dagId = json[safeDagId].dag_id; - const executionDate = json[safeDagId].execution_date; + const logicalDate = json[safeDagId].logical_date; const g = d3.select(`#last-run-${safeDagId}`); // Show last run as a link to the graph view @@ -207,10 +207,10 @@ function lastDagRunsHandler(error, json) { "href", `${graphUrl}?dag_id=${encodeURIComponent( dagId - )}&execution_date=${encodeURIComponent(executionDate)}` + )}&logical_date=${encodeURIComponent(logicalDate)}` ) .html("") - .insert(isoDateToTimeEl.bind(null, executionDate, { title: false })); + .insert(isoDateToTimeEl.bind(null, logicalDate, { title: false })); // Only show the tooltip when we have a last run and add the json to a custom data- attribute g.selectAll("span") diff --git a/airflow/www/static/js/task_instances.js b/airflow/www/static/js/task_instances.js index 47a5dccb6f3b..211bf1dd2ddf 100644 --- a/airflow/www/static/js/task_instances.js +++ b/airflow/www/static/js/task_instances.js @@ -99,8 +99,8 @@ export default function tiTooltip(ti, task, { includeTryNumber = false } = {}) { if (ti.task_id !== undefined) { tt += `Task_id: ${escapeHtml(ti.task_id)}
`; } - if (ti.execution_date !== undefined) { - tt += `Run: ${formatDateTime(ti.execution_date)}
`; + if (ti.logical_date !== undefined) { + tt += `Run: ${formatDateTime(ti.logical_date)}
`; } if (ti.run_id !== undefined) { tt += `Run Id: ${escapeHtml(ti.run_id)}
`; diff --git a/airflow/www/static/js/ti_log.js b/airflow/www/static/js/ti_log.js index 1fd66acee6dd..02064a419d7c 100644 --- a/airflow/www/static/js/ti_log.js +++ b/airflow/www/static/js/ti_log.js @@ -22,7 +22,7 @@ import { AnsiUp } from "ansi_up"; import { getMetaValue } from "./utils"; import { formatDateTime } from "./datetime_utils"; -const executionDate = getMetaValue("execution_date"); +const logicalDate = getMetaValue("logical_date"); const dagId = getMetaValue("dag_id"); const taskId = getMetaValue("task_id"); const mapIndex = getMetaValue("map_index"); @@ -70,7 +70,7 @@ window.scrollBottomLogs = scrollBottom; function autoTailingLog(tryNumber, metadata = null, autoTailing = false) { console.debug( `Auto-tailing log for dag_id: ${dagId}, task_id: ${taskId}, ` + - `execution_date: ${executionDate}, map_index: ${mapIndex}, try_number: ${tryNumber}, ` + + `logical_date: ${logicalDate}, map_index: ${mapIndex}, try_number: ${tryNumber}, ` + `metadata: ${JSON.stringify(metadata)}` ); @@ -81,7 +81,7 @@ function autoTailingLog(tryNumber, metadata = null, autoTailing = false) { dag_id: dagId, task_id: taskId, map_index: mapIndex, - execution_date: executionDate, + logical_date: logicalDate, try_number: tryNumber, metadata: JSON.stringify(metadata), }, @@ -214,7 +214,7 @@ function setDownloadUrl(tryNumber) { const query = new URLSearchParams({ dag_id: dagId, task_id: taskId, - execution_date: executionDate, + logical_date: logicalDate, try_number: tryNumberData, metadata: "null", format: "file", diff --git a/airflow/www/static/js/types/api-generated.ts b/airflow/www/static/js/types/api-generated.ts index c94c20f1d8fe..0f91a2329dd2 100644 --- a/airflow/www/static/js/types/api-generated.ts +++ b/airflow/www/static/js/types/api-generated.ts @@ -1116,7 +1116,7 @@ export interface components { * The value of this field can be set only when creating the object. If you try to modify the * field of an existing object, the request fails with an BAD_REQUEST error. * - * If not provided, a value will be generated based on execution_date. + * If not provided, a value will be generated based on logical_date. * * If the specified dag_run_id is in use, the creation request fails with an ALREADY_EXISTS error. * @@ -1137,18 +1137,6 @@ export interface components { * *New in version 2.2.0* */ logical_date?: string | null; - /** - * Format: date-time - * @deprecated - * @description The execution date. This is the same as logical_date, kept for backwards compatibility. - * If both this field and logical_date are provided but with different values, the request - * will fail with an BAD_REQUEST error. - * - * *Changed in version 2.2.0*: Field becomes nullable. - * - * *Deprecated since version 2.2.0*: Use 'logical_date' instead. - */ - execution_date?: string | null; /** * Format: date-time * @description The start time. The time when DAG run was actually created. @@ -1480,7 +1468,7 @@ export interface components { */ dag_run_id?: string; /** Format: datetime */ - execution_date?: string; + logical_date?: string; /** Format: datetime */ start_date?: string | null; /** Format: datetime */ @@ -1544,7 +1532,7 @@ export interface components { /** @description The DAG ID. */ dag_id?: string; /** Format: datetime */ - execution_date?: string; + logical_date?: string; /** @description The DAG run ID. */ dag_run_id?: string; }; @@ -1586,7 +1574,7 @@ export interface components { /** Format: datetime */ timestamp?: string; /** Format: datetime */ - execution_date?: string; + logical_date?: string; map_index?: number; task_id?: string; dag_id?: string; @@ -2099,11 +2087,11 @@ export interface components { task_id?: string; /** * Format: datetime - * @description The execution date. Either set this or dag_run_id but not both. + * @description The logical date. Either set this or dag_run_id but not both. */ - execution_date?: string; + logical_date?: string; /** - * @description The task instance's DAG run ID. Either set this or execution_date but not both. + * @description The task instance's DAG run ID. Either set this or logical_date but not both. * * *New in version 2.3.0* */ diff --git a/airflow/www/static/js/types/index.ts b/airflow/www/static/js/types/index.ts index 13ee62966214..618706a9e350 100644 --- a/airflow/www/static/js/types/index.ts +++ b/airflow/www/static/js/types/index.ts @@ -56,7 +56,7 @@ interface DagRun { runId: string; runType: "manual" | "backfill" | "scheduled" | "asset_triggered"; state: RunState; - executionDate: string; + logicalDate: string; dataIntervalStart: string; dataIntervalEnd: string; queuedAt: string | null; @@ -109,11 +109,7 @@ interface Task { setupTeardownType?: "setup" | "teardown"; } -type RunOrdering = ( - | "dataIntervalStart" - | "executionDate" - | "dataIntervalEnd" -)[]; +type RunOrdering = ("dataIntervalStart" | "logicalDate" | "dataIntervalEnd")[]; export interface MidEdge { id: string; diff --git a/airflow/www/static/js/utils/index.test.ts b/airflow/www/static/js/utils/index.test.ts index 569d3af98b53..4e367dc08f8c 100644 --- a/airflow/www/static/js/utils/index.test.ts +++ b/airflow/www/static/js/utils/index.test.ts @@ -139,7 +139,7 @@ describe("Test getDagRunLabel", () => { endDate: "2021-11-08T21:17:13.206426+00:00", state: "failed", runType: "scheduled", - executionDate: "2021-12-09T21:14:19.704433+00:00", + logicalDate: "2021-12-09T21:14:19.704433+00:00", lastSchedulingDecision: "2021-11-08T21:14:19.704433+00:00", externalTrigger: false, conf: null, @@ -148,7 +148,7 @@ describe("Test getDagRunLabel", () => { test("Defaults to executionDate", async () => { const runLabel = getDagRunLabel({ dagRun }); - expect(runLabel).toBe(dagRun.executionDate); + expect(runLabel).toBe(dagRun.logicalDate); }); test("Passing an order overrides default", async () => { diff --git a/airflow/www/static/js/utils/index.ts b/airflow/www/static/js/utils/index.ts index 8bef31a8582a..cbebc09c98c1 100644 --- a/airflow/www/static/js/utils/index.ts +++ b/airflow/www/static/js/utils/index.ts @@ -169,7 +169,7 @@ interface RunLabelProps { const getDagRunLabel = ({ dagRun, - ordering = ["executionDate"], + ordering = ["logicalDate"], }: RunLabelProps) => dagRun[ordering[0]]; const getStatusBackgroundColor = (color: string, hasNote: boolean) => diff --git a/airflow/www/templates/airflow/dag.html b/airflow/www/templates/airflow/dag.html index cd212d6620aa..6969993e9199 100644 --- a/airflow/www/templates/airflow/dag.html +++ b/airflow/www/templates/airflow/dag.html @@ -50,10 +50,10 @@ {% set dag_is_paused = dag.get_is_paused() %} {% set base_date_arg = request.args.get('base_date') %} {% set num_runs_arg = request.args.get('num_runs') %} -{% if execution_date is defined %} - {% set execution_date_arg = execution_date %} +{% if logical_date is defined %} + {% set execution_date_arg = logical_date %} {% else %} - {% set execution_date_arg = request.args.get('execution_date') %} + {% set execution_date_arg = request.args.get('logical_date') %} {% endif %} {% block head_meta %} diff --git a/airflow/www/templates/airflow/task_instance.html b/airflow/www/templates/airflow/task_instance.html index 9c5953ae28e8..90e468cc37c5 100644 --- a/airflow/www/templates/airflow/task_instance.html +++ b/airflow/www/templates/airflow/task_instance.html @@ -26,7 +26,7 @@

Task Instance: {{ task_display_name }} - at + at {% if map_index is defined and map_index >= 0 %} Map Index: {{ map_index }} {% endif %} @@ -34,9 +34,9 @@

    {% macro ti_url(endpoint) -%} {% if map_index >= 0 -%} - {{ url_for(endpoint, dag_id=dag.dag_id, task_id=task_id, execution_date=execution_date, map_index=map_index) }} + {{ url_for(endpoint, dag_id=dag.dag_id, task_id=task_id, logical_date=logical_date, map_index=map_index) }} {%- else -%} - {{ url_for(endpoint, dag_id=dag.dag_id, task_id=task_id, execution_date=execution_date) }} + {{ url_for(endpoint, dag_id=dag.dag_id, task_id=task_id, logical_date=logical_date) }} {%- endif -%} {% endmacro -%}
  • diff --git a/airflow/www/templates/airflow/ti_log.html b/airflow/www/templates/airflow/ti_log.html index 0710e973c0f4..fa79555bc00e 100644 --- a/airflow/www/templates/airflow/ti_log.html +++ b/airflow/www/templates/airflow/ti_log.html @@ -24,7 +24,7 @@ {{ super() }} - + {# Time interval to wait before next log fetching. Default 2s. #} diff --git a/airflow/www/templates/airflow/trigger.html b/airflow/www/templates/airflow/trigger.html index 7cdcd337bedd..d68f4fc97b86 100644 --- a/airflow/www/templates/airflow/trigger.html +++ b/airflow/www/templates/airflow/trigger.html @@ -244,10 +244,10 @@

    - + - {{ form.execution_date(class_="form-control", disabled=False) }} + {{ form.logical_date(class_="form-control", disabled=False) }} @@ -273,9 +273,9 @@

    {%- else %}
    - +
    - {{ form.execution_date(class_="form-control", disabled=False) }} + {{ form.logical_date(class_="form-control", disabled=False) }}
    diff --git a/airflow/www/utils.py b/airflow/www/utils.py index 981dc030a210..b30521adaf1b 100644 --- a/airflow/www/utils.py +++ b/airflow/www/utils.py @@ -88,8 +88,8 @@ def get_mapped_instances(task_instance, session): def get_instance_with_map(task_instance, session): if task_instance.map_index == -1: data = alchemy_to_dict(task_instance) - # Fetch execution_date explicitly since it's not a column and a proxy - data["execution_date"] = task_instance.execution_date + # Fetch logical_date explicitly since it's not a column and a proxy + data["logical_date"] = task_instance.logical_date return data mapped_instances = get_mapped_instances(task_instance, session) return get_mapped_summary(task_instance, mapped_instances) @@ -141,7 +141,7 @@ def get_mapped_summary(parent_instance, task_instances): "end_date": group_end_date, "mapped_states": mapped_states, "try_number": parent_instance.try_number, - "execution_date": parent_instance.execution_date, + "logical_date": parent_instance.logical_date, } @@ -174,7 +174,7 @@ def encode_dag_run( "start_date": datetime_to_string(dag_run.start_date), "end_date": datetime_to_string(dag_run.end_date), "state": dag_run.state, - "execution_date": datetime_to_string(dag_run.execution_date), + "logical_date": datetime_to_string(dag_run.logical_date), "data_interval_start": datetime_to_string(dag_run.data_interval_start), "data_interval_end": datetime_to_string(dag_run.data_interval_end), "run_type": dag_run.run_type, @@ -401,7 +401,7 @@ def task_instance_link(attr): task_id = attr.get("task_id") run_id = attr.get("run_id") map_index = attr.get("map_index", None) - execution_date = attr.get("execution_date") or attr.get("dag_run.execution_date") + logical_date = attr.get("logical_date") or attr.get("dag_run.logical_date") if map_index == -1: map_index = None @@ -412,7 +412,7 @@ def task_instance_link(attr): task_id=task_id, dag_run_id=run_id, map_index=map_index, - execution_date=execution_date, + logical_date=logical_date, tab="graph", ) url_root = url_for( @@ -422,7 +422,7 @@ def task_instance_link(attr): root=task_id, dag_run_id=run_id, map_index=map_index, - execution_date=execution_date, + logical_date=logical_date, tab="graph", ) return Markup( @@ -502,10 +502,10 @@ def json_(attr): def dag_link(attr): """Generate a URL to the Graph view for a Dag.""" dag_id = attr.get("dag_id") - execution_date = attr.get("execution_date") or attr.get("dag_run.execution_date") + logical_date = attr.get("logical_date") or attr.get("dag_run.logical_date") if not dag_id: return Markup("None") - url = url_for("Airflow.grid", dag_id=dag_id, execution_date=execution_date) + url = url_for("Airflow.grid", dag_id=dag_id, logical_date=logical_date) return Markup('{}').format(url, dag_id) @@ -513,7 +513,7 @@ def dag_run_link(attr): """Generate a URL to the Graph view for a DagRun.""" dag_id = attr.get("dag_id") run_id = attr.get("run_id") - execution_date = attr.get("execution_date") or attr.get("dag_run.execution_date") + logical_date = attr.get("logical_date") or attr.get("dag_run.logical_date") if not dag_id: return Markup("None") @@ -521,7 +521,7 @@ def dag_run_link(attr): url = url_for( "Airflow.grid", dag_id=dag_id, - execution_date=execution_date, + logical_date=logical_date, dag_run_id=run_id, tab="graph", ) @@ -533,10 +533,10 @@ def _get_run_ordering_expr(name: str) -> ColumnOperators: # Data interval columns are NULL for runs created before 2.3, but SQL's # NULL-sorting logic would make those old runs always appear first. In a # perfect world we'd want to sort by ``get_run_data_interval()``, but that's - # not efficient, so instead the columns are coalesced into execution_date, + # not efficient, so instead the columns are coalesced into logical_date, # which is good enough in most cases. if name in ("data_interval_start", "data_interval_end"): - expr = func.coalesce(expr, DagRun.execution_date) + expr = func.coalesce(expr, DagRun.logical_date) return expr.desc() diff --git a/airflow/www/views.py b/airflow/www/views.py index dd8279b56007..e97e585f753c 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -222,16 +222,16 @@ def get_safe_url(url): def get_date_time_num_runs_dag_runs_form_data(www_request, session, dag): """Get Execution Data, Base Date & Number of runs from a Request.""" - date_time = www_request.args.get("execution_date") + date_time = www_request.args.get("logical_date") run_id = www_request.args.get("run_id") - # First check run id, then check execution date, if not fall back on the latest dagrun + # First check run id, then check logical_date, if not fall back on the latest dagrun if run_id: dagrun = dag.get_dagrun(run_id=run_id, session=session) - date_time = dagrun.execution_date + date_time = dagrun.logical_date elif date_time: date_time = _safe_parse_datetime(date_time) else: - date_time = dag.get_latest_execution_date(session=session) or timezone.utcnow() + date_time = dag.get_latest_logical_date(session=session) or timezone.utcnow() base_date = www_request.args.get("base_date") if base_date: @@ -245,7 +245,7 @@ def get_date_time_num_runs_dag_runs_form_data(www_request, session, dag): num_runs = www_request.args.get("num_runs", default=default_dag_run, type=int) # When base_date has been rounded up because of the DateTimeField widget, we want - # to use the execution_date as the starting point for our query just to ensure a + # to use the logical_date as the starting point for our query just to ensure a # link targeting a specific dag run actually loads that dag run. If there are # more than num_runs dag runs in the "rounded period" then those dagruns would get # loaded and the actual requested run would be excluded by the limit(). Once @@ -256,28 +256,28 @@ def get_date_time_num_runs_dag_runs_form_data(www_request, session, dag): drs = session.scalars( select(DagRun) - .where(DagRun.dag_id == dag.dag_id, DagRun.execution_date <= query_date) - .order_by(desc(DagRun.execution_date)) + .where(DagRun.dag_id == dag.dag_id, DagRun.logical_date <= query_date) + .order_by(desc(DagRun.logical_date)) .limit(num_runs) ).all() dr_choices = [] dr_state = None for dr in drs: - dr_choices.append((dr.execution_date.isoformat(), dr.run_id)) - if date_time == dr.execution_date: + dr_choices.append((dr.logical_date.isoformat(), dr.run_id)) + if date_time == dr.logical_date: dr_state = dr.state # Happens if base_date was changed and the selected dag run is not in result if not dr_state and drs: dr = drs[0] - date_time = dr.execution_date + date_time = dr.logical_date dr_state = dr.state return { "dttm": date_time, "base_date": base_date, "num_runs": num_runs, - "execution_date": date_time.isoformat(), + "logical_date": date_time.isoformat(), "dr_choices": dr_choices, "dr_state": dr_state, } @@ -935,7 +935,7 @@ def index(self): dag_run_subquery = ( select( DagRun.dag_id, - sqla.func.max(DagRun.execution_date).label("max_execution_date"), + sqla.func.max(DagRun.logical_date).label("max_execution_date"), ) .group_by(DagRun.dag_id) .subquery() @@ -1259,7 +1259,7 @@ def task_stats(self, session: Session = NEW_SESSION): if conf.getboolean("webserver", "SHOW_RECENT_STATS_FOR_COMPLETED_RUNS", fallback=True): last_dag_run = ( - select(DagRun.dag_id, sqla.func.max(DagRun.execution_date).label("execution_date")) + select(DagRun.dag_id, sqla.func.max(DagRun.logical_date).label("logical_date")) .join(DagModel, DagModel.dag_id == DagRun.dag_id) .where(DagRun.state != DagRunState.RUNNING, DagModel.is_active) .group_by(DagRun.dag_id) @@ -1281,7 +1281,7 @@ def task_stats(self, session: Session = NEW_SESSION): last_dag_run, and_( last_dag_run.c.dag_id == TaskInstance.dag_id, - last_dag_run.c.execution_date == DagRun.execution_date, + last_dag_run.c.logical_date == DagRun.logical_date, ), ) ) @@ -1336,7 +1336,7 @@ def last_dagruns(self, session: Session = NEW_SESSION): last_runs_subquery = ( select( DagRun.dag_id, - sqla.func.max(DagRun.execution_date).label("max_execution_date"), + sqla.func.max(DagRun.logical_date).label("max_execution_date"), ) .group_by(DagRun.dag_id) .where(DagRun.dag_id.in_(filter_dag_ids)) # Only include accessible/selected DAGs. @@ -1349,14 +1349,14 @@ def last_dagruns(self, session: Session = NEW_SESSION): DagRun.start_date, DagRun.end_date, DagRun.state, - DagRun.execution_date, + DagRun.logical_date, DagRun.data_interval_start, DagRun.data_interval_end, ).join( last_runs_subquery, and_( last_runs_subquery.c.dag_id == DagRun.dag_id, - last_runs_subquery.c.max_execution_date == DagRun.execution_date, + last_runs_subquery.c.max_execution_date == DagRun.logical_date, ), ) ) @@ -1365,7 +1365,7 @@ def last_dagruns(self, session: Session = NEW_SESSION): r.dag_id.replace(".", "__dot__"): { "dag_id": r.dag_id, "state": r.state, - "execution_date": wwwutils.datetime_to_string(r.execution_date), + "logical_date": wwwutils.datetime_to_string(r.logical_date), "start_date": wwwutils.datetime_to_string(r.start_date), "end_date": wwwutils.datetime_to_string(r.end_date), "data_interval_start": wwwutils.datetime_to_string(r.data_interval_start), @@ -1410,14 +1410,14 @@ def rendered_templates(self, session): dag_id = request.args.get("dag_id") task_id = request.args.get("task_id") map_index = request.args.get("map_index", -1, type=int) - execution_date = request.args.get("execution_date") - dttm = _safe_parse_datetime(execution_date) - form = DateTimeForm(data={"execution_date": dttm}) + logical_date = request.args.get("logical_date") + dttm = _safe_parse_datetime(logical_date) + form = DateTimeForm(data={"logical_date": dttm}) root = request.args.get("root", "") logger.info("Retrieving rendered templates.") dag: DAG = get_airflow_app().dag_bag.get_dag(dag_id) - dag_run = dag.get_dagrun(execution_date=dttm) + dag_run = dag.get_dagrun(logical_date=dttm) raw_task = dag.get_task(task_id).prepare_for_execution() no_dagrun = False @@ -1434,7 +1434,7 @@ def rendered_templates(self, session): # "fakes" a temporary DagRun-TaskInstance association (not saved to # database) for presentation only. ti = TaskInstance(raw_task, map_index=map_index) - ti.dag_run = DagRun(dag_id=dag_id, execution_date=dttm) + ti.dag_run = DagRun(dag_id=dag_id, logical_date=dttm) no_dagrun = True else: ti = dag_run.get_task_instance(task_id=task_id, map_index=map_index, session=session) @@ -1451,7 +1451,7 @@ def rendered_templates(self, session): html_dict=html_dict, dag=dag, task_id=task_id, - execution_date=execution_date, + logical_date=logical_date, map_index=map_index, form=form, root=root, @@ -1517,7 +1517,7 @@ def rendered_templates(self, session): dag=dag, task_id=task_id, task_display_name=task.task_display_name, - execution_date=execution_date, + logical_date=logical_date, map_index=map_index, form=form, root=root, @@ -1541,16 +1541,21 @@ def rendered_k8s(self, *, session: Session = NEW_SESSION): if task_id is None: logger.warning("Task id not passed in the request") abort(400) - execution_date = request.args.get("execution_date") - dttm = _safe_parse_datetime(execution_date) - form = DateTimeForm(data={"execution_date": dttm}) + logical_date = request.args.get("logical_date") + dttm = _safe_parse_datetime(logical_date) + form = DateTimeForm(data={"logical_date": dttm}) root = request.args.get("root", "") map_index = request.args.get("map_index", -1, type=int) logger.info("Retrieving rendered k8s.") dag: DAG = get_airflow_app().dag_bag.get_dag(dag_id) task = dag.get_task(task_id) - dag_run = dag.get_dagrun(execution_date=dttm, session=session) + dag_run = dag.get_dagrun( + run_id=session.scalar( + select(DagRun.run_id).where(DagRun.logical_date == dttm).order_by(DagRun.id.desc()).limit(1) + ), + session=session, + ) ti = dag_run.get_task_instance(task_id=task.task_id, map_index=map_index, session=session) if not ti: @@ -1584,7 +1589,7 @@ def rendered_k8s(self, *, session: Session = NEW_SESSION): dag=dag, task_id=task_id, task_display_name=task.task_display_name, - execution_date=execution_date, + logical_date=logical_date, map_index=map_index, form=form, root=root, @@ -1607,6 +1612,8 @@ def rendered_k8s_data(self, *, session: Session = NEW_SESSION): if task_id is None: return {"error": "Task id not passed in the request"}, 404 run_id = request.args.get("run_id") + if run_id is None: + return {"error": "Run id not passed in the request"}, 404 map_index = request.args.get("map_index", -1, type=int) logger.info("Retrieving rendered k8s data.") @@ -1641,7 +1648,7 @@ def get_logs_with_metadata(self, session: Session = NEW_SESSION): """Retrieve logs including metadata.""" dag_id = request.args.get("dag_id") task_id = request.args.get("task_id") - execution_date_str = request.args["execution_date"] + logical_date_str = request.args["logical_date"] map_index = request.args.get("map_index", -1, type=int) try_number = request.args.get("try_number", type=int) metadata_str = request.args.get("metadata", "{}") @@ -1655,10 +1662,10 @@ def get_logs_with_metadata(self, session: Session = NEW_SESSION): # Convert string datetime into actual datetime try: - execution_date = timezone.parse(execution_date_str, strict=True) + logical_date = timezone.parse(logical_date_str, strict=True) except ValueError: error_message = ( - f"Given execution date {execution_date_str!r} could not be identified as a date. " + f"Given logical date {logical_date_str!r} could not be identified as a date. " "Example date format: 2015-11-16T14:34:15+00:00" ) return {"error": error_message}, 400 @@ -1676,7 +1683,7 @@ def get_logs_with_metadata(self, session: Session = NEW_SESSION): .where( TaskInstance.task_id == task_id, TaskInstance.dag_id == dag_id, - TaskInstance.execution_date == execution_date, + TaskInstance.logical_date == logical_date, TaskInstance.map_index == map_index, ) .join(TaskInstance.dag_run) @@ -1722,19 +1729,19 @@ def log(self, session: Session = NEW_SESSION): dag_id = request.args["dag_id"] task_id = request.args.get("task_id") map_index = request.args.get("map_index", -1, type=int) - execution_date = request.args.get("execution_date") + logical_date = request.args.get("logical_date") - if execution_date: - dttm = _safe_parse_datetime(execution_date) + if logical_date: + dttm = _safe_parse_datetime(logical_date) else: dttm = None - form = DateTimeForm(data={"execution_date": dttm}) + form = DateTimeForm(data={"logical_date": dttm}) dag_model = DagModel.get_dagmodel(dag_id) ti: TaskInstance = session.scalar( select(models.TaskInstance) - .filter_by(dag_id=dag_id, task_id=task_id, execution_date=dttm, map_index=map_index) + .filter_by(dag_id=dag_id, task_id=task_id, logical_date=dttm, map_index=map_index) .limit(1) ) @@ -1753,7 +1760,7 @@ def log(self, session: Session = NEW_SESSION): dag_id=dag_id, task_id=task_id, task_display_name=ti.task_display_name, - execution_date=execution_date, + logical_date=logical_date, map_index=map_index, form=form, root=root, @@ -1767,14 +1774,14 @@ def redirect_to_external_log(self, session: Session = NEW_SESSION): """Redirects to external log.""" dag_id = request.args.get("dag_id") task_id = request.args.get("task_id") - execution_date = request.args.get("execution_date") - dttm = _safe_parse_datetime(execution_date) + logical_date = request.args.get("logical_date") + dttm = _safe_parse_datetime(logical_date) map_index = request.args.get("map_index", -1, type=int) try_number = request.args.get("try_number", 1) ti = session.scalar( select(models.TaskInstance) - .filter_by(dag_id=dag_id, task_id=task_id, execution_date=dttm, map_index=map_index) + .filter_by(dag_id=dag_id, task_id=task_id, logical_date=dttm, map_index=map_index) .limit(1) ) @@ -1798,10 +1805,10 @@ def task(self, session: Session = NEW_SESSION): """Retrieve task.""" dag_id = request.args.get("dag_id") task_id = request.args.get("task_id") - execution_date = request.args.get("execution_date") - dttm = _safe_parse_datetime(execution_date) + logical_date = request.args.get("logical_date") + dttm = _safe_parse_datetime(logical_date) map_index = request.args.get("map_index", -1, type=int) - form = DateTimeForm(data={"execution_date": dttm}) + form = DateTimeForm(data={"logical_date": dttm}) root = request.args.get("root", "") dag = get_airflow_app().dag_bag.get_dag(dag_id) @@ -1820,7 +1827,7 @@ def task(self, session: Session = NEW_SESSION): joinedload(TaskInstance.queued_by_job, innerjoin=False), joinedload(TaskInstance.trigger, innerjoin=False), ) - .filter_by(execution_date=dttm, dag_id=dag_id, task_id=task_id, map_index=map_index) + .filter_by(logical_date=dttm, dag_id=dag_id, task_id=task_id, map_index=map_index) ) if ti is None: ti_attrs: list[tuple[str, Any]] | None = None @@ -1908,7 +1915,7 @@ def include_task_attrs(attr_name): dag_run_id=ti.run_id if ti else "", failed_dep_reasons=failed_dep_reasons or no_failed_deps_result, task_id=task_id, - execution_date=execution_date, + logical_date=logical_date, map_index=map_index, special_attrs_rendered=special_attrs_rendered, form=form, @@ -1926,12 +1933,12 @@ def xcom(self, session: Session = NEW_SESSION): dag_id = request.args["dag_id"] task_id = request.args.get("task_id") map_index = request.args.get("map_index", -1, type=int) - # Carrying execution_date through, even though it's irrelevant for + # Carrying logical_date through, even though it's irrelevant for # this context - execution_date = request.args.get("execution_date") - dttm = _safe_parse_datetime(execution_date) + logical_date = request.args.get("logical_date") + dttm = _safe_parse_datetime(logical_date) - form = DateTimeForm(data={"execution_date": dttm}) + form = DateTimeForm(data={"logical_date": dttm}) root = request.args.get("root", "") dag = DagModel.get_dagmodel(dag_id) ti: TaskInstance = session.scalar( @@ -1946,7 +1953,7 @@ def xcom(self, session: Session = NEW_SESSION): select(XCom).where( XCom.dag_id == dag_id, XCom.task_id == task_id, - XCom.execution_date == dttm, + XCom.logical_date == dttm, XCom.map_index == map_index, ) ) @@ -1960,7 +1967,7 @@ def xcom(self, session: Session = NEW_SESSION): task_id=task_id, dag_run_id=ti.run_id if ti else "", task_display_name=ti.task_display_name, - execution_date=execution_date, + logical_date=logical_date, map_index=map_index, form=form, root=root, @@ -2009,7 +2016,7 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): origin = get_safe_url(request.values.get("origin")) unpause = request.values.get("unpause") request_conf = request.values.get("conf") - request_execution_date = request.values.get("execution_date", default=timezone.utcnow().isoformat()) + request_logical_date = request.values.get("logical_date", default=timezone.utcnow().isoformat()) is_dag_run_conf_overrides_params = conf.getboolean("core", "dag_run_conf_overrides_params") dag = get_airflow_app().dag_bag.get_dag(dag_id) dag_orm: DagModel = session.scalar(select(DagModel).where(DagModel.dag_id == dag_id).limit(1)) @@ -2066,14 +2073,14 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): num_recent_confs = conf.getint("webserver", "num_recent_configurations_for_trigger") recent_runs = session.execute( - select(DagRun.conf, func.max(DagRun.run_id).label("run_id"), func.max(DagRun.execution_date)) + select(DagRun.conf, func.max(DagRun.run_id).label("run_id"), func.max(DagRun.logical_date)) .where( DagRun.dag_id == dag_id, DagRun.run_type == DagRunType.MANUAL, DagRun.conf.isnot(None), ) .group_by(DagRun.conf) - .order_by(func.max(DagRun.execution_date).desc()) + .order_by(func.max(DagRun.logical_date).desc()) .limit(num_recent_confs) ) recent_confs = { @@ -2097,7 +2104,7 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): # Populate conf textarea with conf requests parameter, or dag.params default_conf = "" - form = DateTimeForm(data={"execution_date": request_execution_date}) + form = DateTimeForm(data={"logical_date": request_logical_date}) if request_conf: default_conf = request_conf @@ -2125,10 +2132,10 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): ) try: - execution_date = timezone.parse(request_execution_date, strict=True) + logical_date = timezone.parse(request_logical_date, strict=True) except ParserError: - flash("Invalid execution date", "error") - form = DateTimeForm(data={"execution_date": timezone.utcnow().isoformat()}) + flash("Invalid logical date", "error") + form = DateTimeForm(data={"logical_date": timezone.utcnow().isoformat()}) return self.render_template( "airflow/trigger.html", form_fields=form_fields, @@ -2137,12 +2144,12 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): form=form, ) - dr = DagRun.find_duplicate(dag_id=dag_id, run_id=run_id, execution_date=execution_date) + dr = DagRun.find_duplicate(dag_id=dag_id, run_id=run_id, logical_date=logical_date) if dr: if dr.run_id == run_id: message = f"The run ID {run_id} already exists" else: - message = f"The logical date {execution_date} already exists" + message = f"The logical date {logical_date} already exists" flash(message, "error") return redirect(origin) @@ -2155,7 +2162,7 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): "error", ) - form = DateTimeForm(data={"execution_date": execution_date}) + form = DateTimeForm(data={"logical_date": logical_date}) return self.render_template( "airflow/trigger.html", form_fields=form_fields, @@ -2170,7 +2177,7 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): run_conf = json.loads(request_conf) if not isinstance(run_conf, dict): flash("Invalid JSON configuration, must be a dict", "error") - form = DateTimeForm(data={"execution_date": execution_date}) + form = DateTimeForm(data={"logical_date": logical_date}) return self.render_template( "airflow/trigger.html", form_fields=form_fields, @@ -2180,7 +2187,7 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): ) except json.decoder.JSONDecodeError: flash("Invalid JSON configuration, not parseable", "error") - form = DateTimeForm(data={"execution_date": execution_date}) + form = DateTimeForm(data={"logical_date": logical_date}) return self.render_template( "airflow/trigger.html", form_fields=form_fields, @@ -2205,8 +2212,8 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): dag_version = DagVersion.get_latest_version(dag.dag_id) dag_run = dag.create_dagrun( run_type=DagRunType.MANUAL, - execution_date=execution_date, - data_interval=dag.timetable.infer_manual_data_interval(run_after=execution_date), + logical_date=logical_date, + data_interval=dag.timetable.infer_manual_data_interval(run_after=logical_date), state=DagRunState.QUEUED, conf=run_conf, external_trigger=True, @@ -2216,7 +2223,7 @@ def trigger(self, dag_id: str, session: Session = NEW_SESSION): ) except (ValueError, ParamValidationError) as ve: flash(f"{ve}", "error") - form = DateTimeForm(data={"execution_date": execution_date}) + form = DateTimeForm(data={"logical_date": logical_date}) # Take over "bad" submitted fields for new form display for k in form_fields: if k in run_conf: @@ -2316,8 +2323,8 @@ def clear(self, *, session: Session = NEW_SESSION): else: map_indexes = request.form.getlist("map_index", type=int) - execution_date_str = request.form.get("execution_date") - execution_date = _safe_parse_datetime(execution_date_str) + logical_date_str = request.form.get("logical_date") + logical_date = _safe_parse_datetime(logical_date_str) confirmed = request.form.get("confirmed") == "true" upstream = request.form.get("upstream") == "true" downstream = request.form.get("downstream") == "true" @@ -2328,8 +2335,8 @@ def clear(self, *, session: Session = NEW_SESSION): task_ids: list[str | tuple[str, int]] = [] - end_date = execution_date if not future else None - start_date = execution_date if not past else None + end_date = logical_date if not future else None + start_date = logical_date if not past else None locked_dag_run_ids: list[int] = [] @@ -2347,13 +2354,13 @@ def clear(self, *, session: Session = NEW_SESSION): dag_runs_query = select(DagRun.id).where(DagRun.dag_id == dag_id).with_for_update() if start_date is None and end_date is None: - dag_runs_query = dag_runs_query.where(DagRun.execution_date == start_date) + dag_runs_query = dag_runs_query.where(DagRun.logical_date == start_date) else: if start_date is not None: - dag_runs_query = dag_runs_query.where(DagRun.execution_date >= start_date) + dag_runs_query = dag_runs_query.where(DagRun.logical_date >= start_date) if end_date is not None: - dag_runs_query = dag_runs_query.where(DagRun.execution_date <= end_date) + dag_runs_query = dag_runs_query.where(DagRun.logical_date <= end_date) locked_dag_run_ids = session.scalars(dag_runs_query).all() elif task_id: @@ -2673,8 +2680,8 @@ def confirm(self): msg = f"Invalid state {state}, must be either 'success' or 'failed'" return redirect_or_json(origin, msg, status="error", status_code=400) - latest_execution_date = dag.get_latest_execution_date() - if not latest_execution_date: + latest_logical_date = dag.get_latest_logical_date() + if not latest_logical_date: msg = f"Cannot mark tasks as {state}, seem that dag {dag_id} has never run" return redirect_or_json(origin, msg, status="error", status_code=400) @@ -2947,15 +2954,15 @@ def calendar_data(self, session: Session = NEW_SESSION): dag_states = session.execute( select( - func.date(DagRun.execution_date).label("date"), + func.date(DagRun.logical_date).label("date"), DagRun.state, func.max(DagRun.data_interval_start).label("data_interval_start"), func.max(DagRun.data_interval_end).label("data_interval_end"), func.count("*").label("count"), ) .where(DagRun.dag_id == dag.dag_id) - .group_by(func.date(DagRun.execution_date), DagRun.state) - .order_by(func.date(DagRun.execution_date).asc()) + .group_by(func.date(DagRun.logical_date), DagRun.state) + .order_by(func.date(DagRun.logical_date).asc()) ).all() data_dag_states = [ @@ -3054,7 +3061,7 @@ def graph(self, dag_id: str, session: Session = NEW_SESSION): return redirect(url_for("Airflow.index")) dt_nr_dr_data = get_date_time_num_runs_dag_runs_form_data(request, session, dag) dttm = dt_nr_dr_data["dttm"] - dag_run = dag.get_dagrun(execution_date=dttm) + dag_run = dag.get_dagrun(logical_date=dttm) dag_run_id = dag_run.run_id if dag_run else None kwargs = { @@ -3129,7 +3136,7 @@ def gantt(self, dag_id: str, session: Session = NEW_SESSION): dag = get_airflow_app().dag_bag.get_dag(dag_id, session=session) dt_nr_dr_data = get_date_time_num_runs_dag_runs_form_data(request, session, dag) dttm = dt_nr_dr_data["dttm"] - dag_run = dag.get_dagrun(execution_date=dttm) + dag_run = dag.get_dagrun(logical_date=dttm) dag_run_id = dag_run.run_id if dag_run else None kwargs = {**sanitize_args(request.args), "dag_id": dag_id, "tab": "gantt", "dag_run_id": dag_run_id} @@ -3149,7 +3156,7 @@ def extra_links(self, *, session: Session = NEW_SESSION): API: GET Args: dag_id: The id of the dag containing the task in question task_id: The id of the task in question - execution_date: The date of execution of the task + logical_date: The date of execution of the task link_name: The name of the link reference to find the actual URL for Returns: @@ -3161,8 +3168,8 @@ def extra_links(self, *, session: Session = NEW_SESSION): dag_id = request.args.get("dag_id") task_id = request.args.get("task_id") map_index = request.args.get("map_index", -1, type=int) - execution_date = request.args.get("execution_date") - dttm = _safe_parse_datetime(execution_date) + logical_date = request.args.get("logical_date") + dttm = _safe_parse_datetime(logical_date) dag = get_airflow_app().dag_bag.get_dag(dag_id) if not dag or task_id not in dag.task_ids: @@ -3175,7 +3182,7 @@ def extra_links(self, *, session: Session = NEW_SESSION): ti = session.scalar( select(TaskInstance) - .filter_by(dag_id=dag_id, task_id=task_id, execution_date=dttm, map_index=map_index) + .filter_by(dag_id=dag_id, task_id=task_id, logical_date=dttm, map_index=map_index) .options(joinedload(TaskInstance.dag_run)) .limit(1) ) @@ -3226,11 +3233,11 @@ def task_instances(self): dag_id = request.args.get("dag_id") dag = get_airflow_app().dag_bag.get_dag(dag_id) - dttm = request.args.get("execution_date") + dttm = request.args.get("logical_date") if dttm: dttm = _safe_parse_datetime(dttm) else: - return {"error": f"Invalid execution_date {dttm}"}, 400 + return {"error": f"Invalid logical_date {dttm}"}, 400 with create_session() as session: task_instances = { @@ -3265,10 +3272,10 @@ def grid_data(self): try: base_date = timezone.parse(request.args["base_date"], strict=True) except (KeyError, ValueError): - base_date = dag.get_latest_execution_date() or timezone.utcnow() + base_date = dag.get_latest_logical_date() or timezone.utcnow() with create_session() as session: - query = select(DagRun).where(DagRun.dag_id == dag.dag_id, DagRun.execution_date <= base_date) + query = select(DagRun).where(DagRun.dag_id == dag.dag_id, DagRun.logical_date <= base_date) run_types = request.args.getlist("run_type") if run_types: @@ -3400,8 +3407,8 @@ def next_run_assets(self, dag_id): and_( AssetEvent.asset_id == AssetModel.id, ( - AssetEvent.timestamp >= latest_run.execution_date - if latest_run and latest_run.execution_date + AssetEvent.timestamp >= latest_run.logical_date + if latest_run and latest_run.logical_date else True ), ), @@ -3854,8 +3861,8 @@ class XComModelView(AirflowModelView): permissions.ACTION_CAN_ACCESS_MENU, ] - search_columns = ["key", "value", "timestamp", "dag_id", "task_id", "run_id", "execution_date"] - list_columns = ["key", "value", "timestamp", "dag_id", "task_id", "run_id", "map_index", "execution_date"] + search_columns = ["key", "value", "timestamp", "dag_id", "task_id", "run_id", "logical_date"] + list_columns = ["key", "value", "timestamp", "dag_id", "task_id", "run_id", "map_index", "logical_date"] base_order = ("dag_run_id", "desc") order_columns = [ @@ -3866,7 +3873,7 @@ class XComModelView(AirflowModelView): "task_id", "run_id", "map_index", - # "execution_date", # execution_date sorting is not working and crashing the UI, disabled for now. + # "logical_date", # logical_date sorting is not working and crashing the UI, disabled for now. ] base_filters = [["dag_id", DagFilter, list]] @@ -3876,7 +3883,7 @@ class XComModelView(AirflowModelView): "timestamp": wwwutils.datetime_f("timestamp"), "dag_id": wwwutils.dag_link, "map_index": wwwutils.format_map_index, - "execution_date": wwwutils.datetime_f("execution_date"), + "logical_date": wwwutils.datetime_f("logical_date"), } @action("muldelete", "Delete", "Are you sure you want to delete selected records?", single=False) @@ -3889,7 +3896,7 @@ def action_muldelete(self, items): def pre_add(self, item): """Pre add hook.""" - item.execution_date = timezone.make_aware(item.execution_date) + item.logical_date = timezone.make_aware(item.logical_date) item.value = XCom.serialize_value( value=item.value, key=item.key, @@ -3901,7 +3908,7 @@ def pre_add(self, item): def pre_update(self, item): """Pre update hook.""" - item.execution_date = timezone.make_aware(item.execution_date) + item.logical_date = timezone.make_aware(item.logical_date) item.value = XCom.serialize_value( value=item.value, key=item.key, @@ -4721,7 +4728,7 @@ class DagRunModelView(AirflowModelView): list_columns = [ "state", "dag_id", - "execution_date", + "logical_date", "run_id", "run_type", "queued_at", @@ -4735,7 +4742,7 @@ class DagRunModelView(AirflowModelView): search_columns = [ "state", "dag_id", - "execution_date", + "logical_date", "run_id", "run_type", "start_date", @@ -4744,12 +4751,12 @@ class DagRunModelView(AirflowModelView): "external_trigger", ] label_columns = { - "execution_date": "Logical Date", + "logical_date": "Logical Date", } edit_columns = [ "state", "dag_id", - "execution_date", + "logical_date", "start_date", "end_date", "run_id", @@ -4761,7 +4768,7 @@ class DagRunModelView(AirflowModelView): order_columns = [ "state", "dag_id", - "execution_date", + "logical_date", "run_id", "run_type", "queued_at", @@ -4772,7 +4779,7 @@ class DagRunModelView(AirflowModelView): "conf", ] - base_order = ("execution_date", "desc") + base_order = ("logical_date", "desc") base_filters = [["dag_id", DagFilter, list]] @@ -4790,7 +4797,7 @@ def duration_f(self): return difference formatters_columns = { - "execution_date": wwwutils.datetime_f("execution_date"), + "logical_date": wwwutils.datetime_f("logical_date"), "state": wwwutils.state_f, "start_date": wwwutils.datetime_f("start_date"), "end_date": wwwutils.datetime_f("end_date"), @@ -4951,7 +4958,7 @@ class LogModelView(AirflowModelView): "task_id", "run_id", "event", - "execution_date", + "logical_date", "owner", "owner_display_name", "extra", @@ -4962,14 +4969,14 @@ class LogModelView(AirflowModelView): "task_id", "run_id", "event", - "execution_date", + "logical_date", "owner", "owner_display_name", "extra", ] label_columns = { - "execution_date": "Logical Date", + "logical_date": "Logical Date", "owner": "Owner ID", "owner_display_name": "Owner Name", } @@ -4980,7 +4987,7 @@ class LogModelView(AirflowModelView): formatters_columns = { "dttm": wwwutils.datetime_f("dttm"), - "execution_date": wwwutils.datetime_f("execution_date"), + "logical_date": wwwutils.datetime_f("logical_date"), "dag_id": wwwutils.dag_link, } @@ -5007,7 +5014,7 @@ class TaskRescheduleModelView(AirflowModelView): "id", "dag_id", "run_id", - "dag_run.execution_date", + "dag_run.logical_date", "task_id", "map_index", "try_number", @@ -5018,14 +5025,14 @@ class TaskRescheduleModelView(AirflowModelView): ] label_columns = { - "dag_run.execution_date": "Logical Date", + "dag_run.logical_date": "Logical Date", } search_columns = [ "dag_id", "task_id", "run_id", - "execution_date", + "logical_date", "start_date", "end_date", "reschedule_date", @@ -5048,7 +5055,7 @@ def duration_f(self): "task_id": wwwutils.task_instance_link, "start_date": wwwutils.datetime_f("start_date"), "end_date": wwwutils.datetime_f("end_date"), - "dag_run.execution_date": wwwutils.datetime_f("dag_run.execution_date"), + "dag_run.logical_date": wwwutils.datetime_f("dag_run.logical_date"), "reschedule_date": wwwutils.datetime_f("reschedule_date"), "duration": duration_f, "map_index": wwwutils.format_map_index, @@ -5128,7 +5135,7 @@ class TaskInstanceModelView(AirflowModelView): "task_id", "run_id", "map_index", - "dag_run.execution_date", + "dag_run.logical_date", "operator", "start_date", "end_date", @@ -5152,7 +5159,7 @@ class TaskInstanceModelView(AirflowModelView): "task_id", "run_id", "map_index", - "dag_run.execution_date", + "dag_run.logical_date", "operator", "start_date", "end_date", @@ -5167,7 +5174,7 @@ class TaskInstanceModelView(AirflowModelView): "queued_by_job_id", ] # todo: don't use prev_attempted_tries; just use try_number - label_columns = {"dag_run.execution_date": "Logical Date", "prev_attempted_tries": "Try Number"} + label_columns = {"dag_run.logical_date": "Logical Date", "prev_attempted_tries": "Try Number"} search_columns = [ "state", @@ -5175,7 +5182,7 @@ class TaskInstanceModelView(AirflowModelView): "task_id", "run_id", "map_index", - "execution_date", + "logical_date", "operator", "start_date", "end_date", @@ -5192,7 +5199,7 @@ class TaskInstanceModelView(AirflowModelView): edit_columns = [ "dag_id", "task_id", - "execution_date", + "logical_date", "start_date", "end_date", "state", @@ -5243,7 +5250,7 @@ def duration_f(self): "map_index": wwwutils.format_map_index, "hostname": wwwutils.nobr_f("hostname"), "state": wwwutils.state_f, - "dag_run.execution_date": wwwutils.datetime_f("dag_run.execution_date"), + "dag_run.logical_date": wwwutils.datetime_f("dag_run.logical_date"), "start_date": wwwutils.datetime_f("start_date"), "end_date": wwwutils.datetime_f("end_date"), "queued_dttm": wwwutils.datetime_f("queued_dttm"), @@ -5295,8 +5302,8 @@ def _clear_task_instances( # dag.clear returns TIs when in dry run mode downstream_tis_to_clear.extend( dag.clear( - start_date=dag_run.execution_date, - end_date=dag_run.execution_date, + start_date=dag_run.logical_date, + end_date=dag_run.logical_date, task_ids=downstream_task_ids_to_clear, session=session, dry_run=True, diff --git a/dev/perf/dags/sql_perf_dag.py b/dev/perf/dags/sql_perf_dag.py index bfa28ea6b814..2081eb482ae0 100644 --- a/dev/perf/dags/sql_perf_dag.py +++ b/dev/perf/dags/sql_perf_dag.py @@ -36,9 +36,9 @@ def print_context(_, ti, **kwargs): """ - Print the task_id and execution date. + Print the task_id and logical date. """ - print(f"Running {ti.task_id} {ti.execution_date}") + print(f"Running {ti.task_id} {ti.logical_date}") return "Whatever you return gets printed in the logs" diff --git a/dev/perf/scheduler_dag_execution_timing.py b/dev/perf/scheduler_dag_execution_timing.py index cbc4ca6e8fc6..f8760f4c03c8 100755 --- a/dev/perf/scheduler_dag_execution_timing.py +++ b/dev/perf/scheduler_dag_execution_timing.py @@ -68,7 +68,7 @@ def change_state(self, key, state, info=None): super().change_state(key, state, info=info) - dag_id, _, execution_date, __ = key + dag_id, _, logical_date, __ = key if dag_id not in self.dags_to_watch: return @@ -76,15 +76,15 @@ def change_state(self, key, state, info=None): # check the DR.state - so instead we need to check the state of the # tasks in that run - run = self.dags_to_watch[dag_id].runs.get(execution_date) + run = self.dags_to_watch[dag_id].runs.get(logical_date) if not run: import airflow.models - run = airflow.models.DagRun.find(dag_id=dag_id, execution_date=execution_date)[0] - self.dags_to_watch[dag_id].runs[execution_date] = run + run = airflow.models.DagRun.find(dag_id=dag_id, logical_date=logical_date)[0] + self.dags_to_watch[dag_id].runs[logical_date] = run if run and all(t.state == TaskInstanceState.SUCCESS for t in run.get_task_instances()): - self.dags_to_watch[dag_id].runs.pop(execution_date) + self.dags_to_watch[dag_id].runs.pop(logical_date) self.dags_to_watch[dag_id].waiting_for -= 1 if self.dags_to_watch[dag_id].waiting_for == 0: @@ -171,7 +171,7 @@ def create_dag_runs(dag, num_runs, session): logical_date = next_info.logical_date dag.create_dagrun( run_id=f"{id_prefix}{logical_date.isoformat()}", - execution_date=logical_date, + logical_date=logical_date, start_date=timezone.utcnow(), state=DagRunState.RUNNING, external_trigger=False, diff --git a/docs/apache-airflow/templates-ref.rst b/docs/apache-airflow/templates-ref.rst index b29690c096ba..78e5c17f9925 100644 --- a/docs/apache-airflow/templates-ref.rst +++ b/docs/apache-airflow/templates-ref.rst @@ -108,35 +108,6 @@ is an example of accessing a ``task_instance`` object from its task: .. include:: ../shared/template-examples/taskflow.rst -Deprecated variables -------------------------------------------------------- - -The following variables are deprecated. They are kept for backward compatibility, but you should convert -existing code to use other variables instead. - -===================================== ========================================================================== -Deprecated Variable Description -===================================== ========================================================================== -``{{ execution_date }}`` the execution date (logical date), same as ``logical_date`` -``{{ next_execution_date }}`` the logical date of the next scheduled run, - you may be able to use ``data_interval_end`` instead; for manually - triggered dagruns that aren't on a schedule, ``next_execution_date`` is - set to ``logical_date`` -``{{ next_ds }}`` the next execution date as ``YYYY-MM-DD`` if exists, else ``None`` -``{{ next_ds_nodash }}`` the next execution date as ``YYYYMMDD`` if exists, else ``None`` -``{{ prev_execution_date }}`` the logical date of the previous scheduled run (if applicable) -``{{ prev_ds }}`` the previous execution date as ``YYYY-MM-DD`` if exists, else ``None`` -``{{ prev_ds_nodash }}`` the previous execution date as ``YYYYMMDD`` if exists, else ``None`` -``{{ yesterday_ds }}`` the day before the execution date as ``YYYY-MM-DD`` -``{{ yesterday_ds_nodash }}`` the day before the execution date as ``YYYYMMDD`` -``{{ tomorrow_ds }}`` the day after the execution date as ``YYYY-MM-DD`` -``{{ tomorrow_ds_nodash }}`` the day after the execution date as ``YYYYMMDD`` -``{{ prev_execution_date_success }}`` execution date from prior successful DAG run; - you may be able to use ``prev_data_interval_start_success`` instead if - the timetable/schedule you use for the DAG defines ``data_interval_start`` - compatible with the legacy ``execution_date``. -===================================== ========================================================================== - Note that you can access the object's attributes and methods with simple dot notation. Here are some examples of what is possible: ``{{ task.owner }}``, ``{{ task.task_id }}``, ``{{ ti.hostname }}``, ... diff --git a/kubernetes_tests/test_base.py b/kubernetes_tests/test_base.py index 6929128173cb..521a02ddd91d 100644 --- a/kubernetes_tests/test_base.py +++ b/kubernetes_tests/test_base.py @@ -191,7 +191,7 @@ def monitor_task(self, host, dag_run_id, dag_id, task_id, expected_final_state, print(f"The expected state is wrong {state} != {expected_final_state} (expected)!") assert state == expected_final_state - def ensure_dag_expected_state(self, host, execution_date, dag_id, expected_final_state, timeout): + def ensure_dag_expected_state(self, host, logical_date, dag_id, expected_final_state, timeout): tries = 0 state = "" max_tries = max(int(timeout / 5), 1) @@ -207,7 +207,7 @@ def ensure_dag_expected_state(self, host, execution_date, dag_id, expected_final print(f"Received: {result}") state = None for dag_run in result_json["dag_runs"]: - if dag_run["execution_date"] == execution_date: + if dag_run["logical_date"] == logical_date: state = dag_run["state"] check_call(["echo", f"Attempt {tries}: Current state of dag is {state}"]) print(f"Attempt {tries}: Current state of dag is {state}") @@ -271,12 +271,12 @@ def start_job_in_kubernetes(self, dag_id, host): result_json = self.start_dag(dag_id=dag_id, host=host) dag_runs = result_json["dag_runs"] assert len(dag_runs) > 0 - execution_date = None + logical_date = None dag_run_id = None for dag_run in dag_runs: if dag_run["dag_id"] == dag_id: - execution_date = dag_run["execution_date"] + logical_date = dag_run["logical_date"] dag_run_id = dag_run["dag_run_id"] break - assert execution_date is not None, f"No execution_date can be found for the dag with {dag_id}" - return dag_run_id, execution_date + assert logical_date is not None, f"No logical_date can be found for the dag with {dag_id}" + return dag_run_id, logical_date diff --git a/kubernetes_tests/test_kubernetes_executor.py b/kubernetes_tests/test_kubernetes_executor.py index 42b181b44304..a0bd83215618 100644 --- a/kubernetes_tests/test_kubernetes_executor.py +++ b/kubernetes_tests/test_kubernetes_executor.py @@ -31,8 +31,8 @@ class TestKubernetesExecutor(BaseK8STest): @pytest.mark.execution_timeout(300) def test_integration_run_dag(self): dag_id = "example_kubernetes_executor" - dag_run_id, execution_date = self.start_job_in_kubernetes(dag_id, self.host) - print(f"Found the job with execution_date {execution_date}") + dag_run_id, logical_date = self.start_job_in_kubernetes(dag_id, self.host) + print(f"Found the job with logical_date {logical_date}") # Wait some time for the operator to complete self.monitor_task( @@ -46,7 +46,7 @@ def test_integration_run_dag(self): self.ensure_dag_expected_state( host=self.host, - execution_date=execution_date, + logical_date=logical_date, dag_id=dag_id, expected_final_state="success", timeout=300, @@ -56,7 +56,7 @@ def test_integration_run_dag(self): def test_integration_run_dag_with_scheduler_failure(self): dag_id = "example_kubernetes_executor" - dag_run_id, execution_date = self.start_job_in_kubernetes(dag_id, self.host) + dag_run_id, logical_date = self.start_job_in_kubernetes(dag_id, self.host) self._delete_airflow_pod("scheduler") @@ -83,7 +83,7 @@ def test_integration_run_dag_with_scheduler_failure(self): self.ensure_dag_expected_state( host=self.host, - execution_date=execution_date, + logical_date=logical_date, dag_id=dag_id, expected_final_state="success", timeout=300, diff --git a/kubernetes_tests/test_kubernetes_pod_operator.py b/kubernetes_tests/test_kubernetes_pod_operator.py index 99e30c710d2b..9c29f076dcac 100644 --- a/kubernetes_tests/test_kubernetes_pod_operator.py +++ b/kubernetes_tests/test_kubernetes_pod_operator.py @@ -51,13 +51,11 @@ def create_context(task) -> Context: dag = DAG(dag_id="dag", schedule=None) - execution_date = timezone.datetime( - 2016, 1, 1, 1, 0, 0, tzinfo=timezone.parse_timezone("Europe/Amsterdam") - ) + logical_date = timezone.datetime(2016, 1, 1, 1, 0, 0, tzinfo=timezone.parse_timezone("Europe/Amsterdam")) dag_run = DagRun( dag_id=dag.dag_id, - execution_date=execution_date, - run_id=DagRun.generate_run_id(DagRunType.MANUAL, execution_date), + logical_date=logical_date, + run_id=DagRun.generate_run_id(DagRunType.MANUAL, logical_date), ) task_instance = TaskInstance(task=task) task_instance.dag_run = dag_run diff --git a/kubernetes_tests/test_other_executors.py b/kubernetes_tests/test_other_executors.py index 3a1aea16f129..638ca49ad445 100644 --- a/kubernetes_tests/test_other_executors.py +++ b/kubernetes_tests/test_other_executors.py @@ -33,8 +33,8 @@ class TestCeleryAndLocalExecutor(BaseK8STest): def test_integration_run_dag(self): dag_id = "example_bash_operator" - dag_run_id, execution_date = self.start_job_in_kubernetes(dag_id, self.host) - print(f"Found the job with execution_date {execution_date}") + dag_run_id, logical_date = self.start_job_in_kubernetes(dag_id, self.host) + print(f"Found the job with logical_date {logical_date}") # Wait some time for the operator to complete self.monitor_task( @@ -48,7 +48,7 @@ def test_integration_run_dag(self): self.ensure_dag_expected_state( host=self.host, - execution_date=execution_date, + logical_date=logical_date, dag_id=dag_id, expected_final_state="success", timeout=300, @@ -57,7 +57,7 @@ def test_integration_run_dag(self): def test_integration_run_dag_with_scheduler_failure(self): dag_id = "example_xcom" - dag_run_id, execution_date = self.start_job_in_kubernetes(dag_id, self.host) + dag_run_id, logical_date = self.start_job_in_kubernetes(dag_id, self.host) self._delete_airflow_pod("scheduler") @@ -84,7 +84,7 @@ def test_integration_run_dag_with_scheduler_failure(self): self.ensure_dag_expected_state( host=self.host, - execution_date=execution_date, + logical_date=logical_date, dag_id=dag_id, expected_final_state="success", timeout=60, diff --git a/newsfragments/43902.significant.rst b/newsfragments/43902.significant.rst new file mode 100644 index 000000000000..f51eafb8d0a9 --- /dev/null +++ b/newsfragments/43902.significant.rst @@ -0,0 +1,6 @@ +Renamed ``execution_date`` to ``logical_date`` across the codebase to align with Airflow 3.0. + +The shift towards ``logical_date`` helps move away from the limitations of ``execution_date``, particularly with dynamic DAG runs and cases where multiple runs occur at the same time. This change impacts database models, templates, and functions: + +- Renamed columns and function references to ``logical_date``. +- Removed ``execution_date``, ``next_ds``, ``next_ds_nodash``, ``next_execution_date``, ``prev_ds``, ``prev_ds_nodash``, ``prev_execution_date``, ``prev_execution_date_success``, ``tomorrow_ds``, ``yesterday_ds`` and ``yesterday_ds_nodash`` from Airflow ``context``. diff --git a/providers/src/airflow/providers/apache/druid/transfers/hive_to_druid.py b/providers/src/airflow/providers/apache/druid/transfers/hive_to_druid.py index 889bb97f061a..7ae50a25148d 100644 --- a/providers/src/airflow/providers/apache/druid/transfers/hive_to_druid.py +++ b/providers/src/airflow/providers/apache/druid/transfers/hive_to_druid.py @@ -94,7 +94,7 @@ def __init__( self.sql = sql self.druid_datasource = druid_datasource self.ts_dim = ts_dim - self.intervals = intervals or ["{{ ds }}/{{ tomorrow_ds }}"] + self.intervals = intervals or ["{{ ds }}/{{ logical_date.add_days(1) | ds }}"] self.num_shards = num_shards self.target_partition_size = target_partition_size self.query_granularity = query_granularity diff --git a/providers/src/airflow/providers/apache/hive/operators/hive.py b/providers/src/airflow/providers/apache/hive/operators/hive.py index 0804b5f57e7a..aedcb7eded72 100644 --- a/providers/src/airflow/providers/apache/hive/operators/hive.py +++ b/providers/src/airflow/providers/apache/hive/operators/hive.py @@ -111,7 +111,7 @@ def __init__( job_name_template = conf.get_mandatory_value( "hive", "mapred_job_name_template", - fallback="Airflow HiveOperator task for {hostname}.{dag_id}.{task_id}.{execution_date}", + fallback="Airflow HiveOperator task for {hostname}.{dag_id}.{task_id}.{logical_date}", ) self.mapred_job_name_template: str = job_name_template @@ -140,12 +140,12 @@ def execute(self, context: Context) -> None: # set the mapred_job_name if it's not set with dag, task, execution time info if not self.mapred_job_name: ti = context["ti"] - if ti.execution_date is None: - raise RuntimeError("execution_date is None") + if ti.logical_date is None: + raise RuntimeError("logical_date is None") self.hook.mapred_job_name = self.mapred_job_name_template.format( dag_id=ti.dag_id, task_id=ti.task_id, - execution_date=ti.execution_date.isoformat(), + logical_date=ti.logical_date.isoformat(), hostname=ti.hostname.split(".")[0], ) diff --git a/providers/src/airflow/providers/cncf/kubernetes/backcompat/__init__.py b/providers/src/airflow/providers/cncf/kubernetes/backcompat/__init__.py index 13a83393a912..e3ff61d56351 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/backcompat/__init__.py +++ b/providers/src/airflow/providers/cncf/kubernetes/backcompat/__init__.py @@ -14,3 +14,20 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +from __future__ import annotations + + +def get_logical_date_key() -> str: + """ + Get the key for execution/logical date for the appropriate Airflow version. + + This is done by detecting the CLI argument name. There are various ways to + do this, but the CLI key name has a very small import footprint (especially + compared to importing ORM models). + """ + from airflow.cli import cli_config + + if hasattr(cli_config, "ARG_LOGICAL_DATE"): + return "logical_date" + return "execution_date" diff --git a/providers/src/airflow/providers/cncf/kubernetes/cli/kubernetes_command.py b/providers/src/airflow/providers/cncf/kubernetes/cli/kubernetes_command.py index 085a877482dc..1bca856fd32b 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/cli/kubernetes_command.py +++ b/providers/src/airflow/providers/cncf/kubernetes/cli/kubernetes_command.py @@ -25,7 +25,9 @@ from kubernetes import client from kubernetes.client.api_client import ApiClient from kubernetes.client.rest import ApiException +from packaging.version import Version +from airflow import __version__ as airflow_version from airflow.models import DagRun, TaskInstance from airflow.providers.cncf.kubernetes import pod_generator from airflow.providers.cncf.kubernetes.executors.kubernetes_executor import KubeConfig @@ -36,15 +38,21 @@ from airflow.utils.cli import get_dag from airflow.utils.providers_configuration_loader import providers_configuration_loaded +AIRFLOW_VERSION = Version(airflow_version) +AIRFLOW_V_3_0_PLUS = Version(AIRFLOW_VERSION.base_version) >= Version("3.0.0") + @cli_utils.action_cli @providers_configuration_loaded def generate_pod_yaml(args): """Generate yaml files for each task in the DAG. Used for testing output of KubernetesExecutor.""" - execution_date = args.execution_date + logical_date = args.logical_date if AIRFLOW_V_3_0_PLUS else args.execution_date dag = get_dag(subdir=args.subdir, dag_id=args.dag_id) yaml_output_path = args.output_path - dr = DagRun(dag.dag_id, execution_date=execution_date) + if AIRFLOW_V_3_0_PLUS: + dr = DagRun(dag.dag_id, logical_date=logical_date) + else: + dr = DagRun(dag.dag_id, execution_date=logical_date) kube_config = KubeConfig() for task in dag.tasks: ti = TaskInstance(task, None) @@ -55,7 +63,7 @@ def generate_pod_yaml(args): pod_id=create_unique_id(args.dag_id, ti.task_id), try_number=ti.try_number, kube_image=kube_config.kube_image, - date=ti.execution_date, + date=ti.logical_date if AIRFLOW_V_3_0_PLUS else ti.execution_date, args=ti.command_as_list(), pod_override_object=PodGenerator.from_obj(ti.executor_config), scheduler_job_id="worker-config", @@ -64,7 +72,7 @@ def generate_pod_yaml(args): with_mutation_hook=True, ) api_client = ApiClient() - date_string = pod_generator.datetime_to_label_safe_datestring(execution_date) + date_string = pod_generator.datetime_to_label_safe_datestring(logical_date) yaml_file_name = f"{args.dag_id}_{ti.task_id}_{date_string}.yml" os.makedirs(os.path.dirname(yaml_output_path + "/airflow_yaml_output/"), exist_ok=True) with open(yaml_output_path + "/airflow_yaml_output/" + yaml_file_name, "w") as output: diff --git a/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py b/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py index b9240d00377c..3300f3cf96d4 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py +++ b/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py @@ -38,9 +38,14 @@ from kubernetes.dynamic import DynamicClient from sqlalchemy import or_, select, update +try: + from airflow.cli.cli_config import ARG_LOGICAL_DATE +except ImportError: # 2.x compatibility. + from airflow.cli.cli_config import ( # type: ignore[attr-defined, no-redef] + ARG_EXECUTION_DATE as ARG_LOGICAL_DATE, + ) from airflow.cli.cli_config import ( ARG_DAG_ID, - ARG_EXECUTION_DATE, ARG_OUTPUT_PATH, ARG_SUBDIR, ARG_VERBOSE, @@ -118,7 +123,7 @@ help="Generate YAML files for all tasks in DAG. Useful for debugging tasks without " "launching into a cluster", func=lazy_load_command("airflow.providers.cncf.kubernetes.cli.kubernetes_command.generate_pod_yaml"), - args=(ARG_DAG_ID, ARG_EXECUTION_DATE, ARG_SUBDIR, ARG_OUTPUT_PATH, ARG_VERBOSE), + args=(ARG_DAG_ID, ARG_LOGICAL_DATE, ARG_SUBDIR, ARG_OUTPUT_PATH, ARG_VERBOSE), ), ) diff --git a/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py b/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py index 42d2a97dffed..ace65466b23f 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py +++ b/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py @@ -28,6 +28,7 @@ from urllib3.exceptions import ReadTimeoutError from airflow.exceptions import AirflowException +from airflow.providers.cncf.kubernetes.backcompat import get_logical_date_key from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types import ( ADOPTED, ALL_NAMESPACES, @@ -150,6 +151,7 @@ def _run( kwargs["_request_timeout"] = client_timeout kwargs["timeout_seconds"] = server_conn_timeout + logical_date_key = get_logical_date_key() for event in self._pod_events(kube_client=kube_client, query_kwargs=kwargs): task = event["object"] self.log.debug("Event: %s had an event of type %s", task.metadata.name, event["type"]) @@ -159,7 +161,7 @@ def _run( task_instance_related_annotations = { "dag_id": annotations["dag_id"], "task_id": annotations["task_id"], - "execution_date": annotations.get("execution_date"), + logical_date_key: annotations.get(logical_date_key), "run_id": annotations.get("run_id"), "try_number": annotations["try_number"], } diff --git a/providers/src/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py b/providers/src/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py index 813edcc6a6d0..a353b7a5c802 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py +++ b/providers/src/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py @@ -27,6 +27,7 @@ from slugify import slugify from airflow.configuration import conf +from airflow.providers.cncf.kubernetes.backcompat import get_logical_date_key if TYPE_CHECKING: from airflow.models.taskinstancekey import TaskInstanceKey @@ -106,8 +107,10 @@ def annotations_to_key(annotations: dict[str, str]) -> TaskInstanceKey: from airflow.models.taskinstance import TaskInstance, TaskInstanceKey from airflow.settings import Session - if not annotation_run_id and "execution_date" in annotations: - execution_date = pendulum.parse(annotations["execution_date"]) + logical_date_key = get_logical_date_key() + + if not annotation_run_id and logical_date_key in annotations: + logical_date = pendulum.parse(annotations[logical_date_key]) # Do _not_ use create-session, we don't want to expunge session = Session() @@ -117,7 +120,7 @@ def annotations_to_key(annotations: dict[str, str]) -> TaskInstanceKey: .filter( TaskInstance.dag_id == dag_id, TaskInstance.task_id == task_id, - DagRun.execution_date == execution_date, + getattr(DagRun, logical_date_key) == logical_date, ) .scalar() ) diff --git a/providers/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py b/providers/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py index ad529290d8ed..92821a30766a 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py +++ b/providers/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py @@ -198,8 +198,6 @@ def create_labels_for_pod(context: dict | None = None, include_try_number: bool "task_id": ti.task_id, "run_id": run_id, "spark_kubernetes_operator": "True", - # 'execution_date': context['ts'], - # 'try_number': context['ti'].try_number, } # If running on Airflow 2.3+: diff --git a/providers/src/airflow/providers/cncf/kubernetes/pod_generator.py b/providers/src/airflow/providers/cncf/kubernetes/pod_generator.py index 78838d9b4447..ad4123eacc0c 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/pod_generator.py +++ b/providers/src/airflow/providers/cncf/kubernetes/pod_generator.py @@ -41,6 +41,7 @@ AirflowConfigException, AirflowException, ) +from airflow.providers.cncf.kubernetes.backcompat import get_logical_date_key from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import ( POD_NAME_MAX_LENGTH, add_unique_suffix, @@ -329,7 +330,7 @@ def construct_pod( if map_index >= 0: annotations["map_index"] = str(map_index) if date: - annotations["execution_date"] = date.isoformat() + annotations[get_logical_date_key()] = date.isoformat() if run_id: annotations["run_id"] = run_id @@ -344,7 +345,7 @@ def construct_pod( try_number=try_number, airflow_worker=scheduler_job_id, map_index=map_index, - execution_date=date, + logical_date=date, run_id=run_id, ), ), @@ -387,7 +388,7 @@ def build_selector_for_k8s_executor_pod( task_id, try_number, map_index=None, - execution_date=None, + logical_date=None, run_id=None, airflow_worker=None, include_version=False, @@ -402,7 +403,7 @@ def build_selector_for_k8s_executor_pod( task_id=task_id, try_number=try_number, map_index=map_index, - execution_date=execution_date, + logical_date=logical_date, run_id=run_id, airflow_worker=airflow_worker, include_version=include_version, @@ -422,7 +423,7 @@ def build_labels_for_k8s_executor_pod( try_number, airflow_worker=None, map_index=None, - execution_date=None, + logical_date=None, run_id=None, include_version=True, ): @@ -443,8 +444,8 @@ def build_labels_for_k8s_executor_pod( labels["airflow-worker"] = make_safe_label_value(str(airflow_worker)) if map_index is not None and map_index >= 0: labels["map_index"] = str(map_index) - if execution_date: - labels["execution_date"] = datetime_to_label_safe_datestring(execution_date) + if logical_date: + labels[get_logical_date_key()] = datetime_to_label_safe_datestring(logical_date) if run_id: labels["run_id"] = make_safe_label_value(run_id) return labels diff --git a/providers/src/airflow/providers/databricks/plugins/databricks_workflow.py b/providers/src/airflow/providers/databricks/plugins/databricks_workflow.py index 265e93b77867..d0d8a179dd3c 100644 --- a/providers/src/airflow/providers/databricks/plugins/databricks_workflow.py +++ b/providers/src/airflow/providers/databricks/plugins/databricks_workflow.py @@ -195,7 +195,10 @@ def _get_launch_task_key(current_task_key: TaskInstanceKey, task_id: str) -> Tas @provide_session def get_task_instance(operator: BaseOperator, dttm, session: Session = NEW_SESSION) -> TaskInstance: dag_id = operator.dag.dag_id - dag_run = DagRun.find(dag_id, execution_date=dttm)[0] + if hasattr(DagRun, "execution_date"): # Airflow 2.x. + dag_run = DagRun.find(dag_id, execution_date=dttm)[0] # type: ignore[call-arg] + else: + dag_run = DagRun.find(dag_id, logical_date=dttm)[0] ti = ( session.query(TaskInstance) .filter( diff --git a/providers/src/airflow/providers/dbt/cloud/utils/openlineage.py b/providers/src/airflow/providers/dbt/cloud/utils/openlineage.py index ad50552b5146..41b1f9635fd1 100644 --- a/providers/src/airflow/providers/dbt/cloud/utils/openlineage.py +++ b/providers/src/airflow/providers/dbt/cloud/utils/openlineage.py @@ -21,20 +21,32 @@ from contextlib import suppress from typing import TYPE_CHECKING +from packaging.version import parse + from airflow import __version__ as airflow_version if TYPE_CHECKING: + from packaging.version import Version + from airflow.models.taskinstance import TaskInstance from airflow.providers.dbt.cloud.operators.dbt import DbtCloudRunJobOperator from airflow.providers.dbt.cloud.sensors.dbt import DbtCloudJobRunSensor from airflow.providers.openlineage.extractors.base import OperatorLineage +_AIRFLOW_VERSION: Version = parse(parse(airflow_version).base_version) + + +def _get_logical_date(task_instance): + # todo: remove when min airflow version >= 3.0 + if _AIRFLOW_VERSION < parse("3"): + return task_instance.execution_date + return task_instance.logical_date + + def _get_try_number(val): # todo: remove when min airflow version >= 2.10.0 - from packaging.version import parse - - if parse(parse(airflow_version).base_version) < parse("2.10.0"): + if _AIRFLOW_VERSION < parse("2.10.0"): return val.try_number - 1 else: return val.try_number @@ -142,7 +154,7 @@ async def get_artifacts_for_steps(steps, artifacts): parent_run_id = OpenLineageAdapter.build_task_instance_run_id( dag_id=task_instance.dag_id, task_id=operator.task_id, - execution_date=task_instance.execution_date, + logical_date=_get_logical_date(task_instance), try_number=_get_try_number(task_instance), ) diff --git a/providers/src/airflow/providers/elasticsearch/log/es_task_handler.py b/providers/src/airflow/providers/elasticsearch/log/es_task_handler.py index 6fd16ca712a0..c4cc14faee3b 100644 --- a/providers/src/airflow/providers/elasticsearch/log/es_task_handler.py +++ b/providers/src/airflow/providers/elasticsearch/log/es_task_handler.py @@ -32,7 +32,9 @@ import elasticsearch import pendulum from elasticsearch.exceptions import NotFoundError +from packaging.version import Version +from airflow import __version__ as airflow_version from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.models.dagrun import DagRun @@ -44,6 +46,9 @@ from airflow.utils.module_loading import import_string from airflow.utils.session import create_session +AIRFLOW_VERSION = Version(airflow_version) +AIRFLOW_V_3_0_PLUS = Version(AIRFLOW_VERSION.base_version) >= Version("3.0.0") + if TYPE_CHECKING: from datetime import datetime @@ -126,7 +131,7 @@ class ElasticsearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMix To efficiently query and sort Elasticsearch results, this handler assumes each log message has a field `log_id` consists of ti primary keys: - `log_id = {dag_id}-{task_id}-{execution_date}-{try_number}` + `log_id = {dag_id}-{task_id}-{logical_date}-{try_number}` Log messages with specific log_id are sorted based on `offset`, which is a unique integer indicates log message's order. Timestamps here are unreliable because multiple log messages @@ -256,7 +261,7 @@ def _render_log_id(self, ti: TaskInstance | TaskInstanceKey, try_number: int) -> if self.json_format: data_interval_start = self._clean_date(data_interval[0]) data_interval_end = self._clean_date(data_interval[1]) - execution_date = self._clean_date(dag_run.execution_date) + logical_date = self._clean_date(dag_run.logical_date) else: if data_interval[0]: data_interval_start = data_interval[0].isoformat() @@ -266,7 +271,7 @@ def _render_log_id(self, ti: TaskInstance | TaskInstanceKey, try_number: int) -> data_interval_end = data_interval[1].isoformat() else: data_interval_end = "" - execution_date = dag_run.execution_date.isoformat() + logical_date = dag_run.logical_date.isoformat() return log_id_template.format( dag_id=ti.dag_id, @@ -274,7 +279,8 @@ def _render_log_id(self, ti: TaskInstance | TaskInstanceKey, try_number: int) -> run_id=getattr(ti, "run_id", ""), data_interval_start=data_interval_start, data_interval_end=data_interval_end, - execution_date=execution_date, + logical_date=logical_date, + execution_date=logical_date, try_number=try_number, map_index=getattr(ti, "map_index", ""), ) @@ -444,7 +450,7 @@ def set_context(self, ti: TaskInstance, *, identifier: str | None = None) -> Non is_trigger_log_context = getattr(ti, "is_trigger_log_context", None) is_ti_raw = getattr(ti, "raw", None) self.mark_end_on_close = not is_ti_raw and not is_trigger_log_context - + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" if self.json_format: self.formatter = ElasticsearchJSONFormatter( fmt=self.formatter._fmt, @@ -452,7 +458,9 @@ def set_context(self, ti: TaskInstance, *, identifier: str | None = None) -> Non extras={ "dag_id": str(ti.dag_id), "task_id": str(ti.task_id), - "execution_date": self._clean_date(ti.execution_date), + date_key: self._clean_date(ti.logical_date) + if AIRFLOW_V_3_0_PLUS + else self._clean_date(ti.execution_date), "try_number": str(ti.try_number), "log_id": self._render_log_id(ti, ti.try_number), }, @@ -580,7 +588,7 @@ def _get_result(self, hit: dict[Any, Any], parent_class=None) -> Hit: 'container': {'id': 'airflow'}, 'dag_id': 'example_bash_operator', 'ecs': {'version': '8.0.0'}, - 'execution_date': '2023_07_09T07_47_32_000000', + 'logical_date': '2023_07_09T07_47_32_000000', 'filename': 'taskinstance.py', 'input': {'type': 'log'}, 'levelname': 'INFO', diff --git a/providers/src/airflow/providers/google/cloud/log/stackdriver_task_handler.py b/providers/src/airflow/providers/google/cloud/log/stackdriver_task_handler.py index 79449d8de24d..e150419f7e04 100644 --- a/providers/src/airflow/providers/google/cloud/log/stackdriver_task_handler.py +++ b/providers/src/airflow/providers/google/cloud/log/stackdriver_task_handler.py @@ -29,13 +29,18 @@ from google.cloud.logging.handlers.transports import BackgroundThreadTransport, Transport from google.cloud.logging_v2.services.logging_service_v2 import LoggingServiceV2Client from google.cloud.logging_v2.types import ListLogEntriesRequest, ListLogEntriesResponse +from packaging.version import Version +from airflow import __version__ as airflow_version from airflow.exceptions import RemovedInAirflow3Warning from airflow.providers.google.cloud.utils.credentials_provider import get_credentials_and_project_id from airflow.providers.google.common.consts import CLIENT_INFO from airflow.utils.log.trigger_handler import ctx_indiv_trigger from airflow.utils.types import NOTSET, ArgNotSet +AIRFLOW_VERSION = Version(airflow_version) +AIRFLOW_V_3_0_PLUS = Version(AIRFLOW_VERSION.base_version) >= Version("3.0.0") + if TYPE_CHECKING: from google.auth.credentials import Credentials @@ -82,7 +87,7 @@ class StackdriverTaskHandler(logging.Handler): LABEL_TASK_ID = "task_id" LABEL_DAG_ID = "dag_id" - LABEL_EXECUTION_DATE = "execution_date" + LABEL_LOGICAL_DATE = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" LABEL_TRY_NUMBER = "try_number" LOG_VIEWER_BASE_URL = "https://console.cloud.google.com/logs/viewer" LOG_NAME = "Google Stackdriver" @@ -338,7 +343,9 @@ def _task_instance_to_labels(cls, ti: TaskInstance) -> dict[str, str]: return { cls.LABEL_TASK_ID: ti.task_id, cls.LABEL_DAG_ID: ti.dag_id, - cls.LABEL_EXECUTION_DATE: str(ti.execution_date.isoformat()), + cls.LABEL_LOGICAL_DATE: str(ti.logical_date.isoformat()) + if AIRFLOW_V_3_0_PLUS + else str(ti.execution_date.isoformat()), cls.LABEL_TRY_NUMBER: str(ti.try_number), } diff --git a/providers/src/airflow/providers/google/cloud/operators/gcs.py b/providers/src/airflow/providers/google/cloud/operators/gcs.py index 561de4c46edd..ae4344aeb4c8 100644 --- a/providers/src/airflow/providers/google/cloud/operators/gcs.py +++ b/providers/src/airflow/providers/google/cloud/operators/gcs.py @@ -683,7 +683,7 @@ class GCSTimeSpanFileTransformOperator(GoogleCloudBaseOperator): :param source_bucket: The bucket to fetch data from. (templated) :param source_prefix: Prefix string which filters objects whose name begin with - this prefix. Can interpolate execution date and time components. (templated) + this prefix. Can interpolate logical date and time components. (templated) :param source_gcp_conn_id: The connection ID to use connecting to Google Cloud to download files to be processed. :param source_impersonation_chain: Optional service account to impersonate using short-term @@ -697,7 +697,7 @@ class GCSTimeSpanFileTransformOperator(GoogleCloudBaseOperator): :param destination_bucket: The bucket to write data to. (templated) :param destination_prefix: Prefix string for the upload location. - Can interpolate execution date and time components. (templated) + Can interpolate logical date and time components. (templated) :param destination_gcp_conn_id: The connection ID to use connecting to Google Cloud to upload processed files. :param destination_impersonation_chain: Optional service account to impersonate using short-term @@ -794,7 +794,7 @@ def execute(self, context: Context) -> list[str]: orig_start = context["data_interval_start"] orig_end = context["data_interval_end"] except KeyError: - orig_start = pendulum.instance(context["execution_date"]) + orig_start = pendulum.instance(context["logical_date"]) next_dagrun = context["dag"].next_dagrun_info(last_automated_dagrun=None, restricted=False) if next_dagrun and next_dagrun.data_interval and next_dagrun.data_interval.end: orig_end = next_dagrun.data_interval.end diff --git a/providers/src/airflow/providers/google/cloud/operators/workflows.py b/providers/src/airflow/providers/google/cloud/operators/workflows.py index 2a51d7569b4e..86c232bf0567 100644 --- a/providers/src/airflow/providers/google/cloud/operators/workflows.py +++ b/providers/src/airflow/providers/google/cloud/operators/workflows.py @@ -113,7 +113,7 @@ def _workflow_id(self, context): # We are limited by allowed length of workflow_id so # we use hash of whole information - exec_date = context["execution_date"].isoformat() + exec_date = context["logical_date"].isoformat() base = f"airflow_{self.dag_id}_{self.task_id}_{exec_date}_{hash_base}" workflow_id = md5(base.encode()).hexdigest() return re.sub(r"[:\-+.]", "_", workflow_id) diff --git a/providers/src/airflow/providers/google/cloud/sensors/cloud_composer.py b/providers/src/airflow/providers/google/cloud/sensors/cloud_composer.py index 40703c02fe2a..279973d15715 100644 --- a/providers/src/airflow/providers/google/cloud/sensors/cloud_composer.py +++ b/providers/src/airflow/providers/google/cloud/sensors/cloud_composer.py @@ -189,7 +189,7 @@ def __init__( self.deferrable = deferrable self.poll_interval = poll_interval - def _get_execution_dates(self, context) -> tuple[datetime, datetime]: + def _get_logical_dates(self, context) -> tuple[datetime, datetime]: if isinstance(self.execution_range, timedelta): if self.execution_range < timedelta(0): return context["logical_date"], context["logical_date"] - self.execution_range @@ -203,7 +203,7 @@ def _get_execution_dates(self, context) -> tuple[datetime, datetime]: return context["logical_date"] - timedelta(1), context["logical_date"] def poke(self, context: Context) -> bool: - start_date, end_date = self._get_execution_dates(context) + start_date, end_date = self._get_logical_dates(context) if datetime.now(end_date.tzinfo) < end_date: return False @@ -251,7 +251,7 @@ def _check_dag_runs_states( for dag_run in dag_runs: if ( start_date.timestamp() - < parser.parse(dag_run["execution_date"]).timestamp() + < parser.parse(dag_run["logical_date"]).timestamp() < end_date.timestamp() ) and dag_run["state"] not in self.allowed_states: return False @@ -259,7 +259,7 @@ def _check_dag_runs_states( def execute(self, context: Context) -> None: if self.deferrable: - start_date, end_date = self._get_execution_dates(context) + start_date, end_date = self._get_logical_dates(context) self.defer( trigger=CloudComposerDAGRunTrigger( project_id=self.project_id, diff --git a/providers/src/airflow/providers/google/cloud/sensors/gcs.py b/providers/src/airflow/providers/google/cloud/sensors/gcs.py index 2b5782c77ad4..4697debe1ad1 100644 --- a/providers/src/airflow/providers/google/cloud/sensors/gcs.py +++ b/providers/src/airflow/providers/google/cloud/sensors/gcs.py @@ -192,7 +192,7 @@ class GCSObjectUpdateSensor(BaseSensorOperator): :param object: The name of the object to download in the Google cloud storage bucket. :param ts_func: Callback for defining the update condition. The default callback - returns execution_date + schedule_interval. The callback takes the context + returns logical_date + schedule_interval. The callback takes the context as parameter. :param google_cloud_conn_id: The connection ID to use when connecting to Google Cloud Storage. diff --git a/providers/src/airflow/providers/google/cloud/triggers/cloud_composer.py b/providers/src/airflow/providers/google/cloud/triggers/cloud_composer.py index 2334d038e62f..16436a254e3d 100644 --- a/providers/src/airflow/providers/google/cloud/triggers/cloud_composer.py +++ b/providers/src/airflow/providers/google/cloud/triggers/cloud_composer.py @@ -231,7 +231,7 @@ def _check_dag_runs_states( for dag_run in dag_runs: if ( start_date.timestamp() - < parser.parse(dag_run["execution_date"]).timestamp() + < parser.parse(dag_run["logical_date"]).timestamp() < end_date.timestamp() ) and dag_run["state"] not in self.allowed_states: return False diff --git a/providers/src/airflow/providers/http/sensors/http.py b/providers/src/airflow/providers/http/sensors/http.py index ef0d9fd380d0..784284496a95 100644 --- a/providers/src/airflow/providers/http/sensors/http.py +++ b/providers/src/airflow/providers/http/sensors/http.py @@ -48,7 +48,7 @@ class HttpSensor(BaseSensorOperator): def response_check(response, task_instance): # The task_instance is injected, so you can pull data form xcom - # Other context variables such as dag, ds, execution_date are also available. + # Other context variables such as dag, ds, logical_date are also available. xcom_data = task_instance.xcom_pull(task_ids="pushing_task") # In practice you would do something more sensible with this data.. print(xcom_data) diff --git a/providers/src/airflow/providers/openlineage/plugins/adapter.py b/providers/src/airflow/providers/openlineage/plugins/adapter.py index 199df880e79a..688f2d65a547 100644 --- a/providers/src/airflow/providers/openlineage/plugins/adapter.py +++ b/providers/src/airflow/providers/openlineage/plugins/adapter.py @@ -84,7 +84,7 @@ def get_or_create_openlineage_client(self) -> OpenLineageClient: "OpenLineage configuration found. Transport type: `%s`", config.get("type", "no type provided"), ) - self._client = OpenLineageClient(config=config) + self._client = OpenLineageClient(config=config) # type: ignore[call-arg] else: self.log.debug( "OpenLineage configuration not found directly in Airflow. " @@ -128,11 +128,11 @@ def build_task_instance_run_id( dag_id: str, task_id: str, try_number: int, - execution_date: datetime, + logical_date: datetime, ): return str( generate_static_uuid( - instant=execution_date, + instant=logical_date, data=f"{conf.namespace()}.{dag_id}.{task_id}.{try_number}".encode(), ) ) diff --git a/providers/src/airflow/providers/openlineage/plugins/listener.py b/providers/src/airflow/providers/openlineage/plugins/listener.py index 9c568aa37419..7feece437a62 100644 --- a/providers/src/airflow/providers/openlineage/plugins/listener.py +++ b/providers/src/airflow/providers/openlineage/plugins/listener.py @@ -139,11 +139,15 @@ def on_running(): logical_date=dagrun.logical_date, ) + if hasattr(task_instance, "logical_date"): + logical_date = task_instance.logical_date + else: + logical_date = task_instance.execution_date task_uuid = self.adapter.build_task_instance_run_id( dag_id=dag.dag_id, task_id=task.task_id, try_number=task_instance.try_number, - execution_date=task_instance.execution_date, + logical_date=logical_date, ) event_type = RunState.RUNNING.value.lower() operator_name = task.task_type.lower() @@ -213,16 +217,20 @@ def on_task_instance_success( @print_warning(self.log) def on_success(): - parent_run_id = OpenLineageAdapter.build_dag_run_id( + parent_run_id = self.adapter.build_dag_run_id( dag_id=dag.dag_id, logical_date=dagrun.logical_date, ) - task_uuid = OpenLineageAdapter.build_task_instance_run_id( + if hasattr(task_instance, "logical_date"): + logical_date = task_instance.logical_date + else: + logical_date = task_instance.execution_date + task_uuid = self.adapter.build_task_instance_run_id( dag_id=dag.dag_id, task_id=task.task_id, try_number=_get_try_number_success(task_instance), - execution_date=task_instance.execution_date, + logical_date=logical_date, ) event_type = RunState.COMPLETE.value.lower() operator_name = task.task_type.lower() @@ -312,16 +320,20 @@ def _on_task_instance_failed( @print_warning(self.log) def on_failure(): - parent_run_id = OpenLineageAdapter.build_dag_run_id( + parent_run_id = self.adapter.build_dag_run_id( dag_id=dag.dag_id, logical_date=dagrun.logical_date, ) - task_uuid = OpenLineageAdapter.build_task_instance_run_id( + if hasattr(task_instance, "logical_date"): + logical_date = task_instance.logical_date + else: + logical_date = task_instance.execution_date + task_uuid = self.adapter.build_task_instance_run_id( dag_id=dag.dag_id, task_id=task.task_id, try_number=task_instance.try_number, - execution_date=task_instance.execution_date, + logical_date=logical_date, ) event_type = RunState.FAIL.value.lower() operator_name = task.task_type.lower() diff --git a/providers/src/airflow/providers/openlineage/plugins/macros.py b/providers/src/airflow/providers/openlineage/plugins/macros.py index 6bb1699aa789..69af17321e4b 100644 --- a/providers/src/airflow/providers/openlineage/plugins/macros.py +++ b/providers/src/airflow/providers/openlineage/plugins/macros.py @@ -58,11 +58,15 @@ def lineage_run_id(task_instance: TaskInstance): For more information take a look at the guide: :ref:`howto/macros:openlineage` """ + if hasattr(task_instance, "logical_date"): + logical_date = task_instance.logical_date + else: + logical_date = task_instance.execution_date return OpenLineageAdapter.build_task_instance_run_id( dag_id=task_instance.dag_id, task_id=task_instance.task_id, try_number=task_instance.try_number, - execution_date=task_instance.execution_date, + logical_date=logical_date, ) diff --git a/providers/src/airflow/providers/opensearch/log/os_task_handler.py b/providers/src/airflow/providers/opensearch/log/os_task_handler.py index 11865b2b4ff6..ca061f04825d 100644 --- a/providers/src/airflow/providers/opensearch/log/os_task_handler.py +++ b/providers/src/airflow/providers/opensearch/log/os_task_handler.py @@ -29,7 +29,9 @@ import pendulum from opensearchpy import OpenSearch from opensearchpy.exceptions import NotFoundError +from packaging.version import Version +from airflow import __version__ as airflow_version from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.models import DagRun @@ -41,6 +43,9 @@ from airflow.utils.module_loading import import_string from airflow.utils.session import create_session +AIRFLOW_VERSION = Version(airflow_version) +AIRFLOW_V_3_0_PLUS = Version(AIRFLOW_VERSION.base_version) >= Version("3.0.0") + if TYPE_CHECKING: from airflow.models.taskinstance import TaskInstance, TaskInstanceKey USE_PER_RUN_LOG_ID = hasattr(DagRun, "get_log_template") @@ -108,7 +113,7 @@ class OpensearchTaskHandler(FileTaskHandler, ExternalLoggingMixin, LoggingMixin) To efficiently query and sort Elasticsearch results, this handler assumes each log message has a field `log_id` consists of ti primary keys: - `log_id = {dag_id}-{task_id}-{execution_date}-{try_number}` + `log_id = {dag_id}-{task_id}-{logical_date}-{try_number}` Log messages with specific log_id are sorted based on `offset`, which is a unique integer indicates log message's order. Timestamps here are unreliable because multiple log messages @@ -191,7 +196,7 @@ def set_context(self, ti: TaskInstance, *, identifier: str | None = None) -> Non is_trigger_log_context = getattr(ti, "is_trigger_log_context", None) is_ti_raw = getattr(ti, "raw", None) self.mark_end_on_close = not is_ti_raw and not is_trigger_log_context - + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" if self.json_format: self.formatter = OpensearchJSONFormatter( fmt=self.formatter._fmt, @@ -199,7 +204,9 @@ def set_context(self, ti: TaskInstance, *, identifier: str | None = None) -> Non extras={ "dag_id": str(ti.dag_id), "task_id": str(ti.task_id), - "execution_date": self._clean_date(ti.execution_date), + date_key: self._clean_date(ti.logical_date) + if AIRFLOW_V_3_0_PLUS + else self._clean_date(ti.execution_date), "try_number": str(ti.try_number), "log_id": self._render_log_id(ti, ti.try_number), }, @@ -296,7 +303,7 @@ def _render_log_id(self, ti: TaskInstance | TaskInstanceKey, try_number: int) -> if self.json_format: data_interval_start = self._clean_date(data_interval[0]) data_interval_end = self._clean_date(data_interval[1]) - execution_date = self._clean_date(dag_run.execution_date) + logical_date = self._clean_date(dag_run.logical_date) else: if data_interval[0]: data_interval_start = data_interval[0].isoformat() @@ -306,15 +313,15 @@ def _render_log_id(self, ti: TaskInstance | TaskInstanceKey, try_number: int) -> data_interval_end = data_interval[1].isoformat() else: data_interval_end = "" - execution_date = dag_run.execution_date.isoformat() - + logical_date = dag_run.logical_date.isoformat() return log_id_template.format( dag_id=ti.dag_id, task_id=ti.task_id, run_id=getattr(ti, "run_id", ""), data_interval_start=data_interval_start, data_interval_end=data_interval_end, - execution_date=execution_date, + logical_date=logical_date, + execution_date=logical_date, try_number=try_number, map_index=getattr(ti, "map_index", ""), ) @@ -474,7 +481,7 @@ def _get_result(self, hit: dict[Any, Any], parent_class=None) -> Hit: 'container': {'id': 'airflow'}, 'dag_id': 'example_bash_operator', 'ecs': {'version': '8.0.0'}, - 'execution_date': '2023_07_09T07_47_32_000000', + 'logical_date': '2023_07_09T07_47_32_000000', 'filename': 'taskinstance.py', 'input': {'type': 'log'}, 'levelname': 'INFO', diff --git a/providers/src/airflow/providers/presto/hooks/presto.py b/providers/src/airflow/providers/presto/hooks/presto.py index adb4dffa6803..483635c5c6e1 100644 --- a/providers/src/airflow/providers/presto/hooks/presto.py +++ b/providers/src/airflow/providers/presto/hooks/presto.py @@ -22,14 +22,19 @@ from typing import TYPE_CHECKING, Any, Iterable, Mapping, TypeVar import prestodb +from packaging.version import Version from prestodb.exceptions import DatabaseError from prestodb.transaction import IsolationLevel +from airflow import __version__ as airflow_version from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.utils.operator_helpers import AIRFLOW_VAR_NAME_FORMAT_MAPPING, DEFAULT_FORMAT_PREFIX +AIRFLOW_VERSION = Version(airflow_version) +AIRFLOW_V_3_0_PLUS = Version(AIRFLOW_VERSION.base_version) >= Version("3.0.0") + if TYPE_CHECKING: from airflow.models import Connection @@ -37,17 +42,18 @@ def generate_presto_client_info() -> str: - """Return json string with dag_id, task_id, execution_date and try_number.""" + """Return json string with dag_id, task_id, logical_date and try_number.""" context_var = { format_map["default"].replace(DEFAULT_FORMAT_PREFIX, ""): os.environ.get( format_map["env_var_format"], "" ) for format_map in AIRFLOW_VAR_NAME_FORMAT_MAPPING.values() } + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" task_info = { "dag_id": context_var["dag_id"], "task_id": context_var["task_id"], - "execution_date": context_var["execution_date"], + date_key: context_var[date_key], "try_number": context_var["try_number"], "dag_run_id": context_var["dag_run_id"], "dag_owner": context_var["dag_owner"], diff --git a/providers/src/airflow/providers/standard/operators/bash.py b/providers/src/airflow/providers/standard/operators/bash.py index b6a571bd98e4..b991084c295f 100644 --- a/providers/src/airflow/providers/standard/operators/bash.py +++ b/providers/src/airflow/providers/standard/operators/bash.py @@ -97,7 +97,7 @@ class BashOperator(BaseOperator): .. code-block:: python - bash_command = "set -e; python3 script.py '{{ next_execution_date }}'" + bash_command = "set -e; python3 script.py '{{ data_interval_end }}'" .. note:: diff --git a/providers/src/airflow/providers/standard/operators/datetime.py b/providers/src/airflow/providers/standard/operators/datetime.py index 4455b84dd3bb..9b1f7819eca1 100644 --- a/providers/src/airflow/providers/standard/operators/datetime.py +++ b/providers/src/airflow/providers/standard/operators/datetime.py @@ -44,7 +44,7 @@ class BranchDateTimeOperator(BaseBranchOperator): :param target_lower: target lower bound. :param target_upper: target upper bound. :param use_task_logical_date: If ``True``, uses task's logical date to compare with targets. - Execution date is useful for backfilling. If ``False``, uses system's date. + Logical date is useful for backfilling. If ``False``, uses system's date. """ def __init__( diff --git a/providers/src/airflow/providers/standard/operators/python.py b/providers/src/airflow/providers/standard/operators/python.py index 0e8b3843c5ce..8ff6eba0eb06 100644 --- a/providers/src/airflow/providers/standard/operators/python.py +++ b/providers/src/airflow/providers/standard/operators/python.py @@ -114,13 +114,13 @@ class PythonOperator(BaseOperator): function. This set of kwargs correspond exactly to what you can use in your jinja templates. For this to work, you need to define ``**kwargs`` in your function header, or you can add directly the keyword arguments you would like to get - for example with the below code your callable will get - the values of ``ti`` and ``next_ds`` context variables. + the values of ``ti`` context variables. With explicit arguments: .. code-block:: python - def my_python_callable(ti, next_ds): + def my_python_callable(ti): pass With kwargs: @@ -129,7 +129,6 @@ def my_python_callable(ti, next_ds): def my_python_callable(**kwargs): ti = kwargs["ti"] - next_ds = kwargs["next_ds"] :param python_callable: A reference to an object that is callable @@ -310,7 +309,7 @@ def get_tasks_to_skip(): self.skip( dag_run=dag_run, tasks=to_skip, - execution_date=cast("DateTime", dag_run.execution_date), # type: ignore[call-arg] + execution_date=cast("DateTime", dag_run.logical_date), # type: ignore[call-arg, union-attr] map_index=context["ti"].map_index, ) @@ -360,34 +359,36 @@ class _BasePythonVirtualenvOperator(PythonOperator, metaclass=ABCMeta): "ds_nodash", "expanded_ti_count", "inlets", - "next_ds", - "next_ds_nodash", "outlets", - "prev_ds", - "prev_ds_nodash", "run_id", "task_instance_key_str", "test_mode", - "tomorrow_ds", - "tomorrow_ds_nodash", "ts", "ts_nodash", "ts_nodash_with_tz", + # The following should be removed when Airflow 2 support is dropped. + "next_ds", + "next_ds_nodash", + "prev_ds", + "prev_ds_nodash", + "tomorrow_ds", + "tomorrow_ds_nodash", "yesterday_ds", "yesterday_ds_nodash", } PENDULUM_SERIALIZABLE_CONTEXT_KEYS = { "data_interval_end", "data_interval_start", - "execution_date", "logical_date", - "next_execution_date", "prev_data_interval_end_success", "prev_data_interval_start_success", - "prev_execution_date", - "prev_execution_date_success", "prev_start_date_success", "prev_end_date_success", + # The following should be removed when Airflow 2 support is dropped. + "execution_date", + "next_execution_date", + "prev_execution_date", + "prev_execution_date_success", } AIRFLOW_SERIALIZABLE_CONTEXT_KEYS = { @@ -397,7 +398,9 @@ class _BasePythonVirtualenvOperator(PythonOperator, metaclass=ABCMeta): "dag_run", "task", "params", - "triggering_asset_events" if AIRFLOW_V_3_0_PLUS else "triggering_dataset_events", + "triggering_asset_events", + # The following should be removed when Airflow 2 support is dropped. + "triggering_dataset_events", } def __init__( diff --git a/providers/src/airflow/providers/standard/sensors/date_time.py b/providers/src/airflow/providers/standard/sensors/date_time.py index 63917e3c2239..bfc15ab38c46 100644 --- a/providers/src/airflow/providers/standard/sensors/date_time.py +++ b/providers/src/airflow/providers/standard/sensors/date_time.py @@ -54,7 +54,7 @@ class DateTimeSensor(BaseSensorOperator): It handles some cases for which ``TimeSensor`` and ``TimeDeltaSensor`` are not suited. **Example** 1 : - If a task needs to wait for 11am on each ``execution_date``. Using + If a task needs to wait for 11am on each ``logical_date``. Using ``TimeSensor`` or ``TimeDeltaSensor``, all backfill tasks started at 1am have to wait for 10 hours. This is unnecessary, e.g. a backfill task with ``{{ ds }} = '1970-01-01'`` does not need to wait because @@ -69,7 +69,7 @@ class DateTimeSensor(BaseSensorOperator): DateTimeSensor( task_id="wait_for_0100", - target_time="{{ next_execution_date.tomorrow().replace(hour=1) }}", + target_time="{{ data_interval_end.tomorrow().replace(hour=1) }}", ) :param target_time: datetime after which the job succeeds. (templated) diff --git a/providers/src/airflow/providers/trino/hooks/trino.py b/providers/src/airflow/providers/trino/hooks/trino.py index e450c9b67e3d..b2936d976f0e 100644 --- a/providers/src/airflow/providers/trino/hooks/trino.py +++ b/providers/src/airflow/providers/trino/hooks/trino.py @@ -23,15 +23,20 @@ from typing import TYPE_CHECKING, Any, Iterable, Mapping, TypeVar import trino +from packaging.version import Version from trino.exceptions import DatabaseError from trino.transaction import IsolationLevel +from airflow import __version__ as airflow_version from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.providers.common.sql.hooks.sql import DbApiHook from airflow.utils.helpers import exactly_one from airflow.utils.operator_helpers import AIRFLOW_VAR_NAME_FORMAT_MAPPING, DEFAULT_FORMAT_PREFIX +AIRFLOW_VERSION = Version(airflow_version) +AIRFLOW_V_3_0_PLUS = Version(AIRFLOW_VERSION.base_version) >= Version("3.0.0") + if TYPE_CHECKING: from airflow.models import Connection @@ -39,17 +44,18 @@ def generate_trino_client_info() -> str: - """Return json string with dag_id, task_id, execution_date and try_number.""" + """Return json string with dag_id, task_id, logical_date and try_number.""" context_var = { format_map["default"].replace(DEFAULT_FORMAT_PREFIX, ""): os.environ.get( format_map["env_var_format"], "" ) for format_map in AIRFLOW_VAR_NAME_FORMAT_MAPPING.values() } + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" task_info = { "dag_id": context_var["dag_id"], "task_id": context_var["task_id"], - "execution_date": context_var["execution_date"], + date_key: context_var[date_key], "try_number": context_var["try_number"], "dag_run_id": context_var["dag_run_id"], "dag_owner": context_var["dag_owner"], diff --git a/providers/tests/alibaba/cloud/log/test_oss_task_handler.py b/providers/tests/alibaba/cloud/log/test_oss_task_handler.py index db2485e4e707..f3e73e33b005 100644 --- a/providers/tests/alibaba/cloud/log/test_oss_task_handler.py +++ b/providers/tests/alibaba/cloud/log/test_oss_task_handler.py @@ -56,7 +56,7 @@ def task_instance(self, create_task_instance: CreateTaskInstance, dag_maker: Dag self.ti = ti = create_task_instance( dag_id="dag_for_testing_oss_task_handler", task_id="task_for_testing_oss_task_handler", - execution_date=datetime(2020, 1, 1), + logical_date=datetime(2020, 1, 1), state=TaskInstanceState.RUNNING, ) ti.try_number = 1 diff --git a/providers/tests/amazon/aws/links/test_base_aws.py b/providers/tests/amazon/aws/links/test_base_aws.py index 664132cef008..0b629905359e 100644 --- a/providers/tests/amazon/aws/links/test_base_aws.py +++ b/providers/tests/amazon/aws/links/test_base_aws.py @@ -154,7 +154,7 @@ def create_op_and_ti( *, dag_id, task_id, - execution_date=None, + logical_date=None, session=None, **operator_kwargs, ): @@ -166,7 +166,7 @@ def create_op_and_ti( op, dag_id=dag_id, task_id=task_id, - execution_date=execution_date, + logical_date=logical_date, session=session, **operator_kwargs, ), diff --git a/providers/tests/amazon/aws/log/test_cloudwatch_task_handler.py b/providers/tests/amazon/aws/log/test_cloudwatch_task_handler.py index cdca8735b2d3..0e1693648f17 100644 --- a/providers/tests/amazon/aws/log/test_cloudwatch_task_handler.py +++ b/providers/tests/amazon/aws/log/test_cloudwatch_task_handler.py @@ -36,6 +36,7 @@ from airflow.utils.state import State from airflow.utils.timezone import datetime +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars @@ -58,7 +59,10 @@ def setup_tests(self, create_log_template, tmp_path_factory, session): self.remote_log_group = "log_group_name" self.region_name = "us-west-2" self.local_log_location = str(tmp_path_factory.mktemp("local-cloudwatch-log-location")) - create_log_template("{dag_id}/{task_id}/{execution_date}/{try_number}.log") + if AIRFLOW_V_3_0_PLUS: + create_log_template("{dag_id}/{task_id}/{logical_date}/{try_number}.log") + else: + create_log_template("{dag_id}/{task_id}/{execution_date}/{try_number}.log") self.cloudwatch_task_handler = CloudwatchTaskHandler( self.local_log_location, f"arn:aws:logs:{self.region_name}:11111111:log-group:{self.remote_log_group}", @@ -69,7 +73,10 @@ def setup_tests(self, create_log_template, tmp_path_factory, session): task_id = "task_for_testing_cloudwatch_log_handler" self.dag = DAG(dag_id=dag_id, schedule=None, start_date=date) task = EmptyOperator(task_id=task_id, dag=self.dag) - dag_run = DagRun(dag_id=self.dag.dag_id, execution_date=date, run_id="test", run_type="scheduled") + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun(dag_id=self.dag.dag_id, logical_date=date, run_id="test", run_type="scheduled") + else: + dag_run = DagRun(dag_id=self.dag.dag_id, execution_date=date, run_id="test", run_type="scheduled") session.add(dag_run) session.commit() session.refresh(dag_run) diff --git a/providers/tests/amazon/aws/log/test_s3_task_handler.py b/providers/tests/amazon/aws/log/test_s3_task_handler.py index e98cf1846184..45269f283585 100644 --- a/providers/tests/amazon/aws/log/test_s3_task_handler.py +++ b/providers/tests/amazon/aws/log/test_s3_task_handler.py @@ -34,6 +34,7 @@ from airflow.utils.state import State, TaskInstanceState from airflow.utils.timezone import datetime +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars @@ -60,7 +61,10 @@ def setup_tests(self, create_log_template, tmp_path_factory, session): date = datetime(2016, 1, 1) self.dag = DAG("dag_for_testing_s3_task_handler", schedule=None, start_date=date) task = EmptyOperator(task_id="task_for_testing_s3_log_handler", dag=self.dag) - dag_run = DagRun(dag_id=self.dag.dag_id, execution_date=date, run_id="test", run_type="manual") + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun(dag_id=self.dag.dag_id, logical_date=date, run_id="test", run_type="manual") + else: + dag_run = DagRun(dag_id=self.dag.dag_id, execution_date=date, run_id="test", run_type="manual") session.add(dag_run) session.commit() session.refresh(dag_run) diff --git a/providers/tests/amazon/aws/operators/test_athena.py b/providers/tests/amazon/aws/operators/test_athena.py index 1d6e9d58e994..b29f94bbebe4 100644 --- a/providers/tests/amazon/aws/operators/test_athena.py +++ b/providers/tests/amazon/aws/operators/test_athena.py @@ -42,6 +42,7 @@ from airflow.utils.types import DagRunType from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS TEST_DAG_ID = "unit_tests" DEFAULT_DATE = datetime(2018, 1, 1) @@ -232,12 +233,20 @@ def test_return_value( self, mock_conn, mock_run_query, mock_check_query_status, session, clean_dags_and_dagruns ): """Test we return the right value -- that will get put in to XCom by the execution engine""" - dag_run = DagRun( - dag_id=self.dag.dag_id, - execution_date=timezone.utcnow(), - run_id="test", - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=self.dag.dag_id, + logical_date=timezone.utcnow(), + run_id="test", + run_type=DagRunType.MANUAL, + ) + else: + dag_run = DagRun( + dag_id=self.dag.dag_id, + execution_date=timezone.utcnow(), + run_id="test", + run_type=DagRunType.MANUAL, + ) ti = TaskInstance(task=self.athena) ti.dag_run = dag_run session.add(ti) diff --git a/providers/tests/amazon/aws/operators/test_base_aws.py b/providers/tests/amazon/aws/operators/test_base_aws.py index 10d594363e66..d5526b7436e1 100644 --- a/providers/tests/amazon/aws/operators/test_base_aws.py +++ b/providers/tests/amazon/aws/operators/test_base_aws.py @@ -25,6 +25,8 @@ from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator from airflow.utils import timezone +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + TEST_CONN = "aws_test_conn" @@ -116,7 +118,10 @@ def test_execute(self, op_kwargs, dag_maker): with dag_maker("test_aws_base_operator", serialized=True): FakeS3Operator(task_id="fake-task-id", **op_kwargs) - dagrun = dag_maker.create_dagrun(execution_date=timezone.utcnow()) + if AIRFLOW_V_3_0_PLUS: + dagrun = dag_maker.create_dagrun(logical_date=timezone.utcnow()) + else: + dagrun = dag_maker.create_dagrun(execution_date=timezone.utcnow()) tis = {ti.task_id: ti for ti in dagrun.task_instances} tis["fake-task-id"].run() diff --git a/providers/tests/amazon/aws/operators/test_datasync.py b/providers/tests/amazon/aws/operators/test_datasync.py index 52c5863b8aa7..ca909cb48547 100644 --- a/providers/tests/amazon/aws/operators/test_datasync.py +++ b/providers/tests/amazon/aws/operators/test_datasync.py @@ -31,6 +31,7 @@ from airflow.utils.types import DagRunType from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS TEST_DAG_ID = "unit_tests" DEFAULT_DATE = datetime(2018, 1, 1) @@ -351,12 +352,20 @@ def test_return_value(self, mock_get_conn, session, clean_dags_and_dagruns): # ### Begin tests: self.set_up_operator() - dag_run = DagRun( - dag_id=self.dag.dag_id, - execution_date=timezone.utcnow(), - run_id="test", - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=self.dag.dag_id, + logical_date=timezone.utcnow(), + run_id="test", + run_type=DagRunType.MANUAL, + ) + else: + dag_run = DagRun( + dag_id=self.dag.dag_id, + execution_date=timezone.utcnow(), + run_id="test", + run_type=DagRunType.MANUAL, + ) ti = TaskInstance(task=self.datasync) ti.dag_run = dag_run session.add(ti) @@ -555,12 +564,20 @@ def test_return_value(self, mock_get_conn, session, clean_dags_and_dagruns): # ### Begin tests: self.set_up_operator() - dag_run = DagRun( - dag_id=self.dag.dag_id, - execution_date=timezone.utcnow(), - run_id="test", - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=self.dag.dag_id, + logical_date=timezone.utcnow(), + run_id="test", + run_type=DagRunType.MANUAL, + ) + else: + dag_run = DagRun( + dag_id=self.dag.dag_id, + execution_date=timezone.utcnow(), + run_id="test", + run_type=DagRunType.MANUAL, + ) ti = TaskInstance(task=self.datasync) ti.dag_run = dag_run session.add(ti) @@ -661,12 +678,20 @@ def test_return_value(self, mock_get_conn, session, clean_dags_and_dagruns): # ### Begin tests: self.set_up_operator() - dag_run = DagRun( - dag_id=self.dag.dag_id, - execution_date=timezone.utcnow(), - run_id="test", - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=self.dag.dag_id, + logical_date=timezone.utcnow(), + run_id="test", + run_type=DagRunType.MANUAL, + ) + else: + dag_run = DagRun( + dag_id=self.dag.dag_id, + execution_date=timezone.utcnow(), + run_id="test", + run_type=DagRunType.MANUAL, + ) ti = TaskInstance(task=self.datasync) ti.dag_run = dag_run session.add(ti) @@ -839,12 +864,20 @@ def test_return_value(self, mock_get_conn, session, clean_dags_and_dagruns): # ### Begin tests: self.set_up_operator() - dag_run = DagRun( - dag_id=self.dag.dag_id, - execution_date=timezone.utcnow(), - run_id="test", - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=self.dag.dag_id, + logical_date=timezone.utcnow(), + run_id="test", + run_type=DagRunType.MANUAL, + ) + else: + dag_run = DagRun( + dag_id=self.dag.dag_id, + execution_date=timezone.utcnow(), + run_id="test", + run_type=DagRunType.MANUAL, + ) ti = TaskInstance(task=self.datasync) ti.dag_run = dag_run session.add(ti) @@ -941,12 +974,20 @@ def test_return_value(self, mock_get_conn, session, clean_dags_and_dagruns): # ### Begin tests: self.set_up_operator() - dag_run = DagRun( - dag_id=self.dag.dag_id, - execution_date=timezone.utcnow(), - run_id="test", - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=self.dag.dag_id, + logical_date=timezone.utcnow(), + run_id="test", + run_type=DagRunType.MANUAL, + ) + else: + dag_run = DagRun( + dag_id=self.dag.dag_id, + execution_date=timezone.utcnow(), + run_id="test", + run_type=DagRunType.MANUAL, + ) ti = TaskInstance(task=self.datasync) ti.dag_run = dag_run session.add(ti) diff --git a/providers/tests/amazon/aws/operators/test_dms.py b/providers/tests/amazon/aws/operators/test_dms.py index 0c99dc2cd933..8e51b34964fe 100644 --- a/providers/tests/amazon/aws/operators/test_dms.py +++ b/providers/tests/amazon/aws/operators/test_dms.py @@ -36,6 +36,7 @@ from airflow.utils.types import DagRunType from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS TASK_ARN = "test_arn" @@ -279,12 +280,20 @@ def test_describe_tasks_return_value(self, mock_conn, mock_describe_replication_ task_id="describe_tasks", dag=self.dag, describe_tasks_kwargs={"Filters": [self.FILTER]} ) - dag_run = DagRun( - dag_id=self.dag.dag_id, - execution_date=timezone.utcnow(), - run_id="test", - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=self.dag.dag_id, + logical_date=timezone.utcnow(), + run_id="test", + run_type=DagRunType.MANUAL, + ) + else: + dag_run = DagRun( + dag_id=self.dag.dag_id, + execution_date=timezone.utcnow(), + run_id="test", + run_type=DagRunType.MANUAL, + ) ti = TaskInstance(task=describe_task) ti.dag_run = dag_run session.add(ti) diff --git a/providers/tests/amazon/aws/operators/test_emr_add_steps.py b/providers/tests/amazon/aws/operators/test_emr_add_steps.py index 4414ae432707..7ce721be3e7d 100644 --- a/providers/tests/amazon/aws/operators/test_emr_add_steps.py +++ b/providers/tests/amazon/aws/operators/test_emr_add_steps.py @@ -33,6 +33,7 @@ from airflow.utils.types import DagRunType from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS DEFAULT_DATE = timezone.datetime(2017, 1, 1) @@ -98,12 +99,20 @@ def test_validate_mutually_exclusive_args(self, job_flow_id, job_flow_name): @pytest.mark.db_test def test_render_template(self, session, clean_dags_and_dagruns): - dag_run = DagRun( - dag_id=self.operator.dag.dag_id, - execution_date=DEFAULT_DATE, - run_id="test", - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=self.operator.dag.dag_id, + logical_date=DEFAULT_DATE, + run_id="test", + run_type=DagRunType.MANUAL, + ) + else: + dag_run = DagRun( + dag_id=self.operator.dag.dag_id, + execution_date=DEFAULT_DATE, + run_id="test", + run_type=DagRunType.MANUAL, + ) ti = TaskInstance(task=self.operator) ti.dag_run = dag_run session.add(ti) @@ -155,9 +164,20 @@ def test_render_template_from_file(self, mocked_hook_client, session, clean_dags dag=dag, do_xcom_push=False, ) - dag_run = DagRun( - dag_id=dag.dag_id, execution_date=timezone.utcnow(), run_id="test", run_type=DagRunType.MANUAL - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=dag.dag_id, + logical_date=timezone.utcnow(), + run_id="test", + run_type=DagRunType.MANUAL, + ) + else: + dag_run = DagRun( + dag_id=dag.dag_id, + execution_date=timezone.utcnow(), + run_id="test", + run_type=DagRunType.MANUAL, + ) ti = TaskInstance(task=test_task) ti.dag_run = dag_run session.add(ti) diff --git a/providers/tests/amazon/aws/operators/test_emr_create_job_flow.py b/providers/tests/amazon/aws/operators/test_emr_create_job_flow.py index b2f7c8eb48f7..a17304fc6441 100644 --- a/providers/tests/amazon/aws/operators/test_emr_create_job_flow.py +++ b/providers/tests/amazon/aws/operators/test_emr_create_job_flow.py @@ -35,6 +35,7 @@ from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields from providers.tests.amazon.aws.utils.test_waiter import assert_expected_waiter_type +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS TASK_ID = "test_task" @@ -96,12 +97,20 @@ def test_init(self): @pytest.mark.db_test def test_render_template(self, session, clean_dags_and_dagruns): self.operator.job_flow_overrides = self._config - dag_run = DagRun( - dag_id=self.operator.dag_id, - execution_date=DEFAULT_DATE, - run_id="test", - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=self.operator.dag_id, + logical_date=DEFAULT_DATE, + run_id="test", + run_type=DagRunType.MANUAL, + ) + else: + dag_run = DagRun( + dag_id=self.operator.dag_id, + execution_date=DEFAULT_DATE, + run_id="test", + run_type=DagRunType.MANUAL, + ) ti = TaskInstance(task=self.operator) ti.dag_run = dag_run session.add(ti) @@ -134,12 +143,20 @@ def test_render_template_from_file(self, mocked_hook_client, session, clean_dags self.operator.job_flow_overrides = "job.j2.json" self.operator.params = {"releaseLabel": "5.11.0"} - dag_run = DagRun( - dag_id=self.operator.dag_id, - execution_date=DEFAULT_DATE, - run_id="test", - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=self.operator.dag_id, + logical_date=DEFAULT_DATE, + run_id="test", + run_type=DagRunType.MANUAL, + ) + else: + dag_run = DagRun( + dag_id=self.operator.dag_id, + execution_date=DEFAULT_DATE, + run_id="test", + run_type=DagRunType.MANUAL, + ) ti = TaskInstance(task=self.operator) ti.dag_run = dag_run session.add(ti) diff --git a/providers/tests/amazon/aws/operators/test_s3.py b/providers/tests/amazon/aws/operators/test_s3.py index 8c6dbd2df50f..407084083da1 100644 --- a/providers/tests/amazon/aws/operators/test_s3.py +++ b/providers/tests/amazon/aws/operators/test_s3.py @@ -642,7 +642,7 @@ def test_dates_from_template(self, session): for k in keys: conn.upload_fileobj(Bucket=bucket, Key=k, Fileobj=BytesIO(b"input")) - execution_date = utcnow() + logical_date = utcnow() dag = DAG("test_dag", start_date=datetime(2020, 1, 1), schedule=timedelta(days=1)) # use macros.ds_add since it returns a string, not a date op = S3DeleteObjectsOperator( @@ -652,10 +652,14 @@ def test_dates_from_template(self, session): to_datetime="{{ macros.ds_add(ds, 1) }}", dag=dag, ) - - dag_run = DagRun( - dag_id=dag.dag_id, execution_date=execution_date, run_id="test", run_type=DagRunType.MANUAL - ) + if hasattr(DagRun, "execution_date"): # Airflow 2.x. + dag_run = DagRun( + dag_id=dag.dag_id, execution_date=logical_date, run_id="test", run_type=DagRunType.MANUAL + ) + else: + dag_run = DagRun( + dag_id=dag.dag_id, logical_date=logical_date, run_id="test", run_type=DagRunType.MANUAL + ) ti = TaskInstance(task=op) ti.dag_run = dag_run session.add(ti) diff --git a/providers/tests/amazon/aws/operators/test_sagemaker_base.py b/providers/tests/amazon/aws/operators/test_sagemaker_base.py index e6e1fcd5bd90..8be30dabd6fa 100644 --- a/providers/tests/amazon/aws/operators/test_sagemaker_base.py +++ b/providers/tests/amazon/aws/operators/test_sagemaker_base.py @@ -34,6 +34,7 @@ from airflow.utils.types import DagRunType from providers.tests.amazon.aws.utils.test_template_fields import validate_template_fields +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS CONFIG: dict = { "key1": "1", @@ -170,8 +171,8 @@ def test_create_experiment(self, conn_mock, session, clean_dags_and_dagruns): conn_mock().create_experiment.return_value = {"ExperimentArn": "abcdef"} # putting a DAG around the operator so that jinja template gets rendered - execution_date = timezone.datetime(2020, 1, 1) - dag = DAG("test_experiment", schedule=None, start_date=execution_date) + logical_date = timezone.datetime(2020, 1, 1) + dag = DAG("test_experiment", schedule=None, start_date=logical_date) op = SageMakerCreateExperimentOperator( name="the name", description="the desc", @@ -179,9 +180,14 @@ def test_create_experiment(self, conn_mock, session, clean_dags_and_dagruns): task_id="tid", dag=dag, ) - dag_run = DagRun( - dag_id=dag.dag_id, execution_date=execution_date, run_id="test", run_type=DagRunType.MANUAL - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=dag.dag_id, logical_date=logical_date, run_id="test", run_type=DagRunType.MANUAL + ) + else: + dag_run = DagRun( + dag_id=dag.dag_id, execution_date=logical_date, run_id="test", run_type=DagRunType.MANUAL + ) ti = TaskInstance(task=op) ti.dag_run = dag_run session.add(ti) diff --git a/providers/tests/amazon/aws/sensors/test_base_aws.py b/providers/tests/amazon/aws/sensors/test_base_aws.py index f099f5fcfe04..08717bb2c05d 100644 --- a/providers/tests/amazon/aws/sensors/test_base_aws.py +++ b/providers/tests/amazon/aws/sensors/test_base_aws.py @@ -25,6 +25,8 @@ from airflow.providers.amazon.aws.sensors.base_aws import AwsBaseSensor from airflow.utils import timezone +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + TEST_CONN = "aws_test_conn" @@ -118,7 +120,10 @@ def test_execute(self, dag_maker, op_kwargs): with dag_maker("test_aws_base_sensor", serialized=True): FakeDynamoDBSensor(task_id="fake-task-id", **op_kwargs, poke_interval=1) - dagrun = dag_maker.create_dagrun(execution_date=timezone.utcnow()) + if AIRFLOW_V_3_0_PLUS: + dagrun = dag_maker.create_dagrun(logical_date=timezone.utcnow()) + else: + dagrun = dag_maker.create_dagrun(execution_date=timezone.utcnow()) tis = {ti.task_id: ti for ti in dagrun.task_instances} tis["fake-task-id"].run() diff --git a/providers/tests/amazon/aws/sensors/test_ecs.py b/providers/tests/amazon/aws/sensors/test_ecs.py index 52428c378e32..9a5e9313b669 100644 --- a/providers/tests/amazon/aws/sensors/test_ecs.py +++ b/providers/tests/amazon/aws/sensors/test_ecs.py @@ -66,7 +66,7 @@ def create_rendered_task(self, operator_class: type[_Operator], **kwargs) -> _Op operator_class, dag_id=self.dag_id, task_id=self.task_id, - execution_date=timezone.datetime(2021, 12, 21), + logical_date=timezone.datetime(2021, 12, 21), **kwargs, ) self.session.add(ti) diff --git a/providers/tests/amazon/aws/sensors/test_s3.py b/providers/tests/amazon/aws/sensors/test_s3.py index 3c5606dcbe56..3112fdf4c592 100644 --- a/providers/tests/amazon/aws/sensors/test_s3.py +++ b/providers/tests/amazon/aws/sensors/test_s3.py @@ -32,6 +32,8 @@ from airflow.utils import timezone from airflow.utils.types import DagRunType +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + DEFAULT_DATE = datetime(2015, 1, 1) @@ -115,9 +117,9 @@ def test_parse_bucket_key_from_jinja(self, mock_head_object, session, clean_dags Variable.set("test_bucket_key", "s3://bucket/key", session=session) - execution_date = timezone.datetime(2020, 1, 1) + logical_date = timezone.datetime(2020, 1, 1) - dag = DAG("test_s3_key", schedule=None, start_date=execution_date) + dag = DAG("test_s3_key", schedule=None, start_date=logical_date) op = S3KeySensor( task_id="s3_key_sensor", bucket_key="{{ var.value.test_bucket_key }}", @@ -125,9 +127,14 @@ def test_parse_bucket_key_from_jinja(self, mock_head_object, session, clean_dags dag=dag, ) - dag_run = DagRun( - dag_id=dag.dag_id, execution_date=execution_date, run_id="test", run_type=DagRunType.MANUAL - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=dag.dag_id, logical_date=logical_date, run_id="test", run_type=DagRunType.MANUAL + ) + else: + dag_run = DagRun( + dag_id=dag.dag_id, execution_date=logical_date, run_id="test", run_type=DagRunType.MANUAL + ) ti = TaskInstance(task=op) ti.dag_run = dag_run session.add(ti) @@ -146,19 +153,23 @@ def test_parse_list_of_bucket_keys_from_jinja(self, mock_head_object, session, c Variable.set("test_bucket_key", ["s3://bucket/file1", "s3://bucket/file2"], session=session) - execution_date = timezone.datetime(2020, 1, 1) + logical_date = timezone.datetime(2020, 1, 1) - dag = DAG("test_s3_key", schedule=None, start_date=execution_date, render_template_as_native_obj=True) + dag = DAG("test_s3_key", schedule=None, start_date=logical_date, render_template_as_native_obj=True) op = S3KeySensor( task_id="s3_key_sensor", bucket_key="{{ var.value.test_bucket_key }}", bucket_name=None, dag=dag, ) - - dag_run = DagRun( - dag_id=dag.dag_id, execution_date=execution_date, run_id="test", run_type=DagRunType.MANUAL - ) + if hasattr(DagRun, "execution_date"): # Airflow 2.x. + dag_run = DagRun( + dag_id=dag.dag_id, execution_date=logical_date, run_id="test", run_type=DagRunType.MANUAL + ) + else: + dag_run = DagRun( + dag_id=dag.dag_id, logical_date=logical_date, run_id="test", run_type=DagRunType.MANUAL + ) ti = TaskInstance(task=op) ti.dag_run = dag_run session.add(ti) diff --git a/providers/tests/amazon/aws/transfers/test_base.py b/providers/tests/amazon/aws/transfers/test_base.py index a60fdeba0624..b357b9ea8561 100644 --- a/providers/tests/amazon/aws/transfers/test_base.py +++ b/providers/tests/amazon/aws/transfers/test_base.py @@ -25,6 +25,8 @@ from airflow.utils import timezone from airflow.utils.types import DagRunType +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + DEFAULT_DATE = timezone.datetime(2020, 1, 1) @@ -42,12 +44,20 @@ def test_render_template(self, session, clean_dags_and_dagruns): dest_aws_conn_id="{{ ds }}", ) ti = TaskInstance(operator, run_id="something") - ti.dag_run = DagRun( - dag_id=self.dag.dag_id, - run_id="something", - execution_date=timezone.datetime(2020, 1, 1), - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + ti.dag_run = DagRun( + dag_id=self.dag.dag_id, + run_id="something", + logical_date=timezone.datetime(2020, 1, 1), + run_type=DagRunType.MANUAL, + ) + else: + ti.dag_run = DagRun( + dag_id=self.dag.dag_id, + run_id="something", + execution_date=timezone.datetime(2020, 1, 1), + run_type=DagRunType.MANUAL, + ) session.add(ti) session.commit() ti.render_templates() diff --git a/providers/tests/amazon/aws/transfers/test_dynamodb_to_s3.py b/providers/tests/amazon/aws/transfers/test_dynamodb_to_s3.py index bcd0a82632d8..0c9a56cd7d35 100644 --- a/providers/tests/amazon/aws/transfers/test_dynamodb_to_s3.py +++ b/providers/tests/amazon/aws/transfers/test_dynamodb_to_s3.py @@ -33,6 +33,8 @@ from airflow.utils import timezone from airflow.utils.types import DagRunType +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + class TestJSONEncoder: @pytest.mark.parametrize("value", ["102938.3043847474", 1.010001, 10, "100", "1E-128", 1e-128]) @@ -272,12 +274,20 @@ def test_render_template(self, session): dest_aws_conn_id="{{ ds }}", ) ti = TaskInstance(operator, run_id="something") - ti.dag_run = DagRun( - dag_id=dag.dag_id, - run_id="something", - execution_date=timezone.datetime(2020, 1, 1), - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + ti.dag_run = DagRun( + dag_id=dag.dag_id, + run_id="something", + logical_date=timezone.datetime(2020, 1, 1), + run_type=DagRunType.MANUAL, + ) + else: + ti.dag_run = DagRun( + dag_id=dag.dag_id, + run_id="something", + execution_date=timezone.datetime(2020, 1, 1), + run_type=DagRunType.MANUAL, + ) session.add(ti) session.commit() ti.render_templates() diff --git a/providers/tests/amazon/aws/transfers/test_mongo_to_s3.py b/providers/tests/amazon/aws/transfers/test_mongo_to_s3.py index 63afce78c4ff..c461cbc04657 100644 --- a/providers/tests/amazon/aws/transfers/test_mongo_to_s3.py +++ b/providers/tests/amazon/aws/transfers/test_mongo_to_s3.py @@ -26,6 +26,8 @@ from airflow.utils import timezone from airflow.utils.types import DagRunType +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + TASK_ID = "test_mongo_to_s3_operator" MONGO_CONN_ID = "default_mongo" AWS_CONN_ID = "default_s3" @@ -80,12 +82,20 @@ def test_template_field_overrides(self): @pytest.mark.db_test def test_render_template(self, session): - dag_run = DagRun( - dag_id=self.mock_operator.dag_id, - execution_date=DEFAULT_DATE, - run_id="test", - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=self.mock_operator.dag_id, + logical_date=DEFAULT_DATE, + run_id="test", + run_type=DagRunType.MANUAL, + ) + else: + dag_run = DagRun( + dag_id=self.mock_operator.dag_id, + execution_date=DEFAULT_DATE, + run_id="test", + run_type=DagRunType.MANUAL, + ) ti = TaskInstance(task=self.mock_operator) ti.dag_run = dag_run session.add(ti) diff --git a/providers/tests/apache/hive/hooks/test_hive.py b/providers/tests/apache/hive/hooks/test_hive.py index 1e9dc687584c..387339c185e1 100644 --- a/providers/tests/apache/hive/hooks/test_hive.py +++ b/providers/tests/apache/hive/hooks/test_hive.py @@ -42,6 +42,7 @@ MockSubProcess, ) from tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS DEFAULT_DATE = timezone.datetime(2015, 1, 1) DEFAULT_DATE_ISO = DEFAULT_DATE.isoformat() @@ -64,12 +65,13 @@ def test_run_cli(self, mock_popen, mock_temp_dir): mock_popen.return_value = mock_subprocess mock_temp_dir.return_value = "test_run_cli" + envron_name = "AIRFLOW_CTX_LOGICAL_DATE" if AIRFLOW_V_3_0_PLUS else "AIRFLOW_CTX_EXECUTION_DATE" with mock.patch.dict( "os.environ", { "AIRFLOW_CTX_DAG_ID": "test_dag_id", "AIRFLOW_CTX_TASK_ID": "test_task_id", - "AIRFLOW_CTX_EXECUTION_DATE": "2015-01-01T00:00:00+00:00", + envron_name: "2015-01-01T00:00:00+00:00", "AIRFLOW_CTX_TRY_NUMBER": "1", "AIRFLOW_CTX_DAG_RUN_ID": "55", "AIRFLOW_CTX_DAG_OWNER": "airflow", @@ -78,7 +80,7 @@ def test_run_cli(self, mock_popen, mock_temp_dir): ): hook = MockHiveCliHook() hook.run_cli("SHOW DATABASES") - + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" hive_cmd = [ "beeline", "-u", @@ -88,7 +90,7 @@ def test_run_cli(self, mock_popen, mock_temp_dir): "-hiveconf", "airflow.ctx.task_id=test_task_id", "-hiveconf", - "airflow.ctx.execution_date=2015-01-01T00:00:00+00:00", + f"airflow.ctx.{date_key}=2015-01-01T00:00:00+00:00", "-hiveconf", "airflow.ctx.try_number=1", "-hiveconf", @@ -124,21 +126,24 @@ def test_hive_cli_hook_invalid_schema(self): @mock.patch("subprocess.Popen") def test_run_cli_with_hive_conf(self, mock_popen): + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" hql = ( "set key;\n" "set airflow.ctx.dag_id;\nset airflow.ctx.dag_run_id;\n" - "set airflow.ctx.task_id;\nset airflow.ctx.execution_date;\n" + f"set airflow.ctx.task_id;\nset airflow.ctx.{date_key};\n" ) dag_id_ctx_var_name = AIRFLOW_VAR_NAME_FORMAT_MAPPING["AIRFLOW_CONTEXT_DAG_ID"]["env_var_format"] task_id_ctx_var_name = AIRFLOW_VAR_NAME_FORMAT_MAPPING["AIRFLOW_CONTEXT_TASK_ID"]["env_var_format"] - execution_date_ctx_var_name = AIRFLOW_VAR_NAME_FORMAT_MAPPING["AIRFLOW_CONTEXT_EXECUTION_DATE"][ - "env_var_format" - ] + logical_date_ctx_var_name = ( + AIRFLOW_VAR_NAME_FORMAT_MAPPING["AIRFLOW_CONTEXT_LOGICAL_DATE"]["env_var_format"] + if AIRFLOW_V_3_0_PLUS + else AIRFLOW_VAR_NAME_FORMAT_MAPPING["AIRFLOW_CONTEXT_EXECUTION_DATE"]["env_var_format"] + ) dag_run_id_ctx_var_name = AIRFLOW_VAR_NAME_FORMAT_MAPPING["AIRFLOW_CONTEXT_DAG_RUN_ID"][ "env_var_format" ] - + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" mock_output = [ "Connecting to jdbc:hive2://localhost:10000/default", "log4j:WARN No appenders could be found for logger (org.apache.hive.jdbc.Utils).", @@ -177,11 +182,11 @@ def test_run_cli_with_hive_conf(self, mock_popen): "| airflow.ctx.task_id=test_task_id |", "+-----------------------------------+--+", "1 row selected (0.009 seconds)", - "0: jdbc:hive2://localhost:10000/default> set airflow.ctx.execution_date;", + f"0: jdbc:hive2://localhost:10000/default> set airflow.ctx.{date_key};", "+-------------------------------------------------+--+", "| set |", "+-------------------------------------------------+--+", - "| airflow.ctx.execution_date=test_execution_date |", + f"| airflow.ctx.{date_key}=test_{date_key} |", "+-------------------------------------------------+--+", "1 row selected (0.006 seconds)", "0: jdbc:hive2://localhost:10000/default> ", @@ -195,7 +200,7 @@ def test_run_cli_with_hive_conf(self, mock_popen): { dag_id_ctx_var_name: "test_dag_id", task_id_ctx_var_name: "test_task_id", - execution_date_ctx_var_name: "test_execution_date", + logical_date_ctx_var_name: f"test_{date_key}", dag_run_id_ctx_var_name: "test_dag_run_id", }, ): @@ -208,13 +213,13 @@ def test_run_cli_with_hive_conf(self, mock_popen): assert "value" in process_inputs assert "test_dag_id" in process_inputs assert "test_task_id" in process_inputs - assert "test_execution_date" in process_inputs + assert f"test_{date_key}" in process_inputs assert "test_dag_run_id" in process_inputs assert "value" in output assert "test_dag_id" in output assert "test_task_id" in output - assert "test_execution_date" in output + assert f"test_{date_key}" in output assert "test_dag_run_id" in output @mock.patch("airflow.providers.apache.hive.hooks.hive.HiveCliHook.run_cli") @@ -680,12 +685,13 @@ def test_get_records(self): hook = MockHiveServer2Hook() query = f"SELECT * FROM {self.table}" + envron_name = "AIRFLOW_CTX_LOGICAL_DATE" if AIRFLOW_V_3_0_PLUS else "AIRFLOW_CTX_EXECUTION_DATE" with mock.patch.dict( "os.environ", { "AIRFLOW_CTX_DAG_ID": "test_dag_id", "AIRFLOW_CTX_TASK_ID": "HiveHook_3835", - "AIRFLOW_CTX_EXECUTION_DATE": "2015-01-01T00:00:00+00:00", + envron_name: "2015-01-01T00:00:00+00:00", "AIRFLOW_CTX_DAG_RUN_ID": "55", "AIRFLOW_CTX_DAG_OWNER": "airflow", "AIRFLOW_CTX_DAG_EMAIL": "test@airflow.com", @@ -694,11 +700,11 @@ def test_get_records(self): results = hook.get_records(query, schema=self.database) assert results == [(1, 1), (2, 2)] - + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" hook.get_conn.assert_called_with(self.database) hook.mock_cursor.execute.assert_any_call("set airflow.ctx.dag_id=test_dag_id") hook.mock_cursor.execute.assert_any_call("set airflow.ctx.task_id=HiveHook_3835") - hook.mock_cursor.execute.assert_any_call("set airflow.ctx.execution_date=2015-01-01T00:00:00+00:00") + hook.mock_cursor.execute.assert_any_call(f"set airflow.ctx.{date_key}=2015-01-01T00:00:00+00:00") hook.mock_cursor.execute.assert_any_call("set airflow.ctx.dag_run_id=55") hook.mock_cursor.execute.assert_any_call("set airflow.ctx.dag_owner=airflow") hook.mock_cursor.execute.assert_any_call("set airflow.ctx.dag_email=test@airflow.com") @@ -707,12 +713,13 @@ def test_get_pandas_df(self): hook = MockHiveServer2Hook() query = f"SELECT * FROM {self.table}" + envron_name = "AIRFLOW_CTX_LOGICAL_DATE" if AIRFLOW_V_3_0_PLUS else "AIRFLOW_CTX_EXECUTION_DATE" with mock.patch.dict( "os.environ", { "AIRFLOW_CTX_DAG_ID": "test_dag_id", "AIRFLOW_CTX_TASK_ID": "HiveHook_3835", - "AIRFLOW_CTX_EXECUTION_DATE": "2015-01-01T00:00:00+00:00", + envron_name: "2015-01-01T00:00:00+00:00", "AIRFLOW_CTX_DAG_RUN_ID": "55", "AIRFLOW_CTX_DAG_OWNER": "airflow", "AIRFLOW_CTX_DAG_EMAIL": "test@airflow.com", @@ -722,11 +729,11 @@ def test_get_pandas_df(self): assert len(df) == 2 assert df["hive_server_hook.a"].values.tolist() == [1, 2] - + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" hook.get_conn.assert_called_with(self.database) hook.mock_cursor.execute.assert_any_call("set airflow.ctx.dag_id=test_dag_id") hook.mock_cursor.execute.assert_any_call("set airflow.ctx.task_id=HiveHook_3835") - hook.mock_cursor.execute.assert_any_call("set airflow.ctx.execution_date=2015-01-01T00:00:00+00:00") + hook.mock_cursor.execute.assert_any_call(f"set airflow.ctx.{date_key}=2015-01-01T00:00:00+00:00") hook.mock_cursor.execute.assert_any_call("set airflow.ctx.dag_run_id=55") hook.mock_cursor.execute.assert_any_call("set airflow.ctx.dag_owner=airflow") hook.mock_cursor.execute.assert_any_call("set airflow.ctx.dag_email=test@airflow.com") @@ -792,13 +799,13 @@ def test_multi_statements(self): ] hook = MockHiveServer2Hook() - + envron_name = "AIRFLOW_CTX_LOGICAL_DATE" if AIRFLOW_V_3_0_PLUS else "AIRFLOW_CTX_EXECUTION_DATE" with mock.patch.dict( "os.environ", { "AIRFLOW_CTX_DAG_ID": "test_dag_id", "AIRFLOW_CTX_TASK_ID": "HiveHook_3835", - "AIRFLOW_CTX_EXECUTION_DATE": "2015-01-01T00:00:00+00:00", + envron_name: "2015-01-01T00:00:00+00:00", "AIRFLOW_CTX_DAG_RUN_ID": "55", "AIRFLOW_CTX_DAG_OWNER": "airflow", "AIRFLOW_CTX_DAG_EMAIL": "test@airflow.com", @@ -807,31 +814,35 @@ def test_multi_statements(self): results = hook.get_records(sqls, schema=self.database) assert results == [(1, 1), (2, 2)] + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" hook.get_conn.assert_called_with(self.database) hook.mock_cursor.execute.assert_any_call("CREATE TABLE IF NOT EXISTS test_multi_statements (i INT)") hook.mock_cursor.execute.assert_any_call(f"SELECT * FROM {self.table}") hook.mock_cursor.execute.assert_any_call("DROP TABLE test_multi_statements") hook.mock_cursor.execute.assert_any_call("set airflow.ctx.dag_id=test_dag_id") hook.mock_cursor.execute.assert_any_call("set airflow.ctx.task_id=HiveHook_3835") - hook.mock_cursor.execute.assert_any_call("set airflow.ctx.execution_date=2015-01-01T00:00:00+00:00") + hook.mock_cursor.execute.assert_any_call(f"set airflow.ctx.{date_key}=2015-01-01T00:00:00+00:00") hook.mock_cursor.execute.assert_any_call("set airflow.ctx.dag_run_id=55") hook.mock_cursor.execute.assert_any_call("set airflow.ctx.dag_owner=airflow") hook.mock_cursor.execute.assert_any_call("set airflow.ctx.dag_email=test@airflow.com") def test_get_results_with_hive_conf(self): + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" hql = [ "set key", "set airflow.ctx.dag_id", "set airflow.ctx.dag_run_id", "set airflow.ctx.task_id", - "set airflow.ctx.execution_date", + f"set airflow.ctx.{date_key}", ] dag_id_ctx_var_name = AIRFLOW_VAR_NAME_FORMAT_MAPPING["AIRFLOW_CONTEXT_DAG_ID"]["env_var_format"] task_id_ctx_var_name = AIRFLOW_VAR_NAME_FORMAT_MAPPING["AIRFLOW_CONTEXT_TASK_ID"]["env_var_format"] - execution_date_ctx_var_name = AIRFLOW_VAR_NAME_FORMAT_MAPPING["AIRFLOW_CONTEXT_EXECUTION_DATE"][ - "env_var_format" - ] + logical_date_ctx_var_name = ( + AIRFLOW_VAR_NAME_FORMAT_MAPPING["AIRFLOW_CONTEXT_LOGICAL_DATE"]["env_var_format"] + if AIRFLOW_V_3_0_PLUS + else AIRFLOW_VAR_NAME_FORMAT_MAPPING["AIRFLOW_CONTEXT_EXECUTION_DATE"]["env_var_format"] + ) dag_run_id_ctx_var_name = AIRFLOW_VAR_NAME_FORMAT_MAPPING["AIRFLOW_CONTEXT_DAG_RUN_ID"][ "env_var_format" ] @@ -841,7 +852,7 @@ def test_get_results_with_hive_conf(self): { dag_id_ctx_var_name: "test_dag_id", task_id_ctx_var_name: "test_task_id", - execution_date_ctx_var_name: "test_execution_date", + logical_date_ctx_var_name: f"test_{date_key}", dag_run_id_ctx_var_name: "test_dag_run_id", }, ): @@ -853,7 +864,7 @@ def test_get_results_with_hive_conf(self): ("value", "test"), ("test_dag_id", "test"), ("test_task_id", "test"), - ("test_execution_date", "test"), + (f"test_{date_key}", "test"), ("test_dag_run_id", "test"), ] ) @@ -865,7 +876,7 @@ def test_get_results_with_hive_conf(self): assert "value" in output assert "test_dag_id" in output assert "test_task_id" in output - assert "test_execution_date" in output + assert f"test_{date_key}" in output assert "test_dag_run_id" in output diff --git a/providers/tests/apache/hive/operators/test_hive.py b/providers/tests/apache/hive/operators/test_hive.py index cda19f9c2a05..162bc1ef0163 100644 --- a/providers/tests/apache/hive/operators/test_hive.py +++ b/providers/tests/apache/hive/operators/test_hive.py @@ -81,16 +81,16 @@ def test_mapred_job_name(self, mock_hook): op = HiveOperator(task_id="test_mapred_job_name", hql=self.hql, dag=self.dag) fake_run_id = "test_mapred_job_name" - fake_execution_date = timezone.datetime(2018, 6, 19) + fake_logical_date = timezone.datetime(2018, 6, 19) fake_ti = TaskInstance(task=op) - fake_ti.dag_run = DagRun(run_id=fake_run_id, execution_date=fake_execution_date) + fake_ti.dag_run = DagRun(run_id=fake_run_id, logical_date=fake_logical_date) fake_ti.hostname = "fake_hostname" fake_context = {"ti": fake_ti} op.execute(fake_context) assert ( "Airflow HiveOperator task for " - f"{fake_ti.hostname}.{self.dag.dag_id}.{op.task_id}.{fake_execution_date.isoformat()}" + f"{fake_ti.hostname}.{self.dag.dag_id}.{op.task_id}.{fake_logical_date.isoformat()}" == mock_hook.mapred_job_name ) @@ -118,7 +118,7 @@ def test_hive(self, mock_popen, mock_temp_dir): "-hiveconf", "airflow.ctx.task_id=basic_hql", "-hiveconf", - "airflow.ctx.execution_date=2015-01-01T00:00:00+00:00", + "airflow.ctx.logical_date=2015-01-01T00:00:00+00:00", "-hiveconf", "airflow.ctx.dag_run_id=", "-hiveconf", @@ -162,7 +162,7 @@ def test_hive_queues(self, mock_popen, mock_temp_dir): "-hiveconf", "airflow.ctx.task_id=test_hive_queues", "-hiveconf", - "airflow.ctx.execution_date=2015-01-01T00:00:00+00:00", + "airflow.ctx.logical_date=2015-01-01T00:00:00+00:00", "-hiveconf", "airflow.ctx.dag_run_id=", "-hiveconf", @@ -220,7 +220,7 @@ def test_hive_dryrun(self, mock_popen, mock_temp_dir): "-hiveconf", "airflow.ctx.task_id=", "-hiveconf", - "airflow.ctx.execution_date=", + "airflow.ctx.logical_date=", "-hiveconf", "airflow.ctx.dag_run_id=", "-hiveconf", @@ -261,7 +261,7 @@ def test_beeline(self, mock_popen, mock_temp_dir): "-hiveconf", "airflow.ctx.task_id=beeline_hql", "-hiveconf", - "airflow.ctx.execution_date=2015-01-01T00:00:00+00:00", + "airflow.ctx.logical_date=2015-01-01T00:00:00+00:00", "-hiveconf", "airflow.ctx.dag_run_id=", "-hiveconf", diff --git a/providers/tests/apache/hive/transfers/test_hive_to_mysql.py b/providers/tests/apache/hive/transfers/test_hive_to_mysql.py index f7a4e9c01301..be99c90291ae 100644 --- a/providers/tests/apache/hive/transfers/test_hive_to_mysql.py +++ b/providers/tests/apache/hive/transfers/test_hive_to_mysql.py @@ -170,7 +170,7 @@ def test_hive_to_mysql(self): "airflow.ctx.dag_owner": "airflow", "airflow.ctx.dag_id": "test_dag_id", "airflow.ctx.task_id": "hive_to_mysql_check", - "airflow.ctx.execution_date": "2015-01-01T00:00:00+00:00", + "airflow.ctx.logical_date": "2015-01-01T00:00:00+00:00", } assert expected_hive_conf == actual_hive_conf diff --git a/providers/tests/apache/hive/transfers/test_mysql_to_hive.py b/providers/tests/apache/hive/transfers/test_mysql_to_hive.py index 59ffa9d74375..0d1aad8deb51 100644 --- a/providers/tests/apache/hive/transfers/test_mysql_to_hive.py +++ b/providers/tests/apache/hive/transfers/test_mysql_to_hive.py @@ -43,7 +43,7 @@ class TestTransfer: env_vars = { "AIRFLOW_CTX_DAG_ID": "test_dag_id", "AIRFLOW_CTX_TASK_ID": "test_task_id", - "AIRFLOW_CTX_EXECUTION_DATE": "2015-01-01T00:00:00+00:00", + "AIRFLOW_CTX_LOGICAL_DATE": "2015-01-01T00:00:00+00:00", "AIRFLOW_CTX_DAG_RUN_ID": "55", "AIRFLOW_CTX_DAG_OWNER": "airflow", "AIRFLOW_CTX_DAG_EMAIL": "test@airflow.com", diff --git a/providers/tests/apache/kylin/operators/test_kylin_cube.py b/providers/tests/apache/kylin/operators/test_kylin_cube.py index 572e27b9037e..ea0ea67827ab 100644 --- a/providers/tests/apache/kylin/operators/test_kylin_cube.py +++ b/providers/tests/apache/kylin/operators/test_kylin_cube.py @@ -29,6 +29,8 @@ from airflow.utils import timezone from airflow.utils.types import DagRunType +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + DEFAULT_DATE = timezone.datetime(2020, 1, 1) @@ -169,12 +171,20 @@ def test_render_template(self, session): }, ) ti = TaskInstance(operator, run_id="kylin_test") - ti.dag_run = DagRun( - dag_id=self.dag.dag_id, - run_id="kylin_test", - execution_date=DEFAULT_DATE, - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + ti.dag_run = DagRun( + dag_id=self.dag.dag_id, + run_id="kylin_test", + logical_date=DEFAULT_DATE, + run_type=DagRunType.MANUAL, + ) + else: + ti.dag_run = DagRun( + dag_id=self.dag.dag_id, + run_id="kylin_test", + execution_date=DEFAULT_DATE, + run_type=DagRunType.MANUAL, + ) session.add(ti) session.commit() ti.render_templates() diff --git a/providers/tests/apache/livy/operators/test_livy.py b/providers/tests/apache/livy/operators/test_livy.py index be1674e189ff..abec29badace 100644 --- a/providers/tests/apache/livy/operators/test_livy.py +++ b/providers/tests/apache/livy/operators/test_livy.py @@ -448,7 +448,6 @@ def test_spark_params_templating(create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() diff --git a/providers/tests/apache/spark/decorators/test_pyspark.py b/providers/tests/apache/spark/decorators/test_pyspark.py index 03732992e147..e2c9ee9b8995 100644 --- a/providers/tests/apache/spark/decorators/test_pyspark.py +++ b/providers/tests/apache/spark/decorators/test_pyspark.py @@ -105,7 +105,7 @@ def f(spark, sc): ret = f() dr = dag_maker.create_dagrun() - ret.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + ret.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) ti = dr.get_task_instances()[0] assert len(ti.xcom_pull()) == 100 assert config.get("spark.master") == "spark://none" @@ -132,7 +132,7 @@ def f(): ret = f() dr = dag_maker.create_dagrun() - ret.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + ret.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) ti = dr.get_task_instances()[0] assert ti.xcom_pull() == e assert config.get("spark.master") == "local[*]" @@ -156,7 +156,7 @@ def f(spark, sc): ret = f() dr = dag_maker.create_dagrun() - ret.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + ret.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) ti = dr.get_task_instances()[0] assert ti.xcom_pull() assert config.get("spark.remote") == "sc://localhost" @@ -182,7 +182,7 @@ def f(spark, sc): ret = f() dr = dag_maker.create_dagrun() - ret.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + ret.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) ti = dr.get_task_instances()[0] assert ti.xcom_pull() assert config.get("spark.remote") == "sc://localhost/;user_id=connect;token=1234;use_ssl=True" diff --git a/providers/tests/apache/spark/operators/test_spark_jdbc.py b/providers/tests/apache/spark/operators/test_spark_jdbc.py index 980e35994655..c1bcadc1ca28 100644 --- a/providers/tests/apache/spark/operators/test_spark_jdbc.py +++ b/providers/tests/apache/spark/operators/test_spark_jdbc.py @@ -156,7 +156,6 @@ def test_templating_with_create_task_instance_of_operator( # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() diff --git a/providers/tests/apache/spark/operators/test_spark_sql.py b/providers/tests/apache/spark/operators/test_spark_sql.py index 070826bdd427..9154ba349bfa 100644 --- a/providers/tests/apache/spark/operators/test_spark_sql.py +++ b/providers/tests/apache/spark/operators/test_spark_sql.py @@ -76,7 +76,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() diff --git a/providers/tests/apache/spark/operators/test_spark_submit.py b/providers/tests/apache/spark/operators/test_spark_submit.py index 87fb50ff91f4..a6ba854826d0 100644 --- a/providers/tests/apache/spark/operators/test_spark_submit.py +++ b/providers/tests/apache/spark/operators/test_spark_submit.py @@ -27,6 +27,8 @@ from airflow.utils import timezone from airflow.utils.types import DagRunType +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + DEFAULT_DATE = timezone.datetime(2017, 1, 1) @@ -193,12 +195,20 @@ def test_render_template(self, session): # Given operator = SparkSubmitOperator(task_id="spark_submit_job", dag=self.dag, **self._config) ti = TaskInstance(operator, run_id="spark_test") - ti.dag_run = DagRun( - dag_id=self.dag.dag_id, - run_id="spark_test", - execution_date=DEFAULT_DATE, - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + ti.dag_run = DagRun( + dag_id=self.dag.dag_id, + run_id="spark_test", + logical_date=DEFAULT_DATE, + run_type=DagRunType.MANUAL, + ) + else: + ti.dag_run = DagRun( + dag_id=self.dag.dag_id, + run_id="spark_test", + execution_date=DEFAULT_DATE, + run_type=DagRunType.MANUAL, + ) session.add(ti) session.commit() # When @@ -246,7 +256,6 @@ def test_templating_with_create_task_instance_of_operator( # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() diff --git a/providers/tests/celery/log_handlers/test_log_handlers.py b/providers/tests/celery/log_handlers/test_log_handlers.py index d47029cdd4dd..db2ff94c4e1b 100644 --- a/providers/tests/celery/log_handlers/test_log_handlers.py +++ b/providers/tests/celery/log_handlers/test_log_handlers.py @@ -36,12 +36,8 @@ from airflow.utils.timezone import datetime from airflow.utils.types import DagRunType -from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars -if AIRFLOW_V_3_0_PLUS: - pass - pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] DEFAULT_DATE = datetime(2016, 1, 1) @@ -68,12 +64,11 @@ def test__read_for_celery_executor_fallbacks_to_worker(self, create_task_instanc """Test for executors which do not have `get_task_log` method, it fallbacks to reading log from worker""" executor_name = "CeleryExecutor" - ti = create_task_instance( dag_id="dag_for_testing_celery_executor_log_read", task_id="task_for_testing_celery_executor_log_read", run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, ) ti.state = TaskInstanceState.RUNNING ti.try_number = 1 diff --git a/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py b/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py index f5db8b806af4..14785fab37b9 100644 --- a/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py +++ b/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py @@ -28,6 +28,7 @@ from kubernetes.client.rest import ApiException from urllib3 import HTTPResponse +from airflow import __version__ from airflow.exceptions import AirflowException from airflow.executors.executor_constants import ( CELERY_EXECUTOR, @@ -66,6 +67,11 @@ pytestmark = pytest.mark.skip_if_database_isolation_mode +if __version__.startswith("2."): + LOGICAL_DATE_KEY = "execution_date" +else: + LOGICAL_DATE_KEY = "logical_date" + class TestAirflowKubernetesScheduler: @staticmethod @@ -1818,7 +1824,7 @@ def setup_method(self): "task_id": "task", "run_id": "run_id", "try_number": "1", - "execution_date": None, + LOGICAL_DATE_KEY: None, } self.pod = k8s.V1Pod( metadata=k8s.V1ObjectMeta( diff --git a/providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py b/providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py index f896af6cfe95..3639c6e8972c 100644 --- a/providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py +++ b/providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py @@ -81,7 +81,7 @@ def test__read_for_k8s_executor(self, mock_k8s_get_task_log, create_task_instanc dag_id="dag_for_testing_k8s_executor_log_read", task_id="task_for_testing_k8s_executor_log_read", run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, ) ti.state = state ti.triggerer_job = None @@ -122,13 +122,22 @@ def task_callable(ti): executor_config={"pod_override": pod_override}, ) triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - dagrun = dag.create_dagrun( - run_type=DagRunType.MANUAL, - state=State.RUNNING, - execution_date=DEFAULT_DATE, - data_interval=dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE), - **triggered_by_kwargs, - ) + if AIRFLOW_V_3_0_PLUS: + dagrun = dag.create_dagrun( + run_type=DagRunType.MANUAL, + state=State.RUNNING, + logical_date=DEFAULT_DATE, + data_interval=dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE), + **triggered_by_kwargs, + ) + else: + dagrun = dag.create_dagrun( + run_type=DagRunType.MANUAL, + state=State.RUNNING, + execution_date=DEFAULT_DATE, + data_interval=dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE), + **triggered_by_kwargs, + ) ti = TaskInstance(task=task, run_id=dagrun.run_id) ti.try_number = 3 diff --git a/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py b/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py index 24372dcdca4a..7f9b743f3eb2 100644 --- a/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py +++ b/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py @@ -36,6 +36,8 @@ from airflow.utils import db, timezone from airflow.utils.types import DagRunType +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + @patch("airflow.providers.cncf.kubernetes.operators.spark_kubernetes.KubernetesHook") def test_spark_kubernetes_operator(mock_kubernetes_hook, data_file): @@ -169,12 +171,19 @@ def test_spark_kubernetes_operator_hook(mock_kubernetes_hook, data_file): def create_context(task): dag = DAG(dag_id="dag", schedule=None) tzinfo = pendulum.timezone("Europe/Amsterdam") - execution_date = timezone.datetime(2016, 1, 1, 1, 0, 0, tzinfo=tzinfo) - dag_run = DagRun( - dag_id=dag.dag_id, - execution_date=execution_date, - run_id=DagRun.generate_run_id(DagRunType.MANUAL, execution_date), - ) + logical_date = timezone.datetime(2016, 1, 1, 1, 0, 0, tzinfo=tzinfo) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=dag.dag_id, + logical_date=logical_date, + run_id=DagRun.generate_run_id(DagRunType.MANUAL, logical_date), + ) + else: + dag_run = DagRun( + dag_id=dag.dag_id, + execution_date=logical_date, + run_id=DagRun.generate_run_id(DagRunType.MANUAL, logical_date), + ) task_instance = TaskInstance(task=task) task_instance.dag_run = dag_run task_instance.dag_id = dag.dag_id @@ -700,18 +709,17 @@ def test_template_body_templating(create_task_instance_of_operator, session): dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", session=session, - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() ti.render_templates() task: SparkKubernetesOperator = ti.task - assert task.template_body == {"spark": {"foo": "2024-02-01", "bar": "test_template_body_templating_dag"}} + assert task.template_body == {"spark": {"foo": "2016-01-01", "bar": "test_template_body_templating_dag"}} @pytest.mark.db_test def test_resolve_application_file_template_file(dag_maker, tmp_path, session): - execution_date = timezone.datetime(2024, 2, 1, tzinfo=timezone.utc) + logical_date = timezone.datetime(2024, 2, 1, tzinfo=timezone.utc) filename = "test-application-file.yml" (tmp_path / filename).write_text("foo: {{ ds }}\nbar: {{ dag_run.dag_id }}\nspam: egg") @@ -725,8 +733,10 @@ def test_resolve_application_file_template_file(dag_maker, tmp_path, session): kubernetes_conn_id="kubernetes_default_kube_config", task_id="test_template_body_templating_task", ) - - ti = dag_maker.create_dagrun(execution_date=execution_date).task_instances[0] + if AIRFLOW_V_3_0_PLUS: + ti = dag_maker.create_dagrun(logical_date=logical_date).task_instances[0] + else: + ti = dag_maker.create_dagrun(execution_date=logical_date).task_instances[0] session.add(ti) session.commit() ti.render_templates() @@ -751,7 +761,7 @@ def test_resolve_application_file_template_file(dag_maker, tmp_path, session): ], ) def test_resolve_application_file_template_non_dictionary(dag_maker, tmp_path, body, session): - execution_date = timezone.datetime(2024, 2, 1, tzinfo=timezone.utc) + logical_date = timezone.datetime(2024, 2, 1, tzinfo=timezone.utc) filename = "test-application-file.yml" with open((tmp_path / filename), "w") as fp: yaml.safe_dump(body, fp) @@ -766,8 +776,10 @@ def test_resolve_application_file_template_non_dictionary(dag_maker, tmp_path, b kubernetes_conn_id="kubernetes_default_kube_config", task_id="test_template_body_templating_task", ) - - ti = dag_maker.create_dagrun(execution_date=execution_date).task_instances[0] + if AIRFLOW_V_3_0_PLUS: + ti = dag_maker.create_dagrun(logical_date=logical_date).task_instances[0] + else: + ti = dag_maker.create_dagrun(execution_date=logical_date).task_instances[0] session.add(ti) session.commit() ti.render_templates() diff --git a/providers/tests/cncf/kubernetes/test_pod_generator.py b/providers/tests/cncf/kubernetes/test_pod_generator.py index 30d1df2a0d73..635dd2843ffe 100644 --- a/providers/tests/cncf/kubernetes/test_pod_generator.py +++ b/providers/tests/cncf/kubernetes/test_pod_generator.py @@ -37,8 +37,15 @@ ) from airflow.providers.cncf.kubernetes.secret import Secret +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + now = pendulum.now("UTC") +if AIRFLOW_V_3_0_PLUS: + LOGICAL_DATE_KEY = "logical_date" +else: + LOGICAL_DATE_KEY = "execution_date" + class TestPodGenerator: def setup_method(self): @@ -70,15 +77,15 @@ def setup_method(self): Secret("env", "TARGET", "secret_b", "source_b"), ] - self.execution_date = parser.parse("2020-08-24 00:00:00.000000") - self.execution_date_label = datetime_to_label_safe_datestring(self.execution_date) + self.logical_date = parser.parse("2020-08-24 00:00:00.000000") + self.logical_date_label = datetime_to_label_safe_datestring(self.logical_date) self.dag_id = "dag_id" self.task_id = "task_id" self.try_number = 3 self.labels = { "airflow-worker": "uuid", "dag_id": self.dag_id, - "execution_date": self.execution_date_label, + LOGICAL_DATE_KEY: self.logical_date_label, "task_id": self.task_id, "try_number": str(self.try_number), "airflow_version": __version__.replace("+", "-"), @@ -87,7 +94,7 @@ def setup_method(self): self.annotations = { "dag_id": self.dag_id, "task_id": self.task_id, - "execution_date": self.execution_date.isoformat(), + LOGICAL_DATE_KEY: self.logical_date.isoformat(), "try_number": str(self.try_number), } self.metadata = { @@ -335,7 +342,7 @@ def test_construct_pod( pod_id="pod_id", kube_image=config_image, try_number=self.try_number, - date=self.execution_date, + date=self.logical_date, args=["command"], pod_override_object=executor_config, base_worker_pod=worker_config, @@ -372,7 +379,7 @@ def test_construct_pod_mapped_task(self, data_file): try_number=self.try_number, kube_image="", map_index=0, - date=self.execution_date, + date=self.logical_date, args=["command"], pod_override_object=None, base_worker_pod=worker_config, @@ -407,7 +414,7 @@ def test_construct_pod_empty_executor_config(self, data_file): pod_id="pod_id", kube_image="test-image", try_number=3, - date=self.execution_date, + date=self.logical_date, args=["command"], pod_override_object=executor_config, base_worker_pod=worker_config, @@ -447,7 +454,7 @@ def test_construct_pod_attribute_error(self, mock_rand_str, data_file): pod_id="pod_id", kube_image="test-image", try_number=3, - date=self.execution_date, + date=self.logical_date, args=["command"], pod_override_object=executor_config, base_worker_pod=worker_config, @@ -467,7 +474,7 @@ def test_ensure_max_identifier_length(self, mock_rand_str, data_file): pod_id="a" * 512, kube_image="a" * 512, try_number=3, - date=self.execution_date, + date=self.logical_date, args=["command"], namespace="namespace", scheduler_job_id="a" * 512, @@ -699,17 +706,17 @@ def test_validate_pod_generator(self, data_file): pytest.param(dict(map_index=2), {"map_index": "2"}, id="map_index"), pytest.param(dict(run_id="2"), {"run_id": "2"}, id="run_id"), pytest.param( - dict(execution_date=now), - {"execution_date": datetime_to_label_safe_datestring(now)}, + dict(logical_date=now), + {LOGICAL_DATE_KEY: datetime_to_label_safe_datestring(now)}, id="date", ), pytest.param( - dict(airflow_worker=2, map_index=2, run_id="2", execution_date=now), + dict(airflow_worker=2, map_index=2, run_id="2", logical_date=now), { "airflow-worker": "2", "map_index": "2", "run_id": "2", - "execution_date": datetime_to_label_safe_datestring(now), + LOGICAL_DATE_KEY: datetime_to_label_safe_datestring(now), }, id="all", ), diff --git a/providers/tests/cncf/kubernetes/test_template_rendering.py b/providers/tests/cncf/kubernetes/test_template_rendering.py index 8eb43cd1eaeb..4cccefb75066 100644 --- a/providers/tests/cncf/kubernetes/test_template_rendering.py +++ b/providers/tests/cncf/kubernetes/test_template_rendering.py @@ -42,7 +42,7 @@ def test_render_k8s_pod_yaml(pod_mutation_hook, create_task_instance): dag_id="test_render_k8s_pod_yaml", run_id="test_run_id", task_id="op1", - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, ) expected_pod_spec = { diff --git a/providers/tests/common/io/xcom/test_backend.py b/providers/tests/common/io/xcom/test_backend.py index b88340c62e66..5c4afac1f481 100644 --- a/providers/tests/common/io/xcom/test_backend.py +++ b/providers/tests/common/io/xcom/test_backend.py @@ -71,7 +71,7 @@ def task_instance(create_task_instance_of_operator, session): EmptyOperator, dag_id="test-dag-id", task_id="test-task-id", - execution_date=timezone.datetime(2021, 12, 3, 4, 56), + logical_date=timezone.datetime(2021, 12, 3, 4, 56), session=session, ) diff --git a/providers/tests/common/sql/operators/test_sql.py b/providers/tests/common/sql/operators/test_sql.py index 00149a6389a1..443284bbb9b5 100644 --- a/providers/tests/common/sql/operators/test_sql.py +++ b/providers/tests/common/sql/operators/test_sql.py @@ -1144,14 +1144,24 @@ def test_branch_single_value_with_dag_run(self, mock_get_db_hook): self.dag.clear() triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - dr = self.dag.create_dagrun( - run_id="manual__", - start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, - state=State.RUNNING, - data_interval=(DEFAULT_DATE, DEFAULT_DATE), - **triggered_by_kwargs, - ) + if AIRFLOW_V_3_0_PLUS: + dr = self.dag.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + logical_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) + else: + dr = self.dag.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + execution_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) mock_get_records = mock_get_db_hook.return_value.get_first @@ -1186,15 +1196,24 @@ def test_branch_true_with_dag_run(self, mock_get_db_hook): self.branch_2.set_upstream(branch_op) self.dag.clear() triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - - dr = self.dag.create_dagrun( - run_id="manual__", - start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, - state=State.RUNNING, - data_interval=(DEFAULT_DATE, DEFAULT_DATE), - **triggered_by_kwargs, - ) + if AIRFLOW_V_3_0_PLUS: + dr = self.dag.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + logical_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) + else: + dr = self.dag.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + execution_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) mock_get_records = mock_get_db_hook.return_value.get_first @@ -1231,14 +1250,24 @@ def test_branch_false_with_dag_run(self, mock_get_db_hook): self.dag.clear() triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - dr = self.dag.create_dagrun( - run_id="manual__", - start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, - state=State.RUNNING, - data_interval=(DEFAULT_DATE, DEFAULT_DATE), - **triggered_by_kwargs, - ) + if AIRFLOW_V_3_0_PLUS: + dr = self.dag.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + logical_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) + else: + dr = self.dag.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + execution_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) mock_get_records = mock_get_db_hook.return_value.get_first @@ -1276,14 +1305,24 @@ def test_branch_list_with_dag_run(self, mock_get_db_hook): self.dag.clear() triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - dr = self.dag.create_dagrun( - run_id="manual__", - start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, - state=State.RUNNING, - data_interval=(DEFAULT_DATE, DEFAULT_DATE), - **triggered_by_kwargs, - ) + if AIRFLOW_V_3_0_PLUS: + dr = self.dag.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + logical_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) + else: + dr = self.dag.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + execution_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) mock_get_records = mock_get_db_hook.return_value.get_first mock_get_records.return_value = [["1"]] @@ -1318,14 +1357,24 @@ def test_invalid_query_result_with_dag_run(self, mock_get_db_hook): self.dag.clear() triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - self.dag.create_dagrun( - run_id="manual__", - start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, - state=State.RUNNING, - data_interval=(DEFAULT_DATE, DEFAULT_DATE), - **triggered_by_kwargs, - ) + if AIRFLOW_V_3_0_PLUS: + self.dag.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + logical_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) + else: + self.dag.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + execution_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) mock_get_records = mock_get_db_hook.return_value.get_first @@ -1350,15 +1399,24 @@ def test_with_skip_in_branch_downstream_dependencies(self, mock_get_db_hook): branch_op >> self.branch_2 self.dag.clear() triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - - dr = self.dag.create_dagrun( - run_id="manual__", - start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, - state=State.RUNNING, - data_interval=(DEFAULT_DATE, DEFAULT_DATE), - **triggered_by_kwargs, - ) + if AIRFLOW_V_3_0_PLUS: + dr = self.dag.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + logical_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) + else: + dr = self.dag.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + execution_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) mock_get_records = mock_get_db_hook.return_value.get_first @@ -1392,15 +1450,24 @@ def test_with_skip_in_branch_downstream_dependencies2(self, mock_get_db_hook): branch_op >> self.branch_2 self.dag.clear() triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - - dr = self.dag.create_dagrun( - run_id="manual__", - start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, - state=State.RUNNING, - data_interval=(DEFAULT_DATE, DEFAULT_DATE), - **triggered_by_kwargs, - ) + if AIRFLOW_V_3_0_PLUS: + dr = self.dag.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + logical_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) + else: + dr = self.dag.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + execution_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) mock_get_records = mock_get_db_hook.return_value.get_first diff --git a/providers/tests/databricks/operators/test_databricks_copy.py b/providers/tests/databricks/operators/test_databricks_copy.py index b03309acbd2b..60b8cd82c14f 100644 --- a/providers/tests/databricks/operators/test_databricks_copy.py +++ b/providers/tests/databricks/operators/test_databricks_copy.py @@ -21,7 +21,6 @@ from airflow.exceptions import AirflowException from airflow.providers.databricks.operators.databricks_sql import DatabricksCopyIntoOperator -from airflow.utils import timezone DATE = "2017-04-20" TASK_ID = "databricks-sql-operator" @@ -244,7 +243,6 @@ def test_templating(create_task_instance_of_operator, session): file_format="JSON", dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), session=session, ) session.add(ti) diff --git a/providers/tests/dbt/cloud/operators/test_dbt.py b/providers/tests/dbt/cloud/operators/test_dbt.py index a5f8752ffb3e..7aaf5d6870b4 100644 --- a/providers/tests/dbt/cloud/operators/test_dbt.py +++ b/providers/tests/dbt/cloud/operators/test_dbt.py @@ -544,7 +544,6 @@ def test_run_job_operator_link(self, conn_id, account_id, create_task_instance_o ti = create_task_instance_of_operator( DbtCloudRunJobOperator, dag_id="test_dbt_cloud_run_job_op_link", - execution_date=DEFAULT_DATE, task_id="trigger_dbt_cloud_job", dbt_cloud_conn_id=conn_id, job_id=JOB_ID, diff --git a/providers/tests/docker/decorators/test_docker.py b/providers/tests/docker/decorators/test_docker.py index c40e071c402b..5d961d0ec4e2 100644 --- a/providers/tests/docker/decorators/test_docker.py +++ b/providers/tests/docker/decorators/test_docker.py @@ -51,7 +51,7 @@ def f(): ret = f() dr = dag_maker.create_dagrun() - ret.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + ret.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) ti = dr.get_task_instances()[0] assert len(ti.xcom_pull()) == 100 @@ -66,7 +66,7 @@ def f(num_results): ret = f(50) dr = dag_maker.create_dagrun() - ret.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + ret.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) ti = dr.get_task_instances()[0] result = ti.xcom_pull() assert isinstance(result, list) @@ -95,7 +95,7 @@ def return_dict(number: int): ret = return_dict(test_number) dr = dag_maker.create_dagrun() - ret.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + ret.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) ti = dr.get_task_instances()[0] assert ti.xcom_pull(key="number") == test_number + 1 @@ -111,7 +111,7 @@ def f(): ret = f() dr = dag_maker.create_dagrun() - ret.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + ret.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) ti = dr.get_task_instances()[0] assert ti.xcom_pull() is None @@ -163,9 +163,9 @@ def f(exit_code): dr = dag_maker.create_dagrun() if expected_state == TaskInstanceState.FAILED: with pytest.raises(AirflowException): - ret.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + ret.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) else: - ret.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + ret.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) ti = dr.get_task_instances()[0] assert ti.state == expected_state @@ -323,7 +323,7 @@ def f(): dr = dag_maker.create_dagrun() with pytest.raises(AirflowException): - ret.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + ret.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) ti = dr.get_task_instances()[0] assert ti.state == TaskInstanceState.FAILED diff --git a/providers/tests/elasticsearch/log/test_es_task_handler.py b/providers/tests/elasticsearch/log/test_es_task_handler.py index d8d328969148..642f3fe4cdb1 100644 --- a/providers/tests/elasticsearch/log/test_es_task_handler.py +++ b/providers/tests/elasticsearch/log/test_es_task_handler.py @@ -46,6 +46,7 @@ from providers.tests.elasticsearch.log.elasticmock import elasticmock from providers.tests.elasticsearch.log.elasticmock.utilities import SearchFailedException +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_dags, clear_db_runs @@ -55,11 +56,11 @@ ES_PROVIDER_YAML_FILE = AIRFLOW_SOURCES_ROOT_DIR / "airflow" / "providers" / "elasticsearch" / "provider.yaml" -def get_ti(dag_id, task_id, execution_date, create_task_instance): +def get_ti(dag_id, task_id, logical_date, create_task_instance): ti = create_task_instance( dag_id=dag_id, task_id=task_id, - execution_date=execution_date, + logical_date=logical_date, dagrun_state=DagRunState.RUNNING, state=TaskInstanceState.RUNNING, ) @@ -71,18 +72,23 @@ def get_ti(dag_id, task_id, execution_date, create_task_instance): class TestElasticsearchTaskHandler: DAG_ID = "dag_for_testing_es_task_handler" TASK_ID = "task_for_testing_es_log_handler" - EXECUTION_DATE = datetime(2016, 1, 1) + LOGICAL_DATE = datetime(2016, 1, 1) LOG_ID = f"{DAG_ID}-{TASK_ID}-2016-01-01T00:00:00+00:00-1" - JSON_LOG_ID = f"{DAG_ID}-{TASK_ID}-{ElasticsearchTaskHandler._clean_date(EXECUTION_DATE)}-1" + JSON_LOG_ID = f"{DAG_ID}-{TASK_ID}-{ElasticsearchTaskHandler._clean_date(LOGICAL_DATE)}-1" FILENAME_TEMPLATE = "{try_number}.log" @pytest.fixture def ti(self, create_task_instance, create_log_template): - create_log_template(self.FILENAME_TEMPLATE, "{dag_id}-{task_id}-{execution_date}-{try_number}") + create_log_template( + self.FILENAME_TEMPLATE, + "{dag_id}-{task_id}-{logical_date}-{try_number}" + if AIRFLOW_V_3_0_PLUS + else "{dag_id}-{task_id}-{execution_date}-{try_number}", + ) yield get_ti( dag_id=self.DAG_ID, task_id=self.TASK_ID, - execution_date=self.EXECUTION_DATE, + logical_date=self.LOGICAL_DATE, create_task_instance=create_task_instance, ) clear_db_runs() @@ -255,7 +261,7 @@ def test_read_missing_logs(self, seconds, create_task_instance): ti = get_ti( self.DAG_ID, self.TASK_ID, - pendulum.instance(self.EXECUTION_DATE).add(days=1), # so logs are not found + pendulum.instance(self.LOGICAL_DATE).add(days=1), # so logs are not found create_task_instance=create_task_instance, ) ts = pendulum.now().add(seconds=-seconds) @@ -563,8 +569,8 @@ def test_render_log_id(self, ti): assert self.JSON_LOG_ID == self.es_task_handler._render_log_id(ti, 1) def test_clean_date(self): - clean_execution_date = self.es_task_handler._clean_date(datetime(2016, 7, 8, 9, 10, 11, 12)) - assert "2016_07_08T09_10_11_000012" == clean_execution_date + clean_logical_date = self.es_task_handler._clean_date(datetime(2016, 7, 8, 9, 10, 11, 12)) + assert "2016_07_08T09_10_11_000012" == clean_logical_date @pytest.mark.parametrize( "json_format, es_frontend, expected_url", diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_asset_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_asset_endpoint.py index d831adf4d8ef..9a5435dce547 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_asset_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_asset_endpoint.py @@ -168,9 +168,7 @@ def test_delete_should_respond_204(self, session, create_dummy_dag): assert response.status_code == 204 conn = session.query(AssetDagRunQueue).all() assert len(conn) == 0 - _check_last_log( - session, dag_id=dag_id, event="api.delete_dag_asset_queued_event", execution_date=None - ) + _check_last_log(session, dag_id=dag_id, event="api.delete_dag_asset_queued_event", logical_date=None) def test_should_respond_404(self): dag_id = "not_exists" @@ -309,7 +307,7 @@ def test_delete_should_respond_204(self, session, create_dummy_dag): assert response.status_code == 204 conn = session.query(AssetDagRunQueue).all() assert len(conn) == 0 - _check_last_log(session, dag_id=None, event="api.delete_asset_queued_events", execution_date=None) + _check_last_log(session, dag_id=None, event="api.delete_asset_queued_events", logical_date=None) def test_should_respond_404(self): asset_uri = "not_exists" diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py index a32c0a7dbddf..1ae1f742773f 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py @@ -217,7 +217,7 @@ def test_should_respond_200_on_patch_with_granular_dag_access(self, session): environ_overrides={"REMOTE_USER": "test_granular_permissions"}, ) assert response.status_code == 200 - _check_last_log(session, dag_id="TEST_DAG_1", event="api.patch_dag", execution_date=None) + _check_last_log(session, dag_id="TEST_DAG_1", event="api.patch_dag", logical_date=None) def test_validation_error_raises_400(self): patch_body = { diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py index c1f3e71f306b..2ef37491d81c 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_dag_run_endpoint.py @@ -153,7 +153,7 @@ def _create_test_dag_run(self, state=DagRunState.RUNNING, extra_dag=False, commi dag_id="TEST_DAG_ID", run_id=f"TEST_DAG_RUN_ID_{i}", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time) + timedelta(days=i - 1), + logical_date=timezone.parse(self.default_time) + timedelta(days=i - 1), start_date=timezone.parse(self.default_time), external_trigger=True, state=state, @@ -170,7 +170,7 @@ def _create_test_dag_run(self, state=DagRunState.RUNNING, extra_dag=False, commi dag_id=f"TEST_DAG_ID_{i}", run_id=f"TEST_DAG_RUN_ID_{i}", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time_2), + logical_date=timezone.parse(self.default_time_2), start_date=timezone.parse(self.default_time), external_trigger=True, state=state, @@ -203,7 +203,6 @@ def test_should_return_accessible_with_tilde_as_dag_id_and_dag_level_permissions "dag_run_id": "TEST_DAG_RUN_ID_1", "end_date": None, "state": "running", - "execution_date": self.default_time, "logical_date": self.default_time, "external_trigger": True, "start_date": self.default_time, @@ -220,7 +219,6 @@ def test_should_return_accessible_with_tilde_as_dag_id_and_dag_level_permissions "dag_run_id": "TEST_DAG_RUN_ID_2", "end_date": None, "state": "running", - "execution_date": self.default_time_2, "logical_date": self.default_time_2, "external_trigger": True, "start_date": self.default_time, @@ -268,7 +266,7 @@ def test_should_raises_403_unauthorized(self, username): "api/v1/dags/TEST_DAG_ID/dagRuns", json={ "dag_run_id": "TEST_DAG_RUN_ID_1", - "execution_date": self.default_time, + "logical_date": self.default_time, }, environ_overrides={"REMOTE_USER": username}, ) diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_event_log_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_event_log_endpoint.py index 9f00c7943e8b..847bca6fad5d 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_event_log_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_event_log_endpoint.py @@ -63,7 +63,7 @@ def task_instance(session, create_task_instance, request): dag_id="TEST_DAG_ID", task_id="TEST_TASK_ID", run_id="TEST_RUN_ID", - execution_date=request.instance.default_time, + logical_date=request.instance.default_time, ) diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py index a9a25c1f8a59..74279b95dc43 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_task_instance_endpoint.py @@ -109,7 +109,7 @@ class TestTaskInstanceEndpoint: def setup_attrs(self, configured_app, dagbag) -> None: self.default_time = DEFAULT_DATETIME_1 self.ti_init = { - "execution_date": self.default_time, + "logical_date": self.default_time, "state": State.RUNNING, } self.ti_extras = { @@ -145,7 +145,7 @@ def create_task_instances( counter = min(len(task_instances), counter) run_id = "TEST_DAG_RUN_ID" - execution_date = self.ti_init.pop("execution_date", self.default_time) + logical_date = self.ti_init.pop("logical_date", self.default_time) dr = None tis = [] @@ -157,16 +157,16 @@ def create_task_instances( else: self.ti_init.update(task_instances[i]) - if "execution_date" in self.ti_init: + if "logical_date" in self.ti_init: run_id = f"TEST_DAG_RUN_ID_{i}" - execution_date = self.ti_init.pop("execution_date") + logical_date = self.ti_init.pop("logical_date") dr = None if not dr: dr = DagRun( run_id=run_id, dag_id=dag_id, - execution_date=execution_date, + logical_date=logical_date, run_type=DagRunType.MANUAL, state=dag_run_state, ) @@ -247,7 +247,7 @@ def test_return_TI_only_from_readable_dags(self, task_instances, user, expected_ self.create_task_instances( session, task_instances=[ - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=i)} + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=i)} for i in range(task_instances[dag_id]) ], dag_id=dag_id, @@ -379,7 +379,7 @@ def test_should_raise_403_forbidden(self, username): json={ "dry_run": True, "task_id": "print_the_context", - "execution_date": DEFAULT_DATETIME_1.isoformat(), + "logical_date": DEFAULT_DATETIME_1.isoformat(), "include_upstream": True, "include_downstream": True, "include_future": True, diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_xcom_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_xcom_endpoint.py index 16e5d44f89e2..2bb7bf7a6621 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_xcom_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_xcom_endpoint.py @@ -77,11 +77,21 @@ def _compare_xcom_collections(collection1: dict, collection_2: dict): def sort_key(record): return ( - record.get("dag_id"), - record.get("task_id"), - record.get("execution_date"), - record.get("map_index"), - record.get("key"), + ( + record.get("dag_id"), + record.get("task_id"), + record.get("logical_date"), + record.get("map_index"), + record.get("key"), + ) + if AIRFLOW_V_3_0_PLUS + else ( + record.get("dag_id"), + record.get("task_id"), + record.get("execution_date"), + record.get("map_index"), + record.get("key"), + ) ) assert sorted(collection1.get("xcom_entries", []), key=sort_key) == sorted( @@ -117,16 +127,16 @@ class TestGetXComEntries(TestXComEndpoint): def test_should_respond_200_with_tilde_and_granular_dag_access(self): dag_id_1 = "test-dag-id-1" task_id_1 = "test-task-id-1" - execution_date = "2005-04-02T00:00:00+00:00" - execution_date_parsed = timezone.parse(execution_date) - dag_run_id_1 = DagRun.generate_run_id(DagRunType.MANUAL, execution_date_parsed) - self._create_xcom_entries(dag_id_1, dag_run_id_1, execution_date_parsed, task_id_1) + logical_date = "2005-04-02T00:00:00+00:00" + logical_date_parsed = timezone.parse(logical_date) + dag_run_id_1 = DagRun.generate_run_id(DagRunType.MANUAL, logical_date_parsed) + self._create_xcom_entries(dag_id_1, dag_run_id_1, logical_date_parsed, task_id_1) dag_id_2 = "test-dag-id-2" task_id_2 = "test-task-id-2" - run_id_2 = DagRun.generate_run_id(DagRunType.MANUAL, execution_date_parsed) - self._create_xcom_entries(dag_id_2, run_id_2, execution_date_parsed, task_id_2) - self._create_invalid_xcom_entries(execution_date_parsed) + run_id_2 = DagRun.generate_run_id(DagRunType.MANUAL, logical_date_parsed) + self._create_xcom_entries(dag_id_2, run_id_2, logical_date_parsed, task_id_2) + self._create_invalid_xcom_entries(logical_date_parsed) response = self.client.get( "/api/v1/dags/~/dagRuns/~/taskInstances/~/xcomEntries", environ_overrides={"REMOTE_USER": "test_granular_permissions"}, @@ -136,13 +146,14 @@ def test_should_respond_200_with_tilde_and_granular_dag_access(self): response_data = response.json for xcom_entry in response_data["xcom_entries"]: xcom_entry["timestamp"] = "TIMESTAMP" + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" _compare_xcom_collections( response_data, { "xcom_entries": [ { "dag_id": dag_id_1, - "execution_date": execution_date, + date_key: logical_date, "key": "test-xcom-key-1", "task_id": task_id_1, "timestamp": "TIMESTAMP", @@ -150,7 +161,7 @@ def test_should_respond_200_with_tilde_and_granular_dag_access(self): }, { "dag_id": dag_id_1, - "execution_date": execution_date, + date_key: logical_date, "key": "test-xcom-key-2", "task_id": task_id_1, "timestamp": "TIMESTAMP", @@ -161,17 +172,26 @@ def test_should_respond_200_with_tilde_and_granular_dag_access(self): }, ) - def _create_xcom_entries(self, dag_id, run_id, execution_date, task_id, mapped_ti=False): + def _create_xcom_entries(self, dag_id, run_id, logical_date, task_id, mapped_ti=False): with create_session() as session: dag = DagModel(dag_id=dag_id) session.add(dag) - dagrun = DagRun( - dag_id=dag_id, - run_id=run_id, - execution_date=execution_date, - start_date=execution_date, - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + dagrun = DagRun( + dag_id=dag_id, + run_id=run_id, + logical_date=logical_date, + start_date=logical_date, + run_type=DagRunType.MANUAL, + ) + else: + dagrun = DagRun( + dag_id=dag_id, + run_id=run_id, + execution_date=logical_date, + start_date=logical_date, + run_type=DagRunType.MANUAL, + ) session.add(dagrun) if mapped_ti: for i in [0, 1]: @@ -195,28 +215,46 @@ def _create_xcom_entries(self, dag_id, run_id, execution_date, task_id, mapped_t key=key, value="TEST", run_id=run_id, task_id=task_id, dag_id=dag_id, map_index=map_index ) - def _create_invalid_xcom_entries(self, execution_date): + def _create_invalid_xcom_entries(self, logical_date): """ Invalid XCom entries to test join query """ with create_session() as session: dag = DagModel(dag_id="invalid_dag") session.add(dag) - dagrun = DagRun( - dag_id="invalid_dag", - run_id="invalid_run_id", - execution_date=execution_date + timedelta(days=1), - start_date=execution_date, - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + dagrun = DagRun( + dag_id="invalid_dag", + run_id="invalid_run_id", + logical_date=logical_date + timedelta(days=1), + start_date=logical_date, + run_type=DagRunType.MANUAL, + ) + else: + dagrun = DagRun( + dag_id="invalid_dag", + run_id="invalid_run_id", + execution_date=logical_date + timedelta(days=1), + start_date=logical_date, + run_type=DagRunType.MANUAL, + ) session.add(dagrun) - dagrun1 = DagRun( - dag_id="invalid_dag", - run_id="not_this_run_id", - execution_date=execution_date, - start_date=execution_date, - run_type=DagRunType.MANUAL, - ) + if AIRFLOW_V_3_0_PLUS: + dagrun1 = DagRun( + dag_id="invalid_dag", + run_id="not_this_run_id", + logical_date=logical_date, + start_date=logical_date, + run_type=DagRunType.MANUAL, + ) + else: + dagrun1 = DagRun( + dag_id="invalid_dag", + run_id="not_this_run_id", + execution_date=logical_date, + start_date=logical_date, + run_type=DagRunType.MANUAL, + ) session.add(dagrun1) ti = TaskInstance(EmptyOperator(task_id="invalid_task"), run_id="not_this_run_id") ti.dag_id = "invalid_dag" diff --git a/providers/tests/google/cloud/log/test_gcs_task_handler.py b/providers/tests/google/cloud/log/test_gcs_task_handler.py index 22ad5f06af8d..93b68399c203 100644 --- a/providers/tests/google/cloud/log/test_gcs_task_handler.py +++ b/providers/tests/google/cloud/log/test_gcs_task_handler.py @@ -39,7 +39,7 @@ def task_instance(self, create_task_instance, session): self.ti = ti = create_task_instance( dag_id="dag_for_testing_gcs_task_handler", task_id="task_for_testing_gcs_task_handler", - execution_date=datetime(2020, 1, 1), + logical_date=datetime(2020, 1, 1), state=TaskInstanceState.RUNNING, ) ti.try_number = 1 diff --git a/providers/tests/google/cloud/log/test_stackdriver_task_handler.py b/providers/tests/google/cloud/log/test_stackdriver_task_handler.py index 947e6906291b..4f309c9f3434 100644 --- a/providers/tests/google/cloud/log/test_stackdriver_task_handler.py +++ b/providers/tests/google/cloud/log/test_stackdriver_task_handler.py @@ -30,7 +30,7 @@ from airflow.utils import timezone from airflow.utils.state import TaskInstanceState -from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS, AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_dags, clear_db_runs @@ -120,7 +120,7 @@ def task_instance(self, create_task_instance, clean_stackdriver_handlers): self.ti = create_task_instance( dag_id=self.DAG_ID, task_id=self.TASK_ID, - execution_date=timezone.datetime(2016, 1, 1), + logical_date=timezone.datetime(2016, 1, 1), state=TaskInstanceState.RUNNING, ) self.ti.try_number = 1 @@ -148,10 +148,11 @@ def test_should_set_labels(self, mock_client, mock_get_creds_and_project_id): self.logger.info("test-message") stackdriver_task_handler.flush() + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" labels = { "task_id": self.TASK_ID, "dag_id": self.DAG_ID, - "execution_date": "2016-01-01T00:00:00+00:00", + date_key: "2016-01-01T00:00:00+00:00", "try_number": "1", } resource = Resource(type="global", labels={}) @@ -172,10 +173,11 @@ def test_should_append_labels(self, mock_client, mock_get_creds_and_project_id): self.logger.info("test-message") stackdriver_task_handler.flush() + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" labels = { "task_id": self.TASK_ID, "dag_id": self.DAG_ID, - "execution_date": "2016-01-01T00:00:00+00:00", + date_key: "2016-01-01T00:00:00+00:00", "try_number": "1", "product.googleapis.com/task_id": "test-value", } @@ -194,16 +196,20 @@ def test_should_read_logs_for_all_try(self, mock_client, mock_get_creds_and_proj stackdriver_task_handler = self._setup_handler() logs, metadata = stackdriver_task_handler.read(self.ti) + + date_label = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" + + filter_str = ( + 'resource.type="global"\n' + 'logName="projects/project_id/logs/airflow"\n' + 'labels.task_id="task_for_testing_stackdriver_task_handler"\n' + 'labels.dag_id="dag_for_testing_stackdriver_file_task_handler"\n' + f'labels.{date_label}="2016-01-01T00:00:00+00:00"' + ) mock_client.return_value.list_log_entries.assert_called_once_with( request=ListLogEntriesRequest( resource_names=["projects/project_id"], - filter=( - 'resource.type="global"\n' - 'logName="projects/project_id/logs/airflow"\n' - 'labels.task_id="task_for_testing_stackdriver_task_handler"\n' - 'labels.dag_id="dag_for_testing_stackdriver_file_task_handler"\n' - 'labels.execution_date="2016-01-01T00:00:00+00:00"' - ), + filter=filter_str, order_by="timestamp asc", page_size=1000, page_token=None, @@ -224,16 +230,18 @@ def test_should_read_logs_for_task_with_quote(self, mock_client, mock_get_creds_ stackdriver_task_handler = self._setup_handler() logs, metadata = stackdriver_task_handler.read(self.ti) + date_label = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" + filter_str = ( + 'resource.type="global"\n' + 'logName="projects/project_id/logs/airflow"\n' + 'labels.task_id="K\\"OT"\n' + 'labels.dag_id="dag_for_testing_stackdriver_file_task_handler"\n' + f'labels.{date_label}="2016-01-01T00:00:00+00:00"' + ) mock_client.return_value.list_log_entries.assert_called_once_with( request=ListLogEntriesRequest( resource_names=["projects/project_id"], - filter=( - 'resource.type="global"\n' - 'logName="projects/project_id/logs/airflow"\n' - 'labels.task_id="K\\"OT"\n' - 'labels.dag_id="dag_for_testing_stackdriver_file_task_handler"\n' - 'labels.execution_date="2016-01-01T00:00:00+00:00"' - ), + filter=filter_str, order_by="timestamp asc", page_size=1000, page_token=None, @@ -252,17 +260,19 @@ def test_should_read_logs_for_single_try(self, mock_client, mock_get_creds_and_p stackdriver_task_handler = self._setup_handler() logs, metadata = stackdriver_task_handler.read(self.ti, 3) + date_label = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" + filter_str = ( + 'resource.type="global"\n' + 'logName="projects/project_id/logs/airflow"\n' + 'labels.task_id="task_for_testing_stackdriver_task_handler"\n' + 'labels.dag_id="dag_for_testing_stackdriver_file_task_handler"\n' + f'labels.{date_label}="2016-01-01T00:00:00+00:00"\n' + 'labels.try_number="3"' + ) mock_client.return_value.list_log_entries.assert_called_once_with( request=ListLogEntriesRequest( resource_names=["projects/project_id"], - filter=( - 'resource.type="global"\n' - 'logName="projects/project_id/logs/airflow"\n' - 'labels.task_id="task_for_testing_stackdriver_task_handler"\n' - 'labels.dag_id="dag_for_testing_stackdriver_file_task_handler"\n' - 'labels.execution_date="2016-01-01T00:00:00+00:00"\n' - 'labels.try_number="3"' - ), + filter=filter_str, order_by="timestamp asc", page_size=1000, page_token=None, @@ -282,17 +292,19 @@ def test_should_read_logs_with_pagination(self, mock_client, mock_get_creds_and_ stackdriver_task_handler = self._setup_handler() logs, metadata1 = stackdriver_task_handler.read(self.ti, 3) + date_label = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" + filter_str = ( + 'resource.type="global"\n' + 'logName="projects/project_id/logs/airflow"\n' + 'labels.task_id="task_for_testing_stackdriver_task_handler"\n' + 'labels.dag_id="dag_for_testing_stackdriver_file_task_handler"\n' + f'labels.{date_label}="2016-01-01T00:00:00+00:00"\n' + 'labels.try_number="3"' + ) mock_client.return_value.list_log_entries.assert_called_once_with( request=ListLogEntriesRequest( resource_names=["projects/project_id"], - filter=( - '''resource.type="global" -logName="projects/project_id/logs/airflow" -labels.task_id="task_for_testing_stackdriver_task_handler" -labels.dag_id="dag_for_testing_stackdriver_file_task_handler" -labels.execution_date="2016-01-01T00:00:00+00:00" -labels.try_number="3"''' - ), + filter=filter_str, order_by="timestamp asc", page_size=1000, page_token=None, @@ -312,7 +324,7 @@ def test_should_read_logs_with_pagination(self, mock_client, mock_get_creds_and_ 'logName="projects/project_id/logs/airflow"\n' 'labels.task_id="task_for_testing_stackdriver_task_handler"\n' 'labels.dag_id="dag_for_testing_stackdriver_file_task_handler"\n' - 'labels.execution_date="2016-01-01T00:00:00+00:00"\n' + f'labels.{date_label}="2016-01-01T00:00:00+00:00"\n' 'labels.try_number="3"' ), order_by="timestamp asc", @@ -357,19 +369,21 @@ def test_should_read_logs_with_custom_resources(self, mock_client, mock_get_cred mock_client.return_value.list_log_entries.return_value.pages = iter([page]) logs, metadata = stackdriver_task_handler.read(self.ti) + date_label = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" + filter_str = ( + 'resource.type="cloud_composer_environment"\n' + 'logName="projects/project_id/logs/airflow"\n' + 'resource.labels."environment.name"="test-instance"\n' + 'resource.labels.location="europe-west-3"\n' + 'resource.labels.project_id="project_id"\n' + 'labels.task_id="task_for_testing_stackdriver_task_handler"\n' + 'labels.dag_id="dag_for_testing_stackdriver_file_task_handler"\n' + f'labels.{date_label}="2016-01-01T00:00:00+00:00"' + ) mock_client.return_value.list_log_entries.assert_called_once_with( request=ListLogEntriesRequest( resource_names=["projects/project_id"], - filter=( - 'resource.type="cloud_composer_environment"\n' - 'logName="projects/project_id/logs/airflow"\n' - 'resource.labels."environment.name"="test-instance"\n' - 'resource.labels.location="europe-west-3"\n' - 'resource.labels.project_id="project_id"\n' - 'labels.task_id="task_for_testing_stackdriver_task_handler"\n' - 'labels.dag_id="dag_for_testing_stackdriver_file_task_handler"\n' - 'labels.execution_date="2016-01-01T00:00:00+00:00"' - ), + filter=filter_str, order_by="timestamp asc", page_size=1000, page_token=None, @@ -416,12 +430,13 @@ def test_should_return_valid_external_url(self, mock_client, mock_get_creds_and_ assert "global" in parsed_qs["resource"] filter_params = parsed_qs["advancedFilter"][0].splitlines() + date_label = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" expected_filter = [ 'resource.type="global"', 'logName="projects/project_id/logs/airflow"', f'labels.task_id="{self.ti.task_id}"', f'labels.dag_id="{self.DAG_ID}"', - f'labels.execution_date="{self.ti.execution_date.isoformat()}"', + f'labels.{date_label}="{self.ti.logical_date.isoformat() if AIRFLOW_V_3_0_PLUS else self.ti.execution_date.isoformat()}"', f'labels.try_number="{self.ti.try_number}"', ] assert set(expected_filter) == set(filter_params) diff --git a/providers/tests/google/cloud/operators/test_automl.py b/providers/tests/google/cloud/operators/test_automl.py index 86a5c7ac34ad..e0b286a34325 100644 --- a/providers/tests/google/cloud/operators/test_automl.py +++ b/providers/tests/google/cloud/operators/test_automl.py @@ -46,7 +46,6 @@ AutoMLTablesUpdateDatasetOperator, AutoMLTrainModelOperator, ) -from airflow.utils import timezone CREDENTIALS = "test-creds" TASK_ID = "test-automl-hook" @@ -129,7 +128,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -226,7 +224,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -286,7 +283,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), payload={}, ) session.add(ti) @@ -395,7 +391,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -449,7 +444,6 @@ def test_templating(self, create_task_instance_of_operator): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) @@ -490,7 +484,6 @@ def test_templating(self, create_task_instance_of_operator): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) @@ -555,7 +548,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -626,7 +618,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -675,7 +666,6 @@ def test_templating(self, create_task_instance_of_operator): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) @@ -742,7 +732,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -792,7 +781,6 @@ def test_templating(self, create_task_instance_of_operator): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) @@ -841,7 +829,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -911,7 +898,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() diff --git a/providers/tests/google/cloud/operators/test_bigquery.py b/providers/tests/google/cloud/operators/test_bigquery.py index 4269d02377b2..fee7437dc944 100644 --- a/providers/tests/google/cloud/operators/test_bigquery.py +++ b/providers/tests/google/cloud/operators/test_bigquery.py @@ -748,7 +748,6 @@ def test_bigquery_operator_extra_serialized_field_when_single_query( ti = create_task_instance_of_operator( BigQueryExecuteQueryOperator, dag_id=TEST_DAG_ID, - execution_date=DEFAULT_DATE, task_id=TASK_ID, sql="SELECT * FROM test_table", ) @@ -786,7 +785,6 @@ def test_bigquery_operator_extra_serialized_field_when_multiple_queries( ti = create_task_instance_of_operator( BigQueryExecuteQueryOperator, dag_id=TEST_DAG_ID, - execution_date=DEFAULT_DATE, task_id=TASK_ID, sql=["SELECT * FROM test_table", "SELECT * FROM test_table2"], ) @@ -847,9 +845,8 @@ def test_bigquery_operator_extra_link_when_single_query( ): with pytest.warns(AirflowProviderDeprecationWarning): ti = create_task_instance_of_operator( - BigQueryExecuteQueryOperator, + operator_class=BigQueryExecuteQueryOperator, dag_id=TEST_DAG_ID, - execution_date=DEFAULT_DATE, task_id=TASK_ID, sql="SELECT * FROM test_table", ) @@ -871,7 +868,6 @@ def test_bigquery_operator_extra_link_when_multiple_query( ti = create_task_instance_of_operator( BigQueryExecuteQueryOperator, dag_id=TEST_DAG_ID, - execution_date=DEFAULT_DATE, task_id=TASK_ID, sql=["SELECT * FROM test_table", "SELECT * FROM test_table2"], ) diff --git a/providers/tests/google/cloud/operators/test_bigquery_dts.py b/providers/tests/google/cloud/operators/test_bigquery_dts.py index 189c2623fcf3..0bdaeeee7eed 100644 --- a/providers/tests/google/cloud/operators/test_bigquery_dts.py +++ b/providers/tests/google/cloud/operators/test_bigquery_dts.py @@ -32,7 +32,6 @@ PROJECT_ID = "id" - TRANSFER_CONFIG = { "data_source_id": "google_cloud_storage", "destination_dataset_id": "example_dataset", diff --git a/providers/tests/google/cloud/operators/test_cloud_build.py b/providers/tests/google/cloud/operators/test_cloud_build.py index 958190c942ae..1b9eb363ee7e 100644 --- a/providers/tests/google/cloud/operators/test_cloud_build.py +++ b/providers/tests/google/cloud/operators/test_cloud_build.py @@ -404,18 +404,29 @@ def test_do_nothing(self, source_key): assert body == expected_body +def set_execute_complete(session, ti, **next_kwargs): + ti.next_method = "execute_complete" + ti.next_kwargs = next_kwargs + session.flush() + + +@pytest.mark.db_test @mock.patch(CLOUD_BUILD_HOOK_PATH) -def test_async_create_build_fires_correct_trigger_should_execute_successfully(mock_hook): +def test_async_create_build_fires_correct_trigger_should_execute_successfully( + mock_hook, create_task_instance_of_operator, session +): mock_hook.return_value.create_build_without_waiting_for_result.return_value = (BUILD, BUILD_ID) - operator = CloudBuildCreateBuildOperator( + ti = create_task_instance_of_operator( build=BUILD, task_id="id", deferrable=True, + operator_class=CloudBuildCreateBuildOperator, + dag_id="cloud_build_create_build_job", ) with pytest.raises(TaskDeferred) as exc: - operator.execute(create_context(operator)) + ti.task.execute({"ti": ti}) assert isinstance( exc.value.trigger, CloudBuildCreateBuildTrigger @@ -443,19 +454,25 @@ def test_async_create_build_without_wait_should_execute_successfully(mock_hook): mock_hook.return_value.get_build.assert_called_once_with(id_=BUILD_ID, project_id=None, location="global") +@pytest.mark.db_test @mock.patch(CLOUD_BUILD_HOOK_PATH) -def test_async_create_build_correct_logging_should_execute_successfully(mock_hook): +def test_async_create_build_correct_logging_should_execute_successfully( + mock_hook, create_task_instance_of_operator, session +): mock_hook.return_value.create_build_without_waiting_for_result.return_value = (BUILD, BUILD_ID) mock_hook.return_value.get_build.return_value = Build() - operator = CloudBuildCreateBuildOperator( + ti = create_task_instance_of_operator( build=BUILD, task_id="id", deferrable=True, + operator_class=CloudBuildCreateBuildOperator, + dag_id="cloud_build_create_build_logging", ) - with mock.patch.object(operator.log, "info") as mock_log_info: - operator.execute_complete( - context=create_context(operator), + + with mock.patch.object(ti.task.log, "info") as mock_log_info: + ti.task.execute_complete( + context={"ti": ti}, event={ "instance": TEST_BUILD_INSTANCE, "status": "success", @@ -463,6 +480,7 @@ def test_async_create_build_correct_logging_should_execute_successfully(mock_hoo "id_": BUILD_ID, }, ) + mock_log_info.assert_called_with("Cloud Build completed with response %s ", "Build completed") @@ -521,7 +539,7 @@ def create_context(task): logical_date = datetime(2022, 1, 1, 0, 0, 0) dag_run = DagRun( dag_id=dag.dag_id, - execution_date=logical_date, + logical_date=logical_date, run_id=DagRun.generate_run_id(DagRunType.MANUAL, logical_date), ) task_instance = TaskInstance(task=task) diff --git a/providers/tests/google/cloud/operators/test_compute.py b/providers/tests/google/cloud/operators/test_compute.py index 7913143618cb..6aca2c2b7bb7 100644 --- a/providers/tests/google/cloud/operators/test_compute.py +++ b/providers/tests/google/cloud/operators/test_compute.py @@ -229,7 +229,7 @@ def test_insert_instance_should_not_throw_ex_when_name_is_templated(self, mock_h get_instance_obj_mock, ] body_with_templated_name = deepcopy(GCE_INSTANCE_BODY_API_CALL) - body_with_templated_name["name"] = "{{ execution_date }}" + body_with_templated_name["name"] = "{{ logical_date }}" op = ComputeEngineInsertInstanceOperator( project_id=GCP_PROJECT_ID, resource_id=GCE_RESOURCE_ID, diff --git a/providers/tests/google/cloud/operators/test_dataproc.py b/providers/tests/google/cloud/operators/test_dataproc.py index 2ec6ceb9babb..1e27f3503176 100644 --- a/providers/tests/google/cloud/operators/test_dataproc.py +++ b/providers/tests/google/cloud/operators/test_dataproc.py @@ -1082,7 +1082,6 @@ def test_create_cluster_operator_extra_links(dag_maker, create_task_instance_of_ ti = create_task_instance_of_operator( DataprocCreateClusterOperator, dag_id=TEST_DAG_ID, - execution_date=DEFAULT_DATE, task_id=TASK_ID, region=GCP_REGION, project_id=GCP_PROJECT, @@ -1192,7 +1191,6 @@ def test_scale_cluster_operator_extra_links(dag_maker, create_task_instance_of_o ti = create_task_instance_of_operator( DataprocScaleClusterOperator, dag_id=TEST_DAG_ID, - execution_date=DEFAULT_DATE, task_id=TASK_ID, cluster_name=CLUSTER_NAME, project_id=GCP_PROJECT, @@ -1599,7 +1597,6 @@ def test_submit_job_operator_extra_links(mock_hook, dag_maker, create_task_insta ti = create_task_instance_of_operator( DataprocSubmitJobOperator, dag_id=TEST_DAG_ID, - execution_date=DEFAULT_DATE, task_id=TASK_ID, region=GCP_REGION, project_id=GCP_PROJECT, @@ -1807,7 +1804,6 @@ def test_update_cluster_operator_extra_links(dag_maker, create_task_instance_of_ ti = create_task_instance_of_operator( DataprocUpdateClusterOperator, dag_id=TEST_DAG_ID, - execution_date=DEFAULT_DATE, task_id=TASK_ID, region=GCP_REGION, cluster_name=CLUSTER_NAME, @@ -2033,7 +2029,6 @@ def test_instantiate_workflow_operator_extra_links(mock_hook, dag_maker, create_ ti = create_task_instance_of_operator( DataprocInstantiateWorkflowTemplateOperator, dag_id=TEST_DAG_ID, - execution_date=DEFAULT_DATE, task_id=TASK_ID, region=GCP_REGION, project_id=GCP_PROJECT, @@ -2195,7 +2190,6 @@ def test_instantiate_inline_workflow_operator_extra_links( ti = create_task_instance_of_operator( DataprocInstantiateInlineWorkflowTemplateOperator, dag_id=TEST_DAG_ID, - execution_date=DEFAULT_DATE, task_id=TASK_ID, region=GCP_REGION, project_id=GCP_PROJECT, @@ -2524,7 +2518,6 @@ def test_submit_spark_job_operator_extra_links(mock_hook, dag_maker, create_task ti = create_task_instance_of_operator( DataprocSubmitSparkJobOperator, dag_id=TEST_DAG_ID, - execution_date=DEFAULT_DATE, task_id=TASK_ID, region=GCP_REGION, gcp_conn_id=GCP_CONN_ID, diff --git a/providers/tests/google/cloud/operators/test_gcs.py b/providers/tests/google/cloud/operators/test_gcs.py index fcb5fcb92514..a92de4a7de24 100644 --- a/providers/tests/google/cloud/operators/test_gcs.py +++ b/providers/tests/google/cloud/operators/test_gcs.py @@ -408,7 +408,7 @@ def test_execute(self, mock_hook, mock_subprocess, mock_tempdir): ] mock_ti = mock.Mock() context = dict( - execution_date=timespan_start, + logical_date=timespan_start, dag=mock_dag, ti=mock_ti, ) @@ -595,7 +595,7 @@ def test_get_openlineage_facets_on_complete( ), ] context = dict( - execution_date=timespan_start, + logical_date=timespan_start, dag=mock_dag, ti=mock.Mock(), ) diff --git a/providers/tests/google/cloud/operators/test_mlengine.py b/providers/tests/google/cloud/operators/test_mlengine.py index c6dcd3853a75..c10a59f94b18 100644 --- a/providers/tests/google/cloud/operators/test_mlengine.py +++ b/providers/tests/google/cloud/operators/test_mlengine.py @@ -322,7 +322,6 @@ def test_templating(self, create_task_instance_of_operator, session): data_format="data_format", dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -401,7 +400,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -483,7 +481,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -528,7 +525,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -574,7 +570,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -620,7 +615,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -671,7 +665,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -744,7 +737,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -816,7 +808,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -874,7 +865,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -945,7 +935,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -1246,7 +1235,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -1282,8 +1270,11 @@ def test_templating(self, create_task_instance_of_operator, session): TEST_LABELS = {"job_type": "training", "***-version": "v2-5-0-dev0"} +@pytest.mark.db_test @patch(MLENGINE_AI_PATH.format("MLEngineHook")) -def test_async_create_training_job_should_execute_successfully(mock_hook): +def test_async_create_training_job_should_execute_successfully( + mock_hook, create_task_instance_of_operator, session +): """ Asserts that a task is deferred and a MLEngineStartTrainingJobTrigger will be fired when the MLEngineStartTrainingJobOperator is executed in deferrable mode when deferrable=True. @@ -1291,7 +1282,7 @@ def test_async_create_training_job_should_execute_successfully(mock_hook): mock_hook.return_value.create_job_without_waiting_result.return_value = "test_training" with pytest.warns(AirflowProviderDeprecationWarning): - op = MLEngineStartTrainingJobOperator( + ti = create_task_instance_of_operator( task_id=TEST_TASK_ID, project_id=TEST_GCP_PROJECT_ID, region=TEST_REGION, @@ -1304,10 +1295,12 @@ def test_async_create_training_job_should_execute_successfully(mock_hook): training_args=TEST_TRAINING_ARGS, labels=TEST_LABELS, deferrable=True, + operator_class=MLEngineStartTrainingJobOperator, + dag_id="test_async_create_training", ) with pytest.raises(TaskDeferred) as exc: - op.execute(create_context(op)) + ti.task.execute({"ti": ti}) assert isinstance( exc.value.trigger, MLEngineStartTrainingJobTrigger @@ -1342,7 +1335,7 @@ def create_context(task): logical_date = datetime(2022, 1, 1, 0, 0, 0) dag_run = DagRun( dag_id=dag.dag_id, - execution_date=logical_date, + logical_date=logical_date, run_id=DagRun.generate_run_id(DagRunType.MANUAL, logical_date), ) task_instance = TaskInstance(task=task) @@ -1359,11 +1352,15 @@ def create_context(task): } -def test_async_create_training_job_logging_should_execute_successfully(): +@pytest.mark.db_test +def test_async_create_training_job_logging_should_execute_successfully( + create_task_instance_of_operator, session +): """Asserts that logging occurs as expected""" with pytest.warns(AirflowProviderDeprecationWarning): - op = MLEngineStartTrainingJobOperator( + ti = create_task_instance_of_operator( + operator_class=MLEngineStartTrainingJobOperator, task_id=TEST_TASK_ID, project_id=TEST_GCP_PROJECT_ID, region=TEST_REGION, @@ -1376,24 +1373,33 @@ def test_async_create_training_job_logging_should_execute_successfully(): training_args=TEST_TRAINING_ARGS, labels=TEST_LABELS, deferrable=True, + dag_id="test_async_create_training_job_logging", ) - with mock.patch.object(op.log, "info") as mock_log_info: - op.execute_complete( - context=create_context(op), + + with mock.patch.object(ti.task.log, "info") as mock_log_info: + ti.task.execute_complete( + context={"ti": ti}, event={"status": "success", "message": "Job completed", "job_id": TEST_TASK_ID}, ) + mock_log_info.assert_called_with("%s completed with response %s ", TEST_TASK_ID, "Job completed") +@pytest.mark.db_test @patch(MLENGINE_AI_PATH.format("MLEngineHook")) -def test_async_create_training_job_with_conflict_should_execute_successfully(mock_hook): - mock_hook.return_value.create_job_without_waiting_result.return_value = HttpError( +def test_async_create_training_job_with_conflict_should_execute_successfully( + mock_hook, create_task_instance_of_operator, session +): + """Test that the operator defers correctly when a conflict occurs.""" + + mock_hook.return_value.create_job_without_waiting_result.side_effect = HttpError( resp=httplib2.Response({"status": "409"}), content=b"some bytes" ) mock_hook.return_value.get_job.return_value = {"job_id": "test_training"} with pytest.warns(AirflowProviderDeprecationWarning): - op = MLEngineStartTrainingJobOperator( + ti = create_task_instance_of_operator( + operator_class=MLEngineStartTrainingJobOperator, task_id=TEST_TASK_ID, project_id=TEST_GCP_PROJECT_ID, region=TEST_REGION, @@ -1406,9 +1412,11 @@ def test_async_create_training_job_with_conflict_should_execute_successfully(moc training_args=TEST_TRAINING_ARGS, labels=TEST_LABELS, deferrable=True, + dag_id="test_async_create_training_job_with_conflict", ) + with pytest.raises(TaskDeferred): - op.execute(create_context(op)) + ti.task.execute({"ti": ti}) mock_hook.assert_called_once_with( gcp_conn_id="google_cloud_default", @@ -1417,9 +1425,15 @@ def test_async_create_training_job_with_conflict_should_execute_successfully(moc mock_hook.return_value.create_job_without_waiting_result.assert_called_once() -def test_async_create_training_job_should_throw_exception_if_job_id_none(): +@pytest.mark.db_test +def test_async_create_training_job_should_throw_exception_if_job_id_none( + create_task_instance_of_operator, session +): + """Test that the operator throws an exception if the job_id is None.""" + with pytest.warns(AirflowProviderDeprecationWarning): - op = MLEngineStartTrainingJobOperator( + ti = create_task_instance_of_operator( + operator_class=MLEngineStartTrainingJobOperator, task_id=TEST_TASK_ID, project_id=TEST_GCP_PROJECT_ID, region=TEST_REGION, @@ -1432,16 +1446,23 @@ def test_async_create_training_job_should_throw_exception_if_job_id_none(): training_args=TEST_TRAINING_ARGS, labels=TEST_LABELS, deferrable=True, + dag_id="test_async_create_training_job", ) + with pytest.raises( AirflowException, match=r"An unique job id is required for Google MLEngine training job." ): - op.execute(create_context(op)) + ti.task.execute({"ti": ti}) -def test_async_create_training_job_should_throw_exception_if_project_id_none(): +@pytest.mark.db_test +def test_async_create_training_job_should_throw_exception_if_project_id_none( + create_task_instance_of_operator, session +): + """Test that the operator throws an exception if the project_id is None.""" with pytest.warns(AirflowProviderDeprecationWarning): - op = MLEngineStartTrainingJobOperator( + ti = create_task_instance_of_operator( + operator_class=MLEngineStartTrainingJobOperator, task_id=TEST_TASK_ID, project_id=None, region=TEST_REGION, @@ -1454,14 +1475,21 @@ def test_async_create_training_job_should_throw_exception_if_project_id_none(): training_args=TEST_TRAINING_ARGS, labels=TEST_LABELS, deferrable=True, + dag_id="test_async_create_training_job_exception", ) + with pytest.raises(AirflowException, match=r"Google Cloud project id is required."): - op.execute(create_context(op)) + ti.task.execute({"ti": ti}) -def test_async_create_training_job_should_throw_exception_if_custom_none(): +@pytest.mark.db_test +def test_async_create_training_job_should_throw_exception_if_custom_none( + create_task_instance_of_operator, session +): + """Test that the operator throws an exception if master_type is None while master_config is provided.""" with pytest.warns(AirflowProviderDeprecationWarning): - op = MLEngineStartTrainingJobOperator( + ti = create_task_instance_of_operator( + operator_class=MLEngineStartTrainingJobOperator, task_id=TEST_TASK_ID, project_id=TEST_PROJECT_ID, region=TEST_REGION, @@ -1476,14 +1504,22 @@ def test_async_create_training_job_should_throw_exception_if_custom_none(): master_config={"config": "config"}, master_type=None, deferrable=True, + dag_id="test_async_create_training_job_custom", ) + with pytest.raises(AirflowException, match=r"master_type must be set when master_config is provided"): - op.execute(create_context(op)) + ti.task.execute({"ti": ti}) + +@pytest.mark.db_test +def test_async_create_training_job_should_throw_exception_if_package_none( + create_task_instance_of_operator, session +): + """Test that the operator throws an exception if both package_uris and training_python_module are None.""" -def test_async_create_training_job_should_throw_exception_if_package_none(): with pytest.warns(AirflowProviderDeprecationWarning): - op = MLEngineStartTrainingJobOperator( + ti = create_task_instance_of_operator( + operator_class=MLEngineStartTrainingJobOperator, task_id=TEST_TASK_ID, project_id=TEST_PROJECT_ID, region=TEST_REGION, @@ -1496,18 +1532,26 @@ def test_async_create_training_job_should_throw_exception_if_package_none(): training_args=TEST_TRAINING_ARGS, labels=TEST_LABELS, deferrable=True, + dag_id="test_async_create_training_job_package", ) + with pytest.raises( AirflowException, match=r"Either a Python package with a Python module or a custom " r"Docker image should be provided.", ): - op.execute(create_context(op)) + ti.task.execute({"ti": ti}) + +@pytest.mark.db_test +def test_async_create_training_job_should_throw_exception_if_uris_none( + create_task_instance_of_operator, session +): + """Test that the operator throws an exception if package_uris is None.""" -def test_async_create_training_job_should_throw_exception_if_uris_none(): with pytest.warns(AirflowProviderDeprecationWarning): - op = MLEngineStartTrainingJobOperator( + ti = create_task_instance_of_operator( + operator_class=MLEngineStartTrainingJobOperator, task_id=TEST_TASK_ID, project_id=TEST_PROJECT_ID, region=TEST_REGION, @@ -1522,10 +1566,12 @@ def test_async_create_training_job_should_throw_exception_if_uris_none(): master_config={"config": "config"}, master_type="type", deferrable=True, + dag_id="test_async_create_training_job_with_uri", ) + with pytest.raises( AirflowException, match=r"Either a Python package with a Python module or a custom " r"Docker image should be provided.", ): - op.execute(create_context(op)) + ti.task.execute({"ti": ti}) diff --git a/providers/tests/google/cloud/operators/test_vertex_ai.py b/providers/tests/google/cloud/operators/test_vertex_ai.py index f79542006e26..c9e658959aa6 100644 --- a/providers/tests/google/cloud/operators/test_vertex_ai.py +++ b/providers/tests/google/cloud/operators/test_vertex_ai.py @@ -94,7 +94,6 @@ CustomTrainingJobTrigger, RunPipelineJobTrigger, ) -from airflow.utils import timezone VERTEX_AI_PATH = "airflow.providers.google.cloud.operators.vertex_ai.{}" VERTEX_AI_LINKS_PATH = "airflow.providers.google.cloud.links.vertex_ai.{}" @@ -1228,7 +1227,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() @@ -2047,7 +2045,6 @@ def test_templating(self, create_task_instance_of_operator, session): # Other parameters dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) session.add(ti) session.commit() diff --git a/providers/tests/google/cloud/sensors/test_cloud_composer.py b/providers/tests/google/cloud/sensors/test_cloud_composer.py index 1e21898ae6c2..b4c1add409d5 100644 --- a/providers/tests/google/cloud/sensors/test_cloud_composer.py +++ b/providers/tests/google/cloud/sensors/test_cloud_composer.py @@ -46,7 +46,7 @@ "dag_id": "test_dag_id", "run_id": "scheduled__2024-05-22T11:10:00+00:00", "state": state, - "execution_date": "2024-05-22T11:10:00+00:00", + "logical_date": "2024-05-22T11:10:00+00:00", "start_date": "2024-05-22T11:20:01.531988+00:00", "end_date": "2024-05-22T11:20:11.997479+00:00", } diff --git a/providers/tests/google/cloud/transfers/test_gcs_to_bigquery.py b/providers/tests/google/cloud/transfers/test_gcs_to_bigquery.py index 1097faa7491b..0ba2e07bb05e 100644 --- a/providers/tests/google/cloud/transfers/test_gcs_to_bigquery.py +++ b/providers/tests/google/cloud/transfers/test_gcs_to_bigquery.py @@ -17,6 +17,7 @@ # under the License. from __future__ import annotations +import functools import json from unittest import mock from unittest.mock import MagicMock, call @@ -24,11 +25,10 @@ import pytest from google.cloud.bigquery import DEFAULT_RETRY, Table from google.cloud.exceptions import Conflict +from sqlalchemy import select -from airflow.exceptions import AirflowException, TaskDeferred -from airflow.models import DAG -from airflow.models.dagrun import DagRun -from airflow.models.taskinstance import TaskInstance +from airflow.exceptions import AirflowException +from airflow.models.trigger import Trigger from airflow.providers.common.compat.openlineage.facet import ( ColumnLineageDatasetFacet, Dataset, @@ -42,9 +42,8 @@ SymlinksDatasetFacet, ) from airflow.providers.google.cloud.transfers.gcs_to_bigquery import GCSToBigQueryOperator -from airflow.providers.google.cloud.triggers.bigquery import BigQueryInsertJobTrigger +from airflow.utils.state import TaskInstanceState from airflow.utils.timezone import datetime -from airflow.utils.types import DagRunType TASK_ID = "test-gcs-to-bq-operator" TEST_EXPLICIT_DEST = "test-project.dataset.table" @@ -1574,9 +1573,27 @@ def test_get_openlineage_facets_on_complete_full_table_multiple_gcs_uris(self, h assert lineage.job_facets == {} +@pytest.fixture +def create_task_instance(create_task_instance_of_operator, session): + return functools.partial( + create_task_instance_of_operator, + session=session, + operator_class=GCSToBigQueryOperator, + dag_id="adhoc_airflow", + ) + + +@pytest.mark.db_test class TestAsyncGCSToBigQueryOperator: + def _set_execute_complete(self, session, ti, **next_kwargs): + ti.next_method = "execute_complete" + ti.next_kwargs = next_kwargs + session.flush() + @mock.patch(GCS_TO_BQ_PATH.format("BigQueryHook")) - def test_execute_without_external_table_async_should_execute_successfully(self, hook): + def test_execute_without_external_table_async_should_execute_successfully( + self, hook, create_task_instance, session + ): """ Asserts that a task is deferred and a BigQueryInsertJobTrigger will be fired when Operator is executed in deferrable. @@ -1586,7 +1603,7 @@ def test_execute_without_external_table_async_should_execute_successfully(self, hook.return_value.split_tablename.return_value = (PROJECT_ID, DATASET, TABLE) hook.return_value.get_job.return_value.result.return_value = ("1",) - operator = GCSToBigQueryOperator( + ti = create_task_instance( task_id=TASK_ID, bucket=TEST_BUCKET, source_objects=TEST_SOURCE_OBJECTS, @@ -1598,20 +1615,19 @@ def test_execute_without_external_table_async_should_execute_successfully(self, deferrable=True, project_id=JOB_PROJECT_ID, ) + ti.run(session=session) - with pytest.raises(TaskDeferred) as exc: - operator.execute(self.create_context(operator)) - - assert isinstance( - exc.value.trigger, BigQueryInsertJobTrigger - ), "Trigger is not a BigQueryInsertJobTrigger" + assert ti.state == TaskInstanceState.DEFERRED + trigger_cls = session.scalar(select(Trigger.classpath).where(Trigger.id == ti.trigger_id)) + assert trigger_cls == "airflow.providers.google.cloud.triggers.bigquery.BigQueryInsertJobTrigger" - def test_execute_without_external_table_async_should_throw_ex_when_event_status_error(self): + def test_execute_without_external_table_async_should_throw_ex_when_event_status_error( + self, create_task_instance, session + ): """ Tests that an AirflowException is raised in case of error event. """ - - operator = GCSToBigQueryOperator( + ti = create_task_instance( task_id=TASK_ID, bucket=TEST_BUCKET, source_objects=TEST_SOURCE_OBJECTS, @@ -1623,18 +1639,19 @@ def test_execute_without_external_table_async_should_throw_ex_when_event_status_ deferrable=True, project_id=JOB_PROJECT_ID, ) + self._set_execute_complete(session, ti, event={"status": "error", "message": "test failure message"}) + with pytest.raises(AirflowException): - operator.execute_complete( - context=None, event={"status": "error", "message": "test failure message"} - ) + ti.run() @pytest.mark.db_test - def test_execute_logging_without_external_table_async_should_execute_successfully(self): + def test_execute_logging_without_external_table_async_should_execute_successfully( + self, caplog, create_task_instance, session + ): """ Asserts that logging occurs as expected. """ - - operator = GCSToBigQueryOperator( + ti = create_task_instance( task_id=TASK_ID, bucket=TEST_BUCKET, source_objects=TEST_SOURCE_OBJECTS, @@ -1646,17 +1663,20 @@ def test_execute_logging_without_external_table_async_should_execute_successfull deferrable=True, project_id=JOB_PROJECT_ID, ) - with mock.patch.object(operator.log, "info") as mock_log_info: - operator.execute_complete( - context=self.create_context(operator), - event={"status": "success", "message": "Job completed", "job_id": job_id}, - ) + self._set_execute_complete( + session, ti, event={"status": "success", "message": "Job completed", "job_id": job_id} + ) + + with mock.patch.object(ti.task.log, "info") as mock_log_info: + ti.run() mock_log_info.assert_called_with( "%s completed with response %s ", "test-gcs-to-bq-operator", "Job completed" ) @mock.patch(GCS_TO_BQ_PATH.format("BigQueryHook")) - def test_execute_without_external_table_generate_job_id_async_should_execute_successfully(self, hook): + def test_execute_without_external_table_generate_job_id_async_should_execute_successfully( + self, hook, create_task_instance, session + ): hook.return_value.insert_job.side_effect = Conflict("any") hook.return_value.split_tablename.return_value = (PROJECT_ID, DATASET, TABLE) job = MagicMock( @@ -1667,7 +1687,7 @@ def test_execute_without_external_table_generate_job_id_async_should_execute_suc ) hook.return_value.get_job.return_value = job - operator = GCSToBigQueryOperator( + ti = create_task_instance( task_id=TASK_ID, bucket=TEST_BUCKET, source_objects=TEST_SOURCE_OBJECTS, @@ -1681,20 +1701,21 @@ def test_execute_without_external_table_generate_job_id_async_should_execute_suc project_id=JOB_PROJECT_ID, ) - with pytest.raises(TaskDeferred): - operator.execute(self.create_context(operator)) - + ti.run(session=session) + assert ti.state == TaskInstanceState.DEFERRED hook.return_value.generate_job_id.assert_called_once_with( job_id=None, dag_id="adhoc_airflow", task_id=TASK_ID, - logical_date=datetime(2022, 1, 1, 0, 0), + logical_date=datetime(2016, 1, 1, 0, 0), configuration={}, force_rerun=True, ) @mock.patch(GCS_TO_BQ_PATH.format("BigQueryHook")) - def test_execute_without_external_table_reattach_async_should_execute_successfully(self, hook): + def test_execute_without_external_table_reattach_async_should_execute_successfully( + self, hook, create_task_instance, session + ): hook.return_value.generate_job_id.return_value = REAL_JOB_ID hook.return_value.insert_job.side_effect = Conflict("any") @@ -1707,7 +1728,7 @@ def test_execute_without_external_table_reattach_async_should_execute_successful ) hook.return_value.get_job.return_value = job - operator = GCSToBigQueryOperator( + ti = create_task_instance( task_id=TASK_ID, bucket=TEST_BUCKET, source_objects=TEST_SOURCE_OBJECTS, @@ -1722,9 +1743,8 @@ def test_execute_without_external_table_reattach_async_should_execute_successful project_id=JOB_PROJECT_ID, ) - with pytest.raises(TaskDeferred): - operator.execute(self.create_context(operator)) - + ti.run(session=session) + assert ti.state == TaskInstanceState.DEFERRED hook.return_value.get_job.assert_called_once_with( location=TEST_DATASET_LOCATION, job_id=REAL_JOB_ID, @@ -1732,7 +1752,9 @@ def test_execute_without_external_table_reattach_async_should_execute_successful ) @mock.patch(GCS_TO_BQ_PATH.format("BigQueryHook")) - def test_execute_without_external_table_force_rerun_async_should_execute_successfully(self, hook): + def test_execute_without_external_table_force_rerun_async_should_execute_successfully( + self, hook, create_task_instance + ): hook.return_value.generate_job_id.return_value = f"{job_id}_{hash_}" hook.return_value.split_tablename.return_value = (PROJECT_ID, DATASET, TABLE) @@ -1745,7 +1767,7 @@ def test_execute_without_external_table_force_rerun_async_should_execute_success ) hook.return_value.get_job.return_value = job - operator = GCSToBigQueryOperator( + ti = create_task_instance( task_id=TASK_ID, bucket=TEST_BUCKET, source_objects=TEST_SOURCE_OBJECTS, @@ -1761,7 +1783,7 @@ def test_execute_without_external_table_force_rerun_async_should_execute_success ) with pytest.raises(AirflowException) as exc: - operator.execute(self.create_context(operator)) + ti.run() expected_exception_msg = ( f"Job with id: {REAL_JOB_ID} already exists and is in {job.state} state. " @@ -1779,14 +1801,16 @@ def test_execute_without_external_table_force_rerun_async_should_execute_success @mock.patch(GCS_TO_BQ_PATH.format("GCSHook")) @mock.patch(GCS_TO_BQ_PATH.format("BigQueryHook")) - def test_schema_fields_without_external_table_async_should_execute_successfully(self, bq_hook, gcs_hook): + def test_schema_fields_without_external_table_async_should_execute_successfully( + self, bq_hook, gcs_hook, create_task_instance + ): bq_hook.return_value.insert_job.return_value = MagicMock(job_id=REAL_JOB_ID, error_result=False) bq_hook.return_value.generate_job_id.return_value = REAL_JOB_ID bq_hook.return_value.split_tablename.return_value = (PROJECT_ID, DATASET, TABLE) bq_hook.return_value.get_job.return_value.result.return_value = ("1",) gcs_hook.return_value.download.return_value = b"id,name\r\none,Anna" - operator = GCSToBigQueryOperator( + ti = create_task_instance( task_id=TASK_ID, bucket=TEST_BUCKET, source_objects=TEST_SOURCE_OBJECTS, @@ -1799,9 +1823,8 @@ def test_schema_fields_without_external_table_async_should_execute_successfully( deferrable=True, project_id=JOB_PROJECT_ID, ) - - with pytest.raises(TaskDeferred): - operator.execute(self.create_context(operator)) + ti.run() + assert ti.state == TaskInstanceState.DEFERRED calls = [ call( @@ -1840,7 +1863,7 @@ def test_schema_fields_without_external_table_async_should_execute_successfully( @mock.patch(GCS_TO_BQ_PATH.format("GCSHook")) @mock.patch(GCS_TO_BQ_PATH.format("BigQueryHook")) def test_schema_fields_int_without_external_table_async_should_execute_successfully( - self, bq_hook, gcs_hook + self, bq_hook, gcs_hook, create_task_instance ): bq_hook.return_value.insert_job.return_value = MagicMock(job_id=REAL_JOB_ID, error_result=False) bq_hook.return_value.generate_job_id.return_value = REAL_JOB_ID @@ -1848,7 +1871,7 @@ def test_schema_fields_int_without_external_table_async_should_execute_successfu bq_hook.return_value.get_job.return_value.result.return_value = ("1",) gcs_hook.return_value.download.return_value = b"id,name\r\n1,Anna" - operator = GCSToBigQueryOperator( + ti = create_task_instance( task_id=TASK_ID, bucket=TEST_BUCKET, source_objects=TEST_SOURCE_OBJECTS, @@ -1861,9 +1884,8 @@ def test_schema_fields_int_without_external_table_async_should_execute_successfu deferrable=True, project_id=JOB_PROJECT_ID, ) - - with pytest.raises(TaskDeferred): - operator.execute(self.create_context(operator)) + ti.run() + assert ti.state == TaskInstanceState.DEFERRED calls = [ call( @@ -1905,10 +1927,10 @@ def test_schema_fields_int_without_external_table_async_should_execute_successfu bq_hook.return_value.insert_job.assert_has_calls(calls) @mock.patch(GCS_TO_BQ_PATH.format("BigQueryHook")) - def test_execute_complete_reassigns_job_id(self, bq_hook): + def test_execute_complete_reassigns_job_id(self, bq_hook, create_task_instance, session): """Assert that we use job_id from event after deferral.""" - - operator = GCSToBigQueryOperator( + bq_hook.return_value.split_tablename.return_value = "", "", "" + ti = create_task_instance( task_id=TASK_ID, bucket=TEST_BUCKET, source_objects=TEST_SOURCE_OBJECTS, @@ -1916,36 +1938,17 @@ def test_execute_complete_reassigns_job_id(self, bq_hook): deferrable=True, job_id=None, ) - generated_job_id = "123456" - assert operator.job_id is None - - operator.execute_complete( - context=MagicMock(), - event={"status": "success", "message": "Job completed", "job_id": generated_job_id}, - ) - assert operator.job_id == generated_job_id - - def create_context(self, task): - dag = DAG(dag_id="dag", schedule=None) - logical_date = datetime(2022, 1, 1, 0, 0, 0) - dag_run = DagRun( - dag_id=dag.dag_id, - execution_date=logical_date, - run_id=DagRun.generate_run_id(DagRunType.MANUAL, logical_date), - ) - task_instance = TaskInstance(task=task) - task_instance.dag_run = dag_run - task_instance.dag_id = dag.dag_id - task_instance.xcom_push = mock.Mock() - return { - "dag": dag, - "run_id": dag_run.run_id, - "task": task, - "ti": task_instance, - "task_instance": task_instance, - "logical_date": logical_date, + generated_job_id = "123456" + ti.next_method = "execute_complete" + ti.next_kwargs = { + "event": {"status": "success", "message": "Job completed", "job_id": generated_job_id}, } + session.flush() + + assert ti.task.job_id is None + ti.run(session=session) + assert ti.task.job_id == generated_job_id @mock.patch(GCS_TO_BQ_PATH.format("BigQueryHook")) def test_force_delete_should_execute_successfully(self, hook): diff --git a/providers/tests/google/cloud/utils/airflow_util.py b/providers/tests/google/cloud/utils/airflow_util.py index b856081dd10c..891a00780b59 100644 --- a/providers/tests/google/cloud/utils/airflow_util.py +++ b/providers/tests/google/cloud/utils/airflow_util.py @@ -33,7 +33,7 @@ def get_dag_run(dag_id: str = "test_dag_id", run_id: str = "test_dag_id") -> DagRun: dag_run = DagRun( - dag_id=dag_id, run_type="manual", execution_date=timezone.datetime(2022, 1, 1), run_id=run_id + dag_id=dag_id, run_type="manual", logical_date=timezone.datetime(2022, 1, 1), run_id=run_id ) return dag_run @@ -53,11 +53,11 @@ def create_context(task, dag=None): if dag is None: dag = DAG(dag_id="dag") tzinfo = pendulum.timezone("UTC") - execution_date = timezone.datetime(2022, 1, 1, 1, 0, 0, tzinfo=tzinfo) + logical_date = timezone.datetime(2022, 1, 1, 1, 0, 0, tzinfo=tzinfo) dag_run = DagRun( dag_id=dag.dag_id, - execution_date=execution_date, - run_id=DagRun.generate_run_id(DagRunType.MANUAL, execution_date), + logical_date=logical_date, + run_id=DagRun.generate_run_id(DagRunType.MANUAL, logical_date), ) task_instance = TaskInstance(task=task) @@ -65,13 +65,12 @@ def create_context(task, dag=None): task_instance.xcom_push = mock.Mock() return { "dag": dag, - "ts": execution_date.isoformat(), + "ts": logical_date.isoformat(), "task": task, "ti": task_instance, "task_instance": task_instance, "run_id": dag_run.run_id, "dag_run": dag_run, - "execution_date": execution_date, - "data_interval_end": execution_date, - "logical_date": execution_date, + "logical_date": logical_date, + "data_interval_end": logical_date, } diff --git a/providers/tests/http/sensors/test_http.py b/providers/tests/http/sensors/test_http.py index e19a4c929aed..78a11e15bb7c 100644 --- a/providers/tests/http/sensors/test_http.py +++ b/providers/tests/http/sensors/test_http.py @@ -138,7 +138,6 @@ def resp_check(_, logical_date): task_instance = create_task_instance_of_operator( HttpSensor, dag_id="http_sensor_poke_exception", - execution_date=DEFAULT_DATE, task_id="http_sensor_poke_exception", http_conn_id="http_default", endpoint="", diff --git a/providers/tests/microsoft/azure/log/test_wasb_task_handler.py b/providers/tests/microsoft/azure/log/test_wasb_task_handler.py index 5f5493e9ff06..425e4bec68a6 100644 --- a/providers/tests/microsoft/azure/log/test_wasb_task_handler.py +++ b/providers/tests/microsoft/azure/log/test_wasb_task_handler.py @@ -46,7 +46,7 @@ def ti(self, create_task_instance, create_log_template): ti = create_task_instance( dag_id="dag_for_testing_wasb_task_handler", task_id="task_for_testing_wasb_log_handler", - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, start_date=DEFAULT_DATE, dagrun_state=TaskInstanceState.RUNNING, state=TaskInstanceState.RUNNING, diff --git a/providers/tests/microsoft/azure/operators/test_adx.py b/providers/tests/microsoft/azure/operators/test_adx.py index fc88e64d1028..ec63bba5b2a3 100644 --- a/providers/tests/microsoft/azure/operators/test_adx.py +++ b/providers/tests/microsoft/azure/operators/test_adx.py @@ -25,7 +25,6 @@ from airflow.models import DAG from airflow.providers.microsoft.azure.hooks.adx import AzureDataExplorerHook from airflow.providers.microsoft.azure.operators.adx import AzureDataExplorerQueryOperator -from airflow.utils import timezone from airflow.utils.timezone import datetime TEST_DAG_ID = "unit_tests" @@ -93,7 +92,6 @@ def test_azure_data_explorer_query_operator_xcom_push_and_pull( ti = create_task_instance_of_operator( AzureDataExplorerQueryOperator, dag_id="test_azure_data_explorer_query_operator_xcom_push_and_pull", - execution_date=timezone.utcnow(), **MOCK_DATA, ) ti.run() diff --git a/providers/tests/microsoft/azure/operators/test_data_factory.py b/providers/tests/microsoft/azure/operators/test_data_factory.py index ee89941166d6..429aa518e9be 100644 --- a/providers/tests/microsoft/azure/operators/test_data_factory.py +++ b/providers/tests/microsoft/azure/operators/test_data_factory.py @@ -16,6 +16,7 @@ # under the License. from __future__ import annotations +import functools from typing import TYPE_CHECKING from unittest import mock from unittest.mock import MagicMock, patch @@ -37,6 +38,8 @@ from airflow.utils import timezone from airflow.utils.types import DagRunType +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + if TYPE_CHECKING: from airflow.models.baseoperator import BaseOperator @@ -236,7 +239,6 @@ def test_run_pipeline_operator_link(self, resource_group, factory, create_task_i ti = create_task_instance_of_operator( AzureDataFactoryRunPipelineOperator, dag_id="test_adf_run_pipeline_op_link", - execution_date=DEFAULT_DATE, task_id=TASK_ID, azure_data_factory_conn_id=AZURE_DATA_FACTORY_CONN_ID, pipeline_name=PIPELINE_NAME, @@ -267,20 +269,42 @@ def test_run_pipeline_operator_link(self, resource_group, factory, create_task_i ) +@pytest.fixture +def create_task_instance(create_task_instance_of_operator, session): + def _create_task_instance(operator_class, **kwargs): + return functools.partial( + create_task_instance_of_operator, + session=session, + operator_class=operator_class, + dag_id="adhoc_airflow", + )(**kwargs) + + return _create_task_instance + + class TestAzureDataFactoryRunPipelineOperatorWithDeferrable: - OPERATOR = AzureDataFactoryRunPipelineOperator( - task_id="run_pipeline", - pipeline_name="pipeline", - resource_group_name="resource-group-name", - factory_name="factory-name", - parameters={"myParam": "value"}, - deferrable=True, - ) + @pytest.fixture(autouse=True) + def setup_operator(self, create_task_instance): + """Fixture to set up the operator using create_task_instance.""" + self.ti = create_task_instance( + operator_class=AzureDataFactoryRunPipelineOperator, + task_id="run_pipeline", + pipeline_name="pipeline", + resource_group_name="resource-group-name", + factory_name="factory-name", + parameters={"myParam": "value"}, + deferrable=True, + ) def get_dag_run(self, dag_id: str = "test_dag_id", run_id: str = "test_dag_id") -> DagRun: - dag_run = DagRun( - dag_id=dag_id, run_type="manual", execution_date=timezone.datetime(2022, 1, 1), run_id=run_id - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=dag_id, run_type="manual", logical_date=timezone.datetime(2022, 1, 1), run_id=run_id + ) + else: + dag_run = DagRun( # type: ignore[call-arg] + dag_id=dag_id, run_type="manual", execution_date=timezone.datetime(2022, 1, 1), run_id=run_id + ) return dag_run def get_task_instance(self, task: BaseOperator) -> TaskInstance: @@ -298,29 +322,37 @@ def create_context(self, task, dag=None): if dag is None: dag = DAG(dag_id="dag", schedule=None) tzinfo = pendulum.timezone("UTC") - execution_date = timezone.datetime(2022, 1, 1, 1, 0, 0, tzinfo=tzinfo) - dag_run = DagRun( - dag_id=dag.dag_id, - execution_date=execution_date, - run_id=DagRun.generate_run_id(DagRunType.MANUAL, execution_date), - ) + logical_date = timezone.datetime(2022, 1, 1, 1, 0, 0, tzinfo=tzinfo) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=dag.dag_id, + logical_date=logical_date, + run_id=DagRun.generate_run_id(DagRunType.MANUAL, logical_date), + ) + else: + dag_run = DagRun( + dag_id=dag.dag_id, + execution_date=logical_date, + run_id=DagRun.generate_run_id(DagRunType.MANUAL, logical_date), + ) task_instance = TaskInstance(task=task) task_instance.dag_run = dag_run task_instance.xcom_push = mock.Mock() + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" return { "dag": dag, - "ts": execution_date.isoformat(), + "ts": logical_date.isoformat(), "task": task, "ti": task_instance, "task_instance": task_instance, "run_id": dag_run.run_id, "dag_run": dag_run, - "execution_date": execution_date, - "data_interval_end": execution_date, - "logical_date": execution_date, + "data_interval_end": logical_date, + date_key: logical_date, } + @pytest.mark.db_test @mock.patch( "airflow.providers.microsoft.azure.operators.data_factory.AzureDataFactoryRunPipelineOperator" ".defer" @@ -339,9 +371,10 @@ class CreateRunResponse: CreateRunResponse.run_id = AZ_PIPELINE_RUN_ID mock_run_pipeline.return_value = CreateRunResponse - self.OPERATOR.execute(context=self.create_context(self.OPERATOR)) + self.ti.task.execute(context=self.create_context(self.ti.task)) assert not mock_defer.called + @pytest.mark.db_test @pytest.mark.parametrize("status", sorted(AzureDataFactoryPipelineRunStatus.FAILURE_STATES)) @mock.patch( "airflow.providers.microsoft.azure.operators.data_factory.AzureDataFactoryRunPipelineOperator" @@ -363,9 +396,10 @@ class CreateRunResponse: mock_run_pipeline.return_value = CreateRunResponse with pytest.raises(AzureDataFactoryPipelineRunException): - self.OPERATOR.execute(context=self.create_context(self.OPERATOR)) + self.ti.task.execute(context=self.create_context(self.ti.task)) assert not mock_defer.called + @pytest.mark.db_test @pytest.mark.parametrize("status", sorted(AzureDataFactoryPipelineRunStatus.INTERMEDIATE_STATES)) @mock.patch( "airflow.providers.microsoft.azure.hooks.data_factory.AzureDataFactoryHook.get_pipeline_run_status", @@ -381,27 +415,29 @@ class CreateRunResponse: mock_run_pipeline.return_value = CreateRunResponse with pytest.raises(TaskDeferred) as exc: - self.OPERATOR.execute(context=self.create_context(self.OPERATOR)) + self.ti.task.execute(context=self.create_context(self.ti.task)) assert isinstance( exc.value.trigger, AzureDataFactoryTrigger ), "Trigger is not a AzureDataFactoryTrigger" + @pytest.mark.db_test def test_azure_data_factory_run_pipeline_operator_async_execute_complete_success(self): """Assert that execute_complete log success message""" - with mock.patch.object(self.OPERATOR.log, "info") as mock_log_info: - self.OPERATOR.execute_complete( + with mock.patch.object(self.ti.task.log, "info") as mock_log_info: + self.ti.task.execute_complete( context={}, event={"status": "success", "message": "success", "run_id": AZ_PIPELINE_RUN_ID}, ) mock_log_info.assert_called_with("success") + @pytest.mark.db_test def test_azure_data_factory_run_pipeline_operator_async_execute_complete_fail(self): """Assert that execute_complete raise exception on error""" with pytest.raises(AirflowException): - self.OPERATOR.execute_complete( + self.ti.task.execute_complete( context={}, event={"status": "error", "message": "error", "run_id": AZ_PIPELINE_RUN_ID}, ) diff --git a/providers/tests/microsoft/azure/operators/test_powerbi.py b/providers/tests/microsoft/azure/operators/test_powerbi.py index a9171cff39a6..dbb53dbf30bf 100644 --- a/providers/tests/microsoft/azure/operators/test_powerbi.py +++ b/providers/tests/microsoft/azure/operators/test_powerbi.py @@ -131,7 +131,6 @@ def test_powerbi_link(self, create_task_instance_of_operator): ti = create_task_instance_of_operator( PowerBIDatasetRefreshOperator, dag_id="test_powerbi_refresh_op_link", - execution_date=DEFAULT_DATE, task_id=TASK_ID, conn_id=DEFAULT_CONNECTION_CLIENT_SECRET, group_id=GROUP_ID, diff --git a/providers/tests/microsoft/azure/operators/test_synapse.py b/providers/tests/microsoft/azure/operators/test_synapse.py index becfde5f71f2..738e7b3675bb 100644 --- a/providers/tests/microsoft/azure/operators/test_synapse.py +++ b/providers/tests/microsoft/azure/operators/test_synapse.py @@ -280,7 +280,6 @@ def test_run_pipeline_operator_link(self, create_task_instance_of_operator): ti = create_task_instance_of_operator( AzureSynapseRunPipelineOperator, dag_id="test_synapse_run_pipeline_op_link", - execution_date=DEFAULT_DATE, task_id=AZURE_SYNAPSE_PIPELINE_TASK_ID, azure_synapse_conn_id=AZURE_SYNAPSE_CONN_ID, pipeline_name=PIPELINE_NAME, diff --git a/providers/tests/microsoft/azure/sensors/test_wasb.py b/providers/tests/microsoft/azure/sensors/test_wasb.py index 46a386cb44d6..e05f9454c954 100644 --- a/providers/tests/microsoft/azure/sensors/test_wasb.py +++ b/providers/tests/microsoft/azure/sensors/test_wasb.py @@ -79,9 +79,20 @@ def test_poke(self, mock_hook): class TestWasbBlobAsyncSensor: def get_dag_run(self, dag_id: str = "test_dag_id", run_id: str = "test_dag_id") -> DagRun: - dag_run = DagRun( - dag_id=dag_id, run_type="manual", execution_date=timezone.datetime(2022, 1, 1), run_id=run_id - ) + if hasattr(DagRun, "execution_date"): # for 2.x + dag_run = DagRun( # type: ignore[call-arg] + dag_id=dag_id, + run_type="manual", + execution_date=timezone.datetime(2022, 1, 1), + run_id=run_id, + ) + else: + dag_run = DagRun( + dag_id=dag_id, + run_type="manual", + logical_date=timezone.datetime(2022, 1, 1), + run_id=run_id, + ) return dag_run def get_task_instance(self, task: BaseOperator) -> TaskInstance: @@ -97,27 +108,34 @@ def create_context(self, task, dag=None): if dag is None: dag = DAG(dag_id="dag", schedule=None) tzinfo = pendulum.timezone("UTC") - execution_date = timezone.datetime(2022, 1, 1, 1, 0, 0, tzinfo=tzinfo) - dag_run = DagRun( - dag_id=dag.dag_id, - execution_date=execution_date, - run_id=DagRun.generate_run_id(DagRunType.MANUAL, execution_date), - ) + logical_date = timezone.datetime(2022, 1, 1, 1, 0, 0, tzinfo=tzinfo) + if hasattr(DagRun, "execution_date"): # for 2.x + dag_run = DagRun( + dag_id=dag.dag_id, + execution_date=logical_date, + run_id=DagRun.generate_run_id(DagRunType.MANUAL, logical_date), + ) + else: + dag_run = DagRun( + dag_id=dag.dag_id, + logical_date=logical_date, + run_id=DagRun.generate_run_id(DagRunType.MANUAL, logical_date), + ) task_instance = TaskInstance(task=task) task_instance.dag_run = dag_run task_instance.xcom_push = mock.Mock() + date_key = "execution_date" if hasattr(DagRun, "execution_date") else "logical_date" return { "dag": dag, - "ts": execution_date.isoformat(), + "ts": logical_date.isoformat(), "task": task, "ti": task_instance, "task_instance": task_instance, "run_id": dag_run.run_id, "dag_run": dag_run, - "execution_date": execution_date, - "data_interval_end": execution_date, - "logical_date": execution_date, + "data_interval_end": logical_date, + date_key: logical_date, } SENSOR = WasbBlobSensor( @@ -204,9 +222,20 @@ def test_poke(self, mock_hook): class TestWasbPrefixAsyncSensor: def get_dag_run(self, dag_id: str = "test_dag_id", run_id: str = "test_dag_id") -> DagRun: - dag_run = DagRun( - dag_id=dag_id, run_type="manual", execution_date=timezone.datetime(2022, 1, 1), run_id=run_id - ) + if hasattr(DagRun, "execution_date"): # for 2.x + dag_run = DagRun( # type: ignore[call-arg] + dag_id=dag_id, + run_type="manual", + execution_date=timezone.datetime(2022, 1, 1), + run_id=run_id, + ) + else: + dag_run = DagRun( + dag_id=dag_id, + run_type="manual", + logical_date=timezone.datetime(2022, 1, 1), + run_id=run_id, + ) return dag_run def get_task_instance(self, task: BaseOperator) -> TaskInstance: @@ -222,27 +251,34 @@ def create_context(self, task, dag=None): if dag is None: dag = DAG(dag_id="dag", schedule=None) tzinfo = pendulum.timezone("UTC") - execution_date = timezone.datetime(2022, 1, 1, 1, 0, 0, tzinfo=tzinfo) - dag_run = DagRun( - dag_id=dag.dag_id, - execution_date=execution_date, - run_id=DagRun.generate_run_id(DagRunType.MANUAL, execution_date), - ) + logical_date = timezone.datetime(2022, 1, 1, 1, 0, 0, tzinfo=tzinfo) + if hasattr(DagRun, "execution_date"): # for 2.x + dag_run = DagRun( + dag_id=dag.dag_id, + execution_date=logical_date, + run_id=DagRun.generate_run_id(DagRunType.MANUAL, logical_date), + ) + else: + dag_run = DagRun( + dag_id=dag.dag_id, + logical_date=logical_date, + run_id=DagRun.generate_run_id(DagRunType.MANUAL, logical_date), + ) task_instance = TaskInstance(task=task) task_instance.dag_run = dag_run task_instance.xcom_push = mock.Mock() + date_key = "execution_date" if hasattr(DagRun, "execution_date") else "logical_date" return { "dag": dag, - "ts": execution_date.isoformat(), + "ts": logical_date.isoformat(), "task": task, "ti": task_instance, "task_instance": task_instance, "run_id": dag_run.run_id, "dag_run": dag_run, - "execution_date": execution_date, - "data_interval_end": execution_date, - "logical_date": execution_date, + "data_interval_end": logical_date, + date_key: logical_date, } SENSOR = WasbPrefixSensor( diff --git a/providers/tests/openlineage/plugins/test_adapter.py b/providers/tests/openlineage/plugins/test_adapter.py index 73145f9e4b1c..576f6df830c3 100644 --- a/providers/tests/openlineage/plugins/test_adapter.py +++ b/providers/tests/openlineage/plugins/test_adapter.py @@ -52,7 +52,7 @@ from airflow.providers.openlineage.utils.utils import get_airflow_job_facet from airflow.utils.task_group import TaskGroup -from tests_common.test_utils.compat import BashOperator +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS, BashOperator from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test @@ -542,13 +542,22 @@ def test_emit_dag_started_event(mock_stats_incr, mock_stats_timer, generate_stat task_1 = BashOperator(task_id="task_1", bash_command="exit 0;", task_group=tg) # noqa: F841 task_2 = EmptyOperator(task_id="task_2.test.dot", task_group=tg2) # noqa: F841 - dag_run = DagRun( - dag_id=dag_id, - run_id=run_id, - start_date=event_time, - execution_date=event_time, - data_interval=(event_time, event_time), - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=dag_id, + run_id=run_id, + start_date=event_time, + logical_date=event_time, + data_interval=(event_time, event_time), + ) + else: + dag_run = DagRun( + dag_id=dag_id, + run_id=run_id, + start_date=event_time, + execution_date=event_time, + data_interval=(event_time, event_time), + ) dag_run.dag = dag generate_static_uuid.return_value = random_uuid @@ -581,7 +590,6 @@ def test_emit_dag_started_event(mock_stats_incr, mock_stats_timer, generate_stat "start_date": event_time.isoformat(), }, ) - adapter.dag_started( dag_id=dag_id, start_date=event_time, @@ -671,13 +679,21 @@ def test_emit_dag_complete_event( task_2 = EmptyOperator( task_id="task_2.test", ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=dag_id, + run_id=run_id, + start_date=event_time, + logical_date=event_time, + ) + else: + dag_run = DagRun( + dag_id=dag_id, + run_id=run_id, + start_date=event_time, + execution_date=event_time, + ) - dag_run = DagRun( - dag_id=dag_id, - run_id=run_id, - start_date=event_time, - execution_date=event_time, - ) dag_run._state = DagRunState.SUCCESS dag_run.end_date = event_time mocked_fetch_tis.return_value = [ @@ -753,12 +769,20 @@ def test_emit_dag_failed_event( task_1 = BashOperator(task_id="task_1", bash_command="exit 0;") task_2 = EmptyOperator(task_id="task_2.test") - dag_run = DagRun( - dag_id=dag_id, - run_id=run_id, - start_date=event_time, - execution_date=event_time, - ) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=dag_id, + run_id=run_id, + start_date=event_time, + logical_date=event_time, + ) + else: + dag_run = DagRun( + dag_id=dag_id, + run_id=run_id, + start_date=event_time, + execution_date=event_time, + ) dag_run._state = DagRunState.FAILED dag_run.end_date = event_time mocked_fetch_tis.return_value = [ @@ -875,7 +899,7 @@ def test_build_task_instance_run_id_is_valid_uuid(): dag_id="dag_id", task_id="task_id", try_number=1, - execution_date=datetime.datetime.now(), + logical_date=datetime.datetime.now(), ) uuid_result = uuid.UUID(result) assert uuid_result @@ -887,13 +911,13 @@ def test_build_task_instance_run_id_same_input_gives_same_result(): dag_id="dag1", task_id="task1", try_number=1, - execution_date=datetime.datetime(2024, 1, 1, 1, 1, 1), + logical_date=datetime.datetime(2024, 1, 1, 1, 1, 1), ) result2 = OpenLineageAdapter.build_task_instance_run_id( dag_id="dag1", task_id="task1", try_number=1, - execution_date=datetime.datetime(2024, 1, 1, 1, 1, 1), + logical_date=datetime.datetime(2024, 1, 1, 1, 1, 1), ) assert result1 == result2 @@ -903,13 +927,13 @@ def test_build_task_instance_run_id_different_inputs_gives_different_results(): dag_id="dag1", task_id="task1", try_number=1, - execution_date=datetime.datetime.now(), + logical_date=datetime.datetime.now(), ) result2 = OpenLineageAdapter.build_task_instance_run_id( dag_id="dag2", task_id="task2", try_number=2, - execution_date=datetime.datetime.now(), + logical_date=datetime.datetime.now(), ) assert result1 != result2 diff --git a/providers/tests/openlineage/plugins/test_listener.py b/providers/tests/openlineage/plugins/test_listener.py index 79149324e551..d53c23c8b6de 100644 --- a/providers/tests/openlineage/plugins/test_listener.py +++ b/providers/tests/openlineage/plugins/test_listener.py @@ -191,8 +191,8 @@ def _create_listener_and_task_instance() -> tuple[OpenLineageListener, TaskInsta def mock_dag_id(dag_id, logical_date): return f"{logical_date}.{dag_id}" - def mock_task_id(dag_id, task_id, try_number, execution_date): - return f"{execution_date}.{dag_id}.{task_id}.{try_number}" + def mock_task_id(dag_id, task_id, try_number, logical_date): + return f"{logical_date}.{dag_id}.{task_id}.{try_number}" listener = OpenLineageListener() listener.log = mock.Mock() @@ -215,7 +215,10 @@ def mock_task_id(dag_id, task_id, try_number, execution_date): task_instance.dag_run.run_id = "dag_run_run_id" task_instance.dag_run.data_interval_start = None task_instance.dag_run.data_interval_end = None - task_instance.dag_run.execution_date = "logical_date" + if AIRFLOW_V_3_0_PLUS: + task_instance.dag_run.logical_date = "2020-01-01T01:01:01" + else: + task_instance.dag_run.execution_date = "2020-01-01T01:01:01" task_instance.task = mock.Mock() task_instance.task.task_id = "task_id" task_instance.task.dag = mock.Mock() @@ -228,7 +231,7 @@ def mock_task_id(dag_id, task_id, try_number, execution_date): task_instance.state = State.RUNNING task_instance.start_date = dt.datetime(2023, 1, 1, 13, 1, 1) task_instance.end_date = dt.datetime(2023, 1, 3, 13, 1, 1) - task_instance.execution_date = "2020-01-01T01:01:01" + task_instance.logical_date = "2020-01-01T01:01:01" task_instance.next_method = None # Ensure this is None to reach start_task return listener, task_instance @@ -312,10 +315,11 @@ def test_adapter_fail_task_is_called_with_proper_arguments( def mock_dag_id(dag_id, logical_date): return f"{logical_date}.{dag_id}" - def mock_task_id(dag_id, task_id, try_number, execution_date): - return f"{execution_date}.{dag_id}.{task_id}.{try_number}" + def mock_task_id(dag_id, task_id, try_number, logical_date): + return f"{logical_date}.{dag_id}.{task_id}.{try_number}" listener, task_instance = _create_listener_and_task_instance() + task_instance.logical_date = "2020-01-01T01:01:01" mock_get_job_name.return_value = "job_name" mocked_adapter.build_dag_run_id.side_effect = mock_dag_id mocked_adapter.build_task_instance_run_id.side_effect = mock_task_id @@ -348,7 +352,6 @@ def mock_task_id(dag_id, task_id, try_number, execution_date): @mock.patch("airflow.providers.openlineage.conf.debug_mode", return_value=True) @mock.patch("airflow.providers.openlineage.plugins.listener.is_operator_disabled") -@mock.patch("airflow.providers.openlineage.plugins.listener.OpenLineageAdapter") @mock.patch("airflow.providers.openlineage.plugins.listener.get_airflow_run_facet") @mock.patch("airflow.providers.openlineage.plugins.listener.get_user_provided_run_facets") @mock.patch("airflow.providers.openlineage.plugins.listener.get_job_name") @@ -357,7 +360,6 @@ def test_adapter_complete_task_is_called_with_proper_arguments( mock_get_job_name, mock_get_user_provided_run_facets, mock_get_airflow_run_facet, - mocked_adapter, mock_disabled, mock_debug_mode, ): @@ -373,13 +375,13 @@ def test_adapter_complete_task_is_called_with_proper_arguments( def mock_dag_id(dag_id, logical_date): return f"{logical_date}.{dag_id}" - def mock_task_id(dag_id, task_id, try_number, execution_date): - return f"{execution_date}.{dag_id}.{task_id}.{try_number}" + def mock_task_id(dag_id, task_id, try_number, logical_date): + return f"{logical_date}.{dag_id}.{task_id}.{try_number}" listener, task_instance = _create_listener_and_task_instance() mock_get_job_name.return_value = "job_name" - mocked_adapter.build_dag_run_id.side_effect = mock_dag_id - mocked_adapter.build_task_instance_run_id.side_effect = mock_task_id + listener.adapter.build_dag_run_id.side_effect = mock_dag_id + listener.adapter.build_task_instance_run_id.side_effect = mock_task_id mock_get_user_provided_run_facets.return_value = {"custom_user_facet": 2} mock_get_airflow_run_facet.return_value = {"airflow": {"task": "..."}} mock_disabled.return_value = False @@ -417,14 +419,13 @@ def test_on_task_instance_running_correctly_calls_openlineage_adapter_run_id_met listener.adapter.build_task_instance_run_id.assert_called_once_with( dag_id="dag_id", task_id="task_id", - execution_date="2020-01-01T01:01:01", + logical_date="2020-01-01T01:01:01", try_number=1, ) -@mock.patch("airflow.providers.openlineage.plugins.listener.OpenLineageAdapter") @mock.patch("airflow.providers.openlineage.plugins.listener.OpenLineageListener._execute", new=regular_call) -def test_on_task_instance_failed_correctly_calls_openlineage_adapter_run_id_method(mock_adapter): +def test_on_task_instance_failed_correctly_calls_openlineage_adapter_run_id_method(): """Tests the OpenLineageListener's response when a task instance is in the failed state. This test ensures that when an Airflow task instance transitions to the failed state, @@ -437,17 +438,16 @@ def test_on_task_instance_failed_correctly_calls_openlineage_adapter_run_id_meth listener.on_task_instance_failed( previous_state=None, task_instance=task_instance, session=None, **on_task_failed_kwargs ) - mock_adapter.build_task_instance_run_id.assert_called_once_with( + listener.adapter.build_task_instance_run_id.assert_called_once_with( dag_id="dag_id", task_id="task_id", - execution_date="2020-01-01T01:01:01", + logical_date="2020-01-01T01:01:01", try_number=1, ) -@mock.patch("airflow.providers.openlineage.plugins.listener.OpenLineageAdapter") @mock.patch("airflow.providers.openlineage.plugins.listener.OpenLineageListener._execute", new=regular_call) -def test_on_task_instance_success_correctly_calls_openlineage_adapter_run_id_method(mock_adapter): +def test_on_task_instance_success_correctly_calls_openlineage_adapter_run_id_method(): """Tests the OpenLineageListener's response when a task instance is in the success state. This test ensures that when an Airflow task instance transitions to the success state, @@ -456,10 +456,10 @@ def test_on_task_instance_success_correctly_calls_openlineage_adapter_run_id_met """ listener, task_instance = _create_listener_and_task_instance() listener.on_task_instance_success(None, task_instance, None) - mock_adapter.build_task_instance_run_id.assert_called_once_with( + listener.adapter.build_task_instance_run_id.assert_called_once_with( dag_id="dag_id", task_id="task_id", - execution_date="2020-01-01T01:01:01", + logical_date="2020-01-01T01:01:01", try_number=EXPECTED_TRY_NUMBER_1, ) @@ -547,11 +547,10 @@ def test_listener_on_task_instance_running_do_not_call_adapter_when_disabled_ope @mock.patch("airflow.providers.openlineage.plugins.listener.is_operator_disabled") -@mock.patch("airflow.providers.openlineage.plugins.listener.OpenLineageAdapter") @mock.patch("airflow.providers.openlineage.plugins.listener.get_user_provided_run_facets") @mock.patch("airflow.providers.openlineage.plugins.listener.get_job_name") def test_listener_on_task_instance_failed_do_not_call_adapter_when_disabled_operator( - mock_get_job_name, mock_get_user_provided_run_facets, mocked_adapter, mock_disabled + mock_get_job_name, mock_get_user_provided_run_facets, mock_disabled ): listener, task_instance = _create_listener_and_task_instance() mock_get_user_provided_run_facets.return_value = {"custom_facet": 2} @@ -563,18 +562,17 @@ def test_listener_on_task_instance_failed_do_not_call_adapter_when_disabled_oper previous_state=None, task_instance=task_instance, session=None, **on_task_failed_kwargs ) mock_disabled.assert_called_once_with(task_instance.task) - mocked_adapter.build_dag_run_id.assert_not_called() - mocked_adapter.build_task_instance_run_id.assert_not_called() + listener.adapter.build_dag_run_id.assert_not_called() + listener.adapter.build_task_instance_run_id.assert_not_called() listener.extractor_manager.extract_metadata.assert_not_called() listener.adapter.fail_task.assert_not_called() @mock.patch("airflow.providers.openlineage.plugins.listener.is_operator_disabled") -@mock.patch("airflow.providers.openlineage.plugins.listener.OpenLineageAdapter") @mock.patch("airflow.providers.openlineage.plugins.listener.get_user_provided_run_facets") @mock.patch("airflow.providers.openlineage.plugins.listener.get_job_name") def test_listener_on_task_instance_success_do_not_call_adapter_when_disabled_operator( - mock_get_job_name, mock_get_user_provided_run_facets, mocked_adapter, mock_disabled + mock_get_job_name, mock_get_user_provided_run_facets, mock_disabled ): listener, task_instance = _create_listener_and_task_instance() mock_get_user_provided_run_facets.return_value = {"custom_facet": 2} @@ -582,8 +580,8 @@ def test_listener_on_task_instance_success_do_not_call_adapter_when_disabled_ope listener.on_task_instance_success(None, task_instance, None) mock_disabled.assert_called_once_with(task_instance.task) - mocked_adapter.build_dag_run_id.assert_not_called() - mocked_adapter.build_task_instance_run_id.assert_not_called() + listener.adapter.build_dag_run_id.assert_not_called() + listener.adapter.build_task_instance_run_id.assert_not_called() listener.extractor_manager.extract_metadata.assert_not_called() listener.adapter.complete_task.assert_not_called() @@ -667,7 +665,7 @@ def set_result(*args, **kwargs): dag_id="", run_id="", end_date=event_time, - execution_date=callback_future, + logical_date=callback_future, dag_run_state=DagRunState.FAILED, task_ids=["task_id"], msg="", diff --git a/providers/tests/openlineage/plugins/test_macros.py b/providers/tests/openlineage/plugins/test_macros.py index c8ae74aca48c..28eb8c74cd91 100644 --- a/providers/tests/openlineage/plugins/test_macros.py +++ b/providers/tests/openlineage/plugins/test_macros.py @@ -19,6 +19,7 @@ from datetime import datetime, timezone from unittest import mock +from airflow import __version__ from airflow.providers.openlineage.conf import namespace from airflow.providers.openlineage.plugins.macros import ( lineage_job_name, @@ -29,6 +30,11 @@ _DAG_NAMESPACE = namespace() +if __version__.startswith("2."): + LOGICAL_DATE_KEY = "execution_date" +else: + LOGICAL_DATE_KEY = "logical_date" + def test_lineage_job_namespace(): assert lineage_job_namespace() == _DAG_NAMESPACE @@ -39,7 +45,7 @@ def test_lineage_job_name(): dag_id="dag_id", task_id="task_id", try_number=1, - execution_date=datetime(2020, 1, 1, 1, 1, 1, 0, tzinfo=timezone.utc), + **{LOGICAL_DATE_KEY: datetime(2020, 1, 1, 1, 1, 1, 0, tzinfo=timezone.utc)}, ) assert lineage_job_name(task_instance) == "dag_id.task_id" @@ -49,7 +55,7 @@ def test_lineage_run_id(): dag_id="dag_id", task_id="task_id", dag_run=mock.MagicMock(run_id="run_id"), - execution_date=datetime(2020, 1, 1, 1, 1, 1, 0, tzinfo=timezone.utc), + logical_date=datetime(2020, 1, 1, 1, 1, 1, 0, tzinfo=timezone.utc), try_number=1, ) @@ -69,7 +75,7 @@ def test_lineage_parent_id(mock_run_id): dag_id="dag_id", task_id="task_id", try_number=1, - execution_date=datetime(2020, 1, 1, 1, 1, 1, 0, tzinfo=timezone.utc), + **{LOGICAL_DATE_KEY: datetime(2020, 1, 1, 1, 1, 1, 0, tzinfo=timezone.utc)}, ) actual = lineage_parent_id(task_instance) expected = f"{_DAG_NAMESPACE}/dag_id.task_id/run_id" diff --git a/providers/tests/opensearch/log/test_os_task_handler.py b/providers/tests/opensearch/log/test_os_task_handler.py index 36e07893fdbb..af534dfbd590 100644 --- a/providers/tests/opensearch/log/test_os_task_handler.py +++ b/providers/tests/opensearch/log/test_os_task_handler.py @@ -45,6 +45,7 @@ from airflow.utils.timezone import datetime from providers.tests.opensearch.conftest import MockClient +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars from tests_common.test_utils.db import clear_db_dags, clear_db_runs @@ -54,11 +55,11 @@ ES_PROVIDER_YAML_FILE = AIRFLOW_SOURCES_ROOT_DIR / "airflow" / "providers" / "opensearch" / "provider.yaml" -def get_ti(dag_id, task_id, execution_date, create_task_instance): +def get_ti(dag_id, task_id, logical_date, create_task_instance): ti = create_task_instance( dag_id=dag_id, task_id=task_id, - execution_date=execution_date, + logical_date=logical_date, dagrun_state=DagRunState.RUNNING, state=TaskInstanceState.RUNNING, ) @@ -70,18 +71,21 @@ def get_ti(dag_id, task_id, execution_date, create_task_instance): class TestOpensearchTaskHandler: DAG_ID = "dag_for_testing_os_task_handler" TASK_ID = "task_for_testing_os_log_handler" - EXECUTION_DATE = datetime(2016, 1, 1) + LOGICAL_DATE = datetime(2016, 1, 1) LOG_ID = f"{DAG_ID}-{TASK_ID}-2016-01-01T00:00:00+00:00-1" - JSON_LOG_ID = f"{DAG_ID}-{TASK_ID}-{OpensearchTaskHandler._clean_date(EXECUTION_DATE)}-1" + JSON_LOG_ID = f"{DAG_ID}-{TASK_ID}-{OpensearchTaskHandler._clean_date(LOGICAL_DATE)}-1" FILENAME_TEMPLATE = "{try_number}.log" @pytest.fixture def ti(self, create_task_instance, create_log_template): - create_log_template(self.FILENAME_TEMPLATE, "{dag_id}-{task_id}-{execution_date}-{try_number}") + if AIRFLOW_V_3_0_PLUS: + create_log_template(self.FILENAME_TEMPLATE, "{dag_id}-{task_id}-{logical_date}-{try_number}") + else: + create_log_template(self.FILENAME_TEMPLATE, "{dag_id}-{task_id}-{execution_date}-{try_number}") yield get_ti( dag_id=self.DAG_ID, task_id=self.TASK_ID, - execution_date=self.EXECUTION_DATE, + logical_date=self.LOGICAL_DATE, create_task_instance=create_task_instance, ) clear_db_runs() @@ -268,7 +272,7 @@ def test_read_missing_logs(self, seconds, create_task_instance): ti = get_ti( self.DAG_ID, self.TASK_ID, - pendulum.instance(self.EXECUTION_DATE).add(days=1), # so logs are not found + pendulum.instance(self.LOGICAL_DATE).add(days=1), # so logs are not found create_task_instance=create_task_instance, ) ts = pendulum.now().add(seconds=-seconds) diff --git a/providers/tests/papermill/operators/test_papermill.py b/providers/tests/papermill/operators/test_papermill.py index a29d61bab462..5c51e814a9b8 100644 --- a/providers/tests/papermill/operators/test_papermill.py +++ b/providers/tests/papermill/operators/test_papermill.py @@ -200,7 +200,6 @@ def test_render_template(self, create_task_instance_of_operator): # TI Settings dag_id="test_render_template", task_id="render_dag_test", - execution_date=DEFAULT_DATE, ) task = ti.render_templates() diff --git a/providers/tests/presto/hooks/test_presto.py b/providers/tests/presto/hooks/test_presto.py index 74ee349f9d82..cec977b64124 100644 --- a/providers/tests/presto/hooks/test_presto.py +++ b/providers/tests/presto/hooks/test_presto.py @@ -29,11 +29,15 @@ from airflow.models import Connection from airflow.providers.presto.hooks.presto import PrestoHook, generate_presto_client_info +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + def test_generate_airflow_presto_client_info_header(): + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" + environ_date_key = "AIRFLOW_CTX_LOGICAL_DATE" if AIRFLOW_V_3_0_PLUS else "AIRFLOW_CTX_EXECUTION_DATE" env_vars = { "AIRFLOW_CTX_DAG_ID": "dag_id", - "AIRFLOW_CTX_EXECUTION_DATE": "2022-01-01T00:00:00", + environ_date_key: "2022-01-01T00:00:00", "AIRFLOW_CTX_TASK_ID": "task_id", "AIRFLOW_CTX_TRY_NUMBER": "1", "AIRFLOW_CTX_DAG_RUN_ID": "dag_run_id", @@ -42,7 +46,7 @@ def test_generate_airflow_presto_client_info_header(): expected = json.dumps( { "dag_id": "dag_id", - "execution_date": "2022-01-01T00:00:00", + date_key: "2022-01-01T00:00:00", "task_id": "task_id", "try_number": "1", "dag_run_id": "dag_run_id", @@ -157,10 +161,11 @@ def test_http_headers( mock_get_connection.return_value = Connection( login="login", password="password", host="host", schema="hive" ) + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" client = json.dumps( { "dag_id": "dag-id", - "execution_date": "2022-01-01T00:00:00", + date_key: "2022-01-01T00:00:00", "task_id": "task-id", "try_number": "1", "dag_run_id": "dag-run-id", diff --git a/providers/tests/redis/log/test_redis_task_handler.py b/providers/tests/redis/log/test_redis_task_handler.py index b010b18191a4..0f86ff65bf48 100644 --- a/providers/tests/redis/log/test_redis_task_handler.py +++ b/providers/tests/redis/log/test_redis_task_handler.py @@ -29,6 +29,7 @@ from airflow.utils.state import State from airflow.utils.timezone import datetime +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test @@ -40,7 +41,10 @@ def ti(self): date = datetime(2020, 1, 1) dag = DAG(dag_id="dag_for_testing_redis_task_handler", schedule=None, start_date=date) task = EmptyOperator(task_id="task_for_testing_redis_log_handler", dag=dag) - dag_run = DagRun(dag_id=dag.dag_id, execution_date=date, run_id="test", run_type="scheduled") + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun(dag_id=dag.dag_id, logical_date=date, run_id="test", run_type="scheduled") + else: + dag_run = DagRun(dag_id=dag.dag_id, execution_date=date, run_id="test", run_type="scheduled") with create_session() as session: session.add(dag_run) diff --git a/providers/tests/sftp/decorators/sensors/test_sftp.py b/providers/tests/sftp/decorators/sensors/test_sftp.py index 0f30e2da36d2..43283a2abcd4 100644 --- a/providers/tests/sftp/decorators/sensors/test_sftp.py +++ b/providers/tests/sftp/decorators/sensors/test_sftp.py @@ -50,7 +50,7 @@ def f(): ret = f() dr = dag_maker.create_dagrun() - ret.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + ret.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) ti = dr.get_task_instances()[0] assert ti.xcom_pull() == expected_xcom_return @@ -71,7 +71,7 @@ def f(*args, **kwargs): ret = f(*op_args, **op_kwargs) dr = dag_maker.create_dagrun() - ret.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + ret.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) ti = dr.get_task_instances()[0] assert ti.xcom_pull() == expected_xcom_return @@ -97,7 +97,7 @@ def f(): ret = f() dr = dag_maker.create_dagrun() - ret.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + ret.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) ti = dr.get_task_instances()[0] assert ti.xcom_pull() == expected_xcom_return @@ -125,6 +125,6 @@ def f(*args, **kwargs): ret = f(*op_args, **op_kwargs) dr = dag_maker.create_dagrun() - ret.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + ret.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) ti = dr.get_task_instances()[0] assert ti.xcom_pull() == expected_xcom_return diff --git a/providers/tests/sftp/operators/test_sftp.py b/providers/tests/sftp/operators/test_sftp.py index 7b7e79cb881e..b1551e9c4563 100644 --- a/providers/tests/sftp/operators/test_sftp.py +++ b/providers/tests/sftp/operators/test_sftp.py @@ -36,6 +36,7 @@ from airflow.utils import timezone from airflow.utils.timezone import datetime +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.config import conf_vars pytestmark = pytest.mark.db_test @@ -144,7 +145,6 @@ def test_file_transfer_no_intermediate_dir_error_put(self, create_task_instance_ ti2 = create_task_instance_of_operator( SFTPOperator, dag_id="unit_tests_sftp_op_file_transfer_no_intermediate_dir_error_put", - execution_date=timezone.utcnow(), task_id="test_sftp", ssh_hook=self.hook, local_filepath=self.test_local_filepath, @@ -184,8 +184,10 @@ def test_file_transfer_with_intermediate_dir_put(self, dag_maker): command=f"cat {self.test_remote_filepath_int_dir}", do_xcom_push=True, ) - - dagrun = dag_maker.create_dagrun(execution_date=timezone.utcnow()) + if AIRFLOW_V_3_0_PLUS: + dagrun = dag_maker.create_dagrun(logical_date=timezone.utcnow()) + else: + dagrun = dag_maker.create_dagrun(execution_date=timezone.utcnow()) tis = {ti.task_id: ti for ti in dagrun.task_instances} with pytest.warns(AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*"): tis["test_sftp"].run() @@ -218,8 +220,10 @@ def test_json_file_transfer_put(self, dag_maker): command=f"cat {self.test_remote_filepath}", do_xcom_push=True, ) - - dagrun = dag_maker.create_dagrun(execution_date=timezone.utcnow()) + if AIRFLOW_V_3_0_PLUS: + dagrun = dag_maker.create_dagrun(logical_date=timezone.utcnow()) + else: + dagrun = dag_maker.create_dagrun(execution_date=timezone.utcnow()) tis = {ti.task_id: ti for ti in dagrun.task_instances} with pytest.warns(AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*"): tis["put_test_task"].run() @@ -245,12 +249,18 @@ def test_pickle_file_transfer_get(self, dag_maker, create_remote_file_and_cleanu remote_filepath=self.test_remote_filepath, operation=SFTPOperation.GET, ) - - for ti in dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances: - with pytest.warns( - AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*" - ): - ti.run() + if AIRFLOW_V_3_0_PLUS: + for ti in dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances: + with pytest.warns( + AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*" + ): + ti.run() + else: + for ti in dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances: + with pytest.warns( + AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*" + ): + ti.run() # Test the received content. with open(self.test_local_filepath, "rb") as file: @@ -267,12 +277,18 @@ def test_json_file_transfer_get(self, dag_maker, create_remote_file_and_cleanup) remote_filepath=self.test_remote_filepath, operation=SFTPOperation.GET, ) - - for ti in dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances: - with pytest.warns( - AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*" - ): - ti.run() + if AIRFLOW_V_3_0_PLUS: + for ti in dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances: + with pytest.warns( + AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*" + ): + ti.run() + else: + for ti in dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances: + with pytest.warns( + AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*" + ): + ti.run() # Test the received content. content_received = None @@ -291,17 +307,30 @@ def test_file_transfer_no_intermediate_dir_error_get(self, dag_maker, create_rem operation=SFTPOperation.GET, ) - for ti in dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances: - # This should raise an error with "No such file" as the directory - # does not exist. - with ( - pytest.raises(AirflowException) as ctx, - pytest.warns( - AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*" - ), - ): - ti.run() - assert "No such file" in str(ctx.value) + if AIRFLOW_V_3_0_PLUS: + for ti in dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances: + # This should raise an error with "No such file" as the directory + # does not exist. + with ( + pytest.raises(AirflowException) as ctx, + pytest.warns( + AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*" + ), + ): + ti.run() + assert "No such file" in str(ctx.value) + else: + for ti in dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances: + # This should raise an error with "No such file" as the directory + # does not exist. + with ( + pytest.raises(AirflowException) as ctx, + pytest.warns( + AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*" + ), + ): + ti.run() + assert "No such file" in str(ctx.value) @conf_vars({("core", "enable_xcom_pickling"): "True"}) def test_file_transfer_with_intermediate_dir_error_get(self, dag_maker, create_remote_file_and_cleanup): @@ -315,11 +344,18 @@ def test_file_transfer_with_intermediate_dir_error_get(self, dag_maker, create_r create_intermediate_dirs=True, ) - for ti in dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances: - with pytest.warns( - AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*" - ): - ti.run() + if AIRFLOW_V_3_0_PLUS: + for ti in dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances: + with pytest.warns( + AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*" + ): + ti.run() + else: + for ti in dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances: + with pytest.warns( + AirflowProviderDeprecationWarning, match="Parameter `ssh_hook` is deprecated..*" + ): + ti.run() # Test the received content. content_received = None diff --git a/providers/tests/smtp/notifications/test_smtp.py b/providers/tests/smtp/notifications/test_smtp.py index 934a4d406a40..4c559df628e9 100644 --- a/providers/tests/smtp/notifications/test_smtp.py +++ b/providers/tests/smtp/notifications/test_smtp.py @@ -122,7 +122,7 @@ def test_notifier_templated(self, mock_smtphook_hook, dag_maker): @mock.patch("airflow.providers.smtp.notifications.smtp.SmtpHook") def test_notifier_with_defaults(self, mock_smtphook_hook, create_task_instance): - ti = create_task_instance(dag_id="dag", task_id="op", execution_date=timezone.datetime(2018, 1, 1)) + ti = create_task_instance(dag_id="dag", task_id="op", logical_date=timezone.datetime(2018, 1, 1)) context = {"dag": ti.dag_run.dag, "ti": ti} notifier = SmtpNotifier( from_email=conf.get("smtp", "smtp_mail_from"), @@ -147,7 +147,7 @@ def test_notifier_with_defaults(self, mock_smtphook_hook, create_task_instance): @mock.patch("airflow.providers.smtp.notifications.smtp.SmtpHook") def test_notifier_with_nondefault_conf_vars(self, mock_smtphook_hook, create_task_instance): - ti = create_task_instance(dag_id="dag", task_id="op", execution_date=timezone.datetime(2018, 1, 1)) + ti = create_task_instance(dag_id="dag", task_id="op", logical_date=timezone.datetime(2018, 1, 1)) context = {"dag": ti.dag_run.dag, "ti": ti} with ( diff --git a/providers/tests/snowflake/operators/test_snowflake.py b/providers/tests/snowflake/operators/test_snowflake.py index 3ab6ab5f8895..e461a0b6f62c 100644 --- a/providers/tests/snowflake/operators/test_snowflake.py +++ b/providers/tests/snowflake/operators/test_snowflake.py @@ -37,6 +37,8 @@ from airflow.utils import timezone from airflow.utils.types import DagRunType +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + DEFAULT_DATE = timezone.datetime(2015, 1, 1) DEFAULT_DATE_ISO = DEFAULT_DATE.isoformat() DEFAULT_DATE_DS = DEFAULT_DATE_ISO[:10] @@ -176,27 +178,34 @@ def create_context(task, dag=None): if dag is None: dag = DAG(dag_id="dag", schedule=None) tzinfo = pendulum.timezone("UTC") - execution_date = timezone.datetime(2022, 1, 1, 1, 0, 0, tzinfo=tzinfo) - dag_run = DagRun( - dag_id=dag.dag_id, - execution_date=execution_date, - run_id=DagRun.generate_run_id(DagRunType.MANUAL, execution_date), - ) + logical_date = timezone.datetime(2022, 1, 1, 1, 0, 0, tzinfo=tzinfo) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=dag.dag_id, + logical_date=logical_date, + run_id=DagRun.generate_run_id(DagRunType.MANUAL, logical_date), + ) + else: + dag_run = DagRun( + dag_id=dag.dag_id, + execution_date=logical_date, + run_id=DagRun.generate_run_id(DagRunType.MANUAL, logical_date), + ) task_instance = TaskInstance(task=task) task_instance.dag_run = dag_run task_instance.xcom_push = mock.Mock() + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" return { "dag": dag, - "ts": execution_date.isoformat(), + "ts": logical_date.isoformat(), "task": task, "ti": task_instance, "task_instance": task_instance, "run_id": dag_run.run_id, "dag_run": dag_run, - "execution_date": execution_date, - "data_interval_end": execution_date, - "logical_date": execution_date, + "data_interval_end": logical_date, + date_key: logical_date, } diff --git a/providers/tests/standard/operators/test_bash.py b/providers/tests/standard/operators/test_bash.py index 645f7f276b87..d548204e9319 100644 --- a/providers/tests/standard/operators/test_bash.py +++ b/providers/tests/standard/operators/test_bash.py @@ -89,7 +89,7 @@ def test_echo_env_variables( f"{utc_now.isoformat()}\n" f"manual__{utc_now.isoformat()}\n" ) - + date_env_name = "$AIRFLOW_CTX_LOGICAL_DATE" if AIRFLOW_V_3_0_PLUS else "$AIRFLOW_CTX_EXECUTION_DATE" with dag_maker( "bash_op_test", default_args={"owner": "airflow", "retries": 100, "start_date": DEFAULT_DATE}, @@ -104,23 +104,34 @@ def test_echo_env_variables( f"echo $PYTHONPATH>> {tmp_file};" f"echo $AIRFLOW_CTX_DAG_ID >> {tmp_file};" f"echo $AIRFLOW_CTX_TASK_ID>> {tmp_file};" - f"echo $AIRFLOW_CTX_EXECUTION_DATE>> {tmp_file};" + f"echo {date_env_name}>> {tmp_file};" f"echo $AIRFLOW_CTX_DAG_RUN_ID>> {tmp_file};", append_env=append_env, env=user_defined_env, ) - execution_date = utc_now + logical_date = utc_now triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - dag_maker.create_dagrun( - run_type=DagRunType.MANUAL, - execution_date=execution_date, - start_date=utc_now, - state=State.RUNNING, - external_trigger=False, - data_interval=(execution_date, execution_date), - **triggered_by_kwargs, - ) + if AIRFLOW_V_3_0_PLUS: + dag_maker.create_dagrun( + run_type=DagRunType.MANUAL, + logical_date=logical_date, + start_date=utc_now, + state=State.RUNNING, + external_trigger=False, + data_interval=(logical_date, logical_date), + **triggered_by_kwargs, + ) + else: + dag_maker.create_dagrun( + run_type=DagRunType.MANUAL, + execution_date=logical_date, + start_date=utc_now, + state=State.RUNNING, + external_trigger=False, + data_interval=(logical_date, logical_date), + **triggered_by_kwargs, + ) with mock.patch.dict( "os.environ", {"AIRFLOW_HOME": "MY_PATH_TO_AIRFLOW_HOME", "PYTHONPATH": "AWESOME_PYTHONPATH"} @@ -282,12 +293,10 @@ def test_templated_fields(self, create_task_instance_of_operator): # Other parameters dag_id="test_templated_fields_dag", task_id="test_templated_fields_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) ti.render_templates() task: BashOperator = ti.task assert task.bash_command == 'echo "test_templated_fields_dag"' - assert task.env == {"FOO": "2024-02-01"} assert task.cwd == Path(__file__).absolute().parent.as_posix() @pytest.mark.db_test diff --git a/providers/tests/standard/operators/test_datetime.py b/providers/tests/standard/operators/test_datetime.py index b729781f19fd..b51821275dce 100644 --- a/providers/tests/standard/operators/test_datetime.py +++ b/providers/tests/standard/operators/test_datetime.py @@ -80,14 +80,24 @@ def base_tests_setup(self, dag_maker): self.branch_2.set_upstream(self.branch_op) triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - self.dr = dag_maker.create_dagrun( - run_id="manual__", - start_date=DEFAULT_DATE, - execution_date=DEFAULT_DATE, - state=State.RUNNING, - data_interval=(DEFAULT_DATE, DEFAULT_DATE), - **triggered_by_kwargs, - ) + if AIRFLOW_V_3_0_PLUS: + self.dr = dag_maker.create_dagrun( + run_id="manual__", + start_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) + else: + self.dr = dag_maker.create_dagrun( + run_id="manual__", + start_date=DEFAULT_DATE, + execution_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) def teardown_method(self): with create_session() as session: @@ -238,18 +248,28 @@ def test_branch_datetime_operator_lower_comparison_outside_range(self, target_lo ) @time_machine.travel("2020-12-01 09:00:00") def test_branch_datetime_operator_use_task_logical_date(self, dag_maker, target_lower, target_upper): - """Check if BranchDateTimeOperator uses task execution date""" + """Check if BranchDateTimeOperator uses task logical date""" in_between_date = timezone.datetime(2020, 7, 7, 10, 30, 0) self.branch_op.use_task_logical_date = True triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - self.dr = dag_maker.create_dagrun( - run_id="manual_exec_date__", - start_date=in_between_date, - execution_date=in_between_date, - state=State.RUNNING, - data_interval=(in_between_date, in_between_date), - **triggered_by_kwargs, - ) + if AIRFLOW_V_3_0_PLUS: + self.dr = dag_maker.create_dagrun( + run_id="manual_exec_date__", + start_date=in_between_date, + logical_date=in_between_date, + state=State.RUNNING, + data_interval=(in_between_date, in_between_date), + **triggered_by_kwargs, + ) + else: + self.dr = dag_maker.create_dagrun( + run_id="manual_exec_date__", + start_date=in_between_date, + execution_date=in_between_date, + state=State.RUNNING, + data_interval=(in_between_date, in_between_date), + **triggered_by_kwargs, + ) self.branch_op.target_lower = target_lower self.branch_op.target_upper = target_upper diff --git a/providers/tests/standard/operators/test_python.py b/providers/tests/standard/operators/test_python.py index 143c057003d6..3798a7123844 100644 --- a/providers/tests/standard/operators/test_python.py +++ b/providers/tests/standard/operators/test_python.py @@ -149,15 +149,26 @@ def default_kwargs(**kwargs): def create_dag_run(self) -> DagRun: triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - return self.dag_maker.create_dagrun( - state=DagRunState.RUNNING, - start_date=self.dag_maker.start_date, - session=self.dag_maker.session, - execution_date=self.default_date, - run_type=DagRunType.MANUAL, - data_interval=(self.default_date, self.default_date), - **triggered_by_kwargs, # type: ignore - ) + if AIRFLOW_V_3_0_PLUS: + return self.dag_maker.create_dagrun( + state=DagRunState.RUNNING, + start_date=self.dag_maker.start_date, + session=self.dag_maker.session, + logical_date=self.default_date, + run_type=DagRunType.MANUAL, + data_interval=(self.default_date, self.default_date), + **triggered_by_kwargs, # type: ignore + ) + else: + return self.dag_maker.create_dagrun( + state=DagRunState.RUNNING, + start_date=self.dag_maker.start_date, + session=self.dag_maker.session, + execution_date=self.default_date, + run_type=DagRunType.MANUAL, + data_interval=(self.default_date, self.default_date), + **triggered_by_kwargs, # type: ignore + ) def create_ti(self, fn, **kwargs) -> TI: """Create TaskInstance for class defined Operator.""" @@ -167,7 +178,7 @@ def create_ti(self, fn, **kwargs) -> TI: **self.default_kwargs(**kwargs), dag_id=self.dag_id, task_id=self.task_id, - execution_date=self.default_date, + logical_date=self.default_date, ) def run_as_operator(self, fn, **kwargs): @@ -904,21 +915,19 @@ def test_virtualenv_serializable_context_fields(self, create_task_instance): # These are intentionally NOT serialized into the virtual environment: # * Variables pointing to the task instance itself. # * Variables that are accessor instances. - intentionally_excluded_context_keys = [ + intentionally_excluded_context_keys = { "task_instance", "ti", "var", # Accessor for Variable; var->json and var->value. "conn", # Accessor for Connection. - ] + } if AIRFLOW_V_2_9_PLUS: - intentionally_excluded_context_keys.extend( - ["map_index_template"], - ) + intentionally_excluded_context_keys.add("map_index_template") if AIRFLOW_V_2_10_PLUS: - intentionally_excluded_context_keys.extend( - # Accessors for inlet_events and outlet_events - ["inlet_events", "outlet_events"], - ) + intentionally_excluded_context_keys |= { + "inlet_events", + "outlet_events", + } ti = create_task_instance(dag_id=self.dag_id, task_id=self.task_id, schedule=None) context = ti.get_template_context() @@ -929,6 +938,25 @@ def test_virtualenv_serializable_context_fields(self, create_task_instance): *PythonVirtualenvOperator.AIRFLOW_SERIALIZABLE_CONTEXT_KEYS, *intentionally_excluded_context_keys, } + if AIRFLOW_V_3_0_PLUS: + declared_keys -= { + "execution_date", + "next_ds", + "next_ds_nodash", + "prev_ds", + "prev_ds_nodash", + "tomorrow_ds", + "tomorrow_ds_nodash", + "triggering_dataset_events", + "yesterday_ds", + "yesterday_ds_nodash", + "next_execution_date", + "prev_execution_date", + "prev_execution_date_success", + } + else: + declared_keys.remove("triggering_asset_events") + assert set(context) == declared_keys @pytest.mark.parametrize( @@ -1437,27 +1465,16 @@ def f( # basic ds_nodash, inlets, - next_ds, - next_ds_nodash, outlets, params, - prev_ds, - prev_ds_nodash, run_id, task_instance_key_str, test_mode, - tomorrow_ds, - tomorrow_ds_nodash, ts, ts_nodash, ts_nodash_with_tz, - yesterday_ds, - yesterday_ds_nodash, # pendulum-specific - execution_date, - next_execution_date, - prev_execution_date, - prev_execution_date_success, + logical_date, prev_start_date_success, prev_end_date_success, # airflow-specific @@ -1486,26 +1503,15 @@ def f( # basic ds_nodash, inlets, - next_ds, - next_ds_nodash, outlets, - prev_ds, - prev_ds_nodash, run_id, task_instance_key_str, test_mode, - tomorrow_ds, - tomorrow_ds_nodash, ts, ts_nodash, ts_nodash_with_tz, - yesterday_ds, - yesterday_ds_nodash, # pendulum-specific - execution_date, - next_execution_date, - prev_execution_date, - prev_execution_date_success, + logical_date, prev_start_date_success, prev_end_date_success, # other @@ -1530,21 +1536,13 @@ def f( # basic ds_nodash, inlets, - next_ds, - next_ds_nodash, outlets, - prev_ds, - prev_ds_nodash, run_id, task_instance_key_str, test_mode, - tomorrow_ds, - tomorrow_ds_nodash, ts, ts_nodash, ts_nodash_with_tz, - yesterday_ds, - yesterday_ds_nodash, # other **context, ): diff --git a/providers/tests/standard/operators/test_weekday.py b/providers/tests/standard/operators/test_weekday.py index 127d735c3da0..a9eb19189ecc 100644 --- a/providers/tests/standard/operators/test_weekday.py +++ b/providers/tests/standard/operators/test_weekday.py @@ -106,14 +106,24 @@ def test_branch_follow_true(self, weekday, dag_maker): branch_3.set_upstream(branch_op) triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - dr = dag_maker.create_dagrun( - run_id="manual__", - start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, - state=State.RUNNING, - data_interval=(DEFAULT_DATE, DEFAULT_DATE), - **triggered_by_kwargs, - ) + if AIRFLOW_V_3_0_PLUS: + dr = dag_maker.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + logical_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) + else: + dr = dag_maker.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + execution_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) branch_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE) @@ -128,7 +138,7 @@ def test_branch_follow_true(self, weekday, dag_maker): ) @time_machine.travel("2021-01-25") # Monday - def test_branch_follow_true_with_execution_date(self, dag_maker): + def test_branch_follow_true_with_logical_date(self, dag_maker): """Checks if BranchDayOfWeekOperator follows true branch when set use_task_logical_date""" with dag_maker( "branch_day_of_week_operator_test", start_date=DEFAULT_DATE, schedule=INTERVAL, serialized=True @@ -147,14 +157,24 @@ def test_branch_follow_true_with_execution_date(self, dag_maker): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - dr = dag_maker.create_dagrun( - run_id="manual__", - start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, - state=State.RUNNING, - data_interval=(DEFAULT_DATE, DEFAULT_DATE), - **triggered_by_kwargs, - ) + if AIRFLOW_V_3_0_PLUS: + dr = dag_maker.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + logical_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) + else: + dr = dag_maker.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + execution_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) branch_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE) @@ -186,14 +206,24 @@ def test_branch_follow_false(self, dag_maker): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - dr = dag_maker.create_dagrun( - run_id="manual__", - start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, - state=State.RUNNING, - data_interval=(DEFAULT_DATE, DEFAULT_DATE), - **triggered_by_kwargs, - ) + if AIRFLOW_V_3_0_PLUS: + dr = dag_maker.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + logical_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) + else: + dr = dag_maker.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + execution_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) branch_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE) @@ -285,15 +315,24 @@ def test_branch_xcom_push_true_branch(self, dag_maker): branch_2.set_upstream(branch_op) triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} - - dr = dag_maker.create_dagrun( - run_id="manual__", - start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, - state=State.RUNNING, - data_interval=(DEFAULT_DATE, DEFAULT_DATE), - **triggered_by_kwargs, - ) + if AIRFLOW_V_3_0_PLUS: + dr = dag_maker.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + logical_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) + else: + dr = dag_maker.create_dagrun( + run_id="manual__", + start_date=timezone.utcnow(), + execution_date=DEFAULT_DATE, + state=State.RUNNING, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + **triggered_by_kwargs, + ) branch_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE) diff --git a/providers/tests/system/papermill/example_papermill.py b/providers/tests/system/papermill/example_papermill.py index 33b427785cf2..b5504e3f80d9 100644 --- a/providers/tests/system/papermill/example_papermill.py +++ b/providers/tests/system/papermill/example_papermill.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. """ -This DAG will use Papermill to run the notebook "hello_world", based on the execution date +This DAG will use Papermill to run the notebook "hello_world", based on the logical date it will create an output notebook "out-". All fields, including the keys in the parameters, are templated. """ diff --git a/providers/tests/system/papermill/example_papermill_remote_verify.py b/providers/tests/system/papermill/example_papermill_remote_verify.py index 89d482858cb0..d083f8393fda 100644 --- a/providers/tests/system/papermill/example_papermill_remote_verify.py +++ b/providers/tests/system/papermill/example_papermill_remote_verify.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. """ -This DAG will use Papermill to run the notebook "hello_world", based on the execution date +This DAG will use Papermill to run the notebook "hello_world", based on the logical date it will create an output notebook "out-". All fields, including the keys in the parameters, are templated. """ diff --git a/providers/tests/system/papermill/example_papermill_verify.py b/providers/tests/system/papermill/example_papermill_verify.py index 632b209f21ef..347eab2184e3 100644 --- a/providers/tests/system/papermill/example_papermill_verify.py +++ b/providers/tests/system/papermill/example_papermill_verify.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. """ -This DAG will use Papermill to run the notebook "hello_world", based on the execution date +This DAG will use Papermill to run the notebook "hello_world", based on the logical date it will create an output notebook "out-". All fields, including the keys in the parameters, are templated. """ @@ -42,15 +42,15 @@ # [START howto_verify_operator_papermill] @task -def check_notebook(inlets, execution_date): +def check_notebook(inlets, logical_date): """ Verify the message in the notebook """ notebook = sb.read_notebook(inlets[0].url) message = notebook.scraps["message"] - print(f"Message in notebook {message} for {execution_date}") + print(f"Message in notebook {message} for {logical_date}") - if message.data != f"Ran from Airflow at {execution_date}!": + if message.data != f"Ran from Airflow at {logical_date}!": return False return True @@ -66,11 +66,11 @@ def check_notebook(inlets, execution_date): run_this = PapermillOperator( task_id="run_example_notebook", input_nb=os.path.join(os.path.dirname(os.path.realpath(__file__)), "input_notebook.ipynb"), - output_nb="/tmp/out-{{ execution_date }}.ipynb", - parameters={"msgs": "Ran from Airflow at {{ execution_date }}!"}, + output_nb="/tmp/out-{{ logical_date }}.ipynb", + parameters={"msgs": "Ran from Airflow at {{ logical_date }}!"}, ) - run_this >> check_notebook(inlets=AUTO, execution_date="{{ execution_date }}") + run_this >> check_notebook(inlets=AUTO, logical_date="{{ logical_date }}") # [END howto_verify_operator_papermill] from tests_common.test_utils.system_tests import get_test_run # noqa: E402 diff --git a/providers/tests/telegram/operators/test_telegram.py b/providers/tests/telegram/operators/test_telegram.py index 478776bd8845..46916ce56763 100644 --- a/providers/tests/telegram/operators/test_telegram.py +++ b/providers/tests/telegram/operators/test_telegram.py @@ -156,15 +156,15 @@ def test_should_return_templatized_text_field(self, mock_hook): telegram_conn_id="telegram_default", chat_id="-420913222", task_id="telegram", - text="execution date is {{ ds }}", + text="logical date is {{ ds }}", telegram_kwargs={"custom_arg": "value", "text": "should be ignored"}, ) operator.render_template_fields({"ds": "2021-02-04"}) operator.execute(None) - assert operator.text == "execution date is 2021-02-04" + assert operator.text == "logical date is 2021-02-04" assert "text" in operator.telegram_kwargs - assert operator.telegram_kwargs["text"] == "execution date is 2021-02-04" + assert operator.telegram_kwargs["text"] == "logical date is 2021-02-04" def test_should_return_templatized_chat_id_field(self): operator = TelegramOperator( diff --git a/providers/tests/trino/hooks/test_trino.py b/providers/tests/trino/hooks/test_trino.py index 5f6e0961727d..696cf28bd8bf 100644 --- a/providers/tests/trino/hooks/test_trino.py +++ b/providers/tests/trino/hooks/test_trino.py @@ -29,6 +29,8 @@ from airflow.models import Connection from airflow.providers.trino.hooks.trino import TrinoHook +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + HOOK_GET_CONNECTION = "airflow.providers.trino.hooks.trino.TrinoHook.get_connection" BASIC_AUTHENTICATION = "airflow.providers.trino.hooks.trino.trino.auth.BasicAuthentication" KERBEROS_AUTHENTICATION = "airflow.providers.trino.hooks.trino.trino.auth.KerberosAuthentication" @@ -68,10 +70,11 @@ def test_http_headers( mock_get_connection.return_value = Connection( login="login", password="password", host="host", schema="hive" ) + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" client = json.dumps( { "dag_id": "dag-id", - "execution_date": "2022-01-01T00:00:00", + date_key: "2022-01-01T00:00:00", "task_id": "task-id", "try_number": "1", "dag_run_id": "dag-run-id", diff --git a/providers/tests/ydb/operators/test_ydb.py b/providers/tests/ydb/operators/test_ydb.py index 1392253f743d..b5e014eca215 100644 --- a/providers/tests/ydb/operators/test_ydb.py +++ b/providers/tests/ydb/operators/test_ydb.py @@ -26,7 +26,6 @@ from airflow.models.dag import DAG from airflow.providers.common.sql.hooks.sql import fetch_all_handler, fetch_one_handler from airflow.providers.ydb.operators.ydb import YDBExecuteQueryOperator -from airflow.utils import timezone @pytest.mark.db_test @@ -38,7 +37,6 @@ def test_sql_templating(create_task_instance_of_operator): ydb_conn_id="ydb_default1", dag_id="test_template_body_templating_dag", task_id="test_template_body_templating_task", - execution_date=timezone.datetime(2024, 2, 1, tzinfo=timezone.utc), ) ti.render_templates() task: YDBExecuteQueryOperator = ti.task diff --git a/scripts/ci/pre_commit/check_ti_vs_tis_attributes.py b/scripts/ci/pre_commit/check_ti_vs_tis_attributes.py index d1782e629074..c8cb358960f1 100755 --- a/scripts/ci/pre_commit/check_ti_vs_tis_attributes.py +++ b/scripts/ci/pre_commit/check_ti_vs_tis_attributes.py @@ -48,7 +48,7 @@ def compare_attributes(path1, path2): "task_instance_note", "dag_run", "trigger", - "execution_date", + "logical_date", "triggerer_job", "note", "rendered_task_instance_fields", diff --git a/task_sdk/src/airflow/sdk/definitions/baseoperator.py b/task_sdk/src/airflow/sdk/definitions/baseoperator.py index 1eae39576c8e..9b1f64d970e6 100644 --- a/task_sdk/src/airflow/sdk/definitions/baseoperator.py +++ b/task_sdk/src/airflow/sdk/definitions/baseoperator.py @@ -330,16 +330,16 @@ class derived from this one results in the creation of a task object, :param max_retry_delay: maximum delay interval between retries, can be set as ``timedelta`` or ``float`` seconds, which will be converted into ``timedelta``. :param start_date: The ``start_date`` for the task, determines - the ``execution_date`` for the first task instance. The best practice + the ``logical_date`` for the first task instance. The best practice is to have the start_date rounded to your DAG's ``schedule_interval``. Daily jobs have their start_date some day at 00:00:00, hourly jobs have their start_date at 00:00 of a specific hour. Note that Airflow simply looks at the latest - ``execution_date`` and adds the ``schedule_interval`` to determine - the next ``execution_date``. It is also very important + ``logical_date`` and adds the ``schedule_interval`` to determine + the next ``logical_date``. It is also very important to note that different tasks' dependencies need to line up in time. If task A depends on task B and their - start_date are offset in a way that their execution_date don't line + start_date are offset in a way that their logical_date don't line up, A's dependencies will never be met. If you are looking to delay a task, for example running a daily task at 2AM, look into the ``TimeSensor`` and ``TimeDeltaSensor``. We advise against using @@ -454,7 +454,7 @@ class derived from this one results in the creation of a task object, Resources constructor) to their values. :param run_as_user: unix username to impersonate while running the task :param max_active_tis_per_dag: When set, a task will be able to limit the concurrent - runs across execution_dates. + runs across logical_dates. :param max_active_tis_per_dagrun: When set, a task will be able to limit the concurrent task instances per DAG run. :param executor: Which executor to target when running this task. NOT YET SUPPORTED diff --git a/task_sdk/src/airflow/sdk/definitions/dag.py b/task_sdk/src/airflow/sdk/definitions/dag.py index 2573c62eb312..745581b45d79 100644 --- a/task_sdk/src/airflow/sdk/definitions/dag.py +++ b/task_sdk/src/airflow/sdk/definitions/dag.py @@ -655,7 +655,7 @@ def owner(self) -> str: @property def allow_future_exec_dates(self) -> bool: - return settings.ALLOW_FUTURE_EXEC_DATES and not self.timetable.can_be_scheduled + return settings.ALLOW_FUTURE_LOGICAL_DATES and not self.timetable.can_be_scheduled def resolve_template_files(self): for t in self.tasks: diff --git a/tests/api_connexion/endpoints/test_asset_endpoint.py b/tests/api_connexion/endpoints/test_asset_endpoint.py index 4d195b5d870e..9e9371b91ef7 100644 --- a/tests/api_connexion/endpoints/test_asset_endpoint.py +++ b/tests/api_connexion/endpoints/test_asset_endpoint.py @@ -522,7 +522,7 @@ def test_includes_created_dagrun(self, session): dag_id="TEST_DAG_ID", run_id="TEST_DAG_RUN_ID", run_type=DagRunType.ASSET_TRIGGERED, - execution_date=timezone.parse(self.default_time), + logical_date=timezone.parse(self.default_time), start_date=timezone.parse(self.default_time), external_trigger=True, state="success", @@ -604,7 +604,7 @@ def test_should_respond_200(self, session): session, dag_id=None, event="api.create_asset_event", - execution_date=None, + logical_date=None, expected_extra=event_payload, ) @@ -622,7 +622,7 @@ def test_should_mask_sensitive_extra_logs(self, session): session, dag_id=None, event="api.create_asset_event", - execution_date=None, + logical_date=None, expected_extra=expected_extra, ) diff --git a/tests/api_connexion/endpoints/test_connection_endpoint.py b/tests/api_connexion/endpoints/test_connection_endpoint.py index e23a43a14c4f..717e24cfb537 100644 --- a/tests/api_connexion/endpoints/test_connection_endpoint.py +++ b/tests/api_connexion/endpoints/test_connection_endpoint.py @@ -81,7 +81,7 @@ def test_delete_should_respond_204(self, session): assert response.status_code == 204 connection = session.query(Connection).all() assert len(connection) == 0 - _check_last_log(session, dag_id=None, event="api.connection.delete", execution_date=None) + _check_last_log(session, dag_id=None, event="api.connection.delete", logical_date=None) def test_delete_should_respond_404(self): response = self.client.delete( @@ -370,7 +370,7 @@ def test_patch_should_respond_200(self, payload, session): "/api/v1/connections/test-connection-id", json=payload, environ_overrides={"REMOTE_USER": "test"} ) assert response.status_code == 200 - _check_last_log(session, dag_id=None, event="api.connection.edit", execution_date=None) + _check_last_log(session, dag_id=None, event="api.connection.edit", logical_date=None) def test_patch_should_respond_200_with_update_mask(self, session): self._create_connection(session) @@ -535,7 +535,7 @@ def test_post_should_respond_200(self, session): assert len(connection) == 1 assert connection[0].conn_id == "test-connection-id" _check_last_log( - session, dag_id=None, event="api.connection.create", execution_date=None, expected_extra=payload + session, dag_id=None, event="api.connection.create", logical_date=None, expected_extra=payload ) def test_post_should_respond_200_extra_null(self, session): diff --git a/tests/api_connexion/endpoints/test_dag_endpoint.py b/tests/api_connexion/endpoints/test_dag_endpoint.py index 1cd014ccb0cf..b812d190dff5 100644 --- a/tests/api_connexion/endpoints/test_dag_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_endpoint.py @@ -1163,7 +1163,7 @@ def test_should_respond_200_on_patch_is_paused(self, url_safe_serializer, sessio } assert response.json == expected_response _check_last_log( - session, dag_id="TEST_DAG_1", event="api.patch_dag", execution_date=None, expected_extra=payload + session, dag_id="TEST_DAG_1", event="api.patch_dag", logical_date=None, expected_extra=payload ) def test_should_respond_400_on_invalid_request(self): @@ -1377,7 +1377,7 @@ def test_should_respond_200_on_patch_is_paused(self, session, url_safe_serialize ], "total_entries": 2, } == response.json - _check_last_log(session, dag_id=None, event="api.patch_dags", execution_date=None) + _check_last_log(session, dag_id=None, event="api.patch_dags", logical_date=None) def test_should_respond_200_on_patch_is_paused_using_update_mask(self, session, url_safe_serializer): file_token = url_safe_serializer.dumps("/tmp/dag_1.py") @@ -1452,7 +1452,7 @@ def test_should_respond_200_on_patch_is_paused_using_update_mask(self, session, ], "total_entries": 2, } == response.json - _check_last_log(session, dag_id=None, event="api.patch_dags", execution_date=None) + _check_last_log(session, dag_id=None, event="api.patch_dags", logical_date=None) def test_wrong_value_as_update_mask_rasise(self, session): self._create_dag_models(2) @@ -1542,7 +1542,7 @@ def test_only_active_true_returns_active_dags(self, url_safe_serializer, session ], "total_entries": 1, } == response.json - _check_last_log(session, dag_id=None, event="api.patch_dags", execution_date=None) + _check_last_log(session, dag_id=None, event="api.patch_dags", logical_date=None) def test_only_active_false_returns_all_dags(self, url_safe_serializer, session): file_token = url_safe_serializer.dumps("/tmp/dag_1.py") @@ -1613,7 +1613,7 @@ def test_only_active_false_returns_all_dags(self, url_safe_serializer, session): ], "total_entries": 2, } == response.json - _check_last_log(session, dag_id=None, event="api.patch_dags", execution_date=None) + _check_last_log(session, dag_id=None, event="api.patch_dags", logical_date=None) @pytest.mark.parametrize( "url, expected_dag_ids", @@ -2000,7 +2000,7 @@ def test_that_dag_can_be_deleted(self, session): environ_overrides={"REMOTE_USER": "test"}, ) assert response.status_code == 204 - _check_last_log(session, dag_id="TEST_DAG_1", event="api.delete_dag", execution_date=None) + _check_last_log(session, dag_id="TEST_DAG_1", event="api.delete_dag", logical_date=None) def test_raise_when_dag_is_not_found(self): response = self.client.delete( diff --git a/tests/api_connexion/endpoints/test_dag_run_endpoint.py b/tests/api_connexion/endpoints/test_dag_run_endpoint.py index 6bbd63fa8b49..b295c063adc2 100644 --- a/tests/api_connexion/endpoints/test_dag_run_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_run_endpoint.py @@ -104,7 +104,7 @@ def _create_test_dag_run(self, state=DagRunState.RUNNING, extra_dag=False, commi dag_id="TEST_DAG_ID", run_id=f"TEST_DAG_RUN_ID_{i}", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time) + timedelta(days=i - 1), + logical_date=timezone.parse(self.default_time) + timedelta(days=i - 1), start_date=timezone.parse(self.default_time), external_trigger=True, state=state, @@ -121,7 +121,7 @@ def _create_test_dag_run(self, state=DagRunState.RUNNING, extra_dag=False, commi dag_id=f"TEST_DAG_ID_{i}", run_id=f"TEST_DAG_RUN_ID_{i}", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time_2), + logical_date=timezone.parse(self.default_time_2), start_date=timezone.parse(self.default_time), external_trigger=True, state=state, @@ -185,7 +185,7 @@ def test_should_respond_200(self, session): dag_id="TEST_DAG_ID", run_id="TEST_DAG_RUN_ID", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time), + logical_date=timezone.parse(self.default_time), start_date=timezone.parse(self.default_time), external_trigger=True, state="running", @@ -197,7 +197,6 @@ def test_should_respond_200(self, session): "end_date": None, "state": "running", "logical_date": self.default_time, - "execution_date": self.default_time, "external_trigger": True, "start_date": self.default_time, "conf": {}, @@ -236,7 +235,7 @@ def test_should_raises_401_unauthenticated(self, session): dag_id="TEST_DAG_ID", run_id="TEST_DAG_RUN_ID", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time), + logical_date=timezone.parse(self.default_time), start_date=timezone.parse(self.default_time), external_trigger=True, ) @@ -251,7 +250,7 @@ def test_should_raises_401_unauthenticated(self, session): "fields", [ ["dag_run_id", "logical_date"], - ["dag_run_id", "state", "conf", "execution_date"], + ["dag_run_id", "state", "conf", "logical_date"], ], ) def test_should_return_specified_fields(self, session, fields): @@ -259,7 +258,7 @@ def test_should_return_specified_fields(self, session, fields): dag_id="TEST_DAG_ID", run_id="TEST_DAG_RUN_ID", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time), + logical_date=timezone.parse(self.default_time), start_date=timezone.parse(self.default_time), external_trigger=True, state="running", @@ -274,7 +273,6 @@ def test_should_return_specified_fields(self, session, fields): ) assert response.status_code == 200 res_json = response.json - print("get dagRun", res_json) assert len(res_json.keys()) == len(fields) for field in fields: assert field in res_json @@ -284,7 +282,7 @@ def test_should_respond_400_with_not_exists_fields(self, session): dag_id="TEST_DAG_ID", run_id="TEST_DAG_RUN_ID", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time), + logical_date=timezone.parse(self.default_time), start_date=timezone.parse(self.default_time), external_trigger=True, state="running", @@ -309,7 +307,6 @@ def test_should_respond_200(self, session): "dag_run_id": "TEST_DAG_RUN_ID_1", "end_date": None, "state": "running", - "execution_date": self.default_time, "logical_date": self.default_time, "external_trigger": True, "start_date": self.default_time, @@ -326,7 +323,6 @@ def test_should_respond_200(self, session): "dag_run_id": "TEST_DAG_RUN_ID_2", "end_date": None, "state": "running", - "execution_date": self.default_time_2, "logical_date": self.default_time_2, "external_trigger": True, "start_date": self.default_time, @@ -381,7 +377,6 @@ def test_return_correct_results_with_order_by(self, session): "dag_run_id": "TEST_DAG_RUN_ID_2", "end_date": None, "state": "running", - "execution_date": self.default_time_2, "logical_date": self.default_time_2, "external_trigger": True, "start_date": self.default_time, @@ -398,7 +393,6 @@ def test_return_correct_results_with_order_by(self, session): "dag_run_id": "TEST_DAG_RUN_ID_1", "end_date": None, "state": "running", - "execution_date": self.default_time, "logical_date": self.default_time, "external_trigger": True, "start_date": self.default_time, @@ -414,7 +408,7 @@ def test_return_correct_results_with_order_by(self, session): result = session.query(DagRun).all() assert len(result) == 2 response = self.client.get( - "api/v1/dags/TEST_DAG_ID/dagRuns?order_by=-execution_date", + "api/v1/dags/TEST_DAG_ID/dagRuns?order_by=-logical_date", environ_overrides={"REMOTE_USER": "test"}, ) @@ -448,7 +442,7 @@ def test_should_raises_401_unauthenticated(self): "fields", [ ["dag_run_id", "logical_date"], - ["dag_run_id", "state", "conf", "execution_date"], + ["dag_run_id", "state", "conf", "logical_date"], ], ) def test_should_return_specified_fields(self, session, fields): @@ -551,7 +545,7 @@ def _create_dag_runs(self, count): dag_id="TEST_DAG_ID", run_id=f"TEST_DAG_RUN_ID{i}", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time) + timedelta(minutes=i), + logical_date=timezone.parse(self.default_time) + timedelta(minutes=i), start_date=timezone.parse(self.default_time), external_trigger=True, ) @@ -628,7 +622,7 @@ def test_date_filters_gte_and_lte(self, url, expected_dag_run_ids, session): dagrun_models = self._create_dag_runs() session.add_all(dagrun_models) for d in dagrun_models: - d.updated_at = d.execution_date + d.updated_at = d.logical_date session.commit() response = self.client.get(url, environ_overrides={"REMOTE_USER": "test"}) @@ -656,7 +650,7 @@ def _create_dag_runs(self): dag_id="TEST_DAG_ID", run_id=f"TEST_START_EXEC_DAY_1{i}", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(dates[i]), + logical_date=timezone.parse(dates[i]), start_date=timezone.parse(dates[i]), external_trigger=True, state=DagRunState.SUCCESS, @@ -700,7 +694,6 @@ def test_should_respond_200(self): "dag_run_id": "TEST_DAG_RUN_ID_1", "end_date": None, "state": "running", - "execution_date": self.default_time, "logical_date": self.default_time, "external_trigger": True, "start_date": self.default_time, @@ -718,7 +711,6 @@ def test_should_respond_200(self): "dag_run_id": "TEST_DAG_RUN_ID_2", "end_date": None, "state": "running", - "execution_date": self.default_time_2, "logical_date": self.default_time_2, "external_trigger": True, "start_date": self.default_time, @@ -780,7 +772,6 @@ def test_order_by_descending_works(self): "dag_run_id": "TEST_DAG_RUN_ID_2", "end_date": None, "state": "running", - "execution_date": self.default_time_2, "logical_date": self.default_time_2, "external_trigger": True, "start_date": self.default_time, @@ -797,7 +788,6 @@ def test_order_by_descending_works(self): "dag_run_id": "TEST_DAG_RUN_ID_1", "end_date": None, "state": "running", - "execution_date": self.default_time, "logical_date": self.default_time, "external_trigger": True, "start_date": self.default_time, @@ -930,7 +920,7 @@ def _create_dag_runs(self, count): run_id=f"TEST_DAG_RUN_ID{i}", state="running", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time) + timedelta(minutes=i), + logical_date=timezone.parse(self.default_time) + timedelta(minutes=i), start_date=timezone.parse(self.default_time), external_trigger=True, ) @@ -1013,7 +1003,7 @@ def _create_dag_runs(self): dag_id="TEST_DAG_ID", run_id=f"TEST_START_EXEC_DAY_1{i}", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(date), + logical_date=timezone.parse(date), start_date=timezone.parse(date), external_trigger=True, state="success", @@ -1089,7 +1079,7 @@ def test_end_date_gte_lte(self, payload, expected_dag_run_ids): class TestPostDagRun(TestDagRunEndpoint): @time_machine.travel(timezone.utcnow(), tick=False) - @pytest.mark.parametrize("logical_date_field_name", ["execution_date", "logical_date"]) + @pytest.mark.parametrize("logical_date_field_name", ["logical_date"]) @pytest.mark.parametrize( "dag_run_id, logical_date, note, data_interval_start, data_interval_end", [ @@ -1164,7 +1154,6 @@ def test_should_respond_200( "dag_id": "TEST_DAG_ID", "dag_run_id": expected_dag_run_id, "end_date": None, - "execution_date": expected_logical_date, "logical_date": expected_logical_date, "external_trigger": True, "start_date": None, @@ -1178,7 +1167,7 @@ def test_should_respond_200( expected_response_json.update({"triggered_by": "rest_api"} if AIRFLOW_V_3_0_PLUS else {}) assert response.json == expected_response_json - _check_last_log(session, dag_id="TEST_DAG_ID", event="api.post_dag_run", execution_date=None) + _check_last_log(session, dag_id="TEST_DAG_ID", event="api.post_dag_run", logical_date=None) def test_raises_validation_error_for_invalid_request(self): self._create_dag("TEST_DAG_ID") @@ -1216,7 +1205,7 @@ def test_dagrun_creation_exception_is_handled(self, mock_get_dag_version, mock_g mock_get_dag_version.get_latest_version.return_value = mock.MagicMock() response = self.client.post( "api/v1/dags/TEST_DAG_ID/dagRuns", - json={"execution_date": "2020-11-10T08:25:56Z"}, + json={"logical_date": "2020-11-10T08:25:56Z"}, environ_overrides={"REMOTE_USER": "test"}, ) assert response.status_code == 400 @@ -1257,14 +1246,12 @@ def test_should_respond_400_if_a_dag_has_import_errors(self, session): "type": EXCEPTIONS_LINK_MAP[400], } == response.json - def test_should_response_200_for_matching_execution_date_logical_date(self): - execution_date = "2020-11-10T08:25:56.939143+00:00" + def test_should_response_200_for_matching_logical_date(self): logical_date = "2020-11-10T08:25:56.939143+00:00" self._create_dag("TEST_DAG_ID") response = self.client.post( "api/v1/dags/TEST_DAG_ID/dagRuns", json={ - "execution_date": execution_date, "logical_date": logical_date, }, environ_overrides={"REMOTE_USER": "test"}, @@ -1276,7 +1263,6 @@ def test_should_response_200_for_matching_execution_date_logical_date(self): "dag_id": "TEST_DAG_ID", "dag_run_id": dag_run_id, "end_date": None, - "execution_date": execution_date, "logical_date": logical_date, "external_trigger": True, "start_date": None, @@ -1292,19 +1278,6 @@ def test_should_response_200_for_matching_execution_date_logical_date(self): assert response.status_code == 200 assert response.json == expected_response_json - def test_should_response_400_for_conflicting_execution_date_logical_date(self): - execution_date = "2020-11-10T08:25:56.939143+00:00" - logical_date = "2020-11-11T08:25:56.939143+00:00" - self._create_dag("TEST_DAG_ID") - response = self.client.post( - "api/v1/dags/TEST_DAG_ID/dagRuns", - json={"execution_date": execution_date, "logical_date": logical_date}, - environ_overrides={"REMOTE_USER": "test"}, - ) - assert response.status_code == 400 - assert response.json["title"] == "logical_date conflicts with execution_date" - assert response.json["detail"] == (f"'{logical_date}' != '{execution_date}'") - @pytest.mark.parametrize( "data_interval_start, data_interval_end, expected", [ @@ -1337,7 +1310,7 @@ def test_should_response_400_for_missing_start_date_or_end_date( response = self.client.post( "api/v1/dags/TEST_DAG_ID/dagRuns", json={ - "execution_date": "2020-11-10T08:25:56.939143+00:00", + "logical_date": "2020-11-10T08:25:56.939143+00:00", "data_interval_start": data_interval_start, "data_interval_end": data_interval_end, }, @@ -1349,14 +1322,6 @@ def test_should_response_400_for_missing_start_date_or_end_date( @pytest.mark.parametrize( "data, expected", [ - ( - {"execution_date": "2020-11-10T08:25:56.939143"}, - "'2020-11-10T08:25:56.939143' is not a 'date-time' - 'execution_date'", - ), - ( - {"execution_date": "2020-11-10T08:25:56P"}, - "'2020-11-10T08:25:56P' is not a 'date-time' - 'execution_date'", - ), ( {"logical_date": "2020-11-10T08:25:56.939143"}, "'2020-11-10T08:25:56.939143' is not a 'date-time' - 'logical_date'", @@ -1381,7 +1346,7 @@ def test_should_response_400_for_naive_datetime_and_bad_datetime(self, data, exp ( { "dag_run_id": "TEST_DAG_RUN", - "execution_date": "2020-06-11T18:00:00+00:00", + "logical_date": "2020-06-11T18:00:00+00:00", "conf": "some string", }, "'some string' is not of type 'object' - 'conf'", @@ -1399,7 +1364,7 @@ def test_should_response_400_for_non_dict_dagrun_conf(self, data, expected): def test_response_404(self): response = self.client.post( "api/v1/dags/TEST_DAG_ID/dagRuns", - json={"dag_run_id": "TEST_DAG_RUN", "execution_date": self.default_time}, + json={"dag_run_id": "TEST_DAG_RUN", "logical_date": self.default_time}, environ_overrides={"REMOTE_USER": "test"}, ) assert response.status_code == 404 @@ -1417,7 +1382,7 @@ def test_response_404(self): "api/v1/dags/TEST_DAG_ID/dagRuns", { "start_date": "2020-06-11T18:00:00+00:00", - "execution_date": "2020-06-12T18:00:00+00:00", + "logical_date": "2020-06-12T18:00:00+00:00", }, { "detail": "Property is read-only - 'start_date'", @@ -1429,7 +1394,7 @@ def test_response_404(self): ), pytest.param( "api/v1/dags/TEST_DAG_ID/dagRuns", - {"state": "failed", "execution_date": "2020-06-12T18:00:00+00:00"}, + {"state": "failed", "logical_date": "2020-06-12T18:00:00+00:00"}, { "detail": "Property is read-only - 'state'", "status": 400, @@ -1452,7 +1417,7 @@ def test_response_409(self): "api/v1/dags/TEST_DAG_ID/dagRuns", json={ "dag_run_id": "TEST_DAG_RUN_ID_1", - "execution_date": self.default_time_3, + "logical_date": self.default_time_3, }, environ_overrides={"REMOTE_USER": "test"}, ) @@ -1465,14 +1430,14 @@ def test_response_409(self): "type": EXCEPTIONS_LINK_MAP[409], } - def test_response_409_when_execution_date_is_same(self): + def test_response_409_when_logical_date_is_same(self): self._create_test_dag_run() response = self.client.post( "api/v1/dags/TEST_DAG_ID/dagRuns", json={ "dag_run_id": "TEST_DAG_RUN_ID_6", - "execution_date": self.default_time, + "logical_date": self.default_time, }, environ_overrides={"REMOTE_USER": "test"}, ) @@ -1491,7 +1456,7 @@ def test_should_raises_401_unauthenticated(self): "api/v1/dags/TEST_DAG_ID/dagRuns", json={ "dag_run_id": "TEST_DAG_RUN_ID_1", - "execution_date": self.default_time, + "logical_date": self.default_time, }, ) @@ -1503,7 +1468,7 @@ def test_should_raises_403_unauthorized(self): "api/v1/dags/TEST_DAG_ID/dagRuns", json={ "dag_run_id": "TEST_DAG_RUN_ID_1", - "execution_date": self.default_time, + "logical_date": self.default_time, }, environ_overrides={"REMOTE_USER": "test_no_permissions"}, ) @@ -1544,9 +1509,8 @@ def test_should_respond_200(self, state, run_type, dag_maker, session): "dag_id": dag_id, "dag_run_id": dag_run_id, "end_date": dr.end_date.isoformat() if state != State.QUEUED else None, - "execution_date": dr.execution_date.isoformat(), + "logical_date": dr.logical_date.isoformat(), "external_trigger": False, - "logical_date": dr.execution_date.isoformat(), "start_date": dr.start_date.isoformat() if state != State.QUEUED else None, "state": state, "data_interval_start": dr.data_interval_start.isoformat(), @@ -1664,7 +1628,6 @@ def test_should_respond_200(self, dag_maker, session): "dag_id": dag_id, "dag_run_id": dag_run_id, "end_date": None, - "execution_date": dr.execution_date.isoformat(), "external_trigger": False, "logical_date": dr.logical_date.isoformat(), "start_date": None, @@ -1731,7 +1694,7 @@ def test_dry_run(self, dag_maker, session): { "dag_id": dag_id, "dag_run_id": dag_run_id, - "execution_date": dr.execution_date.isoformat(), + "logical_date": dr.logical_date.isoformat(), "task_id": "task_id", } ] @@ -1856,7 +1819,7 @@ def test_should_raises_401_unauthenticated(self, session): dag_id="TEST_DAG_ID", run_id="TEST_DAG_RUN_ID", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time), + logical_date=timezone.parse(self.default_time), start_date=timezone.parse(self.default_time), external_trigger=True, ) @@ -1887,7 +1850,6 @@ def test_should_respond_200(self, dag_maker, session): "dag_id": dr.dag_id, "dag_run_id": dr.run_id, "end_date": dr.end_date.isoformat(), - "execution_date": self.default_time, "external_trigger": True, "logical_date": self.default_time, "start_date": self.default_time, @@ -1917,9 +1879,8 @@ def test_should_respond_200(self, dag_maker, session): "dag_id": dr.dag_id, "dag_run_id": dr.run_id, "end_date": dr.end_date.isoformat(), - "execution_date": self.default_time, - "external_trigger": True, "logical_date": self.default_time, + "external_trigger": True, "start_date": self.default_time, "state": "success", "data_interval_start": None, @@ -1937,7 +1898,7 @@ def test_should_respond_200(self, dag_maker, session): session, dag_id=dr.dag_id, event="api.set_dag_run_note", - execution_date=None, + logical_date=None, expected_extra=payload, ) diff --git a/tests/api_connexion/endpoints/test_dag_stats_endpoint.py b/tests/api_connexion/endpoints/test_dag_stats_endpoint.py index 90db64560a9f..6e3b0ab90d6a 100644 --- a/tests/api_connexion/endpoints/test_dag_stats_endpoint.py +++ b/tests/api_connexion/endpoints/test_dag_stats_endpoint.py @@ -84,7 +84,7 @@ def _create_dag_runs(self, session): dag_id="dag_stats_dag", run_id="test_dag_run_id_1", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time), + logical_date=timezone.parse(self.default_time), start_date=timezone.parse(self.default_time), external_trigger=True, state="running", @@ -93,7 +93,7 @@ def _create_dag_runs(self, session): dag_id="dag_stats_dag", run_id="test_dag_run_id_2", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time) + timedelta(days=1), + logical_date=timezone.parse(self.default_time) + timedelta(days=1), start_date=timezone.parse(self.default_time), external_trigger=True, state="failed", @@ -102,7 +102,7 @@ def _create_dag_runs(self, session): dag_id="dag_stats_dag_2", run_id="test_dag_2_run_id_1", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time), + logical_date=timezone.parse(self.default_time), start_date=timezone.parse(self.default_time), external_trigger=True, state="queued", @@ -111,7 +111,7 @@ def _create_dag_runs(self, session): dag_id="dag_stats_dag_3", run_id="test_dag_3_run_id_1", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time), + logical_date=timezone.parse(self.default_time), start_date=timezone.parse(self.default_time), external_trigger=True, state="success", diff --git a/tests/api_connexion/endpoints/test_event_log_endpoint.py b/tests/api_connexion/endpoints/test_event_log_endpoint.py index 00b5f7e66877..06c9612d8346 100644 --- a/tests/api_connexion/endpoints/test_event_log_endpoint.py +++ b/tests/api_connexion/endpoints/test_event_log_endpoint.py @@ -52,7 +52,7 @@ def task_instance(session, create_task_instance, request): dag_id="TEST_DAG_ID", task_id="TEST_TASK_ID", run_id="TEST_RUN_ID", - execution_date=request.instance.default_time, + logical_date=request.instance.default_time, ) @@ -111,7 +111,7 @@ def test_should_respond_200(self, log_model): "run_id": "TEST_RUN_ID", "map_index": -1, "try_number": 0, - "execution_date": self.default_time.isoformat(), + "logical_date": self.default_time.isoformat(), "owner": "airflow", "when": self.default_time.isoformat(), "extra": None, @@ -162,7 +162,7 @@ def test_should_respond_200(self, session, create_log_model): "run_id": "TEST_RUN_ID", "map_index": -1, "try_number": 0, - "execution_date": self.default_time.isoformat(), + "logical_date": self.default_time.isoformat(), "owner": "airflow", "when": self.default_time.isoformat(), "extra": None, @@ -175,7 +175,7 @@ def test_should_respond_200(self, session, create_log_model): "run_id": "TEST_RUN_ID", "map_index": -1, "try_number": 0, - "execution_date": self.default_time.isoformat(), + "logical_date": self.default_time.isoformat(), "owner": "airflow", "when": self.default_time_2.isoformat(), "extra": None, @@ -188,7 +188,7 @@ def test_should_respond_200(self, session, create_log_model): "run_id": None, "map_index": None, "try_number": None, - "execution_date": None, + "logical_date": None, "owner": "root", "when": self.default_time_2.isoformat(), "extra": '{"host_name": "e24b454f002a"}', @@ -218,7 +218,7 @@ def test_order_eventlogs_by_owner(self, create_log_model, session): "run_id": "TEST_RUN_ID", "map_index": -1, "try_number": 0, - "execution_date": self.default_time.isoformat(), + "logical_date": self.default_time.isoformat(), "owner": "zsh", # Order by name, sort order is descending(-) "when": self.default_time_2.isoformat(), "extra": None, @@ -231,7 +231,7 @@ def test_order_eventlogs_by_owner(self, create_log_model, session): "run_id": None, "map_index": None, "try_number": None, - "execution_date": None, + "logical_date": None, "owner": "root", "when": self.default_time_2.isoformat(), "extra": '{"host_name": "e24b454f002a"}', @@ -244,7 +244,7 @@ def test_order_eventlogs_by_owner(self, create_log_model, session): "run_id": "TEST_RUN_ID", "map_index": -1, "try_number": 0, - "execution_date": self.default_time.isoformat(), + "logical_date": self.default_time.isoformat(), "owner": "airflow", "when": self.default_time.isoformat(), "extra": None, diff --git a/tests/api_connexion/endpoints/test_extra_link_endpoint.py b/tests/api_connexion/endpoints/test_extra_link_endpoint.py index 800006391207..a757e15ecee4 100644 --- a/tests/api_connexion/endpoints/test_extra_link_endpoint.py +++ b/tests/api_connexion/endpoints/test_extra_link_endpoint.py @@ -79,7 +79,7 @@ def setup_attrs(self, configured_app, session) -> None: triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} self.dag.create_dagrun( run_id="TEST_DAG_RUN_ID", - execution_date=self.default_time, + logical_date=self.default_time, run_type=DagRunType.MANUAL, state=DagRunState.SUCCESS, session=session, diff --git a/tests/api_connexion/endpoints/test_log_endpoint.py b/tests/api_connexion/endpoints/test_log_endpoint.py index 248b8a46872b..79c968b54057 100644 --- a/tests/api_connexion/endpoints/test_log_endpoint.py +++ b/tests/api_connexion/endpoints/test_log_endpoint.py @@ -84,7 +84,7 @@ def add_one(x: int): dr = dag_maker.create_dagrun( run_id=self.RUN_ID, run_type=DagRunType.SCHEDULED, - execution_date=timezone.parse(self.default_time), + logical_date=timezone.parse(self.default_time), start_date=timezone.parse(self.default_time), ) @@ -98,7 +98,7 @@ def add_one(x: int): dr2 = dag_maker.create_dagrun( run_id=self.RUN_ID, run_type=DagRunType.SCHEDULED, - execution_date=timezone.parse(self.default_time), + logical_date=timezone.parse(self.default_time), start_date=timezone.parse(self.default_time), ) configured_app.dag_bag.bag_dag(dummy_dag) diff --git a/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py b/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py index 68ecd1e83898..9027c29e5c87 100644 --- a/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py +++ b/tests/api_connexion/endpoints/test_mapped_task_instance_endpoint.py @@ -67,7 +67,7 @@ class TestMappedTaskInstanceEndpoint: def setup_attrs(self, configured_app) -> None: self.default_time = DEFAULT_DATETIME_1 self.ti_init = { - "execution_date": self.default_time, + "logical_date": self.default_time, "state": State.RUNNING, } self.ti_extras = { @@ -210,7 +210,7 @@ def test_mapped_task_instances(self, one_task_with_mapped_tis, session): "dag_run_id": "run_mapped_tis", "duration": None, "end_date": None, - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "executor": "default", "executor_config": "{}", "hostname": "", diff --git a/tests/api_connexion/endpoints/test_pool_endpoint.py b/tests/api_connexion/endpoints/test_pool_endpoint.py index dfe9ded92980..7b639b65c646 100644 --- a/tests/api_connexion/endpoints/test_pool_endpoint.py +++ b/tests/api_connexion/endpoints/test_pool_endpoint.py @@ -262,7 +262,7 @@ def test_response_204(self, session): # Check if the pool is deleted from the db response = self.client.get(f"api/v1/pools/{pool_name}", environ_overrides={"REMOTE_USER": "test"}) assert response.status_code == 404 - _check_last_log(session, dag_id=None, event="api.delete_pool", execution_date=None) + _check_last_log(session, dag_id=None, event="api.delete_pool", logical_date=None) def test_response_404(self): response = self.client.delete("api/v1/pools/invalid_pool", environ_overrides={"REMOTE_USER": "test"}) @@ -309,7 +309,7 @@ def test_response_200(self, session): "description": "test pool", "include_deferred": True, } == response.json - _check_last_log(session, dag_id=None, event="api.post_pool", execution_date=None) + _check_last_log(session, dag_id=None, event="api.post_pool", logical_date=None) def test_response_409(self, session): pool_name = "test_pool_a" @@ -395,7 +395,7 @@ def test_response_200(self, session): "description": None, "include_deferred": False, } == response.json - _check_last_log(session, dag_id=None, event="api.patch_pool", execution_date=None) + _check_last_log(session, dag_id=None, event="api.patch_pool", logical_date=None) @pytest.mark.parametrize( "error_detail, request_json", @@ -592,7 +592,7 @@ def test_patch(self, status_code, url, json, expected_response, session): response = self.client.patch(url, json=json, environ_overrides={"REMOTE_USER": "test"}) assert response.status_code == status_code assert response.json == expected_response - _check_last_log(session, dag_id=None, event="api.patch_pool", execution_date=None) + _check_last_log(session, dag_id=None, event="api.patch_pool", logical_date=None) class TestPatchPoolWithUpdateMask(TestBasePoolEndpoints): @@ -657,7 +657,7 @@ def test_response_200( "description": None, "include_deferred": expected_include_deferred, } == response.json - _check_last_log(session, dag_id=None, event="api.patch_pool", execution_date=None) + _check_last_log(session, dag_id=None, event="api.patch_pool", logical_date=None) @pytest.mark.parametrize( "error_detail, url, patch_json", diff --git a/tests/api_connexion/endpoints/test_task_instance_endpoint.py b/tests/api_connexion/endpoints/test_task_instance_endpoint.py index 1eeefd698e28..a14a0be33a59 100644 --- a/tests/api_connexion/endpoints/test_task_instance_endpoint.py +++ b/tests/api_connexion/endpoints/test_task_instance_endpoint.py @@ -71,7 +71,7 @@ class TestTaskInstanceEndpoint: def setup_attrs(self, configured_app, dagbag) -> None: self.default_time = DEFAULT_DATETIME_1 self.ti_init = { - "execution_date": self.default_time, + "logical_date": self.default_time, "state": State.RUNNING, } self.ti_extras = { @@ -106,7 +106,7 @@ def create_task_instances( counter = min(len(task_instances), counter) run_id = "TEST_DAG_RUN_ID" - execution_date = self.ti_init.pop("execution_date", self.default_time) + logical_date = self.ti_init.pop("logical_date", self.default_time) dr = None tis = [] @@ -118,16 +118,16 @@ def create_task_instances( else: self.ti_init.update(task_instances[i]) - if "execution_date" in self.ti_init: + if "logical_date" in self.ti_init: run_id = f"TEST_DAG_RUN_ID_{i}" - execution_date = self.ti_init.pop("execution_date") + logical_date = self.ti_init.pop("logical_date") dr = None if not dr: dr = DagRun( run_id=run_id, dag_id=dag_id, - execution_date=execution_date, + logical_date=logical_date, run_type=DagRunType.MANUAL, state=dag_run_state, ) @@ -182,7 +182,7 @@ def test_should_respond_200(self, session): "dag_id": "example_python_operator", "duration": 10000.0, "end_date": "2020-01-03T00:00:00+00:00", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "executor": None, "executor_config": "{}", "hostname": "", @@ -240,7 +240,7 @@ def test_should_respond_200_with_task_state_in_deferred(self, session): "dag_id": "example_python_operator", "duration": 10000.0, "end_date": "2020-01-03T00:00:00+00:00", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "executor": None, "executor_config": "{}", "hostname": "", @@ -287,7 +287,7 @@ def test_should_respond_200_with_task_state_in_removed(self, session): "dag_id": "example_python_operator", "duration": 10000.0, "end_date": "2020-01-03T00:00:00+00:00", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "executor": None, "executor_config": "{}", "hostname": "", @@ -330,7 +330,7 @@ def test_should_respond_200_task_instance_with_rendered(self, session): "dag_id": "example_python_operator", "duration": 10000.0, "end_date": "2020-01-03T00:00:00+00:00", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "executor": None, "executor_config": "{}", "hostname": "", @@ -382,7 +382,7 @@ def test_should_respond_200_mapped_task_instance_with_rtif(self, session): "dag_id": "example_python_operator", "duration": 10000.0, "end_date": "2020-01-03T00:00:00+00:00", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "executor": None, "executor_config": "{}", "hostname": "", @@ -464,9 +464,9 @@ class TestGetTaskInstances(TestTaskInstanceEndpoint): [ pytest.param( [ - {"execution_date": DEFAULT_DATETIME_1}, - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1)}, - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2)}, + {"logical_date": DEFAULT_DATETIME_1}, + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1)}, + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2)}, ], False, ( @@ -474,7 +474,7 @@ class TestGetTaskInstances(TestTaskInstanceEndpoint): f"taskInstances?execution_date_lte={QUOTED_DEFAULT_DATETIME_STR_1}" ), 1, - id="test execution date filter", + id="test logical date filter", ), pytest.param( [ @@ -655,7 +655,7 @@ def test_return_TI_only_from_readable_dags(self, session): self.create_task_instances( session, task_instances=[ - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=i)} + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=i)} for i in range(task_instances[dag_id]) ], dag_id=dag_id, @@ -810,12 +810,12 @@ class TestGetTaskInstancesBatch(TestTaskInstanceEndpoint): ), pytest.param( [ - {"execution_date": DEFAULT_DATETIME_1}, - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1)}, - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2)}, - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=3)}, - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=4)}, - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=5)}, + {"logical_date": DEFAULT_DATETIME_1}, + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1)}, + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2)}, + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=3)}, + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=4)}, + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=5)}, ], False, { @@ -824,14 +824,14 @@ class TestGetTaskInstancesBatch(TestTaskInstanceEndpoint): }, 3, "test", - id="with execution date filter", + id="with logical date filter", ), pytest.param( [ - {"execution_date": DEFAULT_DATETIME_1}, - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1)}, - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2)}, - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=3)}, + {"logical_date": DEFAULT_DATETIME_1}, + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1)}, + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2)}, + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=3)}, ], False, { @@ -843,10 +843,10 @@ class TestGetTaskInstancesBatch(TestTaskInstanceEndpoint): ), pytest.param( [ - {"execution_date": DEFAULT_DATETIME_1}, - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1)}, - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2)}, - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=3)}, + {"logical_date": DEFAULT_DATETIME_1}, + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1)}, + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2)}, + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=3)}, ], False, { @@ -1089,13 +1089,13 @@ class TestPostClearTaskInstances(TestTaskInstanceEndpoint): pytest.param( "example_python_operator", [ - {"execution_date": DEFAULT_DATETIME_1, "state": State.FAILED}, + {"logical_date": DEFAULT_DATETIME_1, "state": State.FAILED}, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), "state": State.FAILED, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), "state": State.FAILED, }, ], @@ -1111,13 +1111,13 @@ class TestPostClearTaskInstances(TestTaskInstanceEndpoint): pytest.param( "example_python_operator", [ - {"execution_date": DEFAULT_DATETIME_1, "state": State.FAILED}, + {"logical_date": DEFAULT_DATETIME_1, "state": State.FAILED}, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), "state": State.FAILED, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), "state": State.FAILED, }, ], @@ -1133,13 +1133,13 @@ class TestPostClearTaskInstances(TestTaskInstanceEndpoint): pytest.param( "example_python_operator", [ - {"execution_date": DEFAULT_DATETIME_1, "state": State.RUNNING}, + {"logical_date": DEFAULT_DATETIME_1, "state": State.RUNNING}, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), "state": State.RUNNING, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), "state": State.FAILED, }, ], @@ -1151,13 +1151,13 @@ class TestPostClearTaskInstances(TestTaskInstanceEndpoint): pytest.param( "example_python_operator", [ - {"execution_date": DEFAULT_DATETIME_1, "state": State.FAILED}, + {"logical_date": DEFAULT_DATETIME_1, "state": State.FAILED}, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), "state": State.FAILED, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), "state": State.RUNNING, }, ], @@ -1172,17 +1172,17 @@ class TestPostClearTaskInstances(TestTaskInstanceEndpoint): pytest.param( "example_python_operator", [ - {"execution_date": DEFAULT_DATETIME_1, "state": State.FAILED}, + {"logical_date": DEFAULT_DATETIME_1, "state": State.FAILED}, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), "state": State.FAILED, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), "state": State.FAILED, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=3), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=3), "state": State.FAILED, }, ], @@ -1197,13 +1197,13 @@ class TestPostClearTaskInstances(TestTaskInstanceEndpoint): pytest.param( "example_python_operator", [ - {"execution_date": DEFAULT_DATETIME_1, "state": State.FAILED}, + {"logical_date": DEFAULT_DATETIME_1, "state": State.FAILED}, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), "state": State.FAILED, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), "state": State.RUNNING, }, ], @@ -1236,7 +1236,7 @@ def test_should_respond_200(self, main_dag, task_instances, request_dag, payload session, dag_id=request_dag, event="api.post_clear_task_instances", - execution_date=None, + logical_date=None, expected_extra=payload, ) @@ -1256,7 +1256,7 @@ def test_clear_taskinstance_is_called_with_queued_dr_state(self, mock_clearti, s mock_clearti.assert_called_once_with( [], session, dag=self.app.dag_bag.get_dag(dag_id), dag_run_state=State.QUEUED ) - _check_last_log(session, dag_id=dag_id, event="api.post_clear_task_instances", execution_date=None) + _check_last_log(session, dag_id=dag_id, event="api.post_clear_task_instances", logical_date=None) def test_clear_taskinstance_is_called_with_invalid_task_ids(self, session): """Test that dagrun is running when invalid task_ids are passed to clearTaskInstances API.""" @@ -1277,7 +1277,7 @@ def test_clear_taskinstance_is_called_with_invalid_task_ids(self, session): dagrun.refresh_from_db() assert dagrun.state == "running" assert all(ti.state == "running" for ti in tis) - _check_last_log(session, dag_id=dag_id, event="api.post_clear_task_instances", execution_date=None) + _check_last_log(session, dag_id=dag_id, event="api.post_clear_task_instances", logical_date=None) def test_should_respond_200_with_reset_dag_run(self, session): dag_id = "example_python_operator" @@ -1288,25 +1288,25 @@ def test_should_respond_200_with_reset_dag_run(self, session): "only_running": True, } task_instances = [ - {"execution_date": DEFAULT_DATETIME_1, "state": State.RUNNING}, + {"logical_date": DEFAULT_DATETIME_1, "state": State.RUNNING}, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), "state": State.RUNNING, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), "state": State.RUNNING, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=3), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=3), "state": State.RUNNING, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=4), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=4), "state": State.RUNNING, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=5), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=5), "state": State.RUNNING, }, ] @@ -1330,37 +1330,37 @@ def test_should_respond_200_with_reset_dag_run(self, session): { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_0", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "task_id": "print_the_context", }, { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_1", - "execution_date": "2020-01-02T00:00:00+00:00", + "logical_date": "2020-01-02T00:00:00+00:00", "task_id": "log_sql_query", }, { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_2", - "execution_date": "2020-01-03T00:00:00+00:00", + "logical_date": "2020-01-03T00:00:00+00:00", "task_id": "sleep_for_0", }, { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_3", - "execution_date": "2020-01-04T00:00:00+00:00", + "logical_date": "2020-01-04T00:00:00+00:00", "task_id": "sleep_for_1", }, { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_4", - "execution_date": "2020-01-05T00:00:00+00:00", + "logical_date": "2020-01-05T00:00:00+00:00", "task_id": "sleep_for_2", }, { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_5", - "execution_date": "2020-01-06T00:00:00+00:00", + "logical_date": "2020-01-06T00:00:00+00:00", "task_id": "sleep_for_3", }, ] @@ -1368,7 +1368,7 @@ def test_should_respond_200_with_reset_dag_run(self, session): assert task_instance in response.json["task_instances"] assert 6 == len(response.json["task_instances"]) assert 0 == failed_dag_runs, 0 - _check_last_log(session, dag_id=dag_id, event="api.post_clear_task_instances", execution_date=None) + _check_last_log(session, dag_id=dag_id, event="api.post_clear_task_instances", logical_date=None) def test_should_respond_200_with_dag_run_id(self, session): dag_id = "example_python_operator" @@ -1380,25 +1380,25 @@ def test_should_respond_200_with_dag_run_id(self, session): "dag_run_id": "TEST_DAG_RUN_ID_0", } task_instances = [ - {"execution_date": DEFAULT_DATETIME_1, "state": State.RUNNING}, + {"logical_date": DEFAULT_DATETIME_1, "state": State.RUNNING}, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), "state": State.RUNNING, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), "state": State.RUNNING, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=3), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=3), "state": State.RUNNING, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=4), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=4), "state": State.RUNNING, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=5), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=5), "state": State.RUNNING, }, ] @@ -1420,13 +1420,13 @@ def test_should_respond_200_with_dag_run_id(self, session): { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_0", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "task_id": "print_the_context", }, ] assert response.json["task_instances"] == expected_response assert 1 == len(response.json["task_instances"]) - _check_last_log(session, dag_id=dag_id, event="api.post_clear_task_instances", execution_date=None) + _check_last_log(session, dag_id=dag_id, event="api.post_clear_task_instances", logical_date=None) def test_should_respond_200_with_include_past(self, session): dag_id = "example_python_operator" @@ -1438,25 +1438,25 @@ def test_should_respond_200_with_include_past(self, session): "only_running": True, } task_instances = [ - {"execution_date": DEFAULT_DATETIME_1, "state": State.RUNNING}, + {"logical_date": DEFAULT_DATETIME_1, "state": State.RUNNING}, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), "state": State.RUNNING, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), "state": State.RUNNING, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=3), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=3), "state": State.RUNNING, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=4), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=4), "state": State.RUNNING, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=5), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=5), "state": State.RUNNING, }, ] @@ -1478,44 +1478,44 @@ def test_should_respond_200_with_include_past(self, session): { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_0", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "task_id": "print_the_context", }, { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_1", - "execution_date": "2020-01-02T00:00:00+00:00", + "logical_date": "2020-01-02T00:00:00+00:00", "task_id": "log_sql_query", }, { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_2", - "execution_date": "2020-01-03T00:00:00+00:00", + "logical_date": "2020-01-03T00:00:00+00:00", "task_id": "sleep_for_0", }, { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_3", - "execution_date": "2020-01-04T00:00:00+00:00", + "logical_date": "2020-01-04T00:00:00+00:00", "task_id": "sleep_for_1", }, { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_4", - "execution_date": "2020-01-05T00:00:00+00:00", + "logical_date": "2020-01-05T00:00:00+00:00", "task_id": "sleep_for_2", }, { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_5", - "execution_date": "2020-01-06T00:00:00+00:00", + "logical_date": "2020-01-06T00:00:00+00:00", "task_id": "sleep_for_3", }, ] for task_instance in expected_response: assert task_instance in response.json["task_instances"] assert 6 == len(response.json["task_instances"]) - _check_last_log(session, dag_id=dag_id, event="api.post_clear_task_instances", execution_date=None) + _check_last_log(session, dag_id=dag_id, event="api.post_clear_task_instances", logical_date=None) def test_should_respond_200_with_include_future(self, session): dag_id = "example_python_operator" @@ -1527,25 +1527,25 @@ def test_should_respond_200_with_include_future(self, session): "only_running": False, } task_instances = [ - {"execution_date": DEFAULT_DATETIME_1, "state": State.SUCCESS}, + {"logical_date": DEFAULT_DATETIME_1, "state": State.SUCCESS}, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), "state": State.SUCCESS, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2), "state": State.SUCCESS, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=3), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=3), "state": State.SUCCESS, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=4), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=4), "state": State.SUCCESS, }, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=5), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=5), "state": State.SUCCESS, }, ] @@ -1568,44 +1568,44 @@ def test_should_respond_200_with_include_future(self, session): { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_0", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "task_id": "print_the_context", }, { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_1", - "execution_date": "2020-01-02T00:00:00+00:00", + "logical_date": "2020-01-02T00:00:00+00:00", "task_id": "log_sql_query", }, { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_2", - "execution_date": "2020-01-03T00:00:00+00:00", + "logical_date": "2020-01-03T00:00:00+00:00", "task_id": "sleep_for_0", }, { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_3", - "execution_date": "2020-01-04T00:00:00+00:00", + "logical_date": "2020-01-04T00:00:00+00:00", "task_id": "sleep_for_1", }, { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_4", - "execution_date": "2020-01-05T00:00:00+00:00", + "logical_date": "2020-01-05T00:00:00+00:00", "task_id": "sleep_for_2", }, { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID_5", - "execution_date": "2020-01-06T00:00:00+00:00", + "logical_date": "2020-01-06T00:00:00+00:00", "task_id": "sleep_for_3", }, ] for task_instance in expected_response: assert task_instance in response.json["task_instances"] assert 6 == len(response.json["task_instances"]) - _check_last_log(session, dag_id=dag_id, event="api.post_clear_task_instances", execution_date=None) + _check_last_log(session, dag_id=dag_id, event="api.post_clear_task_instances", logical_date=None) def test_should_respond_404_for_nonexistent_dagrun_id(self, session): dag_id = "example_python_operator" @@ -1617,9 +1617,9 @@ def test_should_respond_404_for_nonexistent_dagrun_id(self, session): "dag_run_id": "TEST_DAG_RUN_ID_100", } task_instances = [ - {"execution_date": DEFAULT_DATETIME_1, "state": State.RUNNING}, + {"logical_date": DEFAULT_DATETIME_1, "state": State.RUNNING}, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), "state": State.RUNNING, }, ] @@ -1642,7 +1642,7 @@ def test_should_respond_404_for_nonexistent_dagrun_id(self, session): response.json["title"] == "Dag Run id TEST_DAG_RUN_ID_100 not found in dag example_python_operator" ) - _check_last_log(session, dag_id=dag_id, event="api.post_clear_task_instances", execution_date=None) + _check_last_log(session, dag_id=dag_id, event="api.post_clear_task_instances", logical_date=None) def test_should_raises_401_unauthenticated(self): response = self.client.post( @@ -1681,9 +1681,9 @@ def test_should_raise_403_forbidden(self): @provide_session def test_should_raise_400_for_naive_and_bad_datetime(self, payload, expected, session): task_instances = [ - {"execution_date": DEFAULT_DATETIME_1, "state": State.RUNNING}, + {"logical_date": DEFAULT_DATETIME_1, "state": State.RUNNING}, { - "execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), + "logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1), "state": State.RUNNING, }, ] @@ -1734,7 +1734,7 @@ def test_should_assert_call_mocked_api(self, mock_set_task_instance_state, sessi json={ "dry_run": True, "task_id": "print_the_context", - "execution_date": DEFAULT_DATETIME_1.isoformat(), + "logical_date": DEFAULT_DATETIME_1.isoformat(), "include_upstream": True, "include_downstream": True, "include_future": True, @@ -1748,7 +1748,7 @@ def test_should_assert_call_mocked_api(self, mock_set_task_instance_state, sessi { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "task_id": "print_the_context", } ] @@ -1758,7 +1758,7 @@ def test_should_assert_call_mocked_api(self, mock_set_task_instance_state, sessi commit=False, downstream=True, run_id=None, - execution_date=DEFAULT_DATETIME_1, + logical_date=DEFAULT_DATETIME_1, future=True, past=True, state="failed", @@ -1797,7 +1797,7 @@ def test_should_assert_call_mocked_api_when_run_id(self, mock_set_task_instance_ { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "task_id": "print_the_context", } ] @@ -1807,7 +1807,7 @@ def test_should_assert_call_mocked_api_when_run_id(self, mock_set_task_instance_ commit=False, downstream=True, run_id=run_id, - execution_date=None, + logical_date=None, future=True, past=True, state="failed", @@ -1820,7 +1820,7 @@ def test_should_assert_call_mocked_api_when_run_id(self, mock_set_task_instance_ "error, code, payload", [ [ - "{'_schema': ['Exactly one of execution_date or dag_run_id must be provided']}", + "{'_schema': ['Exactly one of logical_date or dag_run_id must be provided']}", 400, { "dry_run": True, @@ -1833,13 +1833,13 @@ def test_should_assert_call_mocked_api_when_run_id(self, mock_set_task_instance_ }, ], [ - "Task instance not found for task 'print_the_context' on execution_date " + "Task instance not found for task 'print_the_context' on logical_date " "2021-01-01 00:00:00+00:00", 404, { "dry_run": True, "task_id": "print_the_context", - "execution_date": "2021-01-01T00:00:00+00:00", + "logical_date": "2021-01-01T00:00:00+00:00", "include_upstream": True, "include_downstream": True, "include_future": True, @@ -1862,13 +1862,13 @@ def test_should_assert_call_mocked_api_when_run_id(self, mock_set_task_instance_ }, ], [ - "{'_schema': ['Exactly one of execution_date or dag_run_id must be provided']}", + "{'_schema': ['Exactly one of logical_date or dag_run_id must be provided']}", 400, { "dry_run": True, "task_id": "print_the_context", "dag_run_id": "TEST_DAG_RUN_", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "include_upstream": True, "include_downstream": True, "include_future": True, @@ -1894,7 +1894,7 @@ def test_should_raises_401_unauthenticated(self): json={ "dry_run": True, "task_id": "print_the_context", - "execution_date": DEFAULT_DATETIME_1.isoformat(), + "logical_date": DEFAULT_DATETIME_1.isoformat(), "include_upstream": True, "include_downstream": True, "include_future": True, @@ -1911,7 +1911,7 @@ def test_should_raise_403_forbidden(self): json={ "dry_run": True, "task_id": "print_the_context", - "execution_date": DEFAULT_DATETIME_1.isoformat(), + "logical_date": DEFAULT_DATETIME_1.isoformat(), "include_upstream": True, "include_downstream": True, "include_future": True, @@ -1928,7 +1928,7 @@ def test_should_raise_404_not_found_dag(self): json={ "dry_run": True, "task_id": "print_the_context", - "execution_date": DEFAULT_DATETIME_1.isoformat(), + "logical_date": DEFAULT_DATETIME_1.isoformat(), "include_upstream": True, "include_downstream": True, "include_future": True, @@ -1939,7 +1939,7 @@ def test_should_raise_404_not_found_dag(self): assert response.status_code == 404 @mock.patch("airflow.models.dag.DAG.set_task_instance_state") - def test_should_raise_not_found_if_execution_date_is_wrong(self, mock_set_task_instance_state, session): + def test_should_raise_not_found_if_run_id_is_wrong(self, mock_set_task_instance_state, session): self.create_task_instances(session) date = DEFAULT_DATETIME_1 + dt.timedelta(days=1) response = self.client.post( @@ -1948,7 +1948,7 @@ def test_should_raise_not_found_if_execution_date_is_wrong(self, mock_set_task_i json={ "dry_run": True, "task_id": "print_the_context", - "execution_date": date.isoformat(), + "logical_date": date.isoformat(), "include_upstream": True, "include_downstream": True, "include_future": True, @@ -1958,7 +1958,7 @@ def test_should_raise_not_found_if_execution_date_is_wrong(self, mock_set_task_i ) assert response.status_code == 404 assert response.json["detail"] == ( - f"Task instance not found for task 'print_the_context' on execution_date {date}" + f"Task instance not found for task 'print_the_context' on logical_date {date}" ) assert mock_set_task_instance_state.call_count == 0 @@ -1969,7 +1969,7 @@ def test_should_raise_404_not_found_task(self): json={ "dry_run": True, "task_id": "INVALID_TASK", - "execution_date": DEFAULT_DATETIME_1.isoformat(), + "logical_date": DEFAULT_DATETIME_1.isoformat(), "include_upstream": True, "include_downstream": True, "include_future": True, @@ -1986,7 +1986,7 @@ def test_should_raise_404_not_found_task(self): { "dry_run": True, "task_id": "print_the_context", - "execution_date": "2020-11-10T12:42:39.442973", + "logical_date": "2020-11-10T12:42:39.442973", "include_upstream": True, "include_downstream": True, "include_future": True, @@ -1999,14 +1999,14 @@ def test_should_raise_404_not_found_task(self): { "dry_run": True, "task_id": "print_the_context", - "execution_date": "2020-11-10T12:4opfo", + "logical_date": "2020-11-10T12:4opfo", "include_upstream": True, "include_downstream": True, "include_future": True, "include_past": True, "new_state": "failed", }, - "{'execution_date': ['Not a valid datetime.']}", + "{'logical_date': ['Not a valid datetime.']}", ), ], ) @@ -2053,7 +2053,7 @@ def test_should_call_mocked_api(self, mock_set_task_instance_state, session): assert response.json == { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "task_id": "print_the_context", } @@ -2069,7 +2069,7 @@ def test_should_call_mocked_api(self, mock_set_task_instance_state, session): session, dag_id="example_python_operator", event="api.post_set_task_instances_state", - execution_date=None, + logical_date=None, ) @mock.patch("airflow.models.dag.DAG.set_task_instance_state") @@ -2098,7 +2098,7 @@ def test_should_not_call_mocked_api_for_dry_run(self, mock_set_task_instance_sta assert response.json == { "dag_id": "example_python_operator", "dag_run_id": "TEST_DAG_RUN_ID", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "task_id": "print_the_context", } @@ -2335,7 +2335,7 @@ def test_should_respond_200(self, session): "dag_id": "example_python_operator", "duration": 10000.0, "end_date": "2020-01-03T00:00:00+00:00", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "executor": None, "executor_config": "{}", "hostname": "", @@ -2364,7 +2364,7 @@ def test_should_respond_200(self, session): ti = session.scalars(select(TaskInstance).where(TaskInstance.task_id == "print_the_context")).one() assert ti.task_instance_note.user_id is not None _check_last_log( - session, dag_id="example_python_operator", event="api.set_task_instance_note", execution_date=None + session, dag_id="example_python_operator", event="api.set_task_instance_note", logical_date=None ) def test_should_respond_200_mapped_task_instance_with_rtif(self, session): @@ -2394,7 +2394,7 @@ def test_should_respond_200_mapped_task_instance_with_rtif(self, session): "dag_id": "example_python_operator", "duration": 10000.0, "end_date": "2020-01-03T00:00:00+00:00", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "executor": None, "executor_config": "{}", "hostname": "", @@ -2508,13 +2508,13 @@ def test_should_respond_empty_non_scheduled(self, session): { "dependencies": [ { - "name": "Execution Date", - "reason": "The execution date is 2020-01-01T00:00:00+00:00 but this is " + "name": "Logical Date", + "reason": "The logical date is 2020-01-01T00:00:00+00:00 but this is " "before the task's start date 2021-01-01T00:00:00+00:00.", }, { - "name": "Execution Date", - "reason": "The execution date is 2020-01-01T00:00:00+00:00 but this is " + "name": "Logical Date", + "reason": "The logical date is 2020-01-01T00:00:00+00:00 but this is " "before the task's DAG's start date 2021-01-01T00:00:00+00:00.", }, ], @@ -2525,12 +2525,12 @@ def test_should_respond_empty_non_scheduled(self, session): { "dependencies": [ { - "name": "Execution Date", - "reason": "The execution date is 2020-01-01T00:00:00+00:00 but this is before the task's start date 2021-01-01T00:00:00+00:00.", + "name": "Logical Date", + "reason": "The logical date is 2020-01-01T00:00:00+00:00 but this is before the task's start date 2021-01-01T00:00:00+00:00.", }, { - "name": "Execution Date", - "reason": "The execution date is 2020-01-01T00:00:00+00:00 but this is before the task's DAG's start date 2021-01-01T00:00:00+00:00.", + "name": "Logical Date", + "reason": "The logical date is 2020-01-01T00:00:00+00:00 but this is before the task's DAG's start date 2021-01-01T00:00:00+00:00.", }, {"name": "Task Instance State", "reason": "Task is in the 'None' state."}, ] diff --git a/tests/api_connexion/endpoints/test_variable_endpoint.py b/tests/api_connexion/endpoints/test_variable_endpoint.py index c3c1c91cb8ed..2daeba5ea761 100644 --- a/tests/api_connexion/endpoints/test_variable_endpoint.py +++ b/tests/api_connexion/endpoints/test_variable_endpoint.py @@ -74,7 +74,7 @@ def test_should_delete_variable(self, session): # make sure variable is deleted response = self.client.get("/api/v1/variables/delete_var1", environ_overrides={"REMOTE_USER": "test"}) assert response.status_code == 404 - _check_last_log(session, dag_id=None, event="api.variable.delete", execution_date=None) + _check_last_log(session, dag_id=None, event="api.variable.delete", logical_date=None) def test_should_respond_404_if_key_does_not_exist(self): response = self.client.delete( @@ -236,7 +236,7 @@ def test_should_update_variable(self, session): assert response.status_code == 200 assert response.json == {"key": "var1", "value": "updated", "description": None} _check_last_log( - session, dag_id=None, event="api.variable.edit", execution_date=None, expected_extra=payload + session, dag_id=None, event="api.variable.edit", logical_date=None, expected_extra=payload ) def test_should_update_variable_with_mask(self, session): @@ -248,7 +248,7 @@ def test_should_update_variable_with_mask(self, session): ) assert response.status_code == 200 assert response.json == {"key": "var1", "value": "foo", "description": "after_update"} - _check_last_log(session, dag_id=None, event="api.variable.edit", execution_date=None) + _check_last_log(session, dag_id=None, event="api.variable.edit", logical_date=None) def test_should_reject_invalid_update(self): response = self.client.patch( @@ -331,7 +331,7 @@ def test_should_create_variable(self, description, session): ) assert response.status_code == 200 _check_last_log( - session, dag_id=None, event="api.variable.create", execution_date=None, expected_extra=payload + session, dag_id=None, event="api.variable.create", logical_date=None, expected_extra=payload ) response = self.client.get("/api/v1/variables/var_create", environ_overrides={"REMOTE_USER": "test"}) assert response.json == { @@ -357,7 +357,7 @@ def test_should_create_masked_variable(self, session): session, dag_id=None, event="api.variable.create", - execution_date=None, + logical_date=None, expected_extra=expected_extra, ) response = self.client.get("/api/v1/variables/api_key", environ_overrides={"REMOTE_USER": "test"}) @@ -379,7 +379,7 @@ def test_should_reject_invalid_request(self, session): "type": EXCEPTIONS_LINK_MAP[400], "detail": "{'value': ['Missing data for required field.'], 'v': ['Unknown field.']}", } - _check_last_log(session, dag_id=None, event="api.variable.create", execution_date=None) + _check_last_log(session, dag_id=None, event="api.variable.create", logical_date=None) def test_should_raises_401_unauthenticated(self): response = self.client.post( diff --git a/tests/api_connexion/endpoints/test_xcom_endpoint.py b/tests/api_connexion/endpoints/test_xcom_endpoint.py index a2c267428ac6..aa767ae168b0 100644 --- a/tests/api_connexion/endpoints/test_xcom_endpoint.py +++ b/tests/api_connexion/endpoints/test_xcom_endpoint.py @@ -71,7 +71,7 @@ def sort_key(record): return ( record.get("dag_id"), record.get("task_id"), - record.get("execution_date"), + record.get("logical_date"), record.get("map_index"), record.get("key"), ) @@ -109,11 +109,11 @@ class TestGetXComEntry(TestXComEndpoint): def test_should_respond_200_stringify(self): dag_id = "test-dag-id" task_id = "test-task-id" - execution_date = "2005-04-02T00:00:00+00:00" + logical_date = "2005-04-02T00:00:00+00:00" xcom_key = "test-xcom-key" - execution_date_parsed = timezone.parse(execution_date) - run_id = DagRun.generate_run_id(DagRunType.MANUAL, execution_date_parsed) - self._create_xcom_entry(dag_id, run_id, execution_date_parsed, task_id, xcom_key, {"key": "value"}) + logical_date_parsed = timezone.parse(logical_date) + run_id = DagRun.generate_run_id(DagRunType.MANUAL, logical_date_parsed) + self._create_xcom_entry(dag_id, run_id, logical_date_parsed, task_id, xcom_key, {"key": "value"}) response = self.client.get( f"/api/v1/dags/{dag_id}/dagRuns/{run_id}/taskInstances/{task_id}/xcomEntries/{xcom_key}", environ_overrides={"REMOTE_USER": "test"}, @@ -124,7 +124,7 @@ def test_should_respond_200_stringify(self): current_data["timestamp"] = "TIMESTAMP" assert current_data == { "dag_id": dag_id, - "execution_date": execution_date, + "logical_date": logical_date, "key": xcom_key, "task_id": task_id, "map_index": -1, @@ -135,11 +135,11 @@ def test_should_respond_200_stringify(self): def test_should_respond_200_native(self): dag_id = "test-dag-id" task_id = "test-task-id" - execution_date = "2005-04-02T00:00:00+00:00" + logical_date = "2005-04-02T00:00:00+00:00" xcom_key = "test-xcom-key" - execution_date_parsed = timezone.parse(execution_date) - run_id = DagRun.generate_run_id(DagRunType.MANUAL, execution_date_parsed) - self._create_xcom_entry(dag_id, run_id, execution_date_parsed, task_id, xcom_key, {"key": "value"}) + logical_date_parsed = timezone.parse(logical_date) + run_id = DagRun.generate_run_id(DagRunType.MANUAL, logical_date_parsed) + self._create_xcom_entry(dag_id, run_id, logical_date_parsed, task_id, xcom_key, {"key": "value"}) response = self.client.get( f"/api/v1/dags/{dag_id}/dagRuns/{run_id}/taskInstances/{task_id}/xcomEntries/{xcom_key}?stringify=false", environ_overrides={"REMOTE_USER": "test"}, @@ -150,7 +150,7 @@ def test_should_respond_200_native(self): current_data["timestamp"] = "TIMESTAMP" assert current_data == { "dag_id": dag_id, - "execution_date": execution_date, + "logical_date": logical_date, "key": xcom_key, "task_id": task_id, "map_index": -1, @@ -162,13 +162,13 @@ def test_should_respond_200_native(self): def test_should_respond_200_native_for_pickled(self): dag_id = "test-dag-id" task_id = "test-task-id" - execution_date = "2005-04-02T00:00:00+00:00" + logical_date = "2005-04-02T00:00:00+00:00" xcom_key = "test-xcom-key" - execution_date_parsed = timezone.parse(execution_date) - run_id = DagRun.generate_run_id(DagRunType.MANUAL, execution_date_parsed) + logical_date_parsed = timezone.parse(logical_date) + run_id = DagRun.generate_run_id(DagRunType.MANUAL, logical_date_parsed) value_non_serializable_key = {("201009_NB502104_0421_AHJY23BGXG (SEQ_WF: 138898)", None): 82359} self._create_xcom_entry( - dag_id, run_id, execution_date_parsed, task_id, xcom_key, {"key": value_non_serializable_key} + dag_id, run_id, logical_date_parsed, task_id, xcom_key, {"key": value_non_serializable_key} ) response = self.client.get( f"/api/v1/dags/{dag_id}/dagRuns/{run_id}/taskInstances/{task_id}/xcomEntries/{xcom_key}", @@ -180,7 +180,7 @@ def test_should_respond_200_native_for_pickled(self): current_data["timestamp"] = "TIMESTAMP" assert current_data == { "dag_id": dag_id, - "execution_date": execution_date, + "logical_date": logical_date, "key": xcom_key, "task_id": task_id, "map_index": -1, @@ -191,11 +191,11 @@ def test_should_respond_200_native_for_pickled(self): def test_should_raise_404_for_non_existent_xcom(self): dag_id = "test-dag-id" task_id = "test-task-id" - execution_date = "2005-04-02T00:00:00+00:00" + logical_date = "2005-04-02T00:00:00+00:00" xcom_key = "test-xcom-key" - execution_date_parsed = timezone.parse(execution_date) - run_id = DagRun.generate_run_id(DagRunType.MANUAL, execution_date_parsed) - self._create_xcom_entry(dag_id, run_id, execution_date_parsed, task_id, xcom_key) + logical_date_parsed = timezone.parse(logical_date) + run_id = DagRun.generate_run_id(DagRunType.MANUAL, logical_date_parsed) + self._create_xcom_entry(dag_id, run_id, logical_date_parsed, task_id, xcom_key) response = self.client.get( f"/api/v1/dags/nonexistentdagid/dagRuns/{run_id}/taskInstances/{task_id}/xcomEntries/{xcom_key}", environ_overrides={"REMOTE_USER": "test"}, @@ -206,11 +206,11 @@ def test_should_raise_404_for_non_existent_xcom(self): def test_should_raises_401_unauthenticated(self): dag_id = "test-dag-id" task_id = "test-task-id" - execution_date = "2005-04-02T00:00:00+00:00" + logical_date = "2005-04-02T00:00:00+00:00" xcom_key = "test-xcom-key" - execution_date_parsed = timezone.parse(execution_date) - run_id = DagRun.generate_run_id(DagRunType.MANUAL, execution_date_parsed) - self._create_xcom_entry(dag_id, run_id, execution_date_parsed, task_id, xcom_key) + logical_date_parsed = timezone.parse(logical_date) + run_id = DagRun.generate_run_id(DagRunType.MANUAL, logical_date_parsed) + self._create_xcom_entry(dag_id, run_id, logical_date_parsed, task_id, xcom_key) response = self.client.get( f"/api/v1/dags/{dag_id}/dagRuns/{run_id}/taskInstances/{task_id}/xcomEntries/{xcom_key}" ) @@ -220,12 +220,12 @@ def test_should_raises_401_unauthenticated(self): def test_should_raise_403_forbidden(self): dag_id = "test-dag-id" task_id = "test-task-id" - execution_date = "2005-04-02T00:00:00+00:00" + logical_date = "2005-04-02T00:00:00+00:00" xcom_key = "test-xcom-key" - execution_date_parsed = timezone.parse(execution_date) - run_id = DagRun.generate_run_id(DagRunType.MANUAL, execution_date_parsed) + logical_date_parsed = timezone.parse(logical_date) + run_id = DagRun.generate_run_id(DagRunType.MANUAL, logical_date_parsed) - self._create_xcom_entry(dag_id, run_id, execution_date_parsed, task_id, xcom_key) + self._create_xcom_entry(dag_id, run_id, logical_date_parsed, task_id, xcom_key) response = self.client.get( f"/api/v1/dags/{dag_id}/dagRuns/{run_id}/taskInstances/{task_id}/xcomEntries/{xcom_key}", environ_overrides={"REMOTE_USER": "test_no_permissions"}, @@ -233,14 +233,14 @@ def test_should_raise_403_forbidden(self): assert response.status_code == 403 def _create_xcom_entry( - self, dag_id, run_id, execution_date, task_id, xcom_key, xcom_value="TEST_VALUE", *, backend=XCom + self, dag_id, run_id, logical_date, task_id, xcom_key, xcom_value="TEST_VALUE", *, backend=XCom ): with create_session() as session: dagrun = DagRun( dag_id=dag_id, run_id=run_id, - execution_date=execution_date, - start_date=execution_date, + logical_date=logical_date, + start_date=logical_date, run_type=DagRunType.MANUAL, ) session.add(dagrun) @@ -317,11 +317,11 @@ class TestGetXComEntries(TestXComEndpoint): def test_should_respond_200(self): dag_id = "test-dag-id" task_id = "test-task-id" - execution_date = "2005-04-02T00:00:00+00:00" - execution_date_parsed = timezone.parse(execution_date) - run_id = DagRun.generate_run_id(DagRunType.MANUAL, execution_date_parsed) + logical_date = "2005-04-02T00:00:00+00:00" + logical_date_parsed = timezone.parse(logical_date) + run_id = DagRun.generate_run_id(DagRunType.MANUAL, logical_date_parsed) - self._create_xcom_entries(dag_id, run_id, execution_date_parsed, task_id) + self._create_xcom_entries(dag_id, run_id, logical_date_parsed, task_id) response = self.client.get( f"/api/v1/dags/{dag_id}/dagRuns/{run_id}/taskInstances/{task_id}/xcomEntries", environ_overrides={"REMOTE_USER": "test"}, @@ -337,7 +337,7 @@ def test_should_respond_200(self): "xcom_entries": [ { "dag_id": dag_id, - "execution_date": execution_date, + "logical_date": logical_date, "key": "test-xcom-key-1", "task_id": task_id, "timestamp": "TIMESTAMP", @@ -345,7 +345,7 @@ def test_should_respond_200(self): }, { "dag_id": dag_id, - "execution_date": execution_date, + "logical_date": logical_date, "key": "test-xcom-key-2", "task_id": task_id, "timestamp": "TIMESTAMP", @@ -359,15 +359,15 @@ def test_should_respond_200(self): def test_should_respond_200_with_tilde_and_access_to_all_dags(self): dag_id_1 = "test-dag-id-1" task_id_1 = "test-task-id-1" - execution_date = "2005-04-02T00:00:00+00:00" - execution_date_parsed = timezone.parse(execution_date) - run_id_1 = DagRun.generate_run_id(DagRunType.MANUAL, execution_date_parsed) - self._create_xcom_entries(dag_id_1, run_id_1, execution_date_parsed, task_id_1) + logical_date = "2005-04-02T00:00:00+00:00" + logical_date_parsed = timezone.parse(logical_date) + run_id_1 = DagRun.generate_run_id(DagRunType.MANUAL, logical_date_parsed) + self._create_xcom_entries(dag_id_1, run_id_1, logical_date_parsed, task_id_1) dag_id_2 = "test-dag-id-2" task_id_2 = "test-task-id-2" - run_id_2 = DagRun.generate_run_id(DagRunType.MANUAL, execution_date_parsed) - self._create_xcom_entries(dag_id_2, run_id_2, execution_date_parsed, task_id_2) + run_id_2 = DagRun.generate_run_id(DagRunType.MANUAL, logical_date_parsed) + self._create_xcom_entries(dag_id_2, run_id_2, logical_date_parsed, task_id_2) response = self.client.get( "/api/v1/dags/~/dagRuns/~/taskInstances/~/xcomEntries", @@ -384,7 +384,7 @@ def test_should_respond_200_with_tilde_and_access_to_all_dags(self): "xcom_entries": [ { "dag_id": dag_id_1, - "execution_date": execution_date, + "logical_date": logical_date, "key": "test-xcom-key-1", "task_id": task_id_1, "timestamp": "TIMESTAMP", @@ -392,7 +392,7 @@ def test_should_respond_200_with_tilde_and_access_to_all_dags(self): }, { "dag_id": dag_id_1, - "execution_date": execution_date, + "logical_date": logical_date, "key": "test-xcom-key-2", "task_id": task_id_1, "timestamp": "TIMESTAMP", @@ -400,7 +400,7 @@ def test_should_respond_200_with_tilde_and_access_to_all_dags(self): }, { "dag_id": dag_id_2, - "execution_date": execution_date, + "logical_date": logical_date, "key": "test-xcom-key-1", "task_id": task_id_2, "timestamp": "TIMESTAMP", @@ -408,7 +408,7 @@ def test_should_respond_200_with_tilde_and_access_to_all_dags(self): }, { "dag_id": dag_id_2, - "execution_date": execution_date, + "logical_date": logical_date, "key": "test-xcom-key-2", "task_id": task_id_2, "timestamp": "TIMESTAMP", @@ -422,10 +422,10 @@ def test_should_respond_200_with_tilde_and_access_to_all_dags(self): def test_should_respond_200_with_map_index(self): dag_id = "test-dag-id" task_id = "test-task-id" - execution_date = "2005-04-02T00:00:00+00:00" - execution_date_parsed = timezone.parse(execution_date) - dag_run_id = DagRun.generate_run_id(DagRunType.MANUAL, execution_date_parsed) - self._create_xcom_entries(dag_id, dag_run_id, execution_date_parsed, task_id, mapped_ti=True) + logical_date = "2005-04-02T00:00:00+00:00" + logical_date_parsed = timezone.parse(logical_date) + dag_run_id = DagRun.generate_run_id(DagRunType.MANUAL, logical_date_parsed) + self._create_xcom_entries(dag_id, dag_run_id, logical_date_parsed, task_id, mapped_ti=True) def assert_expected_result(expected_entries, map_index=None): response = self.client.get( @@ -445,7 +445,7 @@ def assert_expected_result(expected_entries, map_index=None): expected_entry1 = { "dag_id": dag_id, - "execution_date": execution_date, + "logical_date": logical_date, "key": "test-xcom-key", "task_id": task_id, "timestamp": "TIMESTAMP", @@ -453,7 +453,7 @@ def assert_expected_result(expected_entries, map_index=None): } expected_entry2 = { "dag_id": dag_id, - "execution_date": execution_date, + "logical_date": logical_date, "key": "test-xcom-key", "task_id": task_id, "timestamp": "TIMESTAMP", @@ -466,10 +466,10 @@ def assert_expected_result(expected_entries, map_index=None): def test_should_respond_200_with_xcom_key(self): dag_id = "test-dag-id" task_id = "test-task-id" - execution_date = "2005-04-02T00:00:00+00:00" - execution_date_parsed = timezone.parse(execution_date) - dag_run_id = DagRun.generate_run_id(DagRunType.MANUAL, execution_date_parsed) - self._create_xcom_entries(dag_id, dag_run_id, execution_date_parsed, task_id, mapped_ti=True) + logical_date = "2005-04-02T00:00:00+00:00" + logical_date_parsed = timezone.parse(logical_date) + dag_run_id = DagRun.generate_run_id(DagRunType.MANUAL, logical_date_parsed) + self._create_xcom_entries(dag_id, dag_run_id, logical_date_parsed, task_id, mapped_ti=True) def assert_expected_result(expected_entries, key=None): response = self.client.get( @@ -488,7 +488,7 @@ def assert_expected_result(expected_entries, key=None): expected_entry1 = { "dag_id": dag_id, - "execution_date": execution_date, + "logical_date": logical_date, "key": "test-xcom-key", "task_id": task_id, "timestamp": "TIMESTAMP", @@ -496,7 +496,7 @@ def assert_expected_result(expected_entries, key=None): } expected_entry2 = { "dag_id": dag_id, - "execution_date": execution_date, + "logical_date": logical_date, "key": "test-xcom-key", "task_id": task_id, "timestamp": "TIMESTAMP", @@ -508,10 +508,10 @@ def assert_expected_result(expected_entries, key=None): def test_should_raises_401_unauthenticated(self): dag_id = "test-dag-id" task_id = "test-task-id" - execution_date = "2005-04-02T00:00:00+00:00" - execution_date_parsed = timezone.parse(execution_date) - run_id = DagRun.generate_run_id(DagRunType.MANUAL, execution_date_parsed) - self._create_xcom_entries(dag_id, run_id, execution_date_parsed, task_id) + logical_date = "2005-04-02T00:00:00+00:00" + logical_date_parsed = timezone.parse(logical_date) + run_id = DagRun.generate_run_id(DagRunType.MANUAL, logical_date_parsed) + self._create_xcom_entries(dag_id, run_id, logical_date_parsed, task_id) response = self.client.get( f"/api/v1/dags/{dag_id}/dagRuns/{run_id}/taskInstances/{task_id}/xcomEntries" @@ -519,15 +519,15 @@ def test_should_raises_401_unauthenticated(self): assert_401(response) - def _create_xcom_entries(self, dag_id, run_id, execution_date, task_id, mapped_ti=False): + def _create_xcom_entries(self, dag_id, run_id, logical_date, task_id, mapped_ti=False): with create_session() as session: dag = DagModel(dag_id=dag_id) session.add(dag) dagrun = DagRun( dag_id=dag_id, run_id=run_id, - execution_date=execution_date, - start_date=execution_date, + logical_date=logical_date, + start_date=logical_date, run_type=DagRunType.MANUAL, ) session.add(dagrun) @@ -553,7 +553,7 @@ def _create_xcom_entries(self, dag_id, run_id, execution_date, task_id, mapped_t key=key, value="TEST", run_id=run_id, task_id=task_id, dag_id=dag_id, map_index=map_index ) - def _create_invalid_xcom_entries(self, execution_date): + def _create_invalid_xcom_entries(self, logical_date): """ Invalid XCom entries to test join query """ @@ -563,16 +563,16 @@ def _create_invalid_xcom_entries(self, execution_date): dagrun = DagRun( dag_id="invalid_dag", run_id="invalid_run_id", - execution_date=execution_date + timedelta(days=1), - start_date=execution_date, + logical_date=logical_date + timedelta(days=1), + start_date=logical_date, run_type=DagRunType.MANUAL, ) session.add(dagrun) dagrun1 = DagRun( dag_id="invalid_dag", run_id="not_this_run_id", - execution_date=execution_date, - start_date=execution_date, + logical_date=logical_date, + start_date=logical_date, run_type=DagRunType.MANUAL, ) session.add(dagrun1) @@ -593,9 +593,9 @@ class TestPaginationGetXComEntries(TestXComEndpoint): def setup_method(self): self.dag_id = "test-dag-id" self.task_id = "test-task-id" - self.execution_date = "2005-04-02T00:00:00+00:00" - self.execution_date_parsed = timezone.parse(self.execution_date) - self.run_id = DagRun.generate_run_id(DagRunType.MANUAL, self.execution_date_parsed) + self.logical_date = "2005-04-02T00:00:00+00:00" + self.logical_date_parsed = timezone.parse(self.logical_date) + self.run_id = DagRun.generate_run_id(DagRunType.MANUAL, self.logical_date_parsed) @pytest.mark.parametrize( "query_params, expected_xcom_ids", @@ -656,8 +656,8 @@ def test_handle_limit_offset(self, query_params, expected_xcom_ids): dagrun = DagRun( dag_id=self.dag_id, run_id=self.run_id, - execution_date=self.execution_date_parsed, - start_date=self.execution_date_parsed, + logical_date=self.logical_date_parsed, + start_date=self.logical_date_parsed, run_type=DagRunType.MANUAL, ) session.add(dagrun) @@ -674,7 +674,7 @@ def test_handle_limit_offset(self, query_params, expected_xcom_ids): run_id=self.run_id, task_id=self.task_id, dag_id=self.dag_id, - timestamp=self.execution_date_parsed, + timestamp=self.logical_date_parsed, ) session.add(xcom) diff --git a/tests/api_connexion/schemas/test_dag_run_schema.py b/tests/api_connexion/schemas/test_dag_run_schema.py index 2c1ec142d508..8d5b266cabde 100644 --- a/tests/api_connexion/schemas/test_dag_run_schema.py +++ b/tests/api_connexion/schemas/test_dag_run_schema.py @@ -62,7 +62,7 @@ def test_serialize(self, session): run_id="my-dag-run", state="running", run_type=DagRunType.MANUAL.value, - execution_date=timezone.parse(self.default_time), + logical_date=timezone.parse(self.default_time), start_date=timezone.parse(self.default_time), conf='{"start": "stop"}', **triggered_by_kwargs, @@ -76,7 +76,6 @@ def test_serialize(self, session): "dag_run_id": "my-dag-run", "end_date": None, "state": "running", - "execution_date": self.default_time, "logical_date": self.default_time, "external_trigger": True, "start_date": self.default_time, @@ -95,30 +94,30 @@ def test_serialize(self, session): "serialized_dagrun, expected_result", [ ( # Conf not provided - {"dag_run_id": "my-dag-run", "execution_date": DEFAULT_TIME}, - {"run_id": "my-dag-run", "execution_date": parse(DEFAULT_TIME)}, + {"dag_run_id": "my-dag-run", "logical_date": DEFAULT_TIME}, + {"run_id": "my-dag-run", "logical_date": parse(DEFAULT_TIME)}, ), ( { "dag_run_id": "my-dag-run", - "execution_date": DEFAULT_TIME, + "logical_date": DEFAULT_TIME, "conf": {"start": "stop"}, }, { "run_id": "my-dag-run", - "execution_date": parse(DEFAULT_TIME), + "logical_date": parse(DEFAULT_TIME), "conf": {"start": "stop"}, }, ), ( { "dag_run_id": "my-dag-run", - "execution_date": DEFAULT_TIME, + "logical_date": DEFAULT_TIME, "conf": '{"start": "stop"}', }, { "run_id": "my-dag-run", - "execution_date": parse(DEFAULT_TIME), + "logical_date": parse(DEFAULT_TIME), "conf": {"start": "stop"}, }, ), @@ -129,13 +128,13 @@ def test_deserialize(self, serialized_dagrun, expected_result): assert result == expected_result def test_autofill_fields(self): - """Dag_run_id and execution_date fields are autogenerated if missing""" + """Dag_run_id and logical_date fields are autogenerated if missing""" serialized_dagrun = {} result = dagrun_schema.load(serialized_dagrun) - assert result == {"execution_date": result["execution_date"], "run_id": result["run_id"]} + assert result == {"logical_date": result["logical_date"], "run_id": result["run_id"]} - def test_invalid_execution_date_raises(self): - serialized_dagrun = {"execution_date": "mydate"} + def test_invalid_logical_date_raises(self): + serialized_dagrun = {"logical_date": "mydate"} with pytest.raises(BadRequest) as ctx: dagrun_schema.load(serialized_dagrun) assert str(ctx.value) == "Incorrect datetime argument" @@ -151,7 +150,7 @@ def test_serialize(self, session): dag_id="my-dag-run", run_id="my-dag-run", state="running", - execution_date=timezone.parse(self.default_time), + logical_date=timezone.parse(self.default_time), run_type=DagRunType.MANUAL.value, start_date=timezone.parse(self.default_time), conf='{"start": "stop"}', @@ -162,7 +161,7 @@ def test_serialize(self, session): dag_id="my-dag-run", run_id="my-dag-run-2", state="running", - execution_date=timezone.parse(self.second_time), + logical_date=timezone.parse(self.second_time), start_date=timezone.parse(self.default_time), run_type=DagRunType.MANUAL.value, **triggered_by_ui_kwargs, @@ -176,7 +175,6 @@ def test_serialize(self, session): "dag_id": "my-dag-run", "dag_run_id": "my-dag-run", "end_date": None, - "execution_date": self.default_time, "logical_date": self.default_time, "external_trigger": True, "state": "running", @@ -196,7 +194,6 @@ def test_serialize(self, session): "dag_run_id": "my-dag-run-2", "end_date": None, "state": "running", - "execution_date": self.second_time, "logical_date": self.second_time, "external_trigger": True, "start_date": self.default_time, diff --git a/tests/api_connexion/schemas/test_event_log_schema.py b/tests/api_connexion/schemas/test_event_log_schema.py index 407ac4140fee..6e9af9fc69a8 100644 --- a/tests/api_connexion/schemas/test_event_log_schema.py +++ b/tests/api_connexion/schemas/test_event_log_schema.py @@ -36,7 +36,7 @@ def task_instance(session, create_task_instance, request): dag_id="TEST_DAG_ID", task_id="TEST_TASK_ID", run_id="TEST_RUN_ID", - execution_date=request.instance.default_time, + logical_date=request.instance.default_time, ) @@ -60,7 +60,7 @@ def test_serialize(self, task_instance): "run_id": "TEST_RUN_ID", "map_index": -1, "try_number": 0, - "execution_date": self.default_time.isoformat(), + "logical_date": self.default_time.isoformat(), "owner": "airflow", "when": self.default_time.isoformat(), "extra": None, @@ -86,7 +86,7 @@ def test_serialize(self, task_instance): "run_id": "TEST_RUN_ID", "map_index": -1, "try_number": 0, - "execution_date": self.default_time.isoformat(), + "logical_date": self.default_time.isoformat(), "owner": "airflow", "when": self.default_time.isoformat(), "extra": None, @@ -99,7 +99,7 @@ def test_serialize(self, task_instance): "run_id": "TEST_RUN_ID", "map_index": -1, "try_number": 0, - "execution_date": self.default_time.isoformat(), + "logical_date": self.default_time.isoformat(), "owner": "airflow", "when": self.default_time2.isoformat(), "extra": None, diff --git a/tests/api_connexion/schemas/test_task_instance_schema.py b/tests/api_connexion/schemas/test_task_instance_schema.py index ff18ad75b544..5297830dca01 100644 --- a/tests/api_connexion/schemas/test_task_instance_schema.py +++ b/tests/api_connexion/schemas/test_task_instance_schema.py @@ -42,7 +42,7 @@ def set_attrs(self, session, dag_maker): with dag_maker(dag_id="TEST_DAG_ID", session=session): self.task = EmptyOperator(task_id="TEST_TASK_ID", start_date=self.default_time) - self.dr = dag_maker.create_dagrun(execution_date=self.default_time) + self.dr = dag_maker.create_dagrun(logical_date=self.default_time) session.flush() self.default_ti_init = { @@ -74,7 +74,7 @@ def test_task_instance_schema_without_rendered(self, session): "dag_id": "TEST_DAG_ID", "duration": 10000.0, "end_date": "2020-01-03T00:00:00+00:00", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "executor": None, "executor_config": "{}", "hostname": "", @@ -166,7 +166,7 @@ class TestSetTaskInstanceStateFormSchema: current_input = { "dry_run": True, "task_id": "print_the_context", - "execution_date": "2020-01-01T00:00:00+00:00", + "logical_date": "2020-01-01T00:00:00+00:00", "include_upstream": True, "include_downstream": True, "include_future": True, @@ -178,7 +178,7 @@ def test_success(self): result = set_task_instance_state_form.load(self.current_input) expected_result = { "dry_run": True, - "execution_date": dt.datetime(2020, 1, 1, 0, 0, tzinfo=dt.timezone(dt.timedelta(0), "+0000")), + "logical_date": dt.datetime(2020, 1, 1, 0, 0, tzinfo=dt.timezone(dt.timedelta(0), "+0000")), "include_downstream": True, "include_future": True, "include_past": True, @@ -194,7 +194,7 @@ def test_dry_run_is_optional(self): result = set_task_instance_state_form.load(self.current_input) expected_result = { "dry_run": True, - "execution_date": dt.datetime(2020, 1, 1, 0, 0, tzinfo=dt.timezone(dt.timedelta(0), "+0000")), + "logical_date": dt.datetime(2020, 1, 1, 0, 0, tzinfo=dt.timezone(dt.timedelta(0), "+0000")), "include_downstream": True, "include_future": True, "include_past": True, @@ -209,9 +209,8 @@ def test_dry_run_is_optional(self): [ {"task_id": None}, {"include_future": "foo"}, - {"execution_date": "NOW"}, {"new_state": "INVALID_STATE"}, - {"execution_date": "2020-01-01T00:00:00+00:00", "dag_run_id": "some-run-id"}, + {"logical_date": "2020-01-01T00:00:00+00:00", "dag_run_id": "some-run-id"}, ], ) def test_validation_error(self, override_data): diff --git a/tests/api_connexion/schemas/test_xcom_schema.py b/tests/api_connexion/schemas/test_xcom_schema.py index 5863dd48c618..24ce27efcf46 100644 --- a/tests/api_connexion/schemas/test_xcom_schema.py +++ b/tests/api_connexion/schemas/test_xcom_schema.py @@ -50,7 +50,7 @@ def sort_key(record): return ( record.get("dag_id"), record.get("task_id"), - record.get("execution_date"), + record.get("logical_date"), record.get("map_index"), record.get("key"), ) @@ -62,11 +62,11 @@ def sort_key(record): @pytest.fixture def create_xcom(create_task_instance, session): - def maker(dag_id, task_id, execution_date, key, map_index=-1, value=None): + def maker(dag_id, task_id, logical_date, key, map_index=-1, value=None): ti = create_task_instance( dag_id=dag_id, task_id=task_id, - execution_date=execution_date, + logical_date=logical_date, map_index=map_index, session=session, ) @@ -77,7 +77,7 @@ def maker(dag_id, task_id, execution_date, key, map_index=-1, value=None): map_index=map_index, key=key, value=value, - timestamp=run.execution_date, + timestamp=run.logical_date, dag_id=run.dag_id, run_id=run.run_id, ) @@ -96,7 +96,7 @@ def test_serialize(self, create_xcom, session): create_xcom( dag_id="test_dag", task_id="test_task_id", - execution_date=self.default_time_parsed, + logical_date=self.default_time_parsed, key="test_key", ) xcom_model = session.query(XCom).first() @@ -104,7 +104,7 @@ def test_serialize(self, create_xcom, session): assert deserialized_xcom == { "key": "test_key", "timestamp": self.default_time, - "execution_date": self.default_time, + "logical_date": self.default_time, "task_id": "test_task_id", "dag_id": "test_dag", "map_index": -1, @@ -114,7 +114,7 @@ def test_deserialize(self): xcom_dump = { "key": "test_key", "timestamp": self.default_time, - "execution_date": self.default_time, + "logical_date": self.default_time, "task_id": "test_task_id", "dag_id": "test_dag", "map_index": 2, @@ -123,7 +123,7 @@ def test_deserialize(self): assert result == { "key": "test_key", "timestamp": self.default_time_parsed, - "execution_date": self.default_time_parsed, + "logical_date": self.default_time_parsed, "task_id": "test_task_id", "dag_id": "test_dag", "map_index": 2, @@ -140,18 +140,18 @@ def test_serialize(self, create_xcom, session): create_xcom( dag_id="test_dag_1", task_id="test_task_id_1", - execution_date=self.time_1, + logical_date=self.time_1, key="test_key_1", ) create_xcom( dag_id="test_dag_2", task_id="test_task_id_2", - execution_date=self.time_2, + logical_date=self.time_2, key="test_key_2", ) xcom_models = session.scalars( select(XCom) - .where(or_(XCom.execution_date == self.time_1, XCom.execution_date == self.time_2)) + .where(or_(XCom.logical_date == self.time_1, XCom.logical_date == self.time_2)) .order_by(XCom.dag_run_id) ).all() deserialized_xcoms = xcom_collection_schema.dump( @@ -167,7 +167,7 @@ def test_serialize(self, create_xcom, session): { "key": "test_key_1", "timestamp": self.default_time_1, - "execution_date": self.default_time_1, + "logical_date": self.default_time_1, "task_id": "test_task_id_1", "dag_id": "test_dag_1", "map_index": -1, @@ -175,7 +175,7 @@ def test_serialize(self, create_xcom, session): { "key": "test_key_2", "timestamp": self.default_time_2, - "execution_date": self.default_time_2, + "logical_date": self.default_time_2, "task_id": "test_task_id_2", "dag_id": "test_dag_2", "map_index": -1, @@ -195,7 +195,7 @@ def test_serialize(self, create_xcom, session): create_xcom( dag_id="test_dag", task_id="test_task_id", - execution_date=self.default_time_parsed, + logical_date=self.default_time_parsed, key="test_key", value=pickle.dumps(b"test_binary"), ) @@ -204,7 +204,7 @@ def test_serialize(self, create_xcom, session): assert deserialized_xcom == { "key": "test_key", "timestamp": self.default_time, - "execution_date": self.default_time, + "logical_date": self.default_time, "task_id": "test_task_id", "dag_id": "test_dag", "value": "test_binary", @@ -216,7 +216,7 @@ def test_deserialize(self): xcom_dump = { "key": "test_key", "timestamp": self.default_time, - "execution_date": self.default_time, + "logical_date": self.default_time, "task_id": "test_task_id", "dag_id": "test_dag", "value": b"test_binary", @@ -225,7 +225,7 @@ def test_deserialize(self): assert result == { "key": "test_key", "timestamp": self.default_time_parsed, - "execution_date": self.default_time_parsed, + "logical_date": self.default_time_parsed, "task_id": "test_task_id", "dag_id": "test_dag", "value": "test_binary", diff --git a/tests/api_fastapi/core_api/routes/public/test_assets.py b/tests/api_fastapi/core_api/routes/public/test_assets.py index 95ad658ba4d9..dd1c971dacbf 100644 --- a/tests/api_fastapi/core_api/routes/public/test_assets.py +++ b/tests/api_fastapi/core_api/routes/public/test_assets.py @@ -85,7 +85,7 @@ def _create_dag_run(session, num: int = 2): dag_id="source_dag_id", run_id=f"source_run_id_{i}", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(default_time), + logical_date=timezone.parse(default_time), start_date=timezone.parse(default_time), data_interval=(timezone.parse(default_time), timezone.parse(default_time)), external_trigger=True, diff --git a/tests/api_fastapi/core_api/routes/public/test_dag_run.py b/tests/api_fastapi/core_api/routes/public/test_dag_run.py index 64c3512e88b7..b2b47a7186cc 100644 --- a/tests/api_fastapi/core_api/routes/public/test_dag_run.py +++ b/tests/api_fastapi/core_api/routes/public/test_dag_run.py @@ -79,7 +79,7 @@ def setup(dag_maker, session=None): state=DAG1_RUN2_STATE, run_type=DAG1_RUN2_RUN_TYPE, triggered_by=DAG1_RUN2_TRIGGERED_BY, - execution_date=EXECUTION_DATE, + logical_date=EXECUTION_DATE, ) with dag_maker( @@ -93,14 +93,14 @@ def setup(dag_maker, session=None): state=DAG2_RUN1_STATE, run_type=DAG2_RUN1_RUN_TYPE, triggered_by=DAG2_RUN1_TRIGGERED_BY, - execution_date=EXECUTION_DATE, + logical_date=EXECUTION_DATE, ) dag_maker.create_dagrun( run_id=DAG2_RUN2_ID, state=DAG2_RUN2_STATE, run_type=DAG2_RUN2_RUN_TYPE, triggered_by=DAG2_RUN2_TRIGGERED_BY, - execution_date=EXECUTION_DATE, + logical_date=EXECUTION_DATE, ) dag_maker.dagbag.sync_to_db() diff --git a/tests/api_fastapi/core_api/routes/public/test_dag_stats.py b/tests/api_fastapi/core_api/routes/public/test_dag_stats.py index e2611addd70c..f4c59d48c313 100644 --- a/tests/api_fastapi/core_api/routes/public/test_dag_stats.py +++ b/tests/api_fastapi/core_api/routes/public/test_dag_stats.py @@ -60,7 +60,7 @@ def _create_dag_and_runs(self, session=None): dag_id=DAG1_ID, run_id="test_dag_run_id_1", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time), + logical_date=timezone.parse(self.default_time), start_date=timezone.parse(self.default_time), external_trigger=True, state="running", @@ -69,7 +69,7 @@ def _create_dag_and_runs(self, session=None): dag_id=dag_1.dag_id, run_id="test_dag_run_id_2", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time) + timedelta(days=1), + logical_date=timezone.parse(self.default_time) + timedelta(days=1), start_date=timezone.parse(self.default_time), external_trigger=True, state="failed", @@ -87,7 +87,7 @@ def _create_dag_and_runs(self, session=None): dag_id=dag_2.dag_id, run_id="test_dag_2_run_id_1", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time), + logical_date=timezone.parse(self.default_time), start_date=timezone.parse(self.default_time), external_trigger=True, state="queued", @@ -105,7 +105,7 @@ def _create_dag_and_runs(self, session=None): dag_id=dag_3.dag_id, run_id="test_dag_3_run_id_1", run_type=DagRunType.MANUAL, - execution_date=timezone.parse(self.default_time), + logical_date=timezone.parse(self.default_time), start_date=timezone.parse(self.default_time), external_trigger=True, state="success", diff --git a/tests/api_fastapi/core_api/routes/public/test_dags.py b/tests/api_fastapi/core_api/routes/public/test_dags.py index e200763c2a72..aef11a5870c6 100644 --- a/tests/api_fastapi/core_api/routes/public/test_dags.py +++ b/tests/api_fastapi/core_api/routes/public/test_dags.py @@ -69,7 +69,7 @@ def _create_deactivated_paused_dag(self, session=None): dagrun_failed = DagRun( dag_id=DAG3_ID, run_id="run1", - execution_date=datetime(2018, 1, 1, 12, 0, 0, tzinfo=timezone.utc), + logical_date=datetime(2018, 1, 1, 12, 0, 0, tzinfo=timezone.utc), start_date=datetime(2018, 1, 1, 12, 0, 0, tzinfo=timezone.utc), run_type=DagRunType.SCHEDULED, state=DagRunState.FAILED, @@ -79,7 +79,7 @@ def _create_deactivated_paused_dag(self, session=None): dagrun_success = DagRun( dag_id=DAG3_ID, run_id="run2", - execution_date=datetime(2019, 1, 1, 12, 0, 0, tzinfo=timezone.utc), + logical_date=datetime(2019, 1, 1, 12, 0, 0, tzinfo=timezone.utc), start_date=datetime(2019, 1, 1, 12, 0, 0, tzinfo=timezone.utc), run_type=DagRunType.MANUAL, state=DagRunState.SUCCESS, diff --git a/tests/api_fastapi/core_api/routes/public/test_event_logs.py b/tests/api_fastapi/core_api/routes/public/test_event_logs.py index 347f587bf5c2..d42f0b4f77e6 100644 --- a/tests/api_fastapi/core_api/routes/public/test_event_logs.py +++ b/tests/api_fastapi/core_api/routes/public/test_event_logs.py @@ -68,7 +68,7 @@ def setup(self, create_task_instance, session=None) -> dict[str, Log]: dag_id=DAG_ID, task_id=TASK_ID, run_id=DAG_RUN_ID, - execution_date=DAG_EXECUTION_DATE, + logical_date=DAG_EXECUTION_DATE, ) normal_log = Log( event=EVENT_NORMAL, @@ -167,8 +167,8 @@ def test_get_event_log(self, test_client, setup, event_log_key, expected_status_ "map_index": event_log.map_index, "try_number": event_log.try_number, "event": expected_body.get("event"), - "logical_date": event_log.execution_date.isoformat().replace("+00:00", "Z") - if event_log.execution_date + "logical_date": event_log.logical_date.isoformat().replace("+00:00", "Z") + if event_log.logical_date else None, "owner": expected_body.get("owner"), "extra": expected_body.get("extra"), @@ -267,7 +267,7 @@ class TestGetEventLogs(TestEventLogsEndpoint): [EVENT_WITH_OWNER_AND_TASK_INSTANCE, TASK_INSTANCE_EVENT, EVENT_WITH_OWNER, EVENT_NORMAL], ), ( - {"order_by": "execution_date"}, + {"order_by": "logical_date"}, 200, 4, [TASK_INSTANCE_EVENT, EVENT_WITH_OWNER_AND_TASK_INSTANCE, EVENT_NORMAL, EVENT_WITH_OWNER], diff --git a/tests/api_fastapi/core_api/routes/public/test_task_instances.py b/tests/api_fastapi/core_api/routes/public/test_task_instances.py index 88d2b62ed79d..12f6e02a1fa1 100644 --- a/tests/api_fastapi/core_api/routes/public/test_task_instances.py +++ b/tests/api_fastapi/core_api/routes/public/test_task_instances.py @@ -63,7 +63,7 @@ def teardown_method(self): def setup_attrs(self, session) -> None: self.default_time = DEFAULT self.ti_init = { - "execution_date": self.default_time, + "logical_date": self.default_time, "state": State.RUNNING, } self.ti_extras = { @@ -98,7 +98,7 @@ def create_task_instances( counter = min(len(task_instances), counter) run_id = "TEST_DAG_RUN_ID" - execution_date = self.ti_init.pop("execution_date", self.default_time) + logical_date = self.ti_init.pop("logical_date", self.default_time) dr = None tis = [] @@ -110,16 +110,16 @@ def create_task_instances( else: self.ti_init.update(task_instances[i]) - if "execution_date" in self.ti_init: + if "logical_date" in self.ti_init: run_id = f"TEST_DAG_RUN_ID_{i}" - execution_date = self.ti_init.pop("execution_date") + logical_date = self.ti_init.pop("logical_date") dr = None if not dr: dr = DagRun( run_id=run_id, dag_id=dag_id, - execution_date=execution_date, + logical_date=logical_date, run_type=DagRunType.MANUAL, state=dag_run_state, ) @@ -462,7 +462,7 @@ class TestGetMappedTaskInstances: def setup_attrs(self) -> None: self.default_time = DEFAULT_DATETIME_1 self.ti_init = { - "execution_date": self.default_time, + "logical_date": self.default_time, "state": State.RUNNING, } self.ti_extras = { @@ -775,9 +775,9 @@ class TestGetTaskInstances(TestTaskInstanceEndpoint): [ pytest.param( [ - {"execution_date": DEFAULT_DATETIME_1}, - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1)}, - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2)}, + {"logical_date": DEFAULT_DATETIME_1}, + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=1)}, + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=2)}, ], False, "/public/dags/example_python_operator/dagRuns/~/taskInstances", @@ -956,7 +956,7 @@ def test_return_TI_only_from_readable_dags(self, test_client, session): self.create_task_instances( session, task_instances=[ - {"execution_date": DEFAULT_DATETIME_1 + dt.timedelta(days=i)} + {"logical_date": DEFAULT_DATETIME_1 + dt.timedelta(days=i)} for i in range(task_instances[dag_id]) ], dag_id=dag_id, @@ -1072,13 +1072,13 @@ def test_should_respond_empty_non_scheduled(self, test_client, session): { "dependencies": [ { - "name": "Execution Date", - "reason": "The execution date is 2020-01-01T00:00:00+00:00 but this is " + "name": "Logical Date", + "reason": "The logical date is 2020-01-01T00:00:00+00:00 but this is " "before the task's start date 2021-01-01T00:00:00+00:00.", }, { - "name": "Execution Date", - "reason": "The execution date is 2020-01-01T00:00:00+00:00 but this is " + "name": "Logical Date", + "reason": "The logical date is 2020-01-01T00:00:00+00:00 but this is " "before the task's DAG's start date 2021-01-01T00:00:00+00:00.", }, ], @@ -1089,12 +1089,12 @@ def test_should_respond_empty_non_scheduled(self, test_client, session): { "dependencies": [ { - "name": "Execution Date", - "reason": "The execution date is 2020-01-01T00:00:00+00:00 but this is before the task's start date 2021-01-01T00:00:00+00:00.", + "name": "Logical Date", + "reason": "The logical date is 2020-01-01T00:00:00+00:00 but this is before the task's start date 2021-01-01T00:00:00+00:00.", }, { - "name": "Execution Date", - "reason": "The execution date is 2020-01-01T00:00:00+00:00 but this is before the task's DAG's start date 2021-01-01T00:00:00+00:00.", + "name": "Logical Date", + "reason": "The logical date is 2020-01-01T00:00:00+00:00 but this is before the task's DAG's start date 2021-01-01T00:00:00+00:00.", }, {"name": "Task Instance State", "reason": "Task is in the 'None' state."}, ] diff --git a/tests/api_fastapi/core_api/routes/public/test_xcom.py b/tests/api_fastapi/core_api/routes/public/test_xcom.py index 58df4903d425..b35cfe8ab331 100644 --- a/tests/api_fastapi/core_api/routes/public/test_xcom.py +++ b/tests/api_fastapi/core_api/routes/public/test_xcom.py @@ -44,8 +44,8 @@ TEST_TASK_ID = "test-task-id" TEST_EXECUTION_DATE = "2005-04-02T00:00:00+00:00" -execution_date_parsed = timezone.parse(TEST_EXECUTION_DATE) -run_id = DagRun.generate_run_id(DagRunType.MANUAL, execution_date_parsed) +logical_date_parsed = timezone.parse(TEST_EXECUTION_DATE) +run_id = DagRun.generate_run_id(DagRunType.MANUAL, logical_date_parsed) @provide_session @@ -65,8 +65,8 @@ def _create_dag_run(session=None) -> None: dagrun = DagRun( dag_id=TEST_DAG_ID, run_id=run_id, - execution_date=execution_date_parsed, - start_date=execution_date_parsed, + logical_date=logical_date_parsed, + start_date=logical_date_parsed, run_type=DagRunType.MANUAL, ) session.add(dagrun) @@ -114,7 +114,7 @@ def test_should_respond_200_stringify(self, test_client): current_data = response.json() assert current_data == { "dag_id": TEST_DAG_ID, - "execution_date": execution_date_parsed.strftime("%Y-%m-%dT%H:%M:%SZ"), + "logical_date": logical_date_parsed.strftime("%Y-%m-%dT%H:%M:%SZ"), "key": TEST_XCOM_KEY, "task_id": TEST_TASK_ID, "map_index": -1, @@ -132,7 +132,7 @@ def test_should_respond_200_native(self, test_client): current_data = response.json() assert current_data == { "dag_id": TEST_DAG_ID, - "execution_date": execution_date_parsed.strftime("%Y-%m-%dT%H:%M:%SZ"), + "logical_date": logical_date_parsed.strftime("%Y-%m-%dT%H:%M:%SZ"), "key": TEST_XCOM_KEY, "task_id": TEST_TASK_ID, "map_index": -1, @@ -151,7 +151,7 @@ def test_should_respond_200_pickled(self, test_client): current_data = response.json() assert current_data == { "dag_id": TEST_DAG_ID, - "execution_date": execution_date_parsed.strftime("%Y-%m-%dT%H:%M:%SZ"), + "logical_date": logical_date_parsed.strftime("%Y-%m-%dT%H:%M:%SZ"), "key": TEST_XCOM_KEY2, "task_id": TEST_TASK_ID, "map_index": -1, diff --git a/tests/api_fastapi/core_api/routes/ui/test_dags.py b/tests/api_fastapi/core_api/routes/ui/test_dags.py index 7258476bf163..bb1331c55299 100644 --- a/tests/api_fastapi/core_api/routes/ui/test_dags.py +++ b/tests/api_fastapi/core_api/routes/ui/test_dags.py @@ -52,7 +52,7 @@ def setup_dag_runs(self, session=None) -> None: run_id=f"run_id_{i+1}", run_type=DagRunType.MANUAL, start_date=start_date, - execution_date=start_date, + logical_date=start_date, state=(DagRunState.FAILED if i % 2 == 0 else DagRunState.SUCCESS), triggered_by=DagRunTriggeredByType.TEST, ) @@ -84,7 +84,6 @@ def test_recent_dag_runs(self, test_client, query_params, expected_ids, expected response = test_client.get("/ui/dags/recent_dag_runs", params=query_params) assert response.status_code == 200 body = response.json() - assert body["total_entries"] == len(expected_ids) required_dag_run_key = [ "run_id", "dag_id", @@ -94,11 +93,11 @@ def test_recent_dag_runs(self, test_client, query_params, expected_ids, expected for recent_dag_runs in body["dags"]: dag_runs = recent_dag_runs["latest_dag_runs"] # check date ordering - previous_execution_date = None + previous_logical_date = None for dag_run in dag_runs: # validate the response for key in required_dag_run_key: assert key in dag_run - if previous_execution_date: - assert previous_execution_date > dag_run["logical_date"] - previous_execution_date = dag_run["logical_date"] + if previous_logical_date: + assert previous_logical_date > dag_run["logical_date"] + previous_logical_date = dag_run["logical_date"] diff --git a/tests/api_fastapi/core_api/routes/ui/test_dashboard.py b/tests/api_fastapi/core_api/routes/ui/test_dashboard.py index e96a31606549..39e8e3386f86 100644 --- a/tests/api_fastapi/core_api/routes/ui/test_dashboard.py +++ b/tests/api_fastapi/core_api/routes/ui/test_dashboard.py @@ -68,7 +68,7 @@ def make_dag_runs(dag_maker, session, time_machine): run_id="run_1", state=DagRunState.SUCCESS, run_type=DagRunType.SCHEDULED, - execution_date=date, + logical_date=date, start_date=date, ) @@ -76,7 +76,7 @@ def make_dag_runs(dag_maker, session, time_machine): run_id="run_2", state=DagRunState.FAILED, run_type=DagRunType.ASSET_TRIGGERED, - execution_date=date + timedelta(days=1), + logical_date=date + timedelta(days=1), start_date=date + timedelta(days=1), ) @@ -84,7 +84,7 @@ def make_dag_runs(dag_maker, session, time_machine): run_id="run_3", state=DagRunState.RUNNING, run_type=DagRunType.SCHEDULED, - execution_date=pendulum.DateTime(2023, 2, 3, 0, 0, 0, tzinfo=pendulum.UTC), + logical_date=pendulum.DateTime(2023, 2, 3, 0, 0, 0, tzinfo=pendulum.UTC), start_date=pendulum.DateTime(2023, 2, 3, 0, 0, 0, tzinfo=pendulum.UTC), ) run3.end_date = None diff --git a/tests/assets/test_manager.py b/tests/assets/test_manager.py index 3310502a97ba..9878cec6f5ba 100644 --- a/tests/assets/test_manager.py +++ b/tests/assets/test_manager.py @@ -66,7 +66,7 @@ def mock_task_instance(): map_index="13", start_date=datetime.now(), end_date=datetime.now(), - execution_date=datetime.now(), + logical_date=datetime.now(), duration=0.1, state="success", try_number=1, diff --git a/tests/cli/commands/test_dag_command.py b/tests/cli/commands/test_dag_command.py index 7acc3a85b934..7b9dd1e63bd1 100644 --- a/tests/cli/commands/test_dag_command.py +++ b/tests/cli/commands/test_dag_command.py @@ -490,7 +490,7 @@ def test_trigger_dag(self): assert dagrun.conf == {"foo": "bar"} # Coerced to UTC. - assert dagrun.execution_date.isoformat(timespec="seconds") == "2021-06-04T01:00:00+00:00" + assert dagrun.logical_date.isoformat(timespec="seconds") == "2021-06-04T01:00:00+00:00" # example_bash_operator runs every day at midnight, so the data interval # should be aligned to the previous day. @@ -517,7 +517,7 @@ def test_trigger_dag_with_microseconds(self): assert dagrun, "DagRun not created" assert dagrun.run_type == DagRunType.MANUAL assert dagrun.external_trigger - assert dagrun.execution_date.isoformat(timespec="microseconds") == "2021-06-04T01:00:00.000001+00:00" + assert dagrun.logical_date.isoformat(timespec="microseconds") == "2021-06-04T01:00:00.000001+00:00" def test_trigger_dag_invalid_conf(self): with pytest.raises(ValueError): @@ -605,7 +605,7 @@ def test_dag_test(self, mock_get_dag): [ mock.call(subdir=cli_args.subdir, dag_id="example_bash_operator"), mock.call().test( - execution_date=timezone.parse(DEFAULT_DATE.isoformat()), + logical_date=timezone.parse(DEFAULT_DATE.isoformat()), run_conf=None, use_executor=False, session=mock.ANY, @@ -616,22 +616,22 @@ def test_dag_test(self, mock_get_dag): @mock.patch("airflow.cli.commands.dag_command.get_dag") def test_dag_test_fail_raise_error(self, mock_get_dag): - execution_date_str = DEFAULT_DATE.isoformat() + logical_date_str = DEFAULT_DATE.isoformat() mock_get_dag.return_value.test.return_value = DagRun( - dag_id="example_bash_operator", execution_date=DEFAULT_DATE, state=DagRunState.FAILED + dag_id="example_bash_operator", logical_date=DEFAULT_DATE, state=DagRunState.FAILED ) - cli_args = self.parser.parse_args(["dags", "test", "example_bash_operator", execution_date_str]) + cli_args = self.parser.parse_args(["dags", "test", "example_bash_operator", logical_date_str]) with pytest.raises(SystemExit, match=r"DagRun failed"): dag_command.dag_test(cli_args) @mock.patch("airflow.cli.commands.dag_command.get_dag") @mock.patch("airflow.utils.timezone.utcnow") - def test_dag_test_no_execution_date(self, mock_utcnow, mock_get_dag): + def test_dag_test_no_logical_date(self, mock_utcnow, mock_get_dag): now = pendulum.now() mock_utcnow.return_value = now cli_args = self.parser.parse_args(["dags", "test", "example_bash_operator"]) - assert cli_args.execution_date is None + assert cli_args.logical_date is None dag_command.dag_test(cli_args) @@ -639,7 +639,7 @@ def test_dag_test_no_execution_date(self, mock_utcnow, mock_get_dag): [ mock.call(subdir=cli_args.subdir, dag_id="example_bash_operator"), mock.call().test( - execution_date=mock.ANY, + logical_date=mock.ANY, run_conf=None, use_executor=False, session=mock.ANY, @@ -666,7 +666,7 @@ def test_dag_test_conf(self, mock_get_dag): [ mock.call(subdir=cli_args.subdir, dag_id="example_bash_operator"), mock.call().test( - execution_date=timezone.parse(DEFAULT_DATE.isoformat()), + logical_date=timezone.parse(DEFAULT_DATE.isoformat()), run_conf={"dag_run_conf_param": "param_value"}, use_executor=False, session=mock.ANY, @@ -690,7 +690,7 @@ def test_dag_test_show_dag(self, mock_get_dag, mock_render_dag): [ mock.call(subdir=cli_args.subdir, dag_id="example_bash_operator"), mock.call().test( - execution_date=timezone.parse(DEFAULT_DATE.isoformat()), + logical_date=timezone.parse(DEFAULT_DATE.isoformat()), run_conf=None, use_executor=False, session=mock.ANY, diff --git a/tests/cli/commands/test_task_command.py b/tests/cli/commands/test_task_command.py index 50e3d393f810..1ba711ca4b22 100644 --- a/tests/cli/commands/test_task_command.py +++ b/tests/cli/commands/test_task_command.py @@ -106,7 +106,7 @@ def setup_class(cls): state=State.NONE, run_id=cls.run_id, run_type=DagRunType.MANUAL, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=data_interval, **triggered_by_kwargs, ) @@ -136,7 +136,7 @@ def test_test(self): @pytest.mark.filterwarnings("ignore::airflow.utils.context.AirflowContextDeprecationWarning") @mock.patch("airflow.utils.timezone.utcnow") - def test_test_no_execution_date(self, mock_utcnow): + def test_test_no_logical_date(self, mock_utcnow): """Test the `airflow test` command""" now = pendulum.now("UTC") mock_utcnow.return_value = now @@ -149,40 +149,6 @@ def test_test_no_execution_date(self, mock_utcnow): # Check that prints, and log messages, are shown assert f"'example_python_operator__print_the_context__{ds}'" in stdout.getvalue() - @pytest.mark.filterwarnings("ignore::airflow.utils.context.AirflowContextDeprecationWarning") - def test_test_with_existing_dag_run(self, caplog): - """Test the `airflow test` command""" - task_id = "print_the_context" - args = self.parser.parse_args(["tasks", "test", self.dag_id, task_id, DEFAULT_DATE.isoformat()]) - with caplog.at_level("INFO", logger="airflow.task"): - task_command.task_test(args) - assert ( - f"Marking task as SUCCESS. dag_id={self.dag_id}, task_id={task_id}, run_id={self.run_id}, " - in caplog.text - ) - - @pytest.mark.enable_redact - @pytest.mark.filterwarnings("ignore::airflow.utils.context.AirflowContextDeprecationWarning") - def test_test_filters_secrets(self, capsys): - """Test ``airflow test`` does not print secrets to stdout. - - Output should be filtered by SecretsMasker. - """ - password = "somepassword1234!" - logging.getLogger("airflow.task").filters[0].add_mask(password) - args = self.parser.parse_args( - ["tasks", "test", "example_python_operator", "print_the_context", "2018-01-01"], - ) - - with mock.patch("airflow.models.TaskInstance.run", side_effect=lambda *_, **__: print(password)): - task_command.task_test(args) - assert capsys.readouterr().out.endswith("***\n") - - not_password = "!4321drowssapemos" - with mock.patch("airflow.models.TaskInstance.run", side_effect=lambda *_, **__: print(not_password)): - task_command.task_test(args) - assert capsys.readouterr().out.endswith(f"{not_password}\n") - def test_cli_test_different_path(self, session, tmp_path): """ When the dag processor has a different dags folder @@ -198,14 +164,14 @@ def test_cli_test_different_path(self, session, tmp_path): dag = dagbag.get_dag("test_dags_folder") dagbag.sync_to_db(session=session) - execution_date = pendulum.now("UTC") - data_interval = dag.timetable.infer_manual_data_interval(run_after=execution_date) + logical_date = pendulum.now("UTC") + data_interval = dag.timetable.infer_manual_data_interval(run_after=logical_date) triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dag.create_dagrun( state=State.NONE, run_id="abc123", run_type=DagRunType.MANUAL, - execution_date=execution_date, + logical_date=logical_date, data_interval=data_interval, session=session, **triggered_by_kwargs, @@ -261,6 +227,54 @@ def test_cli_test_different_path(self, session, tmp_path): # verify that the file was in different location when run assert ti.xcom_pull(ti.task_id) == new_file_path.as_posix() + @mock.patch("airflow.cli.commands.task_command.select") + @mock.patch("sqlalchemy.orm.session.Session.scalars") + @mock.patch("airflow.cli.commands.task_command.DagRun") + def test_task_render_with_custom_timetable(self, mock_dagrun, mock_scalars, mock_select): + """ + when calling `tasks render` on dag with custom timetable, the DagRun object should be created with + data_intervals. + """ + mock_scalars.side_effect = sqlalchemy.exc.NoResultFound + task_command.task_render( + self.parser.parse_args(["tasks", "render", "example_workday_timetable", "run_this", "2022-01-01"]) + ) + assert "data_interval" in mock_dagrun.call_args.kwargs + + @pytest.mark.filterwarnings("ignore::airflow.utils.context.AirflowContextDeprecationWarning") + def test_test_with_existing_dag_run(self, caplog): + """Test the `airflow test` command""" + task_id = "print_the_context" + args = self.parser.parse_args(["tasks", "test", self.dag_id, task_id, DEFAULT_DATE.isoformat()]) + with caplog.at_level("INFO", logger="airflow.task"): + task_command.task_test(args) + assert ( + f"Marking task as SUCCESS. dag_id={self.dag_id}, task_id={task_id}, run_id={self.run_id}, " + in caplog.text + ) + + @pytest.mark.enable_redact + @pytest.mark.filterwarnings("ignore::airflow.utils.context.AirflowContextDeprecationWarning") + def test_test_filters_secrets(self, capsys): + """Test ``airflow test`` does not print secrets to stdout. + + Output should be filtered by SecretsMasker. + """ + password = "somepassword1234!" + logging.getLogger("airflow.task").filters[0].add_mask(password) + args = self.parser.parse_args( + ["tasks", "test", "example_python_operator", "print_the_context", "2018-01-01"], + ) + + with mock.patch("airflow.models.TaskInstance.run", side_effect=lambda *_, **__: print(password)): + task_command.task_test(args) + assert capsys.readouterr().out.endswith("***\n") + + not_password = "!4321drowssapemos" + with mock.patch("airflow.models.TaskInstance.run", side_effect=lambda *_, **__: print(not_password)): + task_command.task_test(args) + assert capsys.readouterr().out.endswith(f"{not_password}\n") + @mock.patch("airflow.cli.commands.task_command.LocalTaskJobRunner") def test_run_with_existing_dag_run_id(self, mock_local_job_runner): """ @@ -589,20 +603,6 @@ def test_mapped_task_render_with_template(self, dag_maker): assert 'echo "2022-01-01"' in output assert 'echo "2022-01-08"' in output - @mock.patch("airflow.cli.commands.task_command.select") - @mock.patch("sqlalchemy.orm.session.Session.scalars") - @mock.patch("airflow.cli.commands.task_command.DagRun") - def test_task_render_with_custom_timetable(self, mock_dagrun, mock_scalars, mock_select): - """ - when calling `tasks render` on dag with custom timetable, the DagRun object should be created with - data_intervals. - """ - mock_scalars.side_effect = sqlalchemy.exc.NoResultFound - task_command.task_render( - self.parser.parse_args(["tasks", "render", "example_workday_timetable", "run_this", "2022-01-01"]) - ) - assert "data_interval" in mock_dagrun.call_args.kwargs - def test_task_state(self): task_command.task_state( self.parser.parse_args( @@ -619,7 +619,7 @@ def test_task_states_for_dag_run(self): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.CLI} if AIRFLOW_V_3_0_PLUS else {} dagrun = dag2.create_dagrun( state=State.RUNNING, - execution_date=default_date2, + logical_date=default_date2, data_interval=data_interval, run_type=DagRunType.MANUAL, external_trigger=True, @@ -648,7 +648,7 @@ def test_task_states_for_dag_run(self): assert len(actual_out) == 1 assert actual_out[0] == { "dag_id": "example_python_operator", - "execution_date": "2016-01-09T00:00:00+00:00", + "logical_date": "2016-01-09T00:00:00+00:00", "task_id": "print_the_context", "state": "success", "start_date": ti_start.isoformat(), @@ -687,20 +687,20 @@ def setup_method(self) -> None: self.run_id = "test_run" self.dag_path = os.path.join(ROOT_FOLDER, "dags", "test_logging_in_dag.py") reset(self.dag_id) - self.execution_date = timezone.datetime(2017, 1, 1) - self.execution_date_str = self.execution_date.isoformat() - self.task_args = ["tasks", "run", self.dag_id, self.task_id, "--local", self.execution_date_str] + self.logical_date = timezone.datetime(2017, 1, 1) + self.logical_date_str = self.logical_date.isoformat() + self.task_args = ["tasks", "run", self.dag_id, self.task_id, "--local", self.logical_date_str] self.log_dir = conf.get_mandatory_value("logging", "base_log_folder") self.log_filename = f"dag_id={self.dag_id}/run_id={self.run_id}/task_id={self.task_id}/attempt=1.log" self.ti_log_file_path = os.path.join(self.log_dir, self.log_filename) self.parser = cli_parser.get_parser() dag = DagBag().get_dag(self.dag_id) - data_interval = dag.timetable.infer_manual_data_interval(run_after=self.execution_date) + data_interval = dag.timetable.infer_manual_data_interval(run_after=self.logical_date) triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} self.dr = dag.create_dagrun( run_id=self.run_id, - execution_date=self.execution_date, + logical_date=self.logical_date, data_interval=data_interval, start_date=timezone.utcnow(), state=State.RUNNING, @@ -868,7 +868,7 @@ def test_logging_with_run_task(self, session): assert ( f"INFO - Marking task as SUCCESS. dag_id={self.dag_id}, " - f"task_id={self.task_id}, run_id={self.run_id}, execution_date=20170101T000000" in logs + f"task_id={self.task_id}, run_id={self.run_id}, logical_date=20170101T000000" in logs ) @pytest.mark.skipif(not hasattr(os, "fork"), reason="Forking not available") @@ -911,7 +911,7 @@ def test_logging_with_run_task_subprocess(self, session): assert f"INFO - Running: ['airflow', 'tasks', 'run', '{self.dag_id}', '{self.task_id}'," in logs assert ( f"INFO - Marking task as SUCCESS. dag_id={self.dag_id}, " - f"task_id={self.task_id}, run_id={self.run_id}, execution_date=20170101T000000" in logs + f"task_id={self.task_id}, run_id={self.run_id}, logical_date=20170101T000000" in logs ) def test_log_file_template_with_run_task(self, session): @@ -995,17 +995,17 @@ def test_context_with_run(): run_id = "test_run" dag_path = os.path.join(ROOT_FOLDER, "dags", "test_parsing_context.py") reset(dag_id) - execution_date = timezone.datetime(2017, 1, 1) - execution_date_str = execution_date.isoformat() - task_args = ["tasks", "run", dag_id, task_id, "--local", execution_date_str] + logical_date = timezone.datetime(2017, 1, 1) + logical_date_str = logical_date.isoformat() + task_args = ["tasks", "run", dag_id, task_id, "--local", logical_date_str] parser = cli_parser.get_parser() dag = DagBag().get_dag(dag_id) - data_interval = dag.timetable.infer_manual_data_interval(run_after=execution_date) + data_interval = dag.timetable.infer_manual_data_interval(run_after=logical_date) triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dag.create_dagrun( run_id=run_id, - execution_date=execution_date, + logical_date=logical_date, data_interval=data_interval, start_date=timezone.utcnow(), state=State.RUNNING, diff --git a/tests/core/test_core.py b/tests/core/test_core.py index 21b5037ed17d..0feaeef20590 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -86,33 +86,6 @@ def sleep_and_catch_other_exceptions(): with pytest.raises(AirflowTaskTimeout): op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) - def test_externally_triggered_dagrun(self, dag_maker): - TI = TaskInstance - - # Create the dagrun between two "scheduled" execution dates of the DAG - execution_date = DEFAULT_DATE + timedelta(days=2) - execution_ds = execution_date.strftime("%Y-%m-%d") - execution_ds_nodash = execution_ds.replace("-", "") - - with dag_maker(schedule=timedelta(weeks=1), serialized=True): - task = EmptyOperator(task_id="test_externally_triggered_dag_context") - dr = dag_maker.create_dagrun( - run_type=DagRunType.SCHEDULED, - execution_date=execution_date, - external_trigger=True, - ) - task.run(start_date=execution_date, end_date=execution_date) - - ti = TI(task=task, run_id=dr.run_id) - ti.refresh_from_db() - context = ti.get_template_context() - - # next_ds should be the execution date for manually triggered runs - with pytest.deprecated_call(): - assert context["next_ds"] == execution_ds - with pytest.deprecated_call(): - assert context["next_ds_nodash"] == execution_ds_nodash - def test_dag_params_and_task_params(self, dag_maker): # This test case guards how params of DAG and Operator work together. # - If any key exists in either DAG's or Operator's params, diff --git a/tests/core/test_example_dags_system.py b/tests/core/test_example_dags_system.py index 5724240eaebf..76c1cefa3144 100644 --- a/tests/core/test_example_dags_system.py +++ b/tests/core/test_example_dags_system.py @@ -135,7 +135,7 @@ def test_dag_root_task_start_date_future(self, dag_maker, session): python_callable=lambda: print("hello"), start_date=fut_start_date, ) - run = get_test_run(dag, execution_date=exec_date) + run = get_test_run(dag, logical_date=exec_date) run() dr = session.scalar(select(DagRun)) tis = dr.task_instances diff --git a/tests/core/test_logging_config.py b/tests/core/test_logging_config.py index 6464e93bdf6b..29dae7ff712a 100644 --- a/tests/core/test_logging_config.py +++ b/tests/core/test_logging_config.py @@ -243,7 +243,7 @@ def test_when_the_config_key_does_not_exists(self): with conf_vars({("logging", "logging_config_class"): None}): with patch.object(logging_config.log, "debug") as mock_debug: logging_config.configure_logging() - mock_debug.assert_any_call("Could not find key logging_config_class in config") + mock_debug.assert_any_call("Unable to load custom logging, using default config instead") # Just default def test_loading_local_settings_without_logging_config(self): diff --git a/tests/core/test_sentry.py b/tests/core/test_sentry.py index 7ec82fd5a56f..5dd592392043 100644 --- a/tests/core/test_sentry.py +++ b/tests/core/test_sentry.py @@ -33,9 +33,9 @@ from tests_common.test_utils.config import conf_vars -EXECUTION_DATE = timezone.utcnow() +LOGICAL_DATE = timezone.utcnow() SCHEDULE_INTERVAL = datetime.timedelta(days=1) -DATA_INTERVAL = (EXECUTION_DATE, EXECUTION_DATE + SCHEDULE_INTERVAL) +DATA_INTERVAL = (LOGICAL_DATE, LOGICAL_DATE + SCHEDULE_INTERVAL) DAG_ID = "test_dag" TASK_ID = "test_task" OPERATOR = "PythonOperator" @@ -46,7 +46,7 @@ "task_id": TASK_ID, "data_interval_start": DATA_INTERVAL[0], "data_interval_end": DATA_INTERVAL[1], - "execution_date": EXECUTION_DATE, + "logical_date": LOGICAL_DATE, "operator": OPERATOR, "try_number": TRY_NUMBER, } @@ -82,7 +82,7 @@ def task_instance(self, dag_maker): with dag_maker(DAG_ID, schedule=SCHEDULE_INTERVAL, serialized=True): task = PythonOperator(task_id=TASK_ID, python_callable=int) - dr = dag_maker.create_dagrun(data_interval=DATA_INTERVAL, execution_date=EXECUTION_DATE) + dr = dag_maker.create_dagrun(data_interval=DATA_INTERVAL, logical_date=LOGICAL_DATE) ti = dr.task_instances[0] ti.state = STATE ti.task = task diff --git a/tests/dag_processing/test_collection.py b/tests/dag_processing/test_collection.py index 4d5a6736ad07..7f7cced9ad1f 100644 --- a/tests/dag_processing/test_collection.py +++ b/tests/dag_processing/test_collection.py @@ -33,7 +33,7 @@ def test_statement_latest_runs_one_dag(): compiled_stmt = str(stmt.compile()) actual = [x.strip() for x in compiled_stmt.splitlines()] expected = [ - "SELECT dag_run.logical_date, dag_run.id, dag_run.dag_id, " + "SELECT dag_run.id, dag_run.dag_id, dag_run.logical_date, " "dag_run.data_interval_start, dag_run.data_interval_end", "FROM dag_run", "WHERE dag_run.dag_id = :dag_id_1 AND dag_run.logical_date = (" @@ -52,7 +52,7 @@ def test_statement_latest_runs_many_dag(): compiled_stmt = str(stmt.compile()) actual = [x.strip() for x in compiled_stmt.splitlines()] expected = [ - "SELECT dag_run.logical_date, dag_run.id, dag_run.dag_id, " + "SELECT dag_run.id, dag_run.dag_id, dag_run.logical_date, " "dag_run.data_interval_start, dag_run.data_interval_end", "FROM dag_run, (SELECT dag_run.dag_id AS dag_id, " "max(dag_run.logical_date) AS max_execution_date", diff --git a/tests/dag_processing/test_processor.py b/tests/dag_processing/test_processor.py index 29aa4e15a388..eb263a47e1d2 100644 --- a/tests/dag_processing/test_processor.py +++ b/tests/dag_processing/test_processor.py @@ -127,7 +127,7 @@ def test_execute_on_failure_callbacks(self, mock_ti_handle_failure): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dagrun = dag.create_dagrun( state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.SCHEDULED, data_interval=dag.infer_automated_data_interval(DEFAULT_DATE), session=session, @@ -165,7 +165,7 @@ def test_execute_on_failure_callbacks_without_dag(self, mock_ti_handle_failure, triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dagrun = dag.create_dagrun( state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.SCHEDULED, data_interval=dag.infer_automated_data_interval(DEFAULT_DATE), session=session, @@ -203,7 +203,7 @@ def test_failure_callbacks_should_not_drop_hostname(self): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dagrun = dag.create_dagrun( state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.SCHEDULED, data_interval=dag.infer_automated_data_interval(DEFAULT_DATE), session=session, @@ -239,7 +239,7 @@ def test_process_file_should_failure_callback(self, monkeypatch, tmp_path, get_t triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dagrun = dag.create_dagrun( state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.SCHEDULED, data_interval=dag.infer_automated_data_interval(DEFAULT_DATE), session=session, diff --git a/tests/dags/test_cli_triggered_dags.py b/tests/dags/test_cli_triggered_dags.py index 530c80f3f7ca..378e2497f281 100644 --- a/tests/dags/test_cli_triggered_dags.py +++ b/tests/dags/test_cli_triggered_dags.py @@ -32,7 +32,7 @@ def fail(): def success(ti=None, *args, **kwargs): - if ti.execution_date != DEFAULT_DATE + timedelta(days=1): + if ti.logical_date != DEFAULT_DATE + timedelta(days=1): fail() diff --git a/tests/decorators/test_branch_external_python.py b/tests/decorators/test_branch_external_python.py index d991f22cd55e..af8bce326579 100644 --- a/tests/decorators/test_branch_external_python.py +++ b/tests/decorators/test_branch_external_python.py @@ -72,12 +72,12 @@ def branch_operator(): branchoperator.set_downstream(task_2) dr = dag_maker.create_dagrun() - df.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) + df.operator.run(start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True) branchoperator.operator.run( - start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True + start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True ) - task_1.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) - task_2.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) + task_1.operator.run(start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True) + task_2.operator.run(start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True) tis = dr.get_task_instances() for ti in tis: diff --git a/tests/decorators/test_branch_python.py b/tests/decorators/test_branch_python.py index 58bb21624604..1768127a405c 100644 --- a/tests/decorators/test_branch_python.py +++ b/tests/decorators/test_branch_python.py @@ -60,12 +60,12 @@ def branch_operator(): branchoperator.set_downstream(task_2) dr = dag_maker.create_dagrun() - df.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) + df.operator.run(start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True) branchoperator.operator.run( - start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True + start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True ) - task_1.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) - task_2.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) + task_1.operator.run(start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True) + task_2.operator.run(start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True) tis = dr.get_task_instances() for ti in tis: diff --git a/tests/decorators/test_branch_virtualenv.py b/tests/decorators/test_branch_virtualenv.py index a5c23de392de..02f5c20e1eb6 100644 --- a/tests/decorators/test_branch_virtualenv.py +++ b/tests/decorators/test_branch_virtualenv.py @@ -79,12 +79,12 @@ def branch_operator(): branchoperator.set_downstream(task_2) dr = dag_maker.create_dagrun() - df.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) + df.operator.run(start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True) branchoperator.operator.run( - start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True + start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True ) - task_1.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) - task_2.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) + task_1.operator.run(start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True) + task_2.operator.run(start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True) tis = dr.get_task_instances() for ti in tis: diff --git a/tests/decorators/test_external_python.py b/tests/decorators/test_external_python.py index 5ed5874e3a55..77baeb013453 100644 --- a/tests/decorators/test_external_python.py +++ b/tests/decorators/test_external_python.py @@ -44,7 +44,7 @@ TI_CONTEXT_ENV_VARS = [ "AIRFLOW_CTX_DAG_ID", "AIRFLOW_CTX_TASK_ID", - "AIRFLOW_CTX_EXECUTION_DATE", + "AIRFLOW_CTX_LOGICAL_DATE", "AIRFLOW_CTX_DAG_RUN_ID", ] diff --git a/tests/decorators/test_python.py b/tests/decorators/test_python.py index 44fa54d7dd80..668d0be99b65 100644 --- a/tests/decorators/test_python.py +++ b/tests/decorators/test_python.py @@ -458,7 +458,7 @@ def return_dict(number: int): dr = self.dag_non_serialized.create_dagrun( run_id=DagRunType.MANUAL, start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, state=State.RUNNING, data_interval=self.dag_non_serialized.timetable.infer_manual_data_interval( run_after=DEFAULT_DATE @@ -523,7 +523,7 @@ def add_num(number: int, num2: int = 2): dr = self.dag_non_serialized.create_dagrun( run_id=DagRunType.MANUAL, start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, state=State.RUNNING, data_interval=self.dag_non_serialized.timetable.infer_manual_data_interval( run_after=DEFAULT_DATE diff --git a/tests/decorators/test_python_virtualenv.py b/tests/decorators/test_python_virtualenv.py index 00964b00818f..81f02351670b 100644 --- a/tests/decorators/test_python_virtualenv.py +++ b/tests/decorators/test_python_virtualenv.py @@ -390,7 +390,7 @@ def in_venv(value: dict[str, _Invalid]) -> _Invalid: ret = in_venv(value) dr = dag_maker.create_dagrun() - ret.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + ret.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) ti = dr.get_task_instances()[0] assert ti.state == TaskInstanceState.SUCCESS diff --git a/tests/decorators/test_sensor.py b/tests/decorators/test_sensor.py index e970894a38ed..3b48d13f5260 100644 --- a/tests/decorators/test_sensor.py +++ b/tests/decorators/test_sensor.py @@ -52,7 +52,7 @@ def dummy_f(): sf >> df dr = dag_maker.create_dagrun() - sf.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) + sf.operator.run(start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True) tis = dr.get_task_instances() assert len(tis) == 2 for ti in tis: @@ -81,7 +81,7 @@ def dummy_f(): sf >> df dr = dag_maker.create_dagrun() - sf.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) + sf.operator.run(start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True) tis = dr.get_task_instances() assert len(tis) == 2 for ti in tis: @@ -107,7 +107,7 @@ def dummy_f(): dr = dag_maker.create_dagrun() with pytest.raises(AirflowSensorTimeout): - sf.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) + sf.operator.run(start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True) tis = dr.get_task_instances() assert len(tis) == 2 @@ -134,7 +134,7 @@ def dummy_f(): dr = dag_maker.create_dagrun() with pytest.raises(AirflowSensorTimeout): - sf.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) + sf.operator.run(start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True) tis = dr.get_task_instances() assert len(tis) == 2 @@ -160,7 +160,7 @@ def dummy_f(): sf >> df dr = dag_maker.create_dagrun() - sf.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) + sf.operator.run(start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True) tis = dr.get_task_instances() assert len(tis) == 2 for ti in tis: @@ -185,7 +185,7 @@ def dummy_f(): sf >> df dr = dag_maker.create_dagrun() - sf.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) + sf.operator.run(start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True) tis = dr.get_task_instances() assert len(tis) == 2 for ti in tis: @@ -213,8 +213,8 @@ def sensor_f(n: int): sf = sensor_f(uf) dr = dag_maker.create_dagrun() - uf.operator.run(start_date=dr.execution_date, end_date=dr.execution_date, ignore_ti_state=True) - sf.operator.run(start_date=dr.execution_date, end_date=dr.execution_date) + uf.operator.run(start_date=dr.logical_date, end_date=dr.logical_date, ignore_ti_state=True) + sf.operator.run(start_date=dr.logical_date, end_date=dr.logical_date) tis = dr.get_task_instances() assert len(tis) == 2 for ti in tis: diff --git a/tests/executors/test_base_executor.py b/tests/executors/test_base_executor.py index be3ad517d70c..b55b74c2c556 100644 --- a/tests/executors/test_base_executor.py +++ b/tests/executors/test_base_executor.py @@ -202,7 +202,7 @@ def setup_dagrun(dag_maker): BaseOperator(task_id="task_2", start_date=start_date) BaseOperator(task_id="task_3", start_date=start_date) - return dag_maker.create_dagrun(execution_date=date) + return dag_maker.create_dagrun(logical_date=date) @pytest.mark.db_test diff --git a/tests/executors/test_local_executor.py b/tests/executors/test_local_executor.py index 7bd4fbec203b..f6ba8dca464d 100644 --- a/tests/executors/test_local_executor.py +++ b/tests/executors/test_local_executor.py @@ -78,14 +78,14 @@ def _test_execute(self, parallelism, success_command, fail_command): success_key = "success {}" assert executor.result_queue.empty() - execution_date = datetime.datetime.now() + logical_date = datetime.datetime.now() for i in range(self.TEST_SUCCESS_COMMANDS): key_id, command = success_key.format(i), success_command - key = key_id, "fake_ti", execution_date, 0 + key = key_id, "fake_ti", logical_date, 0 executor.running.add(key) executor.execute_async(key=key, command=command) - fail_key = "fail", "fake_ti", execution_date, 0 + fail_key = "fail", "fake_ti", logical_date, 0 executor.running.add(fail_key) executor.execute_async(key=fail_key, command=fail_command) @@ -95,7 +95,7 @@ def _test_execute(self, parallelism, success_command, fail_command): for i in range(self.TEST_SUCCESS_COMMANDS): key_id = success_key.format(i) - key = key_id, "fake_ti", execution_date, 0 + key = key_id, "fake_ti", logical_date, 0 assert executor.event_buffer[key][0] == State.SUCCESS assert executor.event_buffer[fail_key][0] == State.FAILED diff --git a/tests/jobs/test_local_task_job.py b/tests/jobs/test_local_task_job.py index 936f773e55fc..cfcd9ecd0b6e 100644 --- a/tests/jobs/test_local_task_job.py +++ b/tests/jobs/test_local_task_job.py @@ -315,7 +315,7 @@ def test_heartbeat_failed_fast(self): dr = dag.create_dagrun( run_id="test_heartbeat_failed_fast_run", state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, start_date=DEFAULT_DATE, session=session, data_interval=data_interval, @@ -353,7 +353,7 @@ def test_mark_success_no_kill(self, caplog, get_test_dag, session): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dr = dag.create_dagrun( state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.SCHEDULED, session=session, data_interval=data_interval, @@ -389,7 +389,7 @@ def test_localtaskjob_double_trigger(self): dr = dag.create_dagrun( run_id="test", state=State.SUCCESS, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, start_date=DEFAULT_DATE, session=session, data_interval=data_interval, @@ -495,7 +495,7 @@ def test_mark_failure_on_failure_callback(self, caplog, get_test_dag): with create_session() as session: dr = dag.create_dagrun( state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.SCHEDULED, session=session, data_interval=data_interval, @@ -532,7 +532,7 @@ def test_dagrun_timeout_logged_in_task_logs(self, caplog, get_test_dag): dr = dag.create_dagrun( state=State.RUNNING, start_date=DEFAULT_DATE, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.SCHEDULED, session=session, data_interval=data_interval, @@ -567,7 +567,7 @@ def test_failure_callback_called_by_airflow_run_raw_process(self, monkeypatch, t with create_session() as session: dr = dag.create_dagrun( state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.SCHEDULED, session=session, data_interval=data_interval, @@ -604,7 +604,7 @@ def test_mark_success_on_success_callback(self, caplog, get_test_dag): with create_session() as session: dr = dag.create_dagrun( state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.SCHEDULED, session=session, data_interval=data_interval, @@ -621,9 +621,7 @@ def test_mark_success_on_success_callback(self, caplog, get_test_dag): # This should run fast because of the return_code=None run_job(job=job, execute_callable=job_runner._execute) ti.refresh_from_db() - assert ( - "State of this instance has been externally set to success. Terminating instance." in caplog.text - ) + assert ti.state == State.SUCCESS, "Task instance was not marked as SUCCESS as expected" @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode def test_success_listeners_executed(self, caplog, get_test_dag): @@ -643,7 +641,7 @@ def test_success_listeners_executed(self, caplog, get_test_dag): with create_session() as session: dr = dag.create_dagrun( state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.SCHEDULED, session=session, data_interval=data_interval, @@ -683,7 +681,7 @@ def test_success_slow_listeners_executed_kill(self, caplog, get_test_dag): with create_session() as session: dr = dag.create_dagrun( state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.SCHEDULED, session=session, data_interval=data_interval, @@ -723,7 +721,7 @@ def test_success_slow_task_not_killed_by_overtime_but_regular_timeout(self, capl with create_session() as session: dr = dag.create_dagrun( state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.SCHEDULED, session=session, data_interval=data_interval, @@ -767,7 +765,7 @@ def test_process_os_signal_calls_on_failure_callback( with create_session() as session: dag.create_dagrun( state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.SCHEDULED, session=session, data_interval=data_interval, @@ -937,8 +935,8 @@ def test_process_sigterm_works_with_retries(self, mp_method, wait_timeout, daemo dag_id = f"test_task_runner_sigterm_{mp_method}_{'' if daemon else 'non_'}daemon" task_id = "test_on_retry_callback" - execution_date = DEFAULT_DATE - run_id = f"test-{execution_date.date().isoformat()}" + logical_date = DEFAULT_DATE + run_id = f"test-{logical_date.date().isoformat()}" tmp_file = tmp_path / "test.txt" # Run LocalTaskJob in separate process proc = mp_context.Process( @@ -948,7 +946,7 @@ def test_process_sigterm_works_with_retries(self, mp_method, wait_timeout, daemo dag_id, task_id, run_id, - execution_date, + logical_date, task_started, retry_callback_called, ), @@ -992,7 +990,7 @@ def _sigterm_local_task_runner( dag_id, task_id, run_id, - execution_date, + logical_date, is_started, callback_value, ): @@ -1012,7 +1010,7 @@ def task_function(): while True: time.sleep(0.25) - with DAG(dag_id=dag_id, schedule=None, start_date=execution_date) as dag: + with DAG(dag_id=dag_id, schedule=None, start_date=logical_date) as dag: task = PythonOperator( task_id=task_id, python_callable=task_function, @@ -1028,7 +1026,7 @@ def task_function(): dag_run = dag.create_dagrun( state=State.RUNNING, run_id=run_id, - execution_date=execution_date, + logical_date=logical_date, data_interval=data_interval, **triggered_by_kwargs, ) diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py index 6ddf4ae16f15..2975fc49df5d 100644 --- a/tests/jobs/test_scheduler_job.py +++ b/tests/jobs/test_scheduler_job.py @@ -362,7 +362,7 @@ def test_process_executor_events_with_no_callback(self, mock_stats_incr, mock_ta with dag_maker(dag_id=dag_id, fileloc="/test_path1/"): task1 = EmptyOperator(task_id=task_id, retries=1) ti1 = dag_maker.create_dagrun( - run_id=run_id, execution_date=DEFAULT_DATE + timedelta(hours=1) + run_id=run_id, logical_date=DEFAULT_DATE + timedelta(hours=1) ).get_task_instance(task1.task_id) mock_stats_incr.reset_mock() @@ -785,19 +785,19 @@ def test_find_executable_task_instances_only_running_dagruns( session.flush() assert total_executed_ti == len(res) - def test_find_executable_task_instances_order_execution_date(self, dag_maker): + def test_find_executable_task_instances_order_logical_date(self, dag_maker): """ - Test that task instances follow execution_date order priority. If two dagruns with - different execution dates are scheduled, tasks with earliest dagrun execution date will first + Test that task instances follow logical_date order priority. If two dagruns with + different logical dates are scheduled, tasks with earliest dagrun logical date will first be executed """ - dag_id_1 = "SchedulerJobTest.test_find_executable_task_instances_order_execution_date-a" - dag_id_2 = "SchedulerJobTest.test_find_executable_task_instances_order_execution_date-b" + dag_id_1 = "SchedulerJobTest.test_find_executable_task_instances_order_logical_date-a" + dag_id_2 = "SchedulerJobTest.test_find_executable_task_instances_order_logical_date-b" task_id = "task-a" session = settings.Session() with dag_maker(dag_id=dag_id_1, max_active_tasks=16, session=session): EmptyOperator(task_id=task_id) - dr1 = dag_maker.create_dagrun(execution_date=DEFAULT_DATE + timedelta(hours=1)) + dr1 = dag_maker.create_dagrun(logical_date=DEFAULT_DATE + timedelta(hours=1)) with dag_maker(dag_id=dag_id_2, max_active_tasks=16, session=session): EmptyOperator(task_id=task_id) @@ -927,9 +927,9 @@ def test_find_executable_task_instances_order_priority_with_pools(self, dag_make session.rollback() - def test_find_executable_task_instances_order_execution_date_and_priority(self, dag_maker): - dag_id_1 = "SchedulerJobTest.test_find_executable_task_instances_order_execution_date_and_priority-a" - dag_id_2 = "SchedulerJobTest.test_find_executable_task_instances_order_execution_date_and_priority-b" + def test_find_executable_task_instances_order_logical_date_and_priority(self, dag_maker): + dag_id_1 = "SchedulerJobTest.test_find_executable_task_instances_order_logical_date_and_priority-a" + dag_id_2 = "SchedulerJobTest.test_find_executable_task_instances_order_logical_date_and_priority-b" task_id = "task-a" session = settings.Session() with dag_maker(dag_id=dag_id_1, max_active_tasks=16, session=session): @@ -938,7 +938,7 @@ def test_find_executable_task_instances_order_execution_date_and_priority(self, with dag_maker(dag_id=dag_id_2, max_active_tasks=16, session=session): EmptyOperator(task_id=task_id, priority_weight=4) - dr2 = dag_maker.create_dagrun(execution_date=DEFAULT_DATE + timedelta(hours=1)) + dr2 = dag_maker.create_dagrun(logical_date=DEFAULT_DATE + timedelta(hours=1)) dr1 = session.merge(dr1, load=False) scheduler_job = Job() @@ -2102,7 +2102,7 @@ def test_adopt_or_reset_orphaned_tasks(self, dag_maker): dr2 = dag.create_dagrun( run_type=DagRunType.BACKFILL_JOB, state=State.RUNNING, - execution_date=DEFAULT_DATE + datetime.timedelta(1), + logical_date=DEFAULT_DATE + datetime.timedelta(1), start_date=DEFAULT_DATE, session=session, data_interval=data_interval, @@ -2503,8 +2503,8 @@ def test_dagrun_timeout_fails_run_and_update_next_dagrun(self, dag_maker): EmptyOperator(task_id="dummy") dr = dag_maker.create_dagrun(start_date=timezone.utcnow() - datetime.timedelta(days=1)) - # check that next_dagrun is dr.execution_date - dag_maker.dag_model.next_dagrun == dr.execution_date + # check that next_dagrun is dr.logical_date + dag_maker.dag_model.next_dagrun == dr.logical_date scheduler_job = Job() self.job_runner = SchedulerJobRunner(job=scheduler_job, subdir=os.devnull) @@ -2519,7 +2519,7 @@ def test_dagrun_timeout_fails_run_and_update_next_dagrun(self, dag_maker): session.refresh(dr) assert dr.state == State.FAILED # check that next_dagrun_create_after has been updated by calculate_dagrun_date_fields - assert dag_maker.dag_model.next_dagrun_create_after == dr.execution_date + timedelta(days=1) + assert dag_maker.dag_model.next_dagrun_create_after == dr.logical_date + timedelta(days=1) # check that no running/queued runs yet assert ( session.query(DagRun).filter(DagRun.state.in_([DagRunState.RUNNING, DagRunState.QUEUED])).count() @@ -2921,7 +2921,7 @@ def test_scheduler_start_date(self, configs): state="success", triggered_by=DagRunTriggeredByType.TIMETABLE, run_id="abc123", - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.BACKFILL_JOB, data_interval=DataInterval(DEFAULT_DATE, DEFAULT_DATE + timedelta(days=1)), ) @@ -3050,9 +3050,7 @@ def test_scheduler_verify_pool_full(self, dag_maker, configs, mock_executor): self.job_runner.processor_agent = mock.MagicMock() # Create 2 dagruns, which will create 2 task instances. - dr = dag_maker.create_dagrun( - run_type=DagRunType.SCHEDULED, - ) + dr = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED) self.job_runner._schedule_dag_run(dr, session) dr = dag_maker.create_dagrun_after(dr, run_type=DagRunType.SCHEDULED, state=State.RUNNING) self.job_runner._schedule_dag_run(dr, session) @@ -3153,7 +3151,7 @@ def _create_dagruns(dag: DAG): for _ in range(30): yield dag.create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=next_info.logical_date, + logical_date=next_info.logical_date, data_interval=next_info.data_interval, state=DagRunState.RUNNING, **triggered_by_kwargs, # type: ignore @@ -3297,7 +3295,7 @@ def test_verify_integrity_if_dag_not_changed(self, dag_maker): session.query(func.count(TaskInstance.task_id)) .filter( TaskInstance.dag_id == dr.dag_id, - TaskInstance.execution_date == dr.execution_date, + TaskInstance.logical_date == dr.logical_date, TaskInstance.task_id == dr.dag.tasks[0].task_id, TaskInstance.state == State.SCHEDULED, ) @@ -3365,7 +3363,7 @@ def test_verify_integrity_if_dag_changed(self, dag_maker): session.query(func.count(TaskInstance.task_id)) .filter( TaskInstance.dag_id == dr.dag_id, - TaskInstance.execution_date == dr.execution_date, + TaskInstance.logical_date == dr.logical_date, TaskInstance.state == State.SCHEDULED, ) .scalar() @@ -3509,7 +3507,7 @@ def test_dag_get_active_runs(self, dag_maker): # We had better get a dag run assert dr is not None - execution_date = dr.execution_date + logical_date = dr.logical_date running_dates = dag1.get_active_runs() @@ -3518,7 +3516,7 @@ def test_dag_get_active_runs(self, dag_maker): except Exception: running_date = "Except" - assert execution_date == running_date, "Running Date must match Execution Date" + assert logical_date == running_date, "Running Date must match Execution Date" def test_list_py_file_paths(self): """ @@ -3717,7 +3715,7 @@ def test_adopt_or_reset_orphaned_tasks_stale_scheduler_jobs(self, dag_maker): dr1 = dag_maker.create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, start_date=timezone.utcnow(), ) @@ -3849,7 +3847,7 @@ def test_should_update_dag_next_dagruns(self, provide_run_count: bool, kwargs: d for index in range(other_runs): dag_maker.create_dagrun( run_id=f"run_{index}", - execution_date=(DEFAULT_DATE + timedelta(days=index)), + logical_date=(DEFAULT_DATE + timedelta(days=index)), start_date=timezone.utcnow(), state=State.RUNNING, run_type=DagRunType.SCHEDULED, @@ -3858,7 +3856,7 @@ def test_should_update_dag_next_dagruns(self, provide_run_count: bool, kwargs: d for index in range(index + 1, index + 1 + backfill_runs): dag_maker.create_dagrun( run_id=f"run_{index}", - execution_date=(DEFAULT_DATE + timedelta(days=index)), + logical_date=(DEFAULT_DATE + timedelta(days=index)), start_date=timezone.utcnow(), state=State.RUNNING, run_type=DagRunType.BACKFILL_JOB, @@ -3905,7 +3903,7 @@ def test_should_update_dag_next_dagruns_after_run_type(self, run_type, expected, run = dag_maker.create_dagrun( run_id="run", run_type=run_type, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, start_date=timezone.utcnow(), state=State.SUCCESS, session=session, @@ -3968,7 +3966,7 @@ def test_create_dag_runs_assets(self, session, dag_maker): BashOperator(task_id="task", bash_command="echo 1", outlets=[asset1]) dr = dag_maker.create_dagrun( run_id="run1", - execution_date=(DEFAULT_DATE + timedelta(days=100)), + logical_date=(DEFAULT_DATE + timedelta(days=100)), data_interval=(DEFAULT_DATE + timedelta(days=10), DEFAULT_DATE + timedelta(days=11)), ) @@ -3986,7 +3984,7 @@ def test_create_dag_runs_assets(self, session, dag_maker): # Create a second event, creation time is more recent, but data interval is older dr = dag_maker.create_dagrun( run_id="run2", - execution_date=(DEFAULT_DATE + timedelta(days=101)), + logical_date=(DEFAULT_DATE + timedelta(days=101)), data_interval=(DEFAULT_DATE + timedelta(days=5), DEFAULT_DATE + timedelta(days=6)), ) @@ -4213,7 +4211,7 @@ def test_bulk_write_to_db_external_trigger_dont_skip_scheduled_run(self, dag_mak session = settings.Session() - # Verify that dag_model.next_dagrun is equal to next execution_date + # Verify that dag_model.next_dagrun is equal to next logical_date dag_model = dag_maker.dag_model assert dag_model.next_dagrun == DEFAULT_DATE assert dag_model.next_dagrun_data_interval_start == DEFAULT_DATE @@ -4228,10 +4226,12 @@ def test_bulk_write_to_db_external_trigger_dont_skip_scheduled_run(self, dag_mak # Verify a DagRun is created with the correct dates # when Scheduler._do_scheduling is run in the Scheduler Loop self.job_runner._do_scheduling(session) - dr1 = dag.get_dagrun(DEFAULT_DATE, session=session) + dr1 = session.scalar( + select(DagRun).where(DagRun.dag_id == dag_model.dag_id).order_by(DagRun.id.asc()).limit(1) + ) assert dr1 is not None - assert dr1.state == State.RUNNING - assert dr1.execution_date == DEFAULT_DATE + assert dr1.state == DagRunState.RUNNING + assert dr1.logical_date == DEFAULT_DATE assert dr1.data_interval_start == DEFAULT_DATE assert dr1.data_interval_end == DEFAULT_DATE + timedelta(minutes=1) @@ -4246,7 +4246,7 @@ def test_bulk_write_to_db_external_trigger_dont_skip_scheduled_run(self, dag_mak triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dr = dag.create_dagrun( state=State.RUNNING, - execution_date=timezone.utcnow(), + logical_date=timezone.utcnow(), run_type=DagRunType.MANUAL, session=session, external_trigger=True, @@ -4268,7 +4268,7 @@ def test_scheduler_create_dag_runs_check_existing_run(self, dag_maker): """ Test that if a dag run exists, scheduler._create_dag_runs does not raise an error. And if a Dag Run does not exist it creates next Dag Run. In both cases the Scheduler - sets next execution date as DagModel.next_dagrun + sets next logical date as DagModel.next_dagrun """ with dag_maker( dag_id="test_scheduler_create_dag_runs_check_existing_run", @@ -4288,7 +4288,7 @@ def test_scheduler_create_dag_runs_check_existing_run(self, dag_maker): dagrun = dag_maker.create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=dag_model.next_dagrun, + logical_date=dag_model.next_dagrun, start_date=timezone.utcnow(), state=State.RUNNING, external_trigger=False, @@ -4306,7 +4306,7 @@ def test_scheduler_create_dag_runs_check_existing_run(self, dag_maker): # Test that this does not raise any error self.job_runner._create_dag_runs([dag_model], session) - # Assert the next dagrun fields are set correctly to next execution date + # Assert the next dagrun fields are set correctly to next logical date assert dag_model.next_dagrun_data_interval_start == DEFAULT_DATE + timedelta(days=1) assert dag_model.next_dagrun_data_interval_end == DEFAULT_DATE + timedelta(days=2) assert dag_model.next_dagrun == DEFAULT_DATE + timedelta(days=1) @@ -4334,7 +4334,7 @@ def test_do_schedule_max_active_runs_dag_timed_out(self, dag_maker): dag_version = DagVersion.get_latest_version(dag.dag_id) run1 = dag.create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, state=State.RUNNING, start_date=timezone.utcnow() - timedelta(seconds=2), session=session, @@ -4348,7 +4348,7 @@ def test_do_schedule_max_active_runs_dag_timed_out(self, dag_maker): run2 = dag.create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE + timedelta(seconds=10), + logical_date=DEFAULT_DATE + timedelta(seconds=10), state=State.QUEUED, session=session, data_interval=data_interval, @@ -4395,7 +4395,7 @@ def test_do_schedule_max_active_runs_task_removed(self, session, dag_maker): run1 = dag_maker.create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE + timedelta(hours=1), + logical_date=DEFAULT_DATE + timedelta(hours=1), state=State.RUNNING, ) @@ -4480,7 +4480,7 @@ def complete_one_dagrun(): session.query(TaskInstance) .join(TaskInstance.dag_run) .filter(TaskInstance.state != State.SUCCESS) - .order_by(DagRun.execution_date) + .order_by(DagRun.logical_date) .first() ) if ti: @@ -4524,11 +4524,11 @@ def complete_one_dagrun(): self.job_runner._do_scheduling(session) complete_one_dagrun() - expected_execution_dates = [datetime.datetime(2016, 1, d, tzinfo=timezone.utc) for d in range(1, 6)] - dagrun_execution_dates = [ - dr.execution_date for dr in session.query(DagRun).order_by(DagRun.execution_date).all() + expected_logical_dates = [datetime.datetime(2016, 1, d, tzinfo=timezone.utc) for d in range(1, 6)] + dagrun_logical_dates = [ + dr.logical_date for dr in session.query(DagRun).order_by(DagRun.logical_date).all() ] - assert dagrun_execution_dates == expected_execution_dates + assert dagrun_logical_dates == expected_logical_dates def test_do_schedule_max_active_runs_and_manual_trigger(self, dag_maker, mock_executors): """ @@ -4573,7 +4573,7 @@ def test_do_schedule_max_active_runs_and_manual_trigger(self, dag_maker, mock_ex dag_maker.create_dagrun( run_type=DagRunType.MANUAL, - execution_date=DEFAULT_DATE + timedelta(hours=1), + logical_date=DEFAULT_DATE + timedelta(hours=1), state=State.QUEUED, session=session, ) @@ -5110,7 +5110,7 @@ def _running_counts(): assert session.scalar(select(func.count()).select_from(DagRun)) == 6 assert session.scalar(select(func.count()).where(DagRun.dag_id == dag1_dag_id)) == 6 - def test_start_queued_dagruns_do_follow_execution_date_order(self, dag_maker): + def test_start_queued_dagruns_do_follow_logical_date_order(self, dag_maker): session = settings.Session() with dag_maker("test_dag1", max_active_runs=1): EmptyOperator(task_id="mytask") @@ -5121,10 +5121,10 @@ def test_start_queued_dagruns_do_follow_execution_date_order(self, dag_maker): run_id=f"dagrun_{i}", run_type=DagRunType.SCHEDULED, state=State.QUEUED, - execution_date=date, + logical_date=date, dag_version=dag_version, ) - date = dr.execution_date + timedelta(hours=1) + date = dr.logical_date + timedelta(hours=1) scheduler_job = Job() self.job_runner = SchedulerJobRunner(job=scheduler_job, subdir=os.devnull) @@ -5166,15 +5166,13 @@ def test_no_dagruns_would_stuck_in_running(self, dag_maker): task1 = EmptyOperator(task_id="dummy_task") dag_version = DagVersion.get_latest_version(dag.dag_id) - dr1_running = dag_maker.create_dagrun( - run_id="dr1_run_1", execution_date=date, dag_version=dag_version - ) + dr1_running = dag_maker.create_dagrun(run_id="dr1_run_1", logical_date=date, dag_version=dag_version) data_interval = dag.infer_automated_data_interval(logical_date) dag_maker.create_dagrun( run_id="dr1_run_2", state=State.QUEUED, dag_version=dag_version, - execution_date=dag.next_dagrun_info( + logical_date=dag.next_dagrun_info( last_automated_dagrun=data_interval, restricted=False ).data_interval.start, ) @@ -5187,20 +5185,20 @@ def test_no_dagruns_would_stuck_in_running(self, dag_maker): dr = dag_maker.create_dagrun( run_id=f"dr2_run_{i+1}", state=State.RUNNING, - execution_date=date, + logical_date=date, dag_version=dag_version, ) - date = dr.execution_date + timedelta(hours=1) + date = dr.logical_date + timedelta(hours=1) dr16 = DagRun.find(run_id="dr2_run_16") - date = dr16[0].execution_date + timedelta(hours=1) + date = dr16[0].logical_date + timedelta(hours=1) for i in range(16, 32): dr = dag_maker.create_dagrun( run_id=f"dr2_run_{i+1}", state=State.QUEUED, - execution_date=date, + logical_date=date, dag_version=dag_version, ) - date = dr.execution_date + timedelta(hours=1) + date = dr.logical_date + timedelta(hours=1) # third dag and dagruns date = timezone.datetime(2021, 1, 1) @@ -5211,20 +5209,20 @@ def test_no_dagruns_would_stuck_in_running(self, dag_maker): dr = dag_maker.create_dagrun( run_id=f"dr3_run_{i+1}", state=State.RUNNING, - execution_date=date, + logical_date=date, dag_version=dag_version, ) - date = dr.execution_date + timedelta(hours=1) + date = dr.logical_date + timedelta(hours=1) dr16 = DagRun.find(run_id="dr3_run_16") - date = dr16[0].execution_date + timedelta(hours=1) + date = dr16[0].logical_date + timedelta(hours=1) for i in range(16, 32): dr = dag_maker.create_dagrun( run_id=f"dr2_run_{i+1}", state=State.QUEUED, - execution_date=date, + logical_date=date, dag_version=dag_version, ) - date = dr.execution_date + timedelta(hours=1) + date = dr.logical_date + timedelta(hours=1) scheduler_job = Job() self.job_runner = SchedulerJobRunner(job=scheduler_job, subdir=os.devnull) @@ -5548,7 +5546,7 @@ def test_timeout_triggers(self, dag_maker): # We don't actually need a linked trigger here; the code doesn't check. dr1 = dag_maker.create_dagrun() dr2 = dag_maker.create_dagrun( - run_id="test2", execution_date=DEFAULT_DATE + datetime.timedelta(seconds=1) + run_id="test2", logical_date=DEFAULT_DATE + datetime.timedelta(seconds=1) ) ti1 = dr1.get_task_instance("dummy1", session) ti2 = dr2.get_task_instance("dummy1", session) @@ -5615,7 +5613,7 @@ def side_effect(*args, **kwargs): # We don't actually need a linked trigger here; the code doesn't check. dr1 = dag_maker.create_dagrun() dr2 = dag_maker.create_dagrun( - run_id="test2", execution_date=DEFAULT_DATE + datetime.timedelta(seconds=1) + run_id="test2", logical_date=DEFAULT_DATE + datetime.timedelta(seconds=1) ) ti1 = dr1.get_task_instance("dummy1", session) ti2 = dr2.get_task_instance("dummy1", session) @@ -5666,7 +5664,7 @@ def test_find_and_purge_zombies(self, load_examples, session): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dag_run = dag.create_dagrun( state=DagRunState.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.SCHEDULED, session=session, data_interval=data_interval, @@ -5729,7 +5727,7 @@ def test_zombie_message(self, load_examples): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dag_run = dag.create_dagrun( state=DagRunState.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.SCHEDULED, session=session, data_interval=data_interval, @@ -5794,7 +5792,7 @@ def test_find_zombies_handle_failure_callbacks_are_correctly_passed_to_dag_proce triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dag_run = dag.create_dagrun( state=DagRunState.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.SCHEDULED, session=session, data_interval=data_interval, @@ -6060,8 +6058,8 @@ def test_catchup_works_correctly(self, dag_maker): self.job_runner._create_dag_runs([dag_maker.dag_model], session) self.job_runner._start_queued_dagruns(session) - # first dagrun execution date is DEFAULT_DATE 2016-01-01T00:00:00+00:00 - dr = DagRun.find(execution_date=DEFAULT_DATE, session=session)[0] + # first dagrun logical date is DEFAULT_DATE 2016-01-01T00:00:00+00:00 + dr = DagRun.find(logical_date=DEFAULT_DATE, session=session)[0] ti = dr.get_task_instance(task_id="dummy") ti.state = State.SUCCESS session.merge(ti) @@ -6081,11 +6079,11 @@ def test_catchup_works_correctly(self, dag_maker): dm = DagModel.get_dagmodel(dag.dag_id) self.job_runner._create_dag_runs([dm], session) - # Check catchup worked correctly by ensuring execution_date is quite new + # Check catchup worked correctly by ensuring logical_date is quite new # Our dag is a daily dag assert ( - session.query(DagRun.execution_date) - .filter(DagRun.execution_date != DEFAULT_DATE) # exclude the first run + session.query(DagRun.logical_date) + .filter(DagRun.logical_date != DEFAULT_DATE) # exclude the first run .scalar() ) > (timezone.utcnow() - timedelta(days=2)) @@ -6096,7 +6094,7 @@ def test_update_dagrun_state_for_paused_dag(self, dag_maker, session): EmptyOperator(task_id="task1") scheduled_run = dag_maker.create_dagrun( - execution_date=datetime.datetime(2022, 1, 1), + logical_date=datetime.datetime(2022, 1, 1), run_type=DagRunType.SCHEDULED, ) scheduled_run.last_scheduling_decision = datetime.datetime.now(timezone.utc) - timedelta(minutes=1) diff --git a/tests/jobs/test_triggerer_job.py b/tests/jobs/test_triggerer_job.py index 939c210dfe75..bae777155bbc 100644 --- a/tests/jobs/test_triggerer_job.py +++ b/tests/jobs/test_triggerer_job.py @@ -96,7 +96,7 @@ def create_trigger_in_db(session, trigger, operator=None): run = DagRun( dag_id=dag_model.dag_id, run_id="test_run", - execution_date=pendulum.datetime(2023, 1, 1), + logical_date=pendulum.datetime(2023, 1, 1), run_type=DagRunType.MANUAL, ) trigger_orm = Trigger.from_object(trigger) @@ -520,7 +520,7 @@ def test_trigger_from_dead_triggerer(session, create_task_instance): session.add(trigger_orm) ti_orm = create_task_instance( task_id="ti_orm", - execution_date=timezone.utcnow(), + logical_date=timezone.utcnow(), run_id="orm_run_id", ) ti_orm.trigger_id = trigger_orm.id @@ -548,7 +548,7 @@ def test_trigger_from_expired_triggerer(session, create_task_instance): session.add(trigger_orm) ti_orm = create_task_instance( task_id="ti_orm", - execution_date=timezone.utcnow(), + logical_date=timezone.utcnow(), run_id="orm_run_id", ) ti_orm.trigger_id = trigger_orm.id diff --git a/tests/lineage/test_lineage.py b/tests/lineage/test_lineage.py index 6a3e11b68b76..2a5c0b1b0836 100644 --- a/tests/lineage/test_lineage.py +++ b/tests/lineage/test_lineage.py @@ -129,7 +129,7 @@ def test_lineage_render(self, dag_maker): op1.inlets.append(file1) op1.outlets.append(file1) - # execution_date is set in the context in order to avoid creating task instances + # logical_date is set in the context in order to avoid creating task instances ctx1 = Context({"ti": TI(task=op1, run_id=dag_run.run_id), "ds": DEFAULT_DATE}) op1.pre_execute(ctx1) @@ -143,7 +143,7 @@ def test_attr_outlet(self, dag_maker): f3s = "/tmp/does_not_exist_3" file3 = File(f3s) - with dag_maker(dag_id="test_prepare_lineage"): + with dag_maker(dag_id="test_prepare_lineage", start_date=DEFAULT_DATE): op1 = EmptyOperator( task_id="leave1", outlets=[a, file3], diff --git a/tests/listeners/test_listeners.py b/tests/listeners/test_listeners.py index 025e84f1cd69..752209052db4 100644 --- a/tests/listeners/test_listeners.py +++ b/tests/listeners/test_listeners.py @@ -52,7 +52,7 @@ DAG_ID = "test_listener_dag" TASK_ID = "test_listener_task" -EXECUTION_DATE = timezone.utcnow() +LOGICAL_DATE = timezone.utcnow() TEST_DAG_FOLDER = os.environ["AIRFLOW__CORE__DAGS_FOLDER"] @@ -139,7 +139,7 @@ def test_listener_captures_failed_taskinstances(create_task_instance_of_operator lm.add_listener(full_listener) ti = create_task_instance_of_operator( - BashOperator, dag_id=DAG_ID, execution_date=EXECUTION_DATE, task_id=TASK_ID, bash_command="exit 1" + BashOperator, dag_id=DAG_ID, logical_date=LOGICAL_DATE, task_id=TASK_ID, bash_command="exit 1" ) with pytest.raises(AirflowException): ti._run_raw_task() @@ -154,7 +154,7 @@ def test_listener_captures_longrunning_taskinstances(create_task_instance_of_ope lm.add_listener(full_listener) ti = create_task_instance_of_operator( - BashOperator, dag_id=DAG_ID, execution_date=EXECUTION_DATE, task_id=TASK_ID, bash_command="sleep 5" + BashOperator, dag_id=DAG_ID, logical_date=LOGICAL_DATE, task_id=TASK_ID, bash_command="sleep 5" ) ti._run_raw_task() diff --git a/tests/models/test_backfill.py b/tests/models/test_backfill.py index a32db07808e8..27be2df009eb 100644 --- a/tests/models/test_backfill.py +++ b/tests/models/test_backfill.py @@ -107,7 +107,7 @@ def test_create_backfill_simple(reverse, existing, dag_maker, session): for date in existing: dag_maker.create_dagrun( run_id=f"scheduled_{date}", - execution_date=timezone.parse(date), + logical_date=timezone.parse(date), session=session, ) session.commit() @@ -186,7 +186,7 @@ def test_reprocess_behavior(reprocess_behavior, run_counts, dag_maker, session): We have two modes whereby when there's an existing run(s) in the range of the backfill, we will create a new run. This test might need to be altered if we change the behavior re multiple - runs of same execution date. But for now, it verifies the current behavior. + runs of same logical date. But for now, it verifies the current behavior. """ dag_id = "backfill-test-reprocess-behavior" with dag_maker(schedule="@daily", dag_id=dag_id) as dag: @@ -212,7 +212,7 @@ def test_reprocess_behavior(reprocess_behavior, run_counts, dag_maker, session): ): dr = dag_maker.create_dagrun( run_id=f"scheduled_{date}-{num}", - execution_date=timezone.parse(date), + logical_date=timezone.parse(date), session=session, state=state, ) @@ -347,7 +347,7 @@ def create_next_run( ) return next_run else: - dr = dag_maker.create_dagrun(execution_date=next_date, run_id="second_run") + dr = dag_maker.create_dagrun(logical_date=next_date, run_id="second_run") return dr @@ -361,7 +361,7 @@ def test_ignore_first_depends_on_past(first_run_type, days_between, catchup, is_ from_date = base_date + timedelta(days=days_between) with dag_maker(dag_id="abc123", serialized=True, catchup=catchup) as dag: op = PythonOperator(task_id="dep_on_past", python_callable=lambda: print, depends_on_past=True) - dr = dag_maker.create_dagrun(execution_date=base_date, run_type=first_run_type) + dr = dag_maker.create_dagrun(logical_date=base_date, run_type=first_run_type) dr.state = DagRunState.FAILED for ti in dr.task_instances: ti.state = TaskInstanceState.FAILED @@ -370,7 +370,7 @@ def test_ignore_first_depends_on_past(first_run_type, days_between, catchup, is_ # let's verify all is as expected session.expunge_all() - first_run = session.scalar(select(DagRun).order_by(DagRun.execution_date).limit(1)) + first_run = session.scalar(select(DagRun).order_by(DagRun.logical_date).limit(1)) assert first_run.state == DagRunState.FAILED tis = first_run.get_task_instances(session=session) assert len(tis) > 0 @@ -387,11 +387,11 @@ def test_ignore_first_depends_on_past(first_run_type, days_between, catchup, is_ # check that it's immediately after the other dag run prior_runs = session.scalars( - select(DagRun.execution_date).where(DagRun.execution_date < next_run.execution_date) + select(DagRun.logical_date).where(DagRun.logical_date < next_run.logical_date) ).all() assert len(prior_runs) == 1 - assert prior_runs[0] == first_run.execution_date - assert prior_runs[0] + timedelta(days=days_between) == next_run.execution_date + assert prior_runs[0] == first_run.logical_date + assert prior_runs[0] + timedelta(days=days_between) == next_run.logical_date # so now the first backfill dag run follows the other one immediately diff --git a/tests/models/test_baseoperator.py b/tests/models/test_baseoperator.py index 1d8e4f4d32a8..00888e4ff353 100644 --- a/tests/models/test_baseoperator.py +++ b/tests/models/test_baseoperator.py @@ -745,22 +745,22 @@ def my_teardown(): def test_get_task_instances(session): import pendulum - first_execution_date = pendulum.datetime(2023, 1, 1) - second_execution_date = pendulum.datetime(2023, 1, 2) - third_execution_date = pendulum.datetime(2023, 1, 3) + first_logical_date = pendulum.datetime(2023, 1, 1) + second_logical_date = pendulum.datetime(2023, 1, 2) + third_logical_date = pendulum.datetime(2023, 1, 3) - test_dag = DAG(dag_id="test_dag", schedule=None, start_date=first_execution_date) + test_dag = DAG(dag_id="test_dag", schedule=None, start_date=first_logical_date) task = BaseOperator(task_id="test_task", dag=test_dag) common_dr_kwargs = { "dag_id": test_dag.dag_id, "run_type": DagRunType.MANUAL, } - dr1 = DagRun(execution_date=first_execution_date, run_id="test_run_id_1", **common_dr_kwargs) + dr1 = DagRun(logical_date=first_logical_date, run_id="test_run_id_1", **common_dr_kwargs) ti_1 = TaskInstance(run_id=dr1.run_id, task=task) - dr2 = DagRun(execution_date=second_execution_date, run_id="test_run_id_2", **common_dr_kwargs) + dr2 = DagRun(logical_date=second_logical_date, run_id="test_run_id_2", **common_dr_kwargs) ti_2 = TaskInstance(run_id=dr2.run_id, task=task) - dr3 = DagRun(execution_date=third_execution_date, run_id="test_run_id_3", **common_dr_kwargs) + dr3 = DagRun(logical_date=third_logical_date, run_id="test_run_id_3", **common_dr_kwargs) ti_3 = TaskInstance(run_id=dr3.run_id, task=task) session.add_all([dr1, dr2, dr3, ti_1, ti_2, ti_3]) session.commit() @@ -768,12 +768,12 @@ def test_get_task_instances(session): # get all task instances assert task.get_task_instances(session=session) == [ti_1, ti_2, ti_3] # get task instances with start_date - assert task.get_task_instances(session=session, start_date=second_execution_date) == [ti_2, ti_3] + assert task.get_task_instances(session=session, start_date=second_logical_date) == [ti_2, ti_3] # get task instances with end_date - assert task.get_task_instances(session=session, end_date=second_execution_date) == [ti_1, ti_2] + assert task.get_task_instances(session=session, end_date=second_logical_date) == [ti_1, ti_2] # get task instances with start_date and end_date assert task.get_task_instances( - session=session, start_date=second_execution_date, end_date=second_execution_date + session=session, start_date=second_logical_date, end_date=second_logical_date ) == [ti_2] diff --git a/tests/models/test_cleartasks.py b/tests/models/test_cleartasks.py index 2afccf8b4677..7771ffe757ab 100644 --- a/tests/models/test_cleartasks.py +++ b/tests/models/test_cleartasks.py @@ -644,7 +644,7 @@ def test_dags_clear(self): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dr = dag.create_dagrun( - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, state=State.RUNNING, run_type=DagRunType.SCHEDULED, session=session, diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index a899c90fded3..28f745b0614e 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -285,7 +285,7 @@ def test_dag_task_custom_weight_strategy(self, cls, expected): dr = dag.create_dagrun( state=None, run_id="test", - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, ) @@ -303,14 +303,14 @@ def test_get_num_task_instances(self): dr1 = test_dag.create_dagrun( state=None, run_id="test1", - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, ) dr2 = test_dag.create_dagrun( state=None, run_id="test2", - execution_date=DEFAULT_DATE + datetime.timedelta(days=1), + logical_date=DEFAULT_DATE + datetime.timedelta(days=1), data_interval=( DEFAULT_DATE + datetime.timedelta(days=1), DEFAULT_DATE + datetime.timedelta(days=1), @@ -320,7 +320,7 @@ def test_get_num_task_instances(self): dr3 = test_dag.create_dagrun( state=None, run_id="test3", - execution_date=DEFAULT_DATE + datetime.timedelta(days=2), + logical_date=DEFAULT_DATE + datetime.timedelta(days=2), data_interval=( DEFAULT_DATE + datetime.timedelta(days=2), DEFAULT_DATE + datetime.timedelta(days=2), @@ -330,7 +330,7 @@ def test_get_num_task_instances(self): dr4 = test_dag.create_dagrun( state=None, run_id="test4", - execution_date=DEFAULT_DATE + datetime.timedelta(days=3), + logical_date=DEFAULT_DATE + datetime.timedelta(days=3), data_interval=( DEFAULT_DATE + datetime.timedelta(days=2), DEFAULT_DATE + datetime.timedelta(days=2), @@ -397,7 +397,7 @@ def dag_run_before(delta_h=0, type=DagRunType.SCHEDULED): **triggered_by_kwargs, ) dagrun.start_date = BASE_DATE + timedelta(hours=delta_h) - dagrun.execution_date = BASE_DATE + timedelta(hours=delta_h) + dagrun.logical_date = BASE_DATE + timedelta(hours=delta_h) return dagrun dr1 = dag_run_before(delta_h=-1, type=DagRunType.MANUAL) # H19 @@ -594,7 +594,7 @@ def test_create_dagrun_when_schedule_is_none_and_empty_start_date(self): dagrun = dag.create_dagrun( state=State.RUNNING, run_type=DagRunType.MANUAL, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, ) @@ -789,7 +789,7 @@ def test_bulk_write_to_db_max_active_runs(self, state): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dr = dag.create_dagrun( state=state, - execution_date=model.next_dagrun, + logical_date=model.next_dagrun, run_type=DagRunType.SCHEDULED, session=session, data_interval=(model.next_dagrun, model.next_dagrun), @@ -1037,15 +1037,15 @@ def test_existing_dag_is_paused_config(self): assert dag.max_consecutive_failed_dag_runs == 2 def test_existing_dag_is_paused_after_limit(self): - def add_failed_dag_run(dag, id, execution_date): + def add_failed_dag_run(dag, id, logical_date): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dag_v = DagVersion.get_latest_version(dag_id=dag.dag_id) dr = dag.create_dagrun( run_type=DagRunType.MANUAL, run_id="run_id_" + id, - execution_date=execution_date, + logical_date=logical_date, state=State.FAILED, - data_interval=(execution_date, execution_date), + data_interval=(logical_date, logical_date), dag_version=dag_v, **triggered_by_kwargs, ) @@ -1126,7 +1126,7 @@ def test_schedule_dag_no_previous_runs(self): dag_run = dag.create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=TEST_DATE, + logical_date=TEST_DATE, state=State.RUNNING, data_interval=(TEST_DATE, TEST_DATE), **triggered_by_kwargs, @@ -1136,8 +1136,8 @@ def test_schedule_dag_no_previous_runs(self): assert dag_run.run_id is not None assert "" != dag_run.run_id assert ( - TEST_DATE == dag_run.execution_date - ), f"dag_run.execution_date did not match expectation: {dag_run.execution_date}" + TEST_DATE == dag_run.logical_date + ), f"dag_run.logical_date did not match expectation: {dag_run.logical_date}" assert State.RUNNING == dag_run.state assert not dag_run.external_trigger dag.clear() @@ -1165,7 +1165,7 @@ def test_dag_handle_callback_crash(self, mock_stats): with create_session() as session: dag_run = dag.create_dagrun( state=State.RUNNING, - execution_date=when, + logical_date=when, run_type=DagRunType.MANUAL, session=session, data_interval=(when, when), @@ -1203,7 +1203,7 @@ def test_dag_handle_callback_with_removed_task(self, dag_maker, session): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dag_run = dag.create_dagrun( state=State.RUNNING, - execution_date=TEST_DATE, + logical_date=TEST_DATE, run_type=DagRunType.MANUAL, session=session, data_interval=(TEST_DATE, TEST_DATE), @@ -1234,7 +1234,7 @@ def test_next_dagrun_after_fake_scheduled_previous(self): dag.create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, state=State.SUCCESS, external_trigger=True, data_interval=(DEFAULT_DATE, DEFAULT_DATE), @@ -1267,7 +1267,7 @@ def test_schedule_dag_once(self): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dag.create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=TEST_DATE, + logical_date=TEST_DATE, state=State.SUCCESS, data_interval=(TEST_DATE, TEST_DATE), **triggered_by_kwargs, @@ -1295,7 +1295,7 @@ def test_fractional_seconds(self): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} run = dag.create_dagrun( run_id="test_" + start_date.isoformat(), - execution_date=start_date, + logical_date=start_date, start_date=start_date, state=State.RUNNING, external_trigger=False, @@ -1305,7 +1305,7 @@ def test_fractional_seconds(self): run.refresh_from_db() - assert start_date == run.execution_date, "dag run execution_date loses precision" + assert start_date == run.logical_date, "dag run logical_date loses precision" assert start_date == run.start_date, "dag run start_date loses precision " self._clean_up(dag_id) @@ -1427,7 +1427,7 @@ def test_create_dagrun_run_id_is_generated(self): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dr = dag.create_dagrun( run_type=DagRunType.MANUAL, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, state=State.NONE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, @@ -1523,7 +1523,7 @@ def test_clear_set_dagrun_state(self, dag_run_state): run_type=DagRunType.BACKFILL_JOB, state=State.FAILED, start_date=DEFAULT_DATE, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, ) @@ -1572,7 +1572,7 @@ def consumer(value): run_type=DagRunType.BACKFILL_JOB, state=State.FAILED, start_date=DEFAULT_DATE, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, session=session, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, @@ -1759,7 +1759,7 @@ def test_clear_dag( run_type=DagRunType.BACKFILL_JOB, state=DagRunState.RUNNING, start_date=DEFAULT_DATE, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, # type: ignore ) @@ -1953,7 +1953,7 @@ def test_next_dagrun_after_auto_align(self): """ Test if the schedule will be auto aligned with the start_date such that if the start_date coincides with the schedule the first - execution_date will be start_date, otherwise it will be start_date + + logical_date will be start_date, otherwise it will be start_date + interval. """ dag = DAG( @@ -2071,7 +2071,7 @@ def test_validate_params_on_trigger_dag(self): dag.create_dagrun( run_id="test_dagrun_missing_param", state=State.RUNNING, - execution_date=TEST_DATE, + logical_date=TEST_DATE, data_interval=(TEST_DATE, TEST_DATE), **triggered_by_kwargs, ) @@ -2083,7 +2083,7 @@ def test_validate_params_on_trigger_dag(self): dag.create_dagrun( run_id="test_dagrun_missing_param", state=State.RUNNING, - execution_date=TEST_DATE, + logical_date=TEST_DATE, conf={"param1": None}, data_interval=(TEST_DATE, TEST_DATE), **triggered_by_kwargs, @@ -2093,7 +2093,7 @@ def test_validate_params_on_trigger_dag(self): dag.create_dagrun( run_id="test_dagrun_missing_param", state=State.RUNNING, - execution_date=TEST_DATE, + logical_date=TEST_DATE, conf={"param1": "hello"}, data_interval=(TEST_DATE, TEST_DATE), **triggered_by_kwargs, @@ -2187,7 +2187,7 @@ def test_dags_needing_dagruns_assets(self, dag_maker, session): dag_maker.create_dagrun( run_type=DagRunType.ASSET_TRIGGERED, state=DagRunState.QUEUED, - execution_date=pendulum.now("UTC"), + logical_date=pendulum.now("UTC"), ) query, _ = DagModel.dags_needing_dagruns(session) dag_models = query.all() @@ -2235,7 +2235,7 @@ def test_dags_needing_dagruns_asset_aliases(self, dag_maker, session): dag_maker.create_dagrun( run_type=DagRunType.ASSET_TRIGGERED, state=DagRunState.QUEUED, - execution_date=pendulum.now("UTC"), + logical_date=pendulum.now("UTC"), ) query, _ = DagModel.dags_needing_dagruns(session) dag_models = query.all() @@ -2479,7 +2479,7 @@ def test_count_number_queries(self, tasks_count): dag.create_dagrun( run_id="test_dagrun_query_count", state=State.RUNNING, - execution_date=TEST_DATE, + logical_date=TEST_DATE, data_interval=(TEST_DATE, TEST_DATE), **triggered_by_kwargs, ) @@ -2554,7 +2554,7 @@ def return_num(num): dr = dag.create_dagrun( run_id=DagRunType.MANUAL.value, start_date=timezone.utcnow(), - execution_date=self.DEFAULT_DATE, + logical_date=self.DEFAULT_DATE, data_interval=(self.DEFAULT_DATE, self.DEFAULT_DATE), state=State.RUNNING, **triggered_by_kwargs, @@ -2584,7 +2584,7 @@ def return_num(num): dr = dag.create_dagrun( run_id=DagRunType.MANUAL.value, start_date=timezone.utcnow(), - execution_date=self.DEFAULT_DATE, + logical_date=self.DEFAULT_DATE, data_interval=(self.DEFAULT_DATE, self.DEFAULT_DATE), state=State.RUNNING, conf={"value": new_value}, @@ -2613,13 +2613,13 @@ def return_num(num): @pytest.mark.parametrize( - "run_id, execution_date", + "run_id, logical_date", [ (None, datetime_tz(2020, 1, 1)), ("test-run-id", None), ], ) -def test_set_task_instance_state(run_id, execution_date, session, dag_maker): +def test_set_task_instance_state(run_id, logical_date, session, dag_maker): """Test that set_task_instance_state updates the TaskInstance state and clear downstream failed""" start_date = datetime_tz(2020, 1, 1) @@ -2629,12 +2629,11 @@ def test_set_task_instance_state(run_id, execution_date, session, dag_maker): task_3 = EmptyOperator(task_id="task_3") task_4 = EmptyOperator(task_id="task_4") task_5 = EmptyOperator(task_id="task_5") - task_1 >> [task_2, task_3, task_4, task_5] dagrun = dag_maker.create_dagrun( run_id=run_id, - execution_date=execution_date, + logical_date=logical_date, state=State.FAILED, run_type=DagRunType.SCHEDULED, ) @@ -2655,17 +2654,15 @@ def get_ti_from_db(task): get_ti_from_db(task_3).state = State.UPSTREAM_FAILED get_ti_from_db(task_4).state = State.FAILED get_ti_from_db(task_5).state = State.SKIPPED - session.flush() altered = dag.set_task_instance_state( task_id=task_1.task_id, run_id=run_id, - execution_date=execution_date, + logical_date=logical_date, state=State.SUCCESS, session=session, ) - # After _mark_task_instance_state, task_1 is marked as SUCCESS ti1 = get_ti_from_db(task_1) assert ti1.state == State.SUCCESS @@ -2716,7 +2713,7 @@ def consumer(value): dr2 = dag_maker.create_dagrun( run_type=DagRunType.SCHEDULED, state=DagRunState.FAILED, - execution_date=DEFAULT_DATE + datetime.timedelta(days=1), + logical_date=DEFAULT_DATE + datetime.timedelta(days=1), ) expand_mapped_task(mapped, dr2.run_id, "make_arg_lists", length=2, session=session) @@ -2758,8 +2755,7 @@ def consumer(value): ] -@pytest.mark.parametrize("run_id, execution_date", [(None, datetime_tz(2020, 1, 1)), ("test-run-id", None)]) -def test_set_task_group_state(run_id, execution_date, session, dag_maker): +def test_set_task_group_state(session, dag_maker): """Test that set_task_group_state updates the TaskGroup state and clear downstream failed""" start_date = datetime_tz(2020, 1, 1) @@ -2782,8 +2778,7 @@ def test_set_task_group_state(run_id, execution_date, session, dag_maker): start >> section_1 >> [task_4, task_5, task_6, task_7, task_8] dagrun = dag_maker.create_dagrun( - run_id=run_id, - execution_date=execution_date, + run_id="test-run-id", state=State.FAILED, run_type=DagRunType.SCHEDULED, ) @@ -2811,8 +2806,7 @@ def get_ti_from_db(task): altered = dag.set_task_group_state( group_id=section_1.group_id, - run_id=run_id, - execution_date=execution_date, + run_id="test-run-id", state=State.SUCCESS, session=session, ) @@ -3097,7 +3091,7 @@ def generate_run_id(self, *, run_type, logical_date, data_interval, **extra) -> dag_run = dag.create_dagrun( run_type=DagRunType.MANUAL, state=DagRunState.QUEUED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, ) @@ -3118,7 +3112,7 @@ def test_create_dagrun_disallow_manual_to_use_automated_run_id(run_id_type: DagR dag.create_dagrun( run_type=DagRunType.MANUAL, run_id=run_id, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), state=DagRunState.QUEUED, **triggered_by_kwargs, # type: ignore diff --git a/tests/models/test_dagrun.py b/tests/models/test_dagrun.py index dc5a8ab66e56..39d6d56ef376 100644 --- a/tests/models/test_dagrun.py +++ b/tests/models/test_dagrun.py @@ -95,23 +95,23 @@ def create_dag_run( dag: DAG, *, task_states: Mapping[str, TaskInstanceState] | None = None, - execution_date: datetime.datetime | None = None, + logical_date: datetime.datetime | None = None, is_backfill: bool = False, state: DagRunState = DagRunState.RUNNING, session: Session, ): now = timezone.utcnow() - execution_date = pendulum.instance(execution_date or now) + logical_date = pendulum.instance(logical_date or now) triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} if is_backfill: run_type = DagRunType.BACKFILL_JOB - data_interval = dag.infer_automated_data_interval(execution_date) + data_interval = dag.infer_automated_data_interval(logical_date) else: run_type = DagRunType.MANUAL - data_interval = dag.timetable.infer_manual_data_interval(run_after=execution_date) + data_interval = dag.timetable.infer_manual_data_interval(run_after=logical_date) dag_run = dag.create_dagrun( run_type=run_type, - execution_date=execution_date, + logical_date=logical_date, data_interval=data_interval, start_date=now, state=state, @@ -132,7 +132,7 @@ def test_clear_task_instances_for_backfill_unfinished_dagrun(self, state, sessio now = timezone.utcnow() dag_id = "test_clear_task_instances_for_backfill_dagrun" dag = DAG(dag_id=dag_id, schedule=datetime.timedelta(days=1), start_date=now) - dag_run = self.create_dag_run(dag, execution_date=now, is_backfill=True, state=state, session=session) + dag_run = self.create_dag_run(dag, logical_date=now, is_backfill=True, state=state, session=session) task0 = EmptyOperator(task_id="backfill_task_0", owner="test", dag=dag) ti0 = TI(task=task0, run_id=dag_run.run_id) @@ -142,7 +142,7 @@ def test_clear_task_instances_for_backfill_unfinished_dagrun(self, state, sessio clear_task_instances(qry, session) session.commit() ti0.refresh_from_db() - dr0 = session.query(DagRun).filter(DagRun.dag_id == dag_id, DagRun.execution_date == now).first() + dr0 = session.query(DagRun).filter(DagRun.dag_id == dag_id, DagRun.logical_date == now).first() assert dr0.state == state assert dr0.clear_number < 1 @@ -151,7 +151,7 @@ def test_clear_task_instances_for_backfill_finished_dagrun(self, state, session) now = timezone.utcnow() dag_id = "test_clear_task_instances_for_backfill_dagrun" dag = DAG(dag_id=dag_id, schedule=datetime.timedelta(days=1), start_date=now) - dag_run = self.create_dag_run(dag, execution_date=now, is_backfill=True, state=state, session=session) + dag_run = self.create_dag_run(dag, logical_date=now, is_backfill=True, state=state, session=session) task0 = EmptyOperator(task_id="backfill_task_0", owner="test", dag=dag) ti0 = TI(task=task0, run_id=dag_run.run_id) @@ -161,7 +161,7 @@ def test_clear_task_instances_for_backfill_finished_dagrun(self, state, session) clear_task_instances(qry, session) session.commit() ti0.refresh_from_db() - dr0 = session.query(DagRun).filter(DagRun.dag_id == dag_id, DagRun.execution_date == now).first() + dr0 = session.query(DagRun).filter(DagRun.dag_id == dag_id, DagRun.logical_date == now).first() assert dr0.state == DagRunState.QUEUED assert dr0.clear_number == 1 @@ -173,7 +173,7 @@ def test_dagrun_find(self, session): dag_id=dag_id1, run_id=dag_id1, run_type=DagRunType.MANUAL, - execution_date=now, + logical_date=now, start_date=now, state=DagRunState.RUNNING, external_trigger=True, @@ -185,7 +185,7 @@ def test_dagrun_find(self, session): dag_id=dag_id2, run_id=dag_id2, run_type=DagRunType.MANUAL, - execution_date=now, + logical_date=now, start_date=now, state=DagRunState.RUNNING, external_trigger=False, @@ -197,8 +197,8 @@ def test_dagrun_find(self, session): assert 1 == len(DagRun.find(dag_id=dag_id1, external_trigger=True)) assert 1 == len(DagRun.find(run_id=dag_id1)) assert 2 == len(DagRun.find(run_id=[dag_id1, dag_id2])) - assert 2 == len(DagRun.find(execution_date=[now, now])) - assert 2 == len(DagRun.find(execution_date=now)) + assert 2 == len(DagRun.find(logical_date=[now, now])) + assert 2 == len(DagRun.find(logical_date=now)) assert 0 == len(DagRun.find(dag_id=dag_id1, external_trigger=False)) assert 0 == len(DagRun.find(dag_id=dag_id2, external_trigger=True)) assert 1 == len(DagRun.find(dag_id=dag_id2, external_trigger=False)) @@ -211,7 +211,7 @@ def test_dagrun_find_duplicate(self, session): dag_id=dag_id, run_id=dag_id, run_type=DagRunType.MANUAL, - execution_date=now, + logical_date=now, start_date=now, state=DagRunState.RUNNING, external_trigger=True, @@ -220,10 +220,10 @@ def test_dagrun_find_duplicate(self, session): session.commit() - assert DagRun.find_duplicate(dag_id=dag_id, run_id=dag_id, execution_date=now) is not None - assert DagRun.find_duplicate(dag_id=dag_id, run_id=dag_id, execution_date=None) is not None - assert DagRun.find_duplicate(dag_id=dag_id, run_id=None, execution_date=now) is not None - assert DagRun.find_duplicate(dag_id=dag_id, run_id=None, execution_date=None) is None + assert DagRun.find_duplicate(dag_id=dag_id, run_id=dag_id, logical_date=now) is not None + assert DagRun.find_duplicate(dag_id=dag_id, run_id=dag_id, logical_date=None) is not None + assert DagRun.find_duplicate(dag_id=dag_id, run_id=None, logical_date=now) is not None + assert DagRun.find_duplicate(dag_id=dag_id, run_id=None, logical_date=None) is None def test_dagrun_success_when_all_skipped(self, session): """ @@ -305,7 +305,7 @@ def test_dagrun_success_conditions(self, session): dr = dag.create_dagrun( run_id="test_dagrun_success_conditions", state=DagRunState.RUNNING, - execution_date=now, + logical_date=now, data_interval=dag.timetable.infer_manual_data_interval(run_after=now), start_date=now, **triggered_by_kwargs, @@ -350,7 +350,7 @@ def test_dagrun_deadlock(self, session): dr = dag.create_dagrun( run_id="test_dagrun_deadlock", state=DagRunState.RUNNING, - execution_date=now, + logical_date=now, data_interval=dag.timetable.infer_manual_data_interval(run_after=now), start_date=now, session=session, @@ -385,7 +385,7 @@ def test_dagrun_no_deadlock_with_restarting(self, session): dr = dag.create_dagrun( run_id="test_dagrun_no_deadlock_with_shutdown", state=DagRunState.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE), start_date=DEFAULT_DATE, **triggered_by_kwargs, @@ -407,7 +407,7 @@ def test_dagrun_no_deadlock_with_depends_on_past(self, session): dr = dag.create_dagrun( run_id="test_dagrun_no_deadlock_1", state=DagRunState.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE), start_date=DEFAULT_DATE, **triggered_by_kwargs, @@ -416,7 +416,7 @@ def test_dagrun_no_deadlock_with_depends_on_past(self, session): dr2 = dag.create_dagrun( run_id="test_dagrun_no_deadlock_2", state=DagRunState.RUNNING, - execution_date=next_date, + logical_date=next_date, data_interval=dag.timetable.infer_manual_data_interval(run_after=next_date), start_date=next_date, **triggered_by_kwargs, @@ -611,7 +611,7 @@ def test_dagrun_set_state_end_date(self, session): dr = dag.create_dagrun( run_id="test_dagrun_set_state_end_date", state=DagRunState.RUNNING, - execution_date=now, + logical_date=now, data_interval=dag.timetable.infer_manual_data_interval(now), start_date=now, **triggered_by_kwargs, @@ -669,7 +669,7 @@ def test_dagrun_update_state_end_date(self, session): dr = dag.create_dagrun( run_id="test_dagrun_update_state_end_date", state=DagRunState.RUNNING, - execution_date=now, + logical_date=now, data_interval=dag.timetable.infer_manual_data_interval(now), start_date=now, **triggered_by_kwargs, @@ -731,7 +731,7 @@ def test_get_task_instance_on_empty_dagrun(self, session): dag_id=dag.dag_id, run_id="test_get_task_instance_on_empty_dagrun", run_type=DagRunType.MANUAL, - execution_date=now, + logical_date=now, start_date=now, state=DagRunState.RUNNING, external_trigger=False, @@ -744,13 +744,13 @@ def test_get_task_instance_on_empty_dagrun(self, session): def test_get_latest_runs(self, session): dag = DAG(dag_id="test_latest_runs_1", schedule=datetime.timedelta(days=1), start_date=DEFAULT_DATE) - self.create_dag_run(dag, execution_date=timezone.datetime(2015, 1, 1), session=session) - self.create_dag_run(dag, execution_date=timezone.datetime(2015, 1, 2), session=session) + self.create_dag_run(dag, logical_date=timezone.datetime(2015, 1, 1), session=session) + self.create_dag_run(dag, logical_date=timezone.datetime(2015, 1, 2), session=session) dagruns = DagRun.get_latest_runs(session) session.close() for dagrun in dagruns: if dagrun.dag_id == "test_latest_runs_1": - assert dagrun.execution_date == timezone.datetime(2015, 1, 2) + assert dagrun.logical_date == timezone.datetime(2015, 1, 2) def test_removed_task_instances_can_be_restored(self, session): def with_all_tasks_removed(dag): @@ -842,13 +842,13 @@ def test_depends_on_past(self, dagbag, session, prev_ti_state, is_ti_success): dag_run_1 = self.create_dag_run( dag, - execution_date=timezone.datetime(2016, 1, 1, 0, 0, 0), + logical_date=timezone.datetime(2016, 1, 1, 0, 0, 0), is_backfill=True, session=session, ) dag_run_2 = self.create_dag_run( dag, - execution_date=timezone.datetime(2016, 1, 2, 0, 0, 0), + logical_date=timezone.datetime(2016, 1, 2, 0, 0, 0), is_backfill=True, session=session, ) @@ -882,13 +882,13 @@ def test_wait_for_downstream(self, dagbag, session, prev_ti_state, is_ti_success # Otherwise ti.previous_ti returns an unpersisted TI dag_run_1 = self.create_dag_run( dag, - execution_date=timezone.datetime(2016, 1, 1, 0, 0, 0), + logical_date=timezone.datetime(2016, 1, 1, 0, 0, 0), is_backfill=True, session=session, ) dag_run_2 = self.create_dag_run( dag, - execution_date=timezone.datetime(2016, 1, 2, 0, 0, 0), + logical_date=timezone.datetime(2016, 1, 2, 0, 0, 0), is_backfill=True, session=session, ) @@ -929,7 +929,7 @@ def test_next_dagruns_to_examine_only_unpaused(self, session, state): dr = dag.create_dagrun( run_type=DagRunType.SCHEDULED, state=state, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=dag.infer_automated_data_interval(DEFAULT_DATE), start_date=DEFAULT_DATE if state == DagRunState.RUNNING else None, session=session, @@ -1002,7 +1002,7 @@ def test_emit_scheduling_delay(self, session, schedule, expected): dag_run = dag.create_dagrun( run_type=DagRunType.SCHEDULED, state=DagRunState.SUCCESS, - execution_date=dag.start_date, + logical_date=dag.start_date, data_interval=dag.infer_automated_data_interval(dag.start_date), start_date=dag.start_date, session=session, @@ -1110,7 +1110,7 @@ def test_verify_integrity_task_start_and_end_date(Stats_incr, session, run_type, dag_run = DagRun( dag_id=dag.dag_id, run_type=run_type, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_id=DagRun.generate_run_id(run_type, DEFAULT_DATE), ) dag_run.dag = dag diff --git a/tests/models/test_param.py b/tests/models/test_param.py index 79fbbeaabb08..d73c8c830f9c 100644 --- a/tests/models/test_param.py +++ b/tests/models/test_param.py @@ -328,7 +328,7 @@ def return_num(num): start_date=timezone.utcnow(), ) - xcom_arg.operator.run(dr.execution_date, dr.execution_date) + xcom_arg.operator.run(dr.logical_date, dr.logical_date) ti = dr.get_task_instances()[0] assert ti.xcom_pull() == self.VALUE @@ -354,7 +354,7 @@ def return_num(num): conf={"value": new_value}, ) - xcom_arg.operator.run(dr.execution_date, dr.execution_date) + xcom_arg.operator.run(dr.logical_date, dr.logical_date) ti = dr.get_task_instances()[0] assert ti.xcom_pull() == new_value @@ -374,7 +374,7 @@ def return_num(num): dr = dag_maker.create_dagrun(run_id=DagRunType.MANUAL.value, start_date=timezone.utcnow()) - xcom_arg.operator.run(dr.execution_date, dr.execution_date) + xcom_arg.operator.run(dr.logical_date, dr.logical_date) ti = dr.get_task_instances()[0] assert ti.xcom_pull() == "test" diff --git a/tests/models/test_renderedtifields.py b/tests/models/test_renderedtifields.py index 62f4be3e7405..0fd7c356abf1 100644 --- a/tests/models/test_renderedtifields.py +++ b/tests/models/test_renderedtifields.py @@ -241,7 +241,7 @@ def test_delete_old_records( rtif_list = [] for num in range(rtif_num): dr = dag_maker.create_dagrun( - run_id=str(num), execution_date=dag.start_date + timedelta(days=num) + run_id=str(num), logical_date=dag.start_date + timedelta(days=num) ) ti = dr.task_instances[0] ti.task = task @@ -283,7 +283,7 @@ def test_delete_old_records_mapped( mapped = BashOperator.partial(task_id="mapped").expand(bash_command=["a", "b"]) for num in range(num_runs): dr = dag_maker.create_dagrun( - run_id=f"run_{num}", execution_date=dag.start_date + timedelta(days=num) + run_id=f"run_{num}", logical_date=dag.start_date + timedelta(days=num) ) mapped.expand_mapped_task(dr.run_id, session=dag_maker.session) @@ -408,9 +408,9 @@ def test_rtif_deletion_stale_data_error(self, dag_maker, session): def run_task(date): run_id = f"abc_{date.to_date_string()}" - dr = session.scalar(select(DagRun).where(DagRun.execution_date == date, DagRun.run_id == run_id)) + dr = session.scalar(select(DagRun).where(DagRun.logical_date == date, DagRun.run_id == run_id)) if not dr: - dr = dag_maker.create_dagrun(execution_date=date, run_id=run_id) + dr = dag_maker.create_dagrun(logical_date=date, run_id=run_id) ti = dr.task_instances[0] ti.state = None ti.try_number += 1 @@ -429,7 +429,7 @@ def run_task(date): # find oldest date date = session.scalar( - select(DagRun.execution_date).join(RTIF.dag_run).order_by(DagRun.execution_date).limit(1) + select(DagRun.logical_date).join(RTIF.dag_run).order_by(DagRun.logical_date).limit(1) ) date = pendulum.instance(date) # rerun the old date. this will fail diff --git a/tests/models/test_skipmixin.py b/tests/models/test_skipmixin.py index 1efb3e2bb493..a3ce1473a302 100644 --- a/tests/models/test_skipmixin.py +++ b/tests/models/test_skipmixin.py @@ -63,7 +63,7 @@ def test_skip(self, mock_now, dag_maker): tasks = [EmptyOperator(task_id="task")] dag_run = dag_maker.create_dagrun( run_type=DagRunType.MANUAL, - execution_date=now, + logical_date=now, state=State.FAILED, ) SkipMixin().skip(dag_run=dag_run, tasks=tasks) diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index 8f15005f43d1..fbe92f006742 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -41,7 +41,6 @@ from airflow import settings from airflow.assets import AssetAlias from airflow.decorators import task, task_group -from airflow.example_dags.plugins.workday import AfterWorkdayTimetable from airflow.exceptions import ( AirflowException, AirflowFailException, @@ -81,7 +80,7 @@ from airflow.providers.standard.sensors.python import PythonSensor from airflow.sensors.base import BaseSensorOperator from airflow.serialization.serialized_objects import SerializedBaseOperator, SerializedDAG -from airflow.settings import TIMEZONE, TracebackSessionForTests +from airflow.settings import TracebackSessionForTests from airflow.stats import Stats from airflow.ti_deps.dep_context import DepContext from airflow.ti_deps.dependencies_deps import REQUEUEABLE_DEPS, RUNNING_DEPS @@ -133,14 +132,14 @@ def wrapper(ti): class CallbackWrapper: task_id: str | None = None dag_id: str | None = None - execution_date: datetime.datetime | None = None + logical_date: datetime.datetime | None = None task_state_in_callback: str | None = None callback_ran = False def wrap_task_instance(self, ti): self.task_id = ti.task_id self.dag_id = ti.dag_id - self.execution_date = ti.execution_date + self.logical_date = ti.logical_date self.task_state_in_callback = "" self.callback_ran = False @@ -360,7 +359,7 @@ def test_not_requeue_non_requeueable_task_instance(self, dag_maker): task_id="test_not_requeue_non_requeueable_task_instance_op", pool="test_pool", ) - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task ti.state = State.QUEUED with create_session() as session: @@ -496,7 +495,7 @@ def raise_skip_exception(): python_callable=raise_skip_exception, ) - dr = dag_maker.create_dagrun(execution_date=timezone.utcnow()) + dr = dag_maker.create_dagrun(logical_date=timezone.utcnow()) ti = dr.task_instances[0] ti.task = task ti.run() @@ -640,7 +639,7 @@ def run_with_error(ti): with contextlib.suppress(AirflowException): ti.run() - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task with create_session() as session: @@ -692,7 +691,7 @@ def run_with_error(ti): with contextlib.suppress(AirflowException): ti.run() - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task assert ti.try_number == 0 @@ -821,7 +820,7 @@ def func(): retry_delay=datetime.timedelta(seconds=0), ) - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task assert ti.try_number == 0 @@ -929,7 +928,7 @@ def func(): retry_delay=datetime.timedelta(seconds=0), ).expand(poke_interval=[0]) - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task assert ti.try_number == 0 @@ -1034,7 +1033,7 @@ def func(): retry_delay=datetime.timedelta(seconds=0), pool="test_pool", ).expand(poke_interval=[0]) - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task assert ti.try_number == 0 @@ -1100,7 +1099,7 @@ def func(): retry_delay=datetime.timedelta(seconds=0), pool="test_pool", ) - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task assert ti.try_number == 0 @@ -1154,7 +1153,7 @@ def test_depends_on_past(self, dag_maker): run_date = task.start_date + datetime.timedelta(days=5) dr = dag_maker.create_dagrun( - execution_date=run_date, + logical_date=run_date, run_type=DagRunType.SCHEDULED, ) @@ -1426,7 +1425,7 @@ def test_check_task_dependencies( assert task.start_date is not None run_date = task.start_date + datetime.timedelta(days=5) - dr = dag_maker.create_dagrun(execution_date=run_date) + dr = dag_maker.create_dagrun(logical_date=run_date) dag_maker.session.commit() ti = dr.get_task_instance(downstream.task_id) ti.task = downstream @@ -1725,9 +1724,9 @@ def test_xcom_pull_after_deferral(self, create_task_instance, session): assert ti.xcom_pull(task_ids="test_xcom", key=key) == value @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode - def test_xcom_pull_different_execution_date(self, create_task_instance): + def test_xcom_pull_different_logical_date(self, create_task_instance): """ - tests xcom fetch behavior with different execution dates, using + tests xcom fetch behavior with different logical dates, using both xcom_pull with "include_prior_dates" and without """ key = "xcom_key" @@ -1739,7 +1738,7 @@ def test_xcom_pull_different_execution_date(self, create_task_instance): task_id="test_xcom", pool="test_xcom", ) - exec_date = ti.dag_run.execution_date + exec_date = ti.dag_run.logical_date ti.run(mark_success=True) ti.xcom_push(key=key, value=value) @@ -1755,7 +1754,7 @@ def test_xcom_pull_different_execution_date(self, create_task_instance): ) ti = TI(task=ti.task, run_id=dr.run_id) ti.run() - # We have set a new execution date (and did not pass in + # We have set a new logical date (and did not pass in # 'include_prior_dates'which means this task should now have a cleared # xcom value assert ti.xcom_pull(task_ids="test_xcom", key=key) is None @@ -1810,7 +1809,7 @@ def test_xcom_push_flag(self, dag_maker): python_callable=lambda: value, do_xcom_push=False, ) - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task ti.run() assert ti.xcom_pull(task_ids=task_id, key=XCOM_RETURN_KEY) is None @@ -1828,7 +1827,7 @@ def test_xcom_without_multiple_outputs(self, dag_maker): python_callable=lambda: value, do_xcom_push=True, ) - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task ti.run() assert ti.xcom_pull(task_ids=task_id, key=XCOM_RETURN_KEY) == value @@ -1847,7 +1846,7 @@ def test_xcom_with_multiple_outputs(self, dag_maker): do_xcom_push=True, multiple_outputs=True, ) - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task ti.run() assert ti.xcom_pull(task_ids=task_id, key=XCOM_RETURN_KEY) == value @@ -1868,7 +1867,7 @@ def test_xcom_with_multiple_outputs_and_no_mapping_result(self, dag_maker): do_xcom_push=True, multiple_outputs=True, ) - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task with pytest.raises(AirflowException) as ctx: ti.run() @@ -1895,7 +1894,7 @@ def post_execute(self, context, result=None): task_id="test_operator", python_callable=lambda: "error", ) - ti = dag_maker.create_dagrun(execution_date=DEFAULT_DATE).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=DEFAULT_DATE).task_instances[0] ti.task = task with pytest.raises(TestError): ti.run() @@ -2026,14 +2025,14 @@ def test_get_num_running_task_instances(self, create_task_instance): dag_id="test_get_num_running_task_instances", task_id="task1", session=session ) - execution_date = DEFAULT_DATE + datetime.timedelta(days=1) + logical_date = DEFAULT_DATE + datetime.timedelta(days=1) triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dr = ti1.task.dag.create_dagrun( - execution_date=execution_date, + logical_date=logical_date, state=None, run_id="2", session=session, - data_interval=(execution_date, execution_date), + data_interval=(logical_date, logical_date), **triggered_by_kwargs, ) assert ti1 in session @@ -2066,13 +2065,13 @@ def test_get_num_running_task_instances_per_dagrun(self, create_task_instance, d MockOperator.partial(task_id="task_3").expand_kwargs([{"a": 1, "b": 2}]) dr1 = dag_maker.create_dagrun( - execution_date=timezone.utcnow(), state=DagRunState.RUNNING, run_id="run_id_1", session=session + logical_date=timezone.utcnow(), state=DagRunState.RUNNING, run_id="run_id_1", session=session ) tis1 = {(ti.task_id, ti.map_index): ti for ti in dr1.task_instances} print(f"tis1: {tis1}") dr2 = dag_maker.create_dagrun( - execution_date=timezone.utcnow(), state=DagRunState.RUNNING, run_id="run_id_2", session=session + logical_date=timezone.utcnow(), state=DagRunState.RUNNING, run_id="run_id_2", session=session ) tis2 = {(ti.task_id, ti.map_index): ti for ti in dr2.task_instances} @@ -2107,7 +2106,7 @@ def test_get_num_running_task_instances_per_dagrun(self, create_task_instance, d assert 1 == tis2[("task_3", 0)].get_num_running_task_instances(session=session, same_dagrun=True) def test_log_url(self, create_task_instance): - ti = create_task_instance(dag_id="my_dag", task_id="op", execution_date=timezone.datetime(2018, 1, 1)) + ti = create_task_instance(dag_id="my_dag", task_id="op", logical_date=timezone.datetime(2018, 1, 1)) expected_url = ( "http://localhost:8080" @@ -2121,14 +2120,14 @@ def test_log_url(self, create_task_instance): def test_mark_success_url(self, create_task_instance): now = pendulum.now("Europe/Brussels") - ti = create_task_instance(dag_id="dag", task_id="op", execution_date=now) + ti = create_task_instance(dag_id="dag", task_id="op", logical_date=now) query = urllib.parse.parse_qs( urllib.parse.urlsplit(ti.mark_success_url).query, keep_blank_values=True, strict_parsing=True ) assert query["dag_id"][0] == "dag" assert query["task_id"][0] == "op" assert query["dag_run_id"][0] == "test" - assert ti.execution_date == now + assert ti.logical_date == now def test_overwrite_params_with_dag_run_conf(self, create_task_instance): ti = create_task_instance() @@ -2160,7 +2159,7 @@ def test_overwrite_params_with_dag_run_conf_none(self, create_task_instance): def test_email_alert(self, mock_send_email, dag_maker, use_native_obj): with dag_maker(dag_id="test_failure_email", render_template_as_native_obj=use_native_obj): task = BashOperator(task_id="test_email_alert", bash_command="exit 1", email="to") - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task with contextlib.suppress(AirflowException): @@ -2187,7 +2186,7 @@ def test_email_alert_with_config(self, mock_send_email, dag_maker): bash_command="exit 1", email="to", ) - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task opener = mock_open(read_data="template: {{ti.task_id}}") @@ -2209,7 +2208,7 @@ def test_email_alert_with_filenotfound_config(self, mock_send_email, dag_maker): bash_command="exit 1", email="to", ) - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task # Run test when the template file is not found @@ -2244,7 +2243,7 @@ def test_email_alert(x): test_email_alert.expand(x=["a", "b"]) # This is 'test_email_alert'. test_email_alert.expand(x=[1, 2, 3]) # This is 'test_email_alert__1'. - dr: DagRun = dag_maker.create_dagrun(execution_date=timezone.utcnow()) + dr: DagRun = dag_maker.create_dagrun(logical_date=timezone.utcnow()) ti = dr.get_task_instance(task_id, map_index=0, session=session) assert ti is not None @@ -2279,7 +2278,7 @@ def test_success_callback_no_race_condition(self, create_task_instance): ti = create_task_instance( on_success_callback=callback_wrapper.success_handler, end_date=timezone.utcnow() + datetime.timedelta(days=10), - execution_date=timezone.utcnow(), + logical_date=timezone.utcnow(), state=State.RUNNING, ) @@ -2399,7 +2398,7 @@ def raise_an_exception(placeholder: int): _ = raise_an_exception.expand(placeholder=[0, 1]) - tis = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances + tis = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances for task_instance in tis: if task_instance.map_index == 0: with pytest.raises(AirflowFailException): @@ -3068,12 +3067,12 @@ def _test_previous_dates_setup( with dag_maker(dag_id=dag_id, schedule=schedule, catchup=catchup, serialized=True): task = EmptyOperator(task_id="task") - def get_test_ti(execution_date: pendulum.DateTime, state: str) -> TI: + def get_test_ti(logical_date: pendulum.DateTime, state: str) -> TI: dr = dag_maker.create_dagrun( - run_id=f"test__{execution_date.isoformat()}", + run_id=f"test__{logical_date.isoformat()}", run_type=DagRunType.SCHEDULED, state=state, - execution_date=execution_date, + logical_date=logical_date, start_date=pendulum.now("UTC"), ) ti = dr.task_instances[0] @@ -3127,18 +3126,18 @@ def test_previous_ti_success(self, schedule, catchup, dag_maker) -> None: assert ti_list[3].get_previous_ti(state=State.SUCCESS).run_id != ti_list[2].run_id @pytest.mark.parametrize("schedule, catchup", _prev_dates_param_list) - def test_previous_execution_date_success(self, schedule, catchup, dag_maker) -> None: + def test_previous_logical_date_success(self, schedule, catchup, dag_maker) -> None: scenario = [State.FAILED, State.SUCCESS, State.FAILED, State.SUCCESS] ti_list = self._test_previous_dates_setup(schedule, catchup, scenario, dag_maker) # vivify for ti in ti_list: - ti.execution_date + ti.logical_date - assert ti_list[0].get_previous_execution_date(state=State.SUCCESS) is None - assert ti_list[1].get_previous_execution_date(state=State.SUCCESS) is None - assert ti_list[3].get_previous_execution_date(state=State.SUCCESS) == ti_list[1].execution_date - assert ti_list[3].get_previous_execution_date(state=State.SUCCESS) != ti_list[2].execution_date + assert ti_list[0].get_previous_logical_date(state=State.SUCCESS) is None + assert ti_list[1].get_previous_logical_date(state=State.SUCCESS) is None + assert ti_list[3].get_previous_logical_date(state=State.SUCCESS) == ti_list[1].logical_date + assert ti_list[3].get_previous_logical_date(state=State.SUCCESS) != ti_list[2].logical_date @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize("schedule, catchup", _prev_dates_param_list) @@ -3167,14 +3166,14 @@ def test_get_previous_start_date_none(self, dag_maker): # should return the start_date of ti_1 (which is None because ti_1 was not run). # It should not raise an error. dagrun_1 = dag_maker.create_dagrun( - execution_date=day_1, + logical_date=day_1, state=State.RUNNING, run_type=DagRunType.MANUAL, **triggered_by_kwargs, ) dagrun_2 = dag_maker.create_dagrun( - execution_date=day_2, + logical_date=day_2, state=State.RUNNING, run_type=DagRunType.MANUAL, data_interval=(day_1, day_2), @@ -3206,7 +3205,7 @@ def test_context_triggering_asset_events(self, create_dummy_dag, session): session.add_all([ds1, ds2]) session.commit() - execution_date = timezone.utcnow() + logical_date = timezone.utcnow() triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} # it's easier to fake a manual run here dag, task1 = create_dummy_dag( @@ -3220,10 +3219,10 @@ def test_context_triggering_asset_events(self, create_dummy_dag, session): dr = dag.create_dagrun( run_id="test2", run_type=DagRunType.ASSET_TRIGGERED, - execution_date=execution_date, + logical_date=logical_date, state=None, session=session, - data_interval=(execution_date, execution_date), + data_interval=(logical_date, logical_date), **triggered_by_kwargs, ) ds1_event = AssetEvent(asset_id=1) @@ -3274,19 +3273,6 @@ def test_template_render(self, create_task_instance, session): result = ti.task.render_template("Task: {{ dag.dag_id }} -> {{ task.task_id }}", template_context) assert result == "Task: test_template_render -> test_template_render_task" - def test_template_render_deprecated(self, create_task_instance, session): - ti = create_task_instance( - dag_id="test_template_render", - task_id="test_template_render_task", - schedule="0 12 * * *", - ) - session.add(ti) - session.commit() - template_context = ti.get_template_context() - with pytest.deprecated_call(): - result = ti.task.render_template("Execution date: {{ execution_date }}", template_context) - assert result.startswith("Execution date: ") - @pytest.mark.skip_if_database_isolation_mode # Does not work in db isolation mode @pytest.mark.parametrize( "content, expected_output", @@ -3395,59 +3381,6 @@ def test_template_with_json_variable_missing(self, create_task_instance, session with pytest.raises(KeyError): ti.task.render_template('{{ var.json.get("missing_variable") }}', context) - @pytest.mark.parametrize( - ("field", "expected"), - [ - ("next_ds", "2016-01-01"), - ("next_ds_nodash", "20160101"), - ("prev_ds", "2015-12-31"), - ("prev_ds_nodash", "20151231"), - ("yesterday_ds", "2015-12-31"), - ("yesterday_ds_nodash", "20151231"), - ("tomorrow_ds", "2016-01-02"), - ("tomorrow_ds_nodash", "20160102"), - ], - ) - def test_deprecated_context(self, field, expected, create_task_instance): - ti = create_task_instance(execution_date=DEFAULT_DATE, serialized=True) - context = ti.get_template_context() - with pytest.deprecated_call() as recorder: - assert context[field] == expected - message_beginning = ( - f"Accessing {field!r} from the template is deprecated and " - f"will be removed in a future version." - ) - - recorded_message = [str(m.message) for m in recorder] - assert len(recorded_message) == 1 - assert recorded_message[0].startswith(message_beginning) - - def test_template_with_custom_timetable_deprecated_context(self, create_task_instance, session): - ti = create_task_instance( - start_date=DEFAULT_DATE, - schedule=AfterWorkdayTimetable(), - run_type=DagRunType.SCHEDULED, - execution_date=timezone.datetime(2021, 9, 6), - data_interval=(timezone.datetime(2021, 9, 6), timezone.datetime(2021, 9, 7)), - ) - session.add(ti) - session.commit() - context = ti.get_template_context() - with pytest.deprecated_call(): - assert context["execution_date"] == pendulum.DateTime(2021, 9, 6, tzinfo=TIMEZONE) - with pytest.deprecated_call(): - assert context["next_ds"] == "2021-09-07" - with pytest.deprecated_call(): - assert context["next_ds_nodash"] == "20210907" - with pytest.deprecated_call(): - assert context["next_execution_date"] == pendulum.DateTime(2021, 9, 7, tzinfo=TIMEZONE) - with pytest.deprecated_call(): - assert context["prev_ds"] is None, "Does not make sense for custom timetable" - with pytest.deprecated_call(): - assert context["prev_ds_nodash"] is None, "Does not make sense for custom timetable" - with pytest.deprecated_call(): - assert context["prev_execution_date"] is None, "Does not make sense for custom timetable" - def test_execute_callback(self, create_task_instance): called = False @@ -3543,15 +3476,15 @@ def test_handle_failure(self, create_dummy_dag, session=None): on_retry_callback=mock_on_retry_1, session=session, ) - execution_date = timezone.utcnow() + logical_date = timezone.utcnow() triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dr = dag.create_dagrun( run_id="test2", run_type=DagRunType.MANUAL, - execution_date=execution_date, + logical_date=logical_date, state=None, session=session, - data_interval=(execution_date, execution_date), + data_interval=(logical_date, logical_date), **triggered_by_kwargs, ) ti1 = dr.get_task_instance(task1.task_id, session=session) @@ -3694,15 +3627,15 @@ def test_handle_failure_fail_stop(self, create_dummy_dag, session=None): session=session, fail_stop=True, ) - execution_date = timezone.utcnow() + logical_date = timezone.utcnow() triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dr = dag.create_dagrun( run_id="test_ff", run_type=DagRunType.MANUAL, - execution_date=execution_date, + logical_date=logical_date, state=None, session=session, - data_interval=(execution_date, execution_date), + data_interval=(logical_date, logical_date), **triggered_by_kwargs, ) @@ -3749,7 +3682,7 @@ def fail(): python_callable=fail, retries=1, ) - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task with contextlib.suppress(AirflowException): ti.run() @@ -3766,7 +3699,7 @@ def fail(): python_callable=fail, retries=1, ) - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task with contextlib.suppress(AirflowException): ti.run() @@ -3784,7 +3717,7 @@ def fail(): python_callable=fail, retries=1, ) - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] ti.task = task mock_log = mock.Mock() mock_get_log.return_value = mock_log @@ -3800,7 +3733,7 @@ def fail(): def _env_var_check_callback(self): assert "test_echo_env_variables" == os.environ["AIRFLOW_CTX_DAG_ID"] assert "hive_in_python_op" == os.environ["AIRFLOW_CTX_TASK_ID"] - assert DEFAULT_DATE.isoformat() == os.environ["AIRFLOW_CTX_EXECUTION_DATE"] + assert DEFAULT_DATE.isoformat() == os.environ["AIRFLOW_CTX_LOGICAL_DATE"] assert DagRun.generate_run_id(DagRunType.MANUAL, DEFAULT_DATE) == os.environ["AIRFLOW_CTX_DAG_RUN_ID"] def test_echo_env_variables(self, dag_maker): @@ -4139,7 +4072,7 @@ def raise_skip_exception(): on_skipped_callback=on_skipped_callback_function, on_success_callback=on_success_callback_function, ) - dr = dag_maker.create_dagrun(execution_date=timezone.utcnow()) + dr = dag_maker.create_dagrun(logical_date=timezone.utcnow()) ti = dr.task_instances[0] ti.task = task ti.run() @@ -4256,7 +4189,7 @@ def timeout(): retries=retries, mode=mode, ) - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] with pytest.raises(AirflowSensorTimeout): ti.run() @@ -4285,7 +4218,7 @@ def timeout(): on_failure_callback=mock_on_failure, retries=retries, ).expand(mode=[mode]) - ti = dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0] + ti = dag_maker.create_dagrun(logical_date=timezone.utcnow()).task_instances[0] with pytest.raises(AirflowSensorTimeout): ti.run() diff --git a/tests/models/test_trigger.py b/tests/models/test_trigger.py index 065575e39004..af6bde061671 100644 --- a/tests/models/test_trigger.py +++ b/tests/models/test_trigger.py @@ -86,7 +86,7 @@ def test_clean_unused(session, create_task_instance): assert session.query(Trigger).count() == 3 # Tie one to a fake TaskInstance that is not deferred, and one to one that is task_instance = create_task_instance( - session=session, task_id="fake", state=State.DEFERRED, execution_date=timezone.utcnow() + session=session, task_id="fake", state=State.DEFERRED, logical_date=timezone.utcnow() ) task_instance.trigger_id = trigger1.id session.add(task_instance) @@ -115,7 +115,7 @@ def test_submit_event(session, create_task_instance): session.commit() # Make a TaskInstance that's deferred and waiting on it task_instance = create_task_instance( - session=session, execution_date=timezone.utcnow(), state=State.DEFERRED + session=session, logical_date=timezone.utcnow(), state=State.DEFERRED ) task_instance.trigger_id = trigger.id task_instance.next_kwargs = {"cheesecake": True} @@ -142,9 +142,7 @@ def test_submit_failure(session, create_task_instance): session.add(trigger) session.commit() # Make a TaskInstance that's deferred and waiting on it - task_instance = create_task_instance( - task_id="fake", execution_date=timezone.utcnow(), state=State.DEFERRED - ) + task_instance = create_task_instance(task_id="fake", logical_date=timezone.utcnow(), state=State.DEFERRED) task_instance.trigger_id = trigger.id session.commit() # Call submit_event @@ -180,7 +178,7 @@ def test_submit_event_task_end(mock_utcnow, session, create_task_instance, event session.commit() # Make a TaskInstance that's deferred and waiting on it task_instance = create_task_instance( - session=session, execution_date=timezone.utcnow(), state=State.DEFERRED + session=session, logical_date=timezone.utcnow(), state=State.DEFERRED ) task_instance.trigger_id = trigger.id session.commit() @@ -249,7 +247,7 @@ def test_assign_unassigned(session, create_task_instance): session.add(trigger_on_healthy_triggerer) ti_trigger_on_healthy_triggerer = create_task_instance( task_id="ti_trigger_on_healthy_triggerer", - execution_date=time_now, + logical_date=time_now, run_id="trigger_on_healthy_triggerer_run_id", ) ti_trigger_on_healthy_triggerer.trigger_id = trigger_on_healthy_triggerer.id @@ -260,7 +258,7 @@ def test_assign_unassigned(session, create_task_instance): session.add(trigger_on_unhealthy_triggerer) ti_trigger_on_unhealthy_triggerer = create_task_instance( task_id="ti_trigger_on_unhealthy_triggerer", - execution_date=time_now + datetime.timedelta(hours=1), + logical_date=time_now + datetime.timedelta(hours=1), run_id="trigger_on_unhealthy_triggerer_run_id", ) ti_trigger_on_unhealthy_triggerer.trigger_id = trigger_on_unhealthy_triggerer.id @@ -271,7 +269,7 @@ def test_assign_unassigned(session, create_task_instance): session.add(trigger_on_killed_triggerer) ti_trigger_on_killed_triggerer = create_task_instance( task_id="ti_trigger_on_killed_triggerer", - execution_date=time_now + datetime.timedelta(hours=2), + logical_date=time_now + datetime.timedelta(hours=2), run_id="trigger_on_killed_triggerer_run_id", ) ti_trigger_on_killed_triggerer.trigger_id = trigger_on_killed_triggerer.id @@ -281,7 +279,7 @@ def test_assign_unassigned(session, create_task_instance): session.add(trigger_unassigned_to_triggerer) ti_trigger_unassigned_to_triggerer = create_task_instance( task_id="ti_trigger_unassigned_to_triggerer", - execution_date=time_now + datetime.timedelta(hours=3), + logical_date=time_now + datetime.timedelta(hours=3), run_id="trigger_unassigned_to_triggerer_run_id", ) ti_trigger_unassigned_to_triggerer.trigger_id = trigger_unassigned_to_triggerer.id @@ -317,38 +315,38 @@ def test_get_sorted_triggers_same_priority_weight(session, create_task_instance) """ Tests that triggers are sorted by the creation_date if they have the same priority. """ - old_execution_date = datetime.datetime( + old_logical_date = datetime.datetime( 2023, 5, 9, 12, 16, 14, 474415, tzinfo=pytz.timezone("Africa/Abidjan") ) trigger_old = Trigger( classpath="airflow.triggers.testing.SuccessTrigger", kwargs={}, - created_date=old_execution_date + datetime.timedelta(seconds=30), + created_date=old_logical_date + datetime.timedelta(seconds=30), ) trigger_old.id = 1 session.add(trigger_old) TI_old = create_task_instance( task_id="old", - execution_date=old_execution_date, + logical_date=old_logical_date, run_id="old_run_id", ) TI_old.priority_weight = 1 TI_old.trigger_id = trigger_old.id session.add(TI_old) - new_execution_date = datetime.datetime( + new_logical_date = datetime.datetime( 2023, 5, 9, 12, 17, 14, 474415, tzinfo=pytz.timezone("Africa/Abidjan") ) trigger_new = Trigger( classpath="airflow.triggers.testing.SuccessTrigger", kwargs={}, - created_date=new_execution_date + datetime.timedelta(seconds=30), + created_date=new_logical_date + datetime.timedelta(seconds=30), ) trigger_new.id = 2 session.add(trigger_new) TI_new = create_task_instance( task_id="new", - execution_date=new_execution_date, + logical_date=new_logical_date, run_id="new_run_id", ) TI_new.priority_weight = 1 @@ -368,38 +366,38 @@ def test_get_sorted_triggers_different_priority_weights(session, create_task_ins """ Tests that triggers are sorted by the priority_weight. """ - old_execution_date = datetime.datetime( + old_logical_date = datetime.datetime( 2023, 5, 9, 12, 16, 14, 474415, tzinfo=pytz.timezone("Africa/Abidjan") ) trigger_old = Trigger( classpath="airflow.triggers.testing.SuccessTrigger", kwargs={}, - created_date=old_execution_date + datetime.timedelta(seconds=30), + created_date=old_logical_date + datetime.timedelta(seconds=30), ) trigger_old.id = 1 session.add(trigger_old) TI_old = create_task_instance( task_id="old", - execution_date=old_execution_date, + logical_date=old_logical_date, run_id="old_run_id", ) TI_old.priority_weight = 1 TI_old.trigger_id = trigger_old.id session.add(TI_old) - new_execution_date = datetime.datetime( + new_logical_date = datetime.datetime( 2023, 5, 9, 12, 17, 14, 474415, tzinfo=pytz.timezone("Africa/Abidjan") ) trigger_new = Trigger( classpath="airflow.triggers.testing.SuccessTrigger", kwargs={}, - created_date=new_execution_date + datetime.timedelta(seconds=30), + created_date=new_logical_date + datetime.timedelta(seconds=30), ) trigger_new.id = 2 session.add(trigger_new) TI_new = create_task_instance( task_id="new", - execution_date=new_execution_date, + logical_date=new_logical_date, run_id="new_run_id", ) TI_new.priority_weight = 2 diff --git a/tests/models/test_xcom.py b/tests/models/test_xcom.py index 87a56c430e6f..d51cf956f09b 100644 --- a/tests/models/test_xcom.py +++ b/tests/models/test_xcom.py @@ -60,13 +60,13 @@ def reset_db(): @pytest.fixture def task_instance_factory(request, session: Session): - def func(*, dag_id, task_id, execution_date): - run_id = DagRun.generate_run_id(DagRunType.SCHEDULED, execution_date) + def func(*, dag_id, task_id, logical_date): + run_id = DagRun.generate_run_id(DagRunType.SCHEDULED, logical_date) run = DagRun( dag_id=dag_id, run_type=DagRunType.SCHEDULED, run_id=run_id, - execution_date=execution_date, + logical_date=logical_date, ) session.add(run) ti = TaskInstance(EmptyOperator(task_id=task_id), run_id=run_id) @@ -90,7 +90,7 @@ def task_instance(task_instance_factory): return task_instance_factory( dag_id="dag", task_id="task_1", - execution_date=timezone.datetime(2021, 12, 3, 4, 56), + logical_date=timezone.datetime(2021, 12, 3, 4, 56), ) @@ -186,7 +186,7 @@ def test_xcom_init_on_load_uses_orm_deserialize_value(self, mock_orm_deserialize key="key", value="value", timestamp=timezone.utcnow(), - execution_date=timezone.utcnow(), + logical_date=timezone.utcnow(), task_id="task_id", dag_id="dag_id", ) @@ -220,7 +220,7 @@ def test_get_one_custom_backend_no_use_orm_deserialize_value(self, task_instance @mock.patch("airflow.models.xcom.conf.getimport") def test_set_serialize_call_current_signature(self, get_import, task_instance): """ - When XCom.serialize_value includes params execution_date, key, dag_id, task_id and run_id, + When XCom.serialize_value includes params logical_date, key, dag_id, task_id and run_id, then XCom.set should pass all of them. """ serialize_watcher = MagicMock() @@ -312,10 +312,10 @@ def test_xcom_get_one(self, session, task_instance): @pytest.fixture def tis_for_xcom_get_one_from_prior_date(self, task_instance_factory, push_simple_json_xcom): date1 = timezone.datetime(2021, 12, 3, 4, 56) - ti1 = task_instance_factory(dag_id="dag", execution_date=date1, task_id="task_1") + ti1 = task_instance_factory(dag_id="dag", logical_date=date1, task_id="task_1") ti2 = task_instance_factory( dag_id="dag", - execution_date=date1 + datetime.timedelta(days=1), + logical_date=date1 + datetime.timedelta(days=1), task_id="task_1", ) @@ -380,8 +380,8 @@ def test_xcom_get_many_multiple_tasks(self, session, task_instance): def tis_for_xcom_get_many_from_prior_dates(self, task_instance_factory, push_simple_json_xcom): date1 = timezone.datetime(2021, 12, 3, 4, 56) date2 = date1 + datetime.timedelta(days=1) - ti1 = task_instance_factory(dag_id="dag", task_id="task_1", execution_date=date1) - ti2 = task_instance_factory(dag_id="dag", task_id="task_1", execution_date=date2) + ti1 = task_instance_factory(dag_id="dag", task_id="task_1", logical_date=date1) + ti2 = task_instance_factory(dag_id="dag", task_id="task_1", logical_date=date2) push_simple_json_xcom(ti=ti1, key="xcom_1", value={"key1": "value1"}) push_simple_json_xcom(ti=ti2, key="xcom_1", value={"key2": "value2"}) return ti1, ti2 @@ -400,7 +400,7 @@ def test_xcom_get_many_from_prior_dates(self, session, tis_for_xcom_get_many_fro # The retrieved XComs should be ordered by logical date, latest first. assert [x.value for x in stored_xcoms] == [{"key2": "value2"}, {"key1": "value1"}] - assert [x.execution_date for x in stored_xcoms] == [ti2.execution_date, ti1.execution_date] + assert [x.logical_date for x in stored_xcoms] == [ti2.logical_date, ti1.logical_date] @pytest.mark.usefixtures("setup_xcom_pickling") @@ -419,7 +419,7 @@ def test_xcom_set(self, session, task_instance): assert stored_xcoms[0].value == {"key": "value"} assert stored_xcoms[0].dag_id == "dag" assert stored_xcoms[0].task_id == "task_1" - assert stored_xcoms[0].execution_date == task_instance.execution_date + assert stored_xcoms[0].logical_date == task_instance.logical_date @pytest.fixture def setup_for_xcom_set_again_replace(self, task_instance, push_simple_json_xcom): diff --git a/tests/operators/test_branch_operator.py b/tests/operators/test_branch_operator.py index 9df1b897b427..eb014222a98b 100644 --- a/tests/operators/test_branch_operator.py +++ b/tests/operators/test_branch_operator.py @@ -75,7 +75,7 @@ def test_without_dag_run(self, dag_maker): branch_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE) - for ti in dag_maker.session.query(TI).filter(TI.dag_id == dag_id, TI.execution_date == DEFAULT_DATE): + for ti in dag_maker.session.query(TI).filter(TI.dag_id == dag_id, TI.logical_date == DEFAULT_DATE): if ti.task_id == "make_choice": assert ti.state == State.SUCCESS elif ti.task_id == "branch_1": @@ -114,7 +114,7 @@ def test_branch_list_without_dag_run(self, dag_maker): "branch_3": State.SKIPPED, } - for ti in dag_maker.session.query(TI).filter(TI.dag_id == dag_id, TI.execution_date == DEFAULT_DATE): + for ti in dag_maker.session.query(TI).filter(TI.dag_id == dag_id, TI.logical_date == DEFAULT_DATE): if ti.task_id in expected: assert ti.state == expected[ti.task_id] else: @@ -137,7 +137,7 @@ def test_with_dag_run(self, dag_maker): dag_maker.create_dagrun( run_type=DagRunType.MANUAL, start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, state=State.RUNNING, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, @@ -151,7 +151,7 @@ def test_with_dag_run(self, dag_maker): "branch_2": State.SKIPPED, } - for ti in dag_maker.session.query(TI).filter(TI.dag_id == dag_id, TI.execution_date == DEFAULT_DATE): + for ti in dag_maker.session.query(TI).filter(TI.dag_id == dag_id, TI.logical_date == DEFAULT_DATE): if ti.task_id in expected: assert ti.state == expected[ti.task_id] else: @@ -175,7 +175,7 @@ def test_with_skip_in_branch_downstream_dependencies(self, dag_maker): dag_maker.create_dagrun( run_type=DagRunType.MANUAL, start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, state=State.RUNNING, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, @@ -189,7 +189,7 @@ def test_with_skip_in_branch_downstream_dependencies(self, dag_maker): "branch_2": State.NONE, } - for ti in dag_maker.session.query(TI).filter(TI.dag_id == dag_id, TI.execution_date == DEFAULT_DATE): + for ti in dag_maker.session.query(TI).filter(TI.dag_id == dag_id, TI.logical_date == DEFAULT_DATE): if ti.task_id in expected: assert ti.state == expected[ti.task_id] else: @@ -213,7 +213,7 @@ def test_xcom_push(self, dag_maker): dag_maker.create_dagrun( run_type=DagRunType.MANUAL, start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, state=State.RUNNING, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, @@ -221,7 +221,7 @@ def test_xcom_push(self, dag_maker): branch_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE) - for ti in dag_maker.session.query(TI).filter(TI.dag_id == dag_id, TI.execution_date == DEFAULT_DATE): + for ti in dag_maker.session.query(TI).filter(TI.dag_id == dag_id, TI.logical_date == DEFAULT_DATE): if ti.task_id == "make_choice": assert ti.xcom_pull(task_ids="make_choice") == "branch_1" @@ -247,7 +247,7 @@ def test_with_dag_run_task_groups(self, dag_maker): dag_maker.create_dagrun( run_type=DagRunType.MANUAL, start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, state=State.RUNNING, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, @@ -255,7 +255,7 @@ def test_with_dag_run_task_groups(self, dag_maker): branch_op.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE) - for ti in dag_maker.session.query(TI).filter(TI.dag_id == dag_id, TI.execution_date == DEFAULT_DATE): + for ti in dag_maker.session.query(TI).filter(TI.dag_id == dag_id, TI.logical_date == DEFAULT_DATE): if ti.task_id == "make_choice": assert ti.state == State.SUCCESS elif ti.task_id == "branch_1": diff --git a/tests/operators/test_latest_only_operator.py b/tests/operators/test_latest_only_operator.py index fbc2cd5b46d8..0182be26f6bd 100644 --- a/tests/operators/test_latest_only_operator.py +++ b/tests/operators/test_latest_only_operator.py @@ -52,7 +52,7 @@ def get_task_instances(task_id): session.query(TaskInstance) .join(TaskInstance.dag_run) .filter(TaskInstance.task_id == task_id) - .order_by(DagRun.execution_date) + .order_by(DagRun.logical_date) .all() ) @@ -100,7 +100,7 @@ def test_skipping_non_latest(self, dag_maker): dag_maker.create_dagrun( run_type=DagRunType.SCHEDULED, start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, state=State.RUNNING, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, @@ -109,7 +109,7 @@ def test_skipping_non_latest(self, dag_maker): dag_maker.create_dagrun( run_type=DagRunType.SCHEDULED, start_date=timezone.utcnow(), - execution_date=timezone.datetime(2016, 1, 1, 12), + logical_date=timezone.datetime(2016, 1, 1, 12), state=State.RUNNING, data_interval=(timezone.datetime(2016, 1, 1, 12), timezone.datetime(2016, 1, 1, 12) + INTERVAL), **triggered_by_kwargs, @@ -118,7 +118,7 @@ def test_skipping_non_latest(self, dag_maker): dag_maker.create_dagrun( run_type=DagRunType.SCHEDULED, start_date=timezone.utcnow(), - execution_date=END_DATE, + logical_date=END_DATE, state=State.RUNNING, data_interval=(END_DATE, END_DATE + INTERVAL), **triggered_by_kwargs, @@ -130,7 +130,7 @@ def test_skipping_non_latest(self, dag_maker): downstream_task3.run(start_date=DEFAULT_DATE, end_date=END_DATE) latest_instances = get_task_instances("latest") - exec_date_to_latest_state = {ti.execution_date: ti.state for ti in latest_instances} + exec_date_to_latest_state = {ti.logical_date: ti.state for ti in latest_instances} assert { timezone.datetime(2016, 1, 1): "success", timezone.datetime(2016, 1, 1, 12): "success", @@ -138,7 +138,7 @@ def test_skipping_non_latest(self, dag_maker): } == exec_date_to_latest_state downstream_instances = get_task_instances("downstream") - exec_date_to_downstream_state = {ti.execution_date: ti.state for ti in downstream_instances} + exec_date_to_downstream_state = {ti.logical_date: ti.state for ti in downstream_instances} assert { timezone.datetime(2016, 1, 1): "skipped", timezone.datetime(2016, 1, 1, 12): "skipped", @@ -146,7 +146,7 @@ def test_skipping_non_latest(self, dag_maker): } == exec_date_to_downstream_state downstream_instances = get_task_instances("downstream_2") - exec_date_to_downstream_state = {ti.execution_date: ti.state for ti in downstream_instances} + exec_date_to_downstream_state = {ti.logical_date: ti.state for ti in downstream_instances} assert { timezone.datetime(2016, 1, 1): None, timezone.datetime(2016, 1, 1, 12): None, @@ -154,7 +154,7 @@ def test_skipping_non_latest(self, dag_maker): } == exec_date_to_downstream_state downstream_instances = get_task_instances("downstream_3") - exec_date_to_downstream_state = {ti.execution_date: ti.state for ti in downstream_instances} + exec_date_to_downstream_state = {ti.logical_date: ti.state for ti in downstream_instances} assert { timezone.datetime(2016, 1, 1): "success", timezone.datetime(2016, 1, 1, 12): "success", @@ -177,28 +177,28 @@ def test_not_skipping_external(self, dag_maker): dag_maker.create_dagrun( run_type=DagRunType.MANUAL, start_date=timezone.utcnow(), - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, state=State.RUNNING, external_trigger=True, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, ) - execution_date = timezone.datetime(2016, 1, 1, 12) + logical_date = timezone.datetime(2016, 1, 1, 12) dag_maker.create_dagrun( run_type=DagRunType.MANUAL, start_date=timezone.utcnow(), - execution_date=execution_date, + logical_date=logical_date, state=State.RUNNING, external_trigger=True, - data_interval=(execution_date, execution_date), + data_interval=(logical_date, logical_date), **triggered_by_kwargs, ) dag_maker.create_dagrun( run_type=DagRunType.MANUAL, start_date=timezone.utcnow(), - execution_date=END_DATE, + logical_date=END_DATE, state=State.RUNNING, external_trigger=True, data_interval=(END_DATE, END_DATE), @@ -210,7 +210,7 @@ def test_not_skipping_external(self, dag_maker): downstream_task2.run(start_date=DEFAULT_DATE, end_date=END_DATE) latest_instances = get_task_instances("latest") - exec_date_to_latest_state = {ti.execution_date: ti.state for ti in latest_instances} + exec_date_to_latest_state = {ti.logical_date: ti.state for ti in latest_instances} assert { timezone.datetime(2016, 1, 1): "success", timezone.datetime(2016, 1, 1, 12): "success", @@ -218,7 +218,7 @@ def test_not_skipping_external(self, dag_maker): } == exec_date_to_latest_state downstream_instances = get_task_instances("downstream") - exec_date_to_downstream_state = {ti.execution_date: ti.state for ti in downstream_instances} + exec_date_to_downstream_state = {ti.logical_date: ti.state for ti in downstream_instances} assert { timezone.datetime(2016, 1, 1): "success", timezone.datetime(2016, 1, 1, 12): "success", @@ -226,7 +226,7 @@ def test_not_skipping_external(self, dag_maker): } == exec_date_to_downstream_state downstream_instances = get_task_instances("downstream_2") - exec_date_to_downstream_state = {ti.execution_date: ti.state for ti in downstream_instances} + exec_date_to_downstream_state = {ti.logical_date: ti.state for ti in downstream_instances} assert { timezone.datetime(2016, 1, 1): "success", timezone.datetime(2016, 1, 1, 12): "success", diff --git a/tests/operators/test_trigger_dagrun.py b/tests/operators/test_trigger_dagrun.py index 6c44357a7794..ec714110eb8f 100644 --- a/tests/operators/test_trigger_dagrun.py +++ b/tests/operators/test_trigger_dagrun.py @@ -193,7 +193,7 @@ def test_trigger_dagrun_twice(self, dag_maker): dag_maker.create_dagrun() dag_run = DagRun( dag_id=TRIGGERED_DAG_ID, - execution_date=utc_now, + logical_date=utc_now, state=State.SUCCESS, run_type="manual", run_id=run_id, @@ -229,7 +229,7 @@ def test_trigger_dagrun_with_scheduled_dag_run(self, dag_maker): run_id = f"scheduled__{utc_now.isoformat()}" dag_run = DagRun( dag_id=TRIGGERED_DAG_ID, - execution_date=utc_now, + logical_date=utc_now, state=State.SUCCESS, run_type="scheduled", run_id=run_id, @@ -374,7 +374,7 @@ def test_trigger_dagrun_with_reset_dag_run_false_fail( def test_trigger_dagrun_with_skip_when_already_exists(self, dag_maker): """Test TriggerDagRunOperator with skip_when_already_exists.""" - execution_date = DEFAULT_DATE + logical_date = DEFAULT_DATE with dag_maker( TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True ) as dag: @@ -387,9 +387,9 @@ def test_trigger_dagrun_with_skip_when_already_exists(self, dag_maker): ) self.re_sync_triggered_dag_to_db(dag, dag_maker) dr: DagRun = dag_maker.create_dagrun() - task.run(start_date=execution_date, end_date=execution_date, ignore_ti_state=True) + task.run(start_date=logical_date, end_date=logical_date, ignore_ti_state=True) assert dr.get_task_instance("test_task").state == TaskInstanceState.SUCCESS - task.run(start_date=execution_date, end_date=execution_date, ignore_ti_state=True) + task.run(start_date=logical_date, end_date=logical_date, ignore_ti_state=True) assert dr.get_task_instance("test_task").state == TaskInstanceState.SKIPPED @pytest.mark.skip_if_database_isolation_mode # Known to be broken in db isolation mode @@ -487,7 +487,7 @@ def test_trigger_dagrun_triggering_itself(self, dag_maker): dagruns = ( dag_maker.session.query(DagRun) .filter(DagRun.dag_id == TEST_DAG_ID) - .order_by(DagRun.execution_date) + .order_by(DagRun.logical_date) .all() ) assert len(dagruns) == 2 @@ -561,7 +561,7 @@ def test_trigger_dagrun_with_wait_for_completion_true_defer_true(self, dag_maker assert len(dagruns) == 1 trigger = DagStateTrigger( dag_id="down_stream", - execution_dates=[DEFAULT_DATE], + logical_dates=[DEFAULT_DATE], poll_interval=20, states=["success", "failed"], ) @@ -595,7 +595,7 @@ def test_trigger_dagrun_with_wait_for_completion_true_defer_true_failure(self, d trigger = DagStateTrigger( dag_id="down_stream", - execution_dates=[DEFAULT_DATE], + logical_dates=[DEFAULT_DATE], poll_interval=20, states=["success", "failed"], ) @@ -633,7 +633,7 @@ def test_trigger_dagrun_with_wait_for_completion_true_defer_true_failure_2(self, trigger = DagStateTrigger( dag_id="down_stream", - execution_dates=[DEFAULT_DATE], + logical_dates=[DEFAULT_DATE], poll_interval=20, states=["success", "failed"], ) @@ -649,7 +649,7 @@ def test_trigger_dagrun_with_wait_for_completion_true_defer_true_failure_2(self, pytest.param(None, id="logical_date=None"), ], ) - def test_dagstatetrigger_execution_dates(self, trigger_logical_date, dag_maker): + def test_dagstatetrigger_logical_dates(self, trigger_logical_date, dag_maker): """Ensure that the DagStateTrigger is called with the triggered DAG's logical date.""" with dag_maker( TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True @@ -674,12 +674,12 @@ def test_dagstatetrigger_execution_dates(self, trigger_logical_date, dag_maker): dagruns = session.query(DagRun).filter(DagRun.dag_id == TRIGGERED_DAG_ID).all() assert len(dagruns) == 1 - assert mock_task_defer.call_args_list[0].kwargs["trigger"].execution_dates == [ + assert mock_task_defer.call_args_list[0].kwargs["trigger"].logical_dates == [ pendulum.instance(dagruns[0].logical_date) ] @pytest.mark.skip_if_database_isolation_mode # Known to be broken in db isolation mode - def test_dagstatetrigger_execution_dates_with_clear_and_reset(self, dag_maker): + def test_dagstatetrigger_logical_dates_with_clear_and_reset(self, dag_maker): """Check DagStateTrigger is called with the triggered DAG's logical date on subsequent defers.""" with dag_maker( TEST_DAG_ID, default_args={"owner": "airflow", "start_date": DEFAULT_DATE}, serialized=True @@ -706,7 +706,7 @@ def test_dagstatetrigger_execution_dates_with_clear_and_reset(self, dag_maker): triggered_logical_date = dagruns[0].logical_date assert len(dagruns) == 1 - assert mock_task_defer.call_args_list[0].kwargs["trigger"].execution_dates == [ + assert mock_task_defer.call_args_list[0].kwargs["trigger"].logical_dates == [ pendulum.instance(triggered_logical_date) ] @@ -724,7 +724,7 @@ def test_dagstatetrigger_execution_dates_with_clear_and_reset(self, dag_maker): assert len(dagruns) == 1 # The second DagStateTrigger call should still use the original `logical_date` value. - assert mock_task_defer.call_args_list[1].kwargs["trigger"].execution_dates == [ + assert mock_task_defer.call_args_list[1].kwargs["trigger"].logical_dates == [ pendulum.instance(triggered_logical_date) ] diff --git a/tests/sensors/test_external_task_sensor.py b/tests/sensors/test_external_task_sensor.py index e2246a2f7513..4cd7b5e5f8c6 100644 --- a/tests/sensors/test_external_task_sensor.py +++ b/tests/sensors/test_external_task_sensor.py @@ -28,7 +28,7 @@ import pytest -from airflow import exceptions, settings +from airflow import settings from airflow.decorators import task as task_deco from airflow.exceptions import AirflowException, AirflowSensorTimeout, AirflowSkipException, TaskDeferred from airflow.models import DagBag, DagRun, TaskInstance @@ -46,7 +46,7 @@ from airflow.serialization.serialized_objects import SerializedBaseOperator from airflow.triggers.external_task import WorkflowTrigger from airflow.utils.hashlib_wrapper import md5 -from airflow.utils.session import create_session, provide_session +from airflow.utils.session import NEW_SESSION, create_session, provide_session from airflow.utils.state import DagRunState, State, TaskInstanceState from airflow.utils.task_group import TaskGroup from airflow.utils.timezone import datetime @@ -458,7 +458,7 @@ def test_external_dag_sensor(self): other_dag.create_dagrun( run_id="test", start_date=DEFAULT_DATE, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, state=State.SUCCESS, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, @@ -478,7 +478,7 @@ def test_external_dag_sensor_log(self, caplog): other_dag.create_dagrun( run_id="test", start_date=DEFAULT_DATE, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, state=State.SUCCESS, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, @@ -498,7 +498,7 @@ def test_external_dag_sensor_soft_fail_as_skipped(self): other_dag.create_dagrun( run_id="test", start_date=DEFAULT_DATE, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, state=State.SUCCESS, data_interval=(DEFAULT_DATE, DEFAULT_DATE), **triggered_by_kwargs, @@ -524,7 +524,7 @@ def test_external_dag_sensor_soft_fail_as_skipped(self): assert task_instances[0].state == State.SKIPPED, "Unexpected external task state" @pytest.mark.skip_if_database_isolation_mode # Test is broken in db isolation mode - def test_external_task_sensor_fn_multiple_execution_dates(self): + def test_external_task_sensor_fn_multiple_logical_dates(self): bash_command_code = """ {% set s=logical_date.time().second %} echo "second is {{ s }}" @@ -562,7 +562,7 @@ def test_external_task_sensor_fn_multiple_execution_dates(self): .filter( TI.dag_id == dag_external_id, TI.state == State.FAILED, - TI.execution_date == DEFAULT_DATE + timedelta(seconds=1), + TI.logical_date == DEFAULT_DATE + timedelta(seconds=1), ) .all() ) @@ -661,7 +661,7 @@ def test_external_task_sensor_fn(self): poke_interval=1, dag=self.dag, ) - with pytest.raises(exceptions.AirflowSensorTimeout): + with pytest.raises(AirflowSensorTimeout): op2.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) @pytest.mark.skip_if_database_isolation_mode # Test is broken in db isolation mode @@ -688,9 +688,8 @@ def test_external_task_sensor_fn_kwargs(self): """Check this task sensor passes multiple args with full context. If no failure, means clean run.""" self.add_time_sensor() - def my_func(dt, ds_nodash, tomorrow_ds_nodash): + def my_func(dt, ds_nodash): assert ds_nodash == dt.strftime("%Y%m%d") - assert tomorrow_ds_nodash == (dt + timedelta(days=1)).strftime("%Y%m%d") return dt + timedelta(0) op1 = ExternalTaskSensor( @@ -1127,7 +1126,7 @@ def test_external_task_sensor_extra_link( ti = create_task_instance_of_operator( ExternalTaskSensor, dag_id="external_task_sensor_extra_links_dag", - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, task_id="external_task_sensor_extra_links_task", external_dag_id=external_dag_id, external_task_id=external_task_id, @@ -1254,7 +1253,7 @@ def dag_bag_parent_child(): task_id="task_0", external_dag_id="child_dag_1", external_task_id="task_1", - execution_date=day_1.isoformat(), + logical_date=day_1.isoformat(), recursion_depth=3, ) @@ -1276,23 +1275,28 @@ def dag_bag_parent_child(): @provide_session -def run_tasks(dag_bag, execution_date=DEFAULT_DATE, session=None): +def run_tasks( + dag_bag: DagBag, + logical_date=DEFAULT_DATE, + session=NEW_SESSION, +) -> tuple[dict[str, DagRun], dict[str, TaskInstance]]: """ Run all tasks in the DAGs in the given dag_bag. Return the TaskInstance objects as a dict keyed by task_id. """ - tis = {} - triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} + runs: dict[str, DagRun] = {} + tis: dict[str, TaskInstance] = {} + triggered_by = DagRunTriggeredByType.TEST if AIRFLOW_V_3_0_PLUS else None for dag in dag_bag.dags.values(): dagrun = dag.create_dagrun( - state=State.RUNNING, - execution_date=execution_date, - start_date=execution_date, + state=DagRunState.RUNNING, + logical_date=logical_date, + start_date=logical_date, run_type=DagRunType.MANUAL, session=session, data_interval=(DEFAULT_DATE, DEFAULT_DATE), - **triggered_by_kwargs, + triggered_by=triggered_by, ) # we use sorting by task_id here because for the test DAG structure of ours # this is equivalent to topological sort. It would not work in general case @@ -1307,7 +1311,7 @@ def run_tasks(dag_bag, execution_date=DEFAULT_DATE, session=None): session.merge(ti) assert_ti_state_equal(ti, State.SUCCESS) - return tis + return runs, tis def assert_ti_state_equal(task_instance, state): @@ -1346,7 +1350,7 @@ def test_external_task_marker_transitive(dag_bag_ext): """ Test clearing tasks across DAGs. """ - tis = run_tasks(dag_bag_ext) + _, tis = run_tasks(dag_bag_ext) dag_0 = dag_bag_ext.get_dag("dag_0") task_a_0 = dag_0.get_task("task_a_0") clear_tasks(dag_bag_ext, dag_0, task_a_0) @@ -1366,12 +1370,12 @@ def test_external_task_marker_clear_activate(dag_bag_parent_child, session): day_1 = DEFAULT_DATE day_2 = DEFAULT_DATE + timedelta(days=1) - run_tasks(dag_bag, execution_date=day_1) - run_tasks(dag_bag, execution_date=day_2) + run_tasks(dag_bag, logical_date=day_1) + run_tasks(dag_bag, logical_date=day_2) # Assert that dagruns of all the affected dags are set to SUCCESS before tasks are cleared. - for dag, execution_date in itertools.product(dag_bag.dags.values(), [day_1, day_2]): - dagrun = dag.get_dagrun(execution_date=execution_date, session=session) + for dag, logical_date in itertools.product(dag_bag.dags.values(), [day_1, day_2]): + dagrun = dag.get_dagrun(logical_date=logical_date, session=session) dagrun.set_state(State.SUCCESS) session.flush() @@ -1381,10 +1385,10 @@ def test_external_task_marker_clear_activate(dag_bag_parent_child, session): # Assert that dagruns of all the affected dags are set to QUEUED after tasks are cleared. # Unaffected dagruns should be left as SUCCESS. - dagrun_0_1 = dag_bag.get_dag("parent_dag_0").get_dagrun(execution_date=day_1, session=session) - dagrun_0_2 = dag_bag.get_dag("parent_dag_0").get_dagrun(execution_date=day_2, session=session) - dagrun_1_1 = dag_bag.get_dag("child_dag_1").get_dagrun(execution_date=day_1, session=session) - dagrun_1_2 = dag_bag.get_dag("child_dag_1").get_dagrun(execution_date=day_2, session=session) + dagrun_0_1 = dag_bag.get_dag("parent_dag_0").get_dagrun(logical_date=day_1, session=session) + dagrun_0_2 = dag_bag.get_dag("parent_dag_0").get_dagrun(logical_date=day_2, session=session) + dagrun_1_1 = dag_bag.get_dag("child_dag_1").get_dagrun(logical_date=day_1, session=session) + dagrun_1_2 = dag_bag.get_dag("child_dag_1").get_dagrun(logical_date=day_2, session=session) assert dagrun_0_1.state == State.QUEUED assert dagrun_0_2.state == State.QUEUED @@ -1401,8 +1405,8 @@ def test_external_task_marker_future(dag_bag_ext): date_0 = DEFAULT_DATE date_1 = DEFAULT_DATE + timedelta(days=1) - tis_date_0 = run_tasks(dag_bag_ext, execution_date=date_0) - tis_date_1 = run_tasks(dag_bag_ext, execution_date=date_1) + _, tis_date_0 = run_tasks(dag_bag_ext, logical_date=date_0) + _, tis_date_1 = run_tasks(dag_bag_ext, logical_date=date_1) dag_0 = dag_bag_ext.get_dag("dag_0") task_a_0 = dag_0.get_task("task_a_0") @@ -1558,7 +1562,7 @@ def dag_bag_multiple(): task_id=f"{daily_task.task_id}_{i}", external_dag_id=daily_dag.dag_id, external_task_id=daily_task.task_id, - execution_date=f"{{{{ macros.ds_add(ds, -1 * {i}) }}}}", + logical_date=f"{{{{ macros.ds_add(ds, -1 * {i}) }}}}", dag=agg_dag, ) begin >> task @@ -1572,11 +1576,11 @@ def test_clear_multiple_external_task_marker(dag_bag_multiple): Test clearing a dag that has multiple ExternalTaskMarker. """ agg_dag = dag_bag_multiple.get_dag("agg_dag") - tis = run_tasks(dag_bag_multiple, execution_date=DEFAULT_DATE) + _, tis = run_tasks(dag_bag_multiple, logical_date=DEFAULT_DATE) session = settings.Session() try: qry = session.query(TaskInstance).filter( - TaskInstance.state == State.NONE, TaskInstance.dag_id.in_(dag_bag_multiple.dag_ids) + TaskInstance.state.is_(None), TaskInstance.dag_id.in_(dag_bag_multiple.dag_ids) ) assert agg_dag.clear(dag_bag=dag_bag_multiple) == len(tis) == qry.count() == 10 finally: @@ -1587,7 +1591,7 @@ def test_clear_multiple_external_task_marker(dag_bag_multiple): def dag_bag_head_tail(): """ Create a DagBag containing one DAG, with task "head" depending on task "tail" of the - previous execution_date. + previous logical_date. 20200501 20200502 20200510 +------+ +------+ +------+ @@ -1615,7 +1619,7 @@ def dag_bag_head_tail(): task_id="tail", external_dag_id=dag.dag_id, external_task_id=head.task_id, - execution_date="{{ macros.ds_add(ds, 1) }}", + logical_date="{{ macros.ds_add(ds, 1) }}", ) head >> body >> tail @@ -1631,11 +1635,11 @@ def test_clear_overlapping_external_task_marker(dag_bag_head_tail, session): # "Run" 10 times. for delta in range(10): - execution_date = DEFAULT_DATE + timedelta(days=delta) + logical_date = DEFAULT_DATE + timedelta(days=delta) dagrun = DagRun( dag_id=dag.dag_id, state=DagRunState.SUCCESS, - execution_date=execution_date, + logical_date=logical_date, run_type=DagRunType.MANUAL, run_id=f"test_{delta}", ) @@ -1653,7 +1657,7 @@ def test_clear_overlapping_external_task_marker(dag_bag_head_tail, session): assert ( dag.clear( start_date=DEFAULT_DATE, - end_date=execution_date, + end_date=logical_date, dag_bag=dag_bag_head_tail, session=session, ) @@ -1665,7 +1669,7 @@ def test_clear_overlapping_external_task_marker(dag_bag_head_tail, session): def dag_bag_head_tail_mapped_tasks(): """ Create a DagBag containing one DAG, with task "head" depending on task "tail" of the - previous execution_date. + previous logical_date. 20200501 20200502 20200510 +------+ +------+ +------+ @@ -1699,7 +1703,7 @@ def dummy_task(x: int): task_id="tail", external_dag_id=dag.dag_id, external_task_id=head.task_id, - execution_date="{{ macros.ds_add(ds, 1) }}", + logical_date="{{ macros.ds_add(ds, 1) }}", ) head >> body >> tail @@ -1715,11 +1719,11 @@ def test_clear_overlapping_external_task_marker_mapped_tasks(dag_bag_head_tail_m # "Run" 10 times. for delta in range(10): - execution_date = DEFAULT_DATE + timedelta(days=delta) + logical_date = DEFAULT_DATE + timedelta(days=delta) dagrun = DagRun( dag_id=dag.dag_id, state=DagRunState.SUCCESS, - execution_date=execution_date, + logical_date=logical_date, run_type=DagRunType.MANUAL, run_id=f"test_{delta}", ) diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index 0111cc669cd0..d10984556ff2 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -1039,7 +1039,7 @@ def test_extra_serialized_field_and_operator_links( # Test all the extra_links are set assert simple_task.extra_links == sorted({*links, "airflow", "github", "google"}) - dr = dag_maker.create_dagrun(execution_date=test_date) + dr = dag_maker.create_dagrun(logical_date=test_date) (ti,) = dr.task_instances XCom.set( key="search_query", @@ -1330,9 +1330,9 @@ def test_task_resources(self): """ from airflow.operators.empty import EmptyOperator - execution_date = datetime(2020, 1, 1) + logical_date = datetime(2020, 1, 1) task_id = "task1" - with DAG("test_task_resources", schedule=None, start_date=execution_date) as dag: + with DAG("test_task_resources", schedule=None, start_date=logical_date) as dag: task = EmptyOperator(task_id=task_id, resources={"cpus": 0.1, "ram": 2048}) SerializedDAG.validate_schema(SerializedDAG.to_dict(dag)) @@ -1347,8 +1347,8 @@ def test_task_group_serialization(self): Test TaskGroup serialization/deserialization. """ - execution_date = datetime(2020, 1, 1) - with DAG("test_task_group_serialization", schedule=None, start_date=execution_date) as dag: + logical_date = datetime(2020, 1, 1) + with DAG("test_task_group_serialization", schedule=None, start_date=logical_date) as dag: task1 = EmptyOperator(task_id="task1") with TaskGroup("group234") as group234: _ = EmptyOperator(task_id="task2") @@ -1403,8 +1403,8 @@ def test_setup_teardown_tasks(self): Test setup and teardown task serialization/deserialization. """ - execution_date = datetime(2020, 1, 1) - with DAG("test_task_group_setup_teardown_tasks", schedule=None, start_date=execution_date) as dag: + logical_date = datetime(2020, 1, 1) + with DAG("test_task_group_setup_teardown_tasks", schedule=None, start_date=logical_date) as dag: EmptyOperator(task_id="setup").as_setup() EmptyOperator(task_id="teardown").as_teardown() @@ -1507,8 +1507,8 @@ def test_deps_sorted(self): from airflow.operators.empty import EmptyOperator from airflow.sensors.external_task import ExternalTaskSensor - execution_date = datetime(2020, 1, 1) - with DAG(dag_id="test_deps_sorted", schedule=None, start_date=execution_date) as dag: + logical_date = datetime(2020, 1, 1) + with DAG(dag_id="test_deps_sorted", schedule=None, start_date=logical_date) as dag: task1 = ExternalTaskSensor( task_id="task1", external_dag_id="external_dag_id", @@ -1537,11 +1537,11 @@ class DummyTriggerRule(BaseTIDep): class DummyTask(BaseOperator): deps = frozenset([*BaseOperator.deps, DummyTriggerRule()]) - execution_date = datetime(2020, 1, 1) + logical_date = datetime(2020, 1, 1) with DAG( dag_id="test_error_on_unregistered_ti_dep_serialization", schedule=None, - start_date=execution_date, + start_date=logical_date, ) as dag: DummyTask(task_id="task1") @@ -1573,8 +1573,8 @@ def test_serialize_and_deserialize_custom_ti_deps(self): class DummyTask(BaseOperator): deps = frozenset([*BaseOperator.deps, CustomTestTriggerRule()]) - execution_date = datetime(2020, 1, 1) - with DAG(dag_id="test_serialize_custom_ti_deps", schedule=None, start_date=execution_date) as dag: + logical_date = datetime(2020, 1, 1) + with DAG(dag_id="test_serialize_custom_ti_deps", schedule=None, start_date=logical_date) as dag: DummyTask(task_id="task1") serialize_op = SerializedBaseOperator.serialize_operator(dag.task_dict["task1"]) @@ -1625,9 +1625,9 @@ def test_derived_dag_deps_sensor(self): class DerivedSensor(ExternalTaskSensor): pass - execution_date = datetime(2020, 1, 1) + logical_date = datetime(2020, 1, 1) for class_ in [ExternalTaskSensor, DerivedSensor]: - with DAG(dag_id="test_derived_dag_deps_sensor", schedule=None, start_date=execution_date) as dag: + with DAG(dag_id="test_derived_dag_deps_sensor", schedule=None, start_date=logical_date) as dag: task1 = class_( task_id="task1", external_dag_id="external_dag_id", @@ -1657,8 +1657,8 @@ def test_dag_deps_assets_with_duplicate_asset(self): d2 = Asset("d2") d3 = Asset("d3") d4 = Asset("d4") - execution_date = datetime(2020, 1, 1) - with DAG(dag_id="test", start_date=execution_date, schedule=[d1, d1, d1, d1, d1]) as dag: + logical_date = datetime(2020, 1, 1) + with DAG(dag_id="test", start_date=logical_date, schedule=[d1, d1, d1, d1, d1]) as dag: ExternalTaskSensor( task_id="task1", external_dag_id="external_dag_id", @@ -1746,8 +1746,8 @@ def test_dag_deps_assets(self): d2 = Asset("d2") d3 = Asset("d3") d4 = Asset("d4") - execution_date = datetime(2020, 1, 1) - with DAG(dag_id="test", start_date=execution_date, schedule=[d1]) as dag: + logical_date = datetime(2020, 1, 1) + with DAG(dag_id="test", start_date=logical_date, schedule=[d1]) as dag: ExternalTaskSensor( task_id="task1", external_dag_id="external_dag_id", @@ -1810,9 +1810,9 @@ def test_derived_dag_deps_operator(self): class DerivedOperator(TriggerDagRunOperator): pass - execution_date = datetime(2020, 1, 1) + logical_date = datetime(2020, 1, 1) for class_ in [TriggerDagRunOperator, DerivedOperator]: - with DAG(dag_id="test_derived_dag_deps_trigger", schedule=None, start_date=execution_date) as dag: + with DAG(dag_id="test_derived_dag_deps_trigger", schedule=None, start_date=logical_date) as dag: task1 = EmptyOperator(task_id="task1") task2 = class_( task_id="task2", @@ -1853,8 +1853,8 @@ def test_task_group_sorted(self): ╲ ╱ end """ - execution_date = datetime(2020, 1, 1) - with DAG(dag_id="test_task_group_sorted", schedule=None, start_date=execution_date) as dag: + logical_date = datetime(2020, 1, 1) + with DAG(dag_id="test_task_group_sorted", schedule=None, start_date=logical_date) as dag: start = EmptyOperator(task_id="start") with TaskGroup("task_group_up1") as task_group_up1: @@ -2557,9 +2557,9 @@ def test_task_resources_serde(): """ from airflow.operators.empty import EmptyOperator - execution_date = datetime(2020, 1, 1) + logical_date = datetime(2020, 1, 1) task_id = "task1" - with DAG("test_task_resources", schedule=None, start_date=execution_date) as _: + with DAG("test_task_resources", schedule=None, start_date=logical_date) as _: task = EmptyOperator(task_id=task_id, resources={"cpus": 0.1, "ram": 2048}) serialized = BaseSerialization.serialize(task) diff --git a/tests/serialization/test_pydantic_models.py b/tests/serialization/test_pydantic_models.py index 7bfc7eda32af..b8dcada14430 100644 --- a/tests/serialization/test_pydantic_models.py +++ b/tests/serialization/test_pydantic_models.py @@ -73,7 +73,7 @@ def test_serializing_pydantic_task_instance(session, create_task_instance): assert deserialized_model.dag_id == dag_id assert deserialized_model.state == State.RUNNING assert deserialized_model.try_number == ti.try_number - assert deserialized_model.execution_date == ti.execution_date + assert deserialized_model.logical_date == ti.logical_date assert deserialized_model.next_kwargs == {"foo": "bar"} @@ -238,17 +238,17 @@ def test_serializing_pydantic_asset_event(session, create_task_instance, create_ with_dagrun_type=DagRunType.MANUAL, session=session, ) - execution_date = timezone.utcnow() + logical_date = timezone.utcnow() TracebackSessionForTests.set_allow_db_access(session, True) triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dr = dag.create_dagrun( run_id="test2", run_type=DagRunType.ASSET_TRIGGERED, - execution_date=execution_date, + logical_date=logical_date, state=None, session=session, - data_interval=(execution_date, execution_date), + data_interval=(logical_date, logical_date), **triggered_by_kwargs, ) asset1_event = AssetEvent(asset_id=1) diff --git a/tests/serialization/test_serialized_objects.py b/tests/serialization/test_serialized_objects.py index 81ba206af467..96f7414b7765 100644 --- a/tests/serialization/test_serialized_objects.py +++ b/tests/serialization/test_serialized_objects.py @@ -143,7 +143,7 @@ class Test: dag_id="test_dag_id", run_id="test_dag_run_id", run_type=DagRunType.MANUAL, - execution_date=timezone.utcnow(), + logical_date=timezone.utcnow(), start_date=timezone.utcnow(), external_trigger=True, state=DagRunState.SUCCESS, @@ -352,7 +352,7 @@ def test_backcompat_deserialize_connection(conn_uri): sample_objects.get(DagRunPydantic), DagRunPydantic, DAT.DAG_RUN, - lambda a, b: equal_time(a.execution_date, b.execution_date) + lambda a, b: equal_time(a.logical_date, b.logical_date) and equal_time(a.start_date, b.start_date), ), # Asset is already serialized by non-Pydantic serialization. Is AssetPydantic needed then? diff --git a/tests/ti_deps/deps/test_not_previously_skipped_dep.py b/tests/ti_deps/deps/test_not_previously_skipped_dep.py index 3d7adca3b97d..5d724992fd4a 100644 --- a/tests/ti_deps/deps/test_not_previously_skipped_dep.py +++ b/tests/ti_deps/deps/test_not_previously_skipped_dep.py @@ -51,7 +51,7 @@ def test_no_parent(session, dag_maker): ): op1 = EmptyOperator(task_id="op1") - (ti1,) = dag_maker.create_dagrun(execution_date=start_date).task_instances + (ti1,) = dag_maker.create_dagrun(logical_date=start_date).task_instances ti1.refresh_from_task(op1) dep = NotPreviouslySkippedDep() diff --git a/tests/ti_deps/deps/test_prev_dagrun_dep.py b/tests/ti_deps/deps/test_prev_dagrun_dep.py index 369ddaebbe40..e2aa2732e043 100644 --- a/tests/ti_deps/deps/test_prev_dagrun_dep.py +++ b/tests/ti_deps/deps/test_prev_dagrun_dep.py @@ -63,7 +63,7 @@ def test_first_task_run_of_new_task(self): dag.create_dagrun( run_id="old_run", state=TaskInstanceState.SUCCESS, - execution_date=old_task.start_date, + logical_date=old_task.start_date, run_type=DagRunType.SCHEDULED, data_interval=(old_task.start_date, old_task.start_date), **triggered_by_kwargs, @@ -78,13 +78,13 @@ def test_first_task_run_of_new_task(self): ) # New DAG run will include 1st TaskInstance of new_task - execution_date = convert_to_utc(datetime(2016, 1, 2)) + logical_date = convert_to_utc(datetime(2016, 1, 2)) dr = dag.create_dagrun( run_id="new_run", state=DagRunState.RUNNING, - execution_date=execution_date, + logical_date=logical_date, run_type=DagRunType.SCHEDULED, - data_interval=(execution_date, execution_date), + data_interval=(logical_date, logical_date), **triggered_by_kwargs, ) @@ -275,7 +275,7 @@ def test_dagrun_dep(mock_get_previous_scheduled_dagrun, kwargs): wait_for_downstream=wait_for_downstream, ) if prev_tis: - prev_dagrun = Mock(execution_date=datetime(2016, 1, 2)) + prev_dagrun = Mock(logical_date=datetime(2016, 1, 2)) else: prev_dagrun = None mock_get_previous_scheduled_dagrun.return_value = prev_dagrun diff --git a/tests/ti_deps/deps/test_ready_to_reschedule_dep.py b/tests/ti_deps/deps/test_ready_to_reschedule_dep.py index 885f045ee37b..802d3424cb6e 100644 --- a/tests/ti_deps/deps/test_ready_to_reschedule_dep.py +++ b/tests/ti_deps/deps/test_ready_to_reschedule_dep.py @@ -71,7 +71,7 @@ def _get_task_instance(self, state, *, map_index=-1): dag_id=self.dag_id, task_id=self.task_id, run_id=self.run_id, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, map_index=map_index, state=state, ) @@ -84,7 +84,7 @@ def _create_task_reschedule(self, ti, minutes: int | list[int]): minutes = [minutes] trs = [] for minutes_timedelta in minutes: - dt = ti.execution_date + timedelta(minutes=minutes_timedelta) + dt = ti.logical_date + timedelta(minutes=minutes_timedelta) trs.append( TaskReschedule( task_id=ti.task_id, diff --git a/tests/ti_deps/deps/test_runnable_exec_date_dep.py b/tests/ti_deps/deps/test_runnable_exec_date_dep.py index 43063001b667..651610605cec 100644 --- a/tests/ti_deps/deps/test_runnable_exec_date_dep.py +++ b/tests/ti_deps/deps/test_runnable_exec_date_dep.py @@ -40,9 +40,8 @@ def clean_db(session): @time_machine.travel("2016-11-01") @pytest.mark.parametrize( - "allow_trigger_in_future,schedule,execution_date,is_met", + "allow_trigger_in_future,schedule,logical_date,is_met", [ - (True, None, datetime(2016, 11, 3), True), (True, "@daily", datetime(2016, 11, 3), False), (False, None, datetime(2016, 11, 3), False), (False, "@daily", datetime(2016, 11, 3), False), @@ -50,20 +49,20 @@ def clean_db(session): (False, None, datetime(2016, 11, 1), True), ], ) -def test_exec_date_dep( +def test_logical_date_dep( dag_maker, session, create_dummy_dag, allow_trigger_in_future, schedule, - execution_date, + logical_date, is_met, ): """ - If the dag's execution date is in the future but (allow_trigger_in_future=False or not schedule) + If the dag's logical date is in the future but (allow_trigger_in_future=False or not schedule) this dep should fail """ - with patch.object(settings, "ALLOW_FUTURE_EXEC_DATES", allow_trigger_in_future): + with patch.object(settings, "ALLOW_FUTURE_LOGICAL_DATES", allow_trigger_in_future): create_dummy_dag( "test_localtaskjob_heartbeat", start_date=datetime(2015, 1, 1), @@ -72,14 +71,14 @@ def test_exec_date_dep( with_dagrun_type=DagRunType.MANUAL, session=session, ) - (ti,) = dag_maker.create_dagrun(execution_date=execution_date).task_instances + (ti,) = dag_maker.create_dagrun(run_id="scheduled", logical_date=logical_date).task_instances assert RunnableExecDateDep().is_met(ti=ti) == is_met @time_machine.travel("2016-01-01") -def test_exec_date_after_end_date(session, dag_maker, create_dummy_dag): +def test_logical_date_after_end_date(session, dag_maker, create_dummy_dag): """ - If the dag's execution date is in the future this dep should fail + If the dag's logical date is in the future this dep should fail """ create_dummy_dag( "test_localtaskjob_heartbeat", @@ -89,38 +88,38 @@ def test_exec_date_after_end_date(session, dag_maker, create_dummy_dag): with_dagrun_type=DagRunType.MANUAL, session=session, ) - (ti,) = dag_maker.create_dagrun(execution_date=datetime(2016, 11, 2)).task_instances + (ti,) = dag_maker.create_dagrun(logical_date=datetime(2016, 11, 2)).task_instances assert not RunnableExecDateDep().is_met(ti=ti) class TestRunnableExecDateDep: - def _get_task_instance(self, execution_date, dag_end_date=None, task_end_date=None): + def _get_task_instance(self, logical_date, dag_end_date=None, task_end_date=None): dag = Mock(end_date=dag_end_date) - dagrun = Mock(execution_date=execution_date) + dagrun = Mock(logical_date=logical_date) task = Mock(dag=dag, end_date=task_end_date) return Mock(task=task, get_dagrun=Mock(return_value=dagrun)) - def test_exec_date_after_task_end_date(self): + def test_logical_date_after_task_end_date(self): """ - If the task instance execution date is after the tasks end date + If the task instance logical date is after the tasks end date this dep should fail """ ti = self._get_task_instance( dag_end_date=datetime(2016, 1, 3), task_end_date=datetime(2016, 1, 1), - execution_date=datetime(2016, 1, 2), + logical_date=datetime(2016, 1, 2), ) assert not RunnableExecDateDep().is_met(ti=ti) def test_exec_date_after_dag_end_date(self): """ - If the task instance execution date is after the dag's end date + If the task instance logical date is after the dag's end date this dep should fail """ ti = self._get_task_instance( dag_end_date=datetime(2016, 1, 1), task_end_date=datetime(2016, 1, 3), - execution_date=datetime(2016, 1, 2), + logical_date=datetime(2016, 1, 2), ) assert not RunnableExecDateDep().is_met(ti=ti) @@ -131,6 +130,6 @@ def test_all_deps_met(self): ti = self._get_task_instance( dag_end_date=datetime(2016, 1, 2), task_end_date=datetime(2016, 1, 2), - execution_date=datetime(2016, 1, 1), + logical_date=datetime(2016, 1, 1), ) assert RunnableExecDateDep().is_met(ti=ti) diff --git a/tests/ti_deps/deps/test_task_concurrency.py b/tests/ti_deps/deps/test_task_concurrency.py index eb5e5a36fa5f..23e6c69699d0 100644 --- a/tests/ti_deps/deps/test_task_concurrency.py +++ b/tests/ti_deps/deps/test_task_concurrency.py @@ -52,7 +52,7 @@ def _get_task(self, **kwargs): def test_concurrency(self, kwargs, num_running_tis, is_task_concurrency_dep_met): task = self._get_task(start_date=datetime(2016, 1, 1), **kwargs) dep_context = DepContext() - ti = Mock(task=task, execution_date=datetime(2016, 1, 1)) + ti = Mock(task=task, logical_date=datetime(2016, 1, 1)) if num_running_tis is not None: ti.get_num_running_task_instances.return_value = num_running_tis assert TaskConcurrencyDep().is_met(ti=ti, dep_context=dep_context) == is_task_concurrency_dep_met diff --git a/tests/triggers/test_external_task.py b/tests/triggers/test_external_task.py index 202eafb0b9e3..4a193c5fef9c 100644 --- a/tests/triggers/test_external_task.py +++ b/tests/triggers/test_external_task.py @@ -44,7 +44,7 @@ async def test_task_workflow_trigger_success(self, mock_get_count): mock_get_count.side_effect = mocked_get_count trigger = WorkflowTrigger( external_dag_id=self.DAG_ID, - execution_dates=[timezone.datetime(2022, 1, 1)], + logical_dates=[timezone.datetime(2022, 1, 1)], external_task_ids=[self.TASK_ID], allowed_states=self.STATES, poke_interval=0.2, @@ -76,7 +76,7 @@ async def test_task_workflow_trigger_failed(self, mock_get_count): mock_get_count.side_effect = mocked_get_count trigger = WorkflowTrigger( external_dag_id=self.DAG_ID, - execution_dates=[timezone.datetime(2022, 1, 1)], + logical_dates=[timezone.datetime(2022, 1, 1)], external_task_ids=[self.TASK_ID], failed_states=self.STATES, poke_interval=0.2, @@ -108,7 +108,7 @@ async def test_task_workflow_trigger_fail_count_eq_0(self, mock_get_count): mock_get_count.return_value = 0 trigger = WorkflowTrigger( external_dag_id=self.DAG_ID, - execution_dates=[timezone.datetime(2022, 1, 1)], + logical_dates=[timezone.datetime(2022, 1, 1)], external_task_ids=[self.TASK_ID], failed_states=self.STATES, poke_interval=0.2, @@ -139,7 +139,7 @@ async def test_task_workflow_trigger_skipped(self, mock_get_count): mock_get_count.side_effect = mocked_get_count trigger = WorkflowTrigger( external_dag_id=self.DAG_ID, - execution_dates=[timezone.datetime(2022, 1, 1)], + logical_dates=[timezone.datetime(2022, 1, 1)], external_task_ids=[self.TASK_ID], skipped_states=self.STATES, poke_interval=0.2, @@ -169,7 +169,7 @@ async def test_task_workflow_trigger_sleep_success(self, mock_sleep, mock_get_co mock_get_count.side_effect = [0, 1] trigger = WorkflowTrigger( external_dag_id=self.DAG_ID, - execution_dates=[timezone.datetime(2022, 1, 1)], + logical_dates=[timezone.datetime(2022, 1, 1)], external_task_ids=[self.TASK_ID], poke_interval=0.2, ) @@ -197,7 +197,7 @@ def test_serialization(self): """ trigger = WorkflowTrigger( external_dag_id=self.DAG_ID, - execution_dates=[timezone.datetime(2022, 1, 1)], + logical_dates=[timezone.datetime(2022, 1, 1)], external_task_ids=[self.TASK_ID], allowed_states=self.STATES, poke_interval=5, @@ -206,7 +206,7 @@ def test_serialization(self): assert classpath == "airflow.triggers.external_task.WorkflowTrigger" assert kwargs == { "external_dag_id": self.DAG_ID, - "execution_dates": [timezone.datetime(2022, 1, 1)], + "logical_dates": [timezone.datetime(2022, 1, 1)], "external_task_ids": [self.TASK_ID], "external_task_group_id": None, "failed_states": None, @@ -234,7 +234,7 @@ async def test_dag_state_trigger(self, session): dag_run = DagRun( dag_id=dag.dag_id, run_type="manual", - execution_date=timezone.datetime(2022, 1, 1), + logical_date=timezone.datetime(2022, 1, 1), run_id=self.RUN_ID, ) session.add(dag_run) @@ -243,7 +243,7 @@ async def test_dag_state_trigger(self, session): trigger = DagStateTrigger( dag_id=dag.dag_id, states=self.STATES, - execution_dates=[timezone.datetime(2022, 1, 1)], + logical_dates=[timezone.datetime(2022, 1, 1)], poll_interval=0.2, ) @@ -267,7 +267,7 @@ def test_serialization(self): trigger = DagStateTrigger( dag_id=self.DAG_ID, states=self.STATES, - execution_dates=[timezone.datetime(2022, 1, 1)], + logical_dates=[timezone.datetime(2022, 1, 1)], poll_interval=5, ) classpath, kwargs = trigger.serialize() @@ -275,7 +275,7 @@ def test_serialization(self): assert kwargs == { "dag_id": self.DAG_ID, "states": self.STATES, - "execution_dates": [timezone.datetime(2022, 1, 1)], + "logical_dates": [timezone.datetime(2022, 1, 1)], "poll_interval": 5, } diff --git a/tests/utils/log/test_log_reader.py b/tests/utils/log/test_log_reader.py index fd8731be5d96..15aea4e863f2 100644 --- a/tests/utils/log/test_log_reader.py +++ b/tests/utils/log/test_log_reader.py @@ -109,7 +109,7 @@ def prepare_db(self, create_task_instance): task_id=self.TASK_ID, start_date=self.DEFAULT_DATE, run_type=DagRunType.SCHEDULED, - execution_date=self.DEFAULT_DATE, + logical_date=self.DEFAULT_DATE, state=TaskInstanceState.RUNNING, ) ti.try_number = 3 @@ -287,15 +287,15 @@ def echo_run_type(dag_run: DagRun, **kwargs): trigger_time = end + datetime.timedelta(hours=4, minutes=29) # Arbitrary. # Create two DAG runs that have the same data interval, but not the same - # execution date, to check if they correctly use different log files. + # logical date, to check if they correctly use different log files. scheduled_dagrun: DagRun = dag_maker.create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=start, + logical_date=start, data_interval=DataInterval(start, end), ) manual_dagrun: DagRun = dag_maker.create_dagrun( run_type=DagRunType.MANUAL, - execution_date=trigger_time, + logical_date=trigger_time, data_interval=DataInterval(start, end), ) diff --git a/tests/utils/test_cli_util.py b/tests/utils/test_cli_util.py index e60146b9558f..097119fbb569 100644 --- a/tests/utils/test_cli_util.py +++ b/tests/utils/test_cli_util.py @@ -45,8 +45,7 @@ class TestCliUtil: def test_metrics_build(self): func_name = "test" - exec_date = timezone.utcnow() - namespace = Namespace(dag_id="foo", task_id="bar", subcommand="test", execution_date=exec_date) + namespace = Namespace(dag_id="foo", task_id="bar", subcommand="test") metrics = cli._build_metrics(func_name, namespace) expected = { @@ -54,7 +53,6 @@ def test_metrics_build(self): "sub_command": "test", "dag_id": "foo", "task_id": "bar", - "execution_date": exec_date, } for k, v in expected.items(): assert v == metrics.get(k) @@ -132,7 +130,7 @@ def test_cli_create_user_supplied_password_is_masked( expected_command = expected_masked_command.split() exec_date = timezone.utcnow() - namespace = Namespace(dag_id="foo", task_id="bar", subcommand="test", execution_date=exec_date) + namespace = Namespace(dag_id="foo", task_id="bar", subcommand="test", logical_date=exec_date) with mock.patch.object(sys, "argv", args), mock.patch( "airflow.utils.session.create_session" ) as mock_create_session: diff --git a/tests/utils/test_dot_renderer.py b/tests/utils/test_dot_renderer.py index 0bf6f75fa6d0..65441ae376ca 100644 --- a/tests/utils/test_dot_renderer.py +++ b/tests/utils/test_dot_renderer.py @@ -98,7 +98,7 @@ def test_should_render_dag_with_task_instances(self, session, dag_maker): task_1 >> task_2 task_1 >> task_3 - tis = {ti.task_id: ti for ti in dag_maker.create_dagrun(execution_date=START_DATE).task_instances} + tis = {ti.task_id: ti for ti in dag_maker.create_dagrun(logical_date=START_DATE).task_instances} tis["first"].state = State.SCHEDULED tis["second"].state = State.SUCCESS tis["third"].state = State.RUNNING @@ -142,7 +142,7 @@ def test_should_render_dag_orientation(self, session, dag_maker): task_1 >> task_2 task_1 >> task_3 - tis = dag_maker.create_dagrun(execution_date=START_DATE).task_instances + tis = dag_maker.create_dagrun(logical_date=START_DATE).task_instances tis[0].state = State.SCHEDULED tis[1].state = State.SUCCESS tis[2].state = State.RUNNING diff --git a/tests/utils/test_helpers.py b/tests/utils/test_helpers.py index ed1b1134d842..2c4f8c9e91dc 100644 --- a/tests/utils/test_helpers.py +++ b/tests/utils/test_helpers.py @@ -63,9 +63,9 @@ def test_render_log_filename(self, create_task_instance): try_number = 1 dag_id = "test_render_log_filename_dag" task_id = "test_render_log_filename_task" - execution_date = timezone.datetime(2016, 1, 1) + logical_date = timezone.datetime(2016, 1, 1) - ti = create_task_instance(dag_id=dag_id, task_id=task_id, execution_date=execution_date) + ti = create_task_instance(dag_id=dag_id, task_id=task_id, logical_date=logical_date) filename_template = "{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log" ts = ti.get_template_context()["ts"] diff --git a/tests/utils/test_log_handlers.py b/tests/utils/test_log_handlers.py index 781e3d853308..6e0e2b215f8e 100644 --- a/tests/utils/test_log_handlers.py +++ b/tests/utils/test_log_handlers.py @@ -100,7 +100,7 @@ def task_callable(ti): dagrun = dag.create_dagrun( run_type=DagRunType.MANUAL, state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE), **triggered_by_kwargs, ) @@ -155,7 +155,7 @@ def task_callable(ti): dagrun = dag.create_dagrun( run_type=DagRunType.MANUAL, state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE), **triggered_by_kwargs, ) @@ -217,7 +217,7 @@ def task_callable(ti): dagrun = dag.create_dagrun( run_type=DagRunType.MANUAL, state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE), **triggered_by_kwargs, ) @@ -280,7 +280,7 @@ def task_callable(ti): dagrun = dag.create_dagrun( run_type=DagRunType.MANUAL, state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE), **triggered_by_kwargs, ) @@ -358,7 +358,7 @@ def test__read_when_local(self, mock_read_local, create_task_instance): dag_id="dag_for_testing_local_log_read", task_id="task_for_testing_local_log_read", run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, ) fth = FileTaskHandler("") actual = fth._read(ti=local_log_file_read, try_number=1) @@ -407,7 +407,7 @@ def test__read_served_logs_checked_when_done_and_no_local_or_remote_logs( dag_id="dag_for_testing_celery_executor_log_read", task_id="task_for_testing_celery_executor_log_read", run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, ) ti.state = TaskInstanceState.SUCCESS # we're testing scenario when task is done with conf_vars({("core", "executor"): executor_name}): @@ -440,7 +440,7 @@ def test_add_triggerer_suffix(self): @pytest.mark.parametrize("is_a_trigger", [True, False]) def test_set_context_trigger(self, create_dummy_dag, dag_maker, is_a_trigger, session, tmp_path): create_dummy_dag(dag_id="test_fth", task_id="dummy") - (ti,) = dag_maker.create_dagrun(execution_date=pendulum.datetime(2023, 1, 1, tz="UTC")).task_instances + (ti,) = dag_maker.create_dagrun(logical_date=pendulum.datetime(2023, 1, 1, tz="UTC")).task_instances assert isinstance(ti, TaskInstance) if is_a_trigger: ti.is_trigger_log_context = True @@ -461,12 +461,12 @@ def test_set_context_trigger(self, create_dummy_dag, dag_maker, is_a_trigger, se class TestFilenameRendering: def test_python_formatting(self, create_log_template, create_task_instance): - create_log_template("{dag_id}/{task_id}/{execution_date}/{try_number}.log") + create_log_template("{dag_id}/{task_id}/{logical_date}/{try_number}.log") filename_rendering_ti = create_task_instance( dag_id="dag_for_testing_filename_rendering", task_id="task_for_testing_filename_rendering", run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, ) expected_filename = ( @@ -483,7 +483,7 @@ def test_jinja_rendering(self, create_log_template, create_task_instance): dag_id="dag_for_testing_filename_rendering", task_id="task_for_testing_filename_rendering", run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, ) expected_filename = ( @@ -501,7 +501,7 @@ def test_log_retrieval_valid(self, create_task_instance): dag_id="dag_for_testing_filename_rendering", task_id="task_for_testing_filename_rendering", run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, ) log_url_ti.hostname = "hostname" actual = FileTaskHandler("")._get_log_retrieval_url(log_url_ti, "DYNAMIC_PATH") @@ -512,7 +512,7 @@ def test_log_retrieval_valid_trigger(self, create_task_instance): dag_id="dag_for_testing_filename_rendering", task_id="task_for_testing_filename_rendering", run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, ) ti.hostname = "hostname" trigger = Trigger("", {}) @@ -544,7 +544,7 @@ def test_log_retrieval_valid_trigger(self, create_task_instance): AIRFLOW_CTX_DAG_OWNER=airflow AIRFLOW_CTX_DAG_ID=simple_async_timedelta AIRFLOW_CTX_TASK_ID=wait -AIRFLOW_CTX_EXECUTION_DATE=2022-11-16T08:05:52.324532+00:00 +AIRFLOW_CTX_LOGICAL_DATE=2022-11-16T08:05:52.324532+00:00 AIRFLOW_CTX_TRY_NUMBER=1 AIRFLOW_CTX_DAG_RUN_ID=manual__2022-11-16T08:05:52.324532+00:00 [2022-11-16T00:05:54.604-0800] {taskinstance.py:1360} INFO - Pausing task as DEFERRED. dag_id=simple_async_timedelta, task_id=wait, execution_date=20221116T080552, start_date=20221116T080554 @@ -601,7 +601,7 @@ def test_interleave_interleaves(): "[2022-11-16T00:05:54.592-0800] {taskinstance.py:1485} INFO - Exporting env vars: AIRFLOW_CTX_DAG_OWNER=airflow", "AIRFLOW_CTX_DAG_ID=simple_async_timedelta", "AIRFLOW_CTX_TASK_ID=wait", - "AIRFLOW_CTX_EXECUTION_DATE=2022-11-16T08:05:52.324532+00:00", + "AIRFLOW_CTX_LOGICAL_DATE=2022-11-16T08:05:52.324532+00:00", "AIRFLOW_CTX_TRY_NUMBER=1", "AIRFLOW_CTX_DAG_RUN_ID=manual__2022-11-16T08:05:52.324532+00:00", "[2022-11-16T00:05:54.604-0800] {taskinstance.py:1360} INFO - Pausing task as DEFERRED. dag_id=simple_async_timedelta, task_id=wait, execution_date=20221116T080552, start_date=20221116T080554", @@ -618,7 +618,7 @@ def test_interleave_interleaves(): "[2022-11-16T00:05:54.592-0800] {taskinstance.py:1485} INFO - Exporting env vars: AIRFLOW_CTX_DAG_OWNER=airflow", "AIRFLOW_CTX_DAG_ID=simple_async_timedelta", "AIRFLOW_CTX_TASK_ID=wait", - "AIRFLOW_CTX_EXECUTION_DATE=2022-11-16T08:05:52.324532+00:00", + "AIRFLOW_CTX_LOGICAL_DATE=2022-11-16T08:05:52.324532+00:00", "AIRFLOW_CTX_TRY_NUMBER=1", "AIRFLOW_CTX_DAG_RUN_ID=manual__2022-11-16T08:05:52.324532+00:00", "[2022-11-16T00:05:54.604-0800] {taskinstance.py:1360} INFO - Pausing task as DEFERRED. dag_id=simple_async_timedelta, task_id=wait, execution_date=20221116T080552, start_date=20221116T080554", @@ -637,7 +637,7 @@ def test_interleave_interleaves(): [2023-01-15T22:36:46.530-0800] {standard_task_runner.py:83} INFO - Running: ['airflow', 'tasks', 'run', 'example_time_delta_sensor_async', 'wait', 'manual__2023-01-16T06:36:43.044492+00:00', '--job-id', '487', '--raw', '--subdir', '/Users/dstandish/code/airflow/airflow/example_dags/example_time_delta_sensor_async.py', '--cfg-path', '/var/folders/7_/1xx0hqcs3txd7kqt0ngfdjth0000gn/T/tmpiwyl54bn', '--no-shut-down-logging'] [2023-01-15T22:36:46.536-0800] {standard_task_runner.py:84} INFO - Job 487: Subtask wait [2023-01-15T22:36:46.624-0800] {task_command.py:417} INFO - Running on host daniels-mbp-2.lan -[2023-01-15T22:36:46.918-0800] {taskinstance.py:1558} INFO - Exporting env vars: AIRFLOW_CTX_DAG_OWNER='airflow' AIRFLOW_CTX_DAG_ID='example_time_delta_sensor_async' AIRFLOW_CTX_TASK_ID='wait' AIRFLOW_CTX_EXECUTION_DATE='2023-01-16T06:36:43.044492+00:00' AIRFLOW_CTX_TRY_NUMBER='1' AIRFLOW_CTX_DAG_RUN_ID='manual__2023-01-16T06:36:43.044492+00:00' +[2023-01-15T22:36:46.918-0800] {taskinstance.py:1558} INFO - Exporting env vars: AIRFLOW_CTX_DAG_OWNER='airflow' AIRFLOW_CTX_DAG_ID='example_time_delta_sensor_async' AIRFLOW_CTX_TASK_ID='wait' AIRFLOW_CTX_LOGICAL_DATE='2023-01-16T06:36:43.044492+00:00' AIRFLOW_CTX_TRY_NUMBER='1' AIRFLOW_CTX_DAG_RUN_ID='manual__2023-01-16T06:36:43.044492+00:00' [2023-01-15T22:36:46.929-0800] {taskinstance.py:1433} INFO - Pausing task as DEFERRED. dag_id=example_time_delta_sensor_async, task_id=wait, execution_date=20230116T063643, start_date=20230116T063646 [2023-01-15T22:36:46.981-0800] {local_task_job.py:218} INFO - Task exited with return code 100 (task deferral) @@ -649,7 +649,7 @@ def test_interleave_interleaves(): [2023-01-15T22:36:46.530-0800] {standard_task_runner.py:83} INFO - Running: ['airflow', 'tasks', 'run', 'example_time_delta_sensor_async', 'wait', 'manual__2023-01-16T06:36:43.044492+00:00', '--job-id', '487', '--raw', '--subdir', '/Users/dstandish/code/airflow/airflow/example_dags/example_time_delta_sensor_async.py', '--cfg-path', '/var/folders/7_/1xx0hqcs3txd7kqt0ngfdjth0000gn/T/tmpiwyl54bn', '--no-shut-down-logging'] [2023-01-15T22:36:46.536-0800] {standard_task_runner.py:84} INFO - Job 487: Subtask wait [2023-01-15T22:36:46.624-0800] {task_command.py:417} INFO - Running on host daniels-mbp-2.lan -[2023-01-15T22:36:46.918-0800] {taskinstance.py:1558} INFO - Exporting env vars: AIRFLOW_CTX_DAG_OWNER='airflow' AIRFLOW_CTX_DAG_ID='example_time_delta_sensor_async' AIRFLOW_CTX_TASK_ID='wait' AIRFLOW_CTX_EXECUTION_DATE='2023-01-16T06:36:43.044492+00:00' AIRFLOW_CTX_TRY_NUMBER='1' AIRFLOW_CTX_DAG_RUN_ID='manual__2023-01-16T06:36:43.044492+00:00' +[2023-01-15T22:36:46.918-0800] {taskinstance.py:1558} INFO - Exporting env vars: AIRFLOW_CTX_DAG_OWNER='airflow' AIRFLOW_CTX_DAG_ID='example_time_delta_sensor_async' AIRFLOW_CTX_TASK_ID='wait' AIRFLOW_CTX_LOGICAL_DATE='2023-01-16T06:36:43.044492+00:00' AIRFLOW_CTX_TRY_NUMBER='1' AIRFLOW_CTX_DAG_RUN_ID='manual__2023-01-16T06:36:43.044492+00:00' [2023-01-15T22:36:46.929-0800] {taskinstance.py:1433} INFO - Pausing task as DEFERRED. dag_id=example_time_delta_sensor_async, task_id=wait, execution_date=20230116T063643, start_date=20230116T063646 [2023-01-15T22:36:46.981-0800] {local_task_job.py:218} INFO - Task exited with return code 100 (task deferral) [2023-01-15T22:37:17.673-0800] {taskinstance.py:1131} INFO - Dependencies all met for dep_context=non-requeueable deps ti= diff --git a/tests/utils/test_operator_helpers.py b/tests/utils/test_operator_helpers.py index 9ebd9d08ae00..ac04050d338b 100644 --- a/tests/utils/test_operator_helpers.py +++ b/tests/utils/test_operator_helpers.py @@ -30,7 +30,7 @@ def setup_method(self): self.dag_id = "dag_id" self.task_id = "task_id" self.try_number = 1 - self.execution_date = "2017-05-21T00:00:00" + self.logical_date = "2017-05-21T00:00:00" self.dag_run_id = "dag_run_id" self.owner = ["owner1", "owner2"] self.email = ["email1@test.com"] @@ -38,14 +38,14 @@ def setup_method(self): "dag_run": mock.MagicMock( name="dag_run", run_id=self.dag_run_id, - execution_date=datetime.strptime(self.execution_date, "%Y-%m-%dT%H:%M:%S"), + logical_date=datetime.strptime(self.logical_date, "%Y-%m-%dT%H:%M:%S"), ), "task_instance": mock.MagicMock( name="task_instance", task_id=self.task_id, dag_id=self.dag_id, try_number=self.try_number, - execution_date=datetime.strptime(self.execution_date, "%Y-%m-%dT%H:%M:%S"), + logical_date=datetime.strptime(self.logical_date, "%Y-%m-%dT%H:%M:%S"), ), "task": mock.MagicMock(name="task", owner=self.owner, email=self.email), } @@ -56,7 +56,7 @@ def test_context_to_airflow_vars_empty_context(self): def test_context_to_airflow_vars_all_context(self): assert operator_helpers.context_to_airflow_vars(self.context) == { "airflow.ctx.dag_id": self.dag_id, - "airflow.ctx.execution_date": self.execution_date, + "airflow.ctx.logical_date": self.logical_date, "airflow.ctx.task_id": self.task_id, "airflow.ctx.dag_run_id": self.dag_run_id, "airflow.ctx.try_number": str(self.try_number), @@ -66,7 +66,7 @@ def test_context_to_airflow_vars_all_context(self): assert operator_helpers.context_to_airflow_vars(self.context, in_env_var_format=True) == { "AIRFLOW_CTX_DAG_ID": self.dag_id, - "AIRFLOW_CTX_EXECUTION_DATE": self.execution_date, + "AIRFLOW_CTX_LOGICAL_DATE": self.logical_date, "AIRFLOW_CTX_TASK_ID": self.task_id, "AIRFLOW_CTX_TRY_NUMBER": str(self.try_number), "AIRFLOW_CTX_DAG_RUN_ID": self.dag_run_id, @@ -102,16 +102,12 @@ def callable1(ds_nodash): return (ds_nodash,) -def callable2(ds_nodash, prev_ds_nodash): - return (ds_nodash, prev_ds_nodash) +def callable3(ds_nodash, *args, **kwargs): + return (ds_nodash, args, kwargs) -def callable3(ds_nodash, prev_ds_nodash, *args, **kwargs): - return (ds_nodash, prev_ds_nodash, args, kwargs) - - -def callable4(ds_nodash, prev_ds_nodash, **kwargs): - return (ds_nodash, prev_ds_nodash, kwargs) +def callable4(ds_nodash, **kwargs): + return (ds_nodash, kwargs) def callable5(**kwargs): @@ -146,9 +142,7 @@ def callable11(*, ds_nodash, **kwargs): KWARGS = { - "prev_ds_nodash": "20191231", "ds_nodash": "20200101", - "tomorrow_ds_nodash": "20200102", } @@ -156,28 +150,6 @@ def callable11(*, ds_nodash, **kwargs): "func,args,kwargs,expected", [ (callable1, (), KWARGS, ("20200101",)), - (callable2, (), KWARGS, ("20200101", "20191231")), - ( - callable3, - (), - KWARGS, - ( - "20200101", - "20191231", - (), - {"tomorrow_ds_nodash": "20200102"}, - ), - ), - ( - callable4, - (), - KWARGS, - ( - "20200101", - "20191231", - {"tomorrow_ds_nodash": "20200102"}, - ), - ), ( callable5, (), @@ -189,20 +161,6 @@ def callable11(*, ds_nodash, **kwargs): (callable8, (1, 2), KWARGS, (1, (2,), KWARGS)), (callable9, (1, 2), KWARGS, ((1, 2), KWARGS)), (callable10, (1,), KWARGS, (1, "20200101")), - ( - callable11, - (), - KWARGS, - ( - ( - "20200101", - { - "prev_ds_nodash": "20191231", - "tomorrow_ds_nodash": "20200102", - }, - ) - ), - ), ], ) def test_make_kwargs_callable(func, args, kwargs, expected): @@ -218,7 +176,7 @@ def func(ds_nodash): kwargs_callable = operator_helpers.make_kwargs_callable(func) args = ["20200101"] - kwargs = {"ds_nodash": "20200101", "tomorrow_ds_nodash": "20200102"} + kwargs = {"ds_nodash": "20200101"} with pytest.raises(ValueError) as exc_info: kwargs_callable(*args, **kwargs) diff --git a/tests/utils/test_sqlalchemy.py b/tests/utils/test_sqlalchemy.py index f32858c064d2..70e60203bddc 100644 --- a/tests/utils/test_sqlalchemy.py +++ b/tests/utils/test_sqlalchemy.py @@ -73,7 +73,7 @@ def test_utc_transformations(self): dag_id = "test_utc_transformations" start_date = utcnow() iso_date = start_date.isoformat() - execution_date = start_date + datetime.timedelta(hours=1, days=1) + logical_date = start_date + datetime.timedelta(hours=1, days=1) dag = DAG(dag_id=dag_id, schedule=datetime.timedelta(days=1), start_date=start_date) dag.clear() @@ -82,17 +82,17 @@ def test_utc_transformations(self): run = dag.create_dagrun( run_id=iso_date, state=State.NONE, - execution_date=execution_date, + logical_date=logical_date, start_date=start_date, session=self.session, - data_interval=dag.timetable.infer_manual_data_interval(run_after=execution_date), + data_interval=dag.timetable.infer_manual_data_interval(run_after=logical_date), **triggered_by_kwargs, ) - assert execution_date == run.execution_date + assert logical_date == run.logical_date assert start_date == run.start_date - assert execution_date.utcoffset().total_seconds() == 0.0 + assert logical_date.utcoffset().total_seconds() == 0.0 assert start_date.utcoffset().total_seconds() == 0.0 assert iso_date == run.run_id @@ -116,7 +116,7 @@ def test_process_bind_param_naive(self): dag.create_dagrun( run_id=start_date.isoformat, state=State.NONE, - execution_date=start_date, + logical_date=start_date, start_date=start_date, session=self.session, data_interval=dag.timetable.infer_manual_data_interval(run_after=start_date), diff --git a/tests/utils/test_state.py b/tests/utils/test_state.py index effd7d28dad4..f8fe9f1fc8c5 100644 --- a/tests/utils/test_state.py +++ b/tests/utils/test_state.py @@ -46,7 +46,7 @@ def test_dagrun_state_enum_escape(): dag.create_dagrun( run_type=DagRunType.SCHEDULED, state=DagRunState.QUEUED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, start_date=DEFAULT_DATE, data_interval=dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE), session=session, diff --git a/tests/utils/test_task_group.py b/tests/utils/test_task_group.py index b0fd7fa6c183..00caac703a40 100644 --- a/tests/utils/test_task_group.py +++ b/tests/utils/test_task_group.py @@ -170,8 +170,8 @@ def my_task(): def test_build_task_group_context_manager(): - execution_date = pendulum.parse("20200101") - with DAG("test_build_task_group_context_manager", schedule=None, start_date=execution_date) as dag: + logical_date = pendulum.parse("20200101") + with DAG("test_build_task_group_context_manager", schedule=None, start_date=logical_date) as dag: task1 = EmptyOperator(task_id="task1") with TaskGroup("group234") as group234: _ = EmptyOperator(task_id="task2") @@ -207,8 +207,8 @@ def test_build_task_group(): This is an alternative syntax to use TaskGroup. It should result in the same TaskGroup as using context manager. """ - execution_date = pendulum.parse("20200101") - dag = DAG("test_build_task_group", schedule=None, start_date=execution_date) + logical_date = pendulum.parse("20200101") + dag = DAG("test_build_task_group", schedule=None, start_date=logical_date) task1 = EmptyOperator(task_id="task1", dag=dag) group234 = TaskGroup("group234", dag=dag) _ = EmptyOperator(task_id="task2", dag=dag, task_group=group234) @@ -241,8 +241,8 @@ def test_build_task_group_with_prefix(): """ Tests that prefix_group_id turns on/off prefixing of task_id with group_id. """ - execution_date = pendulum.parse("20200101") - with DAG("test_build_task_group_with_prefix", schedule=None, start_date=execution_date) as dag: + logical_date = pendulum.parse("20200101") + with DAG("test_build_task_group_with_prefix", schedule=None, start_date=logical_date) as dag: task1 = EmptyOperator(task_id="task1") with TaskGroup("group234", prefix_group_id=False) as group234: task2 = EmptyOperator(task_id="task2") @@ -324,8 +324,8 @@ def task_4(task_2_output, task_3_output): def task_5(): print("task_5") - execution_date = pendulum.parse("20200101") - with DAG("test_build_task_group_with_task_decorator", schedule=None, start_date=execution_date) as dag: + logical_date = pendulum.parse("20200101") + with DAG("test_build_task_group_with_task_decorator", schedule=None, start_date=logical_date) as dag: tsk_1 = task_1() with TaskGroup("group234") as group234: @@ -375,8 +375,8 @@ def test_sub_dag_task_group(): """ Tests dag.partial_subset() updates task_group correctly. """ - execution_date = pendulum.parse("20200101") - with DAG("test_test_task_group_sub_dag", schedule=None, start_date=execution_date) as dag: + logical_date = pendulum.parse("20200101") + with DAG("test_test_task_group_sub_dag", schedule=None, start_date=logical_date) as dag: task1 = EmptyOperator(task_id="task1") with TaskGroup("group234") as group234: _ = EmptyOperator(task_id="task2") @@ -448,8 +448,8 @@ def test_sub_dag_task_group(): def test_dag_edges(): - execution_date = pendulum.parse("20200101") - with DAG("test_dag_edges", schedule=None, start_date=execution_date) as dag: + logical_date = pendulum.parse("20200101") + with DAG("test_dag_edges", schedule=None, start_date=logical_date) as dag: task1 = EmptyOperator(task_id="task1") with TaskGroup("group_a") as group_a: with TaskGroup("group_b") as group_b: @@ -557,8 +557,8 @@ def test_dag_edges(): def test_dag_edges_setup_teardown(): - execution_date = pendulum.parse("20200101") - with DAG("test_dag_edges", schedule=None, start_date=execution_date) as dag: + logical_date = pendulum.parse("20200101") + with DAG("test_dag_edges", schedule=None, start_date=logical_date) as dag: setup1 = EmptyOperator(task_id="setup1").as_setup() teardown1 = EmptyOperator(task_id="teardown1").as_teardown() @@ -589,9 +589,9 @@ def test_dag_edges_setup_teardown_nested(): from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator - execution_date = pendulum.parse("20200101") + logical_date = pendulum.parse("20200101") - with DAG(dag_id="s_t_dag", schedule=None, start_date=execution_date) as dag: + with DAG(dag_id="s_t_dag", schedule=None, start_date=logical_date) as dag: @task def test_task(): @@ -634,31 +634,31 @@ def outer(): def test_duplicate_group_id(): from airflow.exceptions import DuplicateTaskIdFound - execution_date = pendulum.parse("20200101") + logical_date = pendulum.parse("20200101") - with DAG("test_duplicate_group_id", schedule=None, start_date=execution_date): + with DAG("test_duplicate_group_id", schedule=None, start_date=logical_date): _ = EmptyOperator(task_id="task1") with pytest.raises(DuplicateTaskIdFound, match=r".* 'task1' .*"), TaskGroup("task1"): pass - with DAG("test_duplicate_group_id", schedule=None, start_date=execution_date): + with DAG("test_duplicate_group_id", schedule=None, start_date=logical_date): _ = EmptyOperator(task_id="task1") with TaskGroup("group1", prefix_group_id=False): with pytest.raises(DuplicateTaskIdFound, match=r".* 'group1' .*"), TaskGroup("group1"): pass - with DAG("test_duplicate_group_id", schedule=None, start_date=execution_date): + with DAG("test_duplicate_group_id", schedule=None, start_date=logical_date): with TaskGroup("group1", prefix_group_id=False): with pytest.raises(DuplicateTaskIdFound, match=r".* 'group1' .*"): _ = EmptyOperator(task_id="group1") - with DAG("test_duplicate_group_id", schedule=None, start_date=execution_date): + with DAG("test_duplicate_group_id", schedule=None, start_date=logical_date): _ = EmptyOperator(task_id="task1") with TaskGroup("group1"): with pytest.raises(DuplicateTaskIdFound, match=r".* 'group1.downstream_join_id' .*"): _ = EmptyOperator(task_id="downstream_join_id") - with DAG("test_duplicate_group_id", schedule=None, start_date=execution_date): + with DAG("test_duplicate_group_id", schedule=None, start_date=logical_date): _ = EmptyOperator(task_id="task1") with TaskGroup("group1"): with pytest.raises(DuplicateTaskIdFound, match=r".* 'group1.upstream_join_id' .*"): @@ -740,11 +740,11 @@ def section_2(value2): op1 = task_2(task_1(value)) return section_2(op1) - execution_date = pendulum.parse("20201109") + logical_date = pendulum.parse("20201109") with DAG( dag_id="example_nested_task_group_decorator", schedule=None, - start_date=execution_date, + start_date=logical_date, tags=["example"], ) as dag: t_start = task_start() @@ -861,11 +861,11 @@ def section_a(value): """TaskGroup for grouping related Tasks""" return task_3(task_2(task_1(value))) - execution_date = pendulum.parse("20201109") + logical_date = pendulum.parse("20201109") with DAG( dag_id="example_task_group_decorator_mix", schedule=None, - start_date=execution_date, + start_date=logical_date, tags=["example"], ) as dag: t_start = PythonOperator(task_id="task_start", python_callable=task_start, dag=dag) @@ -921,11 +921,11 @@ def section_2(value): """TaskGroup for grouping related Tasks""" return task_3(task_2(task_1(value))) - execution_date = pendulum.parse("20201109") + logical_date = pendulum.parse("20201109") with DAG( dag_id="example_task_group_decorator_mix", schedule=None, - start_date=execution_date, + start_date=logical_date, tags=["example"], ) as dag: t_start = PythonOperator(task_id="task_start", python_callable=task_start, dag=dag) @@ -974,11 +974,11 @@ def section_2(value): def test_default_args(): """Testing TaskGroup with default_args""" - execution_date = pendulum.parse("20201109") + logical_date = pendulum.parse("20201109") with DAG( dag_id="example_task_group_default_args", schedule=None, - start_date=execution_date, + start_date=logical_date, default_args={"owner": "dag"}, ): with TaskGroup("group1", default_args={"owner": "group"}): @@ -1036,11 +1036,11 @@ def task_group2(): def task_group3(): task_end() - execution_date = pendulum.parse("20201109") + logical_date = pendulum.parse("20201109") with DAG( dag_id="example_duplicate_task_group_id", schedule=None, - start_date=execution_date, + start_date=logical_date, tags=["example"], ) as dag: task_group1() @@ -1092,11 +1092,11 @@ def task_group1(name: str): task_1() task_end() - execution_date = pendulum.parse("20201109") + logical_date = pendulum.parse("20201109") with DAG( dag_id="example_multi_call_task_groups", schedule=None, - start_date=execution_date, + start_date=logical_date, tags=["example"], ) as dag: task_group1("Call1") @@ -1255,8 +1255,8 @@ def test_topological_sort2(): def test_topological_nested_groups(): - execution_date = pendulum.parse("20200101") - with DAG("test_dag_edges", schedule=None, start_date=execution_date) as dag: + logical_date = pendulum.parse("20200101") + with DAG("test_dag_edges", schedule=None, start_date=logical_date) as dag: task1 = EmptyOperator(task_id="task1") task5 = EmptyOperator(task_id="task5") with TaskGroup("group_a") as group_a: @@ -1290,8 +1290,8 @@ def nested_topo(group): def test_hierarchical_alphabetical_sort(): - execution_date = pendulum.parse("20200101") - with DAG("test_dag_edges", schedule=None, start_date=execution_date) as dag: + logical_date = pendulum.parse("20200101") + with DAG("test_dag_edges", schedule=None, start_date=logical_date) as dag: task1 = EmptyOperator(task_id="task1") task5 = EmptyOperator(task_id="task5") with TaskGroup("group_c"): @@ -1332,8 +1332,8 @@ def nested(group): def test_topological_group_dep(): - execution_date = pendulum.parse("20200101") - with DAG("test_dag_edges", schedule=None, start_date=execution_date) as dag: + logical_date = pendulum.parse("20200101") + with DAG("test_dag_edges", schedule=None, start_date=logical_date) as dag: task1 = EmptyOperator(task_id="task1") task6 = EmptyOperator(task_id="task6") with TaskGroup("group_a") as group_a: diff --git a/tests/utils/test_task_handler_with_custom_formatter.py b/tests/utils/test_task_handler_with_custom_formatter.py index 103c3543e31e..17de8f6526f0 100644 --- a/tests/utils/test_task_handler_with_custom_formatter.py +++ b/tests/utils/test_task_handler_with_custom_formatter.py @@ -69,7 +69,7 @@ def task_instance(dag_maker): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dagrun = dag_maker.create_dagrun( state=DagRunState.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, run_type=DagRunType.MANUAL, data_interval=dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE), **triggered_by_kwargs, diff --git a/tests/utils/test_types.py b/tests/utils/test_types.py index 151758163406..0c969d1c2064 100644 --- a/tests/utils/test_types.py +++ b/tests/utils/test_types.py @@ -47,7 +47,7 @@ def test_runtype_enum_escape(): dag.create_dagrun( run_type=DagRunType.SCHEDULED, state=State.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, start_date=DEFAULT_DATE, session=session, data_interval=data_interval, diff --git a/tests/www/test_utils.py b/tests/www/test_utils.py index 67e64d601fcf..8837bbaceaa3 100644 --- a/tests/www/test_utils.py +++ b/tests/www/test_utils.py @@ -235,13 +235,13 @@ def test_task_instance_link(self): with cached_app(testing=True).test_request_context(): html = str( utils.task_instance_link( - {"dag_id": "", "task_id": "", "map_index": 1, "execution_date": datetime.now()} + {"dag_id": "", "task_id": "", "map_index": 1, "logical_date": datetime.now()} ) ) html_map_index_none = str( utils.task_instance_link( - {"dag_id": "", "task_id": "", "map_index": -1, "execution_date": datetime.now()} + {"dag_id": "", "task_id": "", "map_index": -1, "logical_date": datetime.now()} ) ) @@ -263,7 +263,7 @@ def test_dag_link(self): from airflow.www.app import cached_app with cached_app(testing=True).test_request_context(): - html = str(utils.dag_link({"dag_id": "", "execution_date": datetime.now()})) + html = str(utils.dag_link({"dag_id": "", "logical_date": datetime.now()})) assert "%3Ca%261%3E" in html assert "" not in html @@ -287,7 +287,7 @@ def test_dag_run_link(self): with cached_app(testing=True).test_request_context(): html = str( - utils.dag_run_link({"dag_id": "", "run_id": "", "execution_date": datetime.now()}) + utils.dag_run_link({"dag_id": "", "run_id": "", "logical_date": datetime.now()}) ) assert "%3Ca%261%3E" in html @@ -509,7 +509,7 @@ def test_wrapped_markdown_with_raw_html(self, html): "data_interval_end": None, "data_interval_start": None, "end_date": None, - "execution_date": None, + "logical_date": None, "external_trigger": None, "last_scheduling_decision": None, "note": None, @@ -625,7 +625,7 @@ def test_dag_run_custom_sqla_interface_delete_no_collateral_damage(dag_maker, se for dag_id, date in itertools.product(dag_ids, dates): with dag_maker(dag_id=dag_id) as dag: dag.create_dagrun( - execution_date=date, + logical_date=date, state="running", run_type="scheduled", data_interval=(date, date), diff --git a/tests/www/views/test_views.py b/tests/www/views/test_views.py index e447746b6305..99bc7e268ce8 100644 --- a/tests/www/views/test_views.py +++ b/tests/www/views/test_views.py @@ -224,11 +224,11 @@ def test_endpoint_should_not_be_unauthenticated(app): "url, content", [ ( - "/taskinstance/list/?_flt_0_execution_date=2018-10-09+22:44:31", + "/taskinstance/list/?_flt_0_logical_date=2018-10-09+22:44:31", "List Task Instance", ), ( - "/taskreschedule/list/?_flt_0_execution_date=2018-10-09+22:44:31", + "/taskreschedule/list/?_flt_0_logical_date=2018-10-09+22:44:31", "List Task Reschedule", ), ], @@ -317,7 +317,6 @@ def test_mark_task_instance_state(test_app): Test that _mark_task_instance_state() does all three things: - Marks the given TaskInstance as SUCCESS; - Clears downstream TaskInstances in FAILED/UPSTREAM_FAILED state; - - Set DagRun to QUEUED. """ from airflow.models.dag import DAG from airflow.models.dagbag import DagBag @@ -345,7 +344,7 @@ def test_mark_task_instance_state(test_app): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dagrun = dag.create_dagrun( start_date=start_date, - execution_date=start_date, + logical_date=start_date, data_interval=(start_date, start_date), state=State.FAILED, run_type=DagRunType.SCHEDULED, @@ -358,7 +357,7 @@ def get_task_instance(session, task): .filter( TaskInstance.dag_id == dag.dag_id, TaskInstance.task_id == task.task_id, - TaskInstance.execution_date == start_date, + TaskInstance.logical_date == start_date, ) .one() ) @@ -448,7 +447,7 @@ def test_mark_task_group_state(test_app): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dagrun = dag.create_dagrun( start_date=start_date, - execution_date=start_date, + logical_date=start_date, data_interval=(start_date, start_date), state=State.FAILED, run_type=DagRunType.SCHEDULED, @@ -461,7 +460,7 @@ def get_task_instance(session, task): .filter( TaskInstance.dag_id == dag.dag_id, TaskInstance.task_id == task.task_id, - TaskInstance.execution_date == start_date, + TaskInstance.logical_date == start_date, ) .one() ) @@ -571,31 +570,31 @@ def test_get_task_stats_from_query(): "url, content", [ ( - "/rendered-templates?execution_date=invalid", + "/rendered-templates?logical_date=invalid", INVALID_DATETIME_RESPONSE, ), ( - "/log?dag_id=tutorial&execution_date=invalid", + "/log?dag_id=tutorial&logical_date=invalid", INVALID_DATETIME_RESPONSE, ), ( - "/redirect_to_external_log?execution_date=invalid", + "/redirect_to_external_log?logical_date=invalid", INVALID_DATETIME_RESPONSE, ), ( - "/task?execution_date=invalid", + "/task?logical_date=invalid", INVALID_DATETIME_RESPONSE, ), ( - "dags/example_bash_operator/graph?execution_date=invalid", + "dags/example_bash_operator/graph?logical_date=invalid", INVALID_DATETIME_RESPONSE, ), ( - "dags/example_bash_operator/gantt?execution_date=invalid", + "dags/example_bash_operator/gantt?logical_date=invalid", INVALID_DATETIME_RESPONSE, ), ( - "extra_links?execution_date=invalid", + "extra_links?logical_date=invalid", INVALID_DATETIME_RESPONSE, ), ], diff --git a/tests/www/views/test_views_acl.py b/tests/www/views/test_views_acl.py index e99aeccb5b4e..331a0e565a14 100644 --- a/tests/www/views/test_views_acl.py +++ b/tests/www/views/test_views_acl.py @@ -152,7 +152,7 @@ def _init_dagruns(acl_app, _reset_dagruns): acl_app.dag_bag.get_dag("example_bash_operator").create_dagrun( run_id=DEFAULT_RUN_ID, run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), start_date=timezone.utcnow(), state=State.RUNNING, @@ -160,7 +160,7 @@ def _init_dagruns(acl_app, _reset_dagruns): ) acl_app.dag_bag.get_dag("example_python_operator").create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, start_date=timezone.utcnow(), data_interval=(DEFAULT_DATE, DEFAULT_DATE), state=State.RUNNING, @@ -592,15 +592,15 @@ def client_dags_tis_logs(acl_app, user_dags_tis_logs): RENDERED_TEMPLATES_URL = ( f"rendered-templates?task_id=runme_0&dag_id=example_bash_operator&" - f"execution_date={urllib.parse.quote_plus(str(DEFAULT_DATE))}" + f"logical_date={urllib.parse.quote_plus(str(DEFAULT_DATE))}" ) TASK_URL = ( f"task?task_id=runme_0&dag_id=example_bash_operator&" - f"execution_date={urllib.parse.quote_plus(str(DEFAULT_DATE))}" + f"logical_date={urllib.parse.quote_plus(str(DEFAULT_DATE))}" ) XCOM_URL = ( f"xcom?task_id=runme_0&dag_id=example_bash_operator&" - f"execution_date={urllib.parse.quote_plus(str(DEFAULT_DATE))}" + f"logical_date={urllib.parse.quote_plus(str(DEFAULT_DATE))}" ) DURATION_URL = "duration?days=30&dag_id=example_bash_operator" TRIES_URL = "tries?days=30&dag_id=example_bash_operator" @@ -609,7 +609,7 @@ def client_dags_tis_logs(acl_app, user_dags_tis_logs): GRID_DATA_URL = "object/grid_data?dag_id=example_bash_operator" LOG_URL = ( f"log?task_id=runme_0&dag_id=example_bash_operator&" - f"execution_date={urllib.parse.quote_plus(str(DEFAULT_DATE))}" + f"logical_date={urllib.parse.quote_plus(str(DEFAULT_DATE))}" ) @@ -805,7 +805,7 @@ def test_success_fail_for_read_only_task_instance_access(client_only_dags_tis): GET_LOGS_WITH_METADATA_URL = ( f"get_logs_with_metadata?task_id=runme_0&dag_id=example_bash_operator&" - f"execution_date={urllib.parse.quote_plus(str(DEFAULT_DATE))}&" + f"logical_date={urllib.parse.quote_plus(str(DEFAULT_DATE))}&" f"try_number=1&metadata=null" ) diff --git a/tests/www/views/test_views_cluster_activity.py b/tests/www/views/test_views_cluster_activity.py index b675a7241ab9..403632f3e4df 100644 --- a/tests/www/views/test_views_cluster_activity.py +++ b/tests/www/views/test_views_cluster_activity.py @@ -68,7 +68,7 @@ def _make_dag_runs(dag_maker, session, time_machine): run_id="run_1", state=DagRunState.SUCCESS, run_type=DagRunType.SCHEDULED, - execution_date=date, + logical_date=date, start_date=date, ) @@ -76,7 +76,7 @@ def _make_dag_runs(dag_maker, session, time_machine): run_id="run_2", state=DagRunState.FAILED, run_type=DagRunType.ASSET_TRIGGERED, - execution_date=date + timedelta(days=1), + logical_date=date + timedelta(days=1), start_date=date + timedelta(days=1), ) @@ -84,7 +84,7 @@ def _make_dag_runs(dag_maker, session, time_machine): run_id="run_3", state=DagRunState.RUNNING, run_type=DagRunType.SCHEDULED, - execution_date=pendulum.DateTime(2023, 2, 3, 0, 0, 0, tzinfo=pendulum.UTC), + logical_date=pendulum.DateTime(2023, 2, 3, 0, 0, 0, tzinfo=pendulum.UTC), start_date=pendulum.DateTime(2023, 2, 3, 0, 0, 0, tzinfo=pendulum.UTC), ) run3.end_date = None diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py index a157e0aea0ca..b588a56128ed 100644 --- a/tests/www/views/test_views_connection.py +++ b/tests/www/views/test_views_connection.py @@ -61,7 +61,7 @@ def _clear_connections(): def test_create_connection(admin_client, session): resp = admin_client.post("/connection/add", data=CONNECTION, follow_redirects=True) check_content_in_response("Added Row", resp) - _check_last_log(session, dag_id=None, event="connection.create", execution_date=None) + _check_last_log(session, dag_id=None, event="connection.create", logical_date=None) def test_connection_id_trailing_blanks(admin_client, session): @@ -108,7 +108,7 @@ def test_all_fields_with_blanks(admin_client, session): @pytest.mark.enable_redact def test_action_logging_connection_masked_secrets(session, admin_client): admin_client.post("/connection/add", data=conn_with_extra(), follow_redirects=True) - _check_last_log_masked_connection(session, dag_id=None, event="connection.create", execution_date=None) + _check_last_log_masked_connection(session, dag_id=None, event="connection.create", logical_date=None) def test_prefill_form_null_extra(): diff --git a/tests/www/views/test_views_dagrun.py b/tests/www/views/test_views_dagrun.py index b8e5d843516e..7b9e7b9ac3a0 100644 --- a/tests/www/views/test_views_dagrun.py +++ b/tests/www/views/test_views_dagrun.py @@ -127,7 +127,7 @@ def test_create_dagrun_permission_denied(session, client_dr_without_dag_run_crea data = { "state": "running", "dag_id": "example_bash_operator", - "execution_date": "2018-07-06 05:06:03", + "logical_date": "2018-07-06 05:06:03", "run_id": "test_list_dagrun_includes_conf", "conf": '{"include": "me"}', } @@ -139,12 +139,12 @@ def test_create_dagrun_permission_denied(session, client_dr_without_dag_run_crea @pytest.fixture def running_dag_run(session): dag = DagBag().get_dag("example_bash_operator") - execution_date = timezone.datetime(2016, 1, 9) + logical_date = timezone.datetime(2016, 1, 9) triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dr = dag.create_dagrun( state="running", - execution_date=execution_date, - data_interval=(execution_date, execution_date), + logical_date=logical_date, + data_interval=(logical_date, logical_date), run_id="test_dag_runs_action", session=session, **triggered_by_kwargs, @@ -162,12 +162,12 @@ def running_dag_run(session): @pytest.fixture def completed_dag_run_with_missing_task(session): dag = DagBag().get_dag("example_bash_operator") - execution_date = timezone.datetime(2016, 1, 9) + logical_date = timezone.datetime(2016, 1, 9) triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dr = dag.create_dagrun( state="success", - execution_date=execution_date, - data_interval=(execution_date, execution_date), + logical_date=logical_date, + data_interval=(logical_date, logical_date), run_id="test_dag_runs_action", session=session, **triggered_by_kwargs, @@ -317,12 +317,12 @@ def dag_run_with_all_done_task(session): # Re-sync the DAG to the DB dag.sync_to_db() - execution_date = timezone.datetime(2016, 1, 9) + logical_date = timezone.datetime(2016, 1, 9) triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dr = dag.create_dagrun( state="running", - execution_date=execution_date, - data_interval=(execution_date, execution_date), + logical_date=logical_date, + data_interval=(logical_date, logical_date), run_id="test_dagrun_failed", session=session, **triggered_by_kwargs, diff --git a/tests/www/views/test_views_decorators.py b/tests/www/views/test_views_decorators.py index 7e00192ab729..c6c849606ef2 100644 --- a/tests/www/views/test_views_decorators.py +++ b/tests/www/views/test_views_decorators.py @@ -61,7 +61,7 @@ def dagruns(bash_dag, xcom_dag): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} bash_dagrun = bash_dag.create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=EXAMPLE_DAG_DEFAULT_DATE, + logical_date=EXAMPLE_DAG_DEFAULT_DATE, data_interval=(EXAMPLE_DAG_DEFAULT_DATE, EXAMPLE_DAG_DEFAULT_DATE), start_date=timezone.utcnow(), state=State.RUNNING, @@ -70,7 +70,7 @@ def dagruns(bash_dag, xcom_dag): xcom_dagrun = xcom_dag.create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=EXAMPLE_DAG_DEFAULT_DATE, + logical_date=EXAMPLE_DAG_DEFAULT_DATE, data_interval=(EXAMPLE_DAG_DEFAULT_DATE, EXAMPLE_DAG_DEFAULT_DATE), start_date=timezone.utcnow(), state=State.RUNNING, @@ -99,7 +99,7 @@ def test_action_logging_robots(session, admin_client): session, event="robots", dag_id=None, - execution_date=None, + logical_date=None, ) @@ -107,7 +107,7 @@ def test_action_logging_post(session, admin_client): form = dict( task_id="runme_1", dag_id="example_bash_operator", - execution_date=EXAMPLE_DAG_DEFAULT_DATE, + logical_date=EXAMPLE_DAG_DEFAULT_DATE, upstream="false", downstream="false", future="false", @@ -122,7 +122,7 @@ def test_action_logging_post(session, admin_client): session, dag_id="example_bash_operator", event="clear", - execution_date=EXAMPLE_DAG_DEFAULT_DATE, + logical_date=EXAMPLE_DAG_DEFAULT_DATE, expected_extra={ "upstream": "false", "downstream": "false", @@ -142,7 +142,7 @@ def test_action_logging_variables_post(session, admin_client): form = dict(key="random", val="random") admin_client.post("/variable/add", data=form) session.commit() - _check_last_log(session, dag_id=None, event="variable.create", execution_date=None) + _check_last_log(session, dag_id=None, event="variable.create", logical_date=None) @pytest.mark.enable_redact @@ -150,4 +150,4 @@ def test_action_logging_variables_masked_secrets(session, admin_client): form = dict(key="x_secret", val="randomval") admin_client.post("/variable/add", data=form) session.commit() - _check_last_log_masked_variable(session, dag_id=None, event="variable.create", execution_date=None) + _check_last_log_masked_variable(session, dag_id=None, event="variable.create", logical_date=None) diff --git a/tests/www/views/test_views_extra_links.py b/tests/www/views/test_views_extra_links.py index 586d38f80791..e3775e2a5498 100644 --- a/tests/www/views/test_views_extra_links.py +++ b/tests/www/views/test_views_extra_links.py @@ -86,12 +86,12 @@ def dag(): @pytest.fixture(scope="module") def create_dag_run(dag): - def _create_dag_run(*, execution_date, session): + def _create_dag_run(*, logical_date, session): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} return dag.create_dagrun( state=DagRunState.RUNNING, - execution_date=execution_date, - data_interval=(execution_date, execution_date), + logical_date=logical_date, + data_interval=(logical_date, logical_date), run_type=DagRunType.MANUAL, session=session, **triggered_by_kwargs, @@ -102,7 +102,7 @@ def _create_dag_run(*, execution_date, session): @pytest.fixture def dag_run(create_dag_run, session): - return create_dag_run(execution_date=DEFAULT_DATE, session=session) + return create_dag_run(logical_date=DEFAULT_DATE, session=session) @pytest.fixture(scope="module", autouse=True) @@ -145,7 +145,7 @@ def _reset_task_instances(): def test_extra_links_works(dag_run, task_1, viewer_client, session): response = viewer_client.get( f"{ENDPOINT}?dag_id={task_1.dag_id}&task_id={task_1.task_id}" - f"&execution_date={STR_DEFAULT_DATE}&link_name=foo-bar", + f"&logical_date={STR_DEFAULT_DATE}&link_name=foo-bar", follow_redirects=True, ) @@ -159,7 +159,7 @@ def test_extra_links_works(dag_run, task_1, viewer_client, session): def test_global_extra_links_works(dag_run, task_1, viewer_client, session): response = viewer_client.get( f"{ENDPOINT}?dag_id={dag_run.dag_id}&task_id={task_1.task_id}" - f"&execution_date={STR_DEFAULT_DATE}&link_name=github", + f"&logical_date={STR_DEFAULT_DATE}&link_name=github", follow_redirects=True, ) @@ -173,7 +173,7 @@ def test_global_extra_links_works(dag_run, task_1, viewer_client, session): def test_operator_extra_link_override_global_extra_link(dag_run, task_1, viewer_client): response = viewer_client.get( f"{ENDPOINT}?dag_id={task_1.dag_id}&task_id={task_1.task_id}" - f"&execution_date={STR_DEFAULT_DATE}&link_name=airflow", + f"&logical_date={STR_DEFAULT_DATE}&link_name=airflow", follow_redirects=True, ) @@ -187,7 +187,7 @@ def test_operator_extra_link_override_global_extra_link(dag_run, task_1, viewer_ def test_extra_links_error_raised(dag_run, task_1, viewer_client): response = viewer_client.get( f"{ENDPOINT}?dag_id={task_1.dag_id}&task_id={task_1.task_id}" - f"&execution_date={STR_DEFAULT_DATE}&link_name=raise_error", + f"&logical_date={STR_DEFAULT_DATE}&link_name=raise_error", follow_redirects=True, ) @@ -201,7 +201,7 @@ def test_extra_links_error_raised(dag_run, task_1, viewer_client): def test_extra_links_no_response(dag_run, task_1, viewer_client): response = viewer_client.get( f"{ENDPOINT}?dag_id={task_1.dag_id}&task_id={task_1.task_id}" - f"&execution_date={STR_DEFAULT_DATE}&link_name=no_response", + f"&logical_date={STR_DEFAULT_DATE}&link_name=no_response", follow_redirects=True, ) @@ -222,7 +222,7 @@ def test_operator_extra_link_override_plugin(dag_run, task_2, viewer_client): """ response = viewer_client.get( f"{ENDPOINT}?dag_id={task_2.dag_id}&task_id={task_2.task_id}" - f"&execution_date={STR_DEFAULT_DATE}&link_name=airflow", + f"&logical_date={STR_DEFAULT_DATE}&link_name=airflow", follow_redirects=True, ) @@ -244,7 +244,7 @@ def test_operator_extra_link_multiple_operators(dag_run, task_2, task_3, viewer_ """ response = viewer_client.get( f"{ENDPOINT}?dag_id={task_2.dag_id}&task_id={task_2.task_id}" - f"&execution_date={STR_DEFAULT_DATE}&link_name=airflow", + f"&logical_date={STR_DEFAULT_DATE}&link_name=airflow", follow_redirects=True, ) @@ -256,7 +256,7 @@ def test_operator_extra_link_multiple_operators(dag_run, task_2, task_3, viewer_ response = viewer_client.get( f"{ENDPOINT}?dag_id={task_3.dag_id}&task_id={task_3.task_id}" - f"&execution_date={STR_DEFAULT_DATE}&link_name=airflow", + f"&logical_date={STR_DEFAULT_DATE}&link_name=airflow", follow_redirects=True, ) @@ -269,7 +269,7 @@ def test_operator_extra_link_multiple_operators(dag_run, task_2, task_3, viewer_ # Also check that the other Operator Link defined for this operator exists response = viewer_client.get( f"{ENDPOINT}?dag_id={task_3.dag_id}&task_id={task_3.task_id}" - f"&execution_date={STR_DEFAULT_DATE}&link_name=google", + f"&logical_date={STR_DEFAULT_DATE}&link_name=google", follow_redirects=True, ) diff --git a/tests/www/views/test_views_grid.py b/tests/www/views/test_views_grid.py index 830cd071abd6..c94539f05587 100644 --- a/tests/www/views/test_views_grid.py +++ b/tests/www/views/test_views_grid.py @@ -90,12 +90,12 @@ def mapped_task_group(arg1): def dag_with_runs(dag_without_runs): date = dag_without_runs.dag.start_date run_1 = dag_without_runs.create_dagrun( - run_id="run_1", state=DagRunState.SUCCESS, run_type=DagRunType.SCHEDULED, execution_date=date + run_id="run_1", state=DagRunState.SUCCESS, run_type=DagRunType.SCHEDULED, logical_date=date ) run_2 = dag_without_runs.create_dagrun( run_id="run_2", run_type=DagRunType.SCHEDULED, - execution_date=date + timedelta(days=1), + logical_date=date + timedelta(days=1), ) return run_1, run_2 @@ -160,7 +160,7 @@ def test_no_runs(admin_client, dag_without_runs): "instances": [], "label": None, }, - "ordering": ["data_interval_end", "execution_date"], + "ordering": ["data_interval_end", "logical_date"], "errors": [], } @@ -231,7 +231,7 @@ def test_one_run(admin_client, dag_with_runs: list[DagRun], session): "data_interval_end": "2016-01-02T00:00:00+00:00", "data_interval_start": "2016-01-01T00:00:00+00:00", "end_date": timezone.utcnow().isoformat(), - "execution_date": "2016-01-01T00:00:00+00:00", + "logical_date": "2016-01-01T00:00:00+00:00", "external_trigger": False, "last_scheduling_decision": None, "note": None, @@ -248,7 +248,7 @@ def test_one_run(admin_client, dag_with_runs: list[DagRun], session): "data_interval_end": "2016-01-03T00:00:00+00:00", "data_interval_start": "2016-01-02T00:00:00+00:00", "end_date": None, - "execution_date": "2016-01-02T00:00:00+00:00", + "logical_date": "2016-01-02T00:00:00+00:00", "external_trigger": False, "last_scheduling_decision": None, "note": None, @@ -409,7 +409,7 @@ def test_one_run(admin_client, dag_with_runs: list[DagRun], session): "instances": [], "label": None, }, - "ordering": ["data_interval_end", "execution_date"], + "ordering": ["data_interval_end", "logical_date"], "errors": [], } @@ -463,7 +463,7 @@ def _expected_task_details(task_id, has_outlet_assets): "instances": [], "label": None, }, - "ordering": ["data_interval_end", "execution_date"], + "ordering": ["data_interval_end", "logical_date"], "errors": [], } diff --git a/tests/www/views/test_views_log.py b/tests/www/views/test_views_log.py index 9c3a8d38ad2a..a757239c553b 100644 --- a/tests/www/views/test_views_log.py +++ b/tests/www/views/test_views_log.py @@ -57,7 +57,7 @@ TASK_ID = "task_for_testing_log_view" DEFAULT_DATE = timezone.datetime(2017, 9, 1) STR_DEFAULT_DATE = urllib.parse.quote(DEFAULT_DATE.strftime("%Y-%m-%dT%H:%M:%S.%f%z")) -ENDPOINT = f"log?dag_id={DAG_ID}&task_id={TASK_ID}&execution_date={STR_DEFAULT_DATE}" +ENDPOINT = f"log?dag_id={DAG_ID}&task_id={TASK_ID}&logical_date={STR_DEFAULT_DATE}" @pytest.fixture(scope="module", autouse=True) @@ -160,7 +160,7 @@ def tis(dags, session): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dagrun = dag.create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), start_date=DEFAULT_DATE, state=DagRunState.RUNNING, @@ -172,7 +172,7 @@ def tis(dags, session): ti.hostname = "localhost" dagrun_removed = dag_removed.create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), start_date=DEFAULT_DATE, state=DagRunState.RUNNING, @@ -257,7 +257,7 @@ def test_get_file_task_log(log_admin_client, tis, state, try_number, num_logs): def test_get_logs_with_metadata_as_download_file(log_admin_client, create_expected_log_file): url_template = ( "get_logs_with_metadata?dag_id={}&" - "task_id={}&execution_date={}&" + "task_id={}&logical_date={}&" "try_number={}&metadata={}&format=file" ) try_number = 1 @@ -289,7 +289,7 @@ def test_get_logs_with_metadata_as_download_file(log_admin_client, create_expect @pytest.fixture def dag_run_with_log_filename(tis): - run_filters = [DagRun.dag_id == DAG_ID, DagRun.execution_date == DEFAULT_DATE] + run_filters = [DagRun.dag_id == DAG_ID, DagRun.logical_date == DEFAULT_DATE] with create_session() as session: log_template = session.merge( LogTemplate(filename=DIFFERENT_LOG_FILENAME, elasticsearch_id="irrelevant") @@ -315,7 +315,7 @@ def test_get_logs_for_changed_filename_format_db( url = ( f"get_logs_with_metadata?dag_id={dag_run_with_log_filename.dag_id}&" f"task_id={TASK_ID}&" - f"execution_date={urllib.parse.quote_plus(dag_run_with_log_filename.logical_date.isoformat())}&" + f"logical_date={urllib.parse.quote_plus(dag_run_with_log_filename.logical_date.isoformat())}&" f"try_number={try_number}&metadata={{}}&format=file" ) response = log_admin_client.get(url) @@ -343,7 +343,7 @@ def test_get_logs_for_changed_filename_format_db( def test_get_logs_with_metadata_as_download_large_file(_, log_admin_client): url_template = ( "get_logs_with_metadata?dag_id={}&" - "task_id={}&execution_date={}&" + "task_id={}&logical_date={}&" "try_number={}&metadata={}&format=file" ) try_number = 1 @@ -365,7 +365,7 @@ def test_get_logs_with_metadata_as_download_large_file(_, log_admin_client): @pytest.mark.parametrize("metadata", ["null", "{}"]) def test_get_logs_with_metadata(log_admin_client, metadata, create_expected_log_file): - url_template = "get_logs_with_metadata?dag_id={}&task_id={}&execution_date={}&try_number={}&metadata={}" + url_template = "get_logs_with_metadata?dag_id={}&task_id={}&logical_date={}&try_number={}&metadata={}" try_number = 1 create_expected_log_file(try_number) response = log_admin_client.get( @@ -390,7 +390,7 @@ def test_get_logs_with_metadata(log_admin_client, metadata, create_expected_log_ def test_get_logs_with_invalid_metadata(log_admin_client): """Test invalid metadata JSON returns error message""" metadata = "invalid" - url_template = "get_logs_with_metadata?dag_id={}&task_id={}&execution_date={}&try_number={}&metadata={}" + url_template = "get_logs_with_metadata?dag_id={}&task_id={}&logical_date={}&try_number={}&metadata={}" response = log_admin_client.get( url_template.format( DAG_ID, @@ -412,7 +412,7 @@ def test_get_logs_with_invalid_metadata(log_admin_client): return_value=(["airflow log line"], [{"end_of_log": True}]), ) def test_get_logs_with_metadata_for_removed_dag(_, log_admin_client): - url_template = "get_logs_with_metadata?dag_id={}&task_id={}&execution_date={}&try_number={}&metadata={}" + url_template = "get_logs_with_metadata?dag_id={}&task_id={}&logical_date={}&try_number={}&metadata={}" response = log_admin_client.get( url_template.format( DAG_ID_REMOVED, @@ -435,7 +435,7 @@ def test_get_logs_with_metadata_for_removed_dag(_, log_admin_client): def test_get_logs_response_with_ti_equal_to_none(log_admin_client): url_template = ( "get_logs_with_metadata?dag_id={}&" - "task_id={}&execution_date={}&" + "task_id={}&logical_date={}&" "try_number={}&metadata={}&format=file" ) try_number = 1 @@ -457,7 +457,7 @@ def test_get_logs_response_with_ti_equal_to_none(log_admin_client): def test_get_logs_with_json_response_format(log_admin_client, create_expected_log_file): url_template = ( "get_logs_with_metadata?dag_id={}&" - "task_id={}&execution_date={}&" + "task_id={}&logical_date={}&" "try_number={}&metadata={}&format=json" ) try_number = 1 @@ -480,7 +480,7 @@ def test_get_logs_with_json_response_format(log_admin_client, create_expected_lo def test_get_logs_invalid_execution_data_format(log_admin_client): url_template = ( "get_logs_with_metadata?dag_id={}&" - "task_id={}&execution_date={}&" + "task_id={}&logical_date={}&" "try_number={}&metadata={}&format=file" ) try_number = 1 @@ -495,7 +495,7 @@ def test_get_logs_invalid_execution_data_format(log_admin_client): assert response.status_code == 400 assert response.json == { "error": ( - "Given execution date 'Tuesday February 27, 2024' could not be identified as a date. " + "Given logical date 'Tuesday February 27, 2024' could not be identified as a date. " "Example date format: 2015-11-16T14:34:15+00:00" ) } @@ -506,7 +506,7 @@ def test_get_logs_for_handler_without_read_method(mock_reader, log_admin_client) type(mock_reader.return_value).supports_read = unittest.mock.PropertyMock(return_value=False) url_template = ( "get_logs_with_metadata?dag_id={}&" - "task_id={}&execution_date={}&" + "task_id={}&logical_date={}&" "try_number={}&metadata={}&format=json" ) try_number = 1 @@ -529,7 +529,7 @@ def test_get_logs_for_handler_without_read_method(mock_reader, log_admin_client) @pytest.mark.parametrize("task_id", ["inexistent", TASK_ID]) def test_redirect_to_external_log_with_local_log_handler(log_admin_client, task_id): """Redirect to home if TI does not exist or if log handler is local""" - url_template = "redirect_to_external_log?dag_id={}&task_id={}&execution_date={}&try_number={}" + url_template = "redirect_to_external_log?dag_id={}&task_id={}&logical_date={}&try_number={}" try_number = 1 url = url_template.format( DAG_ID, @@ -563,7 +563,7 @@ def supports_external_link(self) -> bool: return_value=_ExternalHandler(), ) def test_redirect_to_external_log_with_external_log_handler(_, log_admin_client): - url_template = "redirect_to_external_log?dag_id={}&task_id={}&execution_date={}&try_number={}" + url_template = "redirect_to_external_log?dag_id={}&task_id={}&logical_date={}&try_number={}" try_number = 1 url = url_template.format( DAG_ID, diff --git a/tests/www/views/test_views_rendered.py b/tests/www/views/test_views_rendered.py index 00f2a0c7d163..5b10d12f17c8 100644 --- a/tests/www/views/test_views_rendered.py +++ b/tests/www/views/test_views_rendered.py @@ -143,12 +143,12 @@ def _reset_db(dag, task1, task2, task3, task4, task_secret): @pytest.fixture def create_dag_run(dag, task1, task2, task3, task4, task_secret): - def _create_dag_run(*, execution_date, session): + def _create_dag_run(*, logical_date, session): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dag_run = dag.create_dagrun( state=DagRunState.RUNNING, - execution_date=execution_date, - data_interval=(execution_date, execution_date), + logical_date=logical_date, + data_interval=(logical_date, logical_date), run_type=DagRunType.SCHEDULED, session=session, **triggered_by_kwargs, @@ -184,13 +184,13 @@ def test_rendered_template_view(admin_client, create_dag_run, task1): assert task1.bash_command == "{{ task_instance_key_str }}" with create_session() as session: - dag_run = create_dag_run(execution_date=DEFAULT_DATE, session=session) + dag_run = create_dag_run(logical_date=DEFAULT_DATE, session=session) ti = dag_run.get_task_instance(task1.task_id, session=session) assert ti is not None, "task instance not found" ti.refresh_from_task(task1) session.add(RenderedTaskInstanceFields(ti)) - url = f"rendered-templates?task_id=task1&dag_id=testdag&execution_date={quote_plus(str(DEFAULT_DATE))}" + url = f"rendered-templates?task_id=task1&dag_id=testdag&logical_date={quote_plus(str(DEFAULT_DATE))}" resp = admin_client.get(url, follow_redirects=True) check_content_in_response("testdag__task1__20200301", resp) @@ -205,9 +205,9 @@ def test_rendered_template_view_for_unexecuted_tis(admin_client, create_dag_run, assert task1.bash_command == "{{ task_instance_key_str }}" with create_session() as session: - create_dag_run(execution_date=DEFAULT_DATE, session=session) + create_dag_run(logical_date=DEFAULT_DATE, session=session) - url = f"rendered-templates?task_id=task1&dag_id=testdag&execution_date={quote_plus(str(DEFAULT_DATE))}" + url = f"rendered-templates?task_id=task1&dag_id=testdag&logical_date={quote_plus(str(DEFAULT_DATE))}" resp = admin_client.get(url, follow_redirects=True) check_content_in_response("testdag__task1__20200301", resp) @@ -218,9 +218,9 @@ def test_user_defined_filter_and_macros_raise_error(admin_client, create_dag_run assert task2.bash_command == 'echo {{ fullname("Apache", "Airflow") | hello }}' with create_session() as session: - create_dag_run(execution_date=DEFAULT_DATE, session=session) + create_dag_run(logical_date=DEFAULT_DATE, session=session) - url = f"rendered-templates?task_id=task2&dag_id=testdag&execution_date={quote_plus(str(DEFAULT_DATE))}" + url = f"rendered-templates?task_id=task2&dag_id=testdag&logical_date={quote_plus(str(DEFAULT_DATE))}" resp = admin_client.get(url, follow_redirects=True) assert resp.status_code == 200 @@ -249,14 +249,14 @@ def test_rendered_template_secret(admin_client, create_dag_run, task_secret): assert task_secret.bash_command == "echo {{ var.value.my_secret }} && echo {{ var.value.spam }}" with create_session() as session: - dag_run = create_dag_run(execution_date=DEFAULT_DATE, session=session) + dag_run = create_dag_run(logical_date=DEFAULT_DATE, session=session) ti = dag_run.get_task_instance(task_secret.task_id, session=session) assert ti is not None, "task instance not found" ti.refresh_from_task(task_secret) assert ti.state == TaskInstanceState.QUEUED date = quote_plus(str(DEFAULT_DATE)) - url = f"rendered-templates?task_id=task_secret&dag_id=testdag&execution_date={date}" + url = f"rendered-templates?task_id=task_secret&dag_id=testdag&logical_date={date}" resp = admin_client.get(url, follow_redirects=True) check_content_in_response("***", resp) @@ -337,13 +337,13 @@ def test_rendered_task_detail_env_secret(patch_app, admin_client, request, env, task_secret: BashOperator = dag.get_task(task_id="task1") task_secret.env = env date = quote_plus(str(DEFAULT_DATE)) - url = f"task?task_id=task1&dag_id=testdag&execution_date={date}" + url = f"task?task_id=task1&dag_id=testdag&logical_date={date}" with create_session() as session: triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} dag.create_dagrun( state=DagRunState.RUNNING, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), run_type=DagRunType.SCHEDULED, session=session, @@ -366,9 +366,9 @@ def test_rendered_template_view_for_list_template_field_args(admin_client, creat assert task3.sql == ["SELECT 1;", "SELECT 2;"] with create_session() as session: - create_dag_run(execution_date=DEFAULT_DATE, session=session) + create_dag_run(logical_date=DEFAULT_DATE, session=session) - url = f"rendered-templates?task_id=task3&dag_id=testdag&execution_date={quote_plus(str(DEFAULT_DATE))}" + url = f"rendered-templates?task_id=task3&dag_id=testdag&logical_date={quote_plus(str(DEFAULT_DATE))}" resp = admin_client.get(url, follow_redirects=True) check_content_in_response("List item #0", resp) @@ -384,9 +384,9 @@ def test_rendered_template_view_for_op_args(admin_client, create_dag_run, task4) assert list(task4.op_kwargs.values()) == ["{{ task_instance_key_str }}_kwargs"] with create_session() as session: - create_dag_run(execution_date=DEFAULT_DATE, session=session) + create_dag_run(logical_date=DEFAULT_DATE, session=session) - url = f"rendered-templates?task_id=task4&dag_id=testdag&execution_date={quote_plus(str(DEFAULT_DATE))}" + url = f"rendered-templates?task_id=task4&dag_id=testdag&logical_date={quote_plus(str(DEFAULT_DATE))}" resp = admin_client.get(url, follow_redirects=True) check_content_in_response("testdag__task4__20200301_args", resp) diff --git a/tests/www/views/test_views_task_norun.py b/tests/www/views/test_views_task_norun.py index 4cb195461fcc..ffc08a627c75 100644 --- a/tests/www/views/test_views_task_norun.py +++ b/tests/www/views/test_views_task_norun.py @@ -38,7 +38,7 @@ def _reset_dagruns(): def test_task_view_no_task_instance(admin_client): - url = f"/task?task_id=runme_0&dag_id=example_bash_operator&execution_date={DEFAULT_VAL}" + url = f"/task?task_id=runme_0&dag_id=example_bash_operator&logical_date={DEFAULT_VAL}" resp = admin_client.get(url, follow_redirects=True) assert resp.status_code == 200 html = resp.data.decode("utf-8") @@ -47,7 +47,7 @@ def test_task_view_no_task_instance(admin_client): def test_rendered_templates_view_no_task_instance(admin_client): - url = f"/rendered-templates?task_id=runme_0&dag_id=example_bash_operator&execution_date={DEFAULT_VAL}" + url = f"/rendered-templates?task_id=runme_0&dag_id=example_bash_operator&logical_date={DEFAULT_VAL}" resp = admin_client.get(url, follow_redirects=True) assert resp.status_code == 200 html = resp.data.decode("utf-8") diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index 424fb02979cd..05b55a9f4120 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -84,7 +84,7 @@ def _init_dagruns(app): app.dag_bag.get_dag("example_bash_operator").create_dagrun( run_id=DEFAULT_DAGRUN, run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), start_date=timezone.utcnow(), state=State.RUNNING, @@ -93,7 +93,7 @@ def _init_dagruns(app): app.dag_bag.get_dag("example_python_operator").create_dagrun( run_id=DEFAULT_DAGRUN, run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), start_date=timezone.utcnow(), state=State.RUNNING, @@ -109,7 +109,7 @@ def _init_dagruns(app): app.dag_bag.get_dag("example_xcom").create_dagrun( run_id=DEFAULT_DAGRUN, run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), start_date=timezone.utcnow(), state=State.RUNNING, @@ -118,7 +118,7 @@ def _init_dagruns(app): app.dag_bag.get_dag("latest_only").create_dagrun( run_id=DEFAULT_DAGRUN, run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), start_date=timezone.utcnow(), state=State.RUNNING, @@ -127,7 +127,7 @@ def _init_dagruns(app): app.dag_bag.get_dag("example_task_group").create_dagrun( run_id=DEFAULT_DAGRUN, run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), start_date=timezone.utcnow(), state=State.RUNNING, @@ -178,38 +178,38 @@ def client_ti_without_dag_edit(app): id="delete-dag-button-normal", ), pytest.param( - f"task?task_id=runme_0&dag_id=example_bash_operator&execution_date={DEFAULT_VAL}", + f"task?task_id=runme_0&dag_id=example_bash_operator&logical_date={DEFAULT_VAL}", ["Grid"], id="task-grid-button", ), pytest.param( - f"task?task_id=runme_0&dag_id=example_bash_operator&execution_date={DEFAULT_VAL}", + f"task?task_id=runme_0&dag_id=example_bash_operator&logical_date={DEFAULT_VAL}", ["Task Instance Details"], id="task", ), pytest.param( - f"log?task_id=runme_0&dag_id=example_bash_operator&execution_date={DEFAULT_VAL}", + f"log?task_id=runme_0&dag_id=example_bash_operator&logical_date={DEFAULT_VAL}", ["Grid"], id="log-grid-button", ), pytest.param( - f"xcom?task_id=runme_0&dag_id=example_bash_operator&execution_date={DEFAULT_VAL}", + f"xcom?task_id=runme_0&dag_id=example_bash_operator&logical_date={DEFAULT_VAL}", ["XCom"], id="xcom", ), pytest.param( - f"xcom?task_id=runme_0&dag_id=example_bash_operator&execution_date={DEFAULT_VAL}", + f"xcom?task_id=runme_0&dag_id=example_bash_operator&logical_date={DEFAULT_VAL}", ["Grid"], id="xcom-grid-button", ), pytest.param("xcom/list", ["List XComs"], id="xcom-list"), pytest.param( - f"rendered-templates?task_id=runme_0&dag_id=example_bash_operator&execution_date={DEFAULT_VAL}", + f"rendered-templates?task_id=runme_0&dag_id=example_bash_operator&logical_date={DEFAULT_VAL}", ["Rendered Template"], id="rendered-templates", ), pytest.param( - f"rendered-templates?task_id=runme_0&dag_id=example_bash_operator&execution_date={DEFAULT_VAL}", + f"rendered-templates?task_id=runme_0&dag_id=example_bash_operator&logical_date={DEFAULT_VAL}", ["Grid"], id="rendered-templates-grid-button", ), @@ -362,7 +362,7 @@ def test_views_get(admin_client, url, contents): def test_xcom_return_value_is_not_bytes(admin_client): - url = f"xcom?dag_id=example_bash_operator&task_id=runme_0&execution_date={DEFAULT_VAL}&map_index=-1" + url = f"xcom?dag_id=example_bash_operator&task_id=runme_0&logical_date={DEFAULT_VAL}&map_index=-1" resp = admin_client.get(url, follow_redirects=True) # check that {"x":1} is in the response content = "{'x':1}" @@ -373,7 +373,7 @@ def test_xcom_return_value_is_not_bytes(admin_client): def test_rendered_task_view(admin_client): - url = f"task?task_id=runme_0&dag_id=example_bash_operator&execution_date={DEFAULT_VAL}" + url = f"task?task_id=runme_0&dag_id=example_bash_operator&logical_date={DEFAULT_VAL}" resp = admin_client.get(url, follow_redirects=True) resp_html = resp.data.decode("utf-8") assert resp.status_code == 200 @@ -382,7 +382,7 @@ def test_rendered_task_view(admin_client): def test_rendered_k8s(admin_client): - url = f"rendered-k8s?task_id=runme_0&dag_id=example_bash_operator&execution_date={DEFAULT_VAL}" + url = f"rendered-k8s?task_id=runme_0&dag_id=example_bash_operator&logical_date={DEFAULT_VAL}" with unittest.mock.patch.object(settings, "IS_K8S_OR_K8SCELERY_EXECUTOR", True): resp = admin_client.get(url, follow_redirects=True) check_content_in_response("K8s Pod Spec", resp) @@ -390,7 +390,7 @@ def test_rendered_k8s(admin_client): @conf_vars({("core", "executor"): "LocalExecutor"}) def test_rendered_k8s_without_k8s(admin_client): - url = f"rendered-k8s?task_id=runme_0&dag_id=example_bash_operator&execution_date={DEFAULT_VAL}" + url = f"rendered-k8s?task_id=runme_0&dag_id=example_bash_operator&logical_date={DEFAULT_VAL}" resp = admin_client.get(url, follow_redirects=True) assert 404 == resp.status_code @@ -399,7 +399,7 @@ def test_tree_trigger_origin_tree_view(app, admin_client): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} app.dag_bag.get_dag("test_tree_view").create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), start_date=timezone.utcnow(), state=State.RUNNING, @@ -417,7 +417,7 @@ def test_graph_trigger_origin_grid_view(app, admin_client): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} app.dag_bag.get_dag("test_tree_view").create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), start_date=timezone.utcnow(), state=State.RUNNING, @@ -435,7 +435,7 @@ def test_gantt_trigger_origin_grid_view(app, admin_client): triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {} app.dag_bag.get_dag("test_tree_view").create_dagrun( run_type=DagRunType.SCHEDULED, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, data_interval=(DEFAULT_DATE, DEFAULT_DATE), start_date=timezone.utcnow(), state=State.RUNNING, @@ -554,7 +554,7 @@ def test_code_from_db_all_example_dags(admin_client): dict( task_id="runme_1", dag_id="example_bash_operator", - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, upstream="false", downstream="false", future="false", @@ -568,7 +568,7 @@ def test_code_from_db_all_example_dags(admin_client): dict( group_id="section_1", dag_id="example_task_group", - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, upstream="false", downstream="false", future="false", @@ -597,7 +597,7 @@ def test_dag_never_run(admin_client, url): form = dict( task_id="run_this_last", dag_id=dag_id, - execution_date=DEFAULT_DATE, + logical_date=DEFAULT_DATE, upstream="false", downstream="false", future="false", @@ -797,7 +797,7 @@ def _get_appbuilder_pk_string(model_view_cls, instance) -> str: def test_task_instance_delete(session, admin_client, create_task_instance): task_instance_to_delete = create_task_instance( task_id="test_task_instance_delete", - execution_date=timezone.utcnow(), + logical_date=timezone.utcnow(), state=State.DEFERRED, ) composite_key = _get_appbuilder_pk_string(TaskInstanceModelView, task_instance_to_delete) @@ -811,7 +811,7 @@ def test_task_instance_delete(session, admin_client, create_task_instance): def test_task_instance_delete_permission_denied(session, client_ti_without_dag_edit, create_task_instance): task_instance_to_delete = create_task_instance( task_id="test_task_instance_delete_permission_denied", - execution_date=timezone.utcnow(), + logical_date=timezone.utcnow(), state=State.DEFERRED, session=session, ) @@ -875,7 +875,7 @@ def test_task_instance_clear_downstream(session, admin_client, dag_maker): run_id="run_1", state=DagRunState.SUCCESS, run_type=DagRunType.SCHEDULED, - execution_date=dag_maker.dag.start_date, + logical_date=dag_maker.dag.start_date, start_date=dag_maker.dag.start_date, session=session, **triggered_by_kwargs, @@ -885,7 +885,7 @@ def test_task_instance_clear_downstream(session, admin_client, dag_maker): run_id="run_2", state=DagRunState.SUCCESS, run_type=DagRunType.SCHEDULED, - execution_date=dag_maker.dag.start_date.add(days=1), + logical_date=dag_maker.dag.start_date.add(days=1), start_date=dag_maker.dag.start_date.add(days=1), session=session, **triggered_by_kwargs, @@ -1054,10 +1054,10 @@ def test_get_date_time_num_runs_dag_runs_form_data_graph_view(app, dag_maker, ad """Test the get_date_time_num_runs_dag_runs_form_data function.""" from airflow.www.views import get_date_time_num_runs_dag_runs_form_data - execution_date = pendulum.now(tz="UTC") + logical_date = pendulum.now(tz="UTC") with dag_maker( dag_id="test_get_date_time_num_runs_dag_runs_form_data", - start_date=execution_date, + start_date=logical_date, ) as dag: BashOperator(task_id="task_1", bash_command="echo test") @@ -1073,16 +1073,16 @@ def test_get_date_time_num_runs_dag_runs_form_data_graph_view(app, dag_maker, ad dttm = pendulum.parse(data["dttm"].isoformat()) base_date = pendulum.parse(data["base_date"].isoformat()) - assert dttm.date() == execution_date.date() - assert dttm.time().hour == _safe_parse_datetime(execution_date.time().isoformat()).time().hour - assert dttm.time().minute == _safe_parse_datetime(execution_date.time().isoformat()).time().minute - assert base_date.date() == execution_date.date() + assert dttm.date() == logical_date.date() + assert dttm.time().hour == _safe_parse_datetime(logical_date.time().isoformat()).time().hour + assert dttm.time().minute == _safe_parse_datetime(logical_date.time().isoformat()).time().minute + assert base_date.date() == logical_date.date() def test_task_instances(admin_client): """Test task_instances view.""" resp = admin_client.get( - f"/object/task_instances?dag_id=example_bash_operator&execution_date={STR_DEFAULT_DATE}", + f"/object/task_instances?dag_id=example_bash_operator&logical_date={STR_DEFAULT_DATE}", follow_redirects=True, ) assert resp.status_code == 200 @@ -1092,7 +1092,7 @@ def test_task_instances(admin_client): "dag_id": "example_bash_operator", "duration": None, "end_date": None, - "execution_date": DEFAULT_DATE.isoformat(), + "logical_date": DEFAULT_DATE.isoformat(), "executor": None, "executor_config": {}, "external_executor_id": None, @@ -1129,7 +1129,7 @@ def test_task_instances(admin_client): "dag_id": "example_bash_operator", "duration": None, "end_date": None, - "execution_date": DEFAULT_DATE.isoformat(), + "logical_date": DEFAULT_DATE.isoformat(), "executor": None, "executor_config": {}, "external_executor_id": None, @@ -1166,7 +1166,7 @@ def test_task_instances(admin_client): "dag_id": "example_bash_operator", "duration": None, "end_date": None, - "execution_date": DEFAULT_DATE.isoformat(), + "logical_date": DEFAULT_DATE.isoformat(), "executor": None, "executor_config": {}, "external_executor_id": None, @@ -1203,7 +1203,7 @@ def test_task_instances(admin_client): "dag_id": "example_bash_operator", "duration": None, "end_date": None, - "execution_date": DEFAULT_DATE.isoformat(), + "logical_date": DEFAULT_DATE.isoformat(), "executor": None, "executor_config": {}, "external_executor_id": None, @@ -1240,7 +1240,7 @@ def test_task_instances(admin_client): "dag_id": "example_bash_operator", "duration": None, "end_date": None, - "execution_date": DEFAULT_DATE.isoformat(), + "logical_date": DEFAULT_DATE.isoformat(), "executor": None, "executor_config": {}, "external_executor_id": None, @@ -1277,7 +1277,7 @@ def test_task_instances(admin_client): "dag_id": "example_bash_operator", "duration": None, "end_date": None, - "execution_date": DEFAULT_DATE.isoformat(), + "logical_date": DEFAULT_DATE.isoformat(), "executor": None, "executor_config": {}, "external_executor_id": None, @@ -1314,7 +1314,7 @@ def test_task_instances(admin_client): "dag_id": "example_bash_operator", "duration": None, "end_date": None, - "execution_date": DEFAULT_DATE.isoformat(), + "logical_date": DEFAULT_DATE.isoformat(), "executor": None, "executor_config": {}, "external_executor_id": None, diff --git a/tests/www/views/test_views_trigger_dag.py b/tests/www/views/test_views_trigger_dag.py index 8955bcc16765..17d0b687b857 100644 --- a/tests/www/views/test_views_trigger_dag.py +++ b/tests/www/views/test_views_trigger_dag.py @@ -143,25 +143,25 @@ def test_trigger_dag_conf_not_dict(admin_client): assert run is None -def test_trigger_dag_wrong_execution_date(admin_client): +def test_trigger_dag_wrong_logical_date(admin_client): test_dag_id = "example_bash_operator" response = admin_client.post( - f"dags/{test_dag_id}/trigger", data={"conf": "{}", "execution_date": "not_a_date"} + f"dags/{test_dag_id}/trigger", data={"conf": "{}", "logical_date": "not_a_date"} ) - check_content_in_response("Invalid execution date", response) + check_content_in_response("Invalid logical date", response) with create_session() as session: run = session.query(DagRun).filter(DagRun.dag_id == test_dag_id).first() assert run is None -def test_trigger_dag_execution_date_data_interval(admin_client): +def test_trigger_dag_logical_date_data_interval(admin_client): test_dag_id = "example_bash_operator" exec_date = timezone.utcnow() admin_client.post( - f"dags/{test_dag_id}/trigger", data={"conf": "{}", "execution_date": exec_date.isoformat()} + f"dags/{test_dag_id}/trigger", data={"conf": "{}", "logical_date": exec_date.isoformat()} ) with create_session() as session: @@ -169,7 +169,7 @@ def test_trigger_dag_execution_date_data_interval(admin_client): assert run is not None assert DagRunType.MANUAL in run.run_id assert run.run_type == DagRunType.MANUAL - assert run.execution_date == exec_date + assert run.logical_date == exec_date # Since example_bash_operator runs once per day, the data interval should be # between midnight yesterday and midnight today. diff --git a/tests/www/views/test_views_variable.py b/tests/www/views/test_views_variable.py index aab164b74b75..f085d5021a85 100644 --- a/tests/www/views/test_views_variable.py +++ b/tests/www/views/test_views_variable.py @@ -129,7 +129,7 @@ def test_import_variables_success(session, admin_client): "/variable/varimport", data={"file": (bytes_content, "test.json")}, follow_redirects=True ) check_content_in_response("4 variable(s) successfully updated.", resp) - _check_last_log(session, dag_id=None, event="variables.varimport", execution_date=None) + _check_last_log(session, dag_id=None, event="variables.varimport", logical_date=None) def test_import_variables_override_existing_variables_if_set(session, admin_client, caplog): @@ -144,7 +144,7 @@ def test_import_variables_override_existing_variables_if_set(session, admin_clie follow_redirects=True, ) check_content_in_response("2 variable(s) successfully updated.", resp) - _check_last_log(session, dag_id=None, event="variables.varimport", execution_date=None) + _check_last_log(session, dag_id=None, event="variables.varimport", logical_date=None) def test_import_variables_skips_update_if_set(session, admin_client, caplog): @@ -163,7 +163,7 @@ def test_import_variables_skips_update_if_set(session, admin_client, caplog): check_content_in_response( "The variables with these keys: 'str_key' were skipped because they already exists", resp ) - _check_last_log(session, dag_id=None, event="variables.varimport", execution_date=None) + _check_last_log(session, dag_id=None, event="variables.varimport", logical_date=None) assert "Variable: str_key already exists, skipping." in caplog.text diff --git a/tests_common/pytest_plugin.py b/tests_common/pytest_plugin.py index a27d4e0fa86d..54d9e485e1eb 100644 --- a/tests_common/pytest_plugin.py +++ b/tests_common/pytest_plugin.py @@ -834,6 +834,8 @@ def dag_maker(request) -> Generator[DagMaker, None, None]: from airflow.utils.log.logging_mixin import LoggingMixin + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + class DagFactory(LoggingMixin, DagMaker): _own_session = False @@ -886,8 +888,6 @@ def __exit__(self, type, value, traceback): from airflow.models import DagModel from airflow.models.serialized_dag import SerializedDagModel - from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS - dag = self.dag dag.__exit__(type, value, traceback) if type is not None: @@ -902,7 +902,6 @@ def __exit__(self, type, value, traceback): dag, processor_subdir=self.dag_model.processor_subdir ) sdm = SerializedDagModel.get(dag.dag_id, session=self.session) - from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS and not sdm: from airflow.models.dag_version import DagVersion @@ -936,8 +935,6 @@ def create_dagrun(self, **kwargs): from airflow.utils.state import State from airflow.utils.types import DagRunType - from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType @@ -955,13 +952,24 @@ def create_dagrun(self, **kwargs): if "run_type" not in kwargs: kwargs["run_type"] = DagRunType.from_run_id(kwargs["run_id"]) - if kwargs.get("execution_date") is None: - if kwargs["run_type"] == DagRunType.MANUAL: - kwargs["execution_date"] = self.start_date - else: - kwargs["execution_date"] = dag.next_dagrun_info(None).logical_date + if AIRFLOW_V_3_0_PLUS: + if kwargs.get("logical_date") is None: + if kwargs["run_type"] == DagRunType.MANUAL: + kwargs["logical_date"] = self.start_date + else: + kwargs["logical_date"] = dag.next_dagrun_info(None).logical_date + else: + if kwargs.get("execution_date") is None: + if kwargs["run_type"] == DagRunType.MANUAL: + kwargs["execution_date"] = self.start_date + else: + kwargs["execution_date"] = dag.next_dagrun_info(None).logical_date if "data_interval" not in kwargs: - logical_date = timezone.coerce_datetime(kwargs["execution_date"]) + logical_date = ( + timezone.coerce_datetime(kwargs["logical_date"]) + if AIRFLOW_V_3_0_PLUS + else timezone.coerce_datetime(kwargs["execution_date"]) + ) if kwargs["run_type"] == DagRunType.MANUAL: data_interval = dag.timetable.infer_manual_data_interval(run_after=logical_date) else: @@ -981,10 +989,18 @@ def create_dagrun_after(self, dagrun, **kwargs): next_info = self.dag.next_dagrun_info(self.dag.get_run_data_interval(dagrun)) if next_info is None: raise ValueError(f"cannot create run after {dagrun}") - return self.create_dagrun( - execution_date=next_info.logical_date, - data_interval=next_info.data_interval, - **kwargs, + return ( + self.create_dagrun( + logical_date=next_info.logical_date, + data_interval=next_info.data_interval, + **kwargs, + ) + if AIRFLOW_V_3_0_PLUS + else self.create_dagrun( + execution_date=next_info.logical_date, + data_interval=next_info.data_interval, + **kwargs, + ) ) def __call__( @@ -1045,7 +1061,6 @@ def cleanup(self): return # To isolate problems here with problems from elsewhere on the session object self.session.rollback() - from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS if AIRFLOW_V_3_0_PLUS: from airflow.models.dag_version import DagVersion @@ -1190,7 +1205,7 @@ class CreateTaskInstance(Protocol): def __call__( self, *, - execution_date: datetime = ..., + logical_date: datetime = ..., dagrun_state: DagRunState = ..., state: TaskInstanceState = ..., run_id: str = ..., @@ -1224,8 +1239,10 @@ def create_task_instance(dag_maker: DagMaker, create_dummy_dag: CreateDummyDAG) """ from airflow.operators.empty import EmptyOperator + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + def maker( - execution_date=None, + logical_date=None, dagrun_state=None, state=None, run_id=None, @@ -1251,15 +1268,13 @@ def maker( last_heartbeat_at=None, **kwargs, ) -> TaskInstance: - from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS - if AIRFLOW_V_3_0_PLUS: from airflow.utils.types import DagRunTriggeredByType - if execution_date is None: + if logical_date is None: from airflow.utils import timezone - execution_date = timezone.utcnow() + logical_date = timezone.utcnow() with dag_maker(dag_id, **kwargs): op_kwargs = {} from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS @@ -1280,9 +1295,9 @@ def maker( trigger_rule=trigger_rule, **op_kwargs, ) - + date_key = "logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date" dagrun_kwargs = { - "execution_date": execution_date, + date_key: logical_date, "state": dagrun_state, } dagrun_kwargs.update({"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {}) @@ -1309,20 +1324,22 @@ def maker( @pytest.fixture def create_serialized_task_instance_of_operator(dag_maker: DagMaker): + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + def _create_task_instance( operator_class, *, dag_id, - execution_date=None, + logical_date=None, session=None, **operator_kwargs, ) -> TaskInstance: with dag_maker(dag_id=dag_id, serialized=True, session=session): operator_class(**operator_kwargs) - if execution_date is None: + if logical_date is None: dagrun_kwargs = {} else: - dagrun_kwargs = {"execution_date": execution_date} + dagrun_kwargs = {"logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date": logical_date} (ti,) = dag_maker.create_dagrun(**dagrun_kwargs).task_instances return ti @@ -1337,20 +1354,22 @@ def __call__(self, operator_class: type[BaseOperator], *args, **kwargs) -> TaskI @pytest.fixture def create_task_instance_of_operator(dag_maker: DagMaker) -> CreateTaskInstanceOfOperator: + from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS + def _create_task_instance( operator_class, *, dag_id, - execution_date=None, + logical_date=None, session=None, **operator_kwargs, ) -> TaskInstance: with dag_maker(dag_id=dag_id, session=session, serialized=True): operator_class(**operator_kwargs) - if execution_date is None: + if logical_date is None: dagrun_kwargs = {} else: - dagrun_kwargs = {"execution_date": execution_date} + dagrun_kwargs = {"logical_date" if AIRFLOW_V_3_0_PLUS else "execution_date": logical_date} (ti,) = dag_maker.create_dagrun(**dagrun_kwargs).task_instances return ti diff --git a/tests_common/test_utils/www.py b/tests_common/test_utils/www.py index e9489fcb9ef9..0e3fd43def88 100644 --- a/tests_common/test_utils/www.py +++ b/tests_common/test_utils/www.py @@ -67,20 +67,20 @@ def check_content_not_in_response(text, resp, resp_code=200): assert text not in resp_html -def _check_last_log(session, dag_id, event, execution_date, expected_extra=None): +def _check_last_log(session, dag_id, event, logical_date, expected_extra=None): logs = ( session.query( Log.dag_id, Log.task_id, Log.event, - Log.execution_date, + Log.logical_date, Log.owner, Log.extra, ) .filter( Log.dag_id == dag_id, Log.event == event, - Log.execution_date == execution_date, + Log.logical_date == logical_date, ) .order_by(Log.dttm.desc()) .limit(5) @@ -93,20 +93,20 @@ def _check_last_log(session, dag_id, event, execution_date, expected_extra=None) session.query(Log).delete() -def _check_last_log_masked_connection(session, dag_id, event, execution_date): +def _check_last_log_masked_connection(session, dag_id, event, logical_date): logs = ( session.query( Log.dag_id, Log.task_id, Log.event, - Log.execution_date, + Log.logical_date, Log.owner, Log.extra, ) .filter( Log.dag_id == dag_id, Log.event == event, - Log.execution_date == execution_date, + Log.logical_date == logical_date, ) .order_by(Log.dttm.desc()) .limit(5) @@ -126,20 +126,20 @@ def _check_last_log_masked_connection(session, dag_id, event, execution_date): } -def _check_last_log_masked_variable(session, dag_id, event, execution_date): +def _check_last_log_masked_variable(session, dag_id, event, logical_date): logs = ( session.query( Log.dag_id, Log.task_id, Log.event, - Log.execution_date, + Log.logical_date, Log.owner, Log.extra, ) .filter( Log.dag_id == dag_id, Log.event == event, - Log.execution_date == execution_date, + Log.logical_date == logical_date, ) .order_by(Log.dttm.desc()) .limit(5)