Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

AIP-84: Migrating GET queued asset events for DAG to fastAPI #43934

Merged
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions airflow/api_connexion/endpoints/asset_endpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,7 @@ def delete_dag_asset_queued_event(
)


@mark_fastapi_migration_done
@security.requires_access_asset("GET")
@security.requires_access_dag("GET")
@provide_session
Expand Down
15 changes: 15 additions & 0 deletions airflow/api_fastapi/core_api/datamodels/dags.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,3 +159,18 @@ class DAGTagCollectionResponse(BaseModel):

tags: list[str]
total_entries: int


class QueuedEventResponse(BaseModel):
"""QueuedEvent serializer for responses.."""
amoghrajesh marked this conversation as resolved.
Show resolved Hide resolved

uri: str
dag_id: str
created_at: datetime


class QueuedEventCollectionResponse(BaseModel):
"""QueuedEventCollection serializer for responses."""
amoghrajesh marked this conversation as resolved.
Show resolved Hide resolved

queued_events: list[QueuedEventResponse]
total_entries: int
86 changes: 86 additions & 0 deletions airflow/api_fastapi/core_api/openapi/v1-generated.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1864,6 +1864,57 @@ paths:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/public/dags/{dag_id}/assets/queuedEvent:
get:
tags:
- DAG
summary: Get Dag Asset Queued Events
description: Get queued asset events for a DAG.
operationId: get_dag_asset_queued_events
parameters:
- name: dag_id
in: path
required: true
schema:
type: string
title: Dag Id
- name: before
in: query
required: false
schema:
type: string
title: Before
responses:
'200':
description: Successful Response
content:
application/json:
schema:
$ref: '#/components/schemas/QueuedEventCollectionResponse'
'401':
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPExceptionResponse'
description: Unauthorized
'403':
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPExceptionResponse'
description: Forbidden
'404':
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPExceptionResponse'
description: Not Found
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/public/eventLogs/{event_log_id}:
get:
tags:
Expand Down Expand Up @@ -5473,6 +5524,41 @@ components:
- version
title: ProviderResponse
description: Provider serializer for responses.
QueuedEventCollectionResponse:
properties:
queued_events:
items:
$ref: '#/components/schemas/QueuedEventResponse'
type: array
title: Queued Events
total_entries:
type: integer
title: Total Entries
type: object
required:
- queued_events
- total_entries
title: QueuedEventCollectionResponse
description: QueuedEventCollection serializer for responses.
QueuedEventResponse:
properties:
uri:
type: string
title: Uri
dag_id:
type: string
title: Dag Id
created_at:
type: string
format: date-time
title: Created At
type: object
required:
- uri
- dag_id
- created_at
title: QueuedEventResponse
description: QueuedEvent serializer for responses..
ReprocessBehavior:
type: string
enum:
Expand Down
46 changes: 46 additions & 0 deletions airflow/api_fastapi/core_api/routes/public/dags.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,14 @@
DAGPatchBody,
DAGResponse,
DAGTagCollectionResponse,
QueuedEventCollectionResponse,
QueuedEventResponse,
)
from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc
from airflow.exceptions import AirflowException, DagNotFound
from airflow.models import DAG, DagModel, DagTag
from airflow.models.asset import AssetDagRunQueue, AssetModel
from airflow.utils import timezone

dags_router = AirflowRouter(tags=["DAG"], prefix="/dags")

Expand Down Expand Up @@ -303,3 +307,45 @@ def delete_dag(
status.HTTP_409_CONFLICT, f"Task instances of dag with id: '{dag_id}' are still running"
)
return Response(status_code=status.HTTP_204_NO_CONTENT)


@dags_router.get(
"/{dag_id}/assets/queuedEvent",
responses=create_openapi_http_exception_doc(
[
pierrejeambrun marked this conversation as resolved.
Show resolved Hide resolved
status.HTTP_401_UNAUTHORIZED,
status.HTTP_403_FORBIDDEN,
amoghrajesh marked this conversation as resolved.
Show resolved Hide resolved
status.HTTP_404_NOT_FOUND,
]
),
)
def get_dag_asset_queued_events(
dag_id: str,
session: Annotated[Session, Depends(get_session)],
before: str = Query(None),
pierrejeambrun marked this conversation as resolved.
Show resolved Hide resolved
) -> QueuedEventCollectionResponse:
"""Get queued asset events for a DAG."""
where_clause = [AssetDagRunQueue.target_dag_id == dag_id]
if before:
amoghrajesh marked this conversation as resolved.
Show resolved Hide resolved
before_parsed = timezone.parse(before)
pierrejeambrun marked this conversation as resolved.
Show resolved Hide resolved
where_clause.append(AssetDagRunQueue.created_at < before_parsed)
query = (
select(AssetDagRunQueue, AssetModel.uri)
.join(AssetModel, AssetDagRunQueue.asset_id == AssetModel.id)
.where(*where_clause)
)
result = session.execute(query).all()
amoghrajesh marked this conversation as resolved.
Show resolved Hide resolved
total_entries = len(result)
if not result:
raise HTTPException(status.HTTP_404_NOT_FOUND, f"Queue event with dag_id: `{dag_id}` was not found")
pierrejeambrun marked this conversation as resolved.
Show resolved Hide resolved
queued_events = [
QueuedEventResponse(created_at=adrq.created_at, dag_id=adrq.target_dag_id, uri=uri)
for adrq, uri in result
]
return QueuedEventCollectionResponse(
queued_events=[
QueuedEventResponse.model_validate(queued_event, from_attributes=True)
for queued_event in queued_events
],
total_entries=total_entries,
)
22 changes: 22 additions & 0 deletions airflow/ui/openapi-gen/queries/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -478,6 +478,28 @@ export const UseDagServiceGetDagDetailsKeyFn = (
},
queryKey?: Array<unknown>,
) => [useDagServiceGetDagDetailsKey, ...(queryKey ?? [{ dagId }])];
export type DagServiceGetDagAssetQueuedEventsDefaultResponse = Awaited<
ReturnType<typeof DagService.getDagAssetQueuedEvents>
>;
export type DagServiceGetDagAssetQueuedEventsQueryResult<
TData = DagServiceGetDagAssetQueuedEventsDefaultResponse,
TError = unknown,
> = UseQueryResult<TData, TError>;
export const useDagServiceGetDagAssetQueuedEventsKey =
"DagServiceGetDagAssetQueuedEvents";
export const UseDagServiceGetDagAssetQueuedEventsKeyFn = (
{
before,
dagId,
}: {
before?: string;
dagId: string;
},
queryKey?: Array<unknown>,
) => [
useDagServiceGetDagAssetQueuedEventsKey,
...(queryKey ?? [{ before, dagId }]),
];
export type EventLogServiceGetEventLogDefaultResponse = Awaited<
ReturnType<typeof EventLogService.getEventLog>
>;
Expand Down
26 changes: 26 additions & 0 deletions airflow/ui/openapi-gen/queries/prefetch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -622,6 +622,32 @@ export const prefetchUseDagServiceGetDagDetails = (
queryKey: Common.UseDagServiceGetDagDetailsKeyFn({ dagId }),
queryFn: () => DagService.getDagDetails({ dagId }),
});
/**
* Get Dag Asset Queued Events
* Get queued asset events for a DAG.
* @param data The data for the request.
* @param data.dagId
* @param data.before
* @returns QueuedEventCollectionResponse Successful Response
* @throws ApiError
*/
export const prefetchUseDagServiceGetDagAssetQueuedEvents = (
queryClient: QueryClient,
{
before,
dagId,
}: {
before?: string;
dagId: string;
},
) =>
queryClient.prefetchQuery({
queryKey: Common.UseDagServiceGetDagAssetQueuedEventsKeyFn({
before,
dagId,
}),
queryFn: () => DagService.getDagAssetQueuedEvents({ before, dagId }),
});
/**
* Get Event Log
* @param data The data for the request.
Expand Down
33 changes: 33 additions & 0 deletions airflow/ui/openapi-gen/queries/queries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -766,6 +766,39 @@ export const useDagServiceGetDagDetails = <
queryFn: () => DagService.getDagDetails({ dagId }) as TData,
...options,
});
/**
* Get Dag Asset Queued Events
* Get queued asset events for a DAG.
* @param data The data for the request.
* @param data.dagId
* @param data.before
* @returns QueuedEventCollectionResponse Successful Response
* @throws ApiError
*/
export const useDagServiceGetDagAssetQueuedEvents = <
TData = Common.DagServiceGetDagAssetQueuedEventsDefaultResponse,
TError = unknown,
TQueryKey extends Array<unknown> = unknown[],
>(
{
before,
dagId,
}: {
before?: string;
dagId: string;
},
queryKey?: TQueryKey,
options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">,
) =>
useQuery<TData, TError>({
queryKey: Common.UseDagServiceGetDagAssetQueuedEventsKeyFn(
{ before, dagId },
queryKey,
),
queryFn: () =>
DagService.getDagAssetQueuedEvents({ before, dagId }) as TData,
...options,
});
/**
* Get Event Log
* @param data The data for the request.
Expand Down
33 changes: 33 additions & 0 deletions airflow/ui/openapi-gen/queries/suspense.ts
Original file line number Diff line number Diff line change
Expand Up @@ -751,6 +751,39 @@ export const useDagServiceGetDagDetailsSuspense = <
queryFn: () => DagService.getDagDetails({ dagId }) as TData,
...options,
});
/**
* Get Dag Asset Queued Events
* Get queued asset events for a DAG.
* @param data The data for the request.
* @param data.dagId
* @param data.before
* @returns QueuedEventCollectionResponse Successful Response
* @throws ApiError
*/
export const useDagServiceGetDagAssetQueuedEventsSuspense = <
TData = Common.DagServiceGetDagAssetQueuedEventsDefaultResponse,
TError = unknown,
TQueryKey extends Array<unknown> = unknown[],
>(
{
before,
dagId,
}: {
before?: string;
dagId: string;
},
queryKey?: TQueryKey,
options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">,
) =>
useSuspenseQuery<TData, TError>({
queryKey: Common.UseDagServiceGetDagAssetQueuedEventsKeyFn(
{ before, dagId },
queryKey,
),
queryFn: () =>
DagService.getDagAssetQueuedEvents({ before, dagId }) as TData,
...options,
});
/**
* Get Event Log
* @param data The data for the request.
Expand Down
42 changes: 42 additions & 0 deletions airflow/ui/openapi-gen/requests/schemas.gen.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2844,6 +2844,48 @@ export const $ProviderResponse = {
description: "Provider serializer for responses.",
} as const;

export const $QueuedEventCollectionResponse = {
properties: {
queued_events: {
items: {
$ref: "#/components/schemas/QueuedEventResponse",
},
type: "array",
title: "Queued Events",
},
total_entries: {
type: "integer",
title: "Total Entries",
},
},
type: "object",
required: ["queued_events", "total_entries"],
title: "QueuedEventCollectionResponse",
description: "QueuedEventCollection serializer for responses.",
} as const;

export const $QueuedEventResponse = {
properties: {
uri: {
type: "string",
title: "Uri",
},
dag_id: {
type: "string",
title: "Dag Id",
},
created_at: {
type: "string",
format: "date-time",
title: "Created At",
},
},
type: "object",
required: ["uri", "dag_id", "created_at"],
title: "QueuedEventResponse",
description: "QueuedEvent serializer for responses..",
} as const;

export const $ReprocessBehavior = {
type: "string",
enum: ["failed", "completed", "none"],
Expand Down
Loading