From 2ca67c9b8db3b564caf6e476d6138c13d7ad57cf Mon Sep 17 00:00:00 2001 From: Maurice Yap Date: Fri, 17 Jan 2025 16:51:28 +0000 Subject: [PATCH 01/23] Add description and alerts for job commands on Lookout UI (#4152) Add optional descriptions and alerts for commands for a job displayed on the Lookout UI. These can be specified in Markdown. --- config/lookoutv2/config.yaml | 4 ++ internal/lookout/ui/package.json | 2 + .../sidebar/SidebarTabJobCommands.tsx | 53 +++++++++++++------ .../src/services/lookoutV2/useGetUiConfig.ts | 25 +++++++-- internal/lookout/ui/src/utils.tsx | 28 ++++++++-- internal/lookout/ui/yarn.lock | 14 ++++- internal/lookoutv2/configuration/types.go | 24 ++++++++- 7 files changed, 124 insertions(+), 26 deletions(-) diff --git a/config/lookoutv2/config.yaml b/config/lookoutv2/config.yaml index f423cc5eeed..3a35d84592d 100644 --- a/config/lookoutv2/config.yaml +++ b/config/lookoutv2/config.yaml @@ -32,3 +32,7 @@ uiConfig: template: "kubectl --context {{ runs[runs.length - 1].cluster }} -n {{ namespace }} logs armada-{{ jobId }}-0" - name: Exec template: "kubectl --context {{ runs[runs.length - 1].cluster }} -n {{ namespace }} exec -it armada-{{ jobId }}-0 /bin/sh" + descriptionMd: Execute a command on the job's container. + alertMessageMd: | + This will only work if the container is still running. + alertLevel: info diff --git a/internal/lookout/ui/package.json b/internal/lookout/ui/package.json index 052baa34c44..a0cd1b4347b 100644 --- a/internal/lookout/ui/package.json +++ b/internal/lookout/ui/package.json @@ -38,6 +38,8 @@ "date-fns-tz": "^1.3.7", "js-yaml": "^4.0.0", "lodash": "^4.17.21", + "markdown-to-jsx": "^7.7.3", + "mui-markdown": "^1.2.5", "notistack": "^3.0.1", "oidc-client-ts": "^2.3.0", "prism-react-renderer": "^2.4.1", diff --git a/internal/lookout/ui/src/components/lookoutV2/sidebar/SidebarTabJobCommands.tsx b/internal/lookout/ui/src/components/lookoutV2/sidebar/SidebarTabJobCommands.tsx index 938175564e2..8409370aee2 100644 --- a/internal/lookout/ui/src/components/lookoutV2/sidebar/SidebarTabJobCommands.tsx +++ b/internal/lookout/ui/src/components/lookoutV2/sidebar/SidebarTabJobCommands.tsx @@ -1,6 +1,7 @@ import { OpenInNew } from "@mui/icons-material" -import { Link, Stack } from "@mui/material" +import { Alert, AlertColor, Link, Stack } from "@mui/material" import { template, templateSettings } from "lodash" +import { MuiMarkdown } from "mui-markdown" import { Fragment } from "react/jsx-runtime" import validator from "validator" @@ -11,6 +12,8 @@ import { SPACING } from "../../../styling/spacing" import { CommandSpec } from "../../../utils" import { CodeBlock } from "../../CodeBlock" +const KNOWN_ALERT_COLORS: AlertColor[] = ["success", "info", "warning", "error"] + export interface SidebarTabJobCommandsProps { job: Job commandSpecs: CommandSpec[] @@ -35,27 +38,43 @@ export const SidebarTabJobCommands = ({ job, commandSpecs }: SidebarTabJobComman return ( <> {commandSpecs.map((commandSpec) => { - const { name } = commandSpec + const { name, descriptionMd, alertLevel, alertMessageMd } = commandSpec const commandText = getCommandText(job, commandSpec) + + const alertSeverity: AlertColor = + alertLevel && (KNOWN_ALERT_COLORS as string[]).includes(alertLevel) ? (alertLevel as AlertColor) : "info" + return ( {name} - {validator.isURL(commandText) ? ( - - -
{commandText}
- - - - ) : ( - + {descriptionMd && ( +
+ {descriptionMd} +
+ )} + {alertMessageMd && ( + + {alertMessageMd} + )} +
+ {validator.isURL(commandText) ? ( + + +
{commandText}
+ + + + ) : ( + + )} +
) })} diff --git a/internal/lookout/ui/src/services/lookoutV2/useGetUiConfig.ts b/internal/lookout/ui/src/services/lookoutV2/useGetUiConfig.ts index 683c41b3062..4010d3159a1 100644 --- a/internal/lookout/ui/src/services/lookoutV2/useGetUiConfig.ts +++ b/internal/lookout/ui/src/services/lookoutV2/useGetUiConfig.ts @@ -55,10 +55,27 @@ export const useGetUiConfig = (enabled = true) => { } if (json.CommandSpecs) { - config.commandSpecs = json.CommandSpecs.map(({ Name, Template }: { Name: string; Template: string }) => ({ - name: Name, - template: Template, - })) + config.commandSpecs = json.CommandSpecs.map( + ({ + Name, + Template, + DescriptionMd, + AlertMessageMd, + AlertLevel, + }: { + Name: string + Template: string + DescriptionMd: string + AlertMessageMd: string + AlertLevel: string + }) => ({ + name: Name, + template: Template, + descriptionMd: DescriptionMd, + alertMessageMd: AlertMessageMd, + alertLevel: AlertLevel, + }), + ) } if (json.Backend) config.backend = json.Backend diff --git a/internal/lookout/ui/src/utils.tsx b/internal/lookout/ui/src/utils.tsx index 3c4caf410d7..a458e946f89 100644 --- a/internal/lookout/ui/src/utils.tsx +++ b/internal/lookout/ui/src/utils.tsx @@ -9,9 +9,13 @@ export interface OidcConfig { clientId: string scope: string } + export interface CommandSpec { name: string template: string + descriptionMd?: string + alertMessageMd?: string + alertLevel?: string } export interface UIConfig { @@ -75,9 +79,27 @@ export async function getUIConfig(): Promise { scope: json.Oidc.Scope, } if (json.CommandSpecs) { - config.commandSpecs = json.CommandSpecs.map((c: { Name: string; Template: string }) => { - return { name: c.Name, template: c.Template } - }) + config.commandSpecs = json.CommandSpecs.map( + ({ + Name, + Template, + DescriptionMd, + AlertMessageMd, + AlertLevel, + }: { + Name: string + Template: string + DescriptionMd: string + AlertMessageMd: string + AlertLevel: string + }) => ({ + name: Name, + template: Template, + descriptionMd: DescriptionMd, + alertMessageMd: AlertMessageMd, + alertLevel: AlertLevel, + }), + ) } } if (json.Backend) config.backend = json.Backend diff --git a/internal/lookout/ui/yarn.lock b/internal/lookout/ui/yarn.lock index 462a894822e..7a1cf05c44a 100644 --- a/internal/lookout/ui/yarn.lock +++ b/internal/lookout/ui/yarn.lock @@ -3261,6 +3261,11 @@ magic-string@^0.30.17: dependencies: "@jridgewell/sourcemap-codec" "^1.5.0" +markdown-to-jsx@^7.7.3: + version "7.7.3" + resolved "https://registry.yarnpkg.com/markdown-to-jsx/-/markdown-to-jsx-7.7.3.tgz#c75927252592696e9e8b2a9557628749d8ab023e" + integrity sha512-o35IhJDFP6Fv60zPy+hbvZSQMmgvSGdK5j8NRZ7FeZMY+Bgqw+dSg7SC1ZEzC26++CiOUCqkbq96/c3j/FfTEQ== + math-intrinsics@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/math-intrinsics/-/math-intrinsics-1.1.0.tgz#a0dd74be81e2aa5c2f27e65ce283605ee4e2b7f9" @@ -3325,6 +3330,13 @@ ms@^2.1.1, ms@^2.1.3: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== +mui-markdown@^1.2.5: + version "1.2.5" + resolved "https://registry.yarnpkg.com/mui-markdown/-/mui-markdown-1.2.5.tgz#48e8a800c6707f84b77f56f3e553eb00754f15ff" + integrity sha512-zgLSXxYgHmUkUZ6mp2aM8C1vcoAsCyQLyvvaiSf8AAutNnAXhA8tlBiiGg8hOvX77VQs1A1dssbOyT/W2ytonA== + optionalDependencies: + prism-react-renderer "^2.0.3" + nanoid@^3.3.7: version "3.3.8" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.8.tgz#b1be3030bee36aaff18bacb375e5cce521684baf" @@ -3579,7 +3591,7 @@ pretty-format@^29.0.0, pretty-format@^29.7.0: ansi-styles "^5.0.0" react-is "^18.0.0" -prism-react-renderer@^2.4.1: +prism-react-renderer@^2.0.3, prism-react-renderer@^2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/prism-react-renderer/-/prism-react-renderer-2.4.1.tgz#ac63b7f78e56c8f2b5e76e823a976d5ede77e35f" integrity sha512-ey8Ls/+Di31eqzUxC46h8MksNuGx/n0AAC8uKpwFau4RPDYLuE3EXTp8N8G2vX2N7UC/+IXeNUnlWBGGcAG+Ig== diff --git a/internal/lookoutv2/configuration/types.go b/internal/lookoutv2/configuration/types.go index fa4d399f166..eb3ebfb77d1 100644 --- a/internal/lookoutv2/configuration/types.go +++ b/internal/lookoutv2/configuration/types.go @@ -36,9 +36,31 @@ type PrunerConfig struct { Postgres configuration.PostgresConfig } +// Alert level enum values correspond to the severity levels of the MUI Alert +// component: https://mui.com/material-ui/react-alert/#severity +type AlertLevel string + +const ( + AlertLevelSuccess AlertLevel = "success" + AlertLevelInfo AlertLevel = "info" + AlertLevelWarning AlertLevel = "warning" + AlertLevelError AlertLevel = "error" +) + +// CommandSpec details a command to be displayed on a job's "Commands" sidebar +// tab in the Lookout UI type CommandSpec struct { - Name string + // Name is the title of the command + Name string + // Tempate is the template string for the command Template string + // DescriptionMd is an optional description for the command in Markdown + DescriptionMd string + // AlertMessageMd is an optional message for the command, to be displayed as + // an alert, written in Markdown + AlertMessageMd string + // AlertLevel is the severity level of the alert + AlertLevel AlertLevel } type UIConfig struct { From 474182c09dad80df0fdb3ec0d687adea82640bdc Mon Sep 17 00:00:00 2001 From: Martynas Asipauskas Date: Mon, 20 Jan 2025 09:21:12 +0000 Subject: [PATCH 02/23] Re-enable integration test for failure (#4153) --- .../test_airflow_operator_logic.py | 89 +++++++------------ 1 file changed, 33 insertions(+), 56 deletions(-) diff --git a/third_party/airflow/test/integration/test_airflow_operator_logic.py b/third_party/airflow/test/integration/test_airflow_operator_logic.py index 43a4030e090..68bf3eca3a7 100644 --- a/third_party/airflow/test/integration/test_airflow_operator_logic.py +++ b/third_party/airflow/test/integration/test_airflow_operator_logic.py @@ -78,69 +78,39 @@ def sleep_pod(image: str): ) ], ) - return [ - submit_pb2.JobSubmitRequestItem( - priority=1, pod_spec=pod, namespace=DEFAULT_NAMESPACE - ) - ] - - -def test_success_job(client: ArmadaClient, context: Any, channel_args: GrpcChannelArgs): - job_set_name = f"test-{uuid.uuid1()}" - job = client.submit_jobs( - queue=DEFAULT_QUEUE, - job_set_id=job_set_name, - job_request_items=sleep_pod(image="busybox"), + return submit_pb2.JobSubmitRequestItem( + priority=1, pod_spec=pod, namespace=DEFAULT_NAMESPACE ) - job_id = job.job_response_items[0].job_id - context["ti"].xcom_pull.return_value = { - "armada_queue": DEFAULT_QUEUE, - "armada_job_id": job_id, - "armada_job_set_id": job_set_name, - } - operator = ArmadaOperator( +def armada_operator(image: str, channel_args: GrpcChannelArgs): + job_set_prefix = f"test-{uuid.uuid1()}" + return ArmadaOperator( task_id=DEFAULT_TASK_ID, name="test_job_success", channel_args=channel_args, armada_queue=DEFAULT_QUEUE, - job_request=sleep_pod(image="busybox")[0], + job_request=sleep_pod(image), + job_set_prefix=job_set_prefix, poll_interval=DEFAULT_POLLING_INTERVAL, job_acknowledgement_timeout=DEFAULT_JOB_ACKNOWLEDGEMENT_TIMEOUT, + deferrable=False, ) - operator.execute(context) - response = client.get_job_status([job_id]) - assert JobState(response.job_states[job_id]) == JobState.SUCCEEDED +def test_success_job(client: ArmadaClient, context: Any, channel_args: GrpcChannelArgs): + operator = armada_operator("busybox", channel_args) + operator.execute(context) + job = operator.job_context -@pytest.mark.skip(reason="FIXME: This no longer works we should fix our tests") -def test_bad_job(client: ArmadaClient, context: Any, channel_args: GrpcChannelArgs): - job_set_name = f"test-{uuid.uuid1()}" - job = client.submit_jobs( - queue=DEFAULT_QUEUE, - job_set_id=job_set_name, - job_request_items=sleep_pod(image="NOTACONTAINER"), - ) - job_id = job.job_response_items[0].job_id + assert job.state == JobState.SUCCEEDED + response = client.get_job_status([job.job_id]) + assert JobState(response.job_states[job.job_id]) == JobState.SUCCEEDED - context["ti"].xcom_pull.return_value = { - "armada_queue": DEFAULT_QUEUE, - "armada_job_id": job_id, - "armada_job_set_id": job_set_name, - } - operator = ArmadaOperator( - task_id=DEFAULT_TASK_ID, - name="test_job_failure", - channel_args=channel_args, - armada_queue=DEFAULT_QUEUE, - job_request=sleep_pod(image="busybox")[0], - poll_interval=DEFAULT_POLLING_INTERVAL, - job_acknowledgement_timeout=DEFAULT_JOB_ACKNOWLEDGEMENT_TIMEOUT, - ) +def test_bad_job(client: ArmadaClient, context: Any, channel_args: GrpcChannelArgs): + operator = armada_operator("BADIMAGE", channel_args) try: operator.execute(context) @@ -148,8 +118,13 @@ def test_bad_job(client: ArmadaClient, context: Any, channel_args: GrpcChannelAr "Operator did not raise AirflowException on job failure as expected" ) except AirflowException: # Expected - response = client.get_job_status([job_id]) - assert JobState(response.job_states[job_id]) == JobState.FAILED + # Assert state failed + job = operator.job_context + assert job.state == JobState.FAILED + + # Assert actual job failed too + response = client.get_job_status([job.job_id]) + assert JobState(response.job_states[job.job_id]) == JobState.FAILED except Exception as e: pytest.fail( "Operator did not raise AirflowException on job failure as expected, " @@ -157,7 +132,8 @@ def test_bad_job(client: ArmadaClient, context: Any, channel_args: GrpcChannelAr ) -def success_job( +# Used benchmark parallel execution below +def _success_job( task_number: int, context: Any, channel_args: GrpcChannelArgs, client: ArmadaClient ) -> JobState: operator = ArmadaOperator( @@ -168,6 +144,7 @@ def success_job( job_request=sleep_pod(image="busybox")[0], poll_interval=DEFAULT_POLLING_INTERVAL, job_acknowledgement_timeout=DEFAULT_JOB_ACKNOWLEDGEMENT_TIMEOUT, + deferrable=False, ) operator.execute(context) @@ -181,12 +158,12 @@ def test_parallel_execution( client: ArmadaClient, context: Any, channel_args: GrpcChannelArgs, mocker ): threads = [] - success_job( + _success_job( task_number=0, context=context, channel_args=channel_args, client=client ) for task_number in range(5): t = threading.Thread( - target=success_job, args=[task_number, context, channel_args] + target=_success_job, args=[task_number, context, channel_args] ) t.start() threads.append(t) @@ -200,12 +177,12 @@ def test_parallel_execution_large( client: ArmadaClient, context: Any, channel_args: GrpcChannelArgs, mocker ): threads = [] - success_job( + _success_job( task_number=0, context=context, channel_args=channel_args, client=client ) for task_number in range(80): t = threading.Thread( - target=success_job, args=[task_number, context, channel_args] + target=_success_job, args=[task_number, context, channel_args] ) t.start() threads.append(t) @@ -219,12 +196,12 @@ def test_parallel_execution_huge( client: ArmadaClient, context: Any, channel_args: GrpcChannelArgs, mocker ): threads = [] - success_job( + _success_job( task_number=0, context=context, channel_args=channel_args, client=client ) for task_number in range(500): t = threading.Thread( - target=success_job, args=[task_number, context, channel_args] + target=_success_job, args=[task_number, context, channel_args] ) t.start() threads.append(t) From 4236f64981e0bfb6a7a55b7662d1349338295896 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Mon, 20 Jan 2025 10:21:22 +0000 Subject: [PATCH 03/23] Switch Logging To Zerolog (#4136) * update pulsar mock * go mod tidy * import order * wip * wip * wip * supress logging * lint * lint * wip * wip * wip * wip * wip * wip * wip * remove logrus * remove logrus * fix test * fix armadactl * clean up interfaces * clean up interfaces * wip * added back json logging * lint * fix test * add back log tests * wip * wip * wip * lint * fix skip frames * fixed milli time * formatting * formatting * more tests * lint * colorful * fixes * fixes * fixes * fixes * wip * lint * lint * wip * wip * update docker * wip * add prometheus hook * add prometheus hook * add prometheus hook * go mod tidy * unit test for prometheus * go mod tidy * fix * fix * fix --- cmd/armada-load-tester/cmd/loadtest.go | 4 +- cmd/armada-load-tester/cmd/root.go | 6 +- cmd/binoculars/main.go | 2 +- cmd/executor/main.go | 4 +- cmd/fakeexecutor/main.go | 4 +- cmd/lookoutingesterv2/dbloadtester/main.go | 2 +- cmd/lookoutingesterv2/main.go | 2 +- cmd/lookoutv2/main.go | 2 +- cmd/scheduler/cmd/migrate_database.go | 2 +- cmd/server/main.go | 4 +- cmd/simulator/cmd/root.go | 5 +- go.mod | 7 +- go.sum | 16 +- internal/binoculars/server.go | 2 +- internal/binoculars/service/cordon_test.go | 5 +- internal/binoculars/service/logs.go | 2 +- .../common/armadacontext/armada_context.go | 111 +++++++--- .../armadacontext/armada_context_test.go | 157 +++++++++++++- internal/common/certs/cached_certificate.go | 3 +- internal/common/cluster/kubernetes_client.go | 2 +- internal/common/etcdhealth/etcdhealth.go | 19 +- internal/common/etcdhealth/etcdhealth_test.go | 3 +- internal/common/eventutil/eventutil.go | 2 +- internal/common/grpc/gateway.go | 3 +- internal/common/grpc/grpc.go | 6 +- internal/common/health/http_handler.go | 2 +- .../common/healthmonitor/healthmonitor.go | 3 +- .../healthmonitor/manualhealthmonitor.go | 3 +- .../healthmonitor/multihealthmonitor.go | 5 +- internal/common/ingest/batch.go | 2 +- .../common/ingest/controlplaneevents/utils.go | 2 +- internal/common/ingest/ingestion_pipeline.go | 2 +- internal/common/ingest/jobsetevents/utils.go | 2 +- internal/common/ingest/retry.go | 2 +- internal/common/logging/formatter.go | 13 -- internal/common/logging/formatter_test.go | 24 --- internal/common/logging/global.go | 116 +++++++++++ internal/common/logging/levelwriter.go | 26 +++ internal/common/logging/logger.go | 136 ++++++++++++ internal/common/logging/logger_test.go | 196 ++++++++++++++++++ internal/common/logging/null_logger.go | 14 -- internal/common/logging/prometheus.go | 41 ++++ internal/common/logging/prometheus_test.go | 41 ++++ internal/common/logging/pulsar_adapter.go | 79 +++++++ internal/common/logging/stacktrace.go | 22 -- internal/common/logging/user_input.go | 8 - internal/common/metrics/provider.go | 8 +- internal/common/profiling/http.go | 5 +- internal/common/pulsarutils/publisher.go | 5 +- internal/common/pulsarutils/pulsarclient.go | 2 + internal/common/serve/static.go | 3 +- internal/common/startup.go | 128 ++++++------ internal/eventingester/convert/conversions.go | 2 +- internal/eventingester/ingester.go | 2 +- internal/eventingester/store/eventstore.go | 2 +- internal/executor/application.go | 49 ++--- internal/executor/context/cluster_context.go | 2 +- internal/executor/fake/application.go | 7 +- internal/executor/fake/context/context.go | 2 +- internal/executor/job/job_run_state_store.go | 2 +- .../executor/job/processors/preempt_runs.go | 2 +- .../executor/job/processors/remove_runs.go | 2 +- internal/executor/job/submit.go | 2 +- .../metrics/pod_metrics/cluster_context.go | 2 +- .../podchecks/container_state_checks.go | 6 +- internal/executor/podchecks/event_checks.go | 2 +- internal/executor/podchecks/pod_checks.go | 2 +- .../executor/reporter/job_event_reporter.go | 2 +- .../executor/service/cluster_allocation.go | 5 +- internal/executor/service/job_requester.go | 2 +- internal/executor/service/lease_requester.go | 2 +- .../executor/service/pod_issue_handler.go | 2 +- internal/executor/service/resource_cleanup.go | 2 +- internal/executor/util/pod_util.go | 2 +- .../utilisation/cluster_utilisation.go | 2 +- .../utilisation/job_utilisation_reporter.go | 2 +- .../executor/utilisation/pod_utilisation.go | 2 +- .../pod_utilisation_custom_metrics.go | 2 +- .../pod_utilisation_kubelet_metrics.go | 5 +- .../utilisation/prometheus_scraping.go | 7 +- .../lookoutingesterv2/dbloadtester/queue.go | 4 +- .../dbloadtester/simulator.go | 2 +- internal/lookoutingesterv2/ingester.go | 2 +- .../instructions/instructions.go | 4 +- .../lookoutingesterv2/lookoutdb/insertion.go | 2 +- internal/lookoutv2/application.go | 4 +- internal/lookoutv2/generate/main.go | 2 +- internal/lookoutv2/pruner/pruner.go | 2 +- internal/lookoutv2/repository/getjoberror.go | 2 +- .../repository/getjobrundebugmessage.go | 2 +- .../lookoutv2/repository/getjobrunerror.go | 2 +- internal/lookoutv2/repository/getjobspec.go | 2 +- internal/lookoutv2/repository/querybuilder.go | 4 +- internal/lookoutv2/repository/util.go | 2 +- internal/scheduler/api.go | 5 +- internal/scheduler/metrics.go | 5 +- internal/scheduler/nodedb/nodeiteration.go | 2 +- internal/scheduler/scheduler.go | 17 +- internal/scheduler/schedulerapp.go | 9 +- internal/scheduler/scheduling/context/job.go | 4 +- ..._driven_preempting_queue_scheduler_test.go | 2 +- .../scheduling/preempting_queue_scheduler.go | 20 +- .../preempting_queue_scheduler_test.go | 21 +- .../scheduler/scheduling/scheduling_algo.go | 6 +- internal/scheduler/simulator/simulator.go | 7 +- internal/scheduler/submitcheck.go | 13 +- internal/scheduleringester/ingester.go | 2 +- internal/scheduleringester/instructions.go | 2 +- .../server/event/conversion/conversions.go | 3 +- internal/server/event/event.go | 2 +- internal/server/event/event_repository.go | 2 +- internal/server/server.go | 2 +- internal/server/submit/submit.go | 2 +- pkg/client/auth/kubernetes/authentication.go | 2 +- pkg/client/auth/oidc/device.go | 9 +- pkg/client/auth/oidc/kubernetes.go | 3 +- pkg/client/auth/oidc/pkce.go | 11 +- pkg/client/load-test.go | 11 +- pkg/client/queue/get_all.go | 12 +- pkg/client/watch.go | 8 +- 120 files changed, 1173 insertions(+), 433 deletions(-) delete mode 100644 internal/common/logging/formatter.go delete mode 100644 internal/common/logging/formatter_test.go create mode 100644 internal/common/logging/global.go create mode 100644 internal/common/logging/levelwriter.go create mode 100644 internal/common/logging/logger.go create mode 100644 internal/common/logging/logger_test.go delete mode 100644 internal/common/logging/null_logger.go create mode 100644 internal/common/logging/prometheus.go create mode 100644 internal/common/logging/prometheus_test.go create mode 100644 internal/common/logging/pulsar_adapter.go delete mode 100644 internal/common/logging/stacktrace.go delete mode 100644 internal/common/logging/user_input.go diff --git a/cmd/armada-load-tester/cmd/loadtest.go b/cmd/armada-load-tester/cmd/loadtest.go index 5a2e301490e..f6c09622bcd 100644 --- a/cmd/armada-load-tester/cmd/loadtest.go +++ b/cmd/armada-load-tester/cmd/loadtest.go @@ -6,10 +6,10 @@ import ( "strings" "time" - log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/pkg/client" "github.com/armadaproject/armada/pkg/client/domain" "github.com/armadaproject/armada/pkg/client/util" @@ -72,7 +72,7 @@ var loadtestCmd = &cobra.Command{ loadTestSpec := &domain.LoadTestSpecification{} err := util.BindJsonOrYaml(filePath, loadTestSpec) if err != nil { - log.Error(err) + log.Error(err.Error()) os.Exit(1) } diff --git a/cmd/armada-load-tester/cmd/root.go b/cmd/armada-load-tester/cmd/root.go index 5f2bcfb59c3..0803042410d 100644 --- a/cmd/armada-load-tester/cmd/root.go +++ b/cmd/armada-load-tester/cmd/root.go @@ -3,9 +3,9 @@ package cmd import ( "os" - log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/pkg/client" ) @@ -28,7 +28,7 @@ The location of this file can be passed in using --config argument or picked fro // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { if err := rootCmd.Execute(); err != nil { - log.Error(err) + log.Error(err.Error()) os.Exit(1) } } @@ -37,7 +37,7 @@ var cfgFile string func initConfig() { if err := client.LoadCommandlineArgsFromConfigFile(cfgFile); err != nil { - log.Error(err) + log.Error(err.Error()) os.Exit(1) } } diff --git a/cmd/binoculars/main.go b/cmd/binoculars/main.go index 86389fbc91b..5c0558aeb3a 100644 --- a/cmd/binoculars/main.go +++ b/cmd/binoculars/main.go @@ -8,7 +8,6 @@ import ( "syscall" "github.com/grpc-ecosystem/grpc-gateway/runtime" - log "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/spf13/viper" "google.golang.org/grpc" @@ -19,6 +18,7 @@ import ( "github.com/armadaproject/armada/internal/common/armadacontext" gateway "github.com/armadaproject/armada/internal/common/grpc" "github.com/armadaproject/armada/internal/common/health" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/profiling" api "github.com/armadaproject/armada/pkg/api/binoculars" ) diff --git a/cmd/executor/main.go b/cmd/executor/main.go index 04e53b73f41..a91c0304f49 100644 --- a/cmd/executor/main.go +++ b/cmd/executor/main.go @@ -7,13 +7,13 @@ import ( "syscall" "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/armadaproject/armada/internal/common" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/health" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/profiling" "github.com/armadaproject/armada/internal/executor" "github.com/armadaproject/armada/internal/executor/configuration" @@ -61,7 +61,7 @@ func main() { ) defer shutdownMetricServer() - shutdown, wg := executor.StartUp(armadacontext.Background(), log.NewEntry(log.StandardLogger()), config) + shutdown, wg := executor.StartUp(armadacontext.Background(), config) go func() { <-shutdownChannel shutdown() diff --git a/cmd/fakeexecutor/main.go b/cmd/fakeexecutor/main.go index 831d56b650e..69d9ce03c33 100644 --- a/cmd/fakeexecutor/main.go +++ b/cmd/fakeexecutor/main.go @@ -5,12 +5,12 @@ import ( "os/signal" "syscall" - log "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/armadaproject/armada/internal/common" "github.com/armadaproject/armada/internal/common/armadacontext" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/profiling" "github.com/armadaproject/armada/internal/executor/configuration" "github.com/armadaproject/armada/internal/executor/fake" @@ -54,7 +54,7 @@ func main() { shutdownMetricServer := common.ServeMetrics(config.Metric.Port) defer shutdownMetricServer() - shutdown, wg := fake.StartUp(config, nodes) + shutdown, wg := fake.StartUp(armadacontext.Background(), config, nodes) go func() { <-shutdownChannel shutdown() diff --git a/cmd/lookoutingesterv2/dbloadtester/main.go b/cmd/lookoutingesterv2/dbloadtester/main.go index 8daf0960f2f..87f91fa1f73 100644 --- a/cmd/lookoutingesterv2/dbloadtester/main.go +++ b/cmd/lookoutingesterv2/dbloadtester/main.go @@ -4,13 +4,13 @@ import ( "fmt" "time" - log "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/spf13/viper" "sigs.k8s.io/yaml" "github.com/armadaproject/armada/internal/common" "github.com/armadaproject/armada/internal/common/app" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/lookoutingesterv2/configuration" "github.com/armadaproject/armada/internal/lookoutingesterv2/dbloadtester" ) diff --git a/cmd/lookoutingesterv2/main.go b/cmd/lookoutingesterv2/main.go index b60e58f6241..a0e48a6374f 100644 --- a/cmd/lookoutingesterv2/main.go +++ b/cmd/lookoutingesterv2/main.go @@ -1,11 +1,11 @@ package main import ( - log "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/armadaproject/armada/internal/common" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/lookoutingesterv2" "github.com/armadaproject/armada/internal/lookoutingesterv2/benchmark" "github.com/armadaproject/armada/internal/lookoutingesterv2/configuration" diff --git a/cmd/lookoutv2/main.go b/cmd/lookoutv2/main.go index 48d48bc97b5..da8bb63e57c 100644 --- a/cmd/lookoutv2/main.go +++ b/cmd/lookoutv2/main.go @@ -5,7 +5,6 @@ import ( "os/signal" "syscall" - log "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/spf13/viper" "k8s.io/utils/clock" @@ -13,6 +12,7 @@ import ( "github.com/armadaproject/armada/internal/common" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/database" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/profiling" "github.com/armadaproject/armada/internal/lookoutv2" "github.com/armadaproject/armada/internal/lookoutv2/configuration" diff --git a/cmd/scheduler/cmd/migrate_database.go b/cmd/scheduler/cmd/migrate_database.go index 9e48f5fba5f..407048e31dc 100644 --- a/cmd/scheduler/cmd/migrate_database.go +++ b/cmd/scheduler/cmd/migrate_database.go @@ -4,11 +4,11 @@ import ( "time" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/database" + log "github.com/armadaproject/armada/internal/common/logging" schedulerdb "github.com/armadaproject/armada/internal/scheduler/database" ) diff --git a/cmd/server/main.go b/cmd/server/main.go index 96e51f293e1..3815049164a 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -8,7 +8,6 @@ import ( "syscall" "time" - log "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -17,6 +16,7 @@ import ( gateway "github.com/armadaproject/armada/internal/common/grpc" "github.com/armadaproject/armada/internal/common/health" "github.com/armadaproject/armada/internal/common/logging" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/profiling" "github.com/armadaproject/armada/internal/server" "github.com/armadaproject/armada/internal/server/configuration" @@ -119,6 +119,6 @@ func main() { }() if err := g.Wait(); err != nil { - logging.WithStacktrace(log.NewEntry(log.StandardLogger()), err).Error("Armada server shut down") + logging.WithStacktrace(err).Error("Armada server shut down") } } diff --git a/cmd/simulator/cmd/root.go b/cmd/simulator/cmd/root.go index a1c815633c5..1ddd0e6ba34 100644 --- a/cmd/simulator/cmd/root.go +++ b/cmd/simulator/cmd/root.go @@ -6,13 +6,12 @@ import ( "runtime/pprof" "time" - "github.com/armadaproject/armada/internal/scheduler/simulator/sink" - - log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/armadaproject/armada/internal/common/armadacontext" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/scheduler/simulator" + "github.com/armadaproject/armada/internal/scheduler/simulator/sink" "github.com/armadaproject/armada/internal/scheduler/testfixtures" ) diff --git a/go.mod b/go.mod index c2623ca95ce..f1b9d7a05de 100644 --- a/go.mod +++ b/go.mod @@ -36,12 +36,10 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.20.5 github.com/renstrom/shortuuid v3.0.0+incompatible - github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 - github.com/weaveworks/promrus v1.2.0 golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c golang.org/x/net v0.30.0 golang.org/x/oauth2 v0.23.0 @@ -81,6 +79,7 @@ require ( github.com/prometheus/common v0.60.0 github.com/redis/go-redis/extra/redisprometheus/v9 v9.0.5 github.com/redis/go-redis/v9 v9.7.0 + github.com/rs/zerolog v1.33.0 github.com/segmentio/fasthash v1.0.3 github.com/xitongsys/parquet-go v1.6.2 go.uber.org/mock v0.5.0 @@ -161,7 +160,8 @@ require ( github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-isatty v0.0.18 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/microcosm-cc/bluemonday v1.0.25 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -183,6 +183,7 @@ require ( github.com/rivo/uniseg v0.4.2 // indirect github.com/sagikazarmark/locafero v0.6.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.11.0 // indirect diff --git a/go.sum b/go.sum index 1491cc31df8..789178833e1 100644 --- a/go.sum +++ b/go.sum @@ -114,6 +114,7 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -209,6 +210,7 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a h1:dR8+Q0uO5S2ZBcs2IH6VBKYwSxPo2vYCYq0ot0mu7xA= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -393,8 +395,11 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= -github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= -github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= @@ -503,6 +508,9 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= +github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= @@ -560,8 +568,6 @@ github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFA github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= -github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M= -github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -725,7 +731,9 @@ golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/internal/binoculars/server.go b/internal/binoculars/server.go index bf14abc18f2..d1d604f7c67 100644 --- a/internal/binoculars/server.go +++ b/internal/binoculars/server.go @@ -5,7 +5,6 @@ import ( "sync" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/binoculars/configuration" "github.com/armadaproject/armada/internal/binoculars/server" @@ -13,6 +12,7 @@ import ( "github.com/armadaproject/armada/internal/common/auth" "github.com/armadaproject/armada/internal/common/cluster" grpcCommon "github.com/armadaproject/armada/internal/common/grpc" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/pkg/api/binoculars" ) diff --git a/internal/binoculars/service/cordon_test.go b/internal/binoculars/service/cordon_test.go index 43197004d9e..1245b2ee1f1 100644 --- a/internal/binoculars/service/cordon_test.go +++ b/internal/binoculars/service/cordon_test.go @@ -6,7 +6,8 @@ import ( "fmt" "testing" - "github.com/sirupsen/logrus" + "github.com/armadaproject/armada/internal/common/logging" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" @@ -81,7 +82,7 @@ func TestCordonNode(t *testing.T) { cordonService, client := setupTest(t, cordonConfig, FakePermissionChecker{ReturnValue: true}) ctx := auth.WithPrincipal(context.Background(), principal) - err := cordonService.CordonNode(armadacontext.New(ctx, logrus.NewEntry(logrus.New())), &binoculars.CordonRequest{ + err := cordonService.CordonNode(armadacontext.New(ctx, logging.StdLogger()), &binoculars.CordonRequest{ NodeName: defaultNode.Name, }) assert.Nil(t, err) diff --git a/internal/binoculars/service/logs.go b/internal/binoculars/service/logs.go index 8bc0f38d1be..34ca6b417ce 100644 --- a/internal/binoculars/service/logs.go +++ b/internal/binoculars/service/logs.go @@ -5,13 +5,13 @@ import ( "strings" "time" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/auth" "github.com/armadaproject/armada/internal/common/cluster" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/pkg/api/binoculars" ) diff --git a/internal/common/armadacontext/armada_context.go b/internal/common/armadacontext/armada_context.go index 30fb2b466c9..b96f7cf197f 100644 --- a/internal/common/armadacontext/armada_context.go +++ b/internal/common/armadacontext/armada_context.go @@ -4,60 +4,119 @@ import ( "context" "time" - "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" + + "github.com/armadaproject/armada/internal/common/logging" ) // Context is an extension of Go's context which also includes a logger. This allows us to pass round a contextual logger // while retaining type-safety type Context struct { context.Context - logrus.FieldLogger + logger *logging.Logger +} + +// Debug logs a message at level Debug +func (ctx *Context) Debug(msg string) { + ctx.logger.Debug(msg) +} + +// Info logs a message at level Info +func (ctx *Context) Info(msg string) { + ctx.logger.Info(msg) +} + +// Warn logs a message at level Warn +func (ctx *Context) Warn(msg string) { + ctx.logger.Warn(msg) +} + +// Error logs a message at level Error +func (ctx *Context) Error(msg string) { + ctx.logger.Error(msg) +} + +// Panic logs a message at level Panic +func (ctx *Context) Panic(msg string) { + ctx.logger.Panic(msg) +} + +// Fatal logs a message at level Fatal then the process will exit with status set to 1. +func (ctx *Context) Fatal(msg string) { + ctx.logger.Fatal(msg) +} + +// Debugf logs a message at level Debug. +func (ctx *Context) Debugf(format string, args ...interface{}) { + ctx.logger.Debugf(format, args...) +} + +// Infof logs a message at level Info. +func (ctx *Context) Infof(format string, args ...interface{}) { + ctx.logger.Infof(format, args...) +} + +// Warnf logs a message at level Warn. +func (ctx *Context) Warnf(format string, args ...interface{}) { + ctx.logger.Warnf(format, args...) +} + +// Errorf logs a message at level Error. +func (ctx *Context) Errorf(format string, args ...interface{}) { + ctx.logger.Errorf(format, args...) +} + +// Fatalf logs a message at level Fatal. +func (ctx *Context) Fatalf(format string, args ...interface{}) { + ctx.logger.Fatalf(format, args...) } // Background creates an empty context with a default logger. It is analogous to context.Background() func Background() *Context { return &Context{ - Context: context.Background(), - FieldLogger: logrus.NewEntry(logrus.StandardLogger()), + Context: context.Background(), + logger: logging.StdLogger(), } } // TODO creates an empty context with a default logger. It is analogous to context.TODO() func TODO() *Context { return &Context{ - Context: context.TODO(), - FieldLogger: logrus.NewEntry(logrus.StandardLogger()), + Context: context.TODO(), + logger: logging.StdLogger(), } } -// FromGrpcCtx creates a context where the logger is extracted via ctxlogrus's Extract() method. -// Note that this will result in a no-op logger if a logger hasn't already been inserted into the context via ctxlogrus +// FromGrpcCtx Converts a context.Context to an armadacontext.Context func FromGrpcCtx(ctx context.Context) *Context { armadaCtx, ok := ctx.(*Context) if ok { return armadaCtx } - logger := logrus.NewEntry(logrus.StandardLogger()). + logger := logging.StdLogger(). WithField("user", ctx.Value("user")). WithField("requestId", ctx.Value("requestId")) return New(ctx, logger) } // New returns an armada context that encapsulates both a go context and a logger -func New(ctx context.Context, log *logrus.Entry) *Context { +func New(ctx context.Context, log *logging.Logger) *Context { return &Context{ - Context: ctx, - FieldLogger: log, + Context: ctx, + logger: log, } } +func (ctx *Context) Logger() *logging.Logger { + return ctx.logger.WithCallerSkip(logging.StdSkipFrames - 1) +} + // WithCancel returns a copy of parent with a new Done channel. It is analogous to context.WithCancel() func WithCancel(parent *Context) (*Context, context.CancelFunc) { c, cancel := context.WithCancel(parent.Context) return &Context{ - Context: c, - FieldLogger: parent.FieldLogger, + Context: c, + logger: parent.logger, }, cancel } @@ -66,8 +125,8 @@ func WithCancel(parent *Context) (*Context, context.CancelFunc) { func WithDeadline(parent *Context, d time.Time) (*Context, context.CancelFunc) { c, cancel := context.WithDeadline(parent.Context, d) return &Context{ - Context: c, - FieldLogger: parent.FieldLogger, + Context: c, + logger: parent.logger, }, cancel } @@ -77,18 +136,18 @@ func WithTimeout(parent *Context, timeout time.Duration) (*Context, context.Canc } // WithLogField returns a copy of parent with the supplied key-value added to the logger -func WithLogField(parent *Context, key string, val interface{}) *Context { +func WithLogField(parent *Context, key string, val any) *Context { return &Context{ - Context: parent.Context, - FieldLogger: parent.FieldLogger.WithField(key, val), + Context: parent.Context, + logger: parent.logger.WithField(key, val), } } // WithLogFields returns a copy of parent with the supplied key-values added to the logger -func WithLogFields(parent *Context, fields logrus.Fields) *Context { +func WithLogFields(parent *Context, fields map[string]any) *Context { return &Context{ - Context: parent.Context, - FieldLogger: parent.FieldLogger.WithFields(fields), + Context: parent.Context, + logger: parent.logger.WithFields(fields), } } @@ -96,8 +155,8 @@ func WithLogFields(parent *Context, fields logrus.Fields) *Context { // val. It is analogous to context.WithValue() func WithValue(parent *Context, key, val any) *Context { return &Context{ - Context: context.WithValue(parent, key, val), - FieldLogger: parent.FieldLogger, + Context: context.WithValue(parent, key, val), + logger: parent.logger, } } @@ -106,7 +165,7 @@ func WithValue(parent *Context, key, val any) *Context { func ErrGroup(ctx *Context) (*errgroup.Group, *Context) { group, goctx := errgroup.WithContext(ctx) return group, &Context{ - Context: goctx, - FieldLogger: ctx.FieldLogger, + Context: goctx, + logger: ctx.logger, } } diff --git a/internal/common/armadacontext/armada_context_test.go b/internal/common/armadacontext/armada_context_test.go index ef7c84aa705..60c94fdc441 100644 --- a/internal/common/armadacontext/armada_context_test.go +++ b/internal/common/armadacontext/armada_context_test.go @@ -1,19 +1,31 @@ package armadacontext import ( + "bytes" "context" + "encoding/json" "testing" "time" - "github.com/sirupsen/logrus" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/armadaproject/armada/internal/common/logging" ) -var defaultLogger = logrus.WithField("foo", "bar") +type testLogEntry struct { + Level string `json:"level"` + Message string `json:"message"` + CustomField1 string `json:"customField1,omitempty"` + CustomField2 string `json:"customField2,omitempty"` +} + +var defaultLogger = logging.StdLogger().WithField("foo", "bar") func TestNew(t *testing.T) { ctx := New(context.Background(), defaultLogger) - require.Equal(t, defaultLogger, ctx.FieldLogger) + require.Equal(t, defaultLogger, ctx.logger) require.Equal(t, context.Background(), ctx.Context) } @@ -27,16 +39,128 @@ func TestTODO(t *testing.T) { require.Equal(t, ctx.Context, context.TODO()) } +func TestLogAtLevel(t *testing.T) { + tests := map[string]struct { + logFunction func(ctx *Context) + expectedLevel string + expectedMsg string + }{ + "Debug": { + logFunction: func(ctx *Context) { + ctx.Debug("test message") + }, + expectedMsg: "test message", + expectedLevel: "debug", + }, + "Debugf": { + logFunction: func(ctx *Context) { + ctx.Debugf("test message %d", 1) + }, + expectedMsg: "test message 1", + expectedLevel: "debug", + }, + "Info": { + logFunction: func(ctx *Context) { + ctx.Info("test message") + }, + expectedMsg: "test message", + expectedLevel: "info", + }, + "Infof": { + logFunction: func(ctx *Context) { + ctx.Infof("test message %d", 1) + }, + expectedMsg: "test message 1", + expectedLevel: "info", + }, + "Warn": { + logFunction: func(ctx *Context) { + ctx.Warn("test message") + }, + expectedMsg: "test message", + expectedLevel: "warn", + }, + "Warnf": { + logFunction: func(ctx *Context) { + ctx.Warnf("test message %d", 1) + }, + expectedMsg: "test message 1", + expectedLevel: "warn", + }, + "Error": { + logFunction: func(ctx *Context) { + ctx.Errorf("test message") + }, + expectedMsg: "test message", + expectedLevel: "error", + }, + "Errorf": { + logFunction: func(ctx *Context) { + ctx.Errorf("test message %d", 1) + }, + expectedMsg: "test message 1", + expectedLevel: "error", + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + logger, buf := testLogger() + ctx := New(context.Background(), logger) + tc.logFunction(ctx) + assertLogLineExpected( + t, + &testLogEntry{ + Level: tc.expectedLevel, + Message: tc.expectedMsg, + }, + buf, + ) + }) + } +} + func TestWithLogField(t *testing.T) { - ctx := WithLogField(Background(), "fish", "chips") + logger, buf := testLogger() + ctx := WithLogField(New(context.Background(), logger), "customField1", "foo") + + ctx.Info("test message") + require.Equal(t, context.Background(), ctx.Context) - require.Equal(t, logrus.Fields{"fish": "chips"}, ctx.FieldLogger.(*logrus.Entry).Data) + assertLogLineExpected( + t, + &testLogEntry{ + Level: "info", + Message: "test message", + CustomField1: "foo", + }, + buf, + ) } func TestWithLogFields(t *testing.T) { - ctx := WithLogFields(Background(), logrus.Fields{"fish": "chips", "salt": "pepper"}) + logger, buf := testLogger() + + ctx := WithLogFields( + New(context.Background(), logger), + map[string]interface{}{ + "customField1": "bar", + "customField2": "baz", + }, + ) + + ctx.Info("test message") + require.Equal(t, context.Background(), ctx.Context) - require.Equal(t, logrus.Fields{"fish": "chips", "salt": "pepper"}, ctx.FieldLogger.(*logrus.Entry).Data) + assertLogLineExpected( + t, + &testLogEntry{ + Level: "info", + Message: "test message", + CustomField1: "bar", + CustomField2: "baz", + }, + buf, + ) } func TestWithTimeout(t *testing.T) { @@ -78,3 +202,22 @@ func quiescent(t *testing.T) time.Duration { const arbitraryCleanupMargin = 1 * time.Second return time.Until(deadline) - arbitraryCleanupMargin } + +// testLogger sets up a Zerolog logger that writes to a buffer for testing +func testLogger() (*logging.Logger, *bytes.Buffer) { + var buf bytes.Buffer + baseLogger := zerolog.New(&buf).Level(zerolog.DebugLevel).With().Timestamp().Logger() + logger := logging.FromZerolog(baseLogger) + return logger, &buf +} + +func assertLogLineExpected(t *testing.T, expected *testLogEntry, logOutput *bytes.Buffer) { + var entry testLogEntry + err := json.Unmarshal(logOutput.Bytes(), &entry) + require.NoError(t, err, "Failed to unmarshal log entry") + + assert.Equal(t, expected.Message, entry.Message) + assert.Equal(t, expected.Level, entry.Level) + assert.Equal(t, expected.CustomField1, entry.CustomField1) + assert.Equal(t, expected.CustomField2, entry.CustomField2) +} diff --git a/internal/common/certs/cached_certificate.go b/internal/common/certs/cached_certificate.go index 72b7f6ea250..10ea5ea27da 100644 --- a/internal/common/certs/cached_certificate.go +++ b/internal/common/certs/cached_certificate.go @@ -6,9 +6,8 @@ import ( "sync" "time" - log "github.com/sirupsen/logrus" - "github.com/armadaproject/armada/internal/common/armadacontext" + log "github.com/armadaproject/armada/internal/common/logging" ) type CachedCertificateService struct { diff --git a/internal/common/cluster/kubernetes_client.go b/internal/common/cluster/kubernetes_client.go index 1c280fa55a6..9af32096d6e 100644 --- a/internal/common/cluster/kubernetes_client.go +++ b/internal/common/cluster/kubernetes_client.go @@ -2,13 +2,13 @@ package cluster import ( "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/flowcontrol" "github.com/armadaproject/armada/internal/common/armadaerrors" + log "github.com/armadaproject/armada/internal/common/logging" ) type KubernetesClientProvider interface { diff --git a/internal/common/etcdhealth/etcdhealth.go b/internal/common/etcdhealth/etcdhealth.go index 49be27a22fe..1d836c520c1 100644 --- a/internal/common/etcdhealth/etcdhealth.go +++ b/internal/common/etcdhealth/etcdhealth.go @@ -6,11 +6,9 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/healthmonitor" - "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/metrics" ) @@ -184,10 +182,9 @@ func (srv *EtcdReplicaHealthMonitor) sizeFraction() float64 { return srv.etcdSizeBytes / srv.etcdCapacityBytes } -func (srv *EtcdReplicaHealthMonitor) Run(ctx *armadacontext.Context, log *logrus.Entry) error { - log = log.WithField("service", "EtcdHealthMonitor") - log.Info("starting etcd health monitor") - defer log.Info("stopping etcd health monitor") +func (srv *EtcdReplicaHealthMonitor) Run(ctx *armadacontext.Context) error { + ctx.Info("starting etcd health monitor") + defer ctx.Info("stopping etcd health monitor") ticker := time.NewTicker(srv.scrapeInterval) defer ticker.Stop() for { @@ -196,24 +193,24 @@ func (srv *EtcdReplicaHealthMonitor) Run(ctx *armadacontext.Context, log *logrus return ctx.Err() case <-ticker.C: t := time.Now() - metrics, err := srv.metricsProvider.Collect(ctx, log) + metrics, err := srv.metricsProvider.Collect(ctx) srv.mu.Lock() srv.timeOfMostRecentCollectionAttempt = time.Now() if err != nil { - logging.WithStacktrace(log, err).Errorf("failed to scrape etcd metrics from %s", srv.name) + ctx.Logger().WithStacktrace(err).Errorf("failed to scrape etcd metrics from %s", srv.name) } else { success := true if err := srv.setSizeInUseBytesFromMetrics(metrics); err != nil { success = false - logging.WithStacktrace(log, err).Errorf("failed to scrape etcd metrics from %s", srv.name) + ctx.Logger().WithStacktrace(err).Errorf("failed to scrape etcd metrics from %s", srv.name) } if err := srv.setSizeBytesFromMetrics(metrics); err != nil { success = false - logging.WithStacktrace(log, err).Errorf("failed to scrape etcd metrics from %s", srv.name) + ctx.Logger().WithStacktrace(err).Errorf("failed to scrape etcd metrics from %s", srv.name) } if err := srv.setCapacityBytesFromMetrics(metrics); err != nil { success = false - logging.WithStacktrace(log, err).Errorf("failed to scrape etcd metrics from %s", srv.name) + ctx.Logger().WithStacktrace(err).Errorf("failed to scrape etcd metrics from %s", srv.name) } if success { srv.timeOfMostRecentSuccessfulCollectionAttempt = srv.timeOfMostRecentCollectionAttempt diff --git a/internal/common/etcdhealth/etcdhealth_test.go b/internal/common/etcdhealth/etcdhealth_test.go index 474d4df0e3a..038277e4227 100644 --- a/internal/common/etcdhealth/etcdhealth_test.go +++ b/internal/common/etcdhealth/etcdhealth_test.go @@ -4,7 +4,6 @@ import ( "testing" "time" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/armadaproject/armada/internal/common/armadacontext" @@ -26,7 +25,7 @@ func TestEtcdReplicaHealthMonitor(t *testing.T) { ctx, cancel := armadacontext.WithCancel(armadacontext.Background()) defer cancel() g, ctx := armadacontext.ErrGroup(ctx) - g.Go(func() error { return hm.Run(ctx, logrus.NewEntry(logrus.New())) }) + g.Go(func() error { return hm.Run(ctx) }) // Should still be unavailable due to missing metrics. hm.BlockUntilNextMetricsCollection(ctx) diff --git a/internal/common/eventutil/eventutil.go b/internal/common/eventutil/eventutil.go index 27fc7ef6f55..fa126b155e5 100644 --- a/internal/common/eventutil/eventutil.go +++ b/internal/common/eventutil/eventutil.go @@ -6,13 +6,13 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" networking "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "github.com/armadaproject/armada/internal/common/armadaerrors" + log "github.com/armadaproject/armada/internal/common/logging" protoutil "github.com/armadaproject/armada/internal/common/proto" "github.com/armadaproject/armada/internal/common/slices" "github.com/armadaproject/armada/pkg/api" diff --git a/internal/common/grpc/gateway.go b/internal/common/grpc/gateway.go index 8b6e59dc2ad..9ae6aeddd09 100644 --- a/internal/common/grpc/gateway.go +++ b/internal/common/grpc/gateway.go @@ -10,11 +10,12 @@ import ( "github.com/go-openapi/runtime/middleware" "github.com/grpc-ecosystem/grpc-gateway/runtime" - log "github.com/sirupsen/logrus" "golang.org/x/exp/slices" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" + + log "github.com/armadaproject/armada/internal/common/logging" ) // CreateGatewayHandler configures the gRPC API gateway diff --git a/internal/common/grpc/grpc.go b/internal/common/grpc/grpc.go index 1139e04a427..9cc844a3152 100644 --- a/internal/common/grpc/grpc.go +++ b/internal/common/grpc/grpc.go @@ -15,7 +15,6 @@ import ( grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery" "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -28,6 +27,7 @@ import ( "github.com/armadaproject/armada/internal/common/auth" "github.com/armadaproject/armada/internal/common/certs" "github.com/armadaproject/armada/internal/common/grpc/configuration" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/requestid" ) @@ -89,9 +89,9 @@ func Listen(port uint16, grpcServer *grpc.Server, wg *sync.WaitGroup) { } go func() { - defer log.Println("Stopping server.") + defer log.Infof("Stopping server.") - log.Printf("Grpc listening on %d", port) + log.Infof("Grpc listening on %d", port) if err := grpcServer.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) } diff --git a/internal/common/health/http_handler.go b/internal/common/health/http_handler.go index 5dd2c9b5444..38b1a0f1f82 100644 --- a/internal/common/health/http_handler.go +++ b/internal/common/health/http_handler.go @@ -3,7 +3,7 @@ package health import ( "net/http" - log "github.com/sirupsen/logrus" + log "github.com/armadaproject/armada/internal/common/logging" ) // TODO Doesn't need to exist. Just give a Checker directly. diff --git a/internal/common/healthmonitor/healthmonitor.go b/internal/common/healthmonitor/healthmonitor.go index d5c6b151c1e..2bf7d817b6c 100644 --- a/internal/common/healthmonitor/healthmonitor.go +++ b/internal/common/healthmonitor/healthmonitor.go @@ -2,7 +2,6 @@ package healthmonitor import ( "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common/armadacontext" ) @@ -25,5 +24,5 @@ type HealthMonitor interface { // Run initialises and starts the health checker. // Run may be blocking and should be run within a separate goroutine. // Must be called before IsHealthy() or any prometheus.Collector interface methods. - Run(*armadacontext.Context, *logrus.Entry) error + Run(*armadacontext.Context) error } diff --git a/internal/common/healthmonitor/manualhealthmonitor.go b/internal/common/healthmonitor/manualhealthmonitor.go index 7aa2f525068..46797493df0 100644 --- a/internal/common/healthmonitor/manualhealthmonitor.go +++ b/internal/common/healthmonitor/manualhealthmonitor.go @@ -4,7 +4,6 @@ import ( "sync" "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common/armadacontext" ) @@ -47,7 +46,7 @@ func (srv *ManualHealthMonitor) IsHealthy() (bool, string, error) { } } -func (srv *ManualHealthMonitor) Run(_ *armadacontext.Context, _ *logrus.Entry) error { +func (srv *ManualHealthMonitor) Run(_ *armadacontext.Context) error { return nil } diff --git a/internal/common/healthmonitor/multihealthmonitor.go b/internal/common/healthmonitor/multihealthmonitor.go index a9f03643d10..cf6197918f9 100644 --- a/internal/common/healthmonitor/multihealthmonitor.go +++ b/internal/common/healthmonitor/multihealthmonitor.go @@ -6,7 +6,6 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" "golang.org/x/exp/maps" "github.com/armadaproject/armada/internal/common/armadacontext" @@ -100,11 +99,11 @@ func (srv *MultiHealthMonitor) IsHealthy() (ok bool, reason string, err error) { } // Run initialises prometheus metrics and starts any child health checkers. -func (srv *MultiHealthMonitor) Run(ctx *armadacontext.Context, log *logrus.Entry) error { +func (srv *MultiHealthMonitor) Run(ctx *armadacontext.Context) error { g, ctx := armadacontext.ErrGroup(ctx) for _, healthMonitor := range srv.healthMonitorsByName { healthMonitor := healthMonitor - g.Go(func() error { return healthMonitor.Run(ctx, log) }) + g.Go(func() error { return healthMonitor.Run(ctx) }) } return g.Wait() } diff --git a/internal/common/ingest/batch.go b/internal/common/ingest/batch.go index 02a35f7490c..f2c4867ce10 100644 --- a/internal/common/ingest/batch.go +++ b/internal/common/ingest/batch.go @@ -4,10 +4,10 @@ import ( "sync" "time" - log "github.com/sirupsen/logrus" "k8s.io/utils/clock" "github.com/armadaproject/armada/internal/common/armadacontext" + log "github.com/armadaproject/armada/internal/common/logging" ) // Batcher batches up events from a channel. Batches are created whenever maxItems have been diff --git a/internal/common/ingest/controlplaneevents/utils.go b/internal/common/ingest/controlplaneevents/utils.go index 13182165ba5..836548d27a4 100644 --- a/internal/common/ingest/controlplaneevents/utils.go +++ b/internal/common/ingest/controlplaneevents/utils.go @@ -2,11 +2,11 @@ package controlplaneevents import ( "github.com/apache/pulsar-client-go/pulsar" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common/eventutil" commonmetrics "github.com/armadaproject/armada/internal/common/ingest/metrics" "github.com/armadaproject/armada/internal/common/ingest/utils" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/pkg/controlplaneevents" ) diff --git a/internal/common/ingest/ingestion_pipeline.go b/internal/common/ingest/ingestion_pipeline.go index 160cf677268..a8f39e07ec6 100644 --- a/internal/common/ingest/ingestion_pipeline.go +++ b/internal/common/ingest/ingestion_pipeline.go @@ -7,12 +7,12 @@ import ( "github.com/apache/pulsar-client-go/pulsar" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common/armadacontext" commonconfig "github.com/armadaproject/armada/internal/common/config" commonmetrics "github.com/armadaproject/armada/internal/common/ingest/metrics" "github.com/armadaproject/armada/internal/common/ingest/utils" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/pulsarutils" "github.com/armadaproject/armada/internal/common/util" ) diff --git a/internal/common/ingest/jobsetevents/utils.go b/internal/common/ingest/jobsetevents/utils.go index 0de21b7d27a..a1b36c9d293 100644 --- a/internal/common/ingest/jobsetevents/utils.go +++ b/internal/common/ingest/jobsetevents/utils.go @@ -2,11 +2,11 @@ package jobsetevents import ( "github.com/apache/pulsar-client-go/pulsar" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common/eventutil" commonmetrics "github.com/armadaproject/armada/internal/common/ingest/metrics" "github.com/armadaproject/armada/internal/common/ingest/utils" + log "github.com/armadaproject/armada/internal/common/logging" protoutil "github.com/armadaproject/armada/internal/common/proto" "github.com/armadaproject/armada/pkg/armadaevents" ) diff --git a/internal/common/ingest/retry.go b/internal/common/ingest/retry.go index aa66a7c9166..810b0c346a2 100644 --- a/internal/common/ingest/retry.go +++ b/internal/common/ingest/retry.go @@ -3,7 +3,7 @@ package ingest import ( "time" - log "github.com/sirupsen/logrus" + log "github.com/armadaproject/armada/internal/common/logging" ) // WithRetry executes the supplied action until it either completes successfully or it returns false, indicating that diff --git a/internal/common/logging/formatter.go b/internal/common/logging/formatter.go deleted file mode 100644 index f6db9902424..00000000000 --- a/internal/common/logging/formatter.go +++ /dev/null @@ -1,13 +0,0 @@ -package logging - -import ( - "fmt" - - log "github.com/sirupsen/logrus" -) - -type CommandLineFormatter struct{} - -func (f *CommandLineFormatter) Format(entry *log.Entry) ([]byte, error) { - return []byte(fmt.Sprintf("%s\n", entry.Message)), nil -} diff --git a/internal/common/logging/formatter_test.go b/internal/common/logging/formatter_test.go deleted file mode 100644 index bf79fca89aa..00000000000 --- a/internal/common/logging/formatter_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package logging - -import ( - "testing" - - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" -) - -func TestCommandLineFormatter(t *testing.T) { - commandLineFormatter := new(CommandLineFormatter) - - testMessage := "Test" - expectedOutput := testMessage + "\n" - - event := log.Entry{ - Message: testMessage, - } - - output, err := commandLineFormatter.Format(&event) - - assert.Nil(t, err) - assert.Equal(t, expectedOutput, string(output[:])) -} diff --git a/internal/common/logging/global.go b/internal/common/logging/global.go new file mode 100644 index 00000000000..507e1d49478 --- /dev/null +++ b/internal/common/logging/global.go @@ -0,0 +1,116 @@ +package logging + +import ( + "os" + + "github.com/rs/zerolog" +) + +var ( + StdSkipFrames = 4 + // The global Logger. Comes configured with some sensible defaults for e.g. unit tests, but applications should + // generally configure their own logging config via ReplaceStdLogger + stdLogger = createDefaultLogger() +) + +// ReplaceStdLogger Replaces the global logger. This should be called once at app startup! +func ReplaceStdLogger(l *Logger) { + stdLogger = l.WithCallerSkip(StdSkipFrames) +} + +// StdLogger Returns the default logger +func StdLogger() *Logger { + return stdLogger +} + +// Debug logs a message at level Debug on the default logger. +func Debug(args ...any) { + stdLogger.Debug(args...) +} + +// Info logs a message at level Info on the default logger. +func Info(args ...any) { + stdLogger.Info(args...) +} + +// Warn logs a message at level Warn on the default logger. +func Warn(args ...any) { + stdLogger.Warn(args...) +} + +// Error logs a message at level Error on the default logger. +func Error(args ...any) { + stdLogger.Error(args...) +} + +// Panic logs a message at level Panic on the default logger. +func Panic(args ...any) { + stdLogger.Panic(args...) +} + +// Fatal logs a message at level Fatal on the default logger then the process will exit with status set to 1. +func Fatal(args ...any) { + stdLogger.Fatal(args...) +} + +// Debugf logs a message at level Debug on the default logger. +func Debugf(format string, args ...any) { + stdLogger.Debugf(format, args...) +} + +// Infof logs a message at level Info on the default logger. +func Infof(format string, args ...any) { + stdLogger.Infof(format, args...) +} + +// Warnf logs a message at level Warn on the default logger. +func Warnf(format string, args ...any) { + stdLogger.Warnf(format, args...) +} + +// Errorf logs a message at level Error on the default logger. +func Errorf(format string, args ...any) { + stdLogger.Errorf(format, args...) +} + +// Fatalf logs a message at level Fatal on the default logger then the process will exit with status set to 1. +func Fatalf(format string, args ...any) { + stdLogger.Fatalf(format, args...) +} + +// WithField returns a new Logger with the key-value pair added as a new field +func WithField(key string, value any) *Logger { + return stdLogger.WithField(key, value) +} + +// WithFields returns a new Logger with all key-value pairs in the map added as new fields +func WithFields(args map[string]any) *Logger { + return stdLogger.WithFields(args) +} + +// WithError returns a new Logger with the error added as a field +func WithError(err error) *Logger { + return stdLogger.WithError(err) +} + +// WithStacktrace returns a new Logger with the error and (if available) the stacktrace added as fields +func WithStacktrace(err error) *Logger { + return stdLogger.WithStacktrace(err) +} + +// createDefaultLogger returns a new Logger configured with default settings using zerolog. +func createDefaultLogger() *Logger { + consoleWriter := zerolog.ConsoleWriter{ + Out: os.Stdout, + TimeFormat: "2006-01-02T15:04:05.000Z07:00", + NoColor: true, + } + zerologLogger := zerolog.New(consoleWriter). + Level(zerolog.InfoLevel). + With(). + CallerWithSkipFrameCount(StdSkipFrames). + Timestamp(). + Logger() + + return FromZerolog(zerologLogger) +} diff --git a/internal/common/logging/levelwriter.go b/internal/common/logging/levelwriter.go new file mode 100644 index 00000000000..edf63a298ba --- /dev/null +++ b/internal/common/logging/levelwriter.go @@ -0,0 +1,26 @@ +package logging + +import ( + "io" + + "github.com/rs/zerolog" +) + +type FilteredLevelWriter struct { + writer io.Writer + level zerolog.Level +} + +// Write writes to the underlying Writer. +func (w *FilteredLevelWriter) Write(p []byte) (int, error) { + return w.writer.Write(p) +} + +// WriteLevel calls WriteLevel of the underlying Writer only if the level is equal +// or above the Level. +func (w *FilteredLevelWriter) WriteLevel(level zerolog.Level, p []byte) (int, error) { + if level >= w.level { + return w.writer.Write(p) + } + return len(p), nil +} diff --git a/internal/common/logging/logger.go b/internal/common/logging/logger.go new file mode 100644 index 00000000000..b6e5605f6ea --- /dev/null +++ b/internal/common/logging/logger.go @@ -0,0 +1,136 @@ +package logging + +import ( + "fmt" + + "github.com/pkg/errors" + "github.com/rs/zerolog" +) + +// Logger wraps a zerolog.Logger so that the rest of the code doesn't depend directly on zerolog +type Logger struct { + underlying zerolog.Logger +} + +// NullLogger is Logger that discards all log lines +var NullLogger = &Logger{ + underlying: zerolog.Nop(), +} + +// FromZerolog returns a new Logger backed by the supplied zerolog.Logger +func FromZerolog(l zerolog.Logger) *Logger { + return &Logger{ + underlying: l, + } +} + +// Debug logs a message at level Debug +func (l *Logger) Debug(args ...any) { + l.underlying.Debug().Msg(fmt.Sprint(args...)) +} + +// Info logs a message at level Info +func (l *Logger) Info(args ...any) { + l.underlying.Info().Msg(fmt.Sprint(args...)) +} + +// Warn logs a message at level Warn +func (l *Logger) Warn(args ...any) { + l.underlying.Warn().Msg(fmt.Sprint(args...)) +} + +// Error logs a message at level Error +func (l *Logger) Error(args ...any) { + l.underlying.Error().Msg(fmt.Sprint(args...)) +} + +// Panic logs a message at level Panic and panics +func (l *Logger) Panic(args ...any) { + l.underlying.Panic().Msg(fmt.Sprint(args...)) +} + +// Fatal logs a message at level Fatal and exits the application +func (l *Logger) Fatal(args ...any) { + l.underlying.Fatal().Msg(fmt.Sprint(args...)) +} + +// Debugf logs a formatted message at level Debug +func (l *Logger) Debugf(format string, args ...interface{}) { + l.underlying.Debug().Msgf(format, args...) +} + +// Infof logs a formatted message at level Info +func (l *Logger) Infof(format string, args ...interface{}) { + l.underlying.Info().Msgf(format, args...) +} + +// Warnf logs a formatted message at level Warn +func (l *Logger) Warnf(format string, args ...interface{}) { + l.underlying.Warn().Msgf(format, args...) +} + +// Errorf logs a formatted message at level Error +func (l *Logger) Errorf(format string, args ...interface{}) { + l.underlying.Error().Msgf(format, args...) +} + +// Fatalf logs a formatted message at level Fatal and exits the application +func (l *Logger) Fatalf(format string, args ...interface{}) { + l.underlying.Fatal().Msgf(format, args...) +} + +// WithError returns a new Logger with the error added as a field +func (l *Logger) WithError(err error) *Logger { + return &Logger{ + underlying: l.underlying. + With(). + AnErr("error", err). + CallerWithSkipFrameCount(StdSkipFrames - 1). + Logger(), + } +} + +// WithStacktrace returns a new Logger with the error and (if available) the stacktrace added as fields +func (l *Logger) WithStacktrace(err error) *Logger { + logger := l.WithError(err) + if stackErr, ok := err.(stackTracer); ok { + return logger.WithField("stacktrace", fmt.Sprintf("%v", stackErr.StackTrace())) + } + return logger +} + +// WithField returns a new Logger with the key-value pair added as a new field +func (l *Logger) WithField(key string, value any) *Logger { + return &Logger{ + underlying: l.underlying. + With(). + Interface(key, value). + CallerWithSkipFrameCount(StdSkipFrames - 1). + Logger(), + } +} + +// WithFields returns a new Logger with all key-value pairs in the map added as new fields +func (l *Logger) WithFields(args map[string]any) *Logger { + event := l.underlying.With() + for key, value := range args { + event = event.Interface(key, value) + } + return &Logger{ + underlying: event.CallerWithSkipFrameCount(StdSkipFrames - 1).Logger(), + } +} + +// WithCallerSkip returns a new Logger with the number of callers skipped increased by the skip amount. +// This is needed when building wrappers around the Logger so as to prevent us from always reporting the +// wrapper code as the caller. +func (l *Logger) WithCallerSkip(skip int) *Logger { + return &Logger{ + underlying: l.underlying.With().CallerWithSkipFrameCount(skip).Logger(), + } +} + +// Unexported but considered part of the stable interface of pkg/errors. +type stackTracer interface { + StackTrace() errors.StackTrace +} diff --git a/internal/common/logging/logger_test.go b/internal/common/logging/logger_test.go new file mode 100644 index 00000000000..1f8d9b76d5d --- /dev/null +++ b/internal/common/logging/logger_test.go @@ -0,0 +1,196 @@ +package logging + +import ( + "bytes" + "encoding/json" + "fmt" + "testing" + + "github.com/pkg/errors" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type testLogEntry struct { + Level string `json:"level"` + Message string `json:"message"` + CustomField1 string `json:"customField1,omitempty"` + CustomField2 string `json:"customField2,omitempty"` + Error string `json:"error,omitempty"` + Stacktrace string `json:"stacktrace,omitempty"` +} + +func TestWithField(t *testing.T) { + logger, buf := testLogger() + logger = logger.WithField("customField1", "foo") + + logger.Info("test message") + + assertLogLineExpected( + t, + &testLogEntry{ + Level: "info", + Message: "test message", + CustomField1: "foo", + }, + buf, + ) +} + +func TestWithFields(t *testing.T) { + logger, buf := testLogger() + logger = logger.WithFields( + map[string]any{"customField1": "bar", "customField2": "baz"}, + ) + + logger.Info("test message") + + assertLogLineExpected( + t, + &testLogEntry{ + Level: "info", + Message: "test message", + CustomField1: "bar", + CustomField2: "baz", + }, + buf, + ) +} + +func TestWithError(t *testing.T) { + logger, buf := testLogger() + err := errors.New("test error") + logger = logger.WithError(err) + + logger.Info("test message") + + assertLogLineExpected( + t, + &testLogEntry{ + Level: "info", + Message: "test message", + Error: "test error", + }, + buf, + ) +} + +func TestWithStacktrace(t *testing.T) { + logger, buf := testLogger() + err := errors.New("test error") + logger = logger.WithStacktrace(err) + + logger.Info("test message") + + assertLogLineExpected( + t, + &testLogEntry{ + Level: "info", + Message: "test message", + Error: "test error", + Stacktrace: fmt.Sprintf("%v", err.(stackTracer).StackTrace()), + }, + buf, + ) +} + +func TestLogAtLevel(t *testing.T) { + tests := map[string]struct { + logFunction func(logger *Logger) + expectedLevel string + expectedMsg string + }{ + "Debug": { + logFunction: func(l *Logger) { + l.Debug("test message") + }, + expectedMsg: "test message", + expectedLevel: "debug", + }, + "Debugf": { + logFunction: func(l *Logger) { + l.Debugf("test message %d", 1) + }, + expectedMsg: "test message 1", + expectedLevel: "debug", + }, + "Info": { + logFunction: func(l *Logger) { + l.Info("test message") + }, + expectedMsg: "test message", + expectedLevel: "info", + }, + "Infof": { + logFunction: func(l *Logger) { + l.Infof("test message %d", 1) + }, + expectedMsg: "test message 1", + expectedLevel: "info", + }, + "Warn": { + logFunction: func(l *Logger) { + l.Warn("test message") + }, + expectedMsg: "test message", + expectedLevel: "warn", + }, + "Warnf": { + logFunction: func(l *Logger) { + l.Warnf("test message %d", 1) + }, + expectedMsg: "test message 1", + expectedLevel: "warn", + }, + "Error": { + logFunction: func(l *Logger) { + l.Errorf("test message") + }, + expectedMsg: "test message", + expectedLevel: "error", + }, + "Errorf": { + logFunction: func(l *Logger) { + l.Errorf("test message %d", 1) + }, + expectedMsg: "test message 1", + expectedLevel: "error", + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + logger, buf := testLogger() + tc.logFunction(logger) + assertLogLineExpected( + t, + &testLogEntry{ + Level: tc.expectedLevel, + Message: tc.expectedMsg, + }, + buf, + ) + }) + } +} + +// testLogger sets up a Zerolog logger that writes to a buffer for testing +func testLogger() (*Logger, *bytes.Buffer) { + var buf bytes.Buffer + baseLogger := zerolog.New(&buf).Level(zerolog.DebugLevel).With().Timestamp().Logger() + logger := FromZerolog(baseLogger) + return logger, &buf +} + +func assertLogLineExpected(t *testing.T, expected *testLogEntry, logOutput *bytes.Buffer) { + var entry testLogEntry + err := json.Unmarshal(logOutput.Bytes(), &entry) + require.NoError(t, err, "Failed to unmarshal log entry") + + assert.Equal(t, expected.Message, entry.Message) + assert.Equal(t, expected.Level, entry.Level) + assert.Equal(t, expected.CustomField1, entry.CustomField1) + assert.Equal(t, expected.CustomField2, entry.CustomField2) + assert.Equal(t, expected.Error, entry.Error) + assert.Equal(t, expected.Stacktrace, entry.Stacktrace) +} diff --git a/internal/common/logging/null_logger.go b/internal/common/logging/null_logger.go deleted file mode 100644 index 25326d3691b..00000000000 --- a/internal/common/logging/null_logger.go +++ /dev/null @@ -1,14 +0,0 @@ -package logging - -import ( - "io" - - "github.com/sirupsen/logrus" -) - -var NullLogger = &logrus.Logger{ - Out: io.Discard, - Formatter: new(logrus.TextFormatter), - Hooks: make(logrus.LevelHooks), - Level: logrus.PanicLevel, -} diff --git a/internal/common/logging/prometheus.go b/internal/common/logging/prometheus.go new file mode 100644 index 00000000000..7e5fa3d1814 --- /dev/null +++ b/internal/common/logging/prometheus.go @@ -0,0 +1,41 @@ +package logging + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/rs/zerolog" +) + +// PrometheusHook implements zerolog.Hook +type PrometheusHook struct { + counters map[zerolog.Level]prometheus.Counter +} + +// NewPrometheusHook creates and registers Prometheus counters for each log level. +func NewPrometheusHook() *PrometheusHook { + counters := make(map[zerolog.Level]prometheus.Counter) + + for _, level := range []zerolog.Level{ + zerolog.DebugLevel, + zerolog.InfoLevel, + zerolog.WarnLevel, + zerolog.ErrorLevel, + } { + counter := prometheus.NewCounter(prometheus.CounterOpts{ + Name: "log_messages", + Help: "Total number of log lines logged by level", + ConstLabels: prometheus.Labels{ + "level": level.String(), + }, + }) + // Register the counter with Prometheus. + prometheus.MustRegister(counter) + counters[level] = counter + } + return &PrometheusHook{counters: counters} +} + +func (h *PrometheusHook) Run(_ *zerolog.Event, level zerolog.Level, _ string) { + if counter, ok := h.counters[level]; ok { + counter.Inc() + } +} diff --git a/internal/common/logging/prometheus_test.go b/internal/common/logging/prometheus_test.go new file mode 100644 index 00000000000..bc09b31f861 --- /dev/null +++ b/internal/common/logging/prometheus_test.go @@ -0,0 +1,41 @@ +package logging + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" +) + +// TestPrometheusHook_IncrementsCounter verifies that calling Run on monitored levels +// properly increments the corresponding counter. +func TestPrometheusHook_IncrementsCounter(t *testing.T) { + hook := NewPrometheusHook() + + // Clean up: Unregister the counters to avoid polluting the global registry in further tests. + t.Cleanup(func() { + for _, counter := range hook.counters { + prometheus.Unregister(counter) + } + }) + + // Define the log levels we are monitoring. + levels := []zerolog.Level{ + zerolog.DebugLevel, + zerolog.InfoLevel, + zerolog.WarnLevel, + zerolog.ErrorLevel, + } + + for _, level := range levels { + n := 3 + for i := 0; i < n; i++ { + hook.Run(nil, level, "dummy message") + } + // Verify the counter value. + value := testutil.ToFloat64(hook.counters[level]) + assert.Equal(t, float64(n), value, "expected counter for level %s to be %d, got %f", level, n, value) + } +} diff --git a/internal/common/logging/pulsar_adapter.go b/internal/common/logging/pulsar_adapter.go new file mode 100644 index 00000000000..d4e13d04b59 --- /dev/null +++ b/internal/common/logging/pulsar_adapter.go @@ -0,0 +1,79 @@ +package logging + +import ( + pulsarlog "github.com/apache/pulsar-client-go/pulsar/log" +) + +var pulsarAdapterSkipFrames = StdSkipFrames + +// Wrapper to adapt Logger to the logger interface expected by the pulsar client +type pulsarWrapper struct { + l *Logger +} + +// NewPulsarLogger returns a Logger that can be used by the pulsar client +func NewPulsarLogger() pulsarlog.Logger { + return newPulsarLogger(StdLogger()) +} + +func (p pulsarWrapper) SubLogger(fs pulsarlog.Fields) pulsarlog.Logger { + return newPulsarLogger( + p.l.WithFields(fs), + ) +} + +func (p pulsarWrapper) WithFields(fs pulsarlog.Fields) pulsarlog.Entry { + return newPulsarLogger( + p.l.WithFields(fs), + ) +} + +func (p pulsarWrapper) WithField(name string, value interface{}) pulsarlog.Entry { + return newPulsarLogger( + p.l.WithField(name, value), + ) +} + +func (p pulsarWrapper) WithError(err error) pulsarlog.Entry { + return newPulsarLogger( + p.l.WithError(err), + ) +} + +func (p pulsarWrapper) Debug(args ...any) { + p.l.Debug(args...) +} + +func (p pulsarWrapper) Info(args ...any) { + p.l.Info(args...) +} + +func (p pulsarWrapper) Warn(args ...any) { + p.l.Warn(args...) +} + +func (p pulsarWrapper) Error(args ...any) { + p.l.Error(args...) +} + +func (p pulsarWrapper) Debugf(format string, args ...any) { + p.l.Debugf(format, args...) +} + +func (p pulsarWrapper) Infof(format string, args ...any) { + p.l.Infof(format, args...) +} + +func (p pulsarWrapper) Warnf(format string, args ...any) { + p.l.Warnf(format, args...) +} + +func (p pulsarWrapper) Errorf(format string, args ...any) { + p.l.Errorf(format, args...) +} + +func newPulsarLogger(l *Logger) pulsarlog.Logger { + return &pulsarWrapper{ + l: l.WithCallerSkip(pulsarAdapterSkipFrames), + } +} diff --git a/internal/common/logging/stacktrace.go b/internal/common/logging/stacktrace.go deleted file mode 100644 index 7d546915b31..00000000000 --- a/internal/common/logging/stacktrace.go +++ /dev/null @@ -1,22 +0,0 @@ -package logging - -import ( - "github.com/pkg/errors" - "github.com/sirupsen/logrus" -) - -// Unexported but considered part of the stable interface of pkg/errors. -type stackTracer interface { - StackTrace() errors.StackTrace -} - -// WithStacktrace returns a new logrus.FieldLogger obtained by adding error information and, if available, a stack trace -// as fields to the provided logrus.FieldLogger. -func WithStacktrace(logger logrus.FieldLogger, err error) logrus.FieldLogger { - logger = logger.WithError(err) - if stackErr, ok := err.(stackTracer); ok { - return logger.WithField("stacktrace", stackErr.StackTrace()) - } else { - return logger - } -} diff --git a/internal/common/logging/user_input.go b/internal/common/logging/user_input.go deleted file mode 100644 index 3fb161040b9..00000000000 --- a/internal/common/logging/user_input.go +++ /dev/null @@ -1,8 +0,0 @@ -package logging - -import "strings" - -func SanitizeUserInput(str string) string { - safeStr := strings.Replace(str, "\n", "", -1) - return strings.Replace(safeStr, "\r", "", -1) -} diff --git a/internal/common/metrics/provider.go b/internal/common/metrics/provider.go index 7280f461fa0..28fc08a64ef 100644 --- a/internal/common/metrics/provider.go +++ b/internal/common/metrics/provider.go @@ -8,12 +8,10 @@ import ( "strings" "sync" "time" - - "github.com/sirupsen/logrus" ) type MetricsProvider interface { - Collect(context.Context, *logrus.Entry) (map[string]float64, error) + Collect(context.Context) (map[string]float64, error) } type ManualMetricsProvider struct { @@ -36,7 +34,7 @@ func (srv *ManualMetricsProvider) WithCollectionDelay(d time.Duration) *ManualMe return srv } -func (srv *ManualMetricsProvider) Collect(_ context.Context, _ *logrus.Entry) (map[string]float64, error) { +func (srv *ManualMetricsProvider) Collect(_ context.Context) (map[string]float64, error) { srv.mu.Lock() defer srv.mu.Unlock() if srv.collectionDelay != 0 { @@ -61,7 +59,7 @@ func NewHttpMetricsProvider(url string, client *http.Client) *HttpMetricsProvide } } -func (srv *HttpMetricsProvider) Collect(ctx context.Context, _ *logrus.Entry) (map[string]float64, error) { +func (srv *HttpMetricsProvider) Collect(ctx context.Context) (map[string]float64, error) { req, err := http.NewRequestWithContext(ctx, "GET", srv.url, nil) if err != nil { return nil, err diff --git a/internal/common/profiling/http.go b/internal/common/profiling/http.go index 5557ef190c4..88f60344443 100644 --- a/internal/common/profiling/http.go +++ b/internal/common/profiling/http.go @@ -6,12 +6,11 @@ import ( "net/http" _ "net/http/pprof" - log "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/auth" - "github.com/armadaproject/armada/internal/common/logging" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/profiling/configuration" "github.com/armadaproject/armada/internal/common/serve" ) @@ -54,7 +53,7 @@ func SetupPprof(config *configuration.ProfilingConfig, ctx *armadacontext.Contex serveFunc := func() error { if err := serve.ListenAndServe(ctx, pprofServer); err != nil { - logging.WithStacktrace(ctx, err).Error("pprof server failure") + ctx.Logger().WithStacktrace(err).Error("pprof server failure") } return err } diff --git a/internal/common/pulsarutils/publisher.go b/internal/common/pulsarutils/publisher.go index cb4113f1d4f..90b20839cd9 100644 --- a/internal/common/pulsarutils/publisher.go +++ b/internal/common/pulsarutils/publisher.go @@ -10,7 +10,6 @@ import ( "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/ingest/utils" - "github.com/armadaproject/armada/internal/common/logging" psutils "github.com/armadaproject/armada/internal/common/pulsarutils/utils" ) @@ -78,8 +77,8 @@ func (p *PulsarPublisher[T]) PublishMessages(ctx *armadacontext.Context, events for _, msg := range msgs { p.producer.SendAsync(sendCtx, msg, func(_ pulsar.MessageID, _ *pulsar.ProducerMessage, err error) { if err != nil { - logging. - WithStacktrace(ctx, err). + ctx.Logger(). + WithStacktrace(err). Error("error sending message to Pulsar") errored = true } diff --git a/internal/common/pulsarutils/pulsarclient.go b/internal/common/pulsarutils/pulsarclient.go index 22d53c852f5..477910ee905 100644 --- a/internal/common/pulsarutils/pulsarclient.go +++ b/internal/common/pulsarutils/pulsarclient.go @@ -8,6 +8,7 @@ import ( "github.com/armadaproject/armada/internal/common/armadaerrors" commonconfig "github.com/armadaproject/armada/internal/common/config" + "github.com/armadaproject/armada/internal/common/logging" ) func NewPulsarClient(config *commonconfig.PulsarConfig) (pulsar.Client, error) { @@ -39,5 +40,6 @@ func NewPulsarClient(config *commonconfig.PulsarConfig) (pulsar.Client, error) { TLSAllowInsecureConnection: config.TLSAllowInsecureConnection, MaxConnectionsPerBroker: config.MaxConnectionsPerBroker, Authentication: authentication, + Logger: logging.NewPulsarLogger(), }) } diff --git a/internal/common/serve/static.go b/internal/common/serve/static.go index 662965de0b9..355ae05d8db 100644 --- a/internal/common/serve/static.go +++ b/internal/common/serve/static.go @@ -7,7 +7,6 @@ import ( "github.com/pkg/errors" "github.com/armadaproject/armada/internal/common/armadacontext" - "github.com/armadaproject/armada/internal/common/logging" ) // dirWithIndexFallback is a http.FileSystem that serves the index.html file at @@ -41,7 +40,7 @@ func ListenAndServe(ctx *armadacontext.Context, server *http.Server) error { // Shutdown server on ctx done. <-ctx.Done() if err := server.Shutdown(ctx); err != nil { - logging.WithStacktrace(ctx, err).Errorf("failed to shutdown server serving %s", server.Addr) + ctx.Logger().WithStacktrace(err).Errorf("failed to shutdown server serving %s", server.Addr) } }() if err := server.ListenAndServe(); err != nil { diff --git a/internal/common/startup.go b/internal/common/startup.go index 535fb628c5e..87b05f068de 100644 --- a/internal/common/startup.go +++ b/internal/common/startup.go @@ -3,45 +3,39 @@ package common import ( "crypto/tls" "fmt" + "io" "net/http" "os" - "path" - "runtime" - "strconv" + "path/filepath" "strings" "time" "github.com/mitchellh/mapstructure" - "golang.org/x/exp/slices" - - "github.com/armadaproject/armada/internal/common/certs" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - log "github.com/sirupsen/logrus" + "github.com/rs/zerolog" "github.com/spf13/pflag" "github.com/spf13/viper" - "github.com/weaveworks/promrus" + "golang.org/x/exp/slices" "github.com/armadaproject/armada/internal/common/armadacontext" + "github.com/armadaproject/armada/internal/common/certs" commonconfig "github.com/armadaproject/armada/internal/common/config" - "github.com/armadaproject/armada/internal/common/logging" + log "github.com/armadaproject/armada/internal/common/logging" ) const baseConfigFileName = "config" // RFC3339Millis -const logTimestampFormat = "2006-01-02T15:04:05.999Z07:00" +const logTimestampFormat = "2006-01-02T15:04:05.000Z07:00" func BindCommandlineArguments() { err := viper.BindPFlags(pflag.CommandLine) if err != nil { - log.Error() - os.Exit(-1) + log.Fatalf(err.Error()) } } -// TODO Move code relating to config out of common into a new package internal/serverconfig func LoadConfig(config commonconfig.Config, defaultPath string, overrideConfigs []string) *viper.Viper { v := viper.NewWithOptions(viper.KeyDelimiter("::")) v.SetConfigName(baseConfigFileName) @@ -72,8 +66,7 @@ func LoadConfig(config commonconfig.Config, defaultPath string, overrideConfigs var metadata mapstructure.Metadata customHooks := append(slices.Clone(commonconfig.CustomHooks), func(c *mapstructure.DecoderConfig) { c.Metadata = &metadata }) if err := v.Unmarshal(config, customHooks...); err != nil { - log.Error(err) - os.Exit(-1) + log.Fatal(err) } // Log a warning if there are config keys that don't match a config item in the struct the yaml is decoded into. @@ -90,7 +83,7 @@ func LoadConfig(config commonconfig.Config, defaultPath string, overrideConfigs } if err := config.Validate(); err != nil { - log.Error(commonconfig.FormatValidationErrors(err)) + log.Error(commonconfig.FormatValidationErrors(err).Error()) os.Exit(-1) } @@ -103,58 +96,76 @@ func UnmarshalKey(v *viper.Viper, key string, item interface{}) error { // TODO Move logging-related code out of common into a new package internal/logging func ConfigureCommandLineLogging() { - commandLineFormatter := new(logging.CommandLineFormatter) - log.SetFormatter(commandLineFormatter) - log.SetOutput(os.Stdout) + zerolog.TimestampFieldName = "" + zerolog.LevelFieldName = "" + + // Create a ConsoleWriter that writes to stdout. + // Since we’ve cleared out the field names above, only the message will be printed. + consoleWriter := zerolog.ConsoleWriter{ + Out: os.Stdout, + TimeFormat: "", // No timestamp + } + + l := zerolog.New(consoleWriter).Level(zerolog.InfoLevel).With().Logger() + log.ReplaceStdLogger(log.FromZerolog(l)) } func ConfigureLogging() { - log.SetLevel(readEnvironmentLogLevel()) - log.SetFormatter(readEnvironmentLogFormat()) - log.SetReportCaller(true) - log.SetOutput(os.Stdout) + // needs to be higher or greater precision than the writer format. + zerolog.TimeFieldFormat = logTimestampFormat + + level := readEnvironmentLogLevel() + format := readEnvironmentLogFormat() + + var outStream io.Writer = os.Stdout + + if format != "json" { + outStream = zerolog.ConsoleWriter{ + Out: os.Stdout, + TimeFormat: logTimestampFormat, + FormatLevel: func(i interface{}) string { + return strings.ToUpper(fmt.Sprintf("%s", i)) + }, + FormatCaller: func(i interface{}) string { + return filepath.Base(fmt.Sprintf("%s", i)) + }, + NoColor: format == "text", + } + } + + zerologLogger := zerolog.New(outStream). + Level(level). + Hook(log.NewPrometheusHook()). + With(). + CallerWithSkipFrameCount(log.StdSkipFrames). + Timestamp(). + Logger() + + log.ReplaceStdLogger(log.FromZerolog(zerologLogger)) } -func readEnvironmentLogLevel() log.Level { +func readEnvironmentLogLevel() zerolog.Level { level, ok := os.LookupEnv("LOG_LEVEL") if ok { - logLevel, err := log.ParseLevel(level) + logLevel, err := zerolog.ParseLevel(level) if err == nil { return logLevel } } - return log.InfoLevel + return zerolog.InfoLevel } -func readEnvironmentLogFormat() log.Formatter { - formatStr, ok := os.LookupEnv("LOG_FORMAT") - if !ok { - formatStr = "colourful" - } - - textFormatter := &log.TextFormatter{ - ForceColors: true, - FullTimestamp: true, - TimestampFormat: logTimestampFormat, - CallerPrettyfier: func(frame *runtime.Frame) (function string, file string) { - fileName := path.Base(frame.File) + ":" + strconv.Itoa(frame.Line) - return "", fileName - }, - } - - switch strings.ToLower(formatStr) { - case "json": - return &log.JSONFormatter{TimestampFormat: logTimestampFormat} - case "colourful": - return textFormatter - case "text": - textFormatter.ForceColors = false - textFormatter.DisableColors = true - return textFormatter - default: - println(os.Stderr, fmt.Sprintf("Unknown log format %s, defaulting to colourful format", formatStr)) - return textFormatter +func readEnvironmentLogFormat() string { + format, ok := os.LookupEnv("LOG_FORMAT") + if ok { + format = strings.ToLower(format) + if format == "text" || format == "colourful" || format == "json" { + return format + } else { + _, _ = fmt.Fprintf(os.Stderr, "Invalid log format %s\n", format) + } } + return "colourful" } func ServeMetrics(port uint16) (shutdown func()) { @@ -162,9 +173,6 @@ func ServeMetrics(port uint16) (shutdown func()) { } func ServeMetricsFor(port uint16, gatherer prometheus.Gatherer) (shutdown func()) { - hook := promrus.MustNewPrometheusHook() - log.AddHook(hook) - mux := http.NewServeMux() mux.Handle("/metrics", promhttp.HandlerFor(gatherer, promhttp.HandlerOpts{})) return ServeHttp(port, mux) @@ -192,7 +200,7 @@ func serveHttp(port uint16, mux http.Handler, useTls bool, certFile, keyFile str } go func() { - log.Printf("Starting %s server listening on %d", scheme, port) + log.Infof("Starting %s server listening on %d", scheme, port) var err error if useTls { certWatcher := certs.NewCachedCertificateService(certFile, keyFile, time.Minute) @@ -217,7 +225,7 @@ func serveHttp(port uint16, mux http.Handler, useTls bool, certFile, keyFile str return func() { ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), 5*time.Second) defer cancel() - log.Printf("Stopping %s server listening on %d", scheme, port) + log.Infof("Stopping %s server listening on %d", scheme, port) e := srv.Shutdown(ctx) if e != nil { panic(e) diff --git a/internal/eventingester/convert/conversions.go b/internal/eventingester/convert/conversions.go index 17f69384f40..cbbd629419e 100644 --- a/internal/eventingester/convert/conversions.go +++ b/internal/eventingester/convert/conversions.go @@ -3,7 +3,6 @@ package convert import ( "github.com/gogo/protobuf/proto" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/compress" @@ -11,6 +10,7 @@ import ( "github.com/armadaproject/armada/internal/common/ingest" "github.com/armadaproject/armada/internal/common/ingest/metrics" "github.com/armadaproject/armada/internal/common/ingest/utils" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/eventingester/model" "github.com/armadaproject/armada/pkg/armadaevents" ) diff --git a/internal/eventingester/ingester.go b/internal/eventingester/ingester.go index b6b7c207cbc..bf0185665fb 100644 --- a/internal/eventingester/ingester.go +++ b/internal/eventingester/ingester.go @@ -7,7 +7,6 @@ import ( "github.com/apache/pulsar-client-go/pulsar" "github.com/pkg/errors" "github.com/redis/go-redis/v9" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common" "github.com/armadaproject/armada/internal/common/app" @@ -15,6 +14,7 @@ import ( "github.com/armadaproject/armada/internal/common/compress" "github.com/armadaproject/armada/internal/common/ingest" "github.com/armadaproject/armada/internal/common/ingest/jobsetevents" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/profiling" "github.com/armadaproject/armada/internal/eventingester/configuration" "github.com/armadaproject/armada/internal/eventingester/convert" diff --git a/internal/eventingester/store/eventstore.go b/internal/eventingester/store/eventstore.go index 5682acc8f0d..20a5012a48f 100644 --- a/internal/eventingester/store/eventstore.go +++ b/internal/eventingester/store/eventstore.go @@ -6,10 +6,10 @@ import ( "github.com/hashicorp/go-multierror" "github.com/redis/go-redis/v9" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/ingest" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/eventingester/configuration" "github.com/armadaproject/armada/internal/eventingester/model" ) diff --git a/internal/executor/application.go b/internal/executor/application.go index b737eca2a71..a65cdde84f5 100644 --- a/internal/executor/application.go +++ b/internal/executor/application.go @@ -11,7 +11,6 @@ import ( "github.com/go-playground/validator/v10" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" "google.golang.org/grpc" "google.golang.org/grpc/keepalive" "k8s.io/utils/clock" @@ -39,10 +38,10 @@ import ( "github.com/armadaproject/armada/pkg/executorapi" ) -func StartUp(ctx *armadacontext.Context, log *logrus.Entry, config configuration.ExecutorConfiguration) (func(), *sync.WaitGroup) { +func StartUp(ctx *armadacontext.Context, config configuration.ExecutorConfiguration) (func(), *sync.WaitGroup) { err := validateConfig(config) if err != nil { - log.Errorf("Invalid config: %s", err) + ctx.Errorf("Invalid config: %s", err) os.Exit(-1) } @@ -52,7 +51,7 @@ func StartUp(ctx *armadacontext.Context, log *logrus.Entry, config configuration config.Kubernetes.Burst, ) if err != nil { - log.Errorf("Failed to connect to kubernetes because %s", err) + ctx.Errorf("Failed to connect to kubernetes because %s", err) os.Exit(-1) } @@ -87,17 +86,17 @@ func StartUp(ctx *armadacontext.Context, log *logrus.Entry, config configuration } var etcdClustersHealthMonitoring healthmonitor.HealthMonitor if len(etcdClusterHealthMonitoringByName) > 0 { - log.Info("etcd URLs provided; monitoring etcd health enabled") + ctx.Info("etcd URLs provided; monitoring etcd health enabled") etcdClustersHealthMonitoring = healthmonitor.NewMultiHealthMonitor( "overall_etcd", etcdClusterHealthMonitoringByName, ).WithMetricsPrefix( metrics.ArmadaExecutorMetricsPrefix, ) - g.Go(func() error { return etcdClustersHealthMonitoring.Run(ctx, log) }) + g.Go(func() error { return etcdClustersHealthMonitoring.Run(ctx) }) prometheus.MustRegister(etcdClustersHealthMonitoring) } else { - log.Info("no etcd URLs provided; etcd health isn't monitored") + ctx.Info("no etcd URLs provided; etcd health isn't monitored") } clusterContext := executor_context.NewClusterContext( @@ -113,11 +112,11 @@ func StartUp(ctx *armadacontext.Context, log *logrus.Entry, config configuration taskManager := task.NewBackgroundTaskManager(metrics.ArmadaExecutorMetricsPrefix) taskManager.Register(clusterContext.ProcessPodsToDelete, config.Task.PodDeletionInterval, "pod_deletion") - return StartUpWithContext(log, config, clusterContext, etcdClustersHealthMonitoring, taskManager, wg) + return StartUpWithContext(ctx, config, clusterContext, etcdClustersHealthMonitoring, taskManager, wg) } func StartUpWithContext( - log *logrus.Entry, + ctx *armadacontext.Context, config configuration.ExecutorConfiguration, clusterContext executor_context.ClusterContext, clusterHealthMonitor healthmonitor.HealthMonitor, @@ -133,21 +132,18 @@ func StartUpWithContext( ) if config.Kubernetes.PendingPodChecks == nil { - log.Error("Config error: Missing pending pod checks") - os.Exit(-1) + ctx.Fatalf("Config error: Missing pending pod checks") } pendingPodChecker, err := podchecks.NewPodChecks(*config.Kubernetes.PendingPodChecks) if err != nil { - log.Errorf("Config error in pending pod checks: %s", err) - os.Exit(-1) + ctx.Fatalf("Config error in pending pod checks: %s", err) } - stopExecutorApiComponents := setupExecutorApiComponents(log, config, clusterContext, clusterHealthMonitor, taskManager, pendingPodChecker, nodeInfoService, podUtilisationService) + stopExecutorApiComponents := setupExecutorApiComponents(ctx, config, clusterContext, clusterHealthMonitor, taskManager, pendingPodChecker, nodeInfoService, podUtilisationService) resourceCleanupService, err := service.NewResourceCleanupService(clusterContext, config.Kubernetes) if err != nil { - log.Errorf("Error creating resource cleanup service: %s", err) - os.Exit(-1) + ctx.Fatalf("Error creating resource cleanup service: %s", err) } taskManager.Register(resourceCleanupService.CleanupResources, config.Task.ResourceCleanupInterval, "resource_cleanup") @@ -159,15 +155,15 @@ func StartUpWithContext( clusterContext.Stop() stopExecutorApiComponents() if taskManager.StopAll(10 * time.Second) { - log.Warnf("Graceful shutdown timed out") + ctx.Warnf("Graceful shutdown timed out") } - log.Infof("Shutdown complete") + ctx.Infof("Shutdown complete") wg.Done() }, wg } func setupExecutorApiComponents( - log *logrus.Entry, + ctx *armadacontext.Context, config configuration.ExecutorConfiguration, clusterContext executor_context.ClusterContext, clusterHealthMonitor healthmonitor.HealthMonitor, @@ -178,8 +174,7 @@ func setupExecutorApiComponents( ) func() { conn, err := createConnectionToApi(config.ExecutorApiConnection, config.Client.MaxMessageSizeBytes, config.GRPC) if err != nil { - log.Errorf("Failed to connect to Executor API because: %s", err) - os.Exit(-1) + ctx.Fatalf("Failed to connect to Executor API because: %s", err) } executorApiClient := executorapi.NewExecutorApiClient(conn) @@ -203,8 +198,7 @@ func setupExecutorApiComponents( clock.RealClock{}, 200) if err != nil { - log.Errorf("Failed to create job event reporter: %s", err) - os.Exit(-1) + ctx.Fatalf("Failed to create job event reporter: %s", err) } submitter := job.NewSubmitter( @@ -243,8 +237,7 @@ func setupExecutorApiComponents( config.Kubernetes.StuckTerminatingPodExpiry, ) if err != nil { - log.Errorf("Failed to create pod issue service: %s", err) - os.Exit(-1) + ctx.Fatalf("Failed to create pod issue service: %s", err) } taskManager.Register(podIssueService.HandlePodIssues, config.Task.PodIssueHandlingInterval, "pod_issue_handling") @@ -255,8 +248,7 @@ func setupExecutorApiComponents( taskManager.Register(eventReporter.ReportMissingJobEvents, config.Task.MissingJobEventReconciliationInterval, "event_reconciliation") _, err = pod_metrics.ExposeClusterContextMetrics(clusterContext, clusterUtilisationService, podUtilisationService, nodeInfoService) if err != nil { - log.Errorf("Failed to setup cluster context metrics: %s", err) - os.Exit(-1) + ctx.Fatalf("Failed to setup cluster context metrics: %s", err) } runStateMetricsCollector := runstate.NewJobRunStateStoreMetricsCollector(jobRunState) prometheus.MustRegister(runStateMetricsCollector) @@ -268,8 +260,7 @@ func setupExecutorApiComponents( eventReporter, config.Task.UtilisationEventReportingInterval) if err != nil { - log.Errorf("Failed to pod utilisation reporter: %s", err) - os.Exit(-1) + ctx.Fatalf("Failed to pod utilisation reporter: %s", err) } taskManager.Register( podUtilisationReporter.ReportUtilisationEvents, diff --git a/internal/executor/context/cluster_context.go b/internal/executor/context/cluster_context.go index 53773658e45..25c4cc8176a 100644 --- a/internal/executor/context/cluster_context.go +++ b/internal/executor/context/cluster_context.go @@ -6,7 +6,6 @@ import ( "time" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" networking "k8s.io/api/networking/v1" @@ -27,6 +26,7 @@ import ( "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/cluster" + log "github.com/armadaproject/armada/internal/common/logging" util2 "github.com/armadaproject/armada/internal/common/util" "github.com/armadaproject/armada/internal/executor/configuration" "github.com/armadaproject/armada/internal/executor/domain" diff --git a/internal/executor/fake/application.go b/internal/executor/fake/application.go index 8f26b43405f..ed34153c8e0 100644 --- a/internal/executor/fake/application.go +++ b/internal/executor/fake/application.go @@ -3,8 +3,7 @@ package fake import ( "sync" - "github.com/sirupsen/logrus" - + "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/task" "github.com/armadaproject/armada/internal/executor" "github.com/armadaproject/armada/internal/executor/configuration" @@ -12,11 +11,11 @@ import ( "github.com/armadaproject/armada/internal/executor/metrics" ) -func StartUp(config configuration.ExecutorConfiguration, nodes []*context.NodeSpec) (func(), *sync.WaitGroup) { +func StartUp(ctx *armadacontext.Context, config configuration.ExecutorConfiguration, nodes []*context.NodeSpec) (func(), *sync.WaitGroup) { wg := &sync.WaitGroup{} wg.Add(1) return executor.StartUpWithContext( - logrus.NewEntry(logrus.StandardLogger()), + ctx, config, context.NewFakeClusterContext(config.Application, config.Kubernetes.NodeIdLabel, nodes), nil, diff --git a/internal/executor/fake/context/context.go b/internal/executor/fake/context/context.go index e4ec04a0c9e..612cd33ecd8 100644 --- a/internal/executor/fake/context/context.go +++ b/internal/executor/fake/context/context.go @@ -12,7 +12,6 @@ import ( "github.com/google/uuid" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "golang.org/x/exp/maps" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" @@ -24,6 +23,7 @@ import ( "k8s.io/kubelet/pkg/apis/stats/v1alpha1" "github.com/armadaproject/armada/internal/common/armadacontext" + log "github.com/armadaproject/armada/internal/common/logging" armadaresource "github.com/armadaproject/armada/internal/common/resource" "github.com/armadaproject/armada/internal/executor/configuration" cluster_context "github.com/armadaproject/armada/internal/executor/context" diff --git a/internal/executor/job/job_run_state_store.go b/internal/executor/job/job_run_state_store.go index dfe648fd7eb..646edc1f229 100644 --- a/internal/executor/job/job_run_state_store.go +++ b/internal/executor/job/job_run_state_store.go @@ -4,10 +4,10 @@ import ( "sync" "time" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/executor/context" "github.com/armadaproject/armada/internal/executor/util" ) diff --git a/internal/executor/job/processors/preempt_runs.go b/internal/executor/job/processors/preempt_runs.go index 9d98a73cfde..959c86f9d42 100644 --- a/internal/executor/job/processors/preempt_runs.go +++ b/internal/executor/job/processors/preempt_runs.go @@ -4,10 +4,10 @@ import ( "fmt" "time" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "github.com/armadaproject/armada/internal/common/armadacontext" + log "github.com/armadaproject/armada/internal/common/logging" executorContext "github.com/armadaproject/armada/internal/executor/context" "github.com/armadaproject/armada/internal/executor/domain" "github.com/armadaproject/armada/internal/executor/job" diff --git a/internal/executor/job/processors/remove_runs.go b/internal/executor/job/processors/remove_runs.go index 37942110605..d81f2958e5d 100644 --- a/internal/executor/job/processors/remove_runs.go +++ b/internal/executor/job/processors/remove_runs.go @@ -3,10 +3,10 @@ package processors import ( "time" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "github.com/armadaproject/armada/internal/common/armadacontext" + log "github.com/armadaproject/armada/internal/common/logging" executorContext "github.com/armadaproject/armada/internal/executor/context" "github.com/armadaproject/armada/internal/executor/domain" "github.com/armadaproject/armada/internal/executor/job" diff --git a/internal/executor/job/submit.go b/internal/executor/job/submit.go index 9943a0f3b68..03265ba3395 100644 --- a/internal/executor/job/submit.go +++ b/internal/executor/job/submit.go @@ -6,12 +6,12 @@ import ( "sync" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/armadaproject/armada/internal/common/armadaerrors" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/util" "github.com/armadaproject/armada/internal/executor/configuration" "github.com/armadaproject/armada/internal/executor/context" diff --git a/internal/executor/metrics/pod_metrics/cluster_context.go b/internal/executor/metrics/pod_metrics/cluster_context.go index e86881411b1..028c5f1f86a 100644 --- a/internal/executor/metrics/pod_metrics/cluster_context.go +++ b/internal/executor/metrics/pod_metrics/cluster_context.go @@ -3,11 +3,11 @@ package pod_metrics import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/client-go/tools/cache" + log "github.com/armadaproject/armada/internal/common/logging" armadaresource "github.com/armadaproject/armada/internal/common/resource" "github.com/armadaproject/armada/internal/executor/context" "github.com/armadaproject/armada/internal/executor/domain" diff --git a/internal/executor/podchecks/container_state_checks.go b/internal/executor/podchecks/container_state_checks.go index 51826afd88b..aaaee5faa48 100644 --- a/internal/executor/podchecks/container_state_checks.go +++ b/internal/executor/podchecks/container_state_checks.go @@ -6,11 +6,11 @@ import ( "strings" "time" + v1 "k8s.io/api/core/v1" + + log "github.com/armadaproject/armada/internal/common/logging" config "github.com/armadaproject/armada/internal/executor/configuration/podchecks" "github.com/armadaproject/armada/internal/executor/util" - - log "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" ) type containerStateChecker interface { diff --git a/internal/executor/podchecks/event_checks.go b/internal/executor/podchecks/event_checks.go index ddfe5469c4d..ee0a6a24631 100644 --- a/internal/executor/podchecks/event_checks.go +++ b/internal/executor/podchecks/event_checks.go @@ -6,9 +6,9 @@ import ( "strings" "time" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" + log "github.com/armadaproject/armada/internal/common/logging" config "github.com/armadaproject/armada/internal/executor/configuration/podchecks" ) diff --git a/internal/executor/podchecks/pod_checks.go b/internal/executor/podchecks/pod_checks.go index 496de36fd89..73be56223b1 100644 --- a/internal/executor/podchecks/pod_checks.go +++ b/internal/executor/podchecks/pod_checks.go @@ -5,9 +5,9 @@ import ( "strings" "time" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/slices" config "github.com/armadaproject/armada/internal/executor/configuration/podchecks" "github.com/armadaproject/armada/internal/executor/util" diff --git a/internal/executor/reporter/job_event_reporter.go b/internal/executor/reporter/job_event_reporter.go index 002b2d8d833..3df11d42696 100644 --- a/internal/executor/reporter/job_event_reporter.go +++ b/internal/executor/reporter/job_event_reporter.go @@ -4,11 +4,11 @@ import ( "sync" "time" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/utils/clock" + log "github.com/armadaproject/armada/internal/common/logging" clusterContext "github.com/armadaproject/armada/internal/executor/context" domain2 "github.com/armadaproject/armada/internal/executor/domain" "github.com/armadaproject/armada/internal/executor/job" diff --git a/internal/executor/service/cluster_allocation.go b/internal/executor/service/cluster_allocation.go index 405a7308128..703d41a0526 100644 --- a/internal/executor/service/cluster_allocation.go +++ b/internal/executor/service/cluster_allocation.go @@ -3,12 +3,11 @@ package service import ( "fmt" - "github.com/sirupsen/logrus" - log "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/errors" "github.com/armadaproject/armada/internal/common/healthmonitor" "github.com/armadaproject/armada/internal/common/logging" + log "github.com/armadaproject/armada/internal/common/logging" armadaresource "github.com/armadaproject/armada/internal/common/resource" util2 "github.com/armadaproject/armada/internal/common/util" executorContext "github.com/armadaproject/armada/internal/executor/context" @@ -49,7 +48,7 @@ func (allocationService *ClusterAllocationService) AllocateSpareClusterCapacity( // If a health monitor is provided, avoid leasing jobs when the cluster is unhealthy. if allocationService.clusterHealthMonitor != nil { if ok, reason, err := allocationService.clusterHealthMonitor.IsHealthy(); err != nil { - logging.WithStacktrace(logrus.NewEntry(logrus.StandardLogger()), err).Error("failed to check cluster health") + logging.WithStacktrace(err).Error("failed to check cluster health") return } else if !ok { log.Warnf("cluster is not healthy; will not request more jobs: %s", reason) diff --git a/internal/executor/service/job_requester.go b/internal/executor/service/job_requester.go index 9f06c0c0cc6..f6dd2c38b2d 100644 --- a/internal/executor/service/job_requester.go +++ b/internal/executor/service/job_requester.go @@ -4,10 +4,10 @@ import ( "fmt" "time" - log "github.com/sirupsen/logrus" "golang.org/x/exp/maps" "github.com/armadaproject/armada/internal/common/armadacontext" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/slices" "github.com/armadaproject/armada/internal/executor/configuration" executorContext "github.com/armadaproject/armada/internal/executor/context" diff --git a/internal/executor/service/lease_requester.go b/internal/executor/service/lease_requester.go index c9451962b89..39913fd0e79 100644 --- a/internal/executor/service/lease_requester.go +++ b/internal/executor/service/lease_requester.go @@ -6,11 +6,11 @@ import ( grpcretry "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/retry" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "google.golang.org/grpc" "google.golang.org/grpc/encoding/gzip" "github.com/armadaproject/armada/internal/common/armadacontext" + log "github.com/armadaproject/armada/internal/common/logging" armadaresource "github.com/armadaproject/armada/internal/common/resource" clusterContext "github.com/armadaproject/armada/internal/executor/context" "github.com/armadaproject/armada/pkg/executorapi" diff --git a/internal/executor/service/pod_issue_handler.go b/internal/executor/service/pod_issue_handler.go index c94a5b175e0..72e9ea57531 100644 --- a/internal/executor/service/pod_issue_handler.go +++ b/internal/executor/service/pod_issue_handler.go @@ -6,13 +6,13 @@ import ( "sync" "time" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/kubectl/pkg/describe" "k8s.io/utils/clock" "github.com/armadaproject/armada/internal/common/armadacontext" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/executor/configuration" executorContext "github.com/armadaproject/armada/internal/executor/context" "github.com/armadaproject/armada/internal/executor/job" diff --git a/internal/executor/service/resource_cleanup.go b/internal/executor/service/resource_cleanup.go index 2657d142750..41196e20721 100644 --- a/internal/executor/service/resource_cleanup.go +++ b/internal/executor/service/resource_cleanup.go @@ -4,10 +4,10 @@ import ( "sort" "time" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/executor/configuration" clusterContext "github.com/armadaproject/armada/internal/executor/context" "github.com/armadaproject/armada/internal/executor/util" diff --git a/internal/executor/util/pod_util.go b/internal/executor/util/pod_util.go index 4848f8044cd..72543c14875 100644 --- a/internal/executor/util/pod_util.go +++ b/internal/executor/util/pod_util.go @@ -6,11 +6,11 @@ import ( "strconv" "time" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/util" "github.com/armadaproject/armada/internal/executor/domain" "github.com/armadaproject/armada/internal/server/configuration" diff --git a/internal/executor/utilisation/cluster_utilisation.go b/internal/executor/utilisation/cluster_utilisation.go index a26726a8ea5..e2549731796 100644 --- a/internal/executor/utilisation/cluster_utilisation.go +++ b/internal/executor/utilisation/cluster_utilisation.go @@ -4,9 +4,9 @@ import ( "fmt" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" + log "github.com/armadaproject/armada/internal/common/logging" armadaresource "github.com/armadaproject/armada/internal/common/resource" armadaslices "github.com/armadaproject/armada/internal/common/slices" "github.com/armadaproject/armada/internal/executor/context" diff --git a/internal/executor/utilisation/job_utilisation_reporter.go b/internal/executor/utilisation/job_utilisation_reporter.go index d40ce0919ae..b4387db22bf 100644 --- a/internal/executor/utilisation/job_utilisation_reporter.go +++ b/internal/executor/utilisation/job_utilisation_reporter.go @@ -4,10 +4,10 @@ import ( "sync" "time" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" + log "github.com/armadaproject/armada/internal/common/logging" clusterContext "github.com/armadaproject/armada/internal/executor/context" "github.com/armadaproject/armada/internal/executor/domain" "github.com/armadaproject/armada/internal/executor/reporter" diff --git a/internal/executor/utilisation/pod_utilisation.go b/internal/executor/utilisation/pod_utilisation.go index ae8d43092c5..ec075b198e8 100644 --- a/internal/executor/utilisation/pod_utilisation.go +++ b/internal/executor/utilisation/pod_utilisation.go @@ -5,9 +5,9 @@ import ( "sync" "time" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" + log "github.com/armadaproject/armada/internal/common/logging" armadaresource "github.com/armadaproject/armada/internal/common/resource" commonUtil "github.com/armadaproject/armada/internal/common/util" "github.com/armadaproject/armada/internal/executor/configuration" diff --git a/internal/executor/utilisation/pod_utilisation_custom_metrics.go b/internal/executor/utilisation/pod_utilisation_custom_metrics.go index dffaf5c25c5..bf6102dc1f8 100644 --- a/internal/executor/utilisation/pod_utilisation_custom_metrics.go +++ b/internal/executor/utilisation/pod_utilisation_custom_metrics.go @@ -5,11 +5,11 @@ import ( "net/http" "github.com/prometheus/common/model" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/utils/clock" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/executor/configuration" clusterContext "github.com/armadaproject/armada/internal/executor/context" "github.com/armadaproject/armada/internal/executor/domain" diff --git a/internal/executor/utilisation/pod_utilisation_kubelet_metrics.go b/internal/executor/utilisation/pod_utilisation_kubelet_metrics.go index d7b32d01c7e..4a7aef1ac11 100644 --- a/internal/executor/utilisation/pod_utilisation_kubelet_metrics.go +++ b/internal/executor/utilisation/pod_utilisation_kubelet_metrics.go @@ -4,13 +4,12 @@ import ( "sync" "time" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/kubelet/pkg/apis/stats/v1alpha1" - log "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" - "github.com/armadaproject/armada/internal/common/armadacontext" + log "github.com/armadaproject/armada/internal/common/logging" clusterContext "github.com/armadaproject/armada/internal/executor/context" "github.com/armadaproject/armada/internal/executor/domain" ) diff --git a/internal/executor/utilisation/prometheus_scraping.go b/internal/executor/utilisation/prometheus_scraping.go index 2e71dcf4cac..be98f0964f3 100644 --- a/internal/executor/utilisation/prometheus_scraping.go +++ b/internal/executor/utilisation/prometheus_scraping.go @@ -6,13 +6,12 @@ import ( "net/http" "sync" + "github.com/prometheus/common/expfmt" + "github.com/prometheus/common/model" discovery "k8s.io/api/discovery/v1" + log "github.com/armadaproject/armada/internal/common/logging" commonUtil "github.com/armadaproject/armada/internal/common/util" - - "github.com/prometheus/common/expfmt" - "github.com/prometheus/common/model" - log "github.com/sirupsen/logrus" ) type httpGetter interface { diff --git a/internal/lookoutingesterv2/dbloadtester/queue.go b/internal/lookoutingesterv2/dbloadtester/queue.go index 6ad9775488f..30c60cbe3b5 100644 --- a/internal/lookoutingesterv2/dbloadtester/queue.go +++ b/internal/lookoutingesterv2/dbloadtester/queue.go @@ -8,11 +8,11 @@ import ( "github.com/apache/pulsar-client-go/pulsar" "github.com/google/uuid" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "github.com/armadaproject/armada/internal/common/database/lookout" "github.com/armadaproject/armada/internal/common/ingest/utils" + log "github.com/armadaproject/armada/internal/common/logging" protoutil "github.com/armadaproject/armada/internal/common/proto" "github.com/armadaproject/armada/internal/common/util" "github.com/armadaproject/armada/pkg/armadaevents" @@ -79,7 +79,7 @@ func (q *QueueEventGenerator) Generate(eventsCh chan<- *utils.EventsWithIds[*arm } events, err := q.generateEventsAtTime(i) if err != nil { - log.Panicf("failed to generate events %s", err) + log.Fatalf("failed to generate events %s", err) } if len(events) == 0 { continue diff --git a/internal/lookoutingesterv2/dbloadtester/simulator.go b/internal/lookoutingesterv2/dbloadtester/simulator.go index c15e31825c9..b298bf74827 100644 --- a/internal/lookoutingesterv2/dbloadtester/simulator.go +++ b/internal/lookoutingesterv2/dbloadtester/simulator.go @@ -8,7 +8,6 @@ import ( "time" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "github.com/armadaproject/armada/internal/common/armadacontext" @@ -16,6 +15,7 @@ import ( "github.com/armadaproject/armada/internal/common/database" "github.com/armadaproject/armada/internal/common/ingest" "github.com/armadaproject/armada/internal/common/ingest/utils" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/lookoutingesterv2/configuration" "github.com/armadaproject/armada/internal/lookoutingesterv2/instructions" "github.com/armadaproject/armada/internal/lookoutingesterv2/lookoutdb" diff --git a/internal/lookoutingesterv2/ingester.go b/internal/lookoutingesterv2/ingester.go index 38eceaf7665..feadfce9939 100644 --- a/internal/lookoutingesterv2/ingester.go +++ b/internal/lookoutingesterv2/ingester.go @@ -5,7 +5,6 @@ import ( "github.com/apache/pulsar-client-go/pulsar" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common" "github.com/armadaproject/armada/internal/common/app" @@ -14,6 +13,7 @@ import ( "github.com/armadaproject/armada/internal/common/database" "github.com/armadaproject/armada/internal/common/ingest" "github.com/armadaproject/armada/internal/common/ingest/jobsetevents" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/profiling" "github.com/armadaproject/armada/internal/lookoutingesterv2/configuration" "github.com/armadaproject/armada/internal/lookoutingesterv2/instructions" diff --git a/internal/lookoutingesterv2/instructions/instructions.go b/internal/lookoutingesterv2/instructions/instructions.go index 6be893b85ea..04e85cc13e6 100644 --- a/internal/lookoutingesterv2/instructions/instructions.go +++ b/internal/lookoutingesterv2/instructions/instructions.go @@ -5,7 +5,6 @@ import ( "time" "github.com/gogo/protobuf/proto" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" @@ -15,6 +14,7 @@ import ( "github.com/armadaproject/armada/internal/common/eventutil" "github.com/armadaproject/armada/internal/common/ingest/metrics" "github.com/armadaproject/armada/internal/common/ingest/utils" + log "github.com/armadaproject/armada/internal/common/logging" protoutil "github.com/armadaproject/armada/internal/common/proto" "github.com/armadaproject/armada/internal/common/util" "github.com/armadaproject/armada/internal/lookoutingesterv2/model" @@ -81,7 +81,7 @@ func (c *InstructionConverter) convertSequence( var err error if event.Created == nil { c.metrics.RecordPulsarMessageError(metrics.PulsarMessageErrorProcessing) - log.WithError(err).Warnf("Missing timestamp for event at index %d.", idx) + log.Warnf("Missing timestamp for event at index %d.", idx) continue } ts := protoutil.ToStdTime(event.Created) diff --git a/internal/lookoutingesterv2/lookoutdb/insertion.go b/internal/lookoutingesterv2/lookoutdb/insertion.go index 14af9937ffa..6b68c40d27c 100644 --- a/internal/lookoutingesterv2/lookoutdb/insertion.go +++ b/internal/lookoutingesterv2/lookoutdb/insertion.go @@ -8,12 +8,12 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/armadaerrors" "github.com/armadaproject/armada/internal/common/database/lookout" commonmetrics "github.com/armadaproject/armada/internal/common/ingest/metrics" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/lookoutingesterv2/metrics" "github.com/armadaproject/armada/internal/lookoutingesterv2/model" ) diff --git a/internal/lookoutv2/application.go b/internal/lookoutv2/application.go index bf7c03a9ecd..ec6dbf6b10e 100644 --- a/internal/lookoutv2/application.go +++ b/internal/lookoutv2/application.go @@ -8,12 +8,12 @@ import ( "github.com/go-openapi/runtime/middleware" "github.com/jessevdk/go-flags" "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/compress" "github.com/armadaproject/armada/internal/common/database" + "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/slices" "github.com/armadaproject/armada/internal/lookoutv2/configuration" "github.com/armadaproject/armada/internal/lookoutv2/conversions" @@ -48,7 +48,7 @@ func Serve(configuration configuration.LookoutV2Config) error { // create new service API api := operations.NewLookoutAPI(swaggerSpec) - logger := logrus.NewEntry(logrus.StandardLogger()) + logger := logging.StdLogger() api.Logger = logger.Debugf diff --git a/internal/lookoutv2/generate/main.go b/internal/lookoutv2/generate/main.go index a916b94fd2d..28a607d89df 100644 --- a/internal/lookoutv2/generate/main.go +++ b/internal/lookoutv2/generate/main.go @@ -6,7 +6,7 @@ import ( "os/exec" "path/filepath" - log "github.com/sirupsen/logrus" + log "github.com/armadaproject/armada/internal/common/logging" ) const ( diff --git a/internal/lookoutv2/pruner/pruner.go b/internal/lookoutv2/pruner/pruner.go index 397bb3b1538..425185a3e33 100644 --- a/internal/lookoutv2/pruner/pruner.go +++ b/internal/lookoutv2/pruner/pruner.go @@ -6,10 +6,10 @@ import ( "github.com/hashicorp/go-multierror" "github.com/jackc/pgx/v5" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "k8s.io/utils/clock" "github.com/armadaproject/armada/internal/common/armadacontext" + log "github.com/armadaproject/armada/internal/common/logging" ) func PruneDb( diff --git a/internal/lookoutv2/repository/getjoberror.go b/internal/lookoutv2/repository/getjoberror.go index ebcb14a59fa..d418ed89990 100644 --- a/internal/lookoutv2/repository/getjoberror.go +++ b/internal/lookoutv2/repository/getjoberror.go @@ -4,10 +4,10 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/compress" + log "github.com/armadaproject/armada/internal/common/logging" ) type GetJobErrorRepository interface { diff --git a/internal/lookoutv2/repository/getjobrundebugmessage.go b/internal/lookoutv2/repository/getjobrundebugmessage.go index 5b4841dc7c7..ace8f6d0ca7 100644 --- a/internal/lookoutv2/repository/getjobrundebugmessage.go +++ b/internal/lookoutv2/repository/getjobrundebugmessage.go @@ -4,10 +4,10 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/compress" + log "github.com/armadaproject/armada/internal/common/logging" ) type GetJobRunDebugMessageRepository interface { diff --git a/internal/lookoutv2/repository/getjobrunerror.go b/internal/lookoutv2/repository/getjobrunerror.go index b878c9291fb..b97b0ab8206 100644 --- a/internal/lookoutv2/repository/getjobrunerror.go +++ b/internal/lookoutv2/repository/getjobrunerror.go @@ -4,10 +4,10 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/compress" + log "github.com/armadaproject/armada/internal/common/logging" ) type GetJobRunErrorRepository interface { diff --git a/internal/lookoutv2/repository/getjobspec.go b/internal/lookoutv2/repository/getjobspec.go index 8e51fe89871..ea1216e3a30 100644 --- a/internal/lookoutv2/repository/getjobspec.go +++ b/internal/lookoutv2/repository/getjobspec.go @@ -5,10 +5,10 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/compress" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/pkg/api" ) diff --git a/internal/lookoutv2/repository/querybuilder.go b/internal/lookoutv2/repository/querybuilder.go index 01255998086..18c93c3ed2f 100644 --- a/internal/lookoutv2/repository/querybuilder.go +++ b/internal/lookoutv2/repository/querybuilder.go @@ -5,10 +5,10 @@ import ( "strings" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "golang.org/x/exp/slices" "github.com/armadaproject/armada/internal/common/database/lookout" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/lookoutv2/model" ) @@ -411,7 +411,7 @@ func operatorForMatch(match string) (string, error) { return "<=", nil default: err := errors.Errorf("unsupported match type: %s", match) - logrus.Error(err) + log.Error(err.Error()) return "", err } } diff --git a/internal/lookoutv2/repository/util.go b/internal/lookoutv2/repository/util.go index 1072f91a77b..6c136532626 100644 --- a/internal/lookoutv2/repository/util.go +++ b/internal/lookoutv2/repository/util.go @@ -8,7 +8,6 @@ import ( "github.com/apache/pulsar-client-go/pulsar" "github.com/gogo/protobuf/types" "github.com/google/uuid" - log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/utils/clock" @@ -18,6 +17,7 @@ import ( "github.com/armadaproject/armada/internal/common/database/lookout" "github.com/armadaproject/armada/internal/common/eventutil" "github.com/armadaproject/armada/internal/common/ingest/utils" + log "github.com/armadaproject/armada/internal/common/logging" protoutil "github.com/armadaproject/armada/internal/common/proto" "github.com/armadaproject/armada/internal/common/pulsarutils" "github.com/armadaproject/armada/internal/common/util" diff --git a/internal/scheduler/api.go b/internal/scheduler/api.go index daf8efe8fd8..608faa82a85 100644 --- a/internal/scheduler/api.go +++ b/internal/scheduler/api.go @@ -7,7 +7,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/gogo/protobuf/types" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" v1 "k8s.io/api/core/v1" @@ -17,7 +16,7 @@ import ( "github.com/armadaproject/armada/internal/common/armadaerrors" "github.com/armadaproject/armada/internal/common/auth" "github.com/armadaproject/armada/internal/common/compress" - "github.com/armadaproject/armada/internal/common/logging" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/maps" "github.com/armadaproject/armada/internal/common/pulsarutils" priorityTypes "github.com/armadaproject/armada/internal/common/types" @@ -371,7 +370,7 @@ func (srv *ExecutorApi) executorFromLeaseRequest(ctx *armadacontext.Context, req now := srv.clock.Now().UTC() for _, nodeInfo := range req.Nodes { if node, err := executorapi.NewNodeFromNodeInfo(nodeInfo, req.ExecutorId, srv.allowedPriorities, now); err != nil { - logging.WithStacktrace(ctx, err).Warnf( + ctx.Logger().WithStacktrace(err).Warnf( "skipping node %s from executor %s", nodeInfo.GetName(), req.GetExecutorId(), ) } else { diff --git a/internal/scheduler/metrics.go b/internal/scheduler/metrics.go index 8cc2f3f49d1..a64f040efb5 100644 --- a/internal/scheduler/metrics.go +++ b/internal/scheduler/metrics.go @@ -10,7 +10,6 @@ import ( "k8s.io/utils/clock" "github.com/armadaproject/armada/internal/common/armadacontext" - "github.com/armadaproject/armada/internal/common/logging" armadamaps "github.com/armadaproject/armada/internal/common/maps" commonmetrics "github.com/armadaproject/armada/internal/common/metrics" "github.com/armadaproject/armada/internal/common/resource" @@ -103,8 +102,8 @@ func (c *MetricsCollector) Run(ctx *armadacontext.Context) error { case <-ticker.C(): err := c.refresh(ctx) if err != nil { - logging. - WithStacktrace(ctx, err). + ctx.Logger(). + WithStacktrace(err). Warnf("error refreshing metrics state") } } diff --git a/internal/scheduler/nodedb/nodeiteration.go b/internal/scheduler/nodedb/nodeiteration.go index 3e171007139..d8260270fa2 100644 --- a/internal/scheduler/nodedb/nodeiteration.go +++ b/internal/scheduler/nodedb/nodeiteration.go @@ -6,9 +6,9 @@ import ( "github.com/hashicorp/go-memdb" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "golang.org/x/exp/slices" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/scheduler/internaltypes" ) diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index fd796e1bd82..356817ddaef 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -13,7 +13,6 @@ import ( "k8s.io/utils/clock" "github.com/armadaproject/armada/internal/common/armadacontext" - "github.com/armadaproject/armada/internal/common/logging" protoutil "github.com/armadaproject/armada/internal/common/proto" "github.com/armadaproject/armada/internal/scheduler/database" "github.com/armadaproject/armada/internal/scheduler/jobdb" @@ -153,7 +152,7 @@ func (s *Scheduler) Run(ctx *armadacontext.Context) error { syncContext, cancel := armadacontext.WithTimeout(ctx, 5*time.Minute) err := s.ensureDbUpToDate(syncContext, 1*time.Second) if err != nil { - logging.WithStacktrace(ctx, err).Error("could not become leader") + ctx.Logger().WithStacktrace(err).Error("could not become leader") leaderToken = leader.InvalidLeaderToken() } else { fullUpdate = true @@ -175,7 +174,7 @@ func (s *Scheduler) Run(ctx *armadacontext.Context) error { result, err := s.cycle(ctx, fullUpdate, leaderToken, shouldSchedule) if err != nil { - logging.WithStacktrace(ctx, err).Error("scheduling cycle failure") + ctx.Logger().WithStacktrace(err).Error("scheduling cycle failure") leaderToken = leader.InvalidLeaderToken() } @@ -981,7 +980,9 @@ func (s *Scheduler) initialise(ctx *armadacontext.Context) error { return nil default: if _, _, err := s.syncState(ctx, true); err != nil { - logging.WithStacktrace(ctx, err).Error("failed to initialise; trying again in 1 second") + ctx.Logger(). + WithStacktrace(err). + Error("failed to initialise; trying again in 1 second") time.Sleep(1 * time.Second) } else { ctx.Info("initialisation succeeded") @@ -1008,7 +1009,9 @@ func (s *Scheduler) ensureDbUpToDate(ctx *armadacontext.Context, pollInterval ti default: numSent, err = s.publisher.PublishMarkers(ctx, groupId) if err != nil { - logging.WithStacktrace(ctx, err).Error("Error sending marker messages to pulsar") + ctx.Logger(). + WithStacktrace(err). + Error("Error sending marker messages to pulsar") s.clock.Sleep(pollInterval) } else { messagesSent = true @@ -1024,8 +1027,8 @@ func (s *Scheduler) ensureDbUpToDate(ctx *armadacontext.Context, pollInterval ti default: numReceived, err := s.jobRepository.CountReceivedPartitions(ctx, groupId) if err != nil { - logging. - WithStacktrace(ctx, err). + ctx.Logger(). + WithStacktrace(err). Error("Error querying the database or marker messages") } if numSent == numReceived { diff --git a/internal/scheduler/schedulerapp.go b/internal/scheduler/schedulerapp.go index 70c6067117b..6c21f07cf39 100644 --- a/internal/scheduler/schedulerapp.go +++ b/internal/scheduler/schedulerapp.go @@ -13,7 +13,6 @@ import ( grpc_logging "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -26,7 +25,7 @@ import ( dbcommon "github.com/armadaproject/armada/internal/common/database" grpcCommon "github.com/armadaproject/armada/internal/common/grpc" "github.com/armadaproject/armada/internal/common/health" - "github.com/armadaproject/armada/internal/common/logging" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/profiling" "github.com/armadaproject/armada/internal/common/pulsarutils" "github.com/armadaproject/armada/internal/common/pulsarutils/jobsetevents" @@ -118,8 +117,8 @@ func Run(config schedulerconfig.Configuration) error { defer func() { err := conn.Close() if err != nil { - logging. - WithStacktrace(ctx, err). + ctx.Logger(). + WithStacktrace(err). Warnf("Armada api client didn't close down cleanly") } }() @@ -363,7 +362,7 @@ func loadClusterConfig(ctx *armadacontext.Context) (*rest.Config, error) { return config, err } -// This changes the default logrus grpc logging to log OK messages at trace level +// This changes the default grpc logging to log OK messages at trace level // The reason for doing this are: // - Reduced logging // - We only care about failures, so lets only log failures diff --git a/internal/scheduler/scheduling/context/job.go b/internal/scheduler/scheduling/context/job.go index e9ad86ed71a..81ea403c5ce 100644 --- a/internal/scheduler/scheduling/context/job.go +++ b/internal/scheduler/scheduling/context/job.go @@ -8,11 +8,11 @@ import ( "time" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "golang.org/x/exp/maps" v1 "k8s.io/api/core/v1" "github.com/armadaproject/armada/internal/common/armadacontext" + log "github.com/armadaproject/armada/internal/common/logging" armadamaps "github.com/armadaproject/armada/internal/common/maps" armadaslices "github.com/armadaproject/armada/internal/common/slices" schedulerconfig "github.com/armadaproject/armada/internal/scheduler/configuration" @@ -196,7 +196,7 @@ func JobSchedulingContextsFromJobs[J *jobdb.Job](jobs []J) []*JobSchedulingConte func JobSchedulingContextFromJob(job *jobdb.Job) *JobSchedulingContext { gangInfo, err := GangInfoFromLegacySchedulerJob(job) if err != nil { - logrus.Errorf("failed to extract gang info from job %s: %s", job.Id(), err) + log.Errorf("failed to extract gang info from job %s: %s", job.Id(), err) } return &JobSchedulingContext{ Created: time.Now(), diff --git a/internal/scheduler/scheduling/market_driven_preempting_queue_scheduler_test.go b/internal/scheduler/scheduling/market_driven_preempting_queue_scheduler_test.go index e2363a19b37..24ece4726f3 100644 --- a/internal/scheduler/scheduling/market_driven_preempting_queue_scheduler_test.go +++ b/internal/scheduler/scheduling/market_driven_preempting_queue_scheduler_test.go @@ -351,7 +351,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { cordonedNodes := map[int]bool{} ctx := armadacontext.Background() for i, round := range tc.Rounds { - ctx.FieldLogger = ctx.WithField("round", i) + ctx = armadacontext.WithLogField(ctx, "round", i) ctx.Infof("starting scheduling round %d", i) jobsByNodeId := map[string][]*jobdb.Job{} diff --git a/internal/scheduler/scheduling/preempting_queue_scheduler.go b/internal/scheduler/scheduling/preempting_queue_scheduler.go index 47207a5b8e0..c5609fc7c3f 100644 --- a/internal/scheduler/scheduling/preempting_queue_scheduler.go +++ b/internal/scheduler/scheduling/preempting_queue_scheduler.go @@ -99,7 +99,7 @@ func (sch *PreemptingQueueScheduler) Schedule(ctx *armadacontext.Context) (*Sche scheduledJobsById := make(map[string]*schedulercontext.JobSchedulingContext) // Evict preemptible jobs. - ctx.WithField("stage", "scheduling-algo").Infof("Evicting preemptible jobs") + ctx.Logger().WithField("stage", "scheduling-algo").Infof("Evicting preemptible jobs") evictorResult, inMemoryJobRepo, err := sch.evict( armadacontext.WithLogField(ctx, "stage", "evict for resource balancing"), NewNodeEvictor( @@ -144,14 +144,14 @@ func (sch *PreemptingQueueScheduler) Schedule(ctx *armadacontext.Context) (*Sche if err != nil { return nil, err } - ctx.WithField("stage", "scheduling-algo").Info("Finished evicting preemptible jobs") + ctx.Logger().WithField("stage", "scheduling-algo").Info("Finished evicting preemptible jobs") for _, jctx := range evictorResult.EvictedJctxsByJobId { preemptedJobsById[jctx.JobId] = jctx } maps.Copy(sch.nodeIdByJobId, evictorResult.NodeIdByJobId) // Re-schedule evicted jobs/schedule new jobs. - ctx.WithField("stage", "scheduling-algo").Info("Performing initial scheduling of jobs onto nodes") + ctx.Logger().WithField("stage", "scheduling-algo").Info("Performing initial scheduling of jobs onto nodes") schedulerResult, err := sch.schedule( armadacontext.WithLogField(ctx, "stage", "re-schedule after balancing eviction"), inMemoryJobRepo, @@ -162,7 +162,7 @@ func (sch *PreemptingQueueScheduler) Schedule(ctx *armadacontext.Context) (*Sche if err != nil { return nil, err } - ctx.WithField("stage", "scheduling-algo").Info("Finished initial scheduling of jobs onto nodes") + ctx.Logger().WithField("stage", "scheduling-algo").Info("Finished initial scheduling of jobs onto nodes") for _, jctx := range schedulerResult.ScheduledJobs { if _, ok := preemptedJobsById[jctx.JobId]; ok { delete(preemptedJobsById, jctx.JobId) @@ -173,7 +173,7 @@ func (sch *PreemptingQueueScheduler) Schedule(ctx *armadacontext.Context) (*Sche maps.Copy(sch.nodeIdByJobId, schedulerResult.NodeIdByJobId) // Evict jobs on oversubscribed nodes. - ctx.WithField("stage", "scheduling-algo").Info("Evicting jobs from oversubscribed nodes") + ctx.Logger().WithField("stage", "scheduling-algo").Info("Evicting jobs from oversubscribed nodes") reevictResult, inMemoryJobRepo, err := sch.evict( armadacontext.WithLogField(ctx, "stage", "evict oversubscribed"), NewOversubscribedEvictor( @@ -185,7 +185,7 @@ func (sch *PreemptingQueueScheduler) Schedule(ctx *armadacontext.Context) (*Sche if err != nil { return nil, err } - ctx.WithField("stage", "scheduling-algo").Info("Finished evicting jobs from oversubscribed nodes") + ctx.Logger().WithField("stage", "scheduling-algo").Info("Finished evicting jobs from oversubscribed nodes") scheduledAndEvictedJobsById := armadamaps.FilterKeys( scheduledJobsById, func(jobId string) bool { @@ -205,7 +205,7 @@ func (sch *PreemptingQueueScheduler) Schedule(ctx *armadacontext.Context) (*Sche // Re-schedule evicted jobs/schedule new jobs. // Only necessary if a non-zero number of jobs were evicted. if len(reevictResult.EvictedJctxsByJobId) > 0 { - ctx.WithField("stage", "scheduling-algo").Info("Performing second scheduling ") + ctx.Logger().WithField("stage", "scheduling-algo").Info("Performing second scheduling ") rescheduleSchedulerResult, rescheduleErr := sch.schedule( armadacontext.WithLogField(ctx, "stage", "schedule after oversubscribed eviction"), inMemoryJobRepo, @@ -217,7 +217,7 @@ func (sch *PreemptingQueueScheduler) Schedule(ctx *armadacontext.Context) (*Sche if rescheduleErr != nil { return nil, rescheduleErr } - ctx.WithField("stage", "scheduling-algo").Info("Finished second scheduling pass") + ctx.Logger().WithField("stage", "scheduling-algo").Info("Finished second scheduling pass") for _, jctx := range rescheduleSchedulerResult.ScheduledJobs { if _, ok := preemptedJobsById[jctx.JobId]; ok { delete(preemptedJobsById, jctx.JobId) @@ -231,14 +231,14 @@ func (sch *PreemptingQueueScheduler) Schedule(ctx *armadacontext.Context) (*Sche preemptedJobs := maps.Values(preemptedJobsById) scheduledJobs := maps.Values(scheduledJobsById) - ctx.WithField("stage", "scheduling-algo").Infof("Unbinding %d preempted and %d evicted jobs", len(preemptedJobs), len(maps.Values(scheduledAndEvictedJobsById))) + ctx.Logger().WithField("stage", "scheduling-algo").Infof("Unbinding %d preempted and %d evicted jobs", len(preemptedJobs), len(maps.Values(scheduledAndEvictedJobsById))) if err := sch.unbindJobs(append( slices.Clone(preemptedJobs), maps.Values(scheduledAndEvictedJobsById)...), ); err != nil { return nil, err } - ctx.WithField("stage", "scheduling-algo").Infof("Finished unbinding preempted and evicted jobs") + ctx.Logger().WithField("stage", "scheduling-algo").Infof("Finished unbinding preempted and evicted jobs") PopulatePreemptionDescriptions(preemptedJobs, scheduledJobs) schedulercontext.PrintJobSummary(ctx, "Preempting running jobs;", preemptedJobs) diff --git a/internal/scheduler/scheduling/preempting_queue_scheduler_test.go b/internal/scheduler/scheduling/preempting_queue_scheduler_test.go index 88d70e37257..52c27aca4b0 100644 --- a/internal/scheduler/scheduling/preempting_queue_scheduler_test.go +++ b/internal/scheduler/scheduling/preempting_queue_scheduler_test.go @@ -1,6 +1,7 @@ package scheduling import ( + "context" "fmt" "math/rand" "testing" @@ -25,7 +26,7 @@ import ( "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/nodedb" schedulerconstraints "github.com/armadaproject/armada/internal/scheduler/scheduling/constraints" - "github.com/armadaproject/armada/internal/scheduler/scheduling/context" + schedulingcontext "github.com/armadaproject/armada/internal/scheduler/scheduling/context" "github.com/armadaproject/armada/internal/scheduler/scheduling/fairness" "github.com/armadaproject/armada/internal/scheduler/testfixtures" "github.com/armadaproject/armada/pkg/api" @@ -1939,7 +1940,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { cordonedNodes := map[int]bool{} ctx := armadacontext.Background() for i, round := range tc.Rounds { - ctx.FieldLogger = ctx.WithField("round", i) + ctx = armadacontext.WithLogField(ctx, "round", i) ctx.Infof("starting scheduling round %d", i) jobsByNodeId := map[string][]*jobdb.Job{} @@ -2019,7 +2020,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { tc.SchedulingConfig, ) require.NoError(t, err) - sctx := context.NewSchedulingContext( + sctx := schedulingcontext.NewSchedulingContext( testfixtures.TestPool, fairnessCostProvider, limiter, @@ -2205,7 +2206,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { // which jobs are preempted). slices.SortFunc( result.ScheduledJobs, - func(a, b *context.JobSchedulingContext) int { + func(a, b *schedulingcontext.JobSchedulingContext) int { if a.Job.SubmitTime().Before(b.Job.SubmitTime()) { return -1 } else if b.Job.SubmitTime().Before(a.Job.SubmitTime()) { @@ -2238,7 +2239,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { } } -func jobIdsByQueueFromJobContexts(jctxs []*context.JobSchedulingContext) map[string][]string { +func jobIdsByQueueFromJobContexts(jctxs []*schedulingcontext.JobSchedulingContext) map[string][]string { rv := make(map[string][]string) for _, jctx := range jctxs { job := jctx.Job @@ -2332,9 +2333,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { } for name, tc := range tests { b.Run(name, func(b *testing.B) { - ctx := armadacontext.Background() - ctx.FieldLogger = logging.NullLogger - + ctx := armadacontext.New(context.Background(), logging.NullLogger) jobsByQueue := make(map[string][]*jobdb.Job) priorityFactorByQueue := make(map[string]float64) for i := 0; i < tc.NumQueues; i++ { @@ -2380,7 +2379,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { tc.SchedulingConfig, ) require.NoError(b, err) - sctx := context.NewSchedulingContext( + sctx := schedulingcontext.NewSchedulingContext( testfixtures.TestPool, fairnessCostProvider, limiter, @@ -2426,7 +2425,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { err = jobDbTxn.BatchDelete( armadaslices.Map( result.ScheduledJobs, - func(jctx *context.JobSchedulingContext) string { + func(jctx *schedulingcontext.JobSchedulingContext) string { return jctx.JobId }, ), @@ -2451,7 +2450,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { b.ResetTimer() for n := 0; n < b.N; n++ { - sctx := context.NewSchedulingContext( + sctx := schedulingcontext.NewSchedulingContext( "pool", fairnessCostProvider, limiter, diff --git a/internal/scheduler/scheduling/scheduling_algo.go b/internal/scheduler/scheduling/scheduling_algo.go index e56e01f3d90..90be011edc5 100644 --- a/internal/scheduler/scheduling/scheduling_algo.go +++ b/internal/scheduler/scheduling/scheduling_algo.go @@ -6,13 +6,13 @@ import ( "time" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "golang.org/x/exp/maps" "golang.org/x/exp/slices" "golang.org/x/time/rate" "k8s.io/utils/clock" "github.com/armadaproject/armada/internal/common/armadacontext" + log "github.com/armadaproject/armada/internal/common/logging" armadamaps "github.com/armadaproject/armada/internal/common/maps" armadaslices "github.com/armadaproject/armada/internal/common/slices" "github.com/armadaproject/armada/internal/scheduler/configuration" @@ -130,7 +130,7 @@ func (l *FairSchedulingAlgo) Schedule( start := time.Now() schedulerResult, sctx, err := l.SchedulePool(ctx, fsctx, pool.Name, pool.MarketDriven) - ctx.Infof("Scheduled on executor pool %s in %v with error %v", pool, time.Now().Sub(start), err) + ctx.Infof("Scheduled on executor pool %s in %v with error %v", pool.Name, time.Now().Sub(start), err) if errors.Is(err, context.DeadlineExceeded) { // We've reached the scheduling time limit; @@ -611,7 +611,7 @@ func (l *FairSchedulingAlgo) populateNodeDb(nodeDb *nodedb.NodeDb, currentPoolJo } nodeId := job.LatestRun().NodeId() if _, ok := nodesById[nodeId]; !ok { - logrus.Errorf( + log.Errorf( "job %s assigned to node %s on executor %s, but no such node found", job.Id(), nodeId, job.LatestRun().Executor(), ) diff --git a/internal/scheduler/simulator/simulator.go b/internal/scheduler/simulator/simulator.go index 0305c4cab57..88755abf438 100644 --- a/internal/scheduler/simulator/simulator.go +++ b/internal/scheduler/simulator/simulator.go @@ -523,7 +523,7 @@ func (s *Simulator) pushScheduleEvent(time time.Time) { func (s *Simulator) handleSimulatorEvent(ctx *armadacontext.Context, event Event) error { s.time = event.time - ctx = armadacontext.New(ctx.Context, ctx.FieldLogger.WithField("simulated time", event.time)) + ctx = armadacontext.WithLogField(ctx, "simulated time", event.time) switch e := event.eventSequenceOrScheduleEvent.(type) { case *armadaevents.EventSequence: if err := s.handleEventSequence(ctx, e); err != nil { @@ -607,10 +607,7 @@ func (s *Simulator) handleScheduleEvent(ctx *armadacontext.Context) error { schedulerCtx := ctx if s.SuppressSchedulerLogs { - schedulerCtx = &armadacontext.Context{ - Context: ctx.Context, - FieldLogger: logging.NullLogger, - } + schedulerCtx = armadacontext.New(ctx.Context, logging.NullLogger) } result, err := sch.Schedule(schedulerCtx) if err != nil { diff --git a/internal/scheduler/submitcheck.go b/internal/scheduler/submitcheck.go index d96fd740c2e..40afd8ea91f 100644 --- a/internal/scheduler/submitcheck.go +++ b/internal/scheduler/submitcheck.go @@ -11,7 +11,6 @@ import ( "k8s.io/utils/clock" "github.com/armadaproject/armada/internal/common/armadacontext" - "github.com/armadaproject/armada/internal/common/logging" armadaslices "github.com/armadaproject/armada/internal/common/slices" "github.com/armadaproject/armada/internal/scheduler/configuration" "github.com/armadaproject/armada/internal/scheduler/database" @@ -90,16 +89,16 @@ func (srv *SubmitChecker) Run(ctx *armadacontext.Context) error { func (srv *SubmitChecker) updateExecutors(ctx *armadacontext.Context) { queues, err := srv.queueCache.GetAll(ctx) if err != nil { - logging. - WithStacktrace(ctx, err). + ctx.Logger(). + WithStacktrace(err). Error("Error fetching queues") return } executors, err := srv.executorRepository.GetExecutors(ctx) if err != nil { - logging. - WithStacktrace(ctx, err). + ctx.Logger(). + WithStacktrace(err). Error("Error fetching executors") return } @@ -141,8 +140,8 @@ func (srv *SubmitChecker) updateExecutors(ctx *armadacontext.Context) { nodeDb: nodeDb, } } else { - logging. - WithStacktrace(ctx, err). + ctx.Logger(). + WithStacktrace(err). Warnf("Error constructing nodedb for executor: %s", ex.Id) } } diff --git a/internal/scheduleringester/ingester.go b/internal/scheduleringester/ingester.go index 92e13647b18..fe0578f6028 100644 --- a/internal/scheduleringester/ingester.go +++ b/internal/scheduleringester/ingester.go @@ -6,7 +6,6 @@ import ( "github.com/apache/pulsar-client-go/pulsar" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common" "github.com/armadaproject/armada/internal/common/app" @@ -16,6 +15,7 @@ import ( controlplaneevents_ingest_utils "github.com/armadaproject/armada/internal/common/ingest/controlplaneevents" "github.com/armadaproject/armada/internal/common/ingest/jobsetevents" "github.com/armadaproject/armada/internal/common/ingest/metrics" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/profiling" "github.com/armadaproject/armada/pkg/armadaevents" "github.com/armadaproject/armada/pkg/controlplaneevents" diff --git a/internal/scheduleringester/instructions.go b/internal/scheduleringester/instructions.go index 2c971a95df8..d64a2f0b83e 100644 --- a/internal/scheduleringester/instructions.go +++ b/internal/scheduleringester/instructions.go @@ -6,7 +6,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/google/uuid" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "golang.org/x/exp/maps" "golang.org/x/exp/slices" @@ -14,6 +13,7 @@ import ( "github.com/armadaproject/armada/internal/common/compress" "github.com/armadaproject/armada/internal/common/ingest/metrics" "github.com/armadaproject/armada/internal/common/ingest/utils" + log "github.com/armadaproject/armada/internal/common/logging" protoutil "github.com/armadaproject/armada/internal/common/proto" "github.com/armadaproject/armada/internal/scheduler/adapters" schedulerdb "github.com/armadaproject/armada/internal/scheduler/database" diff --git a/internal/server/event/conversion/conversions.go b/internal/server/event/conversion/conversions.go index 227f9084a93..7db8eaf72d4 100644 --- a/internal/server/event/conversion/conversions.go +++ b/internal/server/event/conversion/conversions.go @@ -3,9 +3,8 @@ package conversion import ( "time" - log "github.com/sirupsen/logrus" - "github.com/armadaproject/armada/internal/common/eventutil" + log "github.com/armadaproject/armada/internal/common/logging" protoutil "github.com/armadaproject/armada/internal/common/proto" "github.com/armadaproject/armada/pkg/api" "github.com/armadaproject/armada/pkg/armadaevents" diff --git a/internal/server/event/event.go b/internal/server/event/event.go index f2cb1b9d843..9c59c7bebfd 100644 --- a/internal/server/event/event.go +++ b/internal/server/event/event.go @@ -6,13 +6,13 @@ import ( "github.com/gogo/protobuf/types" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/armadaerrors" "github.com/armadaproject/armada/internal/common/auth" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/server/event/sequence" "github.com/armadaproject/armada/internal/server/permissions" armadaqueue "github.com/armadaproject/armada/internal/server/queue" diff --git a/internal/server/event/event_repository.go b/internal/server/event/event_repository.go index 124abdd021c..4292f7553b5 100644 --- a/internal/server/event/event_repository.go +++ b/internal/server/event/event_repository.go @@ -10,10 +10,10 @@ import ( pool "github.com/jolestar/go-commons-pool" "github.com/pkg/errors" "github.com/redis/go-redis/v9" - log "github.com/sirupsen/logrus" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/compress" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/server/event/conversion" "github.com/armadaproject/armada/internal/server/event/sequence" "github.com/armadaproject/armada/pkg/api" diff --git a/internal/server/server.go b/internal/server/server.go index 514176e37fb..5a61af58acc 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -12,7 +12,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/redis/go-redis/extra/redisprometheus/v9" "github.com/redis/go-redis/v9" - log "github.com/sirupsen/logrus" "google.golang.org/grpc" "github.com/armadaproject/armada/internal/common/armadacontext" @@ -21,6 +20,7 @@ import ( "github.com/armadaproject/armada/internal/common/database" grpcCommon "github.com/armadaproject/armada/internal/common/grpc" "github.com/armadaproject/armada/internal/common/health" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/pulsarutils" controlplaneeventspulsarutils "github.com/armadaproject/armada/internal/common/pulsarutils/controlplaneevents" "github.com/armadaproject/armada/internal/common/pulsarutils/jobsetevents" diff --git a/internal/server/submit/submit.go b/internal/server/submit/submit.go index 63ea39d1fa6..0541e3f86a6 100644 --- a/internal/server/submit/submit.go +++ b/internal/server/submit/submit.go @@ -6,13 +6,13 @@ import ( "github.com/gogo/protobuf/types" "github.com/gogo/status" - log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "k8s.io/utils/clock" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/auth" "github.com/armadaproject/armada/internal/common/auth/permission" + log "github.com/armadaproject/armada/internal/common/logging" protoutil "github.com/armadaproject/armada/internal/common/proto" "github.com/armadaproject/armada/internal/common/pulsarutils" "github.com/armadaproject/armada/internal/common/slices" diff --git a/pkg/client/auth/kubernetes/authentication.go b/pkg/client/auth/kubernetes/authentication.go index 85f14fe3c19..a3bb4a47f7a 100644 --- a/pkg/client/auth/kubernetes/authentication.go +++ b/pkg/client/auth/kubernetes/authentication.go @@ -4,13 +4,13 @@ import ( "context" "os" - log "github.com/sirupsen/logrus" "golang.org/x/oauth2" authv1 "k8s.io/api/authentication/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/pkg/client/auth/oidc" ) diff --git a/pkg/client/auth/oidc/device.go b/pkg/client/auth/oidc/device.go index e6f9ae63474..3abee6473da 100644 --- a/pkg/client/auth/oidc/device.go +++ b/pkg/client/auth/oidc/device.go @@ -13,8 +13,6 @@ import ( openId "github.com/coreos/go-oidc" "golang.org/x/oauth2" - - "github.com/armadaproject/armada/internal/common/logging" ) type DeviceDetails struct { @@ -152,6 +150,11 @@ func makeErrorForHTTPResponse(resp *http.Response) error { if err != nil { return err } - safeURL := logging.SanitizeUserInput(resp.Request.URL.String()) + safeURL := sanitize(resp.Request.URL.String()) return fmt.Errorf("%s %s returned HTTP %s; \n\n %#q", resp.Request.Method, safeURL, resp.Status, bodyBytes) } + +func sanitize(str string) string { + safeStr := strings.ReplaceAll(str, "\n", "") + return strings.ReplaceAll(safeStr, "\r", "") +} diff --git a/pkg/client/auth/oidc/kubernetes.go b/pkg/client/auth/oidc/kubernetes.go index 3d540e6598d..4e141f4bf4c 100644 --- a/pkg/client/auth/oidc/kubernetes.go +++ b/pkg/client/auth/oidc/kubernetes.go @@ -12,8 +12,9 @@ import ( "strings" "time" - log "github.com/sirupsen/logrus" "golang.org/x/oauth2" + + log "github.com/armadaproject/armada/internal/common/logging" ) type KubernetesDetails struct { diff --git a/pkg/client/auth/oidc/pkce.go b/pkg/client/auth/oidc/pkce.go index f50b7bf2491..7d02a47a366 100644 --- a/pkg/client/auth/oidc/pkce.go +++ b/pkg/client/auth/oidc/pkce.go @@ -12,12 +12,10 @@ import ( "strconv" "strings" - "github.com/sirupsen/logrus" - - "github.com/armadaproject/armada/internal/common/logging" - openId "github.com/coreos/go-oidc" "golang.org/x/oauth2" + + log "github.com/armadaproject/armada/internal/common/logging" ) type PKCEDetails struct { @@ -29,7 +27,6 @@ type PKCEDetails struct { func AuthenticatePkce(config PKCEDetails) (*TokenCredentials, error) { ctx := context.Background() - log := logrus.StandardLogger().WithField("auth", "AuthenticatePkce") result := make(chan *oauth2.Token) errorResult := make(chan error) @@ -97,14 +94,14 @@ func AuthenticatePkce(config PKCEDetails) (*TokenCredentials, error) { go func() { if err := server.Serve(listener); err != nil { - logging.WithStacktrace(log, err).Error("unable to serve") + log.WithStacktrace(err).Error("unable to serve") } }() cmd, err := openBrowser("http://" + localUrl) defer func() { if err := cmd.Process.Kill(); err != nil { - logging.WithStacktrace(log, err).Error("unable to kill process") + log.WithStacktrace(err).Error("unable to kill process") } }() diff --git a/pkg/client/load-test.go b/pkg/client/load-test.go index 448e2000b0d..ac947603289 100644 --- a/pkg/client/load-test.go +++ b/pkg/client/load-test.go @@ -6,13 +6,11 @@ import ( "sync" "time" - "github.com/sirupsen/logrus" - log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "github.com/armadaproject/armada/internal/common" - "github.com/armadaproject/armada/internal/common/logging" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/pkg/api" "github.com/armadaproject/armada/pkg/client/domain" ) @@ -231,7 +229,6 @@ func (apiLoadTester ArmadaLoadTester) monitorJobsUntilCompletion( jobIds chan string, eventChannel chan api.Event, ) []string { - log := logrus.StandardLogger().WithField("Armada", "LoadTester") var submittedIds []string = nil go func() { ids := []string{} @@ -258,7 +255,7 @@ func (apiLoadTester ArmadaLoadTester) monitorJobsUntilCompletion( return nil }) if err != nil { - logging.WithStacktrace(log, err).Error("unable to monitor jobs") + log.WithStacktrace(err).Error("unable to monitor jobs") } return submittedIds } @@ -281,8 +278,6 @@ func createJobSubmitRequestItems(jobDescs []*domain.JobSubmissionDescription) [] } func (apiLoadTester ArmadaLoadTester) cancelRemainingJobs(queue string, jobSetId string) { - log := logrus.StandardLogger().WithField("Armada", "LoadTester") - err := WithSubmitClient(apiLoadTester.apiConnectionDetails, func(client api.SubmitClient) error { timeout, _ := common.ContextWithDefaultTimeout() cancelRequest := &api.JobCancelRequest{ @@ -293,7 +288,7 @@ func (apiLoadTester ArmadaLoadTester) cancelRemainingJobs(queue string, jobSetId return err }) if err != nil { - logging.WithStacktrace(log, err).Error("unable to cancel jobs") + log.WithStacktrace(err).Error("unable to cancel jobs") } } diff --git a/pkg/client/queue/get_all.go b/pkg/client/queue/get_all.go index 2357115792d..785732f02e0 100644 --- a/pkg/client/queue/get_all.go +++ b/pkg/client/queue/get_all.go @@ -6,12 +6,12 @@ import ( "io" "time" - "github.com/armadaproject/armada/pkg/api" - "github.com/armadaproject/armada/pkg/client" - - log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + + log "github.com/armadaproject/armada/internal/common/logging" + "github.com/armadaproject/armada/pkg/api" + "github.com/armadaproject/armada/pkg/client" ) type GetAllAPI func() ([]*api.Queue, error) @@ -35,7 +35,7 @@ func GetAll(getConnectionDetails client.ConnectionDetails) GetAllAPI { allQueues := make([]*api.Queue, 0) queueStream, err := client.GetQueues(ctx, &api.StreamingQueueGetRequest{}) if err != nil { - log.Error(err) + log.Error(err.Error()) return nil, err } @@ -57,7 +57,7 @@ func GetAll(getConnectionDetails client.ConnectionDetails) GetAllAPI { return nil, e } if !isTransportClosingError(e) { - log.Error(e) + log.Error(e.Error()) } break } diff --git a/pkg/client/watch.go b/pkg/client/watch.go index 2b9a96ae498..7b395cf5938 100644 --- a/pkg/client/watch.go +++ b/pkg/client/watch.go @@ -5,10 +5,10 @@ import ( "io" "time" - log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + log "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/common/util" "github.com/armadaproject/armada/pkg/api" "github.com/armadaproject/armada/pkg/client/domain" @@ -71,7 +71,7 @@ func WatchJobSetWithJobIdsFilter( ) if e != nil { - log.Error(e) + log.Error(e.Error()) time.Sleep(5 * time.Second) continue } @@ -94,7 +94,7 @@ func WatchJobSetWithJobIdsFilter( return state } if !isTransportClosingError(e) { - log.Error(e) + log.Error(e.Error()) } time.Sleep(5 * time.Second) break @@ -104,7 +104,7 @@ func WatchJobSetWithJobIdsFilter( event, e := api.UnwrapEvent(msg.Message) if e != nil { // This can mean that the event type reported from server is unknown to the client - log.Error(e) + log.Error(e.Error()) continue } From 48fdf31a60e9119f74ffe75353cc94c61f7a4926 Mon Sep 17 00:00:00 2001 From: JamesMurkin Date: Mon, 20 Jan 2025 13:55:19 +0000 Subject: [PATCH 04/23] [executor] Allow retrying fail pods that haven't had any containers start (#4147) * [executor] Allow retrying fail pods that haven't had any containers start Signed-off-by: JamesMurkin * Split event reporter and state reporter Signed-off-by: JamesMurkin * Improve messages Signed-off-by: JamesMurkin * Add unit tests Signed-off-by: JamesMurkin * Format Signed-off-by: JamesMurkin * FIx error Signed-off-by: JamesMurkin * Use common logger Signed-off-by: JamesMurkin --------- Signed-off-by: JamesMurkin --- internal/executor/application.go | 26 +- .../executor/configuration/podchecks/types.go | 16 ++ internal/executor/configuration/types.go | 1 + .../context/fake/sync_cluster_context.go | 12 + .../podchecks/failedpodchecks/event_checks.go | 68 +++++ .../failedpodchecks/event_checks_test.go | 105 ++++++++ .../podchecks/failedpodchecks/pod_checks.go | 57 +++++ .../failedpodchecks/pod_checks_test.go | 209 ++++++++++++++++ .../failedpodchecks/pod_status_checks.go | 56 +++++ .../failedpodchecks/pod_status_checks_test.go | 99 ++++++++ .../executor/reporter/job_event_reporter.go | 203 +-------------- .../reporter/job_event_reporter_test.go | 42 +--- .../executor/reporter/mocks/testfixtures.go | 6 + .../executor/service/job_state_reporter.go | 233 ++++++++++++++++++ .../service/job_state_reporter_test.go | 217 ++++++++++++++++ .../executor/service/pod_issue_handler.go | 114 ++++++--- .../service/pod_issue_handler_test.go | 134 +++++++++- 17 files changed, 1319 insertions(+), 279 deletions(-) create mode 100644 internal/executor/podchecks/failedpodchecks/event_checks.go create mode 100644 internal/executor/podchecks/failedpodchecks/event_checks_test.go create mode 100644 internal/executor/podchecks/failedpodchecks/pod_checks.go create mode 100644 internal/executor/podchecks/failedpodchecks/pod_checks_test.go create mode 100644 internal/executor/podchecks/failedpodchecks/pod_status_checks.go create mode 100644 internal/executor/podchecks/failedpodchecks/pod_status_checks_test.go create mode 100644 internal/executor/service/job_state_reporter.go create mode 100644 internal/executor/service/job_state_reporter_test.go diff --git a/internal/executor/application.go b/internal/executor/application.go index a65cdde84f5..909f4e65da3 100644 --- a/internal/executor/application.go +++ b/internal/executor/application.go @@ -31,6 +31,7 @@ import ( "github.com/armadaproject/armada/internal/executor/metrics/runstate" "github.com/armadaproject/armada/internal/executor/node" "github.com/armadaproject/armada/internal/executor/podchecks" + "github.com/armadaproject/armada/internal/executor/podchecks/failedpodchecks" "github.com/armadaproject/armada/internal/executor/reporter" "github.com/armadaproject/armada/internal/executor/service" "github.com/armadaproject/armada/internal/executor/utilisation" @@ -191,16 +192,13 @@ func setupExecutorApiComponents( config.Kubernetes.MinimumResourcesMarkedAllocatedToNonArmadaPodsPerNodePriority, ) - eventReporter, stopReporter, err := reporter.NewJobEventReporter( - clusterContext, - jobRunState, - eventSender, - clock.RealClock{}, - 200) + failedPodChecker, err := failedpodchecks.NewPodRetryChecker(config.Kubernetes.FailedPodChecks) if err != nil { - ctx.Fatalf("Failed to create job event reporter: %s", err) + ctx.Fatalf("Config error in failed pod checks: %s", err) } + eventReporter, stopReporter := reporter.NewJobEventReporter(eventSender, clock.RealClock{}, 200) + submitter := job.NewSubmitter( clusterContext, config.Kubernetes.PodDefaults, @@ -228,24 +226,34 @@ func setupExecutorApiComponents( submitter, clusterHealthMonitor, ) - podIssueService, err := service.NewIssueHandler( + podIssueService, err := service.NewPodIssuerHandler( jobRunState, clusterContext, eventReporter, config.Kubernetes.StateChecks, pendingPodChecker, + failedPodChecker, config.Kubernetes.StuckTerminatingPodExpiry, ) if err != nil { ctx.Fatalf("Failed to create pod issue service: %s", err) } + jobStateReporter, err := service.NewJobStateReporter( + clusterContext, + eventReporter, + podIssueService, + ) + if err != nil { + ctx.Fatalf("Failed to create job state reporter: %s", err) + } + taskManager.Register(podIssueService.HandlePodIssues, config.Task.PodIssueHandlingInterval, "pod_issue_handling") taskManager.Register(preemptRunProcessor.Run, config.Task.StateProcessorInterval, "preempt_runs") taskManager.Register(removeRunProcessor.Run, config.Task.StateProcessorInterval, "remove_runs") taskManager.Register(jobRequester.RequestJobsRuns, config.Task.JobLeaseRenewalInterval, "request_runs") taskManager.Register(clusterAllocationService.AllocateSpareClusterCapacity, config.Task.AllocateSpareClusterCapacityInterval, "submit_runs") - taskManager.Register(eventReporter.ReportMissingJobEvents, config.Task.MissingJobEventReconciliationInterval, "event_reconciliation") + taskManager.Register(jobStateReporter.ReportMissingJobEvents, config.Task.MissingJobEventReconciliationInterval, "event_reconciliation") _, err = pod_metrics.ExposeClusterContextMetrics(clusterContext, clusterUtilisationService, podUtilisationService, nodeInfoService) if err != nil { ctx.Fatalf("Failed to setup cluster context metrics: %s", err) diff --git a/internal/executor/configuration/podchecks/types.go b/internal/executor/configuration/podchecks/types.go index 72f6c2488d4..1502ba3cc05 100644 --- a/internal/executor/configuration/podchecks/types.go +++ b/internal/executor/configuration/podchecks/types.go @@ -47,3 +47,19 @@ type ContainerStatusCheck struct { GracePeriod time.Duration Action Action } + +type FailedChecks struct { + Events []PodEventCheck + PodStatuses []PodStatusCheck +} + +type PodEventCheck struct { + Regexp string + Reason string + Type EventType +} + +type PodStatusCheck struct { + Regexp string + Reason string +} diff --git a/internal/executor/configuration/types.go b/internal/executor/configuration/types.go index 42bbd05ed9e..f872598ea97 100644 --- a/internal/executor/configuration/types.go +++ b/internal/executor/configuration/types.go @@ -76,6 +76,7 @@ type KubernetesConfiguration struct { MaxTerminatedPods int PodDefaults *PodDefaults StateChecks StateChecksConfiguration + FailedPodChecks podchecks.FailedChecks PendingPodChecks *podchecks.Checks FatalPodSubmissionErrors []string // Minimum amount of resources marked as allocated to non-Armada pods on each node. diff --git a/internal/executor/context/fake/sync_cluster_context.go b/internal/executor/context/fake/sync_cluster_context.go index c966aed357d..8a71c6d339c 100644 --- a/internal/executor/context/fake/sync_cluster_context.go +++ b/internal/executor/context/fake/sync_cluster_context.go @@ -21,6 +21,7 @@ type SyncFakeClusterContext struct { Events map[string][]*v1.Event AnnotationsAdded map[string]map[string]string podEventHandlers []*cache.ResourceEventHandlerFuncs + GetPodEventsErr error } func NewSyncFakeClusterContext() *SyncFakeClusterContext { @@ -64,6 +65,9 @@ func (c *SyncFakeClusterContext) GetNode(nodeName string) (*v1.Node, error) { } func (c *SyncFakeClusterContext) GetPodEvents(pod *v1.Pod) ([]*v1.Event, error) { + if c.GetPodEventsErr != nil { + return nil, c.GetPodEventsErr + } jobId := util2.ExtractJobId(pod) return c.Events[jobId], nil } @@ -151,6 +155,14 @@ func (c *SyncFakeClusterContext) SimulatePodAddEvent(pod *v1.Pod) { } } +func (c *SyncFakeClusterContext) SimulateUpdateAddEvent(before *v1.Pod, after *v1.Pod) { + for _, h := range c.podEventHandlers { + if h.AddFunc != nil { + h.UpdateFunc(before, after) + } + } +} + type FakeClusterIdentity struct { clusterId string clusterPool string diff --git a/internal/executor/podchecks/failedpodchecks/event_checks.go b/internal/executor/podchecks/failedpodchecks/event_checks.go new file mode 100644 index 00000000000..06e864fadc3 --- /dev/null +++ b/internal/executor/podchecks/failedpodchecks/event_checks.go @@ -0,0 +1,68 @@ +package failedpodchecks + +import ( + "fmt" + "regexp" + + v1 "k8s.io/api/core/v1" + + "github.com/armadaproject/armada/internal/executor/configuration/podchecks" +) + +type eventRetryChecker interface { + IsRetryable(podEvents []*v1.Event) (bool, string) +} + +type podEventCheck struct { + regexp *regexp.Regexp + reason string + eventType podchecks.EventType +} + +type PodEventsChecker struct { + checks []podEventCheck +} + +func NewPodEventsChecker(checks []podchecks.PodEventCheck) (*PodEventsChecker, error) { + podEventChecks := make([]podEventCheck, 0, len(checks)) + + for _, check := range checks { + re, err := regexp.Compile(check.Regexp) + if err != nil { + return nil, fmt.Errorf("cannot parse regexp \"%s\": %+v", check.Regexp, err) + } + if string(check.Type) != v1.EventTypeNormal && string(check.Type) != v1.EventTypeWarning { + return nil, fmt.Errorf("invalid event type: \"%s\"", check.Type) + } + + podEventChecks = append(podEventChecks, podEventCheck{ + regexp: re, + eventType: check.Type, + reason: check.Reason, + }) + } + + return &PodEventsChecker{ + checks: podEventChecks, + }, nil +} + +func (f *PodEventsChecker) IsRetryable(podEvents []*v1.Event) (bool, string) { + for _, check := range f.checks { + for _, podEvent := range podEvents { + if check.reason != "" && check.reason != podEvent.Reason { + continue + } + + if string(check.eventType) != podEvent.Type { + continue + } + + if check.regexp.MatchString(podEvent.Message) { + return true, podEvent.Message + } + } + } + + return false, "" +} diff --git a/internal/executor/podchecks/failedpodchecks/event_checks_test.go b/internal/executor/podchecks/failedpodchecks/event_checks_test.go new file mode 100644 index 00000000000..1f82c12d22c --- /dev/null +++ b/internal/executor/podchecks/failedpodchecks/event_checks_test.go @@ -0,0 +1,105 @@ +package failedpodchecks + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + + "github.com/armadaproject/armada/internal/executor/configuration/podchecks" +) + +func TestPodEventsChecker_IsRetryable(t *testing.T) { + tests := map[string]struct { + events []*v1.Event + checks []podchecks.PodEventCheck + expectedResult bool + expectMessage bool + }{ + "empty checks": { + events: []*v1.Event{{Message: "Failed to allocate", Reason: "reason", Type: "Warning"}}, + expectedResult: false, + expectMessage: false, + }, + "matches message and type": { + events: []*v1.Event{{Message: "Failed to allocate", Reason: "reason", Type: "Warning"}}, + checks: []podchecks.PodEventCheck{{Regexp: "Fail.*", Type: "Warning"}}, + expectedResult: true, + expectMessage: true, + }, + "matches on reason if supplied": { + events: []*v1.Event{{Message: "Failed to allocate", Reason: "reason", Type: "Warning"}}, + checks: []podchecks.PodEventCheck{{Regexp: "Fail.*", Reason: "reason", Type: "Warning"}}, + expectedResult: true, + expectMessage: true, + }, + "matches on reason if supplied - no match": { + events: []*v1.Event{{Message: "Failed to allocate", Reason: "reason", Type: "Warning"}}, + checks: []podchecks.PodEventCheck{{Regexp: "Fail.*", Reason: "reason2", Type: "Warning"}}, + expectedResult: false, + expectMessage: false, + }, + "multiple events and checks": { + events: []*v1.Event{ + {Message: "Failed to allocate", Reason: "reason", Type: "Warning"}, + {Message: "Image pull", Reason: "image", Type: "Normal"}, + {Message: "Image error", Reason: "image", Type: "Warning"}, + }, + checks: []podchecks.PodEventCheck{ + {Regexp: "no.*", Type: "Warning"}, + {Regexp: "Image.*", Reason: "reason", Type: "Warning"}, + {Regexp: "Failed.*", Type: "Warning"}, + {Regexp: "Image err.*", Reason: "image", Type: "Warning"}, + }, + expectedResult: true, + expectMessage: true, + }, + "multiple events and checks - no match": { + events: []*v1.Event{ + {Message: "Failed to allocate", Reason: "reason", Type: "Warning"}, + {Message: "Image pull", Reason: "image", Type: "Normal"}, + {Message: "Image error", Reason: "image", Type: "Warning"}, + }, + checks: []podchecks.PodEventCheck{ + {Regexp: "no.*", Type: "Warning"}, + {Regexp: "Image.*", Reason: "reason", Type: "Warning"}, + {Regexp: "Failed.*", Type: "Normal"}, + {Regexp: "Image err.*", Reason: "image", Type: "Normal"}, + }, + expectedResult: false, + expectMessage: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + checker, err := NewPodEventsChecker(tc.checks) + require.NoError(t, err) + + isRetryable, message := checker.IsRetryable(tc.events) + + assert.Equal(t, tc.expectedResult, isRetryable) + if tc.expectMessage { + assert.NotEmpty(t, message) + } else { + assert.Empty(t, message) + } + }) + } +} + +func TestPodEventsChecker_Initialisation(t *testing.T) { + // Empty + _, err := NewPodEventsChecker([]podchecks.PodEventCheck{}) + assert.NoError(t, err) + // Valid + _, err = NewPodEventsChecker([]podchecks.PodEventCheck{{Regexp: ".*", Type: "Warning"}}) + assert.NoError(t, err) + // Invalid regex + _, err = NewPodEventsChecker([]podchecks.PodEventCheck{{Regexp: "[", Type: "Warning"}}) + assert.Error(t, err) + // Invalid type + _, err = NewPodEventsChecker([]podchecks.PodEventCheck{{Regexp: ".*", Type: "Invalid"}}) + assert.Error(t, err) +} diff --git a/internal/executor/podchecks/failedpodchecks/pod_checks.go b/internal/executor/podchecks/failedpodchecks/pod_checks.go new file mode 100644 index 00000000000..3abb47d8ddf --- /dev/null +++ b/internal/executor/podchecks/failedpodchecks/pod_checks.go @@ -0,0 +1,57 @@ +package failedpodchecks + +import ( + v1 "k8s.io/api/core/v1" + + "github.com/armadaproject/armada/internal/executor/configuration/podchecks" +) + +type RetryChecker interface { + IsRetryable(pod *v1.Pod, podEvents []*v1.Event) (bool, string) +} + +type PodRetryChecker struct { + podStatusChecker podStatusRetryChecker + podEventChecker eventRetryChecker +} + +func NewPodRetryChecker(config podchecks.FailedChecks) (*PodRetryChecker, error) { + podStatusChecker, err := NewPodStatusChecker(config.PodStatuses) + if err != nil { + return nil, err + } + podEventChecker, err := NewPodEventsChecker(config.Events) + if err != nil { + return nil, err + } + + return &PodRetryChecker{ + podEventChecker: podEventChecker, + podStatusChecker: podStatusChecker, + }, nil +} + +func (f *PodRetryChecker) IsRetryable(pod *v1.Pod, podEvents []*v1.Event) (bool, string) { + if hasStartedContainers(pod) { + return false, "" + } + + isRetryable, message := f.podEventChecker.IsRetryable(podEvents) + + if !isRetryable { + isRetryable, message = f.podStatusChecker.IsRetryable(pod) + } + + return isRetryable, message +} + +func hasStartedContainers(pod *v1.Pod) bool { + containers := pod.Status.ContainerStatuses + containers = append(containers, pod.Status.InitContainerStatuses...) + for _, container := range containers { + if container.LastTerminationState.Terminated != nil || container.LastTerminationState.Running != nil { + return true + } + } + return false +} diff --git a/internal/executor/podchecks/failedpodchecks/pod_checks_test.go b/internal/executor/podchecks/failedpodchecks/pod_checks_test.go new file mode 100644 index 00000000000..c5cdd200cc8 --- /dev/null +++ b/internal/executor/podchecks/failedpodchecks/pod_checks_test.go @@ -0,0 +1,209 @@ +package failedpodchecks + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + + "github.com/armadaproject/armada/internal/executor/configuration/podchecks" +) + +func TestPodRetryChecker(t *testing.T) { + podEventChecker := &stubEventRetryChecker{isRetryable: true, message: "event"} + failPodEventChecker := &stubEventRetryChecker{isRetryable: false, message: ""} + + podStatusChecker := &stubPodStatusRetryChecker{isRetryable: true, message: "status"} + failPodStatusChecker := &stubPodStatusRetryChecker{isRetryable: false, message: ""} + + tests := map[string]struct { + podStatusChecker podStatusRetryChecker + podEventChecker eventRetryChecker + expectRetryable bool + }{ + "retryable when all checkers return retryable": { + podStatusChecker: podStatusChecker, + podEventChecker: podEventChecker, + expectRetryable: true, + }, + "retryable when status checker returns retryable": { + podStatusChecker: podStatusChecker, + podEventChecker: failPodEventChecker, + expectRetryable: true, + }, + "retryable when event checker returns retryable": { + podStatusChecker: failPodStatusChecker, + podEventChecker: podEventChecker, + expectRetryable: true, + }, + "not retryable when all checkers return not retryable": { + podStatusChecker: failPodStatusChecker, + podEventChecker: failPodEventChecker, + expectRetryable: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + checker := PodRetryChecker{ + podEventChecker: tc.podEventChecker, + podStatusChecker: tc.podStatusChecker, + } + + isRetryable, message := checker.IsRetryable(&v1.Pod{}, []*v1.Event{}) + + assert.Equal(t, tc.expectRetryable, isRetryable) + if tc.expectRetryable { + assert.NotEmpty(t, message) + } else { + assert.Empty(t, message) + } + }) + } +} + +func TestPodRetryCheckerIsRetryable_ChecksPodHasNotStartedAnyContainers(t *testing.T) { + tests := map[string]struct { + pod *v1.Pod + expectRetryable bool + }{ + "pod with no container statuses - retryable": { + pod: &v1.Pod{}, + expectRetryable: true, + }, + "pod with pending container - retryable": { + pod: &v1.Pod{ + Status: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + LastTerminationState: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, + }, + }, + InitContainerStatuses: []v1.ContainerStatus{ + { + LastTerminationState: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, + }, + }, + }, + }, + expectRetryable: true, + }, + "pod with running init container - not retryable": { + pod: &v1.Pod{ + Status: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + LastTerminationState: v1.ContainerState{Running: &v1.ContainerStateRunning{}}, + }, + }, + }, + }, + expectRetryable: false, + }, + "pod with running container - not retryable": { + pod: &v1.Pod{ + Status: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + LastTerminationState: v1.ContainerState{Running: &v1.ContainerStateRunning{}}, + }, + }, + }, + }, + expectRetryable: false, + }, + "pod with terminated init container - not retryable": { + pod: &v1.Pod{ + Status: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + LastTerminationState: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{}}, + }, + }, + }, + }, + expectRetryable: false, + }, + "pod with terminated container - not retryable": { + pod: &v1.Pod{ + Status: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + LastTerminationState: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{}}, + }, + }, + }, + }, + expectRetryable: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + checker := PodRetryChecker{ + podEventChecker: &stubEventRetryChecker{isRetryable: true, message: "fallthrough"}, + podStatusChecker: &stubPodStatusRetryChecker{isRetryable: true, message: "fallthrough"}, + } + + isRetryable, message := checker.IsRetryable(tc.pod, []*v1.Event{}) + + if tc.expectRetryable { + assert.True(t, isRetryable) + assert.Equal(t, "fallthrough", message) + } else { + assert.False(t, isRetryable) + assert.Empty(t, message) + } + }) + } +} + +func TestPodRetryChecker_SimpleEndToEnd(t *testing.T) { + checker, err := NewPodRetryChecker(podchecks.FailedChecks{ + PodStatuses: []podchecks.PodStatusCheck{{Regexp: "mess.*", Reason: "reason"}}, + Events: []podchecks.PodEventCheck{{Regexp: "Fail.*", Reason: "reason", Type: "Warning"}}, + }) + require.NoError(t, err) + + isRetryable, message := checker.IsRetryable(&v1.Pod{}, []*v1.Event{}) + assert.False(t, isRetryable) + assert.Empty(t, message) + + // Pod matches status checker + isRetryable, message = checker.IsRetryable(makePodWithMessageAndReason("message", "reason"), []*v1.Event{}) + assert.True(t, isRetryable) + assert.NotEmpty(t, message) + + // Pod matches event checker + isRetryable, message = checker.IsRetryable(&v1.Pod{}, []*v1.Event{{Message: "Failed to allocate", Reason: "reason", Type: "Warning"}}) + assert.True(t, isRetryable) + assert.NotEmpty(t, message) +} + +func TestPodRetryChecker_InitialiseEmpty(t *testing.T) { + checker, err := NewPodRetryChecker(podchecks.FailedChecks{}) + require.NoError(t, err) + + isRetryable, message := checker.IsRetryable(&v1.Pod{}, []*v1.Event{}) + assert.False(t, isRetryable) + assert.Empty(t, message) +} + +type stubEventRetryChecker struct { + isRetryable bool + message string +} + +func (s *stubEventRetryChecker) IsRetryable(podEvents []*v1.Event) (bool, string) { + return s.isRetryable, s.message +} + +type stubPodStatusRetryChecker struct { + isRetryable bool + message string +} + +func (s *stubPodStatusRetryChecker) IsRetryable(pod *v1.Pod) (bool, string) { + return s.isRetryable, s.message +} diff --git a/internal/executor/podchecks/failedpodchecks/pod_status_checks.go b/internal/executor/podchecks/failedpodchecks/pod_status_checks.go new file mode 100644 index 00000000000..83d3715495b --- /dev/null +++ b/internal/executor/podchecks/failedpodchecks/pod_status_checks.go @@ -0,0 +1,56 @@ +package failedpodchecks + +import ( + "fmt" + "regexp" + + v1 "k8s.io/api/core/v1" + + "github.com/armadaproject/armada/internal/executor/configuration/podchecks" +) + +type podStatusRetryChecker interface { + IsRetryable(pod *v1.Pod) (bool, string) +} + +type podStatusCheck struct { + regexp *regexp.Regexp + reason string +} + +type PodStatusChecker struct { + checks []podStatusCheck +} + +func NewPodStatusChecker(checks []podchecks.PodStatusCheck) (*PodStatusChecker, error) { + podStatusChecks := make([]podStatusCheck, 0, len(checks)) + + for _, check := range checks { + re, err := regexp.Compile(check.Regexp) + if err != nil { + return nil, fmt.Errorf("cannot parse regexp \"%s\": %+v", check.Regexp, err) + } + + podStatusChecks = append(podStatusChecks, podStatusCheck{ + regexp: re, + reason: check.Reason, + }) + } + return &PodStatusChecker{ + checks: podStatusChecks, + }, nil +} + +func (f *PodStatusChecker) IsRetryable(pod *v1.Pod) (bool, string) { + for _, check := range f.checks { + if check.reason != "" && check.reason != pod.Status.Reason { + continue + } + + if check.regexp.MatchString(pod.Status.Message) { + return true, pod.Status.Message + } + } + + return false, "" +} diff --git a/internal/executor/podchecks/failedpodchecks/pod_status_checks_test.go b/internal/executor/podchecks/failedpodchecks/pod_status_checks_test.go new file mode 100644 index 00000000000..aba540559cf --- /dev/null +++ b/internal/executor/podchecks/failedpodchecks/pod_status_checks_test.go @@ -0,0 +1,99 @@ +package failedpodchecks + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + + "github.com/armadaproject/armada/internal/executor/configuration/podchecks" +) + +func TestPodStatusChecker_IsRetryable(t *testing.T) { + tests := map[string]struct { + input *v1.Pod + checks []podchecks.PodStatusCheck + expectedResult bool + expectMessage bool + }{ + "empty checks": { + input: makePodWithMessageAndReason("message", "reason"), + expectedResult: false, + expectMessage: false, + }, + "matches message on regex": { + input: makePodWithMessageAndReason("message", "reason"), + checks: []podchecks.PodStatusCheck{{Regexp: "mess.*"}}, + expectedResult: true, + expectMessage: true, + }, + "matches on reason if supplied": { + input: makePodWithMessageAndReason("message", "reason"), + checks: []podchecks.PodStatusCheck{{Regexp: "mess.*", Reason: "reason"}}, + expectedResult: true, + expectMessage: true, + }, + "matches on reason if supplied - no match": { + input: makePodWithMessageAndReason("message", "reason"), + checks: []podchecks.PodStatusCheck{{Regexp: "mess.*", Reason: "reason2"}}, + expectedResult: false, + expectMessage: false, + }, + "multiple checks - no match": { + input: makePodWithMessageAndReason("message", "reason"), + checks: []podchecks.PodStatusCheck{ + {Regexp: "reas.*", Reason: ""}, + {Regexp: "reas.*", Reason: "reason"}, + {Regexp: "mess.*", Reason: "reason2"}, + }, + expectedResult: false, + expectMessage: false, + }, + "multiple checks - match": { + input: makePodWithMessageAndReason("message", "reason"), + checks: []podchecks.PodStatusCheck{ + {Regexp: "reas.*", Reason: ""}, + {Regexp: "reas.*", Reason: "reason"}, + {Regexp: "mess.*", Reason: ""}, + }, + expectedResult: true, + expectMessage: true, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + checker, err := NewPodStatusChecker(tc.checks) + require.NoError(t, err) + + isRetryable, message := checker.IsRetryable(tc.input) + + assert.Equal(t, tc.expectedResult, isRetryable) + if tc.expectMessage { + assert.NotEmpty(t, message) + } else { + assert.Empty(t, message) + } + }) + } +} + +func TestPodStatusChecker_Initialisation(t *testing.T) { + // Empty + _, err := NewPodStatusChecker([]podchecks.PodStatusCheck{}) + assert.NoError(t, err) + // Valid + _, err = NewPodStatusChecker([]podchecks.PodStatusCheck{{Regexp: ".*"}}) + assert.NoError(t, err) + // Invalid regex + _, err = NewPodStatusChecker([]podchecks.PodStatusCheck{{Regexp: "["}}) + assert.Error(t, err) +} + +func makePodWithMessageAndReason(message string, reason string) *v1.Pod { + return &v1.Pod{Status: v1.PodStatus{ + Message: message, + Reason: reason, + }} +} diff --git a/internal/executor/reporter/job_event_reporter.go b/internal/executor/reporter/job_event_reporter.go index 3df11d42696..bb9c2c250a1 100644 --- a/internal/executor/reporter/job_event_reporter.go +++ b/internal/executor/reporter/job_event_reporter.go @@ -5,19 +5,16 @@ import ( "time" v1 "k8s.io/api/core/v1" - "k8s.io/client-go/tools/cache" "k8s.io/utils/clock" log "github.com/armadaproject/armada/internal/common/logging" - clusterContext "github.com/armadaproject/armada/internal/executor/context" - domain2 "github.com/armadaproject/armada/internal/executor/domain" - "github.com/armadaproject/armada/internal/executor/job" "github.com/armadaproject/armada/internal/executor/util" ) type EventReporter interface { Report(events []EventMessage) error QueueEvent(event EventMessage, callback func(error)) + HasPendingEvents(pod *v1.Pod) bool } type queuedEvent struct { @@ -31,24 +28,18 @@ type JobEventReporter struct { eventQueued map[string]uint8 eventQueuedMutex sync.Mutex - jobRunStateStore *job.JobRunStateStore - clusterContext clusterContext.ClusterContext - clock clock.WithTicker - maxBatchSize int + clock clock.WithTicker + maxBatchSize int } func NewJobEventReporter( - clusterContext clusterContext.ClusterContext, - jobRunState *job.JobRunStateStore, eventSender EventSender, clock clock.WithTicker, maxBatchSize int, -) (*JobEventReporter, chan bool, error) { +) (*JobEventReporter, chan bool) { stop := make(chan bool) reporter := &JobEventReporter{ eventSender: eventSender, - clusterContext: clusterContext, - jobRunStateStore: jobRunState, eventBuffer: make(chan *queuedEvent, 1000000), eventQueued: map[string]uint8{}, eventQueuedMutex: sync.Mutex{}, @@ -56,94 +47,15 @@ func NewJobEventReporter( maxBatchSize: maxBatchSize, } - _, err := clusterContext.AddPodEventHandler(reporter.podEventHandler()) - if err != nil { - return nil, nil, err - } - go reporter.processEventQueue(stop) - return reporter, stop, nil -} - -func (eventReporter *JobEventReporter) podEventHandler() cache.ResourceEventHandlerFuncs { - return cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - pod, ok := obj.(*v1.Pod) - if !ok { - log.Errorf("Failed to process pod event due to it being an unexpected type. Failed to process %+v", obj) - return - } - go eventReporter.reportCurrentStatus(pod) - }, - UpdateFunc: func(oldObj, newObj interface{}) { - oldPod, ok := oldObj.(*v1.Pod) - if !ok { - log.Errorf("Failed to process pod event due to it being an unexpected type. Failed to process %+v", oldObj) - return - } - newPod, ok := newObj.(*v1.Pod) - if !ok { - log.Errorf("Failed to process pod event due to it being an unexpected type. Failed to process %+v", newObj) - return - } - go eventReporter.reportStatusUpdate(oldPod, newPod) - }, - } + return reporter, stop } func (eventReporter *JobEventReporter) Report(events []EventMessage) error { return eventReporter.eventSender.SendEvents(events) } -func (eventReporter *JobEventReporter) reportStatusUpdate(old *v1.Pod, new *v1.Pod) { - // Don't report status if the pod phase didn't change - if old.Status.Phase == new.Status.Phase { - return - } - // Don't report status change for pods Armada is deleting - // This prevents reporting JobFailed when we delete a pod - for example due to cancellation - if util.IsMarkedForDeletion(new) { - log.Infof("not sending event to report pod %s moving into phase %s as pod is marked for deletion", new.Name, new.Status.Phase) - return - } - eventReporter.reportCurrentStatus(new) -} - -func (eventReporter *JobEventReporter) reportCurrentStatus(pod *v1.Pod) { - if !util.IsManagedPod(pod) { - return - } - if util.HasCurrentStateBeenReported(pod) { - return - } - - event, err := CreateEventForCurrentState(pod, eventReporter.clusterContext.GetClusterId()) - if err != nil { - log.Errorf("Failed to report event: %v", err) - return - } - - eventReporter.QueueEvent(EventMessage{Event: event, JobRunId: util.ExtractJobRunId(pod)}, func(err error) { - if err != nil { - log.Errorf("Failed to report event: %s", err) - return - } - - if util.IsReportingPhaseRequired(pod.Status.Phase) { - err = eventReporter.addAnnotationToMarkStateReported(pod) - if err != nil { - log.Errorf("Failed to add state annotation %s to pod %s: %v", string(pod.Status.Phase), pod.Name, err) - return - } - } - }) - - if pod.Status.Phase == v1.PodRunning && requiresIngressToBeReported(pod) { - eventReporter.attemptToReportIngressInfoEvent(pod) - } -} - func (eventReporter *JobEventReporter) QueueEvent(event EventMessage, callback func(error)) { eventReporter.eventQueuedMutex.Lock() defer eventReporter.eventQueuedMutex.Unlock() @@ -215,112 +127,9 @@ func queuedEventsToEventMessages(queuedEvents []*queuedEvent) []EventMessage { return result } -func (eventReporter *JobEventReporter) addAnnotationToMarkStateReported(pod *v1.Pod) error { - annotations := make(map[string]string) - annotationName := string(pod.Status.Phase) - annotations[annotationName] = time.Now().String() - - return eventReporter.clusterContext.AddAnnotation(pod, annotations) -} - -func (eventReporter *JobEventReporter) addAnnotationToMarkIngressReported(pod *v1.Pod) error { - annotations := make(map[string]string) - annotationName := domain2.IngressReported - annotations[annotationName] = time.Now().String() - - return eventReporter.clusterContext.AddAnnotation(pod, annotations) -} - -func (eventReporter *JobEventReporter) ReportMissingJobEvents() { - allBatchPods, err := eventReporter.clusterContext.GetActiveBatchPods() - if err != nil { - log.Errorf("Failed to reconcile missing job events: %v", err) - return - } - podsWithCurrentPhaseNotReported := filterPodsWithCurrentStateNotReported(allBatchPods) - - for _, pod := range podsWithCurrentPhaseNotReported { - if util.IsReportingPhaseRequired(pod.Status.Phase) && !eventReporter.hasPendingEvents(pod) { - eventReporter.reportCurrentStatus(pod) - } - } - - podWithIngressNotReported := util.FilterPods(allBatchPods, func(pod *v1.Pod) bool { - return pod.Status.Phase == v1.PodRunning && - requiresIngressToBeReported(pod) && - util.HasPodBeenInStateForLongerThanGivenDuration(pod, 15*time.Second) - }) - - for _, pod := range podWithIngressNotReported { - if !eventReporter.hasPendingEvents(pod) { - eventReporter.attemptToReportIngressInfoEvent(pod) - } - } -} - -func (eventReporter *JobEventReporter) attemptToReportIngressInfoEvent(pod *v1.Pod) { - expectedNumberOfServices := util.GetExpectedNumberOfAssociatedServices(pod) - expectedNumberOfIngresses := util.GetExpectedNumberOfAssociatedIngresses(pod) - associatedServices, err := eventReporter.clusterContext.GetServices(pod) - if err != nil { - log.Errorf("Failed to report event JobIngressInfoEvent for pod %s: %v", pod.Name, err) - return - } - associatedIngresses, err := eventReporter.clusterContext.GetIngresses(pod) - if err != nil { - log.Errorf("Failed to report event JobIngressInfoEvent for pod %s: %v", pod.Name, err) - return - } - if len(associatedServices) != expectedNumberOfServices || len(associatedIngresses) != expectedNumberOfIngresses { - log.Warnf("Not reporting JobIngressInfoEvent for pod %s because not all expected associated services "+ - "(current %d, expected %d) or ingresses (current %d, expected %d) exist yet", - pod.Name, len(associatedServices), expectedNumberOfServices, len(associatedIngresses), expectedNumberOfIngresses) - // Don't report ingress info until all expected ingresses exist - return - } - - ingressInfoEvent, err := CreateJobIngressInfoEvent(pod, eventReporter.clusterContext.GetClusterId(), associatedServices, associatedIngresses) - if err != nil { - log.Errorf("Failed to report event JobIngressInfoEvent for pod %s: %v", pod.Name, err) - return - } - eventReporter.QueueEvent(EventMessage{Event: ingressInfoEvent, JobRunId: util.ExtractJobRunId(pod)}, func(err error) { - if err != nil { - log.Errorf("Failed to report event JobIngressInfoEvent for pod %s: %v", pod.Name, err) - return - } - - err = eventReporter.addAnnotationToMarkIngressReported(pod) - if err != nil { - log.Errorf("Failed to add ingress reported annotation %s to pod %s: %v", string(pod.Status.Phase), pod.Name, err) - return - } - }) -} - -func requiresIngressToBeReported(pod *v1.Pod) bool { - if !util.HasIngress(pod) { - return false - } - if _, exists := pod.Annotations[domain2.IngressReported]; exists { - return false - } - return true -} - -func (eventReporter *JobEventReporter) hasPendingEvents(pod *v1.Pod) bool { +func (eventReporter *JobEventReporter) HasPendingEvents(pod *v1.Pod) bool { eventReporter.eventQueuedMutex.Lock() defer eventReporter.eventQueuedMutex.Unlock() id := util.ExtractJobRunId(pod) return eventReporter.eventQueued[id] > 0 } - -func filterPodsWithCurrentStateNotReported(pods []*v1.Pod) []*v1.Pod { - podsWithMissingEvent := make([]*v1.Pod, 0) - for _, pod := range pods { - if !util.HasCurrentStateBeenReported(pod) && util.HasPodBeenInStateForLongerThanGivenDuration(pod, 30*time.Second) { - podsWithMissingEvent = append(podsWithMissingEvent, pod) - } - } - return podsWithMissingEvent -} diff --git a/internal/executor/reporter/job_event_reporter_test.go b/internal/executor/reporter/job_event_reporter_test.go index d90076a24b9..e84d028b358 100644 --- a/internal/executor/reporter/job_event_reporter_test.go +++ b/internal/executor/reporter/job_event_reporter_test.go @@ -14,42 +14,13 @@ import ( clock "k8s.io/utils/clock/testing" util2 "github.com/armadaproject/armada/internal/common/util" - fakecontext "github.com/armadaproject/armada/internal/executor/context/fake" "github.com/armadaproject/armada/internal/executor/domain" - "github.com/armadaproject/armada/internal/executor/job" "github.com/armadaproject/armada/internal/executor/util" "github.com/armadaproject/armada/pkg/armadaevents" ) -func TestRequiresIngressToBeReported_FalseWhenIngressHasBeenReported(t *testing.T) { - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - domain.HasIngress: "true", - domain.IngressReported: time.Now().String(), - }, - }, - } - assert.False(t, requiresIngressToBeReported(pod)) -} - -func TestRequiresIngressToBeReported_FalseWhenNonIngressPod(t *testing.T) { - pod := &v1.Pod{} - assert.False(t, requiresIngressToBeReported(pod)) -} - -func TestRequiresIngressToBeReported_TrueWhenHasIngressButNotIngressReportedAnnotation(t *testing.T) { - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{domain.HasIngress: "true"}, - }, - } - assert.True(t, requiresIngressToBeReported(pod)) -} - func TestJobEventReporter_SendsEventImmediately_OnceNumberOfWaitingEventsMatchesBatchSize(t *testing.T) { - jobEventReporter, eventSender, _, err := setupBatchEventsTest(2) - require.NoError(t, err) + jobEventReporter, eventSender, _ := setupBatchEventsTest(2) pod1 := createPod(1) pod2 := createPod(2) @@ -67,8 +38,7 @@ func TestJobEventReporter_SendsEventImmediately_OnceNumberOfWaitingEventsMatches } func TestJobEventReporter_SendsAllEventsInBuffer_EachBatchTickInterval(t *testing.T) { - jobEventReporter, eventSender, testClock, err := setupBatchEventsTest(2) - require.NoError(t, err) + jobEventReporter, eventSender, testClock := setupBatchEventsTest(2) pod1 := createPod(1) jobEventReporter.QueueEvent(EventMessage{createFailedEvent(t, pod1), util.ExtractJobRunId(pod1)}, func(err error) {}) @@ -90,13 +60,11 @@ func createFailedEvent(t *testing.T, pod *v1.Pod) *armadaevents.EventSequence { return event } -func setupBatchEventsTest(batchSize int) (*JobEventReporter, *FakeEventSender, *clock.FakeClock, error) { - executorContext := fakecontext.NewSyncFakeClusterContext() +func setupBatchEventsTest(batchSize int) (*JobEventReporter, *FakeEventSender, *clock.FakeClock) { eventSender := NewFakeEventSender() - jobRunState := job.NewJobRunStateStore(executorContext) testClock := clock.NewFakeClock(time.Now()) - jobEventReporter, _, err := NewJobEventReporter(executorContext, jobRunState, eventSender, testClock, batchSize) - return jobEventReporter, eventSender, testClock, err + jobEventReporter, _ := NewJobEventReporter(eventSender, testClock, batchSize) + return jobEventReporter, eventSender, testClock } func createPod(index int) *v1.Pod { diff --git a/internal/executor/reporter/mocks/testfixtures.go b/internal/executor/reporter/mocks/testfixtures.go index 4e4499ed428..d308e3828a3 100644 --- a/internal/executor/reporter/mocks/testfixtures.go +++ b/internal/executor/reporter/mocks/testfixtures.go @@ -3,6 +3,8 @@ package mocks import ( "fmt" + v1 "k8s.io/api/core/v1" + "github.com/armadaproject/armada/internal/executor/reporter" ) @@ -27,3 +29,7 @@ func (f *FakeEventReporter) QueueEvent(event reporter.EventMessage, callback fun e := f.Report([]reporter.EventMessage{event}) callback(e) } + +func (f *FakeEventReporter) HasPendingEvents(pod *v1.Pod) bool { + return false +} diff --git a/internal/executor/service/job_state_reporter.go b/internal/executor/service/job_state_reporter.go new file mode 100644 index 00000000000..e753df39e88 --- /dev/null +++ b/internal/executor/service/job_state_reporter.go @@ -0,0 +1,233 @@ +package service + +import ( + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/cache" + + log "github.com/armadaproject/armada/internal/common/logging" + clusterContext "github.com/armadaproject/armada/internal/executor/context" + domain2 "github.com/armadaproject/armada/internal/executor/domain" + "github.com/armadaproject/armada/internal/executor/reporter" + "github.com/armadaproject/armada/internal/executor/util" +) + +type JobStateReporter struct { + eventReporter reporter.EventReporter + clusterContext clusterContext.ClusterContext + podIssueHandler IssueHandler +} + +func NewJobStateReporter( + clusterContext clusterContext.ClusterContext, + eventReporter reporter.EventReporter, + podIssueHandler IssueHandler, +) (*JobStateReporter, error) { + stateReporter := &JobStateReporter{ + eventReporter: eventReporter, + clusterContext: clusterContext, + podIssueHandler: podIssueHandler, + } + + _, err := clusterContext.AddPodEventHandler(stateReporter.podEventHandler()) + if err != nil { + return nil, err + } + + return stateReporter, nil +} + +func (stateReporter *JobStateReporter) podEventHandler() cache.ResourceEventHandlerFuncs { + return cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + pod, ok := obj.(*v1.Pod) + if !ok { + log.Errorf("Failed to process pod event due to it being an unexpected type. Failed to process %+v", obj) + return + } + go stateReporter.reportCurrentStatus(pod) + }, + UpdateFunc: func(oldObj, newObj interface{}) { + oldPod, ok := oldObj.(*v1.Pod) + if !ok { + log.Errorf("Failed to process pod event due to it being an unexpected type. Failed to process %+v", oldObj) + return + } + newPod, ok := newObj.(*v1.Pod) + if !ok { + log.Errorf("Failed to process pod event due to it being an unexpected type. Failed to process %+v", newObj) + return + } + go stateReporter.reportStatusUpdate(oldPod, newPod) + }, + } +} + +func (stateReporter *JobStateReporter) reportStatusUpdate(old *v1.Pod, new *v1.Pod) { + // Don't report status if the pod phase didn't change + if old.Status.Phase == new.Status.Phase { + return + } + // Don't report status change for pods Armada is deleting + // This prevents reporting JobFailed when we delete a pod - for example due to cancellation + if util.IsMarkedForDeletion(new) { + log.Infof("not sending event to report pod %s moving into phase %s as pod is marked for deletion", new.Name, new.Status.Phase) + return + } + stateReporter.reportCurrentStatus(new) +} + +func (stateReporter *JobStateReporter) reportCurrentStatus(pod *v1.Pod) { + if !util.IsManagedPod(pod) { + return + } + if util.HasCurrentStateBeenReported(pod) { + return + } + + event, err := reporter.CreateEventForCurrentState(pod, stateReporter.clusterContext.GetClusterId()) + if err != nil { + log.Errorf("Failed to report event: %v", err) + return + } + + if pod.Status.Phase == v1.PodFailed { + hasIssue := stateReporter.podIssueHandler.HasIssue(util.ExtractJobRunId(pod)) + if hasIssue { + // Pod already being handled by issue handler + return + } + issueAdded, err := stateReporter.podIssueHandler.DetectAndRegisterFailedPodIssue(pod) + if issueAdded { + // Pod already being handled by issue handler + return + } + if err != nil { + log.Errorf("Failed detecting issue on failed pod %s(%s) - %v", pod.Name, util.ExtractJobRunId(pod), err) + // Don't return here, as it is very important we don't block reporting a terminal event (failed) + } + } + + stateReporter.eventReporter.QueueEvent(reporter.EventMessage{Event: event, JobRunId: util.ExtractJobRunId(pod)}, func(err error) { + if err != nil { + log.Errorf("Failed to report event: %s", err) + return + } + + if util.IsReportingPhaseRequired(pod.Status.Phase) { + err = stateReporter.addAnnotationToMarkStateReported(pod) + if err != nil { + log.Errorf("Failed to add state annotation %s to pod %s: %v", string(pod.Status.Phase), pod.Name, err) + return + } + } + }) + + if pod.Status.Phase == v1.PodRunning && requiresIngressToBeReported(pod) { + stateReporter.attemptToReportIngressInfoEvent(pod) + } +} + +func (stateReporter *JobStateReporter) addAnnotationToMarkStateReported(pod *v1.Pod) error { + annotations := make(map[string]string) + annotationName := string(pod.Status.Phase) + annotations[annotationName] = time.Now().String() + + return stateReporter.clusterContext.AddAnnotation(pod, annotations) +} + +func (stateReporter *JobStateReporter) addAnnotationToMarkIngressReported(pod *v1.Pod) error { + annotations := make(map[string]string) + annotationName := domain2.IngressReported + annotations[annotationName] = time.Now().String() + + return stateReporter.clusterContext.AddAnnotation(pod, annotations) +} + +func (stateReporter *JobStateReporter) ReportMissingJobEvents() { + allBatchPods, err := stateReporter.clusterContext.GetActiveBatchPods() + if err != nil { + log.Errorf("Failed to reconcile missing job events: %v", err) + return + } + podsWithCurrentPhaseNotReported := filterPodsWithCurrentStateNotReported(allBatchPods) + + for _, pod := range podsWithCurrentPhaseNotReported { + if util.IsReportingPhaseRequired(pod.Status.Phase) && !stateReporter.eventReporter.HasPendingEvents(pod) { + stateReporter.reportCurrentStatus(pod) + } + } + + podWithIngressNotReported := util.FilterPods(allBatchPods, func(pod *v1.Pod) bool { + return pod.Status.Phase == v1.PodRunning && + requiresIngressToBeReported(pod) && + util.HasPodBeenInStateForLongerThanGivenDuration(pod, 15*time.Second) + }) + + for _, pod := range podWithIngressNotReported { + if !stateReporter.eventReporter.HasPendingEvents(pod) { + stateReporter.attemptToReportIngressInfoEvent(pod) + } + } +} + +func (stateReporter *JobStateReporter) attemptToReportIngressInfoEvent(pod *v1.Pod) { + expectedNumberOfServices := util.GetExpectedNumberOfAssociatedServices(pod) + expectedNumberOfIngresses := util.GetExpectedNumberOfAssociatedIngresses(pod) + associatedServices, err := stateReporter.clusterContext.GetServices(pod) + if err != nil { + log.Errorf("Failed to report event JobIngressInfoEvent for pod %s: %v", pod.Name, err) + return + } + associatedIngresses, err := stateReporter.clusterContext.GetIngresses(pod) + if err != nil { + log.Errorf("Failed to report event JobIngressInfoEvent for pod %s: %v", pod.Name, err) + return + } + if len(associatedServices) != expectedNumberOfServices || len(associatedIngresses) != expectedNumberOfIngresses { + log.Warnf("Not reporting JobIngressInfoEvent for pod %s because not all expected associated services "+ + "(current %d, expected %d) or ingresses (current %d, expected %d) exist yet", + pod.Name, len(associatedServices), expectedNumberOfServices, len(associatedIngresses), expectedNumberOfIngresses) + // Don't report ingress info until all expected ingresses exist + return + } + + ingressInfoEvent, err := reporter.CreateJobIngressInfoEvent(pod, stateReporter.clusterContext.GetClusterId(), associatedServices, associatedIngresses) + if err != nil { + log.Errorf("Failed to report event JobIngressInfoEvent for pod %s: %v", pod.Name, err) + return + } + stateReporter.eventReporter.QueueEvent(reporter.EventMessage{Event: ingressInfoEvent, JobRunId: util.ExtractJobRunId(pod)}, func(err error) { + if err != nil { + log.Errorf("Failed to report event JobIngressInfoEvent for pod %s: %v", pod.Name, err) + return + } + + err = stateReporter.addAnnotationToMarkIngressReported(pod) + if err != nil { + log.Errorf("Failed to add ingress reported annotation %s to pod %s: %v", string(pod.Status.Phase), pod.Name, err) + return + } + }) +} + +func requiresIngressToBeReported(pod *v1.Pod) bool { + if !util.HasIngress(pod) { + return false + } + if _, exists := pod.Annotations[domain2.IngressReported]; exists { + return false + } + return true +} + +func filterPodsWithCurrentStateNotReported(pods []*v1.Pod) []*v1.Pod { + podsWithMissingEvent := make([]*v1.Pod, 0) + for _, pod := range pods { + if !util.HasCurrentStateBeenReported(pod) && util.HasPodBeenInStateForLongerThanGivenDuration(pod, 30*time.Second) { + podsWithMissingEvent = append(podsWithMissingEvent, pod) + } + } + return podsWithMissingEvent +} diff --git a/internal/executor/service/job_state_reporter_test.go b/internal/executor/service/job_state_reporter_test.go new file mode 100644 index 00000000000..ff92a601b6d --- /dev/null +++ b/internal/executor/service/job_state_reporter_test.go @@ -0,0 +1,217 @@ +package service + +import ( + "fmt" + "reflect" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + fakecontext "github.com/armadaproject/armada/internal/executor/context/fake" + "github.com/armadaproject/armada/internal/executor/domain" + "github.com/armadaproject/armada/internal/executor/reporter" + "github.com/armadaproject/armada/internal/executor/reporter/mocks" + "github.com/armadaproject/armada/internal/executor/util" + "github.com/armadaproject/armada/pkg/armadaevents" +) + +func TestRequiresIngressToBeReported_FalseWhenIngressHasBeenReported(t *testing.T) { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + domain.HasIngress: "true", + domain.IngressReported: time.Now().String(), + }, + }, + } + assert.False(t, requiresIngressToBeReported(pod)) +} + +func TestRequiresIngressToBeReported_FalseWhenNonIngressPod(t *testing.T) { + pod := &v1.Pod{} + assert.False(t, requiresIngressToBeReported(pod)) +} + +func TestRequiresIngressToBeReported_TrueWhenHasIngressButNotIngressReportedAnnotation(t *testing.T) { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{domain.HasIngress: "true"}, + }, + } + assert.True(t, requiresIngressToBeReported(pod)) +} + +func TestJobStateReporter_HandlesPodAddEvents(t *testing.T) { + type podAddEventTest struct { + pod *v1.Pod + expectEvent bool + expectAnnotation bool + expectedType reflect.Type + } + + tests := []podAddEventTest{ + {pod: &v1.Pod{Status: v1.PodStatus{Phase: v1.PodPending}}, expectAnnotation: false, expectEvent: false}, + {pod: makeTestPod(v1.PodStatus{Phase: v1.PodPending}), expectEvent: true, expectAnnotation: false, expectedType: reflect.TypeOf(&armadaevents.EventSequence_Event_JobRunAssigned{})}, + {pod: makeTestPod(v1.PodStatus{Phase: v1.PodRunning}), expectEvent: true, expectAnnotation: true, expectedType: reflect.TypeOf(&armadaevents.EventSequence_Event_JobRunRunning{})}, + {pod: makeTestPod(v1.PodStatus{Phase: v1.PodSucceeded}), expectEvent: true, expectAnnotation: true, expectedType: reflect.TypeOf(&armadaevents.EventSequence_Event_JobRunSucceeded{})}, + {pod: makeTestPod(v1.PodStatus{Phase: v1.PodFailed}), expectEvent: true, expectAnnotation: true, expectedType: reflect.TypeOf(&armadaevents.EventSequence_Event_JobRunErrors{})}, + } + + for _, test := range tests { + _, _, eventReporter, fakeClusterContext := setUpJobStateReporterTest(t) + + addPod(t, fakeClusterContext, test.pod) + fakeClusterContext.SimulatePodAddEvent(test.pod) + time.Sleep(time.Millisecond * 100) // Give time for async routine to process message + + if test.expectEvent { + assertExpectedEvents(t, test.pod, eventReporter.ReceivedEvents, test.expectedType) + } else { + assert.Len(t, eventReporter.ReceivedEvents, 0) + } + + if test.expectAnnotation { + assertExpectedAnnotations(t, test.pod, fakeClusterContext) + } else { + assert.Len(t, fakeClusterContext.AnnotationsAdded, 0) + } + } +} + +func TestJobStateReporter_HandlesPodUpdateEvents(t *testing.T) { + type podAddEventTest struct { + before v1.PodPhase + after v1.PodPhase + expectEvent bool + expectedType reflect.Type + } + + tests := []podAddEventTest{ + {before: v1.PodPending, after: v1.PodPending, expectEvent: false}, // No update sent if pod doesn't change phase + {before: v1.PodPending, after: v1.PodRunning, expectEvent: true, expectedType: reflect.TypeOf(&armadaevents.EventSequence_Event_JobRunRunning{})}, + {before: v1.PodPending, after: v1.PodFailed, expectEvent: true, expectedType: reflect.TypeOf(&armadaevents.EventSequence_Event_JobRunErrors{})}, + {before: v1.PodPending, after: v1.PodSucceeded, expectEvent: true, expectedType: reflect.TypeOf(&armadaevents.EventSequence_Event_JobRunSucceeded{})}, + + {before: v1.PodRunning, after: v1.PodRunning, expectEvent: false}, // No update sent if pod doesn't change phase + {before: v1.PodRunning, after: v1.PodFailed, expectEvent: true, expectedType: reflect.TypeOf(&armadaevents.EventSequence_Event_JobRunErrors{})}, + {before: v1.PodRunning, after: v1.PodSucceeded, expectEvent: true, expectedType: reflect.TypeOf(&armadaevents.EventSequence_Event_JobRunSucceeded{})}, + + {before: v1.PodFailed, after: v1.PodFailed, expectEvent: false}, // No update sent if pod doesn't change phase + {before: v1.PodSucceeded, after: v1.PodSucceeded, expectEvent: false}, // No update sent if pod doesn't change phase + } + + for _, test := range tests { + _, _, eventReporter, fakeClusterContext := setUpJobStateReporterTest(t) + + before := makeTestPod(v1.PodStatus{Phase: test.before}) + after := copyWithUpdatedPhase(before, test.after) + + addPod(t, fakeClusterContext, before) + fakeClusterContext.SimulateUpdateAddEvent(before, after) + time.Sleep(time.Millisecond * 100) // Give time for async routine to process message + + if test.expectEvent { + assertExpectedEvents(t, before, eventReporter.ReceivedEvents, test.expectedType) + assertExpectedAnnotations(t, after, fakeClusterContext) + } else { + assert.Len(t, eventReporter.ReceivedEvents, 0) + assert.Len(t, fakeClusterContext.AnnotationsAdded, 0) + } + } +} + +func TestJobStateReporter_HandlesPodUpdateEvents_IgnoreUnmanagedPods(t *testing.T) { + _, _, eventReporter, fakeClusterContext := setUpJobStateReporterTest(t) + + before := &v1.Pod{Status: v1.PodStatus{Phase: v1.PodPending}} + after := &v1.Pod{Status: v1.PodStatus{Phase: v1.PodRunning}} + + fakeClusterContext.SimulateUpdateAddEvent(before, after) + time.Sleep(time.Millisecond * 100) // Give time for async routine to process message + + assert.Len(t, eventReporter.ReceivedEvents, 0) +} + +func TestJobStateReporter_HandlesFailedPod_WithRetryableError(t *testing.T) { + jobStateReporter, _, eventReporter, fakeClusterContext := setUpJobStateReporterTest(t) + + before := makeTestPod(v1.PodStatus{Phase: v1.PodRunning}) + after := copyWithUpdatedPhase(before, v1.PodFailed) + + // Does not send event if issue handler detects an issue + jobStateReporter.podIssueHandler = &stubIssueHandler{detectAndRegisterFailedPodIssueResult: true, detectAndRegisterFailedPodIssueError: nil} + fakeClusterContext.SimulateUpdateAddEvent(before, after) + time.Sleep(time.Millisecond * 100) // Give time for async routine to process message + assert.Len(t, eventReporter.ReceivedEvents, 0) + + // Does not send event if issue handler already knows of issue for run + jobStateReporter.podIssueHandler = &stubIssueHandler{ + runIdsWithIssues: map[string]bool{util.ExtractJobRunId(after): true}, + detectAndRegisterFailedPodIssueResult: false, + detectAndRegisterFailedPodIssueError: nil, + } + fakeClusterContext.SimulateUpdateAddEvent(before, after) + time.Sleep(time.Millisecond * 100) // Give time for async routine to process message + assert.Len(t, eventReporter.ReceivedEvents, 0) + + // Does send event if issue handler errors + jobStateReporter.podIssueHandler = &stubIssueHandler{detectAndRegisterFailedPodIssueResult: false, detectAndRegisterFailedPodIssueError: fmt.Errorf("error")} + fakeClusterContext.SimulateUpdateAddEvent(before, after) + time.Sleep(time.Millisecond * 100) // Give time for async routine to process message + assert.Len(t, eventReporter.ReceivedEvents, 1) + assertExpectedEvents(t, before, eventReporter.ReceivedEvents, reflect.TypeOf(&armadaevents.EventSequence_Event_JobRunErrors{})) +} + +func setUpJobStateReporterTest(t *testing.T) (*JobStateReporter, *stubIssueHandler, *mocks.FakeEventReporter, *fakecontext.SyncFakeClusterContext) { + fakeClusterContext := fakecontext.NewSyncFakeClusterContext() + eventReporter := mocks.NewFakeEventReporter() + issueHandler := &stubIssueHandler{detectAndRegisterFailedPodIssueResult: false, detectAndRegisterFailedPodIssueError: nil} + jobStateReporter, err := NewJobStateReporter(fakeClusterContext, eventReporter, issueHandler) + require.NoError(t, err) + return jobStateReporter, issueHandler, eventReporter, fakeClusterContext +} + +func assertExpectedEvents(t *testing.T, pod *v1.Pod, messages []reporter.EventMessage, expectedType reflect.Type) { + assert.Len(t, messages, 1) + assert.Len(t, messages[0].Event.Events, 1) + + assert.Equal(t, util.ExtractJobRunId(pod), messages[0].JobRunId) + assert.Equal(t, util.ExtractQueue(pod), messages[0].Event.Queue) + assert.Equal(t, util.ExtractJobSet(pod), messages[0].Event.JobSetName) + + event := messages[0].Event.Events[0] + resultType := reflect.TypeOf(event.Event) + assert.Equal(t, expectedType, resultType) +} + +func assertExpectedAnnotations(t *testing.T, pod *v1.Pod, clusterContext *fakecontext.SyncFakeClusterContext) { + jobAnnotations := clusterContext.AnnotationsAdded[util.ExtractJobId(pod)] + assert.Len(t, jobAnnotations, 1) + _, exists := jobAnnotations[string(pod.Status.Phase)] + assert.True(t, exists) +} + +type stubIssueHandler struct { + runIdsWithIssues map[string]bool + detectAndRegisterFailedPodIssueResult bool + detectAndRegisterFailedPodIssueError error +} + +func (s *stubIssueHandler) HasIssue(runId string) bool { + _, exists := s.runIdsWithIssues[runId] + return exists +} + +func (s *stubIssueHandler) DetectAndRegisterFailedPodIssue(pod *v1.Pod) (bool, error) { + return s.detectAndRegisterFailedPodIssueResult, s.detectAndRegisterFailedPodIssueError +} + +func copyWithUpdatedPhase(pod *v1.Pod, newPhase v1.PodPhase) *v1.Pod { + result := pod.DeepCopy() + result.Status.Phase = newPhase + return result +} diff --git a/internal/executor/service/pod_issue_handler.go b/internal/executor/service/pod_issue_handler.go index 72e9ea57531..b4e8e0630f6 100644 --- a/internal/executor/service/pod_issue_handler.go +++ b/internal/executor/service/pod_issue_handler.go @@ -17,6 +17,7 @@ import ( executorContext "github.com/armadaproject/armada/internal/executor/context" "github.com/armadaproject/armada/internal/executor/job" "github.com/armadaproject/armada/internal/executor/podchecks" + "github.com/armadaproject/armada/internal/executor/podchecks/failedpodchecks" "github.com/armadaproject/armada/internal/executor/reporter" "github.com/armadaproject/armada/internal/executor/util" "github.com/armadaproject/armada/pkg/armadaevents" @@ -31,6 +32,7 @@ const ( ActiveDeadlineExceeded ExternallyDeleted ErrorDuringIssueHandling + FailedStartingUp ) type podIssue struct { @@ -62,10 +64,16 @@ type runIssue struct { Reported bool } -type IssueHandler struct { +type IssueHandler interface { + HasIssue(runId string) bool + DetectAndRegisterFailedPodIssue(pod *v1.Pod) (bool, error) +} + +type PodIssueHandler struct { clusterContext executorContext.ClusterContext eventReporter reporter.EventReporter pendingPodChecker podchecks.PodChecker + failedPodChecker failedpodchecks.RetryChecker stateChecksConfig configuration.StateChecksConfiguration stuckTerminatingPodExpiry time.Duration @@ -77,19 +85,21 @@ type IssueHandler struct { clock clock.Clock } -func NewIssueHandler( +func NewPodIssuerHandler( jobRunState job.RunStateStore, clusterContext executorContext.ClusterContext, eventReporter reporter.EventReporter, stateChecksConfig configuration.StateChecksConfiguration, pendingPodChecker podchecks.PodChecker, + failedPodChecker failedpodchecks.RetryChecker, stuckTerminatingPodExpiry time.Duration, -) (*IssueHandler, error) { - issueHandler := &IssueHandler{ +) (*PodIssueHandler, error) { + issueHandler := &PodIssueHandler{ jobRunState: jobRunState, clusterContext: clusterContext, eventReporter: eventReporter, pendingPodChecker: pendingPodChecker, + failedPodChecker: failedPodChecker, stateChecksConfig: stateChecksConfig, stuckTerminatingPodExpiry: stuckTerminatingPodExpiry, knownPodIssues: map[string]*runIssue{}, @@ -114,7 +124,7 @@ func NewIssueHandler( return issueHandler, nil } -func (p *IssueHandler) hasIssue(runId string) bool { +func (p *PodIssueHandler) HasIssue(runId string) bool { p.podIssueMutex.Lock() defer p.podIssueMutex.Unlock() @@ -126,25 +136,65 @@ func (p *IssueHandler) hasIssue(runId string) bool { return exists } -func (p *IssueHandler) registerIssue(issue *runIssue) { +func (p *PodIssueHandler) DetectAndRegisterFailedPodIssue(pod *v1.Pod) (bool, error) { + if !util.IsManagedPod(pod) || pod.Status.Phase != v1.PodFailed { + return false, nil + } + jobId := util.ExtractJobId(pod) + runId := util.ExtractJobRunId(pod) + + podEvents, err := p.clusterContext.GetPodEvents(pod) + if err != nil { + return false, fmt.Errorf("Failed retrieving pod events for pod %s: %v", pod.Name, err) + } + + isRetryable, message := p.failedPodChecker.IsRetryable(pod, podEvents) + if isRetryable { + return p.registerIssue(&runIssue{ + JobId: jobId, + RunId: runId, + PodIssue: &podIssue{ + OriginalPodState: pod.DeepCopy(), + Message: message, + DebugMessage: createDebugMessage(podEvents), + Retryable: true, + DeletionRequested: false, + Type: FailedStartingUp, + }, + Reported: false, + }) + } else { + return false, nil + } +} + +func (p *PodIssueHandler) registerIssue(issue *runIssue) (bool, error) { p.podIssueMutex.Lock() defer p.podIssueMutex.Unlock() runId := issue.RunId if runId == "" { - log.Warnf("Not registering an issue for job %s as run id was empty", issue.JobId) - return + return false, fmt.Errorf("Not registering an issue for job %s as run id was empty", issue.JobId) } _, exists := p.knownPodIssues[issue.RunId] if !exists { log.Infof("Issue for job %s run %s is registered", issue.JobId, issue.RunId) p.knownPodIssues[issue.RunId] = issue + return true, nil } else { log.Warnf("Not registering an issue for job %s (runId %s) as it already has an issue set", issue.JobId, issue.RunId) + return false, nil } } -func (p *IssueHandler) markIssuesResolved(issue *runIssue) { +func (p *PodIssueHandler) attemptToRegisterIssue(issue *runIssue) { + _, err := p.registerIssue(issue) + if err != nil { + log.Warn(err) + } +} + +func (p *PodIssueHandler) markIssuesResolved(issue *runIssue) { p.podIssueMutex.Lock() defer p.podIssueMutex.Unlock() log.Infof("Issue for job %s run %s is resolved", issue.JobId, issue.RunId) @@ -152,11 +202,11 @@ func (p *IssueHandler) markIssuesResolved(issue *runIssue) { delete(p.knownPodIssues, issue.RunId) } -func (p *IssueHandler) markIssueReported(issue *runIssue) { +func (p *PodIssueHandler) markIssueReported(issue *runIssue) { issue.Reported = true } -func (p *IssueHandler) HandlePodIssues() { +func (p *PodIssueHandler) HandlePodIssues() { managedPods, err := p.clusterContext.GetBatchPods() if err != nil { log.WithError(err).Errorf("unable to handle pod issus as failed to load pods") @@ -168,9 +218,9 @@ func (p *IssueHandler) HandlePodIssues() { p.handleKnownIssues(ctx, managedPods) } -func (p *IssueHandler) detectPodIssues(allManagedPods []*v1.Pod) { +func (p *PodIssueHandler) detectPodIssues(allManagedPods []*v1.Pod) { for _, pod := range allManagedPods { - if p.hasIssue(util.ExtractJobRunId(pod)) { + if p.HasIssue(util.ExtractJobRunId(pod)) { continue } if util.IsInTerminalState(pod) && util.HasCurrentStateBeenReported(pod) { @@ -188,7 +238,7 @@ func (p *IssueHandler) detectPodIssues(allManagedPods []*v1.Pod) { Type: StuckTerminating, } - p.registerIssue(&runIssue{ + p.attemptToRegisterIssue(&runIssue{ JobId: util.ExtractJobId(pod), RunId: util.ExtractJobRunId(pod), PodIssue: issue, @@ -204,7 +254,7 @@ func (p *IssueHandler) detectPodIssues(allManagedPods []*v1.Pod) { Type: ActiveDeadlineExceeded, } - p.registerIssue(&runIssue{ + p.attemptToRegisterIssue(&runIssue{ JobId: util.ExtractJobId(pod), RunId: util.ExtractJobRunId(pod), PodIssue: issue, @@ -242,7 +292,7 @@ func (p *IssueHandler) detectPodIssues(allManagedPods []*v1.Pod) { Retryable: retryable, Type: podIssueType, } - p.registerIssue(&runIssue{ + p.attemptToRegisterIssue(&runIssue{ JobId: util.ExtractJobId(pod), RunId: util.ExtractJobRunId(pod), PodIssue: issue, @@ -267,7 +317,7 @@ func createDebugMessage(podEvents []*v1.Event) string { } // Returns true if the pod has been running longer than its activeDeadlineSeconds + grace period -func (p *IssueHandler) hasExceededActiveDeadline(pod *v1.Pod) bool { +func (p *PodIssueHandler) hasExceededActiveDeadline(pod *v1.Pod) bool { if pod.Spec.ActiveDeadlineSeconds == nil { return false } @@ -287,7 +337,7 @@ func (p *IssueHandler) hasExceededActiveDeadline(pod *v1.Pod) bool { return currentRunTimeSeconds > deadline } -func (p *IssueHandler) handleKnownIssues(ctx *armadacontext.Context, allManagedPods []*v1.Pod) { +func (p *PodIssueHandler) handleKnownIssues(ctx *armadacontext.Context, allManagedPods []*v1.Pod) { // Make issues from pods + issues issues := createIssues(allManagedPods, p.knownPodIssues) util.ProcessItemsWithThreadPool(ctx, 20, issues, p.handleRunIssue) @@ -315,7 +365,7 @@ func createIssues(managedPods []*v1.Pod, runIssues map[string]*runIssue) []*issu return result } -func (p *IssueHandler) handleRunIssue(issue *issue) { +func (p *PodIssueHandler) handleRunIssue(issue *issue) { if issue == nil || issue.RunIssue == nil { log.Warnf("issue found with missing issue details") return @@ -330,7 +380,7 @@ func (p *IssueHandler) handleRunIssue(issue *issue) { } } -func (p *IssueHandler) handlePodIssue(issue *issue) { +func (p *PodIssueHandler) handlePodIssue(issue *issue) { hasSelfResolved := hasPodIssueSelfResolved(issue) if hasSelfResolved { log.Infof("Issue for job %s run %s has self resolved", issue.RunIssue.JobId, issue.RunIssue.RunId) @@ -350,7 +400,7 @@ func (p *IssueHandler) handlePodIssue(issue *issue) { // - Report JobFailedEvent // // Once that is done we are free to cleanup the pod -func (p *IssueHandler) handleNonRetryableJobIssue(issue *issue) { +func (p *PodIssueHandler) handleNonRetryableJobIssue(issue *issue) { if !issue.RunIssue.Reported { log.Infof("Handling non-retryable issue detected for job %s run %s", issue.RunIssue.JobId, issue.RunIssue.RunId) podIssue := issue.RunIssue.PodIssue @@ -380,13 +430,13 @@ func (p *IssueHandler) handleNonRetryableJobIssue(issue *issue) { // - Report JobReturnLeaseEvent // // If the pod becomes Running/Completed/Failed in the middle of being deleted - swap this issue to a nonRetryableIssue where it will be Failed -func (p *IssueHandler) handleRetryableJobIssue(issue *issue) { +func (p *PodIssueHandler) handleRetryableJobIssue(issue *issue) { log.Infof("Handling retryable issue for job %s run %s", issue.RunIssue.JobId, issue.RunIssue.RunId) if issue.CurrentPodState != nil { - if issue.CurrentPodState.Status.Phase != v1.PodPending { + if issue.RunIssue.PodIssue.OriginalPodState.Status.Phase == v1.PodPending && issue.CurrentPodState.Status.Phase != v1.PodPending { p.markIssuesResolved(issue.RunIssue) if issue.RunIssue.PodIssue.DeletionRequested { - p.registerIssue(&runIssue{ + p.attemptToRegisterIssue(&runIssue{ JobId: issue.RunIssue.JobId, RunId: issue.RunIssue.RunId, PodIssue: &podIssue{ @@ -404,7 +454,7 @@ func (p *IssueHandler) handleRetryableJobIssue(issue *issue) { } err := p.clusterContext.DeletePodWithCondition(issue.CurrentPodState, func(pod *v1.Pod) bool { - return pod.Status.Phase == v1.PodPending + return pod.Status.Phase == issue.RunIssue.PodIssue.OriginalPodState.Status.Phase }, true) if err != nil { log.Errorf("Failed to delete pod of running job %s because %s", issue.RunIssue.JobId, err) @@ -468,16 +518,16 @@ func createStuckPodMessage(retryable bool, originalMessage string) string { return fmt.Sprintf("Unable to start pod - encountered an unrecoverable problem.\n%s", originalMessage) } -func (p *IssueHandler) handleDeletedPod(pod *v1.Pod) { +func (p *PodIssueHandler) handleDeletedPod(pod *v1.Pod) { jobId := util.ExtractJobId(pod) if jobId != "" { isUnexpectedDeletion := !util.IsMarkedForDeletion(pod) && !util.IsPodFinishedAndReported(pod) if isUnexpectedDeletion { - p.registerIssue(&runIssue{ + p.attemptToRegisterIssue(&runIssue{ JobId: jobId, RunId: util.ExtractJobRunId(pod), PodIssue: &podIssue{ - OriginalPodState: pod, + OriginalPodState: pod.DeepCopy(), Message: "Pod was unexpectedly deleted", Retryable: false, Type: ExternallyDeleted, @@ -487,7 +537,7 @@ func (p *IssueHandler) handleDeletedPod(pod *v1.Pod) { } } -func (p *IssueHandler) handleReconciliationIssue(issue *issue) { +func (p *PodIssueHandler) handleReconciliationIssue(issue *issue) { if issue.RunIssue.ReconciliationIssue == nil { log.Warnf("unexpected trying to process an issue as a reconciliation issue for job %s run %s", issue.RunIssue.JobId, issue.RunIssue.RunId) p.markIssuesResolved(issue.RunIssue) @@ -549,7 +599,7 @@ func (p *IssueHandler) handleReconciliationIssue(issue *issue) { } } -func (p *IssueHandler) detectReconciliationIssues(pods []*v1.Pod) { +func (p *PodIssueHandler) detectReconciliationIssues(pods []*v1.Pod) { runs := p.jobRunState.GetAllWithFilter(func(state *job.RunState) bool { return (state.Phase == job.Active || state.Phase == job.SuccessfulSubmission) && !state.CancelRequested && !state.PreemptionRequested }) @@ -565,10 +615,10 @@ func (p *IssueHandler) detectReconciliationIssues(pods []*v1.Pod) { for _, run := range runs { _, present := runIdsToPod[run.Meta.RunId] if !present { - if p.hasIssue(run.Meta.RunId) { + if p.HasIssue(run.Meta.RunId) { continue } - p.registerIssue(&runIssue{ + p.attemptToRegisterIssue(&runIssue{ JobId: run.Meta.JobId, RunId: run.Meta.RunId, ReconciliationIssue: &reconciliationIssue{ diff --git a/internal/executor/service/pod_issue_handler_test.go b/internal/executor/service/pod_issue_handler_test.go index 72057762eec..3cbcc5238b8 100644 --- a/internal/executor/service/pod_issue_handler_test.go +++ b/internal/executor/service/pod_issue_handler_test.go @@ -21,12 +21,15 @@ import ( "github.com/armadaproject/armada/internal/executor/domain" "github.com/armadaproject/armada/internal/executor/job" "github.com/armadaproject/armada/internal/executor/podchecks" + "github.com/armadaproject/armada/internal/executor/podchecks/failedpodchecks" "github.com/armadaproject/armada/internal/executor/reporter" "github.com/armadaproject/armada/internal/executor/reporter/mocks" "github.com/armadaproject/armada/internal/executor/util" "github.com/armadaproject/armada/pkg/armadaevents" ) +const retryableFailedPodStatusMessage = "retryable" + func TestPodIssueService_DoesNothingIfNoPodsAreFound(t *testing.T) { podIssueService, _, _, eventsReporter, err := setupTestComponents([]*job.RunState{}) require.NoError(t, err) @@ -88,6 +91,112 @@ func TestPodIssueService_DeletesPodAndReportsFailed_IfStuckTerminating(t *testin assert.Contains(t, failedEvent.JobRunErrors.Errors[0].GetPodError().Message, "terminating") } +func TestPodIssueService_HasIssue(t *testing.T) { + podIssueService, _, _, _, err := setupTestComponents([]*job.RunState{}) + require.NoError(t, err) + + issue := &runIssue{ + JobId: "abc", + RunId: "def", + } + + added, err := podIssueService.registerIssue(issue) + assert.True(t, added) + assert.NoError(t, err) + + // Empty input + result := podIssueService.HasIssue("") + assert.False(t, result) + + // unknown id + result = podIssueService.HasIssue("unknown") + assert.False(t, result) + + // known id + result = podIssueService.HasIssue(issue.RunId) + assert.True(t, result) + + // after issue resolve + podIssueService.markIssuesResolved(issue) + result = podIssueService.HasIssue(issue.RunId) + assert.False(t, result) +} + +func TestPodIssueService_DetectAndRegisterFailedPodIssue(t *testing.T) { + failedPodWithRetryableIssue := makeTestPod(v1.PodStatus{Phase: v1.PodFailed}) + failedPodWithRetryableIssue.Status.Message = retryableFailedPodStatusMessage + + failedPodWithNonRetryableIssue := makeTestPod(v1.PodStatus{Phase: v1.PodFailed}) + failedPodWithNonRetryableIssue.Status.Message = "non-retryable" + tests := map[string]struct { + pod *v1.Pod + issueAlreadyExists bool + shouldErrorGettingEvents bool + expectIssueAdded bool + expectError bool + }{ + "FailedPodWithIssue": { + pod: failedPodWithRetryableIssue, + expectIssueAdded: true, + expectError: false, + }, + "FailedPodWithoutIssue": { + pod: failedPodWithNonRetryableIssue, + expectIssueAdded: false, + expectError: false, + }, + "FailedPodWithIssue_IssueAlreadyRegistered": { + pod: failedPodWithRetryableIssue, + issueAlreadyExists: true, + expectIssueAdded: false, + expectError: false, + }, + "FailedPodWithIssue_EventErrors": { + pod: failedPodWithRetryableIssue, + shouldErrorGettingEvents: true, + expectIssueAdded: false, + expectError: true, + }, + "UnmanagedPod": { + pod: &v1.Pod{}, + expectIssueAdded: false, + expectError: false, + }, + "RunningPod": { + pod: makeRunningPod(), + expectIssueAdded: false, + expectError: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + podIssueService, _, fakeClusterContext, _, err := setupTestComponents([]*job.RunState{}) + require.NoError(t, err) + if tc.issueAlreadyExists { + added, err := podIssueService.registerIssue(&runIssue{ + JobId: util.ExtractJobId(tc.pod), + RunId: util.ExtractJobRunId(tc.pod), + }) + assert.True(t, added) + assert.NoError(t, err) + } + if tc.shouldErrorGettingEvents { + fakeClusterContext.GetPodEventsErr = fmt.Errorf("failed getting events") + } + + issueAdded, err := podIssueService.DetectAndRegisterFailedPodIssue(tc.pod) + + assert.Equal(t, tc.expectIssueAdded, issueAdded) + if tc.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + func TestPodIssueService_DeletesPodAndReportsFailed_IfExceedsActiveDeadline(t *testing.T) { startTime := time.Now().Add(-time.Minute * 10) @@ -303,22 +412,24 @@ func TestPodIssueService_DoesNothing_IfSubmittedPodAppears(t *testing.T) { assert.Len(t, runStateStore.GetAll(), 1) } -func setupTestComponents(initialRunState []*job.RunState) (*IssueHandler, *job.JobRunStateStore, *fakecontext.SyncFakeClusterContext, *mocks.FakeEventReporter, error) { +func setupTestComponents(initialRunState []*job.RunState) (*PodIssueHandler, *job.JobRunStateStore, *fakecontext.SyncFakeClusterContext, *mocks.FakeEventReporter, error) { fakeClusterContext := fakecontext.NewSyncFakeClusterContext() eventReporter := mocks.NewFakeEventReporter() - pendingPodChecker := makePodChecker() + pendingPodChecker := makePendingPodChecker() + failedPodChecker := makeFailedPodChecker() runStateStore := job.NewJobRunStateStoreWithInitialState(initialRunState) stateChecksConfig := configuration.StateChecksConfiguration{ DeadlineForSubmittedPodConsideredMissing: time.Minute * 15, DeadlineForActivePodConsideredMissing: time.Minute * 5, } - podIssueHandler, err := NewIssueHandler( + podIssueHandler, err := NewPodIssuerHandler( runStateStore, fakeClusterContext, eventReporter, stateChecksConfig, pendingPodChecker, + failedPodChecker, time.Minute*3, ) @@ -420,7 +531,7 @@ func makeTestPod(status v1.PodStatus) *v1.Pod { return pod } -func makePodChecker() podchecks.PodChecker { +func makePendingPodChecker() podchecks.PodChecker { var cfg podchecksConfig.Checks cfg.Events = []podchecksConfig.EventCheck{ {Regexp: "Image pull has failed", Type: "Warning", GracePeriod: time.Nanosecond, Action: podchecksConfig.ActionFail}, @@ -439,6 +550,21 @@ func makePodChecker() podchecks.PodChecker { return checker } +func makeFailedPodChecker() failedpodchecks.RetryChecker { + checker, err := failedpodchecks.NewPodRetryChecker(podchecksConfig.FailedChecks{ + PodStatuses: []podchecksConfig.PodStatusCheck{ + { + Regexp: fmt.Sprintf("^%s$", retryableFailedPodStatusMessage), + }, + }, + }) + if err != nil { + panic(fmt.Sprintf("Failed to make pod checker: %v", err)) + } + + return checker +} + func addPod(t *testing.T, fakeClusterContext context.ClusterContext, runningPod *v1.Pod) { t.Helper() _, err := fakeClusterContext.SubmitPod(runningPod, "owner-1", []string{}) From ef09d415f6b9af408fe62a8c193dcc6409ecadd3 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Wed, 22 Jan 2025 11:41:20 +0000 Subject: [PATCH 05/23] Scheduler Pruner Helm Chart Now Creates ConfigMap (#4160) --- ...er-pruner-secret.yaml => scheduler-pruner-configmap.yaml} | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) rename deployment/scheduler/templates/{scheduler-pruner-secret.yaml => scheduler-pruner-configmap.yaml} (78%) diff --git a/deployment/scheduler/templates/scheduler-pruner-secret.yaml b/deployment/scheduler/templates/scheduler-pruner-configmap.yaml similarity index 78% rename from deployment/scheduler/templates/scheduler-pruner-secret.yaml rename to deployment/scheduler/templates/scheduler-pruner-configmap.yaml index a7783f03e6a..f95f910b2bd 100644 --- a/deployment/scheduler/templates/scheduler-pruner-secret.yaml +++ b/deployment/scheduler/templates/scheduler-pruner-configmap.yaml @@ -1,13 +1,12 @@ apiVersion: v1 -kind: Secret +kind: ConfigMap metadata: name: {{ include "armada-scheduler-pruner.config.name" . }} namespace: {{ .Release.Namespace }} labels: {{- include "armada-scheduler-pruner.labels.all" . | nindent 4 }} -type: Opaque data: {{ include "armada-scheduler-pruner.config.filename" . }}: | {{- if .Values.pruner.applicationConfig }} -{{ toYaml .Values.pruner.applicationConfig | b64enc | indent 4 }} +{{ toYaml .Values.pruner.applicationConfig | indent 4 }} {{- end }} From 15e0f7154e4eec408ecc94f5f9d539ec87baa657 Mon Sep 17 00:00:00 2001 From: Martynas Asipauskas Date: Thu, 23 Jan 2025 09:31:12 +0000 Subject: [PATCH 06/23] Create 019_add_runs_executor_index.sql (#4161) --- .../database/migrations/019_add_runs_executor_index.sql | 1 + 1 file changed, 1 insertion(+) create mode 100644 internal/scheduler/database/migrations/019_add_runs_executor_index.sql diff --git a/internal/scheduler/database/migrations/019_add_runs_executor_index.sql b/internal/scheduler/database/migrations/019_add_runs_executor_index.sql new file mode 100644 index 00000000000..fa10642587d --- /dev/null +++ b/internal/scheduler/database/migrations/019_add_runs_executor_index.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_filtered ON runs (executor, succeeded, failed, cancelled, serial); From 1e3b7daff3fffa59c1dd0354ab2fb1e88b8290d3 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Mon, 27 Jan 2025 12:07:25 +0000 Subject: [PATCH 07/23] Python Client Uses Queue Service Not Submit Service For Queue Ops (#4166) * use queue service not jobs service * fix test --- client/python/armada_client/asyncio_client.py | 13 ++--- client/python/armada_client/client.py | 15 +++--- client/python/tests/unit/server_mock.py | 48 ++++++++++--------- .../python/tests/unit/test_asyncio_client.py | 3 +- client/python/tests/unit/test_client.py | 3 +- 5 files changed, 44 insertions(+), 38 deletions(-) diff --git a/client/python/armada_client/asyncio_client.py b/client/python/armada_client/asyncio_client.py index 8b76379bedf..ad3b4b9257c 100644 --- a/client/python/armada_client/asyncio_client.py +++ b/client/python/armada_client/asyncio_client.py @@ -105,6 +105,7 @@ def __init__( event_timeout: timedelta = timedelta(minutes=15), ) -> None: self.submit_stub = submit_pb2_grpc.SubmitStub(channel) + self.queue_stub = submit_pb2_grpc.QueueServiceStub(channel) self.event_stub = event_pb2_grpc.EventStub(channel) self.job_stub = job_pb2_grpc.JobsStub(channel) self.event_timeout = event_timeout @@ -390,7 +391,7 @@ async def create_queue(self, queue: submit_pb2.Queue) -> empty_pb2.Empty: :param queue: A queue to create. """ - response = await self.submit_stub.CreateQueue(queue) + response = await self.queue_stub.CreateQueue(queue) return response async def update_queue(self, queue: submit_pb2.Queue) -> empty_pb2.Empty: @@ -400,7 +401,7 @@ async def update_queue(self, queue: submit_pb2.Queue) -> empty_pb2.Empty: :param queue: A queue to update. """ - response = await self.submit_stub.UpdateQueue(queue) + response = await self.queue_stub.UpdateQueue(queue) return response async def create_queues( @@ -413,7 +414,7 @@ async def create_queues( """ queue_list = submit_pb2.QueueList(queues=queues) - response = await self.submit_stub.CreateQueues(queue_list) + response = await self.queue_stub.CreateQueues(queue_list) return response async def update_queues( @@ -426,7 +427,7 @@ async def update_queues( """ queue_list = submit_pb2.QueueList(queues=queues) - response = await self.submit_stub.UpdateQueues(queue_list) + response = await self.queue_stub.UpdateQueues(queue_list) return response async def delete_queue(self, name: str) -> None: @@ -438,7 +439,7 @@ async def delete_queue(self, name: str) -> None: :return: None """ request = submit_pb2.QueueDeleteRequest(name=name) - await self.submit_stub.DeleteQueue(request) + await self.queue_stub.DeleteQueue(request) async def get_queue(self, name: str) -> submit_pb2.Queue: """Get the queue by name. @@ -449,7 +450,7 @@ async def get_queue(self, name: str) -> submit_pb2.Queue: :return: A queue object. See the api definition. """ request = submit_pb2.QueueGetRequest(name=name) - response = await self.submit_stub.GetQueue(request) + response = await self.queue_stub.GetQueue(request) return response @staticmethod diff --git a/client/python/armada_client/client.py b/client/python/armada_client/client.py index 6cc96f81319..004852b0486 100644 --- a/client/python/armada_client/client.py +++ b/client/python/armada_client/client.py @@ -103,8 +103,9 @@ class ArmadaClient: def __init__(self, channel, event_timeout: timedelta = timedelta(minutes=15)): self.submit_stub = submit_pb2_grpc.SubmitStub(channel) self.event_stub = event_pb2_grpc.EventStub(channel) - self.event_timeout = event_timeout self.job_stub = job_pb2_grpc.JobsStub(channel) + self.queue_stub = submit_pb2_grpc.QueueServiceStub(channel) + self.event_timeout = event_timeout def get_job_events_stream( self, @@ -378,7 +379,7 @@ def create_queue(self, queue: submit_pb2.Queue) -> empty_pb2.Empty: :param queue: A queue to create. """ - response = self.submit_stub.CreateQueue(queue) + response = self.queue_stub.CreateQueue(queue) return response def update_queue(self, queue: submit_pb2.Queue) -> empty_pb2.Empty: @@ -388,7 +389,7 @@ def update_queue(self, queue: submit_pb2.Queue) -> empty_pb2.Empty: :param queue: A queue to update. """ - response = self.submit_stub.UpdateQueue(queue) + response = self.queue_stub.UpdateQueue(queue) return response def create_queues( @@ -401,7 +402,7 @@ def create_queues( """ queue_list = submit_pb2.QueueList(queues=queues) - response = self.submit_stub.CreateQueues(queue_list) + response = self.queue_stub.CreateQueues(queue_list) return response def update_queues( @@ -414,7 +415,7 @@ def update_queues( """ queue_list = submit_pb2.QueueList(queues=queues) - response = self.submit_stub.UpdateQueues(queue_list) + response = self.queue_stub.UpdateQueues(queue_list) return response def delete_queue(self, name: str) -> None: @@ -426,7 +427,7 @@ def delete_queue(self, name: str) -> None: :return: None """ request = submit_pb2.QueueDeleteRequest(name=name) - self.submit_stub.DeleteQueue(request) + self.queue_stub.DeleteQueue(request) def get_queue(self, name: str) -> submit_pb2.Queue: """Get the queue by name. @@ -437,7 +438,7 @@ def get_queue(self, name: str) -> submit_pb2.Queue: :return: A queue object. See the api definition. """ request = submit_pb2.QueueGetRequest(name=name) - response = self.submit_stub.GetQueue(request) + response = self.queue_stub.GetQueue(request) return response @staticmethod diff --git a/client/python/tests/unit/server_mock.py b/client/python/tests/unit/server_mock.py index 1bb94bcf053..b3b2731a47c 100644 --- a/client/python/tests/unit/server_mock.py +++ b/client/python/tests/unit/server_mock.py @@ -15,7 +15,7 @@ from armada_client.armada.submit_pb2 import JobState -class SubmitService(submit_pb2_grpc.SubmitServicer): +class QueueService(submit_pb2_grpc.QueueServiceServicer): def CreateQueue(self, request, context): return empty_pb2.Empty() @@ -25,6 +25,30 @@ def DeleteQueue(self, request, context): def GetQueue(self, request, context): return submit_pb2.Queue(name=request.name) + def GetQueueInfo(self, request, context): + return submit_pb2.QueueInfo(name=request.name) + + def CreateQueues(self, request, context): + return submit_pb2.BatchQueueCreateResponse( + failed_queues=[ + submit_pb2.QueueCreateResponse(queue=submit_pb2.Queue(name=queue.name)) + for queue in request.queues + ] + ) + + def UpdateQueues(self, request, context): + return submit_pb2.BatchQueueUpdateResponse( + failed_queues=[ + submit_pb2.QueueUpdateResponse(queue=submit_pb2.Queue(name=queue.name)) + for queue in request.queues + ] + ) + + def UpdateQueue(self, request, context): + return empty_pb2.Empty() + + +class SubmitService(submit_pb2_grpc.SubmitServicer): def SubmitJobs(self, request, context): # read job_ids from request.job_request_items job_ids = [f"job-{i}" for i in range(1, len(request.job_request_items) + 1)] @@ -35,9 +59,6 @@ def SubmitJobs(self, request, context): return submit_pb2.JobSubmitResponse(job_response_items=job_response_items) - def GetQueueInfo(self, request, context): - return submit_pb2.QueueInfo(name=request.name) - def CancelJobs(self, request, context): return submit_pb2.CancellationResult( cancelled_ids=["job-1"], @@ -72,25 +93,6 @@ def ReprioritizeJobs(self, request, context): return submit_pb2.JobReprioritizeResponse(reprioritization_results=results) - def UpdateQueue(self, request, context): - return empty_pb2.Empty() - - def CreateQueues(self, request, context): - return submit_pb2.BatchQueueCreateResponse( - failed_queues=[ - submit_pb2.QueueCreateResponse(queue=submit_pb2.Queue(name=queue.name)) - for queue in request.queues - ] - ) - - def UpdateQueues(self, request, context): - return submit_pb2.BatchQueueUpdateResponse( - failed_queues=[ - submit_pb2.QueueUpdateResponse(queue=submit_pb2.Queue(name=queue.name)) - for queue in request.queues - ] - ) - def Health(self, request, context): return health_pb2.HealthCheckResponse( status=health_pb2.HealthCheckResponse.SERVING diff --git a/client/python/tests/unit/test_asyncio_client.py b/client/python/tests/unit/test_asyncio_client.py index 6f4d8709c23..7d28f54c0eb 100644 --- a/client/python/tests/unit/test_asyncio_client.py +++ b/client/python/tests/unit/test_asyncio_client.py @@ -6,7 +6,7 @@ from armada_client.typings import JobState from armada_client.armada.job_pb2 import JobRunState -from server_mock import EventService, SubmitService, QueryAPIService +from server_mock import EventService, SubmitService, QueueService, QueryAPIService from armada_client.armada import ( event_pb2_grpc, @@ -28,6 +28,7 @@ def server_mock(): server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) submit_pb2_grpc.add_SubmitServicer_to_server(SubmitService(), server) + submit_pb2_grpc.add_QueueServiceServicer_to_server(QueueService(), server) event_pb2_grpc.add_EventServicer_to_server(EventService(), server) job_pb2_grpc.add_JobsServicer_to_server(QueryAPIService(), server) server.add_insecure_port("[::]:50051") diff --git a/client/python/tests/unit/test_client.py b/client/python/tests/unit/test_client.py index 70eba72439b..2f1a252bb78 100644 --- a/client/python/tests/unit/test_client.py +++ b/client/python/tests/unit/test_client.py @@ -5,7 +5,7 @@ from armada_client.typings import JobState from armada_client.armada.job_pb2 import JobRunState -from server_mock import EventService, SubmitService, QueryAPIService +from server_mock import EventService, SubmitService, QueryAPIService, QueueService from armada_client.armada import ( event_pb2_grpc, @@ -27,6 +27,7 @@ def server_mock(): server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) submit_pb2_grpc.add_SubmitServicer_to_server(SubmitService(), server) + submit_pb2_grpc.add_QueueServiceServicer_to_server(QueueService(), server) event_pb2_grpc.add_EventServicer_to_server(EventService(), server) job_pb2_grpc.add_JobsServicer_to_server(QueryAPIService(), server) server.add_insecure_port("[::]:50051") From 5b34935735cab3c8fe28a849359205fa76ba5931 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Mon, 27 Jan 2025 12:19:45 +0000 Subject: [PATCH 08/23] Add A GetQueues Function To The Python Client (#4167) --- client/python/armada_client/asyncio_client.py | 14 ++++++++++++++ client/python/armada_client/client.py | 19 +++++++++++++++++++ client/python/tests/unit/server_mock.py | 10 ++++++++++ .../python/tests/unit/test_asyncio_client.py | 7 +++++++ client/python/tests/unit/test_client.py | 6 ++++++ docs/python_armada_client.md | 18 ++++++++++++++++++ 6 files changed, 74 insertions(+) diff --git a/client/python/armada_client/asyncio_client.py b/client/python/armada_client/asyncio_client.py index ad3b4b9257c..7f94676d78d 100644 --- a/client/python/armada_client/asyncio_client.py +++ b/client/python/armada_client/asyncio_client.py @@ -453,6 +453,20 @@ async def get_queue(self, name: str) -> submit_pb2.Queue: response = await self.queue_stub.GetQueue(request) return response + async def get_queues(self) -> list: + """Retrieves all queues + :return: List containing all queues. + """ + queues = [] + request = submit_pb2.StreamingQueueGetRequest() + async for message in self.queue_stub.GetQueues(request): + event_type = message.WhichOneof("event") + if event_type == "queue": + queues.append(message.queue) + elif event_type == "end": + break + return queues + @staticmethod def unwatch_events(event_stream) -> None: """Closes gRPC event streams diff --git a/client/python/armada_client/client.py b/client/python/armada_client/client.py index 004852b0486..c21fa7d7b7b 100644 --- a/client/python/armada_client/client.py +++ b/client/python/armada_client/client.py @@ -291,6 +291,7 @@ def cancel_jobset( ) -> empty_pb2.Empty: """Cancel jobs in a given queue. + Uses the CancelJobSet RPC to cancel jobs. A filter is used to only cancel jobs in certain states. @@ -441,6 +442,24 @@ def get_queue(self, name: str) -> submit_pb2.Queue: response = self.queue_stub.GetQueue(request) return response + def get_queues(self) -> List[submit_pb2.Queue]: + """Get all queues. + + Uses the GetQueues RPC to get the queues. + + :return: list containing all queues + """ + queues = [] + + request = submit_pb2.StreamingQueueGetRequest() + + for message in self.queue_stub.GetQueues(request): + if message.HasField("queue"): + queues.append(message.queue) + elif message.HasField("end"): + break + return queues + @staticmethod def unwatch_events(event_stream) -> None: """Closes gRPC event streams diff --git a/client/python/tests/unit/server_mock.py b/client/python/tests/unit/server_mock.py index b3b2731a47c..83434dd2439 100644 --- a/client/python/tests/unit/server_mock.py +++ b/client/python/tests/unit/server_mock.py @@ -25,6 +25,16 @@ def DeleteQueue(self, request, context): def GetQueue(self, request, context): return submit_pb2.Queue(name=request.name) + def GetQueues(self, request, context): + queue_names = ["test_queue1", "test_queue2", "test_queue3"] + for name in queue_names: + queue_message = submit_pb2.StreamingQueueMessage( + queue=submit_pb2.Queue(name=name) + ) + yield queue_message + + yield submit_pb2.StreamingQueueMessage(end=submit_pb2.EndMarker()) + def GetQueueInfo(self, request, context): return submit_pb2.QueueInfo(name=request.name) diff --git a/client/python/tests/unit/test_asyncio_client.py b/client/python/tests/unit/test_asyncio_client.py index 7d28f54c0eb..7d847fc5eea 100644 --- a/client/python/tests/unit/test_asyncio_client.py +++ b/client/python/tests/unit/test_asyncio_client.py @@ -176,6 +176,13 @@ async def test_get_queue(aio_client): assert queue.name == "test" +@pytest.mark.asyncio +async def test_get_queues(aio_client): + queues = await aio_client.get_queues() + queue_names = [q.name for q in queues] + assert queue_names == ["test_queue1", "test_queue2", "test_queue3"] + + @pytest.mark.asyncio async def test_delete_queue(aio_client): await aio_client.delete_queue("test") diff --git a/client/python/tests/unit/test_client.py b/client/python/tests/unit/test_client.py index 2f1a252bb78..5bc3785db68 100644 --- a/client/python/tests/unit/test_client.py +++ b/client/python/tests/unit/test_client.py @@ -166,6 +166,12 @@ def test_get_queue(): assert tester.get_queue("test").name == "test" +def test_get_queues(): + queues = tester.get_queues() + queue_names = [q.name for q in queues] + assert queue_names == ["test_queue1", "test_queue2", "test_queue3"] + + def test_delete_queue(): tester.delete_queue("test") diff --git a/docs/python_armada_client.md b/docs/python_armada_client.md index 44cc6ff9abd..602efb51315 100644 --- a/docs/python_armada_client.md +++ b/docs/python_armada_client.md @@ -447,6 +447,24 @@ Uses the GetQueue RPC to get the queue. +#### get_queues() +Get all queues. + +Uses the GetQueues RPC to get the queues. + + +* **Returns** + + list containing all queues + + + +* **Return type** + + *List*[armada.submit_pb2.Queue] + + + #### preempt_jobs(queue, job_set_id, job_id) Preempt jobs in a given queue. From 1d93b7a31b6c9cbb9ae6906719f5616f4c7ba72d Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Mon, 27 Jan 2025 15:22:02 +0000 Subject: [PATCH 09/23] Python CLient v0.4.11 (#4169) --- client/python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/python/pyproject.toml b/client/python/pyproject.toml index 2d3b0783062..cae3ef7f0ec 100644 --- a/client/python/pyproject.toml +++ b/client/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "armada_client" -version = "0.4.10" +version = "0.4.11" description = "Armada gRPC API python client" readme = "README.md" requires-python = ">=3.9" From 1ee719605e05e1397d33900f09b90167230627c0 Mon Sep 17 00:00:00 2001 From: Rob Smith <34475852+robertdavidsmith@users.noreply.github.com> Date: Mon, 27 Jan 2025 17:40:04 +0000 Subject: [PATCH 10/23] Scheduler: floating resources no longer experimental (#4159) Signed-off-by: Robert Smith --- docs/floating_resources.md | 27 +++++++++++++++++++ .../scheduler/configuration/configuration.go | 2 +- internal/scheduler/schedulerapp.go | 4 +-- .../scheduling/gang_scheduler_test.go | 4 +-- internal/scheduler/simulator/simulator.go | 4 +-- 5 files changed, 34 insertions(+), 7 deletions(-) create mode 100644 docs/floating_resources.md diff --git a/docs/floating_resources.md b/docs/floating_resources.md new file mode 100644 index 00000000000..c13a99ee3cb --- /dev/null +++ b/docs/floating_resources.md @@ -0,0 +1,27 @@ +# Floating Resources + +Floating resources are designed to constrain the usage of resources that are not tied to nodes. For example, if you have a fileserver outside your Kubernetes clusters, you may want to limit how many connections to the fileserver can exist at once. In that case you would add config like the below (this goes under the `scheduling` section of the Armada scheduler config). + +``` + floatingResources: + - name: fileserver-connections + resolution: "1" + pools: + - name: cpu + quantity: 1000 + - name: gpu + quantity: 500 +``` +When submitting a job, floating resources are specified in the same way as normal Kubernetes resources such as `cpu`. For example if a job needs 3 cpu cores and opens 10 connections to the fileserver, the job should specify +``` + resources: + requests: + cpu: "3" + fileserver-connections: "10" + limits: + cpu: "3" + fileserver-connections: "10" +``` +The `requests` section is used for scheduling. For floating resources, the `limits` section is not enforced by Armada (this it not possible in the general case). Instead the workload must be trusted to respect its limit. + +If the jobs submitted to Armada request more of a floating resource than is available, they queue just as if they had exceeded the amount available of a standard Kubernetes resource (e.g. `cpu`). Floating resources generally behave like standard Kubernetes resources. They use the same code for queue ordering, pre-emption, etc. diff --git a/internal/scheduler/configuration/configuration.go b/internal/scheduler/configuration/configuration.go index c1866157ede..ff3a71bd447 100644 --- a/internal/scheduler/configuration/configuration.go +++ b/internal/scheduler/configuration/configuration.go @@ -224,7 +224,7 @@ type SchedulingConfig struct { // These can be requested like a normal k8s resource. Note there is no mechanism in armada // to enforce actual usage, it relies on honesty. For example, there is nothing to stop a badly-behaved job // requesting 2 S3 server connections and then opening 10. - ExperimentalFloatingResources []FloatingResourceConfig + FloatingResources []FloatingResourceConfig // WellKnownNodeTypes defines a set of well-known node types used to define "home" and "away" nodes for a given priority class. WellKnownNodeTypes []WellKnownNodeType `validate:"dive"` // Executor that haven't heartbeated in this time period are considered stale. diff --git a/internal/scheduler/schedulerapp.go b/internal/scheduler/schedulerapp.go index 6c21f07cf39..4bd15243fc3 100644 --- a/internal/scheduler/schedulerapp.go +++ b/internal/scheduler/schedulerapp.go @@ -77,14 +77,14 @@ func Run(config schedulerconfig.Configuration) error { // //////////////////////////////////////////////////////////////////////// resourceListFactory, err := internaltypes.NewResourceListFactory( config.Scheduling.SupportedResourceTypes, - config.Scheduling.ExperimentalFloatingResources, + config.Scheduling.FloatingResources, ) if err != nil { return errors.WithMessage(err, "Error with the .scheduling.supportedResourceTypes field in config") } ctx.Infof("Supported resource types: %s", resourceListFactory.SummaryString()) - floatingResourceTypes, err := floatingresources.NewFloatingResourceTypes(config.Scheduling.ExperimentalFloatingResources, resourceListFactory) + floatingResourceTypes, err := floatingresources.NewFloatingResourceTypes(config.Scheduling.FloatingResources, resourceListFactory) if err != nil { return err } diff --git a/internal/scheduler/scheduling/gang_scheduler_test.go b/internal/scheduler/scheduling/gang_scheduler_test.go index 53b5c9121af..2e9f5989dc9 100644 --- a/internal/scheduler/scheduling/gang_scheduler_test.go +++ b/internal/scheduler/scheduling/gang_scheduler_test.go @@ -100,7 +100,7 @@ func TestGangScheduler(t *testing.T) { "floating resources": { SchedulingConfig: func() configuration.SchedulingConfig { cfg := testfixtures.TestSchedulingConfig() - cfg.ExperimentalFloatingResources = testfixtures.TestFloatingResourceConfig + cfg.FloatingResources = testfixtures.TestFloatingResourceConfig return cfg }(), Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), @@ -634,7 +634,7 @@ func TestGangScheduler(t *testing.T) { func(qn string) *api.Queue { return &api.Queue{Name: qn} }, ), ) - floatingResourceTypes, err := floatingresources.NewFloatingResourceTypes(tc.SchedulingConfig.ExperimentalFloatingResources, testfixtures.TestResourceListFactory) + floatingResourceTypes, err := floatingresources.NewFloatingResourceTypes(tc.SchedulingConfig.FloatingResources, testfixtures.TestResourceListFactory) require.NoError(t, err) sch, err := NewGangScheduler(sctx, constraints, floatingResourceTypes, nodeDb, false) require.NoError(t, err) diff --git a/internal/scheduler/simulator/simulator.go b/internal/scheduler/simulator/simulator.go index 88755abf438..a0ca6c2e034 100644 --- a/internal/scheduler/simulator/simulator.go +++ b/internal/scheduler/simulator/simulator.go @@ -126,13 +126,13 @@ func NewSimulator( ) (*Simulator, error) { resourceListFactory, err := internaltypes.NewResourceListFactory( schedulingConfig.SupportedResourceTypes, - schedulingConfig.ExperimentalFloatingResources, + schedulingConfig.FloatingResources, ) if err != nil { return nil, errors.WithMessage(err, "Error with the .scheduling.supportedResourceTypes field in config") } - floatingResourceTypes, err := floatingresources.NewFloatingResourceTypes(schedulingConfig.ExperimentalFloatingResources, resourceListFactory) + floatingResourceTypes, err := floatingresources.NewFloatingResourceTypes(schedulingConfig.FloatingResources, resourceListFactory) if err != nil { return nil, err } From 6cfde089a00c4192e701ec8e8bce77691b8690ce Mon Sep 17 00:00:00 2001 From: Rob Smith <34475852+robertdavidsmith@users.noreply.github.com> Date: Mon, 27 Jan 2025 18:04:39 +0000 Subject: [PATCH 11/23] Scheduler: refactor nodedb stats (#4165) Signed-off-by: Robert Smith --- internal/scheduler/nodedb/nodedb.go | 18 +++--------------- .../scheduler/nodedb/nodeiteration_test.go | 14 ++++++-------- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/internal/scheduler/nodedb/nodedb.go b/internal/scheduler/nodedb/nodedb.go index ba74a66fc8d..c8159253c17 100644 --- a/internal/scheduler/nodedb/nodedb.go +++ b/internal/scheduler/nodedb/nodedb.go @@ -4,8 +4,6 @@ import ( "fmt" "math" "strings" - "sync" - "sync/atomic" "text/tabwriter" "time" @@ -27,9 +25,8 @@ import ( var empty struct{} -func (nodeDb *NodeDb) AddNodeToDb(node *internaltypes.Node) { - nodeDb.mu.Lock() - defer nodeDb.mu.Unlock() +func (nodeDb *NodeDb) addNodeToStats(node *internaltypes.Node) { + nodeDb.numNodes += 1 for key := range nodeDb.indexedNodeLabels { if value, ok := node.GetLabelValue(key); ok { nodeDb.indexedNodeLabelValues[key][value] = empty @@ -42,9 +39,7 @@ func (nodeDb *NodeDb) AddNodeToDb(node *internaltypes.Node) { } func (nodeDb *NodeDb) CreateAndInsertWithJobDbJobsWithTxn(txn *memdb.Txn, jobs []*jobdb.Job, entry *internaltypes.Node) error { - _ = atomic.AddUint64(&nodeDb.numNodes, 1) - - nodeDb.AddNodeToDb(entry) + nodeDb.addNodeToStats(entry) for _, job := range jobs { priority, ok := job.ScheduledAtPriority() @@ -122,9 +117,6 @@ type NodeDb struct { // If not set, no labels are indexed. indexedNodeLabels map[string]bool - // Mutex for the remaining fields of this struct, which are mutated after initialization. - mu sync.Mutex - // Map from indexed label names to the set of values that label takes across all nodes in the NodeDb. indexedNodeLabelValues map[string]map[string]struct{} // Total number of nodes in the db. @@ -305,8 +297,6 @@ func (nodeDb *NodeDb) NumNodes() int { } func (nodeDb *NodeDb) TotalKubernetesResources() internaltypes.ResourceList { - nodeDb.mu.Lock() - defer nodeDb.mu.Unlock() return nodeDb.totalResources } @@ -1126,8 +1116,6 @@ func nodeIndexName(keyIndex int) string { // using a cache to avoid allocating new strings when possible. func (nodeDb *NodeDb) stringFromPodRequirementsNotMetReason(reason PodRequirementsNotMetReason) string { h := reason.Sum64() - nodeDb.mu.Lock() - defer nodeDb.mu.Unlock() if s, ok := nodeDb.podRequirementsNotMetReasonStringCache[h]; ok { return s } else { diff --git a/internal/scheduler/nodedb/nodeiteration_test.go b/internal/scheduler/nodedb/nodeiteration_test.go index f70b1dce23a..b0f1f5063af 100644 --- a/internal/scheduler/nodedb/nodeiteration_test.go +++ b/internal/scheduler/nodedb/nodeiteration_test.go @@ -312,15 +312,15 @@ func TestNodeTypeIterator(t *testing.T) { require.NoError(t, err) entries := make([]*internaltypes.Node, len(tc.nodes)) + txn := nodeDb.Txn(true) for i, node := range tc.nodes { // Set monotonically increasing node IDs to ensure nodes appear in predictable order. newNodeId := fmt.Sprintf("%d", i) entry := testfixtures.WithIdNodes(newNodeId, []*internaltypes.Node{node})[0] - - nodeDb.AddNodeToDb(entry) + require.NoError(t, nodeDb.CreateAndInsertWithJobDbJobsWithTxn(txn, nil, entry)) entries[i] = entry } - require.NoError(t, nodeDb.UpsertMany(entries)) + txn.Commit() indexedResourceRequests := make([]int64, len(testfixtures.TestResources)) @@ -639,20 +639,18 @@ func TestNodeTypesIterator(t *testing.T) { nodeDb, err := newNodeDbWithNodes(nil) require.NoError(t, err) + txn := nodeDb.Txn(true) entries := make([]*internaltypes.Node, len(tc.nodes)) for i, node := range tc.nodes { // Set monotonically increasing node IDs to ensure nodes appear in predictable order. nodeId := fmt.Sprintf("%d", i) entry := testfixtures.WithIdNodes(nodeId, []*internaltypes.Node{node})[0] entry = testfixtures.WithIndexNode(uint64(i), entry) - require.NoError(t, err) - - nodeDb.AddNodeToDb(entry) - + require.NoError(t, nodeDb.CreateAndInsertWithJobDbJobsWithTxn(txn, nil, entry)) entries[i] = entry } - require.NoError(t, nodeDb.UpsertMany(entries)) + txn.Commit() rr := tc.resourceRequests From d43a9d98650cf2e9d95fe99cee115d045bead6aa Mon Sep 17 00:00:00 2001 From: Maurice Yap Date: Wed, 29 Jan 2025 09:33:54 +0000 Subject: [PATCH 12/23] Authenticate requests for scheduling reports (#4170) On the Lookout UI, the fetch requests for scheduling reports require authentication via the user's access token. --- .../lookoutV2/useGetJobSchedulingReport.ts | 16 ++++++++++++++-- .../lookoutV2/useGetQueueSchedulingReport.ts | 16 ++++++++++++++-- .../services/lookoutV2/useGetSchedulingReport.ts | 14 +++++++++++++- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/internal/lookout/ui/src/services/lookoutV2/useGetJobSchedulingReport.ts b/internal/lookout/ui/src/services/lookoutV2/useGetJobSchedulingReport.ts index f1de0ff73d3..f295ceef064 100644 --- a/internal/lookout/ui/src/services/lookoutV2/useGetJobSchedulingReport.ts +++ b/internal/lookout/ui/src/services/lookoutV2/useGetJobSchedulingReport.ts @@ -1,21 +1,33 @@ import { useQuery } from "@tanstack/react-query" import { useGetUiConfig } from "./useGetUiConfig" +import { getAccessToken, getAuthorizationHeaders, useUserManager } from "../../oidc" import { SchedulerReportingApi, Configuration, SchedulerobjectsJobReport } from "../../openapi/schedulerobjects" import { getErrorMessage } from "../../utils" export const useGetJobSchedulingReport = (jobId: string, enabled = true) => { + const userManager = useUserManager() + const { data: uiConfig } = useGetUiConfig(enabled) const armadaApiBaseUrl = uiConfig?.armadaApiBaseUrl - const schedulerReportingApiConfiguration: Configuration = new Configuration({ basePath: armadaApiBaseUrl }) + const schedulerReportingApiConfiguration: Configuration = new Configuration({ + basePath: armadaApiBaseUrl, + credentials: "include", + }) const schedulerReportingApi = new SchedulerReportingApi(schedulerReportingApiConfiguration) return useQuery({ queryKey: ["getJobSchedulingReport", jobId], queryFn: async ({ signal }) => { try { - return await schedulerReportingApi.getJobReport({ jobId }, { signal }) + const headers: HeadersInit = {} + + if (userManager !== undefined) { + Object.assign(headers, getAuthorizationHeaders(await getAccessToken(userManager))) + } + + return await schedulerReportingApi.getJobReport({ jobId }, { headers, signal }) } catch (e) { throw await getErrorMessage(e) } diff --git a/internal/lookout/ui/src/services/lookoutV2/useGetQueueSchedulingReport.ts b/internal/lookout/ui/src/services/lookoutV2/useGetQueueSchedulingReport.ts index 3612c03189e..664da0cda13 100644 --- a/internal/lookout/ui/src/services/lookoutV2/useGetQueueSchedulingReport.ts +++ b/internal/lookout/ui/src/services/lookoutV2/useGetQueueSchedulingReport.ts @@ -1,21 +1,33 @@ import { useQuery } from "@tanstack/react-query" import { useGetUiConfig } from "./useGetUiConfig" +import { getAccessToken, getAuthorizationHeaders, useUserManager } from "../../oidc" import { SchedulerReportingApi, Configuration, SchedulerobjectsQueueReport } from "../../openapi/schedulerobjects" import { getErrorMessage } from "../../utils" export const useGetQueueSchedulingReport = (queueName: string, verbosity: number, enabled = true) => { + const userManager = useUserManager() + const { data: uiConfig } = useGetUiConfig(enabled) const armadaApiBaseUrl = uiConfig?.armadaApiBaseUrl - const schedulerReportingApiConfiguration: Configuration = new Configuration({ basePath: armadaApiBaseUrl }) + const schedulerReportingApiConfiguration: Configuration = new Configuration({ + basePath: armadaApiBaseUrl, + credentials: "include", + }) const schedulerReportingApi = new SchedulerReportingApi(schedulerReportingApiConfiguration) return useQuery({ queryKey: ["getQueueSchedulingReport", queueName, verbosity], queryFn: async ({ signal }) => { try { - return await schedulerReportingApi.getQueueReport({ queueName, verbosity }, { signal }) + const headers: HeadersInit = {} + + if (userManager !== undefined) { + Object.assign(headers, getAuthorizationHeaders(await getAccessToken(userManager))) + } + + return await schedulerReportingApi.getQueueReport({ queueName, verbosity }, { headers, signal }) } catch (e) { throw await getErrorMessage(e) } diff --git a/internal/lookout/ui/src/services/lookoutV2/useGetSchedulingReport.ts b/internal/lookout/ui/src/services/lookoutV2/useGetSchedulingReport.ts index c9c46caa868..1f171748d36 100644 --- a/internal/lookout/ui/src/services/lookoutV2/useGetSchedulingReport.ts +++ b/internal/lookout/ui/src/services/lookoutV2/useGetSchedulingReport.ts @@ -1,20 +1,32 @@ import { useQuery } from "@tanstack/react-query" import { useGetUiConfig } from "./useGetUiConfig" +import { getAccessToken, getAuthorizationHeaders, useUserManager } from "../../oidc" import { SchedulerReportingApi, Configuration, SchedulerobjectsSchedulingReport } from "../../openapi/schedulerobjects" import { getErrorMessage } from "../../utils" export const useGetSchedulingReport = (verbosity: number, enabled = true) => { + const userManager = useUserManager() + const { data: uiConfig } = useGetUiConfig(enabled) const armadaApiBaseUrl = uiConfig?.armadaApiBaseUrl - const schedulerReportingApiConfiguration: Configuration = new Configuration({ basePath: armadaApiBaseUrl }) + const schedulerReportingApiConfiguration: Configuration = new Configuration({ + basePath: armadaApiBaseUrl, + credentials: "include", + }) const schedulerReportingApi = new SchedulerReportingApi(schedulerReportingApiConfiguration) return useQuery({ queryKey: ["getSchedulingReport", verbosity], queryFn: async ({ signal }) => { try { + const headers: HeadersInit = {} + + if (userManager !== undefined) { + Object.assign(headers, getAuthorizationHeaders(await getAccessToken(userManager))) + } + return await schedulerReportingApi.getSchedulingReport({ verbosity }, { signal }) } catch (e) { throw await getErrorMessage(e) From a5afc8af70b7c82fa4bd6ff2bfa638d4424698b9 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Wed, 29 Jan 2025 10:04:20 +0000 Subject: [PATCH 13/23] Fix grpc prometheus metrics (#4172) * fix grpc prom metrics * fix test --------- Co-authored-by: JamesMurkin --- go.mod | 1 - go.sum | 2 -- internal/binoculars/server.go | 3 --- internal/common/grpc/grpc.go | 5 ++--- internal/executor/application.go | 11 +++++++---- internal/scheduler/leader/leader_client.go | 15 ++++++++------- internal/scheduler/leader/leader_client_test.go | 9 +++++---- internal/scheduler/schedulerapp.go | 8 +++++++- internal/server/server.go | 12 +++++++----- 9 files changed, 36 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index f1b9d7a05de..7b83f9caaeb 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/golang/protobuf v1.5.4 github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/hashicorp/go-memdb v1.3.4 github.com/hashicorp/go-multierror v1.1.1 diff --git a/go.sum b/go.sum index 789178833e1..f07df3fa1ae 100644 --- a/go.sum +++ b/go.sum @@ -300,8 +300,6 @@ github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpS github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.2.0 h1:kQ0NI7W1B3HwiN5gAYtY+XFItDPbLBwYRxAqbFTyDes= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.2.0/go.mod h1:zrT2dxOAjNFPRGjTUe2Xmb4q4YdUwVvQFV6xiCSf+z0= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= diff --git a/internal/binoculars/server.go b/internal/binoculars/server.go index d1d604f7c67..643333a457e 100644 --- a/internal/binoculars/server.go +++ b/internal/binoculars/server.go @@ -4,8 +4,6 @@ import ( "os" "sync" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" - "github.com/armadaproject/armada/internal/binoculars/configuration" "github.com/armadaproject/armada/internal/binoculars/server" "github.com/armadaproject/armada/internal/binoculars/service" @@ -48,7 +46,6 @@ func StartUp(config *configuration.BinocularsConfig) (func(), *sync.WaitGroup) { cordonService := service.NewKubernetesCordonService(config.Cordon, permissionsChecker, kubernetesClientProvider) binocularsServer := server.NewBinocularsServer(logService, cordonService) binoculars.RegisterBinocularsServer(grpcServer, binocularsServer) - grpc_prometheus.Register(grpcServer) grpcCommon.Listen(config.GrpcPort, grpcServer, wg) diff --git a/internal/common/grpc/grpc.go b/internal/common/grpc/grpc.go index 9cc844a3152..439242810f3 100644 --- a/internal/common/grpc/grpc.go +++ b/internal/common/grpc/grpc.go @@ -13,8 +13,8 @@ import ( grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/auth" grpc_logging "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery" - "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -76,8 +76,7 @@ func setupPromMetrics() *grpc_prometheus.ServerMetrics { grpc_prometheus.WithHistogramBuckets([]float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}), ), ) - reg := prometheus.NewRegistry() - reg.MustRegister(srvMetrics) + prometheus.MustRegister(srvMetrics) return srvMetrics } diff --git a/internal/executor/application.go b/internal/executor/application.go index 909f4e65da3..29b839a4da3 100644 --- a/internal/executor/application.go +++ b/internal/executor/application.go @@ -9,7 +9,7 @@ import ( "time" "github.com/go-playground/validator/v10" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" "github.com/prometheus/client_golang/prometheus" "google.golang.org/grpc" "google.golang.org/grpc/keepalive" @@ -284,12 +284,15 @@ func setupExecutorApiComponents( } func createConnectionToApi(connectionDetails client.ApiConnectionDetails, maxMessageSizeBytes int, grpcConfig keepalive.ClientParameters) (*grpc.ClientConn, error) { - grpc_prometheus.EnableClientHandlingTimeHistogram() + clientMetrics := grpc_prometheus.NewClientMetrics( + grpc_prometheus.WithClientHandlingTimeHistogram(), + ) + prometheus.MustRegister(clientMetrics) return client.CreateApiConnectionWithCallOptions( &connectionDetails, []grpc.CallOption{grpc.MaxCallRecvMsgSize(maxMessageSizeBytes)}, - grpc.WithChainUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor), - grpc.WithChainStreamInterceptor(grpc_prometheus.StreamClientInterceptor), + grpc.WithChainUnaryInterceptor(clientMetrics.UnaryClientInterceptor()), + grpc.WithChainStreamInterceptor(clientMetrics.StreamClientInterceptor()), grpc.WithKeepaliveParams(grpcConfig), ) } diff --git a/internal/scheduler/leader/leader_client.go b/internal/scheduler/leader/leader_client.go index 8f39b49c15c..1e7436b7377 100644 --- a/internal/scheduler/leader/leader_client.go +++ b/internal/scheduler/leader/leader_client.go @@ -5,7 +5,7 @@ import ( "strings" "sync" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" "github.com/pkg/errors" "google.golang.org/grpc" @@ -24,14 +24,16 @@ type LeaderConnectionProvider struct { leaderConfig configuration.LeaderConfig connectionLock sync.Mutex connectionByName map[string]*grpc.ClientConn + metrics *grpc_prometheus.ClientMetrics } -func NewLeaderConnectionProvider(leaderController LeaderController, leaderConfig configuration.LeaderConfig) *LeaderConnectionProvider { +func NewLeaderConnectionProvider(leaderController LeaderController, leaderConfig configuration.LeaderConfig, metrics *grpc_prometheus.ClientMetrics) *LeaderConnectionProvider { return &LeaderConnectionProvider{ leaderController: leaderController, leaderConfig: leaderConfig, connectionLock: sync.Mutex{}, connectionByName: map[string]*grpc.ClientConn{}, + metrics: metrics, } } @@ -60,7 +62,7 @@ func (l *LeaderConnectionProvider) getClientByName(currentLeaderName string) (*g leaderConnectionDetails := l.leaderConfig.LeaderConnection leaderConnectionDetails.ArmadaUrl = strings.ReplaceAll(leaderConnectionDetails.ArmadaUrl, leaseHolderNameToken, currentLeaderName) - apiConnection, err := createApiConnection(leaderConnectionDetails) + apiConnection, err := createApiConnection(leaderConnectionDetails, l.metrics) if err != nil { return nil, errors.Wrapf(err, "error creating connection to leader") } @@ -69,12 +71,11 @@ func (l *LeaderConnectionProvider) getClientByName(currentLeaderName string) (*g return apiConnection, nil } -func createApiConnection(connectionDetails client.ApiConnectionDetails) (*grpc.ClientConn, error) { - grpc_prometheus.EnableClientHandlingTimeHistogram() +func createApiConnection(connectionDetails client.ApiConnectionDetails, clientMetrics *grpc_prometheus.ClientMetrics) (*grpc.ClientConn, error) { return client.CreateApiConnectionWithCallOptions( &connectionDetails, []grpc.CallOption{}, - grpc.WithChainUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor), - grpc.WithChainStreamInterceptor(grpc_prometheus.StreamClientInterceptor), + grpc.WithChainUnaryInterceptor(clientMetrics.UnaryClientInterceptor()), + grpc.WithChainStreamInterceptor(clientMetrics.StreamClientInterceptor()), ) } diff --git a/internal/scheduler/leader/leader_client_test.go b/internal/scheduler/leader/leader_client_test.go index 6b212be0e3a..39f856a57d0 100644 --- a/internal/scheduler/leader/leader_client_test.go +++ b/internal/scheduler/leader/leader_client_test.go @@ -3,6 +3,7 @@ package leader import ( "testing" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" "github.com/stretchr/testify/assert" "github.com/armadaproject/armada/internal/common/armadacontext" @@ -28,7 +29,7 @@ var templatedLeader = configuration.LeaderConfig{ func TestGetCurrentLeaderClientConnection(t *testing.T) { leaderController := &FakeLeaderController{} - clientProvider := NewLeaderConnectionProvider(leaderController, defaultLeaderConfig) + clientProvider := NewLeaderConnectionProvider(leaderController, defaultLeaderConfig, &grpc_prometheus.ClientMetrics{}) leaderController.LeaderName = "new-leader" isCurrentProcessLeader, result, err := clientProvider.GetCurrentLeaderClientConnection() @@ -40,7 +41,7 @@ func TestGetCurrentLeaderClientConnection(t *testing.T) { func TestGetCurrentLeaderClientConnection_WithTemplatedConnection(t *testing.T) { leaderController := &FakeLeaderController{} - clientProvider := NewLeaderConnectionProvider(leaderController, templatedLeader) + clientProvider := NewLeaderConnectionProvider(leaderController, templatedLeader, &grpc_prometheus.ClientMetrics{}) leaderController.LeaderName = "new-leader" isCurrentProcessLeader, result, err := clientProvider.GetCurrentLeaderClientConnection() @@ -59,7 +60,7 @@ func TestGetCurrentLeaderClientConnection_WithTemplatedConnection(t *testing.T) func TestGetCurrentLeaderClientConnection_NoLeader(t *testing.T) { leaderController := &FakeLeaderController{} - clientProvider := NewLeaderConnectionProvider(leaderController, defaultLeaderConfig) + clientProvider := NewLeaderConnectionProvider(leaderController, defaultLeaderConfig, &grpc_prometheus.ClientMetrics{}) isCurrentProcessLeader, result, err := clientProvider.GetCurrentLeaderClientConnection() assert.Nil(t, result) @@ -69,7 +70,7 @@ func TestGetCurrentLeaderClientConnection_NoLeader(t *testing.T) { func TestGetCurrentLeaderClientConnection_OnCurrentProcessIsLeader(t *testing.T) { leaderController := &FakeLeaderController{} - clientProvider := NewLeaderConnectionProvider(leaderController, defaultLeaderConfig) + clientProvider := NewLeaderConnectionProvider(leaderController, defaultLeaderConfig, &grpc_prometheus.ClientMetrics{}) leaderController.IsCurrentlyLeader = true isCurrentProcessLeader, result, err := clientProvider.GetCurrentLeaderClientConnection() diff --git a/internal/scheduler/schedulerapp.go b/internal/scheduler/schedulerapp.go index 4bd15243fc3..12ab1b321cb 100644 --- a/internal/scheduler/schedulerapp.go +++ b/internal/scheduler/schedulerapp.go @@ -10,6 +10,7 @@ import ( "github.com/apache/pulsar-client-go/pulsar" "github.com/google/uuid" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" grpc_logging "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -224,7 +225,12 @@ func Run(config schedulerconfig.Configuration) error { schedulingContextRepository := reports.NewSchedulingContextRepository() reportServer := reports.NewServer(schedulingContextRepository) - leaderClientConnectionProvider := leader.NewLeaderConnectionProvider(leaderController, config.Leader) + clientMetrics := grpc_prometheus.NewClientMetrics( + grpc_prometheus.WithClientHandlingTimeHistogram(), + ) + prometheus.MustRegister(clientMetrics) + + leaderClientConnectionProvider := leader.NewLeaderConnectionProvider(leaderController, config.Leader, clientMetrics) schedulingSchedulerReportingServer := reports.NewLeaderProxyingSchedulingReportsServer(reportServer, leaderClientConnectionProvider) schedulerobjects.RegisterSchedulerReportingServer(grpcServer, schedulingSchedulerReportingServer) diff --git a/internal/server/server.go b/internal/server/server.go index 5a61af58acc..1d6ba535017 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -7,7 +7,7 @@ import ( "github.com/apache/pulsar-client-go/pulsar" "github.com/google/uuid" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/redis/go-redis/extra/redisprometheus/v9" @@ -194,7 +194,6 @@ func Serve(ctx *armadacontext.Context, config *configuration.ArmadaConfig, healt api.RegisterExecutorServer(grpcServer, executorServer) schedulerobjects.RegisterSchedulerReportingServer(grpcServer, schedulingReportsServer) - grpc_prometheus.Register(grpcServer) // Cancel the errgroup if grpcServer.Serve returns an error. log.Infof("Armada gRPC server listening on %d", config.GrpcPort) @@ -235,11 +234,14 @@ func validateSubmissionConfig(config configuration.SubmissionConfig) error { } func createApiConnection(connectionDetails client.ApiConnectionDetails) (*grpc.ClientConn, error) { - grpc_prometheus.EnableClientHandlingTimeHistogram() + clientMetrics := grpc_prometheus.NewClientMetrics( + grpc_prometheus.WithClientHandlingTimeHistogram(), + ) + prometheus.MustRegister(clientMetrics) return client.CreateApiConnectionWithCallOptions( &connectionDetails, []grpc.CallOption{}, - grpc.WithChainUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor), - grpc.WithChainStreamInterceptor(grpc_prometheus.StreamClientInterceptor), + grpc.WithChainUnaryInterceptor(clientMetrics.UnaryClientInterceptor()), + grpc.WithChainStreamInterceptor(clientMetrics.StreamClientInterceptor()), ) } From fcee91c30d465e022efdd8e90dbad03585001c28 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Wed, 29 Jan 2025 13:15:52 +0000 Subject: [PATCH 14/23] Add External Priority Multipliers (#4174) * Create auction.proto * wip * wip * wip * wip * test * test * lint * log msg * default disable priority multipliers * add missing file --- config/scheduler/config.yaml | 3 +- .../scheduler/configuration/configuration.go | 9 + internal/scheduler/metrics/cycle_metrics.go | 28 + .../mocks/{grpc.go => executor_api.go} | 2 +- internal/scheduler/mocks/generate.go | 3 +- internal/scheduler/mocks/priority_override.go | 63 ++ .../prioritymultiplier/service_provider.go | 92 ++ .../service_provider_test.go | 115 +++ .../prioritymultiplier/static_provider.go | 26 + .../static_provider_test.go | 34 + .../scheduler/prioritymultiplier/types.go | 13 + internal/scheduler/schedulerapp.go | 15 + .../scheduler/scheduling/context/queue.go | 3 + .../scheduling/context/scheduling.go | 5 +- .../scheduling/context/scheduling_test.go | 7 +- .../scheduling/gang_scheduler_test.go | 1 + ..._driven_preempting_queue_scheduler_test.go | 1 + .../preempting_queue_scheduler_test.go | 5 +- .../scheduling/queue_scheduler_test.go | 2 +- .../scheduler/scheduling/scheduling_algo.go | 32 +- .../scheduling/scheduling_algo_test.go | 3 + internal/scheduler/simulator/simulator.go | 1 + magefiles/proto.go | 4 +- pkg/priorityoverride/prioritymultiplier.pb.go | 855 ++++++++++++++++++ pkg/priorityoverride/prioritymultiplier.proto | 20 + 25 files changed, 1325 insertions(+), 17 deletions(-) rename internal/scheduler/mocks/{grpc.go => executor_api.go} (97%) create mode 100644 internal/scheduler/mocks/priority_override.go create mode 100644 internal/scheduler/prioritymultiplier/service_provider.go create mode 100644 internal/scheduler/prioritymultiplier/service_provider_test.go create mode 100644 internal/scheduler/prioritymultiplier/static_provider.go create mode 100644 internal/scheduler/prioritymultiplier/static_provider_test.go create mode 100644 internal/scheduler/prioritymultiplier/types.go create mode 100644 pkg/priorityoverride/prioritymultiplier.pb.go create mode 100644 pkg/priorityoverride/prioritymultiplier.proto diff --git a/config/scheduler/config.yaml b/config/scheduler/config.yaml index d3591db942a..6b76cadff12 100644 --- a/config/scheduler/config.yaml +++ b/config/scheduler/config.yaml @@ -27,6 +27,8 @@ pulsar: armadaApi: armadaUrl: "server:50051" forceNoTls: true +priorityMultiplier: + enabled: false postgres: connection: host: postgres @@ -118,4 +120,3 @@ scheduling: experimentalIndicativePricing: basePrice: 100.0 basePriority: 500.0 - diff --git a/internal/scheduler/configuration/configuration.go b/internal/scheduler/configuration/configuration.go index ff3a71bd447..f34ff30b0ba 100644 --- a/internal/scheduler/configuration/configuration.go +++ b/internal/scheduler/configuration/configuration.go @@ -55,6 +55,8 @@ type Configuration struct { DatabaseFetchSize int `validate:"required"` // Frequency at which queues will be fetched from the API QueueRefreshPeriod time.Duration `validate:"required"` + // Allows queue priority multipliers to be fetched from an external source + PriorityMultiplier PriorityMultiplierConfig } type LeaderConfig struct { @@ -295,3 +297,10 @@ type ExperimentalIndicativePricing struct { BasePrice float64 BasePriority float64 } + +type PriorityMultiplierConfig struct { + Enabled bool + UpdateFrequency time.Duration + ServiceUrl string + ForceNoTls bool +} diff --git a/internal/scheduler/metrics/cycle_metrics.go b/internal/scheduler/metrics/cycle_metrics.go index db515925f74..6f0cf8c54f6 100644 --- a/internal/scheduler/metrics/cycle_metrics.go +++ b/internal/scheduler/metrics/cycle_metrics.go @@ -27,6 +27,8 @@ type cycleMetrics struct { fairnessError *prometheus.GaugeVec demand *prometheus.GaugeVec cappedDemand *prometheus.GaugeVec + queueWeight *prometheus.GaugeVec + rawQueueWeight *prometheus.GaugeVec scheduleCycleTime prometheus.Histogram reconciliationCycleTime prometheus.Histogram gangsConsidered *prometheus.GaugeVec @@ -106,6 +108,22 @@ func newCycleMetrics() *cycleMetrics { poolAndQueueLabels, ) + queueWeight := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: prefix + "queue_weight", + Help: "Weight of the queue after multipliers have been applied", + }, + poolAndQueueLabels, + ) + + rawQueueWeight := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: prefix + "raw_queue_weight", + Help: "Weight of the queue before multipliers have been applied", + }, + poolAndQueueLabels, + ) + fairnessError := prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: prefix + "fairness_error", @@ -212,6 +230,8 @@ func newCycleMetrics() *cycleMetrics { actualShare: actualShare, demand: demand, cappedDemand: cappedDemand, + queueWeight: queueWeight, + rawQueueWeight: rawQueueWeight, fairnessError: fairnessError, scheduleCycleTime: scheduleCycleTime, gangsConsidered: gangsConsidered, @@ -242,6 +262,8 @@ func newCycleMetrics() *cycleMetrics { evictedJobs, evictedResources, spotPrice, + queueWeight, + rawQueueWeight, }, reconciliationCycleTime: reconciliationCycleTime, } @@ -286,6 +308,8 @@ func (m *cycleMetrics) ReportSchedulerResult(result scheduling.SchedulerResult) m.actualShare.WithLabelValues(pool, queue).Set(actualShare) m.demand.WithLabelValues(pool, queue).Set(demand) m.cappedDemand.WithLabelValues(pool, queue).Set(cappedDemand) + m.queueWeight.WithLabelValues(pool, queue).Set(queueContext.Weight) + m.rawQueueWeight.WithLabelValues(pool, queue).Set(queueContext.RawWeight) } m.fairnessError.WithLabelValues(pool).Set(schedContext.FairnessError()) m.spotPrice.WithLabelValues(pool).Set(schedContext.SpotPrice) @@ -331,6 +355,8 @@ func (m *cycleMetrics) describe(ch chan<- *prometheus.Desc) { m.fairnessError.Describe(ch) m.demand.Describe(ch) m.cappedDemand.Describe(ch) + m.queueWeight.Describe(ch) + m.rawQueueWeight.Describe(ch) m.scheduleCycleTime.Describe(ch) m.gangsConsidered.Describe(ch) m.gangsScheduled.Describe(ch) @@ -357,6 +383,8 @@ func (m *cycleMetrics) collect(ch chan<- prometheus.Metric) { m.fairnessError.Collect(ch) m.demand.Collect(ch) m.cappedDemand.Collect(ch) + m.rawQueueWeight.Collect(ch) + m.queueWeight.Collect(ch) m.scheduleCycleTime.Collect(ch) m.gangsConsidered.Collect(ch) m.gangsScheduled.Collect(ch) diff --git a/internal/scheduler/mocks/grpc.go b/internal/scheduler/mocks/executor_api.go similarity index 97% rename from internal/scheduler/mocks/grpc.go rename to internal/scheduler/mocks/executor_api.go index 0fd7e6881c2..60682336c02 100644 --- a/internal/scheduler/mocks/grpc.go +++ b/internal/scheduler/mocks/executor_api.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -destination=./grpc.go -package=schedulermocks github.com/armadaproject/armada/pkg/executorapi ExecutorApi_LeaseJobRunsServer +// mockgen -destination=./executor_api.go -package=schedulermocks github.com/armadaproject/armada/pkg/executorapi ExecutorApi_LeaseJobRunsServer // // Package schedulermocks is a generated GoMock package. diff --git a/internal/scheduler/mocks/generate.go b/internal/scheduler/mocks/generate.go index 06e5544c852..ddc43aa6bfb 100644 --- a/internal/scheduler/mocks/generate.go +++ b/internal/scheduler/mocks/generate.go @@ -4,6 +4,7 @@ package schedulermocks //go:generate mockgen -destination=./leases_getter.go -package=schedulermocks "k8s.io/client-go/kubernetes/typed/coordination/v1" LeasesGetter,LeaseInterface //go:generate mockgen -destination=./job_repository.go -package=schedulermocks "github.com/armadaproject/armada/internal/scheduler/database" JobRepository //go:generate mockgen -destination=./executor_repository.go -package=schedulermocks "github.com/armadaproject/armada/internal/scheduler/database" ExecutorRepository -//go:generate mockgen -destination=./grpc.go -package=schedulermocks "github.com/armadaproject/armada/pkg/executorapi" ExecutorApi_LeaseJobRunsServer +//go:generate mockgen -destination=./executor_api.go -package=schedulermocks "github.com/armadaproject/armada/pkg/executorapi" ExecutorApi_LeaseJobRunsServer //go:generate mockgen -destination=./queue_cache.go -package=schedulermocks "github.com/armadaproject/armada/internal/scheduler/queue" QueueCache //go:generate mockgen -destination=./api.go -package=schedulermocks "github.com/armadaproject/armada/pkg/api" SubmitClient,Submit_GetQueuesClient +//go:generate mockgen -destination=./priority_override.go -package=schedulermocks "github.com/armadaproject/armada/pkg/priorityoverride" PriorityMultiplierServiceClient diff --git a/internal/scheduler/mocks/priority_override.go b/internal/scheduler/mocks/priority_override.go new file mode 100644 index 00000000000..5f5ac636fe9 --- /dev/null +++ b/internal/scheduler/mocks/priority_override.go @@ -0,0 +1,63 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/armadaproject/armada/pkg/priorityoverride (interfaces: PriorityMultiplierServiceClient) +// +// Generated by this command: +// +// mockgen -destination=./priority_override.go -package=schedulermocks github.com/armadaproject/armada/pkg/priorityoverride PriorityMultiplierServiceClient +// + +// Package schedulermocks is a generated GoMock package. +package schedulermocks + +import ( + context "context" + reflect "reflect" + + priorityoverride "github.com/armadaproject/armada/pkg/priorityoverride" + gomock "go.uber.org/mock/gomock" + grpc "google.golang.org/grpc" +) + +// MockPriorityMultiplierServiceClient is a mock of PriorityMultiplierServiceClient interface. +type MockPriorityMultiplierServiceClient struct { + ctrl *gomock.Controller + recorder *MockPriorityMultiplierServiceClientMockRecorder + isgomock struct{} +} + +// MockPriorityMultiplierServiceClientMockRecorder is the mock recorder for MockPriorityMultiplierServiceClient. +type MockPriorityMultiplierServiceClientMockRecorder struct { + mock *MockPriorityMultiplierServiceClient +} + +// NewMockPriorityMultiplierServiceClient creates a new mock instance. +func NewMockPriorityMultiplierServiceClient(ctrl *gomock.Controller) *MockPriorityMultiplierServiceClient { + mock := &MockPriorityMultiplierServiceClient{ctrl: ctrl} + mock.recorder = &MockPriorityMultiplierServiceClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPriorityMultiplierServiceClient) EXPECT() *MockPriorityMultiplierServiceClientMockRecorder { + return m.recorder +} + +// GetPriorityMultipliers mocks base method. +func (m *MockPriorityMultiplierServiceClient) GetPriorityMultipliers(ctx context.Context, in *priorityoverride.PriorityMultiplierRequest, opts ...grpc.CallOption) (*priorityoverride.PriorityMultiplierResponse, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetPriorityMultipliers", varargs...) + ret0, _ := ret[0].(*priorityoverride.PriorityMultiplierResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPriorityMultipliers indicates an expected call of GetPriorityMultipliers. +func (mr *MockPriorityMultiplierServiceClientMockRecorder) GetPriorityMultipliers(ctx, in any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPriorityMultipliers", reflect.TypeOf((*MockPriorityMultiplierServiceClient)(nil).GetPriorityMultipliers), varargs...) +} diff --git a/internal/scheduler/prioritymultiplier/service_provider.go b/internal/scheduler/prioritymultiplier/service_provider.go new file mode 100644 index 00000000000..1b129acf93b --- /dev/null +++ b/internal/scheduler/prioritymultiplier/service_provider.go @@ -0,0 +1,92 @@ +package prioritymultiplier + +import ( + "fmt" + "sync/atomic" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + + "github.com/armadaproject/armada/internal/common/armadacontext" + schedulerconfig "github.com/armadaproject/armada/internal/scheduler/configuration" + "github.com/armadaproject/armada/pkg/priorityoverride" +) + +func NewServiceClient(config schedulerconfig.PriorityMultiplierConfig) (priorityoverride.PriorityMultiplierServiceClient, error) { + creds := credentials.NewClientTLSFromCert(nil, "") + if config.ForceNoTls { + creds = insecure.NewCredentials() + } + client, err := grpc.NewClient(config.ServiceUrl, grpc.WithTransportCredentials(creds)) + if err != nil { + return nil, err + } + return priorityoverride.NewPriorityMultiplierServiceClient(client), nil +} + +// ServiceProvider is an implementation of Provider that fetches priority multipliers from the Priority Multiplier Service. +// We cache the multipliers in memory so that we can continue scheduling even if the API is unavailable +type ServiceProvider struct { + updateFrequency time.Duration + apiClient priorityoverride.PriorityMultiplierServiceClient + multipliers atomic.Pointer[map[multiplierKey]float64] +} + +func NewServiceProvider(apiClient priorityoverride.PriorityMultiplierServiceClient, updateFrequency time.Duration) *ServiceProvider { + return &ServiceProvider{ + updateFrequency: updateFrequency, + apiClient: apiClient, + multipliers: atomic.Pointer[map[multiplierKey]float64]{}, + } +} + +func (p *ServiceProvider) Run(ctx *armadacontext.Context) error { + if err := p.fetchMultipliers(ctx); err != nil { + ctx.Warnf("Error fetching multipliers: %v", err) + } + ticker := time.NewTicker(p.updateFrequency) + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + if err := p.fetchMultipliers(ctx); err != nil { + ctx.Warnf("Error fetching multipliers: %v", err) + } + } + } +} + +func (p *ServiceProvider) Ready() bool { + return p.multipliers.Load() != nil +} + +func (p *ServiceProvider) Multiplier(pool, queue string) (float64, error) { + multipliers := p.multipliers.Load() + if multipliers == nil { + return 0, fmt.Errorf("no multipliers available") + } + multiplier, ok := (*multipliers)[multiplierKey{pool: pool, queue: queue}] + if !ok { + return 1.0, nil + } + return multiplier, nil +} + +func (p *ServiceProvider) fetchMultipliers(ctx *armadacontext.Context) error { + resp, err := p.apiClient.GetPriorityMultipliers(ctx, &priorityoverride.PriorityMultiplierRequest{}) + if err != nil { + return err + } + multipliers := make(map[multiplierKey]float64) + for _, poolMultipliers := range resp.PoolPriorityMultipliers { + for queue, multiplier := range poolMultipliers.Multipliers { + key := multiplierKey{pool: poolMultipliers.Pool, queue: queue} + multipliers[key] = multiplier + } + } + p.multipliers.Store(&multipliers) + return nil +} diff --git a/internal/scheduler/prioritymultiplier/service_provider_test.go b/internal/scheduler/prioritymultiplier/service_provider_test.go new file mode 100644 index 00000000000..4d5019fd609 --- /dev/null +++ b/internal/scheduler/prioritymultiplier/service_provider_test.go @@ -0,0 +1,115 @@ +package prioritymultiplier + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/armadaproject/armada/internal/common/armadacontext" + schedulermocks "github.com/armadaproject/armada/internal/scheduler/mocks" + "github.com/armadaproject/armada/pkg/priorityoverride" +) + +func TestFetch(t *testing.T) { + tests := map[string]struct { + multipliers []*priorityoverride.PoolPriorityMultipliers + expectedMultipliers map[multiplierKey]float64 + }{ + "No Multipliers": { + multipliers: []*priorityoverride.PoolPriorityMultipliers{}, + expectedMultipliers: map[multiplierKey]float64{ + {pool: "testpool", queue: "testQueue"}: 1.0, + }, + }, + "One Multiplier, One Pool": { + multipliers: []*priorityoverride.PoolPriorityMultipliers{ + { + Pool: "testpool", + Multipliers: map[string]float64{ + "testqueue": 2.0, + }, + }, + }, + expectedMultipliers: map[multiplierKey]float64{ + {pool: "testpool", queue: "testqueue"}: 2.0, + {pool: "otherpool", queue: "testqueue"}: 1.0, + {pool: "testpool", queue: "otherqueue"}: 1.0, + }, + }, + "Multiple Multipliers, Multiple pools": { + multipliers: []*priorityoverride.PoolPriorityMultipliers{ + { + Pool: "testpool", + Multipliers: map[string]float64{ + "testqueue": 2.0, + "testqueue2": 3.0, + }, + }, + { + Pool: "testpool2", + Multipliers: map[string]float64{ + "testqueue": 4.0, + "testqueue2": 5.0, + }, + }, + }, + expectedMultipliers: map[multiplierKey]float64{ + {pool: "testpool", queue: "testqueue"}: 2.0, + {pool: "testpool", queue: "testqueue2"}: 3.0, + {pool: "testpool2", queue: "testqueue"}: 4.0, + {pool: "testpool2", queue: "testqueue2"}: 5.0, + }, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + ctx := armadacontext.Background() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := schedulermocks.NewMockPriorityMultiplierServiceClient(ctrl) + mockClient.EXPECT(). + GetPriorityMultipliers(gomock.Any(), gomock.Any()). + Return(&priorityoverride.PriorityMultiplierResponse{PoolPriorityMultipliers: tc.multipliers}, nil) + + provider := NewServiceProvider(mockClient, 1*time.Millisecond) + err := provider.fetchMultipliers(ctx) + require.NoError(t, err) + + for k, expectedMultiplier := range tc.expectedMultipliers { + multiplier, err := provider.Multiplier(k.pool, k.queue) + require.NoError(t, err) + assert.Equal(t, expectedMultiplier, multiplier) + } + }) + } +} + +func TestReady(t *testing.T) { + ctx := armadacontext.Background() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := schedulermocks.NewMockPriorityMultiplierServiceClient(ctrl) + mockClient.EXPECT(). + GetPriorityMultipliers(gomock.Any(), gomock.Any()). + Return(&priorityoverride.PriorityMultiplierResponse{PoolPriorityMultipliers: []*priorityoverride.PoolPriorityMultipliers{}}, nil) + provider := NewServiceProvider(mockClient, 1*time.Millisecond) + + // Not ready before fetch + assert.False(t, provider.Ready()) + _, err := provider.Multiplier("pool", "queue") + assert.Error(t, err) + + // Do the fetch + err = provider.fetchMultipliers(ctx) + require.NoError(t, err) + + // Ready after Fetch + assert.True(t, provider.Ready()) + _, err = provider.Multiplier("pool", "queue") + assert.NoError(t, err) +} diff --git a/internal/scheduler/prioritymultiplier/static_provider.go b/internal/scheduler/prioritymultiplier/static_provider.go new file mode 100644 index 00000000000..d8468e14e14 --- /dev/null +++ b/internal/scheduler/prioritymultiplier/static_provider.go @@ -0,0 +1,26 @@ +package prioritymultiplier + +// StaticProvider is Provider that loads priority overrides from a static map +type StaticProvider struct { + multipliers map[multiplierKey]float64 +} + +func NewStaticProvider(multipliers map[multiplierKey]float64) Provider { + return &StaticProvider{multipliers: multipliers} +} + +func NewNoOpProvider() Provider { + return NewStaticProvider(map[multiplierKey]float64{}) +} + +func (s *StaticProvider) Ready() bool { + return true +} + +func (s *StaticProvider) Multiplier(pool, queue string) (float64, error) { + multiplier, ok := s.multipliers[multiplierKey{pool: pool, queue: queue}] + if !ok { + return 1.0, nil + } + return multiplier, nil +} diff --git a/internal/scheduler/prioritymultiplier/static_provider_test.go b/internal/scheduler/prioritymultiplier/static_provider_test.go new file mode 100644 index 00000000000..267552b9614 --- /dev/null +++ b/internal/scheduler/prioritymultiplier/static_provider_test.go @@ -0,0 +1,34 @@ +package prioritymultiplier + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNoOp(t *testing.T) { + testCases := []multiplierKey{ + { + pool: "testPool", + queue: "testQueue", + }, + { + pool: "", + queue: "", + }, + { + pool: "testPool2", + queue: "testQueue2", + }, + } + + provider := NewNoOpProvider() + for _, testCase := range testCases { + testName := fmt.Sprintf("%s_%s", testCase.pool, testCase.queue) + assert.True(t, provider.Ready(), testName) + multiplier, err := provider.Multiplier(testCase.pool, testCase.queue) + assert.NoError(t, err, testName) + assert.Equal(t, 1.0, multiplier, testName) + } +} diff --git a/internal/scheduler/prioritymultiplier/types.go b/internal/scheduler/prioritymultiplier/types.go new file mode 100644 index 00000000000..310c270435b --- /dev/null +++ b/internal/scheduler/prioritymultiplier/types.go @@ -0,0 +1,13 @@ +package prioritymultiplier + +type multiplierKey struct { + pool string + queue string +} + +// Provider provides per-pool-per-queue priority multipliers +// These can be used to increase the priority of a given queue over its base level +type Provider interface { + Ready() bool + Multiplier(pool, queue string) (float64, error) +} diff --git a/internal/scheduler/schedulerapp.go b/internal/scheduler/schedulerapp.go index 12ab1b321cb..cccb4187b3f 100644 --- a/internal/scheduler/schedulerapp.go +++ b/internal/scheduler/schedulerapp.go @@ -40,6 +40,7 @@ import ( "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/leader" "github.com/armadaproject/armada/internal/scheduler/metrics" + "github.com/armadaproject/armada/internal/scheduler/prioritymultiplier" "github.com/armadaproject/armada/internal/scheduler/queue" "github.com/armadaproject/armada/internal/scheduler/reports" "github.com/armadaproject/armada/internal/scheduler/scheduling" @@ -127,6 +128,19 @@ func Run(config schedulerconfig.Configuration) error { queueCache := queue.NewQueueCache(armadaClient, config.QueueRefreshPeriod) services = append(services, func() error { return queueCache.Run(ctx) }) + // //////////////////////////////////////////////////////////////////////// + // Priority Multiplier + // //////////////////////////////////////////////////////////////////////// + priorityMultiplierProvider := prioritymultiplier.NewNoOpProvider() + if config.PriorityMultiplier.Enabled { + ctx.Infof("Priority Multiplier Service configured, will fetch overrides from %s", config.PriorityMultiplier.ServiceUrl) + priorityMultiplierClient, err := prioritymultiplier.NewServiceClient(config.PriorityMultiplier) + if err != nil { + return errors.WithMessage(err, "Error creating priority multiplier client") + } + priorityMultiplierProvider = prioritymultiplier.NewServiceProvider(priorityMultiplierClient, config.PriorityMultiplier.UpdateFrequency) + } + // //////////////////////////////////////////////////////////////////////// // Pulsar // //////////////////////////////////////////////////////////////////////// @@ -259,6 +273,7 @@ func Run(config schedulerconfig.Configuration) error { schedulingContextRepository, resourceListFactory, floatingResourceTypes, + priorityMultiplierProvider, ) if err != nil { return errors.WithMessage(err, "error creating scheduling algo") diff --git a/internal/scheduler/scheduling/context/queue.go b/internal/scheduler/scheduling/context/queue.go index 918028a99f8..d26dbdf43b1 100644 --- a/internal/scheduler/scheduling/context/queue.go +++ b/internal/scheduler/scheduling/context/queue.go @@ -27,6 +27,9 @@ type QueueSchedulingContext struct { Queue string // Determines the fair share of this queue relative to other queues. Weight float64 + // Raw Weight of the queue before any priority boosts. + // This is purely informational as all scheduling decisions are made using Weight + RawWeight float64 // Limits job scheduling rate for this queue. // Use the "Started" time to ensure limiter state remains constant within each scheduling round. Limiter *rate.Limiter diff --git a/internal/scheduler/scheduling/context/scheduling.go b/internal/scheduler/scheduling/context/scheduling.go index cfed7c1f0f3..8aa296ecf50 100644 --- a/internal/scheduler/scheduling/context/scheduling.go +++ b/internal/scheduler/scheduling/context/scheduling.go @@ -90,7 +90,9 @@ func (sctx *SchedulingContext) ClearUnfeasibleSchedulingKeys() { } func (sctx *SchedulingContext) AddQueueSchedulingContext( - queue string, weight float64, + queue string, + weight float64, + rawWeight float64, initialAllocatedByPriorityClass map[string]internaltypes.ResourceList, demand internaltypes.ResourceList, cappedDemand internaltypes.ResourceList, @@ -120,6 +122,7 @@ func (sctx *SchedulingContext) AddQueueSchedulingContext( Created: time.Now(), Queue: queue, Weight: weight, + RawWeight: rawWeight, Limiter: limiter, Allocated: allocated, Demand: demand, diff --git a/internal/scheduler/scheduling/context/scheduling_test.go b/internal/scheduler/scheduling/context/scheduling_test.go index 38461849b2f..d3249197421 100644 --- a/internal/scheduler/scheduling/context/scheduling_test.go +++ b/internal/scheduler/scheduling/context/scheduling_test.go @@ -35,7 +35,8 @@ func TestSchedulingContextAccounting(t *testing.T) { }, } for _, queue := range []string{"A", "B"} { - err := sctx.AddQueueSchedulingContext(queue, priorityFactorByQueue[queue], allocatedByQueueAndPriorityClass[queue], internaltypes.ResourceList{}, internaltypes.ResourceList{}, nil) + priorityFactor := priorityFactorByQueue[queue] + err := sctx.AddQueueSchedulingContext(queue, priorityFactor, priorityFactor, allocatedByQueueAndPriorityClass[queue], internaltypes.ResourceList{}, internaltypes.ResourceList{}, nil) require.NoError(t, err) } @@ -182,7 +183,7 @@ func TestCalculateFairShares(t *testing.T) { ) for qName, q := range tc.queueCtxs { err = sctx.AddQueueSchedulingContext( - qName, q.Weight, map[string]internaltypes.ResourceList{}, q.Demand, q.Demand, nil) + qName, q.Weight, q.Weight, map[string]internaltypes.ResourceList{}, q.Demand, q.Demand, nil) require.NoError(t, err) } sctx.UpdateFairShares() @@ -269,7 +270,7 @@ func TestCalculateTheoreticalShare(t *testing.T) { ) for qName, q := range tc.queueCtxs { err = sctx.AddQueueSchedulingContext( - qName, q.Weight, map[string]internaltypes.ResourceList{}, q.Demand, q.Demand, nil) + qName, q.Weight, q.Weight, map[string]internaltypes.ResourceList{}, q.Demand, q.Demand, nil) require.NoError(t, err) } theoreticalShare := sctx.CalculateTheoreticalShare(tc.basePriority) diff --git a/internal/scheduler/scheduling/gang_scheduler_test.go b/internal/scheduler/scheduling/gang_scheduler_test.go index 2e9f5989dc9..d0566f64fd0 100644 --- a/internal/scheduler/scheduling/gang_scheduler_test.go +++ b/internal/scheduler/scheduling/gang_scheduler_test.go @@ -615,6 +615,7 @@ func TestGangScheduler(t *testing.T) { err := sctx.AddQueueSchedulingContext( queue, priorityFactor, + priorityFactor, nil, internaltypes.ResourceList{}, internaltypes.ResourceList{}, diff --git a/internal/scheduler/scheduling/market_driven_preempting_queue_scheduler_test.go b/internal/scheduler/scheduling/market_driven_preempting_queue_scheduler_test.go index 24ece4726f3..1f929dc743f 100644 --- a/internal/scheduler/scheduling/market_driven_preempting_queue_scheduler_test.go +++ b/internal/scheduler/scheduling/market_driven_preempting_queue_scheduler_test.go @@ -445,6 +445,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { err := sctx.AddQueueSchedulingContext( queue, weight, + weight, allocatedByQueueAndPriorityClass[queue], queueDemand, queueDemand, diff --git a/internal/scheduler/scheduling/preempting_queue_scheduler_test.go b/internal/scheduler/scheduling/preempting_queue_scheduler_test.go index 52c27aca4b0..9729625f7c1 100644 --- a/internal/scheduler/scheduling/preempting_queue_scheduler_test.go +++ b/internal/scheduler/scheduling/preempting_queue_scheduler_test.go @@ -2034,6 +2034,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { err := sctx.AddQueueSchedulingContext( queue, weight, + weight, allocatedByQueueAndPriorityClass[queue], queueDemand, queueDemand, @@ -2387,7 +2388,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { ) for queue, priorityFactor := range priorityFactorByQueue { weight := 1 / priorityFactor - err := sctx.AddQueueSchedulingContext(queue, weight, make(map[string]internaltypes.ResourceList), + err := sctx.AddQueueSchedulingContext(queue, weight, weight, make(map[string]internaltypes.ResourceList), internaltypes.ResourceList{}, internaltypes.ResourceList{}, limiterByQueue[queue]) require.NoError(b, err) } @@ -2458,7 +2459,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { ) for queue, priorityFactor := range priorityFactorByQueue { weight := 1 / priorityFactor - err := sctx.AddQueueSchedulingContext(queue, weight, allocatedByQueueAndPriorityClass[queue], + err := sctx.AddQueueSchedulingContext(queue, weight, weight, allocatedByQueueAndPriorityClass[queue], internaltypes.ResourceList{}, internaltypes.ResourceList{}, limiterByQueue[queue]) require.NoError(b, err) } diff --git a/internal/scheduler/scheduling/queue_scheduler_test.go b/internal/scheduler/scheduling/queue_scheduler_test.go index 14e0454386a..e0fd1efebb5 100644 --- a/internal/scheduler/scheduling/queue_scheduler_test.go +++ b/internal/scheduler/scheduling/queue_scheduler_test.go @@ -504,7 +504,7 @@ func TestQueueScheduler(t *testing.T) { for _, q := range tc.Queues { weight := 1.0 / float64(q.PriorityFactor) err := sctx.AddQueueSchedulingContext( - q.Name, weight, + q.Name, weight, weight, tc.InitialAllocatedByQueueAndPriorityClass[q.Name], internaltypes.ResourceList{}, internaltypes.ResourceList{}, diff --git a/internal/scheduler/scheduling/scheduling_algo.go b/internal/scheduler/scheduling/scheduling_algo.go index 90be011edc5..481015091c9 100644 --- a/internal/scheduler/scheduling/scheduling_algo.go +++ b/internal/scheduler/scheduling/scheduling_algo.go @@ -21,6 +21,7 @@ import ( "github.com/armadaproject/armada/internal/scheduler/internaltypes" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/nodedb" + "github.com/armadaproject/armada/internal/scheduler/prioritymultiplier" "github.com/armadaproject/armada/internal/scheduler/queue" "github.com/armadaproject/armada/internal/scheduler/reports" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" @@ -43,6 +44,7 @@ type FairSchedulingAlgo struct { schedulingConfig configuration.SchedulingConfig executorRepository database.ExecutorRepository queueCache queue.QueueCache + queueMultiplierProvider prioritymultiplier.Provider schedulingContextRepository *reports.SchedulingContextRepository // Global job scheduling rate-limiter. limiter *rate.Limiter @@ -63,6 +65,7 @@ func NewFairSchedulingAlgo( schedulingContextRepository *reports.SchedulingContextRepository, resourceListFactory *internaltypes.ResourceListFactory, floatingResourceTypes *floatingresources.FloatingResourceTypes, + queueMultiplierProvider prioritymultiplier.Provider, ) (*FairSchedulingAlgo, error) { if _, ok := config.PriorityClasses[config.DefaultPriorityClassName]; !ok { return nil, errors.Errorf( @@ -74,6 +77,7 @@ func NewFairSchedulingAlgo( schedulingConfig: config, executorRepository: executorRepository, queueCache: queueCache, + queueMultiplierProvider: queueMultiplierProvider, schedulingContextRepository: schedulingContextRepository, limiter: rate.NewLimiter(rate.Limit(config.MaximumSchedulingRate), config.MaximumSchedulingBurst), limiterByQueue: make(map[string]*rate.Limiter), @@ -108,6 +112,12 @@ func (l *FairSchedulingAlgo) Schedule( return overallSchedulerResult, nil } + // Exit immediately if priority multipliers are not ready + if !l.queueMultiplierProvider.Ready() { + ctx.Warn("queue multipliers are not ready; exiting") + return overallSchedulerResult, nil + } + for _, pool := range l.schedulingConfig.Pools { select { case <-ctx.Done(): @@ -473,10 +483,15 @@ func (l *FairSchedulingAlgo) constructSchedulingContext( if allocationByQueueAndPriorityClass != nil { allocatedByPriorityClass = allocationByQueueAndPriorityClass[queue.Name] } - var weight float64 = 1 + var rawWeight float64 = 1 if queue.PriorityFactor > 0 { - weight = 1 / queue.PriorityFactor + rawWeight = 1 / queue.PriorityFactor } + multiplier, err := l.queueMultiplierProvider.Multiplier(pool, queue.Name) + if err != nil { + return nil, err + } + weight := rawWeight * multiplier queueLimiter, ok := l.limiterByQueue[queue.Name] if !ok { @@ -487,7 +502,7 @@ func (l *FairSchedulingAlgo) constructSchedulingContext( l.limiterByQueue[queue.Name] = queueLimiter } - if err := sctx.AddQueueSchedulingContext(queue.Name, weight, allocatedByPriorityClass, internaltypes.RlMapSumValues(demand), internaltypes.RlMapSumValues(cappedDemand), queueLimiter); err != nil { + if err := sctx.AddQueueSchedulingContext(queue.Name, weight, rawWeight, allocatedByPriorityClass, internaltypes.RlMapSumValues(demand), internaltypes.RlMapSumValues(cappedDemand), queueLimiter); err != nil { return nil, err } } @@ -499,12 +514,17 @@ func (l *FairSchedulingAlgo) constructSchedulingContext( continue } - var weight float64 = 1 + var rawWeight float64 = 1 if queue.PriorityFactor > 0 { - weight = 1 / queue.PriorityFactor + rawWeight = 1 / queue.PriorityFactor + } + multiplier, err := l.queueMultiplierProvider.Multiplier(pool, queue.Name) + if err != nil { + return nil, err } + weight := rawWeight * multiplier - if err := sctx.AddQueueSchedulingContext(schedulercontext.CalculateAwayQueueName(queue.Name), weight, allocation, internaltypes.ResourceList{}, internaltypes.ResourceList{}, nil); err != nil { + if err := sctx.AddQueueSchedulingContext(schedulercontext.CalculateAwayQueueName(queue.Name), weight, rawWeight, allocation, internaltypes.ResourceList{}, internaltypes.ResourceList{}, nil); err != nil { return nil, err } } diff --git a/internal/scheduler/scheduling/scheduling_algo_test.go b/internal/scheduler/scheduling/scheduling_algo_test.go index 8ea52c14b7e..6e8b260666a 100644 --- a/internal/scheduler/scheduling/scheduling_algo_test.go +++ b/internal/scheduler/scheduling/scheduling_algo_test.go @@ -21,6 +21,7 @@ import ( "github.com/armadaproject/armada/internal/scheduler/jobdb" schedulermocks "github.com/armadaproject/armada/internal/scheduler/mocks" "github.com/armadaproject/armada/internal/scheduler/nodedb" + "github.com/armadaproject/armada/internal/scheduler/prioritymultiplier" "github.com/armadaproject/armada/internal/scheduler/reports" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" "github.com/armadaproject/armada/internal/scheduler/testfixtures" @@ -493,6 +494,7 @@ func TestSchedule(t *testing.T) { schedulingContextRepo, testfixtures.TestResourceListFactory, testfixtures.TestEmptyFloatingResources, + prioritymultiplier.NewNoOpProvider(), ) require.NoError(t, err) @@ -658,6 +660,7 @@ func BenchmarkNodeDbConstruction(b *testing.B) { nil, testfixtures.TestResourceListFactory, testfixtures.TestEmptyFloatingResources, + prioritymultiplier.NewNoOpProvider(), ) require.NoError(b, err) b.StartTimer() diff --git a/internal/scheduler/simulator/simulator.go b/internal/scheduler/simulator/simulator.go index a0ca6c2e034..98ccdd4f143 100644 --- a/internal/scheduler/simulator/simulator.go +++ b/internal/scheduler/simulator/simulator.go @@ -579,6 +579,7 @@ func (s *Simulator) handleScheduleEvent(ctx *armadacontext.Context) error { err := sctx.AddQueueSchedulingContext( queue.Name, queue.Weight, + queue.Weight, s.accounting.allocationByPoolAndQueueAndPriorityClass[pool][queue.Name], demand, demand, diff --git a/magefiles/proto.go b/magefiles/proto.go index 40e6ceb917d..5753f181642 100644 --- a/magefiles/proto.go +++ b/magefiles/proto.go @@ -101,6 +101,7 @@ func protoGenerate() error { "pkg/api/binoculars/*.proto", "pkg/api/schedulerobjects/*.proto", "pkg/executorapi/*.proto", + "pkg/priorityoverride/*.proto", } for _, pattern := range patterns { matches, err := filepath.Glob(pattern) @@ -175,7 +176,8 @@ func protoGenerate() error { return err } - err = sh.Run("goimports", "-w", "-local", "github.com/armadaproject/armada", "./pkg/api/", "./pkg/armadaevents/", "./pkg/controlplaneevents/", "./internal/scheduler/schedulerobjects/", "./pkg/executorapi/", "./pkg/api/schedulerobjects/") + err = sh.Run("goimports", "-w", "-local", "github.com/armadaproject/armada", "./pkg/api/", "./pkg/armadaevents/", + "./pkg/controlplaneevents/", "./internal/scheduler/schedulerobjects/", "./pkg/executorapi/", "./pkg/api/schedulerobjects/", "./pkg/priorityoverride/") if err != nil { return err } diff --git a/pkg/priorityoverride/prioritymultiplier.pb.go b/pkg/priorityoverride/prioritymultiplier.pb.go new file mode 100644 index 00000000000..3655a3aa9bb --- /dev/null +++ b/pkg/priorityoverride/prioritymultiplier.pb.go @@ -0,0 +1,855 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: pkg/priorityoverride/prioritymultiplier.proto + +package priorityoverride + +import ( + context "context" + encoding_binary "encoding/binary" + fmt "fmt" + io "io" + math "math" + math_bits "math/bits" + + proto "github.com/gogo/protobuf/proto" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type PriorityMultiplierRequest struct { +} + +func (m *PriorityMultiplierRequest) Reset() { *m = PriorityMultiplierRequest{} } +func (m *PriorityMultiplierRequest) String() string { return proto.CompactTextString(m) } +func (*PriorityMultiplierRequest) ProtoMessage() {} +func (*PriorityMultiplierRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_8d112cf26c3c73e1, []int{0} +} +func (m *PriorityMultiplierRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *PriorityMultiplierRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_PriorityMultiplierRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *PriorityMultiplierRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_PriorityMultiplierRequest.Merge(m, src) +} +func (m *PriorityMultiplierRequest) XXX_Size() int { + return m.Size() +} +func (m *PriorityMultiplierRequest) XXX_DiscardUnknown() { + xxx_messageInfo_PriorityMultiplierRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_PriorityMultiplierRequest proto.InternalMessageInfo + +type PoolPriorityMultipliers struct { + Pool string `protobuf:"bytes,1,opt,name=pool,proto3" json:"pool,omitempty"` + Multipliers map[string]float64 `protobuf:"bytes,2,rep,name=multipliers,proto3" json:"multipliers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"fixed64,2,opt,name=value,proto3"` +} + +func (m *PoolPriorityMultipliers) Reset() { *m = PoolPriorityMultipliers{} } +func (m *PoolPriorityMultipliers) String() string { return proto.CompactTextString(m) } +func (*PoolPriorityMultipliers) ProtoMessage() {} +func (*PoolPriorityMultipliers) Descriptor() ([]byte, []int) { + return fileDescriptor_8d112cf26c3c73e1, []int{1} +} +func (m *PoolPriorityMultipliers) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *PoolPriorityMultipliers) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_PoolPriorityMultipliers.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *PoolPriorityMultipliers) XXX_Merge(src proto.Message) { + xxx_messageInfo_PoolPriorityMultipliers.Merge(m, src) +} +func (m *PoolPriorityMultipliers) XXX_Size() int { + return m.Size() +} +func (m *PoolPriorityMultipliers) XXX_DiscardUnknown() { + xxx_messageInfo_PoolPriorityMultipliers.DiscardUnknown(m) +} + +var xxx_messageInfo_PoolPriorityMultipliers proto.InternalMessageInfo + +func (m *PoolPriorityMultipliers) GetPool() string { + if m != nil { + return m.Pool + } + return "" +} + +func (m *PoolPriorityMultipliers) GetMultipliers() map[string]float64 { + if m != nil { + return m.Multipliers + } + return nil +} + +type PriorityMultiplierResponse struct { + PoolPriorityMultipliers []*PoolPriorityMultipliers `protobuf:"bytes,1,rep,name=poolPriorityMultipliers,proto3" json:"poolPriorityMultipliers,omitempty"` +} + +func (m *PriorityMultiplierResponse) Reset() { *m = PriorityMultiplierResponse{} } +func (m *PriorityMultiplierResponse) String() string { return proto.CompactTextString(m) } +func (*PriorityMultiplierResponse) ProtoMessage() {} +func (*PriorityMultiplierResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_8d112cf26c3c73e1, []int{2} +} +func (m *PriorityMultiplierResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *PriorityMultiplierResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_PriorityMultiplierResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *PriorityMultiplierResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_PriorityMultiplierResponse.Merge(m, src) +} +func (m *PriorityMultiplierResponse) XXX_Size() int { + return m.Size() +} +func (m *PriorityMultiplierResponse) XXX_DiscardUnknown() { + xxx_messageInfo_PriorityMultiplierResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_PriorityMultiplierResponse proto.InternalMessageInfo + +func (m *PriorityMultiplierResponse) GetPoolPriorityMultipliers() []*PoolPriorityMultipliers { + if m != nil { + return m.PoolPriorityMultipliers + } + return nil +} + +func init() { + proto.RegisterType((*PriorityMultiplierRequest)(nil), "api.PriorityMultiplierRequest") + proto.RegisterType((*PoolPriorityMultipliers)(nil), "api.PoolPriorityMultipliers") + proto.RegisterMapType((map[string]float64)(nil), "api.PoolPriorityMultipliers.MultipliersEntry") + proto.RegisterType((*PriorityMultiplierResponse)(nil), "api.PriorityMultiplierResponse") +} + +func init() { + proto.RegisterFile("pkg/priorityoverride/prioritymultiplier.proto", fileDescriptor_8d112cf26c3c73e1) +} + +var fileDescriptor_8d112cf26c3c73e1 = []byte{ + // 349 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x52, 0x3d, 0x4f, 0xc3, 0x30, + 0x10, 0xad, 0x53, 0x40, 0xc2, 0x15, 0x5f, 0xe6, 0xa3, 0x69, 0x41, 0x76, 0x29, 0x02, 0x15, 0x89, + 0x06, 0xa9, 0x2c, 0x88, 0x31, 0x12, 0x62, 0x42, 0xaa, 0xca, 0x04, 0x5b, 0x81, 0x03, 0x99, 0xa6, + 0xb5, 0x71, 0xdc, 0x48, 0x19, 0xf8, 0x0f, 0x2c, 0x4c, 0xfc, 0x21, 0xc6, 0x8e, 0x4c, 0x11, 0x6a, + 0xb7, 0xfc, 0x0a, 0xd4, 0xb4, 0x2a, 0x16, 0x24, 0x6c, 0xf6, 0xdd, 0xf3, 0xbb, 0xf7, 0x7c, 0x0f, + 0xd7, 0x65, 0xe7, 0xf1, 0x58, 0x2a, 0x2e, 0x14, 0xd7, 0xa1, 0x08, 0x40, 0x29, 0x7e, 0x0f, 0xb3, + 0x42, 0xb7, 0xef, 0x69, 0x2e, 0x3d, 0x0e, 0xca, 0x91, 0x4a, 0x68, 0x41, 0xf2, 0x6d, 0xc9, 0xab, + 0xdb, 0xb8, 0xd4, 0x9c, 0x02, 0x2e, 0x67, 0x80, 0x16, 0x3c, 0xf7, 0xc1, 0xd7, 0xd5, 0x37, 0x0b, + 0x17, 0x9b, 0x42, 0x78, 0x7f, 0x11, 0x3e, 0x39, 0xc0, 0x73, 0x52, 0x08, 0xcf, 0x46, 0x15, 0x54, + 0x5b, 0x74, 0x49, 0x1c, 0xb1, 0xe5, 0xf1, 0xfd, 0x48, 0x74, 0xb9, 0x86, 0xae, 0xd4, 0x61, 0x2b, + 0xe9, 0x93, 0x27, 0x5c, 0xf8, 0x99, 0xec, 0xdb, 0x56, 0x25, 0x5f, 0x2b, 0x34, 0xea, 0x4e, 0x5b, + 0x72, 0x27, 0x83, 0xda, 0x31, 0xce, 0xe7, 0x3d, 0xad, 0x42, 0xb7, 0x14, 0x47, 0x6c, 0xd3, 0x60, + 0x31, 0x86, 0x98, 0xe4, 0xe5, 0x07, 0xbc, 0xfa, 0xfb, 0x2d, 0xd9, 0xc3, 0xf9, 0x0e, 0x84, 0x53, + 0x99, 0x6b, 0x71, 0xc4, 0x96, 0x3a, 0x10, 0x1a, 0x04, 0xe3, 0x2e, 0x39, 0xc4, 0xf3, 0x41, 0xdb, + 0xeb, 0x83, 0x6d, 0x55, 0x50, 0x0d, 0xb9, 0xeb, 0x71, 0xc4, 0x56, 0x92, 0x82, 0x01, 0x9c, 0x20, + 0xce, 0xac, 0x53, 0x54, 0x7d, 0x47, 0xb8, 0x9c, 0xf6, 0x6b, 0xbe, 0x14, 0x3d, 0x1f, 0xc8, 0x0b, + 0x2e, 0xca, 0x74, 0x6b, 0x36, 0x4a, 0xec, 0xef, 0xfc, 0x67, 0xdf, 0xdd, 0x8f, 0x23, 0xb6, 0x9b, + 0x41, 0x60, 0xe8, 0xc9, 0x9a, 0xd1, 0x08, 0xd2, 0x56, 0x7a, 0x05, 0x2a, 0xe0, 0x77, 0x40, 0xae, + 0xf1, 0xd6, 0x05, 0xe8, 0xb4, 0x85, 0xd2, 0x89, 0xa8, 0xac, 0x30, 0x94, 0x59, 0x66, 0x7f, 0x62, + 0xdb, 0x75, 0x3e, 0x86, 0x14, 0x0d, 0x86, 0x14, 0x7d, 0x0d, 0x29, 0x7a, 0x1d, 0xd1, 0xdc, 0x60, + 0x44, 0x73, 0x9f, 0x23, 0x9a, 0xbb, 0xd9, 0x48, 0x0b, 0xe6, 0xed, 0x42, 0x12, 0xc3, 0x93, 0xef, + 0x00, 0x00, 0x00, 0xff, 0xff, 0x8c, 0x8a, 0xca, 0xdd, 0xb7, 0x02, 0x00, 0x00, +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// PriorityMultiplierServiceClient is the client API for PriorityMultiplierService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type PriorityMultiplierServiceClient interface { + GetPriorityMultipliers(ctx context.Context, in *PriorityMultiplierRequest, opts ...grpc.CallOption) (*PriorityMultiplierResponse, error) +} + +type priorityMultiplierServiceClient struct { + cc *grpc.ClientConn +} + +func NewPriorityMultiplierServiceClient(cc *grpc.ClientConn) PriorityMultiplierServiceClient { + return &priorityMultiplierServiceClient{cc} +} + +func (c *priorityMultiplierServiceClient) GetPriorityMultipliers(ctx context.Context, in *PriorityMultiplierRequest, opts ...grpc.CallOption) (*PriorityMultiplierResponse, error) { + out := new(PriorityMultiplierResponse) + err := c.cc.Invoke(ctx, "/api.PriorityMultiplierService/GetPriorityMultipliers", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// PriorityMultiplierServiceServer is the server API for PriorityMultiplierService service. +type PriorityMultiplierServiceServer interface { + GetPriorityMultipliers(context.Context, *PriorityMultiplierRequest) (*PriorityMultiplierResponse, error) +} + +// UnimplementedPriorityMultiplierServiceServer can be embedded to have forward compatible implementations. +type UnimplementedPriorityMultiplierServiceServer struct { +} + +func (*UnimplementedPriorityMultiplierServiceServer) GetPriorityMultipliers(ctx context.Context, req *PriorityMultiplierRequest) (*PriorityMultiplierResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetPriorityMultipliers not implemented") +} + +func RegisterPriorityMultiplierServiceServer(s *grpc.Server, srv PriorityMultiplierServiceServer) { + s.RegisterService(&_PriorityMultiplierService_serviceDesc, srv) +} + +func _PriorityMultiplierService_GetPriorityMultipliers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PriorityMultiplierRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PriorityMultiplierServiceServer).GetPriorityMultipliers(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/api.PriorityMultiplierService/GetPriorityMultipliers", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PriorityMultiplierServiceServer).GetPriorityMultipliers(ctx, req.(*PriorityMultiplierRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _PriorityMultiplierService_serviceDesc = grpc.ServiceDesc{ + ServiceName: "api.PriorityMultiplierService", + HandlerType: (*PriorityMultiplierServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetPriorityMultipliers", + Handler: _PriorityMultiplierService_GetPriorityMultipliers_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "pkg/priorityoverride/prioritymultiplier.proto", +} + +func (m *PriorityMultiplierRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PriorityMultiplierRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PriorityMultiplierRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + return len(dAtA) - i, nil +} + +func (m *PoolPriorityMultipliers) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PoolPriorityMultipliers) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PoolPriorityMultipliers) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Multipliers) > 0 { + for k := range m.Multipliers { + v := m.Multipliers[k] + baseI := i + i -= 8 + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(v)))) + i-- + dAtA[i] = 0x11 + i -= len(k) + copy(dAtA[i:], k) + i = encodeVarintPrioritymultiplier(dAtA, i, uint64(len(k))) + i-- + dAtA[i] = 0xa + i = encodeVarintPrioritymultiplier(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0x12 + } + } + if len(m.Pool) > 0 { + i -= len(m.Pool) + copy(dAtA[i:], m.Pool) + i = encodeVarintPrioritymultiplier(dAtA, i, uint64(len(m.Pool))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *PriorityMultiplierResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PriorityMultiplierResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PriorityMultiplierResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.PoolPriorityMultipliers) > 0 { + for iNdEx := len(m.PoolPriorityMultipliers) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.PoolPriorityMultipliers[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrioritymultiplier(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func encodeVarintPrioritymultiplier(dAtA []byte, offset int, v uint64) int { + offset -= sovPrioritymultiplier(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *PriorityMultiplierRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + return n +} + +func (m *PoolPriorityMultipliers) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Pool) + if l > 0 { + n += 1 + l + sovPrioritymultiplier(uint64(l)) + } + if len(m.Multipliers) > 0 { + for k, v := range m.Multipliers { + _ = k + _ = v + mapEntrySize := 1 + len(k) + sovPrioritymultiplier(uint64(len(k))) + 1 + 8 + n += mapEntrySize + 1 + sovPrioritymultiplier(uint64(mapEntrySize)) + } + } + return n +} + +func (m *PriorityMultiplierResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.PoolPriorityMultipliers) > 0 { + for _, e := range m.PoolPriorityMultipliers { + l = e.Size() + n += 1 + l + sovPrioritymultiplier(uint64(l)) + } + } + return n +} + +func sovPrioritymultiplier(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozPrioritymultiplier(x uint64) (n int) { + return sovPrioritymultiplier(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *PriorityMultiplierRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrioritymultiplier + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PriorityMultiplierRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PriorityMultiplierRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipPrioritymultiplier(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrioritymultiplier + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PoolPriorityMultipliers) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrioritymultiplier + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PoolPriorityMultipliers: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PoolPriorityMultipliers: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Pool", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrioritymultiplier + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrioritymultiplier + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrioritymultiplier + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Pool = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Multipliers", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrioritymultiplier + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrioritymultiplier + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrioritymultiplier + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Multipliers == nil { + m.Multipliers = make(map[string]float64) + } + var mapkey string + var mapvalue float64 + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrioritymultiplier + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrioritymultiplier + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrioritymultiplier + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return ErrInvalidLengthPrioritymultiplier + } + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var mapvaluetemp uint64 + if (iNdEx + 8) > l { + return io.ErrUnexpectedEOF + } + mapvaluetemp = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) + iNdEx += 8 + mapvalue = math.Float64frombits(mapvaluetemp) + } else { + iNdEx = entryPreIndex + skippy, err := skipPrioritymultiplier(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrioritymultiplier + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.Multipliers[mapkey] = mapvalue + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrioritymultiplier(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrioritymultiplier + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PriorityMultiplierResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrioritymultiplier + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PriorityMultiplierResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PriorityMultiplierResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PoolPriorityMultipliers", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrioritymultiplier + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrioritymultiplier + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrioritymultiplier + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PoolPriorityMultipliers = append(m.PoolPriorityMultipliers, &PoolPriorityMultipliers{}) + if err := m.PoolPriorityMultipliers[len(m.PoolPriorityMultipliers)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrioritymultiplier(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrioritymultiplier + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipPrioritymultiplier(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowPrioritymultiplier + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowPrioritymultiplier + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowPrioritymultiplier + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthPrioritymultiplier + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupPrioritymultiplier + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthPrioritymultiplier + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthPrioritymultiplier = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowPrioritymultiplier = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupPrioritymultiplier = fmt.Errorf("proto: unexpected end of group") +) diff --git a/pkg/priorityoverride/prioritymultiplier.proto b/pkg/priorityoverride/prioritymultiplier.proto new file mode 100644 index 00000000000..4d1ce9aa293 --- /dev/null +++ b/pkg/priorityoverride/prioritymultiplier.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; +package api; +option go_package = "pkg/priorityoverride"; + +message PriorityMultiplierRequest { + // Empty for now; future versions may include filtering parameters +} + +message PoolPriorityMultipliers { + string pool = 1; + map multipliers = 2; +} + +message PriorityMultiplierResponse { + repeated PoolPriorityMultipliers poolPriorityMultipliers = 1; +} + +service PriorityMultiplierService { + rpc GetPriorityMultipliers (PriorityMultiplierRequest) returns (PriorityMultiplierResponse); +} From 4f14d0a97c4be8dc32e6949f229b46c05c5f5938 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Thu, 30 Jan 2025 15:30:09 +0000 Subject: [PATCH 15/23] actually start up the service provider (#4177) --- internal/scheduler/schedulerapp.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/scheduler/schedulerapp.go b/internal/scheduler/schedulerapp.go index cccb4187b3f..a20d05a2ae1 100644 --- a/internal/scheduler/schedulerapp.go +++ b/internal/scheduler/schedulerapp.go @@ -138,7 +138,9 @@ func Run(config schedulerconfig.Configuration) error { if err != nil { return errors.WithMessage(err, "Error creating priority multiplier client") } - priorityMultiplierProvider = prioritymultiplier.NewServiceProvider(priorityMultiplierClient, config.PriorityMultiplier.UpdateFrequency) + provider := prioritymultiplier.NewServiceProvider(priorityMultiplierClient, config.PriorityMultiplier.UpdateFrequency) + services = append(services, func() error { return provider.Run(ctx) }) + priorityMultiplierProvider = provider } // //////////////////////////////////////////////////////////////////////// From 3db3eeab1ca0b88e52adb1bb631f4939be90fb86 Mon Sep 17 00:00:00 2001 From: Gary Edwards <66478610+geaere@users.noreply.github.com> Date: Fri, 31 Jan 2025 14:08:03 +0000 Subject: [PATCH 16/23] Add average utilisation to job events (#342) --- internal/common/resource/resource.go | 13 + internal/executor/domain/resources.go | 87 ++- internal/executor/domain/resources_test.go | 94 ++- .../metrics/pod_metrics/cluster_context.go | 2 +- internal/executor/reporter/event.go | 5 +- .../utilisation/cluster_utilisation.go | 2 +- .../utilisation/job_utilisation_reporter.go | 20 +- .../job_utilisation_reporter_test.go | 16 +- .../executor/utilisation/pod_utilisation.go | 6 +- .../pod_utilisation_custom_metrics.go | 2 +- .../pod_utilisation_custom_metrics_test.go | 15 +- .../pod_utilisation_kubelet_metrics.go | 8 +- pkg/armadaevents/events.pb.go | 676 +++++++++++------- pkg/armadaevents/events.proto | 1 + 14 files changed, 624 insertions(+), 323 deletions(-) diff --git a/internal/common/resource/resource.go b/internal/common/resource/resource.go index 00265f544eb..32317648e95 100644 --- a/internal/common/resource/resource.go +++ b/internal/common/resource/resource.go @@ -204,6 +204,19 @@ func (a ComputeResources) Sub(b ComputeResources) { } } +func (a ComputeResources) Div(b map[string]int64) { + for k, v := range b { + existing, ok := a[k] + if ok { + if existing.Format == resource.DecimalSI { + a[k] = resource.NewMilliQuantity(existing.MilliValue()/v, existing.Format).DeepCopy() + } else { + a[k] = resource.NewQuantity(existing.Value()/v, existing.Format).DeepCopy() + } + } + } +} + func (a ComputeResources) DeepCopy() ComputeResources { targetComputeResource := make(ComputeResources) diff --git a/internal/executor/domain/resources.go b/internal/executor/domain/resources.go index 4d00d3cd366..1f0a8d634a7 100644 --- a/internal/executor/domain/resources.go +++ b/internal/executor/domain/resources.go @@ -1,31 +1,96 @@ package domain -import armadaresource "github.com/armadaproject/armada/internal/common/resource" +import ( + "k8s.io/apimachinery/pkg/api/resource" + + armadaresource "github.com/armadaproject/armada/internal/common/resource" +) type UtilisationData struct { - CurrentUsage armadaresource.ComputeResources - CumulativeUsage armadaresource.ComputeResources + currentUsage armadaresource.ComputeResources + maxCurrentUsage armadaresource.ComputeResources + sumCurrentUsage armadaresource.ComputeResources + numDataPoints map[string]int64 + cumulativeUsage armadaresource.ComputeResources +} + +func NewUtilisationData( + resources map[string]resource.Quantity, +) *UtilisationData { + ret := EmptyUtilisationData() + for k, v := range resources { + ret.UpdateCurrentUsage(k, v) + } + return ret } func EmptyUtilisationData() *UtilisationData { return &UtilisationData{ - CurrentUsage: armadaresource.ComputeResources{}, - CumulativeUsage: armadaresource.ComputeResources{}, + currentUsage: armadaresource.ComputeResources{}, + maxCurrentUsage: armadaresource.ComputeResources{}, + sumCurrentUsage: armadaresource.ComputeResources{}, + numDataPoints: map[string]int64{}, + cumulativeUsage: armadaresource.ComputeResources{}, + } +} + +func (a *UtilisationData) UpdateCurrentUsage(resource string, quantity resource.Quantity) { + a.currentUsage[resource] = quantity + a.maxCurrentUsage[resource] = quantity + a.sumCurrentUsage[resource] = quantity + a.numDataPoints[resource] = 1 +} + +func (a *UtilisationData) Process(b *UtilisationData) { + a.currentUsage = b.currentUsage.DeepCopy() + a.maxCurrentUsage.Max(b.maxCurrentUsage) + a.sumCurrentUsage.Add(b.sumCurrentUsage) + a.cumulativeUsage.Max(b.cumulativeUsage) + + for k, v := range b.numDataPoints { + existing, ok := a.numDataPoints[k] + if ok { + a.numDataPoints[k] = existing + v + } else { + a.numDataPoints[k] = v + } } } -func (a *UtilisationData) Max(b *UtilisationData) { - a.CurrentUsage.Max(b.CurrentUsage) - a.CumulativeUsage.Max(b.CumulativeUsage) +func (u *UtilisationData) GetCurrentUsage() armadaresource.ComputeResources { + return u.currentUsage +} + +func (u *UtilisationData) GetMaxUsage() armadaresource.ComputeResources { + return u.maxCurrentUsage +} + +func (u *UtilisationData) GetAvgUsage() armadaresource.ComputeResources { + avg := u.sumCurrentUsage.DeepCopy() + avg.Div(u.numDataPoints) + return avg +} + +func (u *UtilisationData) GetCumulativeUsage() armadaresource.ComputeResources { + return u.cumulativeUsage } func (u *UtilisationData) DeepCopy() *UtilisationData { + numDataPoints := map[string]int64{} + + for k, v := range u.numDataPoints { + numDataPoints[k] = v + } + return &UtilisationData{ - CurrentUsage: u.CurrentUsage.DeepCopy(), - CumulativeUsage: u.CumulativeUsage.DeepCopy(), + currentUsage: u.currentUsage.DeepCopy(), + maxCurrentUsage: u.maxCurrentUsage.DeepCopy(), + sumCurrentUsage: u.sumCurrentUsage.DeepCopy(), + numDataPoints: numDataPoints, + cumulativeUsage: u.cumulativeUsage.DeepCopy(), } } func (u *UtilisationData) IsEmpty() bool { - return len(u.CumulativeUsage) == 0 && len(u.CurrentUsage) == 0 + return len(u.cumulativeUsage) == 0 && len(u.currentUsage) == 0 } diff --git a/internal/executor/domain/resources_test.go b/internal/executor/domain/resources_test.go index 5ce50f7ab65..004f01fa849 100644 --- a/internal/executor/domain/resources_test.go +++ b/internal/executor/domain/resources_test.go @@ -9,37 +9,89 @@ import ( armadaresource "github.com/armadaproject/armada/internal/common/resource" ) -func TestUtilisationData_Max(t *testing.T) { +func compareQuantityMaps(a, b map[string]resource.Quantity) bool { + if len(a) != len(b) { + return false + } + for key, aVal := range a { + bVal, exists := b[key] + if !exists { + return false + } + if aVal.String() != bVal.String() { + return false + } + } + return true +} + +func TestUtilisationData_Aggregations(t *testing.T) { data := &UtilisationData{ - CurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("10")}, - CumulativeUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("5")}, + currentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("10")}, + maxCurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("10")}, + sumCurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("10")}, + numDataPoints: map[string]int64{"cpu": 1, "memory": 1}, + cumulativeUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("5")}, } data2 := &UtilisationData{ - CurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("2"), "memory": resource.MustParse("1")}, - CumulativeUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("10")}, + currentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("2"), "memory": resource.MustParse("1")}, + maxCurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("2"), "memory": resource.MustParse("1")}, + sumCurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("2"), "memory": resource.MustParse("1")}, + numDataPoints: map[string]int64{"cpu": 1, "memory": 1}, + cumulativeUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("10")}, } expected := &UtilisationData{ - CurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("2"), "memory": resource.MustParse("10")}, - CumulativeUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("10")}, + currentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("2"), "memory": resource.MustParse("1")}, + maxCurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("2"), "memory": resource.MustParse("10")}, + sumCurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("3"), "memory": resource.MustParse("11")}, + numDataPoints: map[string]int64{"cpu": 2, "memory": 2}, + cumulativeUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("10")}, } - max := data - max.Max(data2) - assert.Equal(t, expected.CurrentUsage, max.CurrentUsage) - assert.Equal(t, expected.CumulativeUsage, max.CumulativeUsage) + agg := data + agg.Process(data2) + + assert.True(t, compareQuantityMaps(expected.currentUsage, agg.currentUsage)) + assert.True(t, compareQuantityMaps(expected.maxCurrentUsage, agg.maxCurrentUsage)) + assert.True(t, compareQuantityMaps(expected.sumCurrentUsage, agg.sumCurrentUsage)) + assert.Equal(t, expected.numDataPoints, agg.numDataPoints) + assert.True(t, compareQuantityMaps(expected.cumulativeUsage, agg.cumulativeUsage)) +} + +func TestUtilisationData_GetAvg(t *testing.T) { + data := &UtilisationData{ + currentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("100m"), "memory": resource.MustParse("10Mi")}, + maxCurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("100m"), "memory": resource.MustParse("100Mi")}, + sumCurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("160m"), "memory": resource.MustParse("110Mi")}, + numDataPoints: map[string]int64{"cpu": 2, "memory": 2}, + cumulativeUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("160m")}, + } + + expected := armadaresource.ComputeResources{"cpu": resource.MustParse("80m"), "memory": resource.MustParse("55Mi")} + + assert.True(t, compareQuantityMaps(expected, data.GetAvgUsage())) } func TestUtilisationData_Max_WithEmpty(t *testing.T) { currentUsage := armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} + maxCurrentUsage := armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} + sumCurrentUsage := armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} + numDataPoints := map[string]int64{"cpu": 1, "memory": 1} cumulativeUsage := armadaresource.ComputeResources{"cpu": resource.MustParse("10")} data := &UtilisationData{ - CurrentUsage: currentUsage.DeepCopy(), - CumulativeUsage: cumulativeUsage.DeepCopy(), + currentUsage: currentUsage.DeepCopy(), + maxCurrentUsage: maxCurrentUsage.DeepCopy(), + sumCurrentUsage: sumCurrentUsage.DeepCopy(), + numDataPoints: numDataPoints, + cumulativeUsage: cumulativeUsage.DeepCopy(), } - max := EmptyUtilisationData() - max.Max(data) - assert.Equal(t, data.CurrentUsage, max.CurrentUsage) - assert.Equal(t, data.CumulativeUsage, max.CumulativeUsage) + agg := EmptyUtilisationData() + agg.Process(data) + assert.Equal(t, data.currentUsage, agg.currentUsage) + assert.Equal(t, data.maxCurrentUsage, agg.maxCurrentUsage) + assert.Equal(t, data.sumCurrentUsage, agg.sumCurrentUsage) + assert.Equal(t, data.numDataPoints, agg.numDataPoints) + assert.Equal(t, data.cumulativeUsage, agg.cumulativeUsage) } func TestUtilisationData_IsEmpty(t *testing.T) { @@ -47,15 +99,15 @@ func TestUtilisationData_IsEmpty(t *testing.T) { assert.True(t, data.IsEmpty()) cumulativeUsageNotEmpty := EmptyUtilisationData() - cumulativeUsageNotEmpty.CumulativeUsage = armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} + cumulativeUsageNotEmpty.cumulativeUsage = armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} assert.False(t, cumulativeUsageNotEmpty.IsEmpty()) currentUsageNotEmpty := EmptyUtilisationData() - currentUsageNotEmpty.CurrentUsage = armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} + currentUsageNotEmpty.currentUsage = armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} assert.False(t, currentUsageNotEmpty.IsEmpty()) allNotEmpty := EmptyUtilisationData() - allNotEmpty.CumulativeUsage = armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} - allNotEmpty.CurrentUsage = armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} + allNotEmpty.cumulativeUsage = armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} + allNotEmpty.currentUsage = armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} assert.False(t, allNotEmpty.IsEmpty()) } diff --git a/internal/executor/metrics/pod_metrics/cluster_context.go b/internal/executor/metrics/pod_metrics/cluster_context.go index 028c5f1f86a..925f2eebde8 100644 --- a/internal/executor/metrics/pod_metrics/cluster_context.go +++ b/internal/executor/metrics/pod_metrics/cluster_context.go @@ -197,7 +197,7 @@ func (m *ClusterContextMetrics) Collect(metrics chan<- prometheus.Metric) { nodeTypeMetric[phase].count++ nodeTypeMetric[phase].resourceRequest.Add(request) - nodeTypeMetric[phase].resourceUsage.Add(usage.CurrentUsage) + nodeTypeMetric[phase].resourceUsage.Add(usage.GetCurrentUsage()) } m.setEmptyMetrics(podMetrics) diff --git a/internal/executor/reporter/event.go b/internal/executor/reporter/event.go index a11e954f47c..2fd1d3618af 100644 --- a/internal/executor/reporter/event.go +++ b/internal/executor/reporter/event.go @@ -345,8 +345,9 @@ func CreateJobUtilisationEvent(pod *v1.Pod, utilisationData *domain.UtilisationD }, }, }, - MaxResourcesForPeriod: utilisationData.CurrentUsage.ToProtoMap(), - TotalCumulativeUsage: utilisationData.CumulativeUsage.ToProtoMap(), + MaxResourcesForPeriod: utilisationData.GetMaxUsage().ToProtoMap(), + AvgResourcesForPeriod: utilisationData.GetAvgUsage().ToProtoMap(), + TotalCumulativeUsage: utilisationData.GetCumulativeUsage().ToProtoMap(), }, }, }) diff --git a/internal/executor/utilisation/cluster_utilisation.go b/internal/executor/utilisation/cluster_utilisation.go index e2549731796..3d87d80e7b3 100644 --- a/internal/executor/utilisation/cluster_utilisation.go +++ b/internal/executor/utilisation/cluster_utilisation.go @@ -376,7 +376,7 @@ func (clusterUtilisationService *ClusterUtilisationService) getPodUtilisationByQ if _, ok := result[queue][pool]; !ok { result[queue][pool] = armadaresource.ComputeResources{} } - result[queue][pool].Add(podUsage.CurrentUsage) + result[queue][pool].Add(podUsage.GetCurrentUsage()) } return result } diff --git a/internal/executor/utilisation/job_utilisation_reporter.go b/internal/executor/utilisation/job_utilisation_reporter.go index b4387db22bf..2c6b58585ea 100644 --- a/internal/executor/utilisation/job_utilisation_reporter.go +++ b/internal/executor/utilisation/job_utilisation_reporter.go @@ -25,9 +25,9 @@ type UtilisationEventReporter struct { } type podUtilisationInfo struct { - lastReported time.Time - pod *v1.Pod - utilisationMax *domain.UtilisationData + lastReported time.Time + pod *v1.Pod + utilisationData *domain.UtilisationData } func NewUtilisationEventReporter( @@ -84,12 +84,12 @@ func (r *UtilisationEventReporter) ReportUtilisationEvents() { reportingTime := now.Add(-r.reportingInterval) for _, info := range r.podInfo { currentUtilisation := r.podUtilisation.GetPodUtilisation(info.pod) - info.utilisationMax.Max(currentUtilisation) + info.utilisationData.Process(currentUtilisation) if info.lastReported.Before(reportingTime) { reported := r.reportUsage(info) if reported { info.lastReported = now - info.utilisationMax = domain.EmptyUtilisationData() + info.utilisationData = domain.EmptyUtilisationData() } } } @@ -107,9 +107,9 @@ func (r *UtilisationEventReporter) updatePod(pod *v1.Pod) { _, exists := r.podInfo[pod.Name] if !exists { r.podInfo[pod.Name] = &podUtilisationInfo{ - lastReported: time.Now(), - pod: pod, - utilisationMax: r.podUtilisation.GetPodUtilisation(pod), + lastReported: time.Now(), + pod: pod, + utilisationData: r.podUtilisation.GetPodUtilisation(pod), } } } @@ -132,10 +132,10 @@ func (r *UtilisationEventReporter) deletePod(pod *v1.Pod) { } func (r *UtilisationEventReporter) reportUsage(info *podUtilisationInfo) bool { - if info.utilisationMax.IsEmpty() { + if info.utilisationData.IsEmpty() { return false } - event, err := reporter.CreateJobUtilisationEvent(info.pod, info.utilisationMax, r.clusterContext.GetClusterId()) + event, err := reporter.CreateJobUtilisationEvent(info.pod, info.utilisationData, r.clusterContext.GetClusterId()) if err != nil { log.Errorf("failed to create utilisation event %s", err) return false diff --git a/internal/executor/utilisation/job_utilisation_reporter_test.go b/internal/executor/utilisation/job_utilisation_reporter_test.go index 39c23d5463d..3dcb15cedc2 100644 --- a/internal/executor/utilisation/job_utilisation_reporter_test.go +++ b/internal/executor/utilisation/job_utilisation_reporter_test.go @@ -22,21 +22,18 @@ import ( "github.com/armadaproject/armada/pkg/armadaevents" ) -var testPodResources = domain.UtilisationData{ - CurrentUsage: armadaresource.ComputeResources{ +var testPodResources = domain.NewUtilisationData( + map[string]resource.Quantity{ "cpu": resource.MustParse("1"), "memory": resource.MustParse("640Ki"), }, - CumulativeUsage: armadaresource.ComputeResources{ - "cpu": resource.MustParse("10"), - }, -} +) func TestUtilisationEventReporter_ReportUtilisationEvents(t *testing.T) { reportingPeriod := 100 * time.Millisecond clusterContext := fakeContext.NewFakeClusterContext(configuration.ApplicationConfiguration{ClusterId: "test", Pool: "pool"}, "kubernetes.io/hostname", nil) fakeEventReporter := &mocks.FakeEventReporter{} - fakeUtilisationService := &fakePodUtilisationService{data: &testPodResources} + fakeUtilisationService := &fakePodUtilisationService{data: testPodResources} eventReporter, err := NewUtilisationEventReporter(clusterContext, fakeUtilisationService, fakeEventReporter, reportingPeriod) require.NoError(t, err) @@ -61,8 +58,9 @@ func TestUtilisationEventReporter_ReportUtilisationEvents(t *testing.T) { _, ok = fakeEventReporter.ReceivedEvents[1].Event.Events[0].Event.(*armadaevents.EventSequence_Event_ResourceUtilisation) assert.True(t, ok) - assert.Equal(t, testPodResources.CurrentUsage, armadaresource.FromProtoMap(event1.ResourceUtilisation.MaxResourcesForPeriod)) - assert.Equal(t, testPodResources.CumulativeUsage, armadaresource.FromProtoMap(event1.ResourceUtilisation.TotalCumulativeUsage)) + assert.Equal(t, testPodResources.GetMaxUsage(), armadaresource.FromProtoMap(event1.ResourceUtilisation.MaxResourcesForPeriod)) + assert.Equal(t, testPodResources.GetAvgUsage(), armadaresource.FromProtoMap(event1.ResourceUtilisation.AvgResourcesForPeriod)) + assert.Equal(t, testPodResources.GetCumulativeUsage(), armadaresource.FromProtoMap(event1.ResourceUtilisation.TotalCumulativeUsage)) event1CreatedTime := fakeEventReporter.ReceivedEvents[0].Event.Events[0].Created event2CreatedTime := fakeEventReporter.ReceivedEvents[1].Event.Events[0].Created diff --git a/internal/executor/utilisation/pod_utilisation.go b/internal/executor/utilisation/pod_utilisation.go index ec075b198e8..a95c4dccdaf 100644 --- a/internal/executor/utilisation/pod_utilisation.go +++ b/internal/executor/utilisation/pod_utilisation.go @@ -8,7 +8,6 @@ import ( v1 "k8s.io/api/core/v1" log "github.com/armadaproject/armada/internal/common/logging" - armadaresource "github.com/armadaproject/armada/internal/common/resource" commonUtil "github.com/armadaproject/armada/internal/common/util" "github.com/armadaproject/armada/internal/executor/configuration" clusterContext "github.com/armadaproject/armada/internal/executor/context" @@ -115,10 +114,7 @@ func (q *PodUtilisationServiceImpl) RefreshUtilisationData() { podNameToUtilisationData := map[string]*domain.UtilisationData{} for _, podName := range podNames { - podNameToUtilisationData[podName] = &domain.UtilisationData{ - CurrentUsage: armadaresource.ComputeResources{}, - CumulativeUsage: armadaresource.ComputeResources{}, - } + podNameToUtilisationData[podName] = domain.EmptyUtilisationData() } for _, fetcher := range q.fetchers { diff --git a/internal/executor/utilisation/pod_utilisation_custom_metrics.go b/internal/executor/utilisation/pod_utilisation_custom_metrics.go index bf6102dc1f8..f5a335adb0b 100644 --- a/internal/executor/utilisation/pod_utilisation_custom_metrics.go +++ b/internal/executor/utilisation/pod_utilisation_custom_metrics.go @@ -65,7 +65,7 @@ func updateMetric(metricSamples model.Vector, metric configuration.CustomUsageMe if metric.Multiplier > 0 { val *= metric.Multiplier } - podData.CurrentUsage[metric.Name] = toQuantity(val) + podData.UpdateCurrentUsage(metric.Name, toQuantity(val)) } } } diff --git a/internal/executor/utilisation/pod_utilisation_custom_metrics_test.go b/internal/executor/utilisation/pod_utilisation_custom_metrics_test.go index dfd3da85448..0d3ce631331 100644 --- a/internal/executor/utilisation/pod_utilisation_custom_metrics_test.go +++ b/internal/executor/utilisation/pod_utilisation_custom_metrics_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/api/resource" - armadaresource "github.com/armadaproject/armada/internal/common/resource" "github.com/armadaproject/armada/internal/executor/configuration" "github.com/armadaproject/armada/internal/executor/domain" ) @@ -43,20 +42,18 @@ func TestUpdateMetrics(t *testing.T) { updateMetrics(samples, config, podNameToUtilisationData) expectedUtilisationData := map[string]*domain.UtilisationData{ - "pod1": { - CurrentUsage: armadaresource.ComputeResources{ + "pod1": domain.NewUtilisationData( + map[string]resource.Quantity{ "gpu": makeQuantity(1), "accelerator-memory-copy-util": makeQuantity(4), }, - CumulativeUsage: armadaresource.ComputeResources{}, - }, - "pod2": { - CurrentUsage: armadaresource.ComputeResources{ + ), + "pod2": domain.NewUtilisationData( + map[string]resource.Quantity{ "gpu": makeMilliQuantity(550), "accelerator-memory-copy-util": makeQuantity(5), }, - CumulativeUsage: armadaresource.ComputeResources{}, - }, + ), "pod3": domain.EmptyUtilisationData(), } diff --git a/internal/executor/utilisation/pod_utilisation_kubelet_metrics.go b/internal/executor/utilisation/pod_utilisation_kubelet_metrics.go index 4a7aef1ac11..6fe31369c37 100644 --- a/internal/executor/utilisation/pod_utilisation_kubelet_metrics.go +++ b/internal/executor/utilisation/pod_utilisation_kubelet_metrics.go @@ -54,15 +54,15 @@ func (m *podUtilisationKubeletMetrics) fetch(nodes []*v1.Node, podNameToUtilisat func updatePodStats(podStats *v1alpha1.PodStats, utilisationData *domain.UtilisationData) { if podStats.CPU != nil && podStats.CPU.UsageNanoCores != nil { - utilisationData.CurrentUsage["cpu"] = *resource.NewScaledQuantity(int64(*podStats.CPU.UsageNanoCores), -9) + utilisationData.UpdateCurrentUsage("cpu", *resource.NewScaledQuantity(int64(*podStats.CPU.UsageNanoCores), -9)) } if podStats.CPU != nil && podStats.CPU.UsageCoreNanoSeconds != nil { - utilisationData.CumulativeUsage["cpu"] = *resource.NewScaledQuantity(int64(*podStats.CPU.UsageCoreNanoSeconds), -9) + utilisationData.UpdateCurrentUsage("cpu", *resource.NewScaledQuantity(int64(*podStats.CPU.UsageCoreNanoSeconds), -9)) } if podStats.Memory != nil && podStats.Memory.WorkingSetBytes != nil { - utilisationData.CurrentUsage["memory"] = *resource.NewQuantity(int64(*podStats.Memory.WorkingSetBytes), resource.BinarySI) + utilisationData.UpdateCurrentUsage("memory", *resource.NewQuantity(int64(*podStats.Memory.WorkingSetBytes), resource.BinarySI)) } if podStats.EphemeralStorage != nil && podStats.EphemeralStorage.UsedBytes != nil { - utilisationData.CurrentUsage["ephemeral-storage"] = *resource.NewQuantity(int64(*podStats.EphemeralStorage.UsedBytes), resource.BinarySI) + utilisationData.UpdateCurrentUsage("ephemeral-storage", *resource.NewQuantity(int64(*podStats.EphemeralStorage.UsedBytes), resource.BinarySI)) } } diff --git a/pkg/armadaevents/events.pb.go b/pkg/armadaevents/events.pb.go index 9a306a32f64..f148a8a56bc 100644 --- a/pkg/armadaevents/events.pb.go +++ b/pkg/armadaevents/events.pb.go @@ -549,6 +549,7 @@ type ResourceUtilisation struct { TotalCumulativeUsage map[string]*resource.Quantity `protobuf:"bytes,5,rep,name=total_cumulative_usage,json=totalCumulativeUsage,proto3" json:"totalCumulativeUsage,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` JobId string `protobuf:"bytes,6,opt,name=job_id,json=jobId,proto3" json:"jobId,omitempty"` RunId string `protobuf:"bytes,7,opt,name=run_id,json=runId,proto3" json:"runId,omitempty"` + AvgResourcesForPeriod map[string]*resource.Quantity `protobuf:"bytes,8,rep,name=avg_resources_for_period,json=avgResourcesForPeriod,proto3" json:"avgResourcesForPeriod,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (m *ResourceUtilisation) Reset() { *m = ResourceUtilisation{} } @@ -619,6 +620,13 @@ func (m *ResourceUtilisation) GetRunId() string { return "" } +func (m *ResourceUtilisation) GetAvgResourcesForPeriod() map[string]*resource.Quantity { + if m != nil { + return m.AvgResourcesForPeriod + } + return nil +} + // A request to run an Armada job. Each job consists of a set of Kubernetes objects, // one of which is the main object (typically a pod spec.) and has a priority associated with it. // When the main object exits, all other objects are cleaned up. @@ -3644,6 +3652,7 @@ func init() { proto.RegisterType((*EventSequence)(nil), "armadaevents.EventSequence") proto.RegisterType((*EventSequence_Event)(nil), "armadaevents.EventSequence.Event") proto.RegisterType((*ResourceUtilisation)(nil), "armadaevents.ResourceUtilisation") + proto.RegisterMapType((map[string]*resource.Quantity)(nil), "armadaevents.ResourceUtilisation.AvgResourcesForPeriodEntry") proto.RegisterMapType((map[string]*resource.Quantity)(nil), "armadaevents.ResourceUtilisation.MaxResourcesForPeriodEntry") proto.RegisterMapType((map[string]*resource.Quantity)(nil), "armadaevents.ResourceUtilisation.TotalCumulativeUsageEntry") proto.RegisterType((*SubmitJob)(nil), "armadaevents.SubmitJob") @@ -3699,237 +3708,238 @@ func init() { func init() { proto.RegisterFile("pkg/armadaevents/events.proto", fileDescriptor_6aab92ca59e015f8) } var fileDescriptor_6aab92ca59e015f8 = []byte{ - // 3666 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x3b, 0x4d, 0x6f, 0x1c, 0xc7, - 0x95, 0xea, 0xf9, 0x9e, 0x37, 0xe4, 0x70, 0x54, 0xfc, 0xd0, 0x88, 0xb6, 0x38, 0xd4, 0xc8, 0xbb, - 0x96, 0x0c, 0x7b, 0x28, 0xcb, 0xeb, 0x85, 0x3f, 0x16, 0x36, 0x38, 0x12, 0x2d, 0x89, 0x12, 0x25, - 0x7a, 0x28, 0x79, 0xb5, 0x0b, 0x03, 0xe3, 0x9e, 0xe9, 0xe2, 0xb0, 0xc5, 0x99, 0xee, 0x76, 0x7f, - 0xd0, 0x24, 0x60, 0xec, 0xda, 0x0b, 0xef, 0x9e, 0x7d, 0x59, 0x20, 0xc8, 0x25, 0xbe, 0xe4, 0xe0, - 0x20, 0x39, 0x26, 0xd7, 0x5c, 0x73, 0x08, 0x02, 0x5f, 0x02, 0x04, 0x41, 0x3c, 0x08, 0x6c, 0xe4, - 0x32, 0x87, 0xfc, 0x86, 0xa0, 0x3e, 0xba, 0xbb, 0xaa, 0xa7, 0x46, 0x24, 0x65, 0x51, 0x70, 0x9c, - 0x13, 0xd9, 0xef, 0xbb, 0xeb, 0xbd, 0x7a, 0xfd, 0xde, 0xab, 0x1a, 0x38, 0xe7, 0xec, 0xf6, 0x56, - 0x74, 0x77, 0xa0, 0x1b, 0x3a, 0xde, 0xc3, 0x96, 0xef, 0xad, 0xb0, 0x3f, 0x0d, 0xc7, 0xb5, 0x7d, - 0x1b, 0x4d, 0x89, 0xa8, 0xc5, 0xfa, 0xee, 0x6b, 0x5e, 0xc3, 0xb4, 0x57, 0x74, 0xc7, 0x5c, 0xe9, - 0xda, 0x2e, 0x5e, 0xd9, 0x7b, 0x79, 0xa5, 0x87, 0x2d, 0xec, 0xea, 0x3e, 0x36, 0x18, 0xc7, 0xe2, - 0x45, 0x81, 0xc6, 0xc2, 0xfe, 0x47, 0xb6, 0xbb, 0x6b, 0x5a, 0x3d, 0x15, 0x65, 0xad, 0x67, 0xdb, - 0xbd, 0x3e, 0x5e, 0xa1, 0x4f, 0x9d, 0x60, 0x7b, 0xc5, 0x37, 0x07, 0xd8, 0xf3, 0xf5, 0x81, 0xc3, - 0x09, 0xfe, 0x25, 0x16, 0x35, 0xd0, 0xbb, 0x3b, 0xa6, 0x85, 0xdd, 0x83, 0x15, 0x6a, 0xaf, 0x63, - 0xae, 0xb8, 0xd8, 0xb3, 0x03, 0xb7, 0x8b, 0xc7, 0xc4, 0xbe, 0x61, 0x5a, 0x3e, 0x76, 0x2d, 0xbd, - 0xbf, 0xe2, 0x75, 0x77, 0xb0, 0x11, 0xf4, 0xb1, 0x1b, 0xff, 0x67, 0x77, 0x1e, 0xe2, 0xae, 0xef, - 0x8d, 0x01, 0x18, 0x6f, 0xfd, 0xeb, 0x79, 0x98, 0x5e, 0x23, 0xef, 0xba, 0x85, 0x3f, 0x0c, 0xb0, - 0xd5, 0xc5, 0xe8, 0x12, 0x64, 0x3f, 0x0c, 0x70, 0x80, 0xab, 0xda, 0xb2, 0x76, 0xb1, 0xd8, 0x9c, - 0x1d, 0x0d, 0x6b, 0x33, 0x14, 0xf0, 0xa2, 0x3d, 0x30, 0x7d, 0x3c, 0x70, 0xfc, 0x83, 0x16, 0xa3, - 0x40, 0x6f, 0xc0, 0xd4, 0x43, 0xbb, 0xd3, 0xf6, 0xb0, 0xdf, 0xb6, 0xf4, 0x01, 0xae, 0xa6, 0x28, - 0x47, 0x75, 0x34, 0xac, 0xcd, 0x3d, 0xb4, 0x3b, 0x5b, 0xd8, 0xbf, 0xa3, 0x0f, 0x44, 0x36, 0x88, - 0xa1, 0xe8, 0x25, 0xc8, 0x07, 0x1e, 0x76, 0xdb, 0xa6, 0x51, 0x4d, 0x53, 0xb6, 0xb9, 0xd1, 0xb0, - 0x56, 0x21, 0xa0, 0x9b, 0x86, 0xc0, 0x92, 0x63, 0x10, 0xf4, 0x22, 0xe4, 0x7a, 0xae, 0x1d, 0x38, - 0x5e, 0x35, 0xb3, 0x9c, 0x0e, 0xa9, 0x19, 0x44, 0xa4, 0x66, 0x10, 0x74, 0x17, 0x72, 0xcc, 0x81, - 0xd5, 0xec, 0x72, 0xfa, 0x62, 0xe9, 0xca, 0xf9, 0x86, 0xe8, 0xd5, 0x86, 0xf4, 0xc2, 0xec, 0x89, - 0x09, 0x64, 0x78, 0x51, 0x20, 0x8f, 0x83, 0x5f, 0xcd, 0x42, 0x96, 0xd2, 0xa1, 0x5b, 0x90, 0xef, - 0xba, 0x98, 0xac, 0x7e, 0x15, 0x2d, 0x6b, 0x17, 0x4b, 0x57, 0x16, 0x1b, 0xcc, 0xab, 0x8d, 0xd0, - 0xab, 0x8d, 0x7b, 0xa1, 0x57, 0x9b, 0xf3, 0xa3, 0x61, 0xed, 0x34, 0x27, 0x17, 0xa4, 0x86, 0x12, - 0xd0, 0x26, 0x14, 0xbd, 0xa0, 0x33, 0x30, 0xfd, 0x75, 0xbb, 0x43, 0xd7, 0xbb, 0x74, 0xe5, 0x8c, - 0x6c, 0xea, 0x56, 0x88, 0x6e, 0x9e, 0x19, 0x0d, 0x6b, 0xb3, 0x11, 0x75, 0x2c, 0xed, 0xc6, 0xa9, - 0x56, 0x2c, 0x04, 0xed, 0xc0, 0x8c, 0x8b, 0x1d, 0xd7, 0xb4, 0x5d, 0xd3, 0x37, 0x3d, 0x4c, 0xe4, - 0xa6, 0xa8, 0xdc, 0x73, 0xb2, 0xdc, 0x96, 0x4c, 0xd4, 0x3c, 0x37, 0x1a, 0xd6, 0xce, 0x26, 0x38, - 0x25, 0x1d, 0x49, 0xb1, 0xc8, 0x07, 0x94, 0x00, 0x6d, 0x61, 0x9f, 0xfa, 0xb2, 0x74, 0x65, 0xf9, - 0x91, 0xca, 0xb6, 0xb0, 0xdf, 0x5c, 0x1e, 0x0d, 0x6b, 0xcf, 0x8e, 0xf3, 0x4b, 0x2a, 0x15, 0xf2, - 0x51, 0x1f, 0x2a, 0x22, 0xd4, 0x20, 0x2f, 0x98, 0xa1, 0x3a, 0x97, 0x26, 0xeb, 0x24, 0x54, 0xcd, - 0xa5, 0xd1, 0xb0, 0xb6, 0x98, 0xe4, 0x95, 0xf4, 0x8d, 0x49, 0x26, 0xfe, 0xe9, 0xea, 0x56, 0x17, - 0xf7, 0x89, 0x9a, 0xac, 0xca, 0x3f, 0x57, 0x43, 0x34, 0xf3, 0x4f, 0x44, 0x2d, 0xfb, 0x27, 0x02, - 0xa3, 0xf7, 0x61, 0x2a, 0x7a, 0x20, 0xeb, 0x95, 0xe3, 0x31, 0xa4, 0x16, 0x4a, 0x56, 0x6a, 0x71, - 0x34, 0xac, 0x2d, 0x88, 0x3c, 0x92, 0x68, 0x49, 0x5a, 0x2c, 0xbd, 0xcf, 0x56, 0x26, 0x3f, 0x59, - 0x3a, 0xa3, 0x10, 0xa5, 0xf7, 0xc7, 0x57, 0x44, 0x92, 0x46, 0xa4, 0x93, 0x0d, 0x1c, 0x74, 0xbb, - 0x18, 0x1b, 0xd8, 0xa8, 0x16, 0x54, 0xd2, 0xd7, 0x05, 0x0a, 0x26, 0x5d, 0xe4, 0x91, 0xa5, 0x8b, - 0x18, 0xb2, 0xd6, 0x0f, 0xed, 0xce, 0x9a, 0xeb, 0xda, 0xae, 0x57, 0x2d, 0xaa, 0xd6, 0x7a, 0x3d, - 0x44, 0xb3, 0xb5, 0x8e, 0xa8, 0xe5, 0xb5, 0x8e, 0xc0, 0xdc, 0xde, 0x56, 0x60, 0xdd, 0xc6, 0xba, - 0x87, 0x8d, 0x2a, 0x4c, 0xb0, 0x37, 0xa2, 0x88, 0xec, 0x8d, 0x20, 0x63, 0xf6, 0x46, 0x18, 0x64, - 0x40, 0x99, 0x3d, 0xaf, 0x7a, 0x9e, 0xd9, 0xb3, 0xb0, 0x51, 0x2d, 0x51, 0xf9, 0xcf, 0xaa, 0xe4, - 0x87, 0x34, 0xcd, 0x67, 0x47, 0xc3, 0x5a, 0x55, 0xe6, 0x93, 0x74, 0x24, 0x64, 0xa2, 0x0f, 0x60, - 0x9a, 0x41, 0x5a, 0x81, 0x65, 0x99, 0x56, 0xaf, 0x3a, 0x45, 0x95, 0x3c, 0xa3, 0x52, 0xc2, 0x49, - 0x9a, 0xcf, 0x8c, 0x86, 0xb5, 0x33, 0x12, 0x97, 0xa4, 0x42, 0x16, 0x48, 0x32, 0x06, 0x03, 0xc4, - 0x8e, 0x9d, 0x56, 0x65, 0x8c, 0x75, 0x99, 0x88, 0x65, 0x8c, 0x04, 0xa7, 0x9c, 0x31, 0x12, 0xc8, - 0xd8, 0x1f, 0xdc, 0xc9, 0xe5, 0xc9, 0xfe, 0xe0, 0x7e, 0x16, 0xfc, 0xa1, 0x70, 0xb5, 0x24, 0x0d, - 0x7d, 0xa2, 0xc1, 0xbc, 0xe7, 0xeb, 0x96, 0xa1, 0xf7, 0x6d, 0x0b, 0xdf, 0xb4, 0x7a, 0x2e, 0xf6, - 0xbc, 0x9b, 0xd6, 0xb6, 0x5d, 0xad, 0x50, 0x3d, 0x17, 0x12, 0x89, 0x55, 0x45, 0xda, 0xbc, 0x30, - 0x1a, 0xd6, 0x6a, 0x4a, 0x29, 0x92, 0x66, 0xb5, 0x22, 0xb4, 0x0f, 0xb3, 0xe1, 0x47, 0xfa, 0xbe, - 0x6f, 0xf6, 0x4d, 0x4f, 0xf7, 0x4d, 0xdb, 0xaa, 0x9e, 0xa6, 0xfa, 0xcf, 0x27, 0xf3, 0xd3, 0x18, - 0x61, 0xf3, 0xfc, 0x68, 0x58, 0x3b, 0xa7, 0x90, 0x20, 0xe9, 0x56, 0xa9, 0x88, 0x9d, 0xb8, 0xe9, - 0x62, 0x42, 0x88, 0x8d, 0xea, 0xec, 0x64, 0x27, 0x46, 0x44, 0xa2, 0x13, 0x23, 0xa0, 0xca, 0x89, - 0x11, 0x92, 0x68, 0x72, 0x74, 0xd7, 0x37, 0x89, 0xda, 0x0d, 0xdd, 0xdd, 0xc5, 0x6e, 0x75, 0x4e, - 0xa5, 0x69, 0x53, 0x26, 0x62, 0x9a, 0x12, 0x9c, 0xb2, 0xa6, 0x04, 0x12, 0x7d, 0xae, 0x81, 0x6c, - 0x9a, 0x69, 0x5b, 0x2d, 0xf2, 0xd1, 0xf6, 0xc8, 0xeb, 0xcd, 0x53, 0xa5, 0xcf, 0x3f, 0xe2, 0xf5, - 0x44, 0xf2, 0xe6, 0xf3, 0xa3, 0x61, 0xed, 0xc2, 0x44, 0x69, 0x92, 0x21, 0x93, 0x95, 0xa2, 0x07, - 0x50, 0x22, 0x48, 0x4c, 0xcb, 0x1f, 0xa3, 0xba, 0x40, 0x6d, 0x38, 0x3b, 0x6e, 0x03, 0x27, 0x68, - 0x9e, 0x1d, 0x0d, 0x6b, 0xf3, 0x02, 0x87, 0xa4, 0x47, 0x14, 0x85, 0x3e, 0xd3, 0x80, 0x04, 0xba, - 0xea, 0x4d, 0xcf, 0x50, 0x2d, 0xcf, 0x8d, 0x69, 0x51, 0xbd, 0xe6, 0x73, 0xa3, 0x61, 0x6d, 0x59, - 0x2d, 0x47, 0xd2, 0x3d, 0x41, 0x57, 0x1c, 0x47, 0xd1, 0x47, 0xa2, 0x5a, 0x9d, 0x1c, 0x47, 0x11, - 0x91, 0x18, 0x47, 0x11, 0x50, 0x15, 0x47, 0x11, 0x92, 0x27, 0x83, 0xf7, 0xf4, 0xbe, 0x69, 0xd0, - 0x62, 0xea, 0xec, 0x84, 0x64, 0x10, 0x51, 0x44, 0xc9, 0x20, 0x82, 0x8c, 0x25, 0x83, 0x98, 0x36, - 0x0f, 0x59, 0x2a, 0xa2, 0xfe, 0xc7, 0x1c, 0xcc, 0x2a, 0xb6, 0x1a, 0xc2, 0x30, 0x1d, 0xee, 0xa3, - 0xb6, 0x49, 0x92, 0x44, 0x5a, 0xb5, 0xca, 0xb7, 0x82, 0x0e, 0x76, 0x2d, 0xec, 0x63, 0x2f, 0x94, - 0x41, 0xb3, 0x04, 0xb5, 0xc4, 0x15, 0x20, 0x42, 0x6d, 0x37, 0x25, 0xc2, 0xd1, 0x8f, 0x35, 0xa8, - 0x0e, 0xf4, 0xfd, 0x76, 0x08, 0xf4, 0xda, 0xdb, 0xb6, 0xdb, 0x76, 0xb0, 0x6b, 0xda, 0x06, 0xad, - 0x64, 0x4b, 0x57, 0xfe, 0xed, 0xd0, 0xbc, 0xd0, 0xd8, 0xd0, 0xf7, 0x43, 0xb0, 0xf7, 0x8e, 0xed, - 0x6e, 0x52, 0xf6, 0x35, 0xcb, 0x77, 0x0f, 0x58, 0xc2, 0x1a, 0xa8, 0xf0, 0x82, 0x4d, 0xf3, 0x4a, - 0x02, 0xf4, 0xff, 0x1a, 0x2c, 0xf8, 0xb6, 0xaf, 0xf7, 0xdb, 0xdd, 0x60, 0x10, 0xf4, 0x75, 0xdf, - 0xdc, 0xc3, 0xed, 0xc0, 0xd3, 0x7b, 0x98, 0x97, 0xcd, 0x6f, 0x1e, 0x6e, 0xda, 0x3d, 0xc2, 0x7f, - 0x35, 0x62, 0xbf, 0x4f, 0xb8, 0x99, 0x65, 0xf5, 0xd1, 0xb0, 0xb6, 0xe4, 0x2b, 0xd0, 0x82, 0x61, - 0x73, 0x2a, 0x3c, 0x7a, 0x01, 0x72, 0xa4, 0xad, 0x30, 0x0d, 0x5a, 0x1d, 0xf1, 0x16, 0xe4, 0xa1, - 0xdd, 0x91, 0x1a, 0x83, 0x2c, 0x05, 0x10, 0x5a, 0x37, 0xb0, 0x08, 0x6d, 0x3e, 0xa6, 0x75, 0x03, - 0x4b, 0xa6, 0xa5, 0x80, 0xc5, 0x2f, 0x34, 0x58, 0x9c, 0xbc, 0x94, 0xe8, 0x02, 0xa4, 0x77, 0xf1, - 0x01, 0x6f, 0x7b, 0x4e, 0x8f, 0x86, 0xb5, 0xe9, 0x5d, 0x7c, 0x20, 0x48, 0x21, 0x58, 0xf4, 0x1f, - 0x90, 0xdd, 0xd3, 0xfb, 0x01, 0xe6, 0x55, 0x75, 0xa3, 0xc1, 0x3a, 0xb6, 0x86, 0xd8, 0xb1, 0x35, - 0x9c, 0xdd, 0x1e, 0x01, 0x34, 0x42, 0xaf, 0x37, 0xde, 0x0d, 0x74, 0xcb, 0x37, 0xfd, 0x03, 0x66, - 0x1e, 0x15, 0x20, 0x9a, 0x47, 0x01, 0x6f, 0xa4, 0x5e, 0xd3, 0x16, 0x7f, 0xa2, 0xc1, 0xd9, 0x89, - 0x4b, 0xfa, 0x7d, 0xb0, 0x70, 0x3d, 0x53, 0xd0, 0x2a, 0xa9, 0xf5, 0x4c, 0x21, 0x55, 0x49, 0xd7, - 0x7f, 0x9e, 0x87, 0x62, 0xd4, 0xa0, 0xa0, 0x1b, 0x50, 0x31, 0xb0, 0x11, 0x38, 0x7d, 0xb3, 0x4b, - 0x63, 0x83, 0x38, 0x85, 0x75, 0x84, 0x34, 0x3b, 0x48, 0x38, 0xc9, 0x3d, 0x33, 0x09, 0x14, 0xba, - 0x02, 0x05, 0x5e, 0x88, 0x1f, 0xd0, 0x7d, 0x39, 0xdd, 0x5c, 0x18, 0x0d, 0x6b, 0x28, 0x84, 0x09, - 0xac, 0x11, 0x1d, 0x6a, 0x01, 0xb0, 0xce, 0x76, 0x03, 0xfb, 0x3a, 0x6f, 0x09, 0xaa, 0x72, 0xfc, - 0xde, 0x8d, 0xf0, 0xac, 0x47, 0x8d, 0xe9, 0xc5, 0x1e, 0x35, 0x86, 0xa2, 0xf7, 0x01, 0x06, 0xba, - 0x69, 0x31, 0x3e, 0x5e, 0xff, 0xd7, 0x27, 0x65, 0x88, 0x8d, 0x88, 0x92, 0x49, 0x8f, 0x39, 0x45, - 0xe9, 0x31, 0x14, 0xdd, 0x85, 0x3c, 0xef, 0xc5, 0xab, 0x39, 0xba, 0xdd, 0x96, 0x26, 0x89, 0xe6, - 0x62, 0x69, 0x37, 0xc9, 0x59, 0xc4, 0x6e, 0x92, 0x83, 0xc8, 0xb2, 0xf5, 0xcd, 0x6d, 0xec, 0x9b, - 0x03, 0x4c, 0x77, 0x03, 0x5f, 0xb6, 0x10, 0x26, 0x2e, 0x5b, 0x08, 0x43, 0xaf, 0x01, 0xe8, 0xfe, - 0x86, 0xed, 0xf9, 0x77, 0xad, 0x2e, 0xa6, 0x15, 0x7d, 0x81, 0x99, 0x1f, 0x43, 0x45, 0xf3, 0x63, - 0x28, 0x7a, 0x13, 0x4a, 0x0e, 0xff, 0x82, 0x74, 0xfa, 0x98, 0x56, 0xec, 0x05, 0xf6, 0xc1, 0x13, - 0xc0, 0x02, 0xaf, 0x48, 0x8d, 0xae, 0xc3, 0x4c, 0xd7, 0xb6, 0xba, 0x81, 0xeb, 0x62, 0xab, 0x7b, - 0xb0, 0xa5, 0x6f, 0x63, 0x5a, 0x9d, 0x17, 0x58, 0xa8, 0x24, 0x50, 0x62, 0xa8, 0x24, 0x50, 0xe8, - 0x55, 0x28, 0x46, 0x93, 0x0d, 0x5a, 0x80, 0x17, 0x79, 0xa3, 0x1c, 0x02, 0x05, 0xe6, 0x98, 0x92, - 0x18, 0x6f, 0x7a, 0xd7, 0x78, 0xd0, 0x61, 0x5a, 0x54, 0x73, 0xe3, 0x05, 0xb0, 0x68, 0xbc, 0x00, - 0x16, 0xf2, 0x53, 0xf9, 0xd0, 0xfc, 0xf4, 0x5f, 0x30, 0x8f, 0xf7, 0x49, 0xbe, 0x1f, 0x60, 0xcb, - 0xd7, 0xfb, 0x9b, 0xae, 0xc9, 0xbe, 0x0c, 0xd5, 0x19, 0x55, 0x51, 0xba, 0xa6, 0x22, 0x65, 0x39, - 0x5e, 0x29, 0x45, 0xcc, 0xf1, 0x4a, 0x82, 0x68, 0xbb, 0x4e, 0x57, 0xca, 0xf5, 0x5b, 0x30, 0xaf, - 0x54, 0x40, 0x02, 0xa7, 0x63, 0x1a, 0xf4, 0x99, 0x26, 0x17, 0x8d, 0x05, 0x4e, 0x08, 0x13, 0x03, - 0x27, 0x84, 0xd5, 0x7f, 0xab, 0xc1, 0x9c, 0x2a, 0xf8, 0x13, 0x1b, 0x51, 0x7b, 0x22, 0x1b, 0xf1, - 0x3d, 0x28, 0x38, 0xb6, 0xd1, 0xf6, 0x1c, 0xdc, 0xe5, 0x69, 0x2d, 0xb1, 0x0d, 0x37, 0x6d, 0x63, - 0xcb, 0xc1, 0xdd, 0x7f, 0x37, 0xfd, 0x9d, 0xd5, 0x3d, 0xdb, 0x34, 0x6e, 0x9b, 0x1e, 0xdf, 0x2f, - 0x0e, 0xc3, 0x48, 0xb5, 0x42, 0x9e, 0x03, 0x9b, 0x05, 0xc8, 0x31, 0x2d, 0xf5, 0xdf, 0xa5, 0xa1, - 0x92, 0xdc, 0x70, 0x7f, 0x4f, 0xaf, 0x82, 0x1e, 0x40, 0xde, 0x64, 0xad, 0x08, 0x2f, 0x65, 0xfe, - 0x49, 0x48, 0xfc, 0x8d, 0x78, 0x2e, 0xd9, 0xd8, 0x7b, 0xb9, 0xc1, 0x7b, 0x16, 0xba, 0x04, 0x54, - 0x32, 0xe7, 0x94, 0x25, 0x73, 0x20, 0x6a, 0x41, 0xde, 0xc3, 0xee, 0x1e, 0x09, 0x0e, 0x96, 0x56, - 0x6b, 0xa2, 0xe4, 0xae, 0xed, 0x62, 0x22, 0x73, 0x8b, 0x91, 0xc4, 0x32, 0x39, 0x8f, 0x2c, 0x93, - 0x03, 0xd1, 0x7b, 0x50, 0xec, 0xda, 0xd6, 0xb6, 0xd9, 0xdb, 0xd0, 0x1d, 0x9e, 0x58, 0xcf, 0xa9, - 0xa4, 0x5e, 0x0d, 0x89, 0xf8, 0x78, 0x25, 0x7c, 0x4c, 0x8c, 0x57, 0x22, 0xaa, 0xd8, 0xa1, 0x7f, - 0xcd, 0x00, 0xc4, 0xce, 0x41, 0xaf, 0x43, 0x09, 0xef, 0xe3, 0x6e, 0xe0, 0xdb, 0x74, 0xe4, 0xa8, - 0xc5, 0x93, 0xca, 0x10, 0x2c, 0xed, 0x5e, 0x88, 0xa1, 0x24, 0xc5, 0x58, 0xfa, 0x00, 0x7b, 0x8e, - 0xde, 0x0d, 0x47, 0x9c, 0xd4, 0x98, 0x08, 0x28, 0xa6, 0x98, 0x08, 0x88, 0xfe, 0x19, 0x32, 0x74, - 0x28, 0xca, 0xa6, 0x9b, 0x68, 0x34, 0xac, 0x95, 0x2d, 0x79, 0x1c, 0x4a, 0xf1, 0xe8, 0x6d, 0x98, - 0xde, 0x8d, 0x02, 0x8f, 0xd8, 0x96, 0xa1, 0x0c, 0xb4, 0xc6, 0x8c, 0x11, 0x92, 0x75, 0x53, 0x22, - 0x1c, 0x6d, 0x43, 0x49, 0xb7, 0x2c, 0xdb, 0xa7, 0x5f, 0xcf, 0x70, 0xe2, 0x79, 0x69, 0x52, 0x98, - 0x36, 0x56, 0x63, 0x5a, 0x56, 0xa8, 0xd1, 0xb4, 0x27, 0x48, 0x10, 0xd3, 0x9e, 0x00, 0x46, 0x2d, - 0xc8, 0xf5, 0xf5, 0x0e, 0xee, 0x87, 0x9f, 0xab, 0xe7, 0x26, 0xaa, 0xb8, 0x4d, 0xc9, 0x98, 0x74, - 0x3a, 0x57, 0x65, 0x7c, 0xe2, 0x5c, 0x95, 0x41, 0x16, 0xb7, 0xa1, 0x92, 0xb4, 0xe7, 0x68, 0x55, - 0xce, 0x25, 0xb1, 0xca, 0x29, 0x1e, 0x5a, 0x57, 0xe9, 0x50, 0x12, 0x8c, 0x3a, 0x09, 0x15, 0xf5, - 0x2f, 0x35, 0x98, 0x53, 0xed, 0x5d, 0xb4, 0x21, 0xec, 0x78, 0x8d, 0x4f, 0x6f, 0x14, 0xa1, 0xce, - 0x79, 0x27, 0x6c, 0xf5, 0x78, 0xa3, 0x37, 0xa1, 0x6c, 0xd9, 0x06, 0x6e, 0xeb, 0x44, 0x41, 0xdf, - 0xf4, 0xfc, 0x6a, 0x8a, 0x4e, 0xc4, 0xe9, 0xd4, 0x87, 0x60, 0x56, 0x43, 0x84, 0xc0, 0x3d, 0x2d, - 0x21, 0xea, 0x1f, 0xc1, 0x4c, 0x62, 0x26, 0x2b, 0xd5, 0x5c, 0xa9, 0x23, 0xd6, 0x5c, 0xf1, 0x87, - 0x30, 0x7d, 0xd8, 0x87, 0x90, 0x7d, 0x88, 0xea, 0xff, 0x9b, 0x82, 0x92, 0xd0, 0x20, 0xa3, 0x87, - 0x30, 0xc3, 0x3f, 0xca, 0xa6, 0xd5, 0x63, 0x8d, 0x58, 0x8a, 0x7f, 0x18, 0xc7, 0x0e, 0x2c, 0xd6, - 0xed, 0xce, 0x56, 0x44, 0x4b, 0x3f, 0x8c, 0x74, 0x98, 0xe6, 0x49, 0x30, 0x41, 0x71, 0x59, 0xc6, - 0xa0, 0x07, 0xb0, 0x10, 0x38, 0xa4, 0x3d, 0x6c, 0x7b, 0x7c, 0xf4, 0xdf, 0xb6, 0x82, 0x41, 0x07, - 0xbb, 0xd4, 0xfa, 0x2c, 0x6b, 0x58, 0x18, 0x45, 0x78, 0x36, 0x70, 0x87, 0xe2, 0xc5, 0x86, 0x45, - 0x85, 0x17, 0xd6, 0x21, 0x73, 0xc4, 0x75, 0xb8, 0x01, 0x68, 0x7c, 0x28, 0x2e, 0xf9, 0x40, 0x3b, - 0x9a, 0x0f, 0xea, 0xfb, 0x50, 0x49, 0x8e, 0xba, 0x9f, 0x92, 0x2f, 0x77, 0xa1, 0x18, 0x0d, 0xaa, - 0xd1, 0x8b, 0x90, 0x73, 0xb1, 0xee, 0xd9, 0x16, 0xdf, 0x2d, 0x74, 0xdb, 0x33, 0x88, 0xb8, 0xed, - 0x19, 0xe4, 0x31, 0x94, 0xdd, 0x83, 0x29, 0xb6, 0x48, 0xef, 0x98, 0x7d, 0x1f, 0xbb, 0xe8, 0x1a, - 0xe4, 0x3c, 0x5f, 0xf7, 0xb1, 0x57, 0xd5, 0x96, 0xd3, 0x17, 0xcb, 0x57, 0x16, 0xc6, 0xa7, 0xd0, - 0x04, 0xcd, 0xec, 0x60, 0x94, 0xa2, 0x1d, 0x0c, 0x52, 0xff, 0x1f, 0x0d, 0xa6, 0xc4, 0x61, 0xfb, - 0x93, 0x11, 0x7b, 0xbc, 0xc5, 0xa8, 0x5b, 0xa1, 0x0d, 0x7c, 0xcc, 0x7e, 0xd2, 0x4b, 0xf9, 0xa5, - 0xc6, 0xd6, 0x32, 0x9a, 0xcb, 0xf6, 0xe2, 0x59, 0x08, 0xd9, 0x28, 0x1e, 0x4d, 0x28, 0x47, 0x9d, - 0x85, 0xd0, 0xb4, 0x23, 0xb1, 0x8b, 0x69, 0x47, 0x42, 0x3c, 0x86, 0xad, 0x5f, 0x64, 0xa9, 0xad, - 0xf1, 0xd4, 0x3d, 0xf1, 0x1d, 0x4f, 0x1f, 0xe3, 0x3b, 0xfe, 0x12, 0xe4, 0x69, 0xe2, 0x8c, 0xb6, - 0x29, 0x5d, 0x58, 0x02, 0x92, 0x4f, 0x1c, 0x19, 0xe4, 0x11, 0xe9, 0x22, 0xfb, 0x1d, 0xd3, 0x45, - 0x1b, 0xce, 0xee, 0xe8, 0x5e, 0x3b, 0x4c, 0x70, 0x46, 0x5b, 0xf7, 0xdb, 0xd1, 0x7e, 0xcd, 0xd1, - 0x56, 0x84, 0xce, 0xf1, 0x76, 0x74, 0x6f, 0x2b, 0xa4, 0x59, 0xf5, 0x37, 0xc7, 0x77, 0xef, 0x82, - 0x9a, 0x02, 0xdd, 0x87, 0x79, 0xb5, 0xf0, 0x3c, 0xb5, 0x9c, 0x8e, 0x99, 0xbd, 0x47, 0x4a, 0x9e, - 0x55, 0xa0, 0xd1, 0xa7, 0x1a, 0x54, 0xc9, 0x97, 0xcc, 0xc5, 0x1f, 0x06, 0xa6, 0x8b, 0x49, 0x17, - 0xe1, 0xb5, 0xed, 0x3d, 0xec, 0xf6, 0xf5, 0x03, 0x7e, 0x62, 0x73, 0x7e, 0x3c, 0x6d, 0x6f, 0xda, - 0x46, 0x4b, 0x60, 0x60, 0xaf, 0xe6, 0xc8, 0xc0, 0xbb, 0x4c, 0x88, 0xf8, 0x6a, 0x6a, 0x0a, 0x21, - 0x84, 0xe0, 0x18, 0xb3, 0xa1, 0xd2, 0x61, 0xb3, 0x21, 0x52, 0xad, 0x39, 0xb6, 0xdd, 0xa7, 0x9d, - 0x20, 0xaf, 0xd6, 0xc8, 0xb3, 0x58, 0xad, 0x91, 0x67, 0x71, 0xfc, 0xb1, 0x9e, 0x29, 0x14, 0x2a, - 0xc5, 0xfa, 0xd7, 0x1a, 0x94, 0xe5, 0x43, 0x9e, 0xf1, 0x0d, 0x95, 0x3e, 0xf1, 0x0d, 0x95, 0x39, - 0xc6, 0x6a, 0x64, 0x0f, 0x5b, 0x0d, 0x69, 0xc8, 0xf3, 0x27, 0x0d, 0xa6, 0xa5, 0xf3, 0xa5, 0x1f, - 0xd6, 0xeb, 0xfd, 0x28, 0x05, 0x0b, 0x6a, 0x53, 0x4f, 0xa4, 0xfd, 0xbb, 0x01, 0xa4, 0x90, 0xbb, - 0x19, 0x17, 0x3a, 0xf3, 0x63, 0xdd, 0x1f, 0x5d, 0xa6, 0xb0, 0x0a, 0x1c, 0x3b, 0x7a, 0x0a, 0xd9, - 0xd1, 0x03, 0x28, 0x99, 0xc2, 0x21, 0x57, 0x5a, 0x75, 0x16, 0x21, 0x1e, 0x6d, 0xb1, 0xe9, 0xc6, - 0x84, 0x03, 0x2d, 0x51, 0x54, 0x33, 0x07, 0x19, 0x52, 0x89, 0xd5, 0xf7, 0x20, 0xcf, 0xcd, 0x41, - 0xaf, 0x40, 0x91, 0xe6, 0x4e, 0xda, 0xd1, 0xb0, 0xb2, 0x99, 0x96, 0x14, 0x04, 0x98, 0xb8, 0xe4, - 0x51, 0x08, 0x61, 0xe8, 0x5f, 0x01, 0x48, 0xba, 0xe0, 0x59, 0x33, 0x45, 0x73, 0x0f, 0xed, 0x9c, - 0x1c, 0xdb, 0x18, 0x4b, 0x95, 0xc5, 0x08, 0x58, 0xff, 0x45, 0x0a, 0x4a, 0xe2, 0xb1, 0xda, 0x63, - 0x29, 0xff, 0x18, 0xc2, 0xae, 0xb6, 0xad, 0x1b, 0x06, 0xf9, 0x8b, 0xc3, 0x0f, 0xdb, 0xca, 0xc4, - 0x45, 0x0a, 0xff, 0x5f, 0x0d, 0x39, 0x58, 0x0f, 0x43, 0xaf, 0x0e, 0x98, 0x09, 0x94, 0xa0, 0xb5, - 0x92, 0xc4, 0x2d, 0xee, 0xc2, 0xbc, 0x52, 0x94, 0xd8, 0x79, 0x64, 0x9f, 0x54, 0xe7, 0xf1, 0xd3, - 0x2c, 0xcc, 0x2b, 0x8f, 0x33, 0x13, 0x11, 0x9c, 0x7e, 0x22, 0x11, 0xfc, 0x7f, 0x9a, 0x6a, 0x65, - 0xd9, 0x59, 0xc6, 0xeb, 0x47, 0x38, 0x63, 0x7d, 0x52, 0x6b, 0x2c, 0x87, 0x45, 0xf6, 0xb1, 0x62, - 0x32, 0x77, 0xd4, 0x98, 0x44, 0x97, 0x59, 0x13, 0x47, 0x75, 0xb1, 0x93, 0x86, 0x70, 0x87, 0x26, - 0x54, 0xe5, 0x39, 0x88, 0xf4, 0xf5, 0x21, 0x07, 0x1b, 0x1d, 0x14, 0xe2, 0xbe, 0x9e, 0xd3, 0x24, - 0xa7, 0x07, 0x53, 0x22, 0x5c, 0xc8, 0x7e, 0xc5, 0x63, 0x64, 0x3f, 0x38, 0xf4, 0x18, 0xe4, 0x69, - 0xc6, 0xa6, 0x94, 0x6a, 0x87, 0x1a, 0xcc, 0x24, 0x6e, 0x11, 0xfc, 0xb0, 0xbe, 0x25, 0x9f, 0x68, - 0x50, 0x8c, 0x2e, 0xa9, 0xa0, 0x55, 0xc8, 0x61, 0x76, 0xd1, 0x81, 0xa5, 0x9d, 0xd9, 0xc4, 0xac, - 0x97, 0xe0, 0xf8, 0xb5, 0xb3, 0xc4, 0xdd, 0x86, 0x16, 0x67, 0x7c, 0x8c, 0x82, 0xf9, 0x97, 0x5a, - 0x58, 0x30, 0x8f, 0x59, 0x91, 0xfe, 0xee, 0x56, 0x9c, 0xdc, 0xd2, 0xfd, 0xba, 0x08, 0x59, 0x6a, - 0x0b, 0x69, 0x5e, 0x7d, 0xec, 0x0e, 0x4c, 0x4b, 0xef, 0xd3, 0x50, 0x2c, 0xb0, 0x5d, 0x1d, 0xc2, - 0xc4, 0x5d, 0x1d, 0xc2, 0xd0, 0x0e, 0xcc, 0xc4, 0x23, 0x31, 0x2a, 0x46, 0x7d, 0xeb, 0xed, 0x96, - 0x4c, 0xc4, 0x4e, 0x1b, 0x12, 0x9c, 0xf2, 0xb1, 0x75, 0x02, 0x89, 0x0c, 0x28, 0x77, 0x6d, 0xcb, - 0xd7, 0x4d, 0x0b, 0xbb, 0x4c, 0x51, 0x5a, 0x75, 0xeb, 0xe7, 0xaa, 0x44, 0xc3, 0x06, 0x15, 0x32, - 0x9f, 0x7c, 0xeb, 0x47, 0xc6, 0xa1, 0x0f, 0x60, 0x3a, 0x6c, 0x5c, 0x98, 0x92, 0x8c, 0xea, 0xd6, - 0xcf, 0x9a, 0x48, 0xc2, 0x36, 0x83, 0xc4, 0x25, 0xdf, 0xfa, 0x91, 0x50, 0xa8, 0x0f, 0x15, 0xc7, - 0x36, 0xee, 0x5b, 0xbc, 0x5c, 0xd7, 0x3b, 0x7d, 0xcc, 0xe7, 0xb0, 0x4b, 0x63, 0x05, 0x89, 0x44, - 0xc5, 0x12, 0x75, 0x92, 0x57, 0xbe, 0x47, 0x97, 0xc4, 0xa2, 0xf7, 0x61, 0xaa, 0x4f, 0xfa, 0xb7, - 0xb5, 0x7d, 0xc7, 0x74, 0xb1, 0xa1, 0xbe, 0xf5, 0x76, 0x5b, 0xa0, 0x60, 0x69, 0x52, 0xe4, 0x91, - 0x0f, 0xfb, 0x45, 0x0c, 0xf1, 0xfe, 0x40, 0xdf, 0x6f, 0x05, 0x96, 0xb7, 0xb6, 0xcf, 0x6f, 0x30, - 0xe5, 0x55, 0xde, 0xdf, 0x90, 0x89, 0x98, 0xf7, 0x13, 0x9c, 0xb2, 0xf7, 0x13, 0x48, 0x74, 0x9b, - 0x7e, 0x05, 0x98, 0x4b, 0xd8, 0xed, 0xb7, 0x85, 0xb1, 0xd5, 0x62, 0xde, 0x60, 0x03, 0x17, 0xfe, - 0x24, 0x09, 0x8d, 0x24, 0x70, 0x1f, 0xd0, 0xd7, 0x6e, 0x61, 0x3f, 0x70, 0x2d, 0x6c, 0xf0, 0x36, - 0x6a, 0xdc, 0x07, 0x12, 0x55, 0xe4, 0x03, 0x09, 0x3a, 0xe6, 0x03, 0x09, 0x8b, 0x3e, 0x86, 0xb9, - 0xc4, 0x5d, 0x1e, 0xf6, 0x1e, 0x25, 0xd5, 0x21, 0xc4, 0xba, 0x82, 0x92, 0x75, 0xbc, 0x2a, 0x19, - 0x92, 0x66, 0xa5, 0x16, 0xa2, 0xbd, 0xa7, 0x5b, 0xbd, 0x75, 0xbb, 0x23, 0xc7, 0xdc, 0x94, 0x4a, - 0xfb, 0x75, 0x05, 0x25, 0xd3, 0xae, 0x92, 0x21, 0x6b, 0x57, 0x51, 0x44, 0xf7, 0x76, 0x48, 0x11, - 0x13, 0xdd, 0x6f, 0x53, 0xdd, 0xdb, 0x61, 0x04, 0xc2, 0xbd, 0x1d, 0x06, 0x50, 0xdc, 0xdb, 0xe1, - 0x94, 0x85, 0x70, 0x58, 0x53, 0x7f, 0x17, 0x66, 0x12, 0xe9, 0x05, 0xbd, 0x05, 0xd1, 0x6d, 0x90, - 0x7b, 0x07, 0x4e, 0x58, 0xbb, 0x4a, 0xb7, 0x47, 0x08, 0x5c, 0x75, 0x7b, 0x84, 0xc0, 0xeb, 0x9f, - 0x67, 0xa0, 0x10, 0x46, 0xd4, 0x89, 0x74, 0x23, 0x2b, 0x90, 0x1f, 0x60, 0x8f, 0xde, 0xf8, 0x48, - 0xc5, 0x45, 0x0d, 0x07, 0x89, 0x45, 0x0d, 0x07, 0xc9, 0x35, 0x57, 0xfa, 0xb1, 0x6a, 0xae, 0xcc, - 0x91, 0x6b, 0x2e, 0x4c, 0x0f, 0x89, 0x85, 0xbc, 0x18, 0x1e, 0x6e, 0x3c, 0x3a, 0xd9, 0x86, 0x47, - 0xc8, 0x22, 0x63, 0xe2, 0x08, 0x59, 0x44, 0xa1, 0x5d, 0x38, 0x2d, 0x1c, 0xc0, 0xf0, 0xd1, 0x1b, - 0xc9, 0x50, 0xe5, 0xc9, 0x27, 0xf2, 0x2d, 0x4a, 0xc5, 0xf6, 0xe1, 0x6e, 0x02, 0x2a, 0x16, 0xad, - 0x49, 0x1c, 0x09, 0x09, 0x03, 0x77, 0x82, 0xde, 0x06, 0x5f, 0xf6, 0x7c, 0x1c, 0x12, 0x22, 0x5c, - 0x0c, 0x09, 0x11, 0x5e, 0xff, 0x4b, 0x0a, 0xca, 0xf2, 0xfb, 0x9e, 0x48, 0x60, 0xbc, 0x02, 0x45, - 0xbc, 0x6f, 0xfa, 0xed, 0xae, 0x6d, 0x60, 0xde, 0xb9, 0x51, 0x3f, 0x13, 0xe0, 0x55, 0xdb, 0x90, - 0xfc, 0x1c, 0xc2, 0xc4, 0x68, 0x4a, 0x1f, 0x29, 0x9a, 0xe2, 0x49, 0x67, 0xe6, 0x08, 0x93, 0x4e, - 0xa5, 0x9f, 0x8a, 0x27, 0xe3, 0xa7, 0xfa, 0x57, 0x29, 0xa8, 0x24, 0xd3, 0xee, 0xf7, 0x63, 0x0b, - 0xca, 0xbb, 0x29, 0x7d, 0xe4, 0xdd, 0xf4, 0x36, 0x4c, 0x93, 0xca, 0x4c, 0xf7, 0x7d, 0x7e, 0x41, - 0x34, 0x43, 0x8b, 0x2b, 0x96, 0x8d, 0x02, 0x6b, 0x35, 0x84, 0x4b, 0xd9, 0x48, 0x80, 0x8f, 0x85, - 0x6e, 0xf6, 0x98, 0xa1, 0xfb, 0x69, 0x0a, 0xa6, 0x37, 0x6d, 0xe3, 0x1e, 0x2b, 0xda, 0xfc, 0xef, - 0xcb, 0x7a, 0x3e, 0xcd, 0x94, 0x56, 0x9f, 0x81, 0x69, 0xa9, 0x6a, 0xab, 0x7f, 0xc6, 0xe2, 0x4c, - 0xfe, 0x5c, 0xfd, 0xe3, 0xad, 0x4b, 0x19, 0xa6, 0xc4, 0xf2, 0xaf, 0xde, 0x84, 0x99, 0x44, 0xb5, - 0x26, 0xbe, 0x80, 0x76, 0x94, 0x17, 0xa8, 0x5f, 0x83, 0x39, 0x55, 0x19, 0x23, 0x64, 0x1d, 0xed, - 0x08, 0xa7, 0x33, 0xd7, 0x61, 0x4e, 0x55, 0x8e, 0x1c, 0xdf, 0x9c, 0xb7, 0xf8, 0xc9, 0x27, 0x2b, - 0x1c, 0x8e, 0xcf, 0xff, 0xfb, 0xa8, 0x7b, 0x8e, 0x2f, 0x63, 0xbf, 0x03, 0x15, 0x27, 0x7c, 0x68, - 0xf3, 0x1e, 0x8d, 0x6d, 0x4b, 0xda, 0x71, 0x44, 0xb8, 0xf5, 0x44, 0xb3, 0x56, 0x96, 0x31, 0xb2, - 0x1c, 0xde, 0xbf, 0xe5, 0x14, 0x72, 0x5a, 0x89, 0x46, 0xae, 0x2c, 0x63, 0x84, 0xa5, 0xcd, 0x1f, - 0xbe, 0xb4, 0xb4, 0xff, 0xcb, 0x92, 0xa6, 0x79, 0x26, 0x71, 0x59, 0x1c, 0x5d, 0x86, 0x02, 0xfd, - 0x25, 0x57, 0xdc, 0xf9, 0xd2, 0xd5, 0xa1, 0x30, 0xc9, 0x80, 0x3c, 0x07, 0xa1, 0x57, 0xa1, 0x18, - 0xdd, 0x1f, 0xe7, 0x67, 0x9e, 0x2c, 0xee, 0x42, 0xa0, 0x14, 0x77, 0x21, 0x90, 0x37, 0xcd, 0xff, - 0x0d, 0x67, 0x27, 0xde, 0x1c, 0x3f, 0x4e, 0x0f, 0x2e, 0x74, 0xbf, 0x99, 0x63, 0x75, 0xbf, 0xfb, - 0xb0, 0xa0, 0xbe, 0xd0, 0x2d, 0x68, 0x4f, 0x1d, 0xaa, 0x3d, 0x5e, 0xfd, 0xf4, 0x11, 0x57, 0x3f, - 0x55, 0xdf, 0xa5, 0xe3, 0x82, 0xe8, 0xe2, 0x34, 0xba, 0x04, 0x59, 0xc7, 0xb6, 0xfb, 0x1e, 0xbf, - 0x54, 0x40, 0xd5, 0x51, 0x80, 0xa8, 0x8e, 0x02, 0x1e, 0x63, 0x38, 0x11, 0x84, 0x11, 0x1c, 0x5f, - 0x03, 0x7f, 0x0a, 0xab, 0xfb, 0xc2, 0x65, 0x28, 0x84, 0x07, 0xb7, 0x08, 0x20, 0xf7, 0xee, 0xfd, - 0xb5, 0xfb, 0x6b, 0xd7, 0x2a, 0xa7, 0x50, 0x09, 0xf2, 0x9b, 0x6b, 0x77, 0xae, 0xdd, 0xbc, 0x73, - 0xbd, 0xa2, 0x91, 0x87, 0xd6, 0xfd, 0x3b, 0x77, 0xc8, 0x43, 0xea, 0x85, 0xdb, 0xe2, 0x65, 0x30, - 0x5e, 0xb9, 0x4d, 0x41, 0x61, 0xd5, 0x71, 0x68, 0x0a, 0x61, 0xbc, 0x6b, 0x7b, 0x26, 0xd9, 0xc9, - 0x15, 0x0d, 0xe5, 0x21, 0x7d, 0xf7, 0xee, 0x46, 0x25, 0x85, 0xe6, 0xa0, 0x72, 0x0d, 0xeb, 0x46, - 0xdf, 0xb4, 0x70, 0x98, 0xb7, 0x2a, 0xe9, 0xe6, 0xc3, 0xdf, 0x7c, 0xb3, 0xa4, 0x7d, 0xf5, 0xcd, - 0x92, 0xf6, 0xe7, 0x6f, 0x96, 0xb4, 0xcf, 0xbf, 0x5d, 0x3a, 0xf5, 0xd5, 0xb7, 0x4b, 0xa7, 0xfe, - 0xf0, 0xed, 0xd2, 0xa9, 0xff, 0xbc, 0xdc, 0x33, 0xfd, 0x9d, 0xa0, 0xd3, 0xe8, 0xda, 0x03, 0xfe, - 0x93, 0x54, 0xc7, 0xb5, 0x49, 0x82, 0xe0, 0x4f, 0x2b, 0xc9, 0xdf, 0xaa, 0xfe, 0x2c, 0x75, 0x6e, - 0x95, 0x3e, 0x6e, 0x32, 0xba, 0xc6, 0x4d, 0xbb, 0xc1, 0x00, 0xf4, 0xd7, 0x89, 0x5e, 0x27, 0x47, - 0x7f, 0x85, 0xf8, 0xca, 0xdf, 0x02, 0x00, 0x00, 0xff, 0xff, 0xf9, 0x44, 0xa5, 0x08, 0xe6, 0x3a, - 0x00, 0x00, + // 3695 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x3b, 0x4b, 0x6f, 0x1b, 0xd7, + 0xd5, 0x1e, 0xbe, 0x79, 0xa8, 0x07, 0x7d, 0xf5, 0x30, 0xad, 0xc4, 0xa2, 0x4c, 0xe7, 0x4b, 0xec, + 0x20, 0xa1, 0x1c, 0xe7, 0xcb, 0x87, 0x3c, 0x3e, 0x24, 0x10, 0x6d, 0xc5, 0xb6, 0x6c, 0xd9, 0x0a, + 0x65, 0xa7, 0x6e, 0x11, 0x80, 0x19, 0x72, 0xae, 0xa8, 0xb1, 0xc8, 0x99, 0xc9, 0x3c, 0x14, 0x09, + 0x08, 0xda, 0xa4, 0x48, 0xbb, 0xce, 0xa6, 0x40, 0x91, 0x4d, 0xb3, 0xe9, 0x22, 0x45, 0xbb, 0x6c, + 0xb7, 0xdd, 0x76, 0x51, 0x14, 0xd9, 0x14, 0xe8, 0xa2, 0x21, 0x8a, 0x04, 0xdd, 0x70, 0xd1, 0xdf, + 0x50, 0xdc, 0xc7, 0xcc, 0xdc, 0x3b, 0xbc, 0xb4, 0x24, 0xc7, 0x32, 0xdc, 0x74, 0x25, 0xcd, 0x79, + 0xde, 0x39, 0xe7, 0xdc, 0x33, 0xe7, 0x9c, 0x7b, 0x09, 0x67, 0x9c, 0x9d, 0xee, 0xb2, 0xee, 0xf6, + 0x75, 0x43, 0xc7, 0xbb, 0xd8, 0xf2, 0xbd, 0x65, 0xf6, 0xa7, 0xee, 0xb8, 0xb6, 0x6f, 0xa3, 0x09, + 0x11, 0xb5, 0x50, 0xdb, 0x79, 0xd5, 0xab, 0x9b, 0xf6, 0xb2, 0xee, 0x98, 0xcb, 0x1d, 0xdb, 0xc5, + 0xcb, 0xbb, 0x2f, 0x2d, 0x77, 0xb1, 0x85, 0x5d, 0xdd, 0xc7, 0x06, 0xe3, 0x58, 0x38, 0x2f, 0xd0, + 0x58, 0xd8, 0xff, 0xd0, 0x76, 0x77, 0x4c, 0xab, 0xab, 0xa2, 0xac, 0x76, 0x6d, 0xbb, 0xdb, 0xc3, + 0xcb, 0xf4, 0xa9, 0x1d, 0x6c, 0x2d, 0xfb, 0x66, 0x1f, 0x7b, 0xbe, 0xde, 0x77, 0x38, 0xc1, 0xff, + 0xc6, 0xa2, 0xfa, 0x7a, 0x67, 0xdb, 0xb4, 0xb0, 0xbb, 0xbf, 0x4c, 0xd7, 0xeb, 0x98, 0xcb, 0x2e, + 0xf6, 0xec, 0xc0, 0xed, 0xe0, 0x11, 0xb1, 0xaf, 0x9b, 0x96, 0x8f, 0x5d, 0x4b, 0xef, 0x2d, 0x7b, + 0x9d, 0x6d, 0x6c, 0x04, 0x3d, 0xec, 0xc6, 0xff, 0xd9, 0xed, 0xfb, 0xb8, 0xe3, 0x7b, 0x23, 0x00, + 0xc6, 0x5b, 0xfb, 0x7a, 0x0e, 0x26, 0x57, 0xc9, 0xbb, 0x6e, 0xe2, 0x0f, 0x02, 0x6c, 0x75, 0x30, + 0xba, 0x00, 0xd9, 0x0f, 0x02, 0x1c, 0xe0, 0x8a, 0xb6, 0xa4, 0x9d, 0x2f, 0x36, 0x66, 0x86, 0x83, + 0xea, 0x34, 0x05, 0xbc, 0x60, 0xf7, 0x4d, 0x1f, 0xf7, 0x1d, 0x7f, 0xbf, 0xc9, 0x28, 0xd0, 0xeb, + 0x30, 0x71, 0xdf, 0x6e, 0xb7, 0x3c, 0xec, 0xb7, 0x2c, 0xbd, 0x8f, 0x2b, 0x29, 0xca, 0x51, 0x19, + 0x0e, 0xaa, 0xb3, 0xf7, 0xed, 0xf6, 0x26, 0xf6, 0x6f, 0xe9, 0x7d, 0x91, 0x0d, 0x62, 0x28, 0x7a, + 0x11, 0xf2, 0x81, 0x87, 0xdd, 0x96, 0x69, 0x54, 0xd2, 0x94, 0x6d, 0x76, 0x38, 0xa8, 0x96, 0x09, + 0xe8, 0xba, 0x21, 0xb0, 0xe4, 0x18, 0x04, 0xbd, 0x00, 0xb9, 0xae, 0x6b, 0x07, 0x8e, 0x57, 0xc9, + 0x2c, 0xa5, 0x43, 0x6a, 0x06, 0x11, 0xa9, 0x19, 0x04, 0xdd, 0x86, 0x1c, 0x73, 0x60, 0x25, 0xbb, + 0x94, 0x3e, 0x5f, 0xba, 0x74, 0xb6, 0x2e, 0x7a, 0xb5, 0x2e, 0xbd, 0x30, 0x7b, 0x62, 0x02, 0x19, + 0x5e, 0x14, 0xc8, 0xe3, 0xe0, 0x0f, 0x33, 0x90, 0xa5, 0x74, 0xe8, 0x06, 0xe4, 0x3b, 0x2e, 0x26, + 0xd6, 0xaf, 0xa0, 0x25, 0xed, 0x7c, 0xe9, 0xd2, 0x42, 0x9d, 0x79, 0xb5, 0x1e, 0x7a, 0xb5, 0x7e, + 0x27, 0xf4, 0x6a, 0x63, 0x6e, 0x38, 0xa8, 0x9e, 0xe4, 0xe4, 0x82, 0xd4, 0x50, 0x02, 0xda, 0x80, + 0xa2, 0x17, 0xb4, 0xfb, 0xa6, 0xbf, 0x66, 0xb7, 0xa9, 0xbd, 0x4b, 0x97, 0x4e, 0xc9, 0x4b, 0xdd, + 0x0c, 0xd1, 0x8d, 0x53, 0xc3, 0x41, 0x75, 0x26, 0xa2, 0x8e, 0xa5, 0x5d, 0x3b, 0xd1, 0x8c, 0x85, + 0xa0, 0x6d, 0x98, 0x76, 0xb1, 0xe3, 0x9a, 0xb6, 0x6b, 0xfa, 0xa6, 0x87, 0x89, 0xdc, 0x14, 0x95, + 0x7b, 0x46, 0x96, 0xdb, 0x94, 0x89, 0x1a, 0x67, 0x86, 0x83, 0xea, 0xe9, 0x04, 0xa7, 0xa4, 0x23, + 0x29, 0x16, 0xf9, 0x80, 0x12, 0xa0, 0x4d, 0xec, 0x53, 0x5f, 0x96, 0x2e, 0x2d, 0x3d, 0x50, 0xd9, + 0x26, 0xf6, 0x1b, 0x4b, 0xc3, 0x41, 0xf5, 0xe9, 0x51, 0x7e, 0x49, 0xa5, 0x42, 0x3e, 0xea, 0x41, + 0x59, 0x84, 0x1a, 0xe4, 0x05, 0x33, 0x54, 0xe7, 0xe2, 0x78, 0x9d, 0x84, 0xaa, 0xb1, 0x38, 0x1c, + 0x54, 0x17, 0x92, 0xbc, 0x92, 0xbe, 0x11, 0xc9, 0xc4, 0x3f, 0x1d, 0xdd, 0xea, 0xe0, 0x1e, 0x51, + 0x93, 0x55, 0xf9, 0xe7, 0x72, 0x88, 0x66, 0xfe, 0x89, 0xa8, 0x65, 0xff, 0x44, 0x60, 0xf4, 0x1e, + 0x4c, 0x44, 0x0f, 0xc4, 0x5e, 0x39, 0x1e, 0x43, 0x6a, 0xa1, 0xc4, 0x52, 0x0b, 0xc3, 0x41, 0x75, + 0x5e, 0xe4, 0x91, 0x44, 0x4b, 0xd2, 0x62, 0xe9, 0x3d, 0x66, 0x99, 0xfc, 0x78, 0xe9, 0x8c, 0x42, + 0x94, 0xde, 0x1b, 0xb5, 0x88, 0x24, 0x8d, 0x48, 0x27, 0x1b, 0x38, 0xe8, 0x74, 0x30, 0x36, 0xb0, + 0x51, 0x29, 0xa8, 0xa4, 0xaf, 0x09, 0x14, 0x4c, 0xba, 0xc8, 0x23, 0x4b, 0x17, 0x31, 0xc4, 0xd6, + 0xf7, 0xed, 0xf6, 0xaa, 0xeb, 0xda, 0xae, 0x57, 0x29, 0xaa, 0x6c, 0xbd, 0x16, 0xa2, 0x99, 0xad, + 0x23, 0x6a, 0xd9, 0xd6, 0x11, 0x98, 0xaf, 0xb7, 0x19, 0x58, 0x37, 0xb1, 0xee, 0x61, 0xa3, 0x02, + 0x63, 0xd6, 0x1b, 0x51, 0x44, 0xeb, 0x8d, 0x20, 0x23, 0xeb, 0x8d, 0x30, 0xc8, 0x80, 0x29, 0xf6, + 0xbc, 0xe2, 0x79, 0x66, 0xd7, 0xc2, 0x46, 0xa5, 0x44, 0xe5, 0x3f, 0xad, 0x92, 0x1f, 0xd2, 0x34, + 0x9e, 0x1e, 0x0e, 0xaa, 0x15, 0x99, 0x4f, 0xd2, 0x91, 0x90, 0x89, 0xde, 0x87, 0x49, 0x06, 0x69, + 0x06, 0x96, 0x65, 0x5a, 0xdd, 0xca, 0x04, 0x55, 0xf2, 0x94, 0x4a, 0x09, 0x27, 0x69, 0x3c, 0x35, + 0x1c, 0x54, 0x4f, 0x49, 0x5c, 0x92, 0x0a, 0x59, 0x20, 0xc9, 0x18, 0x0c, 0x10, 0x3b, 0x76, 0x52, + 0x95, 0x31, 0xd6, 0x64, 0x22, 0x96, 0x31, 0x12, 0x9c, 0x72, 0xc6, 0x48, 0x20, 0x63, 0x7f, 0x70, + 0x27, 0x4f, 0x8d, 0xf7, 0x07, 0xf7, 0xb3, 0xe0, 0x0f, 0x85, 0xab, 0x25, 0x69, 0xe8, 0x63, 0x0d, + 0xe6, 0x3c, 0x5f, 0xb7, 0x0c, 0xbd, 0x67, 0x5b, 0xf8, 0xba, 0xd5, 0x75, 0xb1, 0xe7, 0x5d, 0xb7, + 0xb6, 0xec, 0x4a, 0x99, 0xea, 0x39, 0x97, 0x48, 0xac, 0x2a, 0xd2, 0xc6, 0xb9, 0xe1, 0xa0, 0x5a, + 0x55, 0x4a, 0x91, 0x34, 0xab, 0x15, 0xa1, 0x3d, 0x98, 0x09, 0x3f, 0xd2, 0x77, 0x7d, 0xb3, 0x67, + 0x7a, 0xba, 0x6f, 0xda, 0x56, 0xe5, 0x24, 0xd5, 0x7f, 0x36, 0x99, 0x9f, 0x46, 0x08, 0x1b, 0x67, + 0x87, 0x83, 0xea, 0x19, 0x85, 0x04, 0x49, 0xb7, 0x4a, 0x45, 0xec, 0xc4, 0x0d, 0x17, 0x13, 0x42, + 0x6c, 0x54, 0x66, 0xc6, 0x3b, 0x31, 0x22, 0x12, 0x9d, 0x18, 0x01, 0x55, 0x4e, 0x8c, 0x90, 0x44, + 0x93, 0xa3, 0xbb, 0xbe, 0x49, 0xd4, 0xae, 0xeb, 0xee, 0x0e, 0x76, 0x2b, 0xb3, 0x2a, 0x4d, 0x1b, + 0x32, 0x11, 0xd3, 0x94, 0xe0, 0x94, 0x35, 0x25, 0x90, 0xe8, 0x33, 0x0d, 0xe4, 0xa5, 0x99, 0xb6, + 0xd5, 0x24, 0x1f, 0x6d, 0x8f, 0xbc, 0xde, 0x1c, 0x55, 0xfa, 0xdc, 0x03, 0x5e, 0x4f, 0x24, 0x6f, + 0x3c, 0x37, 0x1c, 0x54, 0xcf, 0x8d, 0x95, 0x26, 0x2d, 0x64, 0xbc, 0x52, 0x74, 0x0f, 0x4a, 0x04, + 0x89, 0x69, 0xf9, 0x63, 0x54, 0xe6, 0xe9, 0x1a, 0x4e, 0x8f, 0xae, 0x81, 0x13, 0x34, 0x4e, 0x0f, + 0x07, 0xd5, 0x39, 0x81, 0x43, 0xd2, 0x23, 0x8a, 0x42, 0x9f, 0x6a, 0x40, 0x02, 0x5d, 0xf5, 0xa6, + 0xa7, 0xa8, 0x96, 0x67, 0x46, 0xb4, 0xa8, 0x5e, 0xf3, 0x99, 0xe1, 0xa0, 0xba, 0xa4, 0x96, 0x23, + 0xe9, 0x1e, 0xa3, 0x2b, 0x8e, 0xa3, 0xe8, 0x23, 0x51, 0xa9, 0x8c, 0x8f, 0xa3, 0x88, 0x48, 0x8c, + 0xa3, 0x08, 0xa8, 0x8a, 0xa3, 0x08, 0xc9, 0x93, 0xc1, 0xbb, 0x7a, 0xcf, 0x34, 0x68, 0x31, 0x75, + 0x7a, 0x4c, 0x32, 0x88, 0x28, 0xa2, 0x64, 0x10, 0x41, 0x46, 0x92, 0x41, 0x4c, 0x9b, 0x87, 0x2c, + 0x15, 0x51, 0xfb, 0xbc, 0x08, 0x33, 0x8a, 0xad, 0x86, 0x30, 0x4c, 0x86, 0xfb, 0xa8, 0x65, 0x92, + 0x24, 0x91, 0x56, 0x59, 0xf9, 0x46, 0xd0, 0xc6, 0xae, 0x85, 0x7d, 0xec, 0x85, 0x32, 0x68, 0x96, + 0xa0, 0x2b, 0x71, 0x05, 0x88, 0x50, 0xdb, 0x4d, 0x88, 0x70, 0xf4, 0xb9, 0x06, 0x95, 0xbe, 0xbe, + 0xd7, 0x0a, 0x81, 0x5e, 0x6b, 0xcb, 0x76, 0x5b, 0x0e, 0x76, 0x4d, 0xdb, 0xa0, 0x95, 0x6c, 0xe9, + 0xd2, 0xff, 0x1f, 0x98, 0x17, 0xea, 0xeb, 0xfa, 0x5e, 0x08, 0xf6, 0xde, 0xb6, 0xdd, 0x0d, 0xca, + 0xbe, 0x6a, 0xf9, 0xee, 0x3e, 0x4b, 0x58, 0x7d, 0x15, 0x5e, 0x58, 0xd3, 0x9c, 0x92, 0x00, 0xfd, + 0x42, 0x83, 0x79, 0xdf, 0xf6, 0xf5, 0x5e, 0xab, 0x13, 0xf4, 0x83, 0x9e, 0xee, 0x9b, 0xbb, 0xb8, + 0x15, 0x78, 0x7a, 0x17, 0xf3, 0xb2, 0xf9, 0x8d, 0x83, 0x97, 0x76, 0x87, 0xf0, 0x5f, 0x8e, 0xd8, + 0xef, 0x12, 0x6e, 0xb6, 0xb2, 0xda, 0x70, 0x50, 0x5d, 0xf4, 0x15, 0x68, 0x61, 0x61, 0xb3, 0x2a, + 0x3c, 0x7a, 0x1e, 0x72, 0xa4, 0xad, 0x30, 0x0d, 0x5a, 0x1d, 0xf1, 0x16, 0xe4, 0xbe, 0xdd, 0x96, + 0x1a, 0x83, 0x2c, 0x05, 0x10, 0x5a, 0x37, 0xb0, 0x08, 0x6d, 0x3e, 0xa6, 0x75, 0x03, 0x4b, 0xa6, + 0xa5, 0x00, 0xea, 0x0c, 0x7d, 0xb7, 0xab, 0x76, 0x46, 0xe1, 0xb0, 0xce, 0x58, 0xd9, 0xed, 0x3e, + 0xd0, 0x19, 0xba, 0x0a, 0x2f, 0x3a, 0x43, 0x49, 0xb0, 0xf0, 0x85, 0x06, 0x0b, 0xe3, 0xfd, 0x8c, + 0xce, 0x41, 0x7a, 0x07, 0xef, 0xf3, 0x9e, 0xec, 0xe4, 0x70, 0x50, 0x9d, 0xdc, 0xc1, 0xfb, 0x82, + 0x54, 0x82, 0x45, 0x3f, 0x84, 0xec, 0xae, 0xde, 0x0b, 0x30, 0x2f, 0xf9, 0xeb, 0x75, 0xd6, 0x4e, + 0xd6, 0xc5, 0x76, 0xb2, 0xee, 0xec, 0x74, 0x09, 0xa0, 0x1e, 0x5a, 0xa1, 0xfe, 0x4e, 0xa0, 0x5b, + 0xbe, 0xe9, 0xef, 0x33, 0xdb, 0x51, 0x01, 0xa2, 0xed, 0x28, 0xe0, 0xf5, 0xd4, 0xab, 0xda, 0xc2, + 0xaf, 0x34, 0x38, 0x3d, 0xd6, 0xdf, 0x4f, 0xc4, 0x0a, 0x89, 0x11, 0xc7, 0xfb, 0xe7, 0x49, 0x58, + 0xe2, 0x5a, 0xa6, 0xa0, 0x95, 0x53, 0x6b, 0x99, 0x42, 0xaa, 0x9c, 0xae, 0xfd, 0x36, 0x0f, 0xc5, + 0xa8, 0xc1, 0x43, 0xd7, 0xa0, 0x6c, 0x60, 0x23, 0x70, 0x7a, 0x66, 0x87, 0x46, 0x1a, 0x09, 0x6a, + 0xd6, 0x51, 0xd3, 0xec, 0x2a, 0xe1, 0xa4, 0xf0, 0x9e, 0x4e, 0xa0, 0xd0, 0x25, 0x28, 0xf0, 0x46, + 0x66, 0x9f, 0xe6, 0xb5, 0xc9, 0xc6, 0xfc, 0x70, 0x50, 0x45, 0x21, 0x4c, 0x60, 0x8d, 0xe8, 0x50, + 0x13, 0x80, 0x4d, 0x06, 0xd6, 0xb1, 0xaf, 0xf3, 0x96, 0xaa, 0x22, 0xef, 0x86, 0xdb, 0x11, 0x9e, + 0xf5, 0xf8, 0x31, 0xbd, 0xd8, 0xe3, 0xc7, 0x50, 0xf4, 0x1e, 0x40, 0x5f, 0x37, 0x2d, 0xc6, 0xc7, + 0xfb, 0xa7, 0xda, 0xb8, 0x0c, 0xbb, 0x1e, 0x51, 0x32, 0xe9, 0x31, 0xa7, 0x28, 0x3d, 0x86, 0xa2, + 0xdb, 0x90, 0xe7, 0xb3, 0x8c, 0x4a, 0x8e, 0x6e, 0xde, 0xc5, 0x71, 0xa2, 0xb9, 0x58, 0xda, 0x8d, + 0x73, 0x16, 0xb1, 0x1b, 0xe7, 0x20, 0x62, 0xb6, 0x9e, 0xb9, 0x85, 0x7d, 0xb3, 0x8f, 0x69, 0x36, + 0xe1, 0x66, 0x0b, 0x61, 0xa2, 0xd9, 0x42, 0x18, 0x7a, 0x15, 0x40, 0xf7, 0xd7, 0x6d, 0xcf, 0xbf, + 0x6d, 0x75, 0x30, 0xed, 0x88, 0x0a, 0x6c, 0xf9, 0x31, 0x54, 0x5c, 0x7e, 0x0c, 0x45, 0x6f, 0x40, + 0xc9, 0xe1, 0x5f, 0xe0, 0x76, 0x0f, 0xd3, 0x8e, 0xa7, 0xc0, 0x0a, 0x06, 0x01, 0x2c, 0xf0, 0x8a, + 0xd4, 0xe8, 0x2a, 0x4c, 0x77, 0x6c, 0xab, 0x13, 0xb8, 0x2e, 0xb6, 0x3a, 0xfb, 0x9b, 0xfa, 0x16, + 0xa6, 0xdd, 0x4d, 0x81, 0x85, 0x4a, 0x02, 0x25, 0x86, 0x4a, 0x02, 0x85, 0x5e, 0x81, 0x62, 0x34, + 0x19, 0xa2, 0x0d, 0x4c, 0x91, 0x0f, 0x1a, 0x42, 0xa0, 0xc0, 0x1c, 0x53, 0x92, 0xc5, 0x9b, 0xde, + 0x15, 0x1e, 0x74, 0x98, 0x36, 0x25, 0x7c, 0xf1, 0x02, 0x58, 0x5c, 0xbc, 0x00, 0x16, 0xf2, 0xfb, + 0xd4, 0x81, 0xf9, 0xfd, 0xc7, 0x30, 0x87, 0xf7, 0x48, 0x8a, 0xee, 0x63, 0xcb, 0xd7, 0x7b, 0x1b, + 0xae, 0xc9, 0xbe, 0xac, 0x95, 0x69, 0x55, 0x51, 0xbf, 0xaa, 0x22, 0x65, 0x69, 0x59, 0x29, 0x45, + 0x4c, 0xcb, 0x4a, 0x82, 0x68, 0xbb, 0x4e, 0x96, 0xa7, 0x6a, 0x37, 0x60, 0x4e, 0xa9, 0x80, 0x04, + 0x4e, 0xdb, 0x34, 0xe8, 0x33, 0x4d, 0x2e, 0x1a, 0x0b, 0x9c, 0x10, 0x26, 0x06, 0x4e, 0x08, 0xab, + 0xfd, 0x59, 0x83, 0x59, 0x55, 0xf0, 0x27, 0x36, 0xa2, 0xf6, 0x48, 0x36, 0xe2, 0xbb, 0x50, 0x70, + 0x6c, 0xa3, 0xe5, 0x39, 0xb8, 0xc3, 0xd3, 0x5a, 0x62, 0x1b, 0x6e, 0xd8, 0xc6, 0xa6, 0x83, 0x3b, + 0x3f, 0x30, 0xfd, 0xed, 0x95, 0x5d, 0xdb, 0x34, 0x6e, 0x9a, 0x1e, 0xdf, 0x2f, 0x0e, 0xc3, 0x48, + 0xb5, 0x56, 0x9e, 0x03, 0x1b, 0x05, 0xc8, 0x31, 0x2d, 0xb5, 0xbf, 0xa4, 0xa1, 0x9c, 0xdc, 0x70, + 0xff, 0x49, 0xaf, 0x82, 0xee, 0x41, 0xde, 0x64, 0xad, 0x1c, 0x2f, 0x05, 0xff, 0x47, 0x48, 0xfc, + 0xf5, 0x78, 0xae, 0x5b, 0xdf, 0x7d, 0xa9, 0xce, 0x7b, 0x3e, 0x6a, 0x02, 0x2a, 0x99, 0x73, 0xca, + 0x92, 0x39, 0x10, 0x35, 0x21, 0xef, 0x61, 0x77, 0x97, 0x04, 0x07, 0x4b, 0xab, 0x55, 0x51, 0x72, + 0xc7, 0x76, 0x31, 0x91, 0xb9, 0xc9, 0x48, 0x62, 0x99, 0x9c, 0x47, 0x96, 0xc9, 0x81, 0xe8, 0x5d, + 0x28, 0x76, 0x6c, 0x6b, 0xcb, 0xec, 0xae, 0xeb, 0x0e, 0x4f, 0xac, 0x67, 0x54, 0x52, 0x2f, 0x87, + 0x44, 0x7c, 0x3c, 0x15, 0x3e, 0x26, 0xc6, 0x53, 0x11, 0x55, 0xec, 0xd0, 0x7f, 0x65, 0x00, 0x62, + 0xe7, 0xa0, 0xd7, 0xa0, 0x84, 0xf7, 0x70, 0x27, 0xf0, 0x6d, 0x3a, 0xb2, 0xd5, 0xe2, 0x49, 0x6f, + 0x08, 0x96, 0x76, 0x2f, 0xc4, 0x50, 0x92, 0x62, 0x2c, 0xbd, 0x8f, 0x3d, 0x47, 0xef, 0x84, 0x23, + 0x62, 0xba, 0x98, 0x08, 0x28, 0xa6, 0x98, 0x08, 0x88, 0x9e, 0x85, 0x0c, 0x1d, 0x2a, 0xb3, 0xe9, + 0x30, 0x1a, 0x0e, 0xaa, 0x53, 0x96, 0x3c, 0x4e, 0xa6, 0x78, 0xf4, 0x16, 0x4c, 0xee, 0x44, 0x81, + 0x47, 0xd6, 0x96, 0xa1, 0x0c, 0xb4, 0x46, 0x8f, 0x11, 0xd2, 0xea, 0x26, 0x44, 0x38, 0xda, 0x82, + 0x92, 0x6e, 0x59, 0xb6, 0x4f, 0xbf, 0x9e, 0xe1, 0xc4, 0xf8, 0xc2, 0xb8, 0x30, 0xad, 0xaf, 0xc4, + 0xb4, 0xac, 0xea, 0xa3, 0x69, 0x4f, 0x90, 0x20, 0xa6, 0x3d, 0x01, 0x8c, 0x9a, 0x90, 0xeb, 0xe9, + 0x6d, 0xdc, 0x0b, 0x3f, 0x57, 0xcf, 0x8c, 0x55, 0x71, 0x93, 0x92, 0x31, 0xe9, 0x74, 0x2e, 0xcd, + 0xf8, 0xc4, 0xb9, 0x34, 0x83, 0x2c, 0x6c, 0x41, 0x39, 0xb9, 0x9e, 0xc3, 0x55, 0x39, 0x17, 0xc4, + 0x2a, 0xa7, 0x78, 0x60, 0x61, 0xa5, 0x43, 0x49, 0x58, 0xd4, 0x71, 0xa8, 0xa8, 0x7d, 0xa9, 0xc1, + 0xac, 0x6a, 0xef, 0xa2, 0x75, 0x61, 0xc7, 0x6b, 0x7c, 0xfa, 0xa5, 0x08, 0x75, 0xce, 0x3b, 0x66, + 0xab, 0xc7, 0x1b, 0xbd, 0x01, 0x53, 0x96, 0x6d, 0xe0, 0x96, 0x4e, 0x14, 0xf4, 0x4c, 0xcf, 0xaf, + 0xa4, 0xe8, 0x89, 0x02, 0x9d, 0x9a, 0x11, 0xcc, 0x4a, 0x88, 0x10, 0xb8, 0x27, 0x25, 0x44, 0xed, + 0x43, 0x98, 0x4e, 0xcc, 0xb4, 0xa5, 0x9a, 0x2b, 0x75, 0xc8, 0x9a, 0x2b, 0xfe, 0x10, 0xa6, 0x0f, + 0xfa, 0x10, 0xb2, 0x0f, 0x51, 0xed, 0x67, 0x29, 0x28, 0x09, 0x03, 0x06, 0x74, 0x1f, 0xa6, 0xf9, + 0x47, 0xd9, 0xb4, 0xba, 0xac, 0x91, 0x4d, 0xf1, 0x0f, 0xe3, 0xc8, 0x81, 0xcf, 0x9a, 0xdd, 0xde, + 0x8c, 0x68, 0xe9, 0x87, 0x91, 0x0e, 0x23, 0x3d, 0x09, 0x26, 0x28, 0x9e, 0x92, 0x31, 0xe8, 0x1e, + 0xcc, 0x07, 0x0e, 0x69, 0xaf, 0x5b, 0x1e, 0x3f, 0x3a, 0x69, 0x59, 0x41, 0xbf, 0x8d, 0x5d, 0xba, + 0xfa, 0x2c, 0x6b, 0xf8, 0x18, 0x45, 0x78, 0xb6, 0x72, 0x8b, 0xe2, 0xc5, 0x86, 0x4f, 0x85, 0x17, + 0xec, 0x90, 0x39, 0xa4, 0x1d, 0xae, 0x01, 0x1a, 0x3d, 0x54, 0x90, 0x7c, 0xa0, 0x1d, 0xce, 0x07, + 0xb5, 0x3d, 0x28, 0x27, 0x8f, 0x0a, 0x1e, 0x93, 0x2f, 0x77, 0xa0, 0x18, 0x0d, 0xfa, 0xd1, 0x0b, + 0x90, 0x73, 0xb1, 0xee, 0xd9, 0x16, 0xdf, 0x2d, 0x74, 0xdb, 0x33, 0x88, 0xb8, 0xed, 0x19, 0xe4, + 0x21, 0x94, 0xdd, 0x81, 0x09, 0x66, 0xa4, 0xb7, 0xcd, 0x9e, 0x8f, 0x5d, 0x74, 0x05, 0x72, 0x9e, + 0xaf, 0xfb, 0xd8, 0xab, 0x68, 0x4b, 0xe9, 0xf3, 0x53, 0x97, 0xe6, 0x47, 0xa7, 0xf8, 0x04, 0xcd, + 0xd6, 0xc1, 0x28, 0xc5, 0x75, 0x30, 0x48, 0xed, 0xa7, 0x1a, 0x4c, 0x88, 0x87, 0x15, 0x8f, 0x46, + 0xec, 0xd1, 0x8c, 0x51, 0xb3, 0xc2, 0x35, 0xf0, 0x63, 0x8a, 0xe3, 0x36, 0xe5, 0x97, 0x1a, 0xb3, + 0x65, 0x34, 0xd7, 0xee, 0xc6, 0xb3, 0x24, 0xb2, 0x51, 0x3c, 0x9a, 0x50, 0x0e, 0x3b, 0x4b, 0xa2, + 0x69, 0x47, 0x62, 0x17, 0xd3, 0x8e, 0x84, 0x78, 0x88, 0xb5, 0x7e, 0x91, 0xa5, 0x6b, 0x8d, 0x4f, + 0x2d, 0x12, 0xdf, 0xf1, 0xf4, 0x11, 0xbe, 0xe3, 0x2f, 0x42, 0x9e, 0x26, 0xce, 0x68, 0x9b, 0x52, + 0xc3, 0x12, 0x90, 0x7c, 0x62, 0xcb, 0x20, 0x0f, 0x48, 0x17, 0xd9, 0xef, 0x98, 0x2e, 0x5a, 0x70, + 0x7a, 0x5b, 0xf7, 0x5a, 0x61, 0x82, 0x33, 0x5a, 0xba, 0xdf, 0x8a, 0xf6, 0x6b, 0x8e, 0xb6, 0x22, + 0x74, 0x0e, 0xba, 0xad, 0x7b, 0x9b, 0x21, 0xcd, 0x8a, 0xbf, 0x31, 0xba, 0x7b, 0xe7, 0xd5, 0x14, + 0xe8, 0x2e, 0xcc, 0xa9, 0x85, 0xe7, 0xe9, 0xca, 0xe9, 0x98, 0xde, 0x7b, 0xa0, 0xe4, 0x19, 0x05, + 0x1a, 0x7d, 0xa2, 0x41, 0x85, 0x7c, 0xc9, 0x5c, 0xfc, 0x41, 0x60, 0xba, 0x98, 0x74, 0x11, 0x5e, + 0xcb, 0xde, 0xc5, 0x6e, 0x4f, 0xdf, 0xe7, 0x27, 0x5e, 0x67, 0x47, 0xd3, 0xf6, 0x86, 0x6d, 0x34, + 0x05, 0x06, 0xf6, 0x6a, 0x8e, 0x0c, 0xbc, 0xcd, 0x84, 0x88, 0xaf, 0xa6, 0xa6, 0x10, 0x42, 0x08, + 0x8e, 0x30, 0x5b, 0x2b, 0x1d, 0x38, 0x5b, 0x7b, 0x16, 0x32, 0x8e, 0x6d, 0xf7, 0x68, 0x27, 0xc8, + 0xab, 0x35, 0xf2, 0x2c, 0x56, 0x6b, 0xe4, 0x59, 0x1c, 0x7f, 0xac, 0x65, 0x0a, 0x85, 0x72, 0xb1, + 0xf6, 0xb5, 0x06, 0x53, 0xf2, 0x21, 0xd9, 0xe8, 0x86, 0x4a, 0x1f, 0xfb, 0x86, 0xca, 0x1c, 0xc1, + 0x1a, 0xd9, 0x83, 0xac, 0x21, 0x0d, 0x79, 0xfe, 0xae, 0xc1, 0xa4, 0x74, 0x3e, 0xf7, 0xfd, 0x7a, + 0xbd, 0x5f, 0xa6, 0x60, 0x5e, 0xbd, 0xd4, 0x63, 0x69, 0xff, 0xae, 0x01, 0x29, 0xe4, 0xae, 0xc7, + 0x85, 0xce, 0xdc, 0x48, 0xf7, 0x47, 0xcd, 0x14, 0x56, 0x81, 0x23, 0x47, 0x77, 0x21, 0x3b, 0xba, + 0x07, 0x25, 0x53, 0x38, 0x24, 0x4c, 0xab, 0xce, 0x72, 0xc4, 0xa3, 0x41, 0x36, 0xdd, 0x18, 0x73, + 0x20, 0x28, 0x8a, 0x6a, 0xe4, 0x20, 0x43, 0x2a, 0xb1, 0xda, 0x2e, 0xe4, 0xf9, 0x72, 0xd0, 0xcb, + 0x50, 0xa4, 0xb9, 0x93, 0x76, 0x34, 0xac, 0x6c, 0xa6, 0x25, 0x05, 0x01, 0x26, 0x2e, 0xc9, 0x14, + 0x42, 0x18, 0xfa, 0x3f, 0x00, 0x92, 0x2e, 0x78, 0xd6, 0x4c, 0xd1, 0xdc, 0x43, 0x3b, 0x27, 0xc7, + 0x36, 0x46, 0x52, 0x65, 0x31, 0x02, 0xd6, 0x7e, 0x97, 0x82, 0x92, 0x78, 0x2c, 0xf9, 0x50, 0xca, + 0x3f, 0x82, 0xb0, 0xab, 0x6d, 0xe9, 0x86, 0x41, 0xfe, 0xe2, 0xf0, 0xc3, 0xb6, 0x3c, 0xd6, 0x48, + 0xe1, 0xff, 0x2b, 0x21, 0x07, 0xeb, 0x61, 0xe8, 0xd5, 0x0b, 0x33, 0x81, 0x12, 0xb4, 0x96, 0x93, + 0xb8, 0x85, 0x1d, 0x98, 0x53, 0x8a, 0x12, 0x3b, 0x8f, 0xec, 0xa3, 0xea, 0x3c, 0x7e, 0x9d, 0x85, + 0x39, 0xe5, 0x71, 0x70, 0x22, 0x82, 0xd3, 0x8f, 0x24, 0x82, 0x7f, 0xae, 0xa9, 0x2c, 0xcb, 0xce, + 0x82, 0x5e, 0x3b, 0xc4, 0x19, 0xf5, 0xa3, 0xb2, 0xb1, 0x1c, 0x16, 0xd9, 0x87, 0x8a, 0xc9, 0xdc, + 0x61, 0x63, 0x12, 0x5d, 0x64, 0x4d, 0x1c, 0xd5, 0xc5, 0x4e, 0x6a, 0xc2, 0x1d, 0x9a, 0x50, 0x95, + 0xe7, 0x20, 0xd2, 0xd7, 0x87, 0x1c, 0x6c, 0x74, 0x50, 0x88, 0xfb, 0x7a, 0x4e, 0x93, 0x9c, 0x1e, + 0x4c, 0x88, 0x70, 0x21, 0xfb, 0x15, 0x8f, 0x90, 0xfd, 0xe0, 0xa0, 0xec, 0xf7, 0x58, 0x63, 0x53, + 0x4a, 0xb5, 0x03, 0x0d, 0xa6, 0x13, 0xb7, 0x30, 0xbe, 0x5f, 0xdf, 0x92, 0x8f, 0x35, 0x28, 0x46, + 0x97, 0x7c, 0xd0, 0x0a, 0xe4, 0x30, 0xbb, 0x28, 0xc2, 0xd2, 0xce, 0x4c, 0x62, 0xd6, 0x4b, 0x70, + 0xfc, 0xda, 0x5e, 0xe2, 0x6e, 0x48, 0x93, 0x33, 0x3e, 0x44, 0xc1, 0xfc, 0x7b, 0x2d, 0x2c, 0x98, + 0x47, 0x56, 0x91, 0xfe, 0xee, 0xab, 0x38, 0x3e, 0xd3, 0xfd, 0xb1, 0x08, 0x59, 0xba, 0x16, 0xd2, + 0xbc, 0xfa, 0xd8, 0xed, 0x9b, 0x96, 0xde, 0xa3, 0xa1, 0x58, 0x60, 0xbb, 0x3a, 0x84, 0x89, 0xbb, + 0x3a, 0x84, 0xa1, 0x6d, 0x98, 0x8e, 0x47, 0x62, 0x54, 0x8c, 0xfa, 0xd6, 0xe0, 0x0d, 0x99, 0x88, + 0x9d, 0x36, 0x24, 0x38, 0xe5, 0x63, 0xff, 0x04, 0x12, 0x19, 0x30, 0xd5, 0xb1, 0x2d, 0x5f, 0x37, + 0x2d, 0xec, 0x32, 0x45, 0x69, 0xd5, 0xad, 0xa9, 0xcb, 0x12, 0x0d, 0x1b, 0x54, 0xc8, 0x7c, 0xf2, + 0xad, 0x29, 0x19, 0x87, 0xde, 0x87, 0xc9, 0xb0, 0x71, 0x61, 0x4a, 0x32, 0xaa, 0x5b, 0x53, 0xab, + 0x22, 0x09, 0xdb, 0x0c, 0x12, 0x97, 0x7c, 0x6b, 0x4a, 0x42, 0xa1, 0x1e, 0x94, 0x1d, 0xdb, 0xb8, + 0x6b, 0xf1, 0x72, 0x5d, 0x6f, 0xf7, 0x30, 0x9f, 0xc3, 0x2e, 0x8e, 0x14, 0x24, 0x12, 0x15, 0x4b, + 0xd4, 0x49, 0x5e, 0xf9, 0x1e, 0x62, 0x12, 0x8b, 0xde, 0x83, 0x89, 0x1e, 0xe9, 0xdf, 0x56, 0xf7, + 0x1c, 0xd3, 0xc5, 0x86, 0xfa, 0xd6, 0xe0, 0x4d, 0x81, 0x82, 0xa5, 0x49, 0x91, 0x47, 0xbe, 0x2c, + 0x21, 0x62, 0x88, 0xf7, 0xfb, 0xfa, 0x5e, 0x33, 0xb0, 0xbc, 0xd5, 0x3d, 0x7e, 0x03, 0x2c, 0xaf, + 0xf2, 0xfe, 0xba, 0x4c, 0xc4, 0xbc, 0x9f, 0xe0, 0x94, 0xbd, 0x9f, 0x40, 0xa2, 0x9b, 0xf4, 0x2b, + 0xc0, 0x5c, 0xc2, 0x6e, 0x0f, 0xce, 0x8f, 0x58, 0x8b, 0x79, 0x83, 0x0d, 0x5c, 0xf8, 0x93, 0x24, + 0x34, 0x92, 0xc0, 0x7d, 0x40, 0x5f, 0xbb, 0x89, 0xfd, 0xc0, 0xb5, 0xb0, 0xc1, 0xdb, 0xa8, 0x51, + 0x1f, 0x48, 0x54, 0x91, 0x0f, 0x24, 0xe8, 0x88, 0x0f, 0x24, 0x2c, 0xfa, 0x08, 0x66, 0x13, 0x77, + 0xa1, 0xd8, 0x7b, 0x94, 0x54, 0x87, 0x10, 0x6b, 0x0a, 0x4a, 0xd6, 0xf1, 0xaa, 0x64, 0x48, 0x9a, + 0x95, 0x5a, 0x88, 0xf6, 0xae, 0x6e, 0x75, 0xd7, 0xec, 0xb6, 0x1c, 0x73, 0x13, 0x2a, 0xed, 0x57, + 0x15, 0x94, 0x4c, 0xbb, 0x4a, 0x86, 0xac, 0x5d, 0x45, 0x11, 0xdd, 0x7b, 0x22, 0x45, 0x4c, 0x74, + 0x3f, 0x50, 0x75, 0xef, 0x89, 0x11, 0x08, 0xf7, 0x9e, 0x18, 0x40, 0x71, 0xef, 0x89, 0x53, 0x16, + 0xc2, 0x61, 0x4d, 0xed, 0x1d, 0x98, 0x4e, 0xa4, 0x17, 0xf4, 0x26, 0x44, 0xb7, 0x69, 0xee, 0xec, + 0x3b, 0x61, 0xed, 0x2a, 0xdd, 0xbe, 0x21, 0x70, 0xd5, 0xed, 0x1b, 0x02, 0xaf, 0x7d, 0x96, 0x81, + 0x42, 0x18, 0x51, 0xc7, 0xd2, 0x8d, 0x2c, 0x43, 0xbe, 0x8f, 0x3d, 0x7a, 0x63, 0x26, 0x15, 0x17, + 0x35, 0x1c, 0x24, 0x16, 0x35, 0x1c, 0x24, 0xd7, 0x5c, 0xe9, 0x87, 0xaa, 0xb9, 0x32, 0x87, 0xae, + 0xb9, 0x30, 0x3d, 0x24, 0x16, 0xf2, 0x62, 0x78, 0xb8, 0xf1, 0xe0, 0x64, 0x1b, 0x1e, 0x21, 0x8b, + 0x8c, 0x89, 0x23, 0x64, 0x11, 0x85, 0x76, 0xe0, 0xa4, 0x70, 0x00, 0xc3, 0x47, 0x6f, 0x24, 0x43, + 0x4d, 0x8d, 0x3f, 0x91, 0x6f, 0x52, 0x2a, 0xb6, 0x0f, 0x77, 0x12, 0x50, 0xb1, 0x68, 0x4d, 0xe2, + 0x48, 0x48, 0x18, 0xb8, 0x1d, 0x74, 0xd7, 0xb9, 0xd9, 0xf3, 0x71, 0x48, 0x88, 0x70, 0x31, 0x24, + 0x44, 0x78, 0xed, 0x9f, 0x29, 0x98, 0x92, 0xdf, 0xf7, 0x58, 0x02, 0xe3, 0x65, 0x28, 0xe2, 0x3d, + 0xd3, 0x6f, 0x75, 0x6c, 0x03, 0xf3, 0xce, 0x8d, 0xfa, 0x99, 0x00, 0x2f, 0xdb, 0x86, 0xe4, 0xe7, + 0x10, 0x26, 0x46, 0x53, 0xfa, 0x50, 0xd1, 0x14, 0x4f, 0x3a, 0x33, 0x87, 0x98, 0x74, 0x2a, 0xfd, + 0x54, 0x3c, 0x1e, 0x3f, 0xd5, 0xbe, 0x4a, 0x41, 0x39, 0x99, 0x76, 0x9f, 0x8c, 0x2d, 0x28, 0xef, + 0xa6, 0xf4, 0xa1, 0x77, 0xd3, 0x5b, 0x30, 0x49, 0x2a, 0x33, 0xdd, 0xf7, 0xf9, 0x05, 0xdb, 0x0c, + 0x2d, 0xae, 0x58, 0x36, 0x0a, 0xac, 0x95, 0x10, 0x2e, 0x65, 0x23, 0x01, 0x3e, 0x12, 0xba, 0xd9, + 0x23, 0x86, 0xee, 0x27, 0x29, 0x98, 0xdc, 0xb0, 0x8d, 0x3b, 0xac, 0x68, 0xf3, 0x9f, 0x14, 0x7b, + 0x3e, 0xce, 0x94, 0x56, 0x9b, 0x86, 0x49, 0xa9, 0x6a, 0xab, 0x7d, 0xca, 0xe2, 0x4c, 0xfe, 0x5c, + 0xfd, 0xf7, 0xd9, 0x65, 0x0a, 0x26, 0xc4, 0xf2, 0xaf, 0xd6, 0x80, 0xe9, 0x44, 0xb5, 0x26, 0xbe, + 0x80, 0x76, 0x98, 0x17, 0xa8, 0x5d, 0x81, 0x59, 0x55, 0x19, 0x23, 0x64, 0x1d, 0xed, 0x10, 0xa7, + 0x33, 0x57, 0x61, 0x56, 0x55, 0x8e, 0x1c, 0x7d, 0x39, 0x6f, 0xf2, 0x93, 0x4f, 0x56, 0x38, 0x1c, + 0x9d, 0xff, 0xaf, 0x51, 0xf7, 0x1c, 0x5f, 0x66, 0x7f, 0x1b, 0xca, 0x4e, 0xf8, 0xd0, 0xe2, 0x3d, + 0x1a, 0xdb, 0x96, 0xb4, 0xe3, 0x88, 0x70, 0x6b, 0x89, 0x66, 0x6d, 0x4a, 0xc6, 0xc8, 0x72, 0x78, + 0xff, 0x96, 0x53, 0xc8, 0x69, 0x26, 0x1a, 0xb9, 0x29, 0x19, 0x23, 0x98, 0x36, 0x7f, 0xb0, 0x69, + 0x69, 0xff, 0x97, 0x25, 0x4d, 0xf3, 0x74, 0xe2, 0xb2, 0x3d, 0xba, 0x08, 0x05, 0xfa, 0x4b, 0xb8, + 0xb8, 0xf3, 0xa5, 0xd6, 0xa1, 0x30, 0x69, 0x01, 0x79, 0x0e, 0x42, 0xaf, 0x40, 0x31, 0xba, 0x7f, + 0xcf, 0xcf, 0x3c, 0x59, 0xdc, 0x85, 0x40, 0x29, 0xee, 0x42, 0x20, 0x6f, 0x9a, 0x7f, 0x02, 0xa7, + 0xc7, 0xde, 0xbc, 0x3f, 0x4a, 0x0f, 0x2e, 0x74, 0xbf, 0x99, 0x23, 0x75, 0xbf, 0x7b, 0x30, 0xaf, + 0xbe, 0x10, 0x2f, 0x68, 0x4f, 0x1d, 0xa8, 0x3d, 0xb6, 0x7e, 0xfa, 0x90, 0xd6, 0x4f, 0xd5, 0x76, + 0xe8, 0xb8, 0x20, 0xba, 0x78, 0x8e, 0x2e, 0x40, 0xd6, 0xb1, 0xed, 0x9e, 0xc7, 0x2f, 0x15, 0x50, + 0x75, 0x14, 0x20, 0xaa, 0xa3, 0x80, 0x87, 0x18, 0x4e, 0x04, 0x61, 0x04, 0xc7, 0xd7, 0xe8, 0x1f, + 0x83, 0x75, 0x9f, 0xbf, 0x08, 0x85, 0xf0, 0xe0, 0x16, 0x01, 0xe4, 0xde, 0xb9, 0xbb, 0x7a, 0x77, + 0xf5, 0x4a, 0xf9, 0x04, 0x2a, 0x41, 0x7e, 0x63, 0xf5, 0xd6, 0x95, 0xeb, 0xb7, 0xae, 0x96, 0x35, + 0xf2, 0xd0, 0xbc, 0x7b, 0xeb, 0x16, 0x79, 0x48, 0x3d, 0x7f, 0x53, 0xbc, 0x0c, 0xc6, 0x2b, 0xb7, + 0x09, 0x28, 0xac, 0x38, 0x0e, 0x4d, 0x21, 0x8c, 0x77, 0x75, 0xd7, 0x24, 0x3b, 0xb9, 0xac, 0xa1, + 0x3c, 0xa4, 0x6f, 0xdf, 0x5e, 0x2f, 0xa7, 0xd0, 0x2c, 0x94, 0xaf, 0x60, 0xdd, 0xe8, 0x99, 0x16, + 0x0e, 0xf3, 0x56, 0x39, 0xdd, 0xb8, 0xff, 0xa7, 0x6f, 0x16, 0xb5, 0xaf, 0xbe, 0x59, 0xd4, 0xfe, + 0xf1, 0xcd, 0xa2, 0xf6, 0xd9, 0xb7, 0x8b, 0x27, 0xbe, 0xfa, 0x76, 0xf1, 0xc4, 0xdf, 0xbe, 0x5d, + 0x3c, 0xf1, 0xa3, 0x8b, 0x5d, 0xd3, 0xdf, 0x0e, 0xda, 0xf5, 0x8e, 0xdd, 0xe7, 0x3f, 0xe9, 0x75, + 0x5c, 0x9b, 0x24, 0x08, 0xfe, 0xb4, 0x9c, 0xfc, 0xad, 0xef, 0x6f, 0x52, 0x67, 0x56, 0xe8, 0xe3, + 0x06, 0xa3, 0xab, 0x5f, 0xb7, 0xeb, 0x0c, 0x40, 0x7f, 0xdd, 0xe9, 0xb5, 0x73, 0xf4, 0x57, 0x9c, + 0x2f, 0xff, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x5a, 0x65, 0xa8, 0x22, 0x26, 0x3c, 0x00, 0x00, } func (m *EventSequence) Marshal() (dAtA []byte, err error) { @@ -4566,6 +4576,32 @@ func (m *ResourceUtilisation) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.AvgResourcesForPeriod) > 0 { + for k := range m.AvgResourcesForPeriod { + v := m.AvgResourcesForPeriod[k] + baseI := i + if v != nil { + { + size, err := v.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintEvents(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + i -= len(k) + copy(dAtA[i:], k) + i = encodeVarintEvents(dAtA, i, uint64(len(k))) + i-- + dAtA[i] = 0xa + i = encodeVarintEvents(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0x42 + } + } if len(m.RunId) > 0 { i -= len(m.RunId) copy(dAtA[i:], m.RunId) @@ -5349,20 +5385,20 @@ func (m *JobSetFilter) MarshalToSizedBuffer(dAtA []byte) (int, error) { var l int _ = l if len(m.States) > 0 { - dAtA41 := make([]byte, len(m.States)*10) - var j40 int + dAtA42 := make([]byte, len(m.States)*10) + var j41 int for _, num := range m.States { for num >= 1<<7 { - dAtA41[j40] = uint8(uint64(num)&0x7f | 0x80) + dAtA42[j41] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j40++ + j41++ } - dAtA41[j40] = uint8(num) - j40++ + dAtA42[j41] = uint8(num) + j41++ } - i -= j40 - copy(dAtA[i:], dAtA41[:j40]) - i = encodeVarintEvents(dAtA, i, uint64(j40)) + i -= j41 + copy(dAtA[i:], dAtA42[:j41]) + i = encodeVarintEvents(dAtA, i, uint64(j41)) i-- dAtA[i] = 0xa } @@ -5397,20 +5433,20 @@ func (m *CancelJobSet) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x12 } if len(m.States) > 0 { - dAtA43 := make([]byte, len(m.States)*10) - var j42 int + dAtA44 := make([]byte, len(m.States)*10) + var j43 int for _, num := range m.States { for num >= 1<<7 { - dAtA43[j42] = uint8(uint64(num)&0x7f | 0x80) + dAtA44[j43] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j42++ + j43++ } - dAtA43[j42] = uint8(num) - j42++ + dAtA44[j43] = uint8(num) + j43++ } - i -= j42 - copy(dAtA[i:], dAtA43[:j42]) - i = encodeVarintEvents(dAtA, i, uint64(j42)) + i -= j43 + copy(dAtA[i:], dAtA44[:j43]) + i = encodeVarintEvents(dAtA, i, uint64(j43)) i-- dAtA[i] = 0xa } @@ -7485,6 +7521,19 @@ func (m *ResourceUtilisation) Size() (n int) { if l > 0 { n += 1 + l + sovEvents(uint64(l)) } + if len(m.AvgResourcesForPeriod) > 0 { + for k, v := range m.AvgResourcesForPeriod { + _ = k + _ = v + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovEvents(uint64(l)) + } + mapEntrySize := 1 + len(k) + sovEvents(uint64(len(k))) + l + n += mapEntrySize + 1 + sovEvents(uint64(mapEntrySize)) + } + } return n } @@ -10108,6 +10157,135 @@ func (m *ResourceUtilisation) Unmarshal(dAtA []byte) error { } m.RunId = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AvgResourcesForPeriod", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.AvgResourcesForPeriod == nil { + m.AvgResourcesForPeriod = make(map[string]*resource.Quantity) + } + var mapkey string + var mapvalue *resource.Quantity + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthEvents + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return ErrInvalidLengthEvents + } + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var mapmsglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapmsglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if mapmsglen < 0 { + return ErrInvalidLengthEvents + } + postmsgIndex := iNdEx + mapmsglen + if postmsgIndex < 0 { + return ErrInvalidLengthEvents + } + if postmsgIndex > l { + return io.ErrUnexpectedEOF + } + mapvalue = &resource.Quantity{} + if err := mapvalue.Unmarshal(dAtA[iNdEx:postmsgIndex]); err != nil { + return err + } + iNdEx = postmsgIndex + } else { + iNdEx = entryPreIndex + skippy, err := skipEvents(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthEvents + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.AvgResourcesForPeriod[mapkey] = mapvalue + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipEvents(dAtA[iNdEx:]) diff --git a/pkg/armadaevents/events.proto b/pkg/armadaevents/events.proto index 8905344026b..b8e9eb50101 100644 --- a/pkg/armadaevents/events.proto +++ b/pkg/armadaevents/events.proto @@ -115,6 +115,7 @@ message ResourceUtilisation { map total_cumulative_usage = 5; string job_id = 6; string run_id = 7; + map avg_resources_for_period = 8; } From 680550e8f59e7d87c2df79b85ad7734f2bd45aca Mon Sep 17 00:00:00 2001 From: Gary Edwards <66478610+geaere@users.noreply.github.com> Date: Fri, 31 Jan 2025 14:57:50 +0000 Subject: [PATCH 17/23] Gary E to list of maintainers (#343) (#4179) --- .mergify.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mergify.yml b/.mergify.yml index c9ee5a0e548..e4c1d8077fc 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -9,6 +9,6 @@ pull_request_rules: - "#approved-reviews-by>=2" - and: - "#approved-reviews-by>=1" - - "author~=^(d80tb7|dave[-]gantenbein|dejanzele|eleanorpratt|JamesMurkin|mauriceyap|masipauskas|MustafaI|zuqq|richscott|robertdavidsmith|samclark|suprjinx)" + - "author~=^(d80tb7|dave[-]gantenbein|dejanzele|eleanorpratt|geaere|JamesMurkin|mauriceyap|masipauskas|MustafaI|zuqq|richscott|robertdavidsmith|samclark|suprjinx)" title: Two are checks required. From 907733849700d1db05aac07602e1ae5ef30082be Mon Sep 17 00:00:00 2001 From: Maurice Yap Date: Fri, 31 Jan 2025 17:23:38 +0000 Subject: [PATCH 18/23] Fix auth scheduling reports (#4180) In the Lookout UI, this fixes the authentication of requests to get scheduling reports. --- .../services/lookoutV2/useGetJobSchedulingReport.ts | 11 +++++------ .../services/lookoutV2/useGetQueueSchedulingReport.ts | 11 +++++------ .../src/services/lookoutV2/useGetSchedulingReport.ts | 11 +++++------ 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/internal/lookout/ui/src/services/lookoutV2/useGetJobSchedulingReport.ts b/internal/lookout/ui/src/services/lookoutV2/useGetJobSchedulingReport.ts index f295ceef064..82f710d50a3 100644 --- a/internal/lookout/ui/src/services/lookoutV2/useGetJobSchedulingReport.ts +++ b/internal/lookout/ui/src/services/lookoutV2/useGetJobSchedulingReport.ts @@ -21,13 +21,12 @@ export const useGetJobSchedulingReport = (jobId: string, enabled = true) => { queryKey: ["getJobSchedulingReport", jobId], queryFn: async ({ signal }) => { try { - const headers: HeadersInit = {} + const accessToken = userManager === undefined ? undefined : await getAccessToken(userManager) - if (userManager !== undefined) { - Object.assign(headers, getAuthorizationHeaders(await getAccessToken(userManager))) - } - - return await schedulerReportingApi.getJobReport({ jobId }, { headers, signal }) + return await schedulerReportingApi.getJobReport( + { jobId }, + { headers: accessToken ? getAuthorizationHeaders(accessToken) : undefined, signal }, + ) } catch (e) { throw await getErrorMessage(e) } diff --git a/internal/lookout/ui/src/services/lookoutV2/useGetQueueSchedulingReport.ts b/internal/lookout/ui/src/services/lookoutV2/useGetQueueSchedulingReport.ts index 664da0cda13..09299121b55 100644 --- a/internal/lookout/ui/src/services/lookoutV2/useGetQueueSchedulingReport.ts +++ b/internal/lookout/ui/src/services/lookoutV2/useGetQueueSchedulingReport.ts @@ -21,13 +21,12 @@ export const useGetQueueSchedulingReport = (queueName: string, verbosity: number queryKey: ["getQueueSchedulingReport", queueName, verbosity], queryFn: async ({ signal }) => { try { - const headers: HeadersInit = {} + const accessToken = userManager === undefined ? undefined : await getAccessToken(userManager) - if (userManager !== undefined) { - Object.assign(headers, getAuthorizationHeaders(await getAccessToken(userManager))) - } - - return await schedulerReportingApi.getQueueReport({ queueName, verbosity }, { headers, signal }) + return await schedulerReportingApi.getQueueReport( + { queueName, verbosity }, + { headers: accessToken ? getAuthorizationHeaders(accessToken) : undefined, signal }, + ) } catch (e) { throw await getErrorMessage(e) } diff --git a/internal/lookout/ui/src/services/lookoutV2/useGetSchedulingReport.ts b/internal/lookout/ui/src/services/lookoutV2/useGetSchedulingReport.ts index 1f171748d36..2633596a532 100644 --- a/internal/lookout/ui/src/services/lookoutV2/useGetSchedulingReport.ts +++ b/internal/lookout/ui/src/services/lookoutV2/useGetSchedulingReport.ts @@ -21,13 +21,12 @@ export const useGetSchedulingReport = (verbosity: number, enabled = true) => { queryKey: ["getSchedulingReport", verbosity], queryFn: async ({ signal }) => { try { - const headers: HeadersInit = {} + const accessToken = userManager === undefined ? undefined : await getAccessToken(userManager) - if (userManager !== undefined) { - Object.assign(headers, getAuthorizationHeaders(await getAccessToken(userManager))) - } - - return await schedulerReportingApi.getSchedulingReport({ verbosity }, { signal }) + return await schedulerReportingApi.getSchedulingReport( + { verbosity }, + { headers: accessToken ? getAuthorizationHeaders(accessToken) : undefined, signal }, + ) } catch (e) { throw await getErrorMessage(e) } From 7215ddad4774d186835d70fe492ed5b379bdf86e Mon Sep 17 00:00:00 2001 From: JamesMurkin Date: Mon, 3 Feb 2025 16:12:04 +0000 Subject: [PATCH 19/23] Fix stale cycle metrics by resetting each round (#4176) Signed-off-by: JamesMurkin --- internal/scheduler/metrics/cycle_metrics.go | 280 +++++++++--------- .../scheduler/metrics/cycle_metrics_test.go | 99 ++++--- 2 files changed, 193 insertions(+), 186 deletions(-) diff --git a/internal/scheduler/metrics/cycle_metrics.go b/internal/scheduler/metrics/cycle_metrics.go index 6f0cf8c54f6..6a299818faa 100644 --- a/internal/scheduler/metrics/cycle_metrics.go +++ b/internal/scheduler/metrics/cycle_metrics.go @@ -1,6 +1,7 @@ package metrics import ( + "sync/atomic" "time" "github.com/prometheus/client_golang/prometheus" @@ -15,51 +16,28 @@ var ( poolQueueAndResourceLabels = []string{poolLabel, queueLabel, resourceLabel} ) -type cycleMetrics struct { - leaderMetricsEnabled bool - - scheduledJobs *prometheus.CounterVec - premptedJobs *prometheus.CounterVec - consideredJobs *prometheus.GaugeVec - fairShare *prometheus.GaugeVec - adjustedFairShare *prometheus.GaugeVec - actualShare *prometheus.GaugeVec - fairnessError *prometheus.GaugeVec - demand *prometheus.GaugeVec - cappedDemand *prometheus.GaugeVec - queueWeight *prometheus.GaugeVec - rawQueueWeight *prometheus.GaugeVec - scheduleCycleTime prometheus.Histogram - reconciliationCycleTime prometheus.Histogram - gangsConsidered *prometheus.GaugeVec - gangsScheduled *prometheus.GaugeVec - firstGangQueuePosition *prometheus.GaugeVec - lastGangQueuePosition *prometheus.GaugeVec - perQueueCycleTime *prometheus.GaugeVec - loopNumber *prometheus.GaugeVec - evictedJobs *prometheus.GaugeVec - evictedResources *prometheus.GaugeVec - spotPrice *prometheus.GaugeVec - allResettableMetrics []resettableMetric +type perCycleMetrics struct { + consideredJobs *prometheus.GaugeVec + fairShare *prometheus.GaugeVec + adjustedFairShare *prometheus.GaugeVec + actualShare *prometheus.GaugeVec + fairnessError *prometheus.GaugeVec + demand *prometheus.GaugeVec + cappedDemand *prometheus.GaugeVec + queueWeight *prometheus.GaugeVec + rawQueueWeight *prometheus.GaugeVec + gangsConsidered *prometheus.GaugeVec + gangsScheduled *prometheus.GaugeVec + firstGangQueuePosition *prometheus.GaugeVec + lastGangQueuePosition *prometheus.GaugeVec + perQueueCycleTime *prometheus.GaugeVec + loopNumber *prometheus.GaugeVec + evictedJobs *prometheus.GaugeVec + evictedResources *prometheus.GaugeVec + spotPrice *prometheus.GaugeVec } -func newCycleMetrics() *cycleMetrics { - scheduledJobs := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: prefix + "scheduled_jobs", - Help: "Number of events scheduled", - }, - queueAndPriorityClassLabels, - ) - - premptedJobs := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: prefix + "preempted_jobs", - Help: "Number of jobs preempted", - }, - queueAndPriorityClassLabels, - ) - +func newPerCycleMetrics() *perCycleMetrics { consideredJobs := prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: prefix + "considered_jobs", @@ -132,22 +110,6 @@ func newCycleMetrics() *cycleMetrics { []string{poolLabel}, ) - scheduleCycleTime := prometheus.NewHistogram( - prometheus.HistogramOpts{ - Name: prefix + "schedule_cycle_times", - Help: "Cycle time when in a scheduling round.", - Buckets: prometheus.ExponentialBuckets(10.0, 1.1, 110), - }, - ) - - reconciliationCycleTime := prometheus.NewHistogram( - prometheus.HistogramOpts{ - Name: prefix + "reconciliation_cycle_times", - Help: "Cycle time when in a scheduling round.", - Buckets: prometheus.ExponentialBuckets(10.0, 1.1, 110), - }, - ) - gangsConsidered := prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: prefix + "gangs_considered", @@ -220,10 +182,7 @@ func newCycleMetrics() *cycleMetrics { poolLabels, ) - return &cycleMetrics{ - leaderMetricsEnabled: true, - scheduledJobs: scheduledJobs, - premptedJobs: premptedJobs, + return &perCycleMetrics{ consideredJobs: consideredJobs, fairShare: fairShare, adjustedFairShare: adjustedFairShare, @@ -233,7 +192,6 @@ func newCycleMetrics() *cycleMetrics { queueWeight: queueWeight, rawQueueWeight: rawQueueWeight, fairnessError: fairnessError, - scheduleCycleTime: scheduleCycleTime, gangsConsidered: gangsConsidered, gangsScheduled: gangsScheduled, firstGangQueuePosition: firstGangQueuePosition, @@ -243,30 +201,62 @@ func newCycleMetrics() *cycleMetrics { evictedJobs: evictedJobs, evictedResources: evictedResources, spotPrice: spotPrice, - allResettableMetrics: []resettableMetric{ - scheduledJobs, - premptedJobs, - consideredJobs, - fairShare, - adjustedFairShare, - actualShare, - demand, - cappedDemand, - fairnessError, - gangsConsidered, - gangsScheduled, - firstGangQueuePosition, - lastGangQueuePosition, - perQueueCycleTime, - loopNumber, - evictedJobs, - evictedResources, - spotPrice, - queueWeight, - rawQueueWeight, + } +} + +type cycleMetrics struct { + leaderMetricsEnabled bool + + scheduledJobs *prometheus.CounterVec + premptedJobs *prometheus.CounterVec + scheduleCycleTime prometheus.Histogram + reconciliationCycleTime prometheus.Histogram + latestCycleMetrics atomic.Pointer[perCycleMetrics] +} + +func newCycleMetrics() *cycleMetrics { + scheduledJobs := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: prefix + "scheduled_jobs", + Help: "Number of events scheduled", }, + queueAndPriorityClassLabels, + ) + + premptedJobs := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: prefix + "preempted_jobs", + Help: "Number of jobs preempted", + }, + queueAndPriorityClassLabels, + ) + + scheduleCycleTime := prometheus.NewHistogram( + prometheus.HistogramOpts{ + Name: prefix + "schedule_cycle_times", + Help: "Cycle time when in a scheduling round.", + Buckets: prometheus.ExponentialBuckets(10.0, 1.1, 110), + }, + ) + + reconciliationCycleTime := prometheus.NewHistogram( + prometheus.HistogramOpts{ + Name: prefix + "reconciliation_cycle_times", + Help: "Cycle time when in a scheduling round.", + Buckets: prometheus.ExponentialBuckets(10.0, 1.1, 110), + }, + ) + + cycleMetrics := &cycleMetrics{ + leaderMetricsEnabled: true, + scheduledJobs: scheduledJobs, + premptedJobs: premptedJobs, + scheduleCycleTime: scheduleCycleTime, reconciliationCycleTime: reconciliationCycleTime, + latestCycleMetrics: atomic.Pointer[perCycleMetrics]{}, } + cycleMetrics.latestCycleMetrics.Store(newPerCycleMetrics()) + return cycleMetrics } func (m *cycleMetrics) enableLeaderMetrics() { @@ -279,9 +269,9 @@ func (m *cycleMetrics) disableLeaderMetrics() { } func (m *cycleMetrics) resetLeaderMetrics() { - for _, metric := range m.allResettableMetrics { - metric.Reset() - } + m.premptedJobs.Reset() + m.scheduledJobs.Reset() + m.latestCycleMetrics.Store(newPerCycleMetrics()) } func (m *cycleMetrics) ReportScheduleCycleTime(cycleTime time.Duration) { @@ -294,6 +284,7 @@ func (m *cycleMetrics) ReportReconcileCycleTime(cycleTime time.Duration) { func (m *cycleMetrics) ReportSchedulerResult(result scheduling.SchedulerResult) { // Metrics that depend on pool + currentCycle := newPerCycleMetrics() for _, schedContext := range result.SchedulingContexts { pool := schedContext.Pool for queue, queueContext := range schedContext.QueueSchedulingContexts { @@ -302,17 +293,17 @@ func (m *cycleMetrics) ReportSchedulerResult(result scheduling.SchedulerResult) demand := schedContext.FairnessCostProvider.UnweightedCostFromAllocation(queueContext.Demand) cappedDemand := schedContext.FairnessCostProvider.UnweightedCostFromAllocation(queueContext.CappedDemand) - m.consideredJobs.WithLabelValues(pool, queue).Set(jobsConsidered) - m.fairShare.WithLabelValues(pool, queue).Set(queueContext.FairShare) - m.adjustedFairShare.WithLabelValues(pool, queue).Set(queueContext.AdjustedFairShare) - m.actualShare.WithLabelValues(pool, queue).Set(actualShare) - m.demand.WithLabelValues(pool, queue).Set(demand) - m.cappedDemand.WithLabelValues(pool, queue).Set(cappedDemand) - m.queueWeight.WithLabelValues(pool, queue).Set(queueContext.Weight) - m.rawQueueWeight.WithLabelValues(pool, queue).Set(queueContext.RawWeight) + currentCycle.consideredJobs.WithLabelValues(pool, queue).Set(jobsConsidered) + currentCycle.fairShare.WithLabelValues(pool, queue).Set(queueContext.FairShare) + currentCycle.adjustedFairShare.WithLabelValues(pool, queue).Set(queueContext.AdjustedFairShare) + currentCycle.actualShare.WithLabelValues(pool, queue).Set(actualShare) + currentCycle.demand.WithLabelValues(pool, queue).Set(demand) + currentCycle.cappedDemand.WithLabelValues(pool, queue).Set(cappedDemand) + currentCycle.queueWeight.WithLabelValues(pool, queue).Set(queueContext.Weight) + currentCycle.rawQueueWeight.WithLabelValues(pool, queue).Set(queueContext.RawWeight) } - m.fairnessError.WithLabelValues(pool).Set(schedContext.FairnessError()) - m.spotPrice.WithLabelValues(pool).Set(schedContext.SpotPrice) + currentCycle.fairnessError.WithLabelValues(pool).Set(schedContext.FairnessError()) + currentCycle.spotPrice.WithLabelValues(pool).Set(schedContext.SpotPrice) } for _, jobCtx := range result.ScheduledJobs { @@ -325,48 +316,51 @@ func (m *cycleMetrics) ReportSchedulerResult(result scheduling.SchedulerResult) for pool, schedulingStats := range result.PerPoolSchedulingStats { for queue, s := range schedulingStats.StatsPerQueue { - m.gangsConsidered.WithLabelValues(pool, queue).Set(float64(s.GangsConsidered)) - m.gangsScheduled.WithLabelValues(pool, queue).Set(float64(s.GangsScheduled)) - m.firstGangQueuePosition.WithLabelValues(pool, queue).Set(float64(s.FirstGangConsideredQueuePosition)) - m.lastGangQueuePosition.WithLabelValues(pool, queue).Set(float64(s.LastGangScheduledQueuePosition)) - m.perQueueCycleTime.WithLabelValues(pool, queue).Set(float64(s.Time.Milliseconds())) + currentCycle.gangsConsidered.WithLabelValues(pool, queue).Set(float64(s.GangsConsidered)) + currentCycle.gangsScheduled.WithLabelValues(pool, queue).Set(float64(s.GangsScheduled)) + currentCycle.firstGangQueuePosition.WithLabelValues(pool, queue).Set(float64(s.FirstGangConsideredQueuePosition)) + currentCycle.lastGangQueuePosition.WithLabelValues(pool, queue).Set(float64(s.LastGangScheduledQueuePosition)) + currentCycle.perQueueCycleTime.WithLabelValues(pool, queue).Set(float64(s.Time.Milliseconds())) } - m.loopNumber.WithLabelValues(pool).Set(float64(schedulingStats.LoopNumber)) + currentCycle.loopNumber.WithLabelValues(pool).Set(float64(schedulingStats.LoopNumber)) for queue, s := range schedulingStats.EvictorResult.GetStatsPerQueue() { - m.evictedJobs.WithLabelValues(pool, queue).Set(float64(s.EvictedJobCount)) + currentCycle.evictedJobs.WithLabelValues(pool, queue).Set(float64(s.EvictedJobCount)) for _, r := range s.EvictedResources.GetResources() { - m.evictedResources.WithLabelValues(pool, queue, r.Name).Set(float64(r.RawValue)) + currentCycle.evictedResources.WithLabelValues(pool, queue, r.Name).Set(float64(r.RawValue)) } } } + m.latestCycleMetrics.Store(currentCycle) } func (m *cycleMetrics) describe(ch chan<- *prometheus.Desc) { if m.leaderMetricsEnabled { m.scheduledJobs.Describe(ch) m.premptedJobs.Describe(ch) - m.consideredJobs.Describe(ch) - m.fairShare.Describe(ch) - m.adjustedFairShare.Describe(ch) - m.actualShare.Describe(ch) - m.fairnessError.Describe(ch) - m.demand.Describe(ch) - m.cappedDemand.Describe(ch) - m.queueWeight.Describe(ch) - m.rawQueueWeight.Describe(ch) m.scheduleCycleTime.Describe(ch) - m.gangsConsidered.Describe(ch) - m.gangsScheduled.Describe(ch) - m.firstGangQueuePosition.Describe(ch) - m.lastGangQueuePosition.Describe(ch) - m.perQueueCycleTime.Describe(ch) - m.loopNumber.Describe(ch) - m.evictedJobs.Describe(ch) - m.evictedResources.Describe(ch) - m.spotPrice.Describe(ch) + + currentCycle := m.latestCycleMetrics.Load() + currentCycle.consideredJobs.Describe(ch) + currentCycle.fairShare.Describe(ch) + currentCycle.adjustedFairShare.Describe(ch) + currentCycle.actualShare.Describe(ch) + currentCycle.fairnessError.Describe(ch) + currentCycle.demand.Describe(ch) + currentCycle.cappedDemand.Describe(ch) + currentCycle.queueWeight.Describe(ch) + currentCycle.rawQueueWeight.Describe(ch) + currentCycle.gangsConsidered.Describe(ch) + currentCycle.gangsScheduled.Describe(ch) + currentCycle.firstGangQueuePosition.Describe(ch) + currentCycle.lastGangQueuePosition.Describe(ch) + currentCycle.perQueueCycleTime.Describe(ch) + currentCycle.loopNumber.Describe(ch) + currentCycle.evictedJobs.Describe(ch) + currentCycle.evictedResources.Describe(ch) + currentCycle.spotPrice.Describe(ch) } m.reconciliationCycleTime.Describe(ch) @@ -376,25 +370,27 @@ func (m *cycleMetrics) collect(ch chan<- prometheus.Metric) { if m.leaderMetricsEnabled { m.scheduledJobs.Collect(ch) m.premptedJobs.Collect(ch) - m.consideredJobs.Collect(ch) - m.fairShare.Collect(ch) - m.adjustedFairShare.Collect(ch) - m.actualShare.Collect(ch) - m.fairnessError.Collect(ch) - m.demand.Collect(ch) - m.cappedDemand.Collect(ch) - m.rawQueueWeight.Collect(ch) - m.queueWeight.Collect(ch) m.scheduleCycleTime.Collect(ch) - m.gangsConsidered.Collect(ch) - m.gangsScheduled.Collect(ch) - m.firstGangQueuePosition.Collect(ch) - m.lastGangQueuePosition.Collect(ch) - m.perQueueCycleTime.Collect(ch) - m.loopNumber.Collect(ch) - m.evictedJobs.Collect(ch) - m.evictedResources.Collect(ch) - m.spotPrice.Collect(ch) + + currentCycle := m.latestCycleMetrics.Load() + currentCycle.consideredJobs.Collect(ch) + currentCycle.fairShare.Collect(ch) + currentCycle.adjustedFairShare.Collect(ch) + currentCycle.actualShare.Collect(ch) + currentCycle.fairnessError.Collect(ch) + currentCycle.demand.Collect(ch) + currentCycle.cappedDemand.Collect(ch) + currentCycle.rawQueueWeight.Collect(ch) + currentCycle.queueWeight.Collect(ch) + currentCycle.gangsConsidered.Collect(ch) + currentCycle.gangsScheduled.Collect(ch) + currentCycle.firstGangQueuePosition.Collect(ch) + currentCycle.lastGangQueuePosition.Collect(ch) + currentCycle.perQueueCycleTime.Collect(ch) + currentCycle.loopNumber.Collect(ch) + currentCycle.evictedJobs.Collect(ch) + currentCycle.evictedResources.Collect(ch) + currentCycle.spotPrice.Collect(ch) } m.reconciliationCycleTime.Collect(ch) diff --git a/internal/scheduler/metrics/cycle_metrics_test.go b/internal/scheduler/metrics/cycle_metrics_test.go index 4c07b001413..4fbebfadfa7 100644 --- a/internal/scheduler/metrics/cycle_metrics_test.go +++ b/internal/scheduler/metrics/cycle_metrics_test.go @@ -61,31 +61,27 @@ func TestReportStateTransitions(t *testing.T) { poolQueue := []string{"pool1", "queue1"} - consideredJobs := testutil.ToFloat64(m.consideredJobs.WithLabelValues(poolQueue...)) + consideredJobs := testutil.ToFloat64(m.latestCycleMetrics.Load().consideredJobs.WithLabelValues(poolQueue...)) assert.Equal(t, 3.0, consideredJobs, "consideredJobs") - allocated := testutil.ToFloat64(m.actualShare.WithLabelValues(poolQueue...)) + allocated := testutil.ToFloat64(m.latestCycleMetrics.Load().actualShare.WithLabelValues(poolQueue...)) assert.InDelta(t, 0.1, allocated, epsilon, "allocated") - demand := testutil.ToFloat64(m.demand.WithLabelValues(poolQueue...)) + demand := testutil.ToFloat64(m.latestCycleMetrics.Load().demand.WithLabelValues(poolQueue...)) assert.InDelta(t, 0.2, demand, epsilon, "demand") - cappedDemand := testutil.ToFloat64(m.cappedDemand.WithLabelValues(poolQueue...)) + cappedDemand := testutil.ToFloat64(m.latestCycleMetrics.Load().cappedDemand.WithLabelValues(poolQueue...)) assert.InDelta(t, 0.15, cappedDemand, epsilon, "cappedDemand") - adjustedFairShare := testutil.ToFloat64(m.adjustedFairShare.WithLabelValues(poolQueue...)) + adjustedFairShare := testutil.ToFloat64(m.latestCycleMetrics.Load().adjustedFairShare.WithLabelValues(poolQueue...)) assert.InDelta(t, 0.15, adjustedFairShare, epsilon, "adjustedFairShare") - fairnessError := testutil.ToFloat64(m.fairnessError.WithLabelValues("pool1")) + fairnessError := testutil.ToFloat64(m.latestCycleMetrics.Load().fairnessError.WithLabelValues("pool1")) assert.InDelta(t, 0.05, fairnessError, epsilon, "fairnessError") } -func TestResetLeaderMetrics(t *testing.T) { +func TestResetLeaderMetrics_Counters(t *testing.T) { m := newCycleMetrics() - - poolLabelValues := []string{"pool1"} - poolQueueLabelValues := []string{"pool1", "queue1"} - poolQueueResouceLabelValues := []string{"pool1", "queue1", "cpu"} queuePriorityClassLabelValues := []string{"pool1", "priorityClass1"} testResetCounter := func(vec *prometheus.CounterVec, labelValues []string) { @@ -96,32 +92,47 @@ func TestResetLeaderMetrics(t *testing.T) { counterVal = testutil.ToFloat64(vec.WithLabelValues(labelValues...)) assert.Equal(t, 0.0, counterVal) } - testResetGauge := func(vec *prometheus.GaugeVec, labelValues []string) { + + testResetCounter(m.scheduledJobs, queuePriorityClassLabelValues) + testResetCounter(m.premptedJobs, queuePriorityClassLabelValues) +} + +func TestResetLeaderMetrics_ResetsLatestCycleMetrics(t *testing.T) { + m := newCycleMetrics() + poolLabelValues := []string{"pool1"} + poolQueueLabelValues := []string{"pool1", "queue1"} + poolQueueResourceLabelValues := []string{"pool1", "queue1", "cpu"} + + testResetGauge := func(getVec func(metrics *cycleMetrics) *prometheus.GaugeVec, labelValues []string) { + vec := getVec(m) vec.WithLabelValues(labelValues...).Inc() counterVal := testutil.ToFloat64(vec.WithLabelValues(labelValues...)) assert.Equal(t, 1.0, counterVal) m.resetLeaderMetrics() + vec = getVec(m) counterVal = testutil.ToFloat64(vec.WithLabelValues(labelValues...)) assert.Equal(t, 0.0, counterVal) } - testResetCounter(m.scheduledJobs, queuePriorityClassLabelValues) - testResetCounter(m.premptedJobs, queuePriorityClassLabelValues) - testResetGauge(m.consideredJobs, poolQueueLabelValues) - testResetGauge(m.fairShare, poolQueueLabelValues) - testResetGauge(m.adjustedFairShare, poolQueueLabelValues) - testResetGauge(m.actualShare, poolQueueLabelValues) - testResetGauge(m.fairnessError, []string{"pool1"}) - testResetGauge(m.demand, poolQueueLabelValues) - testResetGauge(m.cappedDemand, poolQueueLabelValues) - testResetGauge(m.gangsConsidered, poolQueueLabelValues) - testResetGauge(m.gangsScheduled, poolQueueLabelValues) - testResetGauge(m.firstGangQueuePosition, poolQueueLabelValues) - testResetGauge(m.lastGangQueuePosition, poolQueueLabelValues) - testResetGauge(m.perQueueCycleTime, poolQueueLabelValues) - testResetGauge(m.loopNumber, poolLabelValues) - testResetGauge(m.evictedJobs, poolQueueLabelValues) - testResetGauge(m.evictedResources, poolQueueResouceLabelValues) + testResetGauge(func(metrics *cycleMetrics) *prometheus.GaugeVec { return m.latestCycleMetrics.Load().consideredJobs }, poolQueueLabelValues) + testResetGauge(func(metrics *cycleMetrics) *prometheus.GaugeVec { return m.latestCycleMetrics.Load().fairShare }, poolQueueLabelValues) + testResetGauge(func(metrics *cycleMetrics) *prometheus.GaugeVec { return m.latestCycleMetrics.Load().adjustedFairShare }, poolQueueLabelValues) + testResetGauge(func(metrics *cycleMetrics) *prometheus.GaugeVec { return m.latestCycleMetrics.Load().actualShare }, poolQueueLabelValues) + testResetGauge(func(metrics *cycleMetrics) *prometheus.GaugeVec { return m.latestCycleMetrics.Load().fairnessError }, []string{"pool1"}) + testResetGauge(func(metrics *cycleMetrics) *prometheus.GaugeVec { return m.latestCycleMetrics.Load().demand }, poolQueueLabelValues) + testResetGauge(func(metrics *cycleMetrics) *prometheus.GaugeVec { return m.latestCycleMetrics.Load().cappedDemand }, poolQueueLabelValues) + testResetGauge(func(metrics *cycleMetrics) *prometheus.GaugeVec { return m.latestCycleMetrics.Load().gangsConsidered }, poolQueueLabelValues) + testResetGauge(func(metrics *cycleMetrics) *prometheus.GaugeVec { return m.latestCycleMetrics.Load().gangsScheduled }, poolQueueLabelValues) + testResetGauge(func(metrics *cycleMetrics) *prometheus.GaugeVec { + return m.latestCycleMetrics.Load().firstGangQueuePosition + }, poolQueueLabelValues) + testResetGauge(func(metrics *cycleMetrics) *prometheus.GaugeVec { + return m.latestCycleMetrics.Load().lastGangQueuePosition + }, poolQueueLabelValues) + testResetGauge(func(metrics *cycleMetrics) *prometheus.GaugeVec { return m.latestCycleMetrics.Load().perQueueCycleTime }, poolQueueLabelValues) + testResetGauge(func(metrics *cycleMetrics) *prometheus.GaugeVec { return m.latestCycleMetrics.Load().loopNumber }, poolLabelValues) + testResetGauge(func(metrics *cycleMetrics) *prometheus.GaugeVec { return m.latestCycleMetrics.Load().evictedJobs }, poolQueueLabelValues) + testResetGauge(func(metrics *cycleMetrics) *prometheus.GaugeVec { return m.latestCycleMetrics.Load().evictedResources }, poolQueueResourceLabelValues) } func TestDisableLeaderMetrics(t *testing.T) { @@ -133,23 +144,23 @@ func TestDisableLeaderMetrics(t *testing.T) { collect := func(m *cycleMetrics) []prometheus.Metric { m.scheduledJobs.WithLabelValues(queuePriorityClassLabelValues...).Inc() m.premptedJobs.WithLabelValues(queuePriorityClassLabelValues...).Inc() - m.consideredJobs.WithLabelValues(poolQueueLabelValues...).Inc() - m.fairShare.WithLabelValues(poolQueueLabelValues...).Inc() - m.adjustedFairShare.WithLabelValues(poolQueueLabelValues...).Inc() - m.actualShare.WithLabelValues(poolQueueLabelValues...).Inc() - m.fairnessError.WithLabelValues("pool1").Inc() - m.demand.WithLabelValues(poolQueueLabelValues...).Inc() - m.cappedDemand.WithLabelValues(poolQueueLabelValues...).Inc() + m.latestCycleMetrics.Load().consideredJobs.WithLabelValues(poolQueueLabelValues...).Inc() + m.latestCycleMetrics.Load().fairShare.WithLabelValues(poolQueueLabelValues...).Inc() + m.latestCycleMetrics.Load().adjustedFairShare.WithLabelValues(poolQueueLabelValues...).Inc() + m.latestCycleMetrics.Load().actualShare.WithLabelValues(poolQueueLabelValues...).Inc() + m.latestCycleMetrics.Load().fairnessError.WithLabelValues("pool1").Inc() + m.latestCycleMetrics.Load().demand.WithLabelValues(poolQueueLabelValues...).Inc() + m.latestCycleMetrics.Load().cappedDemand.WithLabelValues(poolQueueLabelValues...).Inc() m.scheduleCycleTime.Observe(float64(1000)) m.reconciliationCycleTime.Observe(float64(1000)) - m.gangsConsidered.WithLabelValues("pool1", "queue1").Inc() - m.gangsScheduled.WithLabelValues("pool1", "queue1").Inc() - m.firstGangQueuePosition.WithLabelValues("pool1", "queue1").Inc() - m.lastGangQueuePosition.WithLabelValues("pool1", "queue1").Inc() - m.perQueueCycleTime.WithLabelValues("pool1", "queue1").Inc() - m.loopNumber.WithLabelValues("pool1").Inc() - m.evictedJobs.WithLabelValues("pool1", "queue1").Inc() - m.evictedResources.WithLabelValues("pool1", "queue1", "cpu").Inc() + m.latestCycleMetrics.Load().gangsConsidered.WithLabelValues("pool1", "queue1").Inc() + m.latestCycleMetrics.Load().gangsScheduled.WithLabelValues("pool1", "queue1").Inc() + m.latestCycleMetrics.Load().firstGangQueuePosition.WithLabelValues("pool1", "queue1").Inc() + m.latestCycleMetrics.Load().lastGangQueuePosition.WithLabelValues("pool1", "queue1").Inc() + m.latestCycleMetrics.Load().perQueueCycleTime.WithLabelValues("pool1", "queue1").Inc() + m.latestCycleMetrics.Load().loopNumber.WithLabelValues("pool1").Inc() + m.latestCycleMetrics.Load().evictedJobs.WithLabelValues("pool1", "queue1").Inc() + m.latestCycleMetrics.Load().evictedResources.WithLabelValues("pool1", "queue1", "cpu").Inc() ch := make(chan prometheus.Metric, 1000) m.collect(ch) From 4cfc872fb4831c589c7b343fad68eb77e77170de Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Mon, 3 Feb 2025 19:40:03 +0000 Subject: [PATCH 20/23] Allow logging to be configured by file (#4155) * update pulsar mock * go mod tidy * import order * wip * wip * wip * supress logging * lint * lint * wip * wip * wip * wip * wip * wip * wip * remove logrus * remove logrus * fix test * fix armadactl * clean up interfaces * clean up interfaces * wip * added back json logging * lint * fix test * add back log tests * wip * wip * wip * lint * fix skip frames * fixed milli time * formatting * formatting * more tests * lint * colorful * fixes * fixes * fixes * fixes * wip * lint * lint * wip * wip * update docker * wip * add logfile config * add prometheus hook * add prometheus hook * add prometheus hook * go mod tidy * unit test for prometheus * go mod tidy * go mod tidy * added development logging * fix * fix * fix * merged master * lint * minor fixes --- .goreleaser.yml | 12 ++ .run/Event Ingester.run.xml | 3 +- .run/Executor.run.xml | 3 +- .run/Lookout Ingester V2.run.xml | 1 + .run/LookoutV2.run.xml | 1 + .run/Scheduler Ingester.run.xml | 3 +- .run/Scheduler.run.xml | 1 + .run/Server.run.xml | 1 + .run/lookoutv2PostgresMigration.run.xml | 3 +- .run/schedulerPostgresMigration.run.xml | 1 + build/binoculars/Dockerfile | 1 + build/bundles/armada/Dockerfile | 3 + build/bundles/lookout/Dockerfile | 3 + build/eventingester/Dockerfile | 2 + build/executor/Dockerfile | 1 + build/fakeexecutor/Dockerfile | 1 + build/lookoutingesterv2/Dockerfile | 1 + build/lookoutv2/Dockerfile | 1 + build/scheduler/Dockerfile | 1 + build/scheduleringester/Dockerfile | 1 + build/server/Dockerfile | 1 + cmd/armada-load-tester/main.go | 4 +- cmd/armadactl/main.go | 4 +- cmd/binoculars/main.go | 2 +- cmd/eventingester/main.go | 3 +- cmd/executor/main.go | 2 +- cmd/fakeexecutor/main.go | 2 +- cmd/lookoutingesterv2/dbloadtester/main.go | 2 +- cmd/lookoutingesterv2/main.go | 2 +- cmd/lookoutv2/main.go | 4 +- cmd/scheduler/main.go | 3 +- cmd/scheduleringester/main.go | 6 +- cmd/server/main.go | 2 +- cmd/simulator/cmd/root.go | 4 +- cmd/testsuite/main.go | 4 +- config/logging-dev.yaml | 13 ++ config/logging.yaml | 13 ++ go.mod | 3 +- go.sum | 3 + internal/common/logging/application.go | 186 +++++++++++++++++++++ internal/common/logging/cli.go | 23 +++ internal/common/logging/config.go | 117 +++++++++++++ internal/common/logging/global.go | 2 +- internal/common/startup.go | 80 --------- 44 files changed, 422 insertions(+), 107 deletions(-) create mode 100644 config/logging-dev.yaml create mode 100644 config/logging.yaml create mode 100644 internal/common/logging/application.go create mode 100644 internal/common/logging/cli.go create mode 100644 internal/common/logging/config.go diff --git a/.goreleaser.yml b/.goreleaser.yml index c55622020d4..51a56c20e22 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -197,6 +197,7 @@ dockers: - scheduler - scheduleringester extra_files: + - config/logging.yaml - config/server/config.yaml - config/executor/config.yaml - config/binoculars/config.yaml @@ -217,6 +218,7 @@ dockers: - lookoutv2 - lookoutingesterv2 extra_files: + - config/logging.yaml - config/lookoutv2/config.yaml - config/lookoutingesterv2/config.yaml - internal/lookout/ui @@ -245,6 +247,7 @@ dockers: - lookoutingesterv2 - fakeexecutor extra_files: + - config/logging.yaml - config/server/config.yaml - config/executor/config.yaml - config/binoculars/config.yaml @@ -271,6 +274,7 @@ dockers: - server extra_files: - config/server/config.yaml + - config/logging.yaml dockerfile: ./build/server/Dockerfile - id: executor @@ -285,6 +289,7 @@ dockers: - executor extra_files: - config/executor/config.yaml + - config/logging.yaml dockerfile: ./build/executor/Dockerfile - id: fakeexecutor @@ -299,6 +304,7 @@ dockers: - fakeexecutor extra_files: - config/executor/config.yaml + - config/logging.yaml dockerfile: ./build/fakeexecutor/Dockerfile - id: armadaloadtester @@ -337,6 +343,7 @@ dockers: - lookoutingesterv2 extra_files: - config/lookoutingesterv2/config.yaml + - config/logging.yaml dockerfile: ./build/lookoutingesterv2/Dockerfile - id: lookoutv2 @@ -357,6 +364,7 @@ dockers: - pkg/api/schedulerobjects/api.swagger.json - config/lookoutv2/config.yaml - config/lookoutingesterv2/config.yaml + - config/logging.yaml dockerfile: ./build/lookoutv2/Dockerfile - id: eventingester @@ -371,6 +379,7 @@ dockers: - eventingester extra_files: - config/eventingester/config.yaml + - config/logging.yaml dockerfile: ./build/eventingester/Dockerfile - id: scheduler @@ -385,6 +394,7 @@ dockers: - scheduler extra_files: - config/scheduler/config.yaml + - config/logging.yaml dockerfile: ./build/scheduler/Dockerfile - id: scheduleringester @@ -399,6 +409,7 @@ dockers: - scheduleringester extra_files: - config/scheduleringester/config.yaml + - config/logging.yaml dockerfile: ./build/scheduleringester/Dockerfile - id: binoculars @@ -413,6 +424,7 @@ dockers: - binoculars extra_files: - config/binoculars/config.yaml + - config/logging.yaml dockerfile: ./build/binoculars/Dockerfile - id: armadactl diff --git a/.run/Event Ingester.run.xml b/.run/Event Ingester.run.xml index a1bf12c1763..87246d38b64 100644 --- a/.run/Event Ingester.run.xml +++ b/.run/Event Ingester.run.xml @@ -3,6 +3,7 @@ + @@ -13,4 +14,4 @@ - \ No newline at end of file + diff --git a/.run/Executor.run.xml b/.run/Executor.run.xml index 8fb53f874a5..70698729834 100644 --- a/.run/Executor.run.xml +++ b/.run/Executor.run.xml @@ -6,6 +6,7 @@ + @@ -17,4 +18,4 @@ - \ No newline at end of file + diff --git a/.run/Lookout Ingester V2.run.xml b/.run/Lookout Ingester V2.run.xml index 58e7bdbdfdd..f522609ffb0 100644 --- a/.run/Lookout Ingester V2.run.xml +++ b/.run/Lookout Ingester V2.run.xml @@ -3,6 +3,7 @@ + diff --git a/.run/LookoutV2.run.xml b/.run/LookoutV2.run.xml index d15d0832d3a..a85ddf17726 100644 --- a/.run/LookoutV2.run.xml +++ b/.run/LookoutV2.run.xml @@ -4,6 +4,7 @@ + diff --git a/.run/Scheduler Ingester.run.xml b/.run/Scheduler Ingester.run.xml index 86c7b5d46bc..00240a5a9df 100644 --- a/.run/Scheduler Ingester.run.xml +++ b/.run/Scheduler Ingester.run.xml @@ -3,6 +3,7 @@ + @@ -12,4 +13,4 @@ - \ No newline at end of file + diff --git a/.run/Scheduler.run.xml b/.run/Scheduler.run.xml index d9beb38c734..3b549dbd567 100644 --- a/.run/Scheduler.run.xml +++ b/.run/Scheduler.run.xml @@ -6,6 +6,7 @@ + diff --git a/.run/Server.run.xml b/.run/Server.run.xml index e773635dc2c..556b8aebcf5 100644 --- a/.run/Server.run.xml +++ b/.run/Server.run.xml @@ -7,6 +7,7 @@ + diff --git a/.run/lookoutv2PostgresMigration.run.xml b/.run/lookoutv2PostgresMigration.run.xml index 1ce1eea1d08..b8f4290c280 100644 --- a/.run/lookoutv2PostgresMigration.run.xml +++ b/.run/lookoutv2PostgresMigration.run.xml @@ -5,6 +5,7 @@ + @@ -14,4 +15,4 @@ - \ No newline at end of file + diff --git a/.run/schedulerPostgresMigration.run.xml b/.run/schedulerPostgresMigration.run.xml index 13779dfae75..2f9b03e5de1 100644 --- a/.run/schedulerPostgresMigration.run.xml +++ b/.run/schedulerPostgresMigration.run.xml @@ -9,6 +9,7 @@ + diff --git a/build/binoculars/Dockerfile b/build/binoculars/Dockerfile index 8d58cc1d228..dda83838f48 100644 --- a/build/binoculars/Dockerfile +++ b/build/binoculars/Dockerfile @@ -9,6 +9,7 @@ USER armada COPY binoculars /app/ COPY config/binoculars/config.yaml /app/config/binoculars/config.yaml +COPY config/logging.yaml /app/config/logging.yaml WORKDIR /app diff --git a/build/bundles/armada/Dockerfile b/build/bundles/armada/Dockerfile index 0955bd18d75..d4b870bd8ea 100644 --- a/build/bundles/armada/Dockerfile +++ b/build/bundles/armada/Dockerfile @@ -7,6 +7,9 @@ LABEL org.opencontainers.image.url=https://hub.docker.com/r/gresearch/armada RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada USER armada +# Logging (share between all components) +COPY config/logging.yaml /app/config/logging.yaml + # Server COPY config/server/config.yaml /app/config/server/config.yaml diff --git a/build/bundles/lookout/Dockerfile b/build/bundles/lookout/Dockerfile index c01574341f8..5d5354bafc9 100644 --- a/build/bundles/lookout/Dockerfile +++ b/build/bundles/lookout/Dockerfile @@ -8,6 +8,9 @@ LABEL org.opencontainers.image.url=https://hub.docker.com/r/gresearch/armada-loo RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada USER armada +# Logging (shared between both components) +COPY config/logging.yaml /app/config/logging.yaml + COPY config/lookoutingesterv2/config.yaml /app/config/lookoutingesterv2/config.yaml COPY config/lookoutv2/config.yaml /app/config/lookoutv2/config.yaml diff --git a/build/eventingester/Dockerfile b/build/eventingester/Dockerfile index 2b1a8bb8d0b..236bbd21ec7 100644 --- a/build/eventingester/Dockerfile +++ b/build/eventingester/Dockerfile @@ -9,5 +9,7 @@ RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada USER armada COPY eventingester /app/ COPY config/eventingester/config.yaml /app/config/eventingester/config.yaml +COPY config/logging.yaml /app/config/logging.yaml + WORKDIR /app ENTRYPOINT ["./eventingester"] diff --git a/build/executor/Dockerfile b/build/executor/Dockerfile index 343896ad9d7..b1fdfded70b 100644 --- a/build/executor/Dockerfile +++ b/build/executor/Dockerfile @@ -10,6 +10,7 @@ USER armada COPY executor /app/ COPY config/executor/config.yaml /app/config/executor/config.yaml +COPY config/logging.yaml /app/config/logging.yaml WORKDIR /app diff --git a/build/fakeexecutor/Dockerfile b/build/fakeexecutor/Dockerfile index efdb47f30aa..db31d788344 100644 --- a/build/fakeexecutor/Dockerfile +++ b/build/fakeexecutor/Dockerfile @@ -9,6 +9,7 @@ USER armada COPY fakeexecutor /app/ COPY config/executor/config.yaml /app/config/executor/config.yaml +COPY config/logging.yaml /app/config/logging.yaml WORKDIR /app diff --git a/build/lookoutingesterv2/Dockerfile b/build/lookoutingesterv2/Dockerfile index c2f953facd2..e5c70002b22 100644 --- a/build/lookoutingesterv2/Dockerfile +++ b/build/lookoutingesterv2/Dockerfile @@ -10,6 +10,7 @@ USER armada COPY lookoutingesterv2 /app/ COPY config/lookoutingesterv2/config.yaml /app/config/lookoutingesterv2/config.yaml +COPY config/logging.yaml /app/config/logging.yaml WORKDIR /app diff --git a/build/lookoutv2/Dockerfile b/build/lookoutv2/Dockerfile index 6f374c3bebe..a8dde1600c9 100644 --- a/build/lookoutv2/Dockerfile +++ b/build/lookoutv2/Dockerfile @@ -27,5 +27,6 @@ COPY lookoutv2 /app/ COPY config/lookoutv2/config.yaml /app/config/lookoutv2/config.yaml COPY lookoutingesterv2 /app/ COPY config/lookoutingesterv2/ /app/config/lookoutingesterv2 +COPY config/logging.yaml /app/config/logging.yaml WORKDIR /app ENTRYPOINT ["./lookoutv2"] diff --git a/build/scheduler/Dockerfile b/build/scheduler/Dockerfile index 6aa5524bb7f..8713880e0fa 100644 --- a/build/scheduler/Dockerfile +++ b/build/scheduler/Dockerfile @@ -9,5 +9,6 @@ RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada USER armada COPY scheduler /app/ COPY config/scheduler/config.yaml /app/config/scheduler/config.yaml +COPY config/logging.yaml /app/config/logging.yaml WORKDIR /app ENTRYPOINT ["./scheduler"] diff --git a/build/scheduleringester/Dockerfile b/build/scheduleringester/Dockerfile index cf86787e726..233445a40c4 100644 --- a/build/scheduleringester/Dockerfile +++ b/build/scheduleringester/Dockerfile @@ -9,5 +9,6 @@ RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada USER armada COPY scheduleringester /app/ COPY config/scheduleringester/config.yaml /app/config/scheduleringester/config.yaml +COPY config/logging.yaml /app/config/logging.yaml WORKDIR /app ENTRYPOINT ["./scheduleringester"] diff --git a/build/server/Dockerfile b/build/server/Dockerfile index 4453057c381..fa8def55ef2 100644 --- a/build/server/Dockerfile +++ b/build/server/Dockerfile @@ -10,6 +10,7 @@ USER armada COPY server /app/ COPY config/server/config.yaml /app/config/server/config.yaml +COPY config/logging.yaml /app/config/logging.yaml WORKDIR /app diff --git a/cmd/armada-load-tester/main.go b/cmd/armada-load-tester/main.go index c025b3830c2..85bb12b862c 100644 --- a/cmd/armada-load-tester/main.go +++ b/cmd/armada-load-tester/main.go @@ -2,10 +2,10 @@ package main import ( "github.com/armadaproject/armada/cmd/armada-load-tester/cmd" - "github.com/armadaproject/armada/internal/common" + "github.com/armadaproject/armada/internal/common/logging" ) func main() { - common.ConfigureCommandLineLogging() + logging.ConfigureCliLogging() cmd.Execute() } diff --git a/cmd/armadactl/main.go b/cmd/armadactl/main.go index b90993fc2ff..2168fe69c66 100644 --- a/cmd/armadactl/main.go +++ b/cmd/armadactl/main.go @@ -4,12 +4,12 @@ import ( "os" "github.com/armadaproject/armada/cmd/armadactl/cmd" - "github.com/armadaproject/armada/internal/common" + "github.com/armadaproject/armada/internal/common/logging" ) // Config is handled by cmd/params.go func main() { - common.ConfigureCommandLineLogging() + logging.ConfigureCliLogging() root := cmd.RootCmd() if err := root.Execute(); err != nil { // We don't need to log the error here because cobra has already done this for us diff --git a/cmd/binoculars/main.go b/cmd/binoculars/main.go index 5c0558aeb3a..cd4e0221766 100644 --- a/cmd/binoculars/main.go +++ b/cmd/binoculars/main.go @@ -35,7 +35,7 @@ func init() { } func main() { - common.ConfigureLogging() + log.MustConfigureApplicationLogging() common.BindCommandlineArguments() var config configuration.BinocularsConfig diff --git a/cmd/eventingester/main.go b/cmd/eventingester/main.go index 93690f33b4d..c6cca9d2791 100644 --- a/cmd/eventingester/main.go +++ b/cmd/eventingester/main.go @@ -1,6 +1,7 @@ package main import ( + "github.com/armadaproject/armada/internal/common/logging" "github.com/armadaproject/armada/internal/eventingester" "github.com/spf13/pflag" @@ -24,7 +25,7 @@ func init() { } func main() { - common.ConfigureLogging() + logging.MustConfigureApplicationLogging() common.BindCommandlineArguments() var config configuration.EventIngesterConfiguration diff --git a/cmd/executor/main.go b/cmd/executor/main.go index a91c0304f49..75567416a1f 100644 --- a/cmd/executor/main.go +++ b/cmd/executor/main.go @@ -31,7 +31,7 @@ func init() { } func main() { - common.ConfigureLogging() + log.MustConfigureApplicationLogging() common.BindCommandlineArguments() var config configuration.ExecutorConfiguration diff --git a/cmd/fakeexecutor/main.go b/cmd/fakeexecutor/main.go index 69d9ce03c33..61b97a38d7e 100644 --- a/cmd/fakeexecutor/main.go +++ b/cmd/fakeexecutor/main.go @@ -29,7 +29,7 @@ func init() { } func main() { - common.ConfigureLogging() + log.MustConfigureApplicationLogging() common.BindCommandlineArguments() var config configuration.ExecutorConfiguration diff --git a/cmd/lookoutingesterv2/dbloadtester/main.go b/cmd/lookoutingesterv2/dbloadtester/main.go index 87f91fa1f73..95e4b834673 100644 --- a/cmd/lookoutingesterv2/dbloadtester/main.go +++ b/cmd/lookoutingesterv2/dbloadtester/main.go @@ -45,7 +45,7 @@ const ReportTemplate string = ` ` func main() { - common.ConfigureLogging() + log.MustConfigureApplicationLogging() common.BindCommandlineArguments() var config configuration.LookoutIngesterV2Configuration diff --git a/cmd/lookoutingesterv2/main.go b/cmd/lookoutingesterv2/main.go index a0e48a6374f..70be8d713f2 100644 --- a/cmd/lookoutingesterv2/main.go +++ b/cmd/lookoutingesterv2/main.go @@ -27,7 +27,7 @@ func init() { } func main() { - common.ConfigureLogging() + log.MustConfigureApplicationLogging() common.BindCommandlineArguments() var config configuration.LookoutIngesterV2Configuration diff --git a/cmd/lookoutv2/main.go b/cmd/lookoutv2/main.go index da8bb63e57c..584aac3edf3 100644 --- a/cmd/lookoutv2/main.go +++ b/cmd/lookoutv2/main.go @@ -117,7 +117,7 @@ func prune(ctx *armadacontext.Context, config configuration.LookoutV2Config) { } func main() { - common.ConfigureLogging() + log.MustConfigureApplicationLogging() common.BindCommandlineArguments() var config configuration.LookoutV2Config @@ -148,7 +148,7 @@ func main() { restapi.UIConfig = config.UIConfig if err := lookoutv2.Serve(config); err != nil { - log.Error(err) + log.Error(err.Error()) os.Exit(1) } } diff --git a/cmd/scheduler/main.go b/cmd/scheduler/main.go index f895b3301cd..7249a2c54d9 100644 --- a/cmd/scheduler/main.go +++ b/cmd/scheduler/main.go @@ -6,10 +6,11 @@ import ( "github.com/armadaproject/armada/cmd/scheduler/cmd" "github.com/armadaproject/armada/internal/common" + "github.com/armadaproject/armada/internal/common/logging" ) func main() { - common.ConfigureLogging() + logging.MustConfigureApplicationLogging() common.BindCommandlineArguments() if err := cmd.RootCmd().Execute(); err != nil { os.Exit(1) diff --git a/cmd/scheduleringester/main.go b/cmd/scheduleringester/main.go index 9aa5207a2ce..625b2a97da6 100644 --- a/cmd/scheduleringester/main.go +++ b/cmd/scheduleringester/main.go @@ -7,9 +7,9 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" - "github.com/armadaproject/armada/internal/scheduleringester" - "github.com/armadaproject/armada/internal/common" + "github.com/armadaproject/armada/internal/common/logging" + "github.com/armadaproject/armada/internal/scheduleringester" ) const CustomConfigLocation string = "config" @@ -24,7 +24,7 @@ func init() { } func main() { - common.ConfigureLogging() + logging.MustConfigureApplicationLogging() common.BindCommandlineArguments() var config scheduleringester.Configuration diff --git a/cmd/server/main.go b/cmd/server/main.go index 3815049164a..38286f393b8 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -36,7 +36,7 @@ func init() { } func main() { - common.ConfigureLogging() + logging.MustConfigureApplicationLogging() common.BindCommandlineArguments() // TODO Load relevant config in one place: don't use viper here and in LoadConfig. diff --git a/cmd/simulator/cmd/root.go b/cmd/simulator/cmd/root.go index 1ddd0e6ba34..1e758e23afb 100644 --- a/cmd/simulator/cmd/root.go +++ b/cmd/simulator/cmd/root.go @@ -138,11 +138,11 @@ func runSimulations(cmd *cobra.Command, args []string) error { log.Infof("Will write profiling information to %s", profilingFile) f, err := os.Create(profilingFile) if err != nil { - log.Fatal(err) + log.Fatal(err.Error()) } err = pprof.StartCPUProfile(f) if err != nil { - log.Fatal(err) + log.Fatal(err.Error()) } defer pprof.StopCPUProfile() } diff --git a/cmd/testsuite/main.go b/cmd/testsuite/main.go index 2c069956574..f6961a60f71 100644 --- a/cmd/testsuite/main.go +++ b/cmd/testsuite/main.go @@ -4,12 +4,12 @@ import ( "os" "github.com/armadaproject/armada/cmd/testsuite/cmd" - "github.com/armadaproject/armada/internal/common" + "github.com/armadaproject/armada/internal/common/logging" ) // Config is handled by cmd/params.go func main() { - common.ConfigureCommandLineLogging() + logging.ConfigureCliLogging() err := cmd.RootCmd().Execute() if err != nil { os.Exit(1) diff --git a/config/logging-dev.yaml b/config/logging-dev.yaml new file mode 100644 index 00000000000..4d5b15de924 --- /dev/null +++ b/config/logging-dev.yaml @@ -0,0 +1,13 @@ +console: + level: INFO + format: colourful +file: + enabled: false + level: INFO + format: json + logfile: app.log + rotation: + maxSizeMb: 5 + maxBackups: 3 + maxAgeDays: 7 + compress: false diff --git a/config/logging.yaml b/config/logging.yaml new file mode 100644 index 00000000000..2ab93296349 --- /dev/null +++ b/config/logging.yaml @@ -0,0 +1,13 @@ +console: + level: INFO + format: text +file: + enabled: false + level: INFO + format: json + logfile: app.log + rotation: + maxSizeMb: 5 + maxBackups: 3 + maxAgeDays: 7 + compress: false diff --git a/go.mod b/go.mod index 7b83f9caaeb..a0059ae638d 100644 --- a/go.mod +++ b/go.mod @@ -86,6 +86,8 @@ require ( golang.org/x/time v0.5.0 google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 gopkg.in/inf.v0 v0.9.1 + gopkg.in/natefinch/lumberjack.v2 v2.0.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -206,7 +208,6 @@ require ( google.golang.org/protobuf v1.35.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/cli-runtime v0.26.15 // indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect diff --git a/go.sum b/go.sum index f07df3fa1ae..57e76d0342f 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/AthenZ/athenz v1.10.4/go.mod h1:ZKAbcckIMkqD2UKqBU2amZoynztPrgYcsmZ93 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= @@ -873,6 +875,7 @@ gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eR gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= diff --git a/internal/common/logging/application.go b/internal/common/logging/application.go new file mode 100644 index 00000000000..ec9bc9051f6 --- /dev/null +++ b/internal/common/logging/application.go @@ -0,0 +1,186 @@ +package logging + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/pkg/errors" + "github.com/rs/zerolog" + "gopkg.in/natefinch/lumberjack.v2" +) + +const ( + defaultConfigPath = "config/logging.yaml" + configPathEnvVar = "ARMADA_LOG_CONFIG" + RFC3339Milli = "2006-01-02T15:04:05.000Z07:00" + + // legacy env vars: will be removed in a future release + legacyLogFormatEnvVar = "LOG_FORMAT" + legacyLogLevelEnvVar = "LOG_LEVEL" +) + +// MustConfigureApplicationLogging sets up logging suitable for an application. Logging configuration is loaded from +// a filepath given by the ARMADA_LOG_CONFIG environmental variable or from config/logging.yaml if this var is unset. +// Note that this function will immediately shut down the application if it fails. +func MustConfigureApplicationLogging() { + err := ConfigureApplicationLogging() + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, "Error initializing logging: "+err.Error()) + os.Exit(1) + } +} + +// ConfigureApplicationLogging sets up logging suitable for an application. Logging configuration is loaded from +// a filepath given by the ARMADA_LOG_CONFIG environmental variable or from config/logging.yaml if this var is unset. +func ConfigureApplicationLogging() error { + // Set some global logging properties + zerolog.TimeFieldFormat = RFC3339Milli // needs to be higher or greater precision than the writer format. + zerolog.CallerMarshalFunc = shortCallerEncoder + + // Load config file + configPath := getEnv(configPathEnvVar, defaultConfigPath) + logConfig, err := readConfig(configPath) + if err != nil { + return err + } + + // Console logging + var writers []io.Writer + consoleLogger, err := createConsoleLogger(logConfig) + if err != nil { + return err + } + writers = append(writers, consoleLogger) + + // File logging + if logConfig.File.Enabled { + fileLogger, err := createFileLogger(logConfig) + if err != nil { + return err + } + writers = append(writers, fileLogger) + } + + // Combine loggers + multiWriter := zerolog.MultiLevelWriter(writers...) + logger := zerolog. + New(multiWriter). + Hook(NewPrometheusHook()). + With(). + Timestamp(). + Logger() + + // Set our new logger to be the default + ReplaceStdLogger(FromZerolog(logger)) + return nil +} + +func createFileLogger(logConfig Config) (*FilteredLevelWriter, error) { + level, err := zerolog.ParseLevel(logConfig.File.Level) + if err != nil { + return nil, err + } + + // Set up lumberjack for log rotation + lumberjackLogger := &lumberjack.Logger{ + Filename: logConfig.File.LogFile, + MaxSize: logConfig.File.Rotation.MaxSizeMb, + MaxBackups: logConfig.File.Rotation.MaxBackups, + MaxAge: logConfig.File.Rotation.MaxAgeDays, + Compress: logConfig.File.Rotation.Compress, + } + + return createWriter(lumberjackLogger, level, logConfig.File.Format) +} + +func createConsoleLogger(logConfig Config) (*FilteredLevelWriter, error) { + level, err := zerolog.ParseLevel(logConfig.Console.Level) + if err != nil { + return nil, err + } + return createWriter(os.Stdout, level, logConfig.Console.Format) +} + +func createWriter(out io.Writer, level zerolog.Level, format LogFormat) (*FilteredLevelWriter, error) { + level = overrideLevelFromEnv(level) + format = overrideFormatFromEnv(format) + + switch format { + case FormatJSON: + return &FilteredLevelWriter{ + level: level, + writer: out, + }, nil + case FormatColourful, FormatText: + return &FilteredLevelWriter{ + level: level, + writer: zerolog.ConsoleWriter{ + Out: out, + TimeFormat: RFC3339Milli, + FormatLevel: func(i interface{}) string { + return strings.ToUpper(fmt.Sprintf("%s", i)) + }, + FormatCaller: func(i interface{}) string { + return filepath.Base(fmt.Sprintf("%s", i)) + }, + NoColor: format == FormatText, + }, + }, nil + } + return nil, errors.Errorf("unknown log format: %s", format) +} + +func shortCallerEncoder(_ uintptr, file string, line int) string { + short := file + for i := len(file) - 1; i > 0; i-- { + if file[i] == '/' { + short = file[i+1:] + break + } + } + file = short + return file + ":" + strconv.Itoa(line) +} + +func getEnv(key, defaultValue string) string { + if value, exists := os.LookupEnv(key); exists { + return value + } + return defaultValue +} + +// Sets level from legacy env var. Will be removed in a subsequent release +func overrideLevelFromEnv(level zerolog.Level) zerolog.Level { + levelOverrideStr, ok := os.LookupEnv(legacyLogLevelEnvVar) + if !ok { + return level + } + if ok { + levelOverride, err := zerolog.ParseLevel(levelOverrideStr) + if err == nil { + return levelOverride + } + } + return level +} + +// Sets format from legacy env var. Will be removed in a subsequent release +func overrideFormatFromEnv(format LogFormat) LogFormat { + levelFormatStr, ok := os.LookupEnv(legacyLogFormatEnvVar) + if !ok { + return format + } + switch strings.ToLower(levelFormatStr) { + case "json": + return FormatJSON + case "colourful": + return FormatColourful + case "text": + return FormatText + } + return format +} diff --git a/internal/common/logging/cli.go b/internal/common/logging/cli.go new file mode 100644 index 00000000000..f923c7db8bc --- /dev/null +++ b/internal/common/logging/cli.go @@ -0,0 +1,23 @@ +package logging + +import ( + "os" + + "github.com/rs/zerolog" +) + +func ConfigureCliLogging() { + zerolog.TimestampFieldName = "" + zerolog.LevelFieldName = "" + + // Create a ConsoleWriter that writes to stdout. + // Since we’ve cleared out the field names above, only the message will be printed. + consoleWriter := zerolog.ConsoleWriter{ + Out: os.Stdout, + TimeFormat: "", // No timestamp + } + + // Create a logger that logs at InfoLevel and above. + l := zerolog.New(consoleWriter).Level(zerolog.InfoLevel).With().Logger() + ReplaceStdLogger(FromZerolog(l)) +} diff --git a/internal/common/logging/config.go b/internal/common/logging/config.go new file mode 100644 index 00000000000..94a9fb3c167 --- /dev/null +++ b/internal/common/logging/config.go @@ -0,0 +1,117 @@ +package logging + +import ( + "fmt" + "os" + "strings" + + "github.com/pkg/errors" + "github.com/rs/zerolog" + "gopkg.in/yaml.v3" +) + +type LogFormat string + +// Allowed values for LogFormat. +const ( + FormatText LogFormat = "text" + FormatJSON LogFormat = "json" + FormatColourful LogFormat = "colourful" +) + +// Config defines Armada logging configuration. +type Config struct { + // Defines configuration for console logging on stdout + Console struct { + // Log level, e.g. INFO, ERROR etc + Level string `yaml:"level"` + // Logging format, either text, json or colourful + Format LogFormat `yaml:"format"` + } `yaml:"console"` + // Defines configuration for file logging + File struct { + // Whether file logging is enabled. + Enabled bool `yaml:"enabled"` + // Log level, e.g. INFO, ERROR etc + Level string `yaml:"level"` + // Logging format, either text, json or or colourful + Format LogFormat `yaml:"format"` + // The Location of the logfile on disk + LogFile string `yaml:"logfile"` + // Log Rotation Options + Rotation struct { + // Whether Log Rotation is enabled + Enabled bool `yaml:"enabled"` + // Maximum size in megabytes of the log file before it gets rotated + MaxSizeMb int `yaml:"maxSizeMb"` + // Maximum number of old log files to retain + MaxBackups int `yaml:"maxBackups"` + // Maximum number of days to retain old log files + MaxAgeDays int `yaml:"maxAgeDays"` + // Whether to compress rotated log files + Compress bool `yaml:"compress"` + } `yaml:"rotation"` + } `yaml:"file"` +} + +func validate(c Config) error { + _, err := zerolog.ParseLevel(c.Console.Level) + if err != nil { + return err + } + + if c.File.Enabled { + _, err := zerolog.ParseLevel(c.File.Level) + if err != nil { + return err + } + + rotation := c.File.Rotation + if rotation.Enabled { + if rotation.MaxSizeMb <= 0 { + return errors.New("rotation.maxSizeMb must be greater than zero") + } + if rotation.MaxBackups <= 0 { + return errors.New("rotation.maxBackups must be greater than zero") + } + if rotation.MaxAgeDays <= 0 { + return errors.New("rotation.maxAgeDays must be greater than zero") + } + } + } + + return nil +} + +func readConfig(configFilePath string) (Config, error) { + yamlConfig, err := os.ReadFile(configFilePath) + if err != nil { + return Config{}, errors.Wrap(err, "failed to read log config file") + } + + var config Config + err = yaml.Unmarshal(yamlConfig, &config) + if err != nil { + return Config{}, errors.Wrap(err, "failed to unmarshall log config file") + } + err = validate(config) + if err != nil { + return Config{}, errors.Wrap(err, "invalid log configuration") + } + return config, nil +} + +func (lf *LogFormat) UnmarshalYAML(value *yaml.Node) error { + var s string + if err := value.Decode(&s); err != nil { + return err + } + s = strings.ToLower(s) + switch LogFormat(s) { + case FormatText, FormatJSON, FormatColourful: + *lf = LogFormat(s) + return nil + default: + return fmt.Errorf("invalid log format %q: valid values are %q, %q, or %q", s, FormatText, FormatJSON, FormatColourful) + } +} diff --git a/internal/common/logging/global.go b/internal/common/logging/global.go index 507e1d49478..a67ecaa507b 100644 --- a/internal/common/logging/global.go +++ b/internal/common/logging/global.go @@ -102,7 +102,7 @@ func WithStacktrace(err error) *Logger { func createDefaultLogger() *Logger { consoleWriter := zerolog.ConsoleWriter{ Out: os.Stdout, - TimeFormat: "2006-01-02T15:04:05.000Z07:00", + TimeFormat: RFC3339Milli, NoColor: true, } zerologLogger := zerolog.New(consoleWriter). diff --git a/internal/common/startup.go b/internal/common/startup.go index 87b05f068de..2cde6a2ecb8 100644 --- a/internal/common/startup.go +++ b/internal/common/startup.go @@ -3,17 +3,14 @@ package common import ( "crypto/tls" "fmt" - "io" "net/http" "os" - "path/filepath" "strings" "time" "github.com/mitchellh/mapstructure" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/rs/zerolog" "github.com/spf13/pflag" "github.com/spf13/viper" "golang.org/x/exp/slices" @@ -26,9 +23,6 @@ import ( const baseConfigFileName = "config" -// RFC3339Millis -const logTimestampFormat = "2006-01-02T15:04:05.000Z07:00" - func BindCommandlineArguments() { err := viper.BindPFlags(pflag.CommandLine) if err != nil { @@ -94,80 +88,6 @@ func UnmarshalKey(v *viper.Viper, key string, item interface{}) error { return v.UnmarshalKey(key, item, commonconfig.CustomHooks...) } -// TODO Move logging-related code out of common into a new package internal/logging -func ConfigureCommandLineLogging() { - zerolog.TimestampFieldName = "" - zerolog.LevelFieldName = "" - - // Create a ConsoleWriter that writes to stdout. - // Since we’ve cleared out the field names above, only the message will be printed. - consoleWriter := zerolog.ConsoleWriter{ - Out: os.Stdout, - TimeFormat: "", // No timestamp - } - - l := zerolog.New(consoleWriter).Level(zerolog.InfoLevel).With().Logger() - log.ReplaceStdLogger(log.FromZerolog(l)) -} - -func ConfigureLogging() { - // needs to be higher or greater precision than the writer format. - zerolog.TimeFieldFormat = logTimestampFormat - - level := readEnvironmentLogLevel() - format := readEnvironmentLogFormat() - - var outStream io.Writer = os.Stdout - - if format != "json" { - outStream = zerolog.ConsoleWriter{ - Out: os.Stdout, - TimeFormat: logTimestampFormat, - FormatLevel: func(i interface{}) string { - return strings.ToUpper(fmt.Sprintf("%s", i)) - }, - FormatCaller: func(i interface{}) string { - return filepath.Base(fmt.Sprintf("%s", i)) - }, - NoColor: format == "text", - } - } - - zerologLogger := zerolog.New(outStream). - Level(level). - Hook(log.NewPrometheusHook()). - With(). - CallerWithSkipFrameCount(log.StdSkipFrames). - Timestamp(). - Logger() - - log.ReplaceStdLogger(log.FromZerolog(zerologLogger)) -} - -func readEnvironmentLogLevel() zerolog.Level { - level, ok := os.LookupEnv("LOG_LEVEL") - if ok { - logLevel, err := zerolog.ParseLevel(level) - if err == nil { - return logLevel - } - } - return zerolog.InfoLevel -} - -func readEnvironmentLogFormat() string { - format, ok := os.LookupEnv("LOG_FORMAT") - if ok { - format = strings.ToLower(format) - if format == "text" || format == "colourful" || format == "json" { - return format - } else { - _, _ = fmt.Fprintf(os.Stderr, "Invalid log format %s\n", format) - } - } - return "colourful" -} - func ServeMetrics(port uint16) (shutdown func()) { return ServeMetricsFor(port, prometheus.DefaultGatherer) } From 7c96fa0f31dbb926f9f696c8ad1523ffac5aba7d Mon Sep 17 00:00:00 2001 From: JamesMurkin Date: Tue, 4 Feb 2025 17:45:37 +0000 Subject: [PATCH 21/23] Fix scheduler initialisation startup race (#4132) * Fix scheduler initialisation startup race Currently the SubmitCheck.Run loads all the queues from the queue cache + executors from the database, this relies on QueueCache already having the queues loaded As SubmitCheck.Run and QueueCache.Run are called at the same time in separate go routines, sometimes SubmitCheck.Run happens first and can't find the queues - When this happens, it blocks scheduling for 1 minute (until next SubmitCheck executor refresh) I've added an Initialise method to QueueCache so we can call this during component creation, preventing the race described above Signed-off-by: JamesMurkin * Change executor update frequency - test if this fixes CI * Fix ci Signed-off-by: JamesMurkin * Make sure queue cache and submit check are initialise on startup Signed-off-by: JamesMurkin * Revert config changes Signed-off-by: JamesMurkin * Respect JobLeaseRequestTimeout, reduce CI JobLeaseRequestTimeout to 5s, remove 70s sleep Signed-off-by: JamesMurkin * Format Signed-off-by: JamesMurkin * Set scheduler depending on server Signed-off-by: JamesMurkin * Wait for postgres to start running Signed-off-by: JamesMurkin * Remove server dependency Signed-off-by: JamesMurkin * Improve startup tests Signed-off-by: JamesMurkin * Fix func calls Signed-off-by: JamesMurkin * Move to separate funcs for checking running/ready Signed-off-by: JamesMurkin * Add timeouts + log errors on startup rather than exit Signed-off-by: JamesMurkin * Pass in timeout Signed-off-by: JamesMurkin * Set longer timeout Signed-off-by: JamesMurkin --------- Signed-off-by: JamesMurkin --- developer/env/docker/executor.env | 1 + internal/executor/application.go | 1 + internal/executor/service/job_requester.go | 5 +- .../executor/service/job_requester_test.go | 16 ++++- internal/scheduler/queue/queue_cache.go | 9 +++ internal/scheduler/schedulerapp.go | 12 ++++ internal/scheduler/submitcheck.go | 34 +++++++---- internal/scheduler/submitcheck_test.go | 61 ++++++++++++++++++- magefiles/ci.go | 5 +- magefiles/developer.go | 45 +++++++++++--- magefiles/main.go | 8 ++- 11 files changed, 172 insertions(+), 25 deletions(-) diff --git a/developer/env/docker/executor.env b/developer/env/docker/executor.env index 598509ffa42..0998e394de8 100644 --- a/developer/env/docker/executor.env +++ b/developer/env/docker/executor.env @@ -1,2 +1,3 @@ ARMADA_EXECUTORAPICONNECTION_ARMADAURL="scheduler:50052" ARMADA_EXECUTORAPICONNECTION_FORCENOTLS=true +ARMADA_APPLICATION_JOBLEASEREQUESTTIMEOUT=5s diff --git a/internal/executor/application.go b/internal/executor/application.go index 29b839a4da3..46e8f8be553 100644 --- a/internal/executor/application.go +++ b/internal/executor/application.go @@ -218,6 +218,7 @@ func setupExecutorApiComponents( clusterUtilisationService, config.Kubernetes.PodDefaults, config.Application.MaxLeasedJobs, + config.Application.JobLeaseRequestTimeout, ) clusterAllocationService := service.NewClusterAllocationService( clusterContext, diff --git a/internal/executor/service/job_requester.go b/internal/executor/service/job_requester.go index f6dd2c38b2d..f10b9f20ed3 100644 --- a/internal/executor/service/job_requester.go +++ b/internal/executor/service/job_requester.go @@ -25,6 +25,7 @@ type JobRequester struct { podDefaults *configuration.PodDefaults jobRunStateStore job.RunStateStore maxLeasedJobs int + maxRequestDuration time.Duration } func NewJobRequester( @@ -35,6 +36,7 @@ func NewJobRequester( utilisationService utilisation.UtilisationService, podDefaults *configuration.PodDefaults, maxLeasedJobs int, + maxRequestDuration time.Duration, ) *JobRequester { return &JobRequester{ leaseRequester: leaseRequester, @@ -44,6 +46,7 @@ func NewJobRequester( clusterId: clusterId, podDefaults: podDefaults, maxLeasedJobs: maxLeasedJobs, + maxRequestDuration: maxRequestDuration, } } @@ -53,7 +56,7 @@ func (r *JobRequester) RequestJobsRuns() { log.Errorf("Failed to create lease request because %s", err) return } - ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), 30*time.Second) + ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), r.maxRequestDuration) defer cancel() leaseResponse, err := r.leaseRequester.LeaseJobRuns(ctx, leaseRequest) if err != nil { diff --git a/internal/executor/service/job_requester_test.go b/internal/executor/service/job_requester_test.go index 22b2210cf3a..7455d5ae77e 100644 --- a/internal/executor/service/job_requester_test.go +++ b/internal/executor/service/job_requester_test.go @@ -3,6 +3,7 @@ package service import ( "fmt" "testing" + "time" "github.com/google/uuid" "github.com/stretchr/testify/assert" @@ -23,7 +24,10 @@ import ( "github.com/armadaproject/armada/pkg/executorapi" ) -const defaultMaxLeasedJobs int = 5 +const ( + defaultMaxLeasedJobs int = 5 + defaultMaxRequestDuration = 30 * time.Second +) func TestRequestJobsRuns_HandlesLeaseRequestError(t *testing.T) { jobRequester, eventReporter, leaseRequester, stateStore, _ := setupJobRequesterTest([]*job.RunState{}) @@ -257,7 +261,15 @@ func setupJobRequesterTest(initialJobRuns []*job.RunState) (*JobRequester, *mock utilisationService.ClusterAvailableCapacityReport = &utilisation.ClusterAvailableCapacityReport{ AvailableCapacity: &armadaresource.ComputeResources{}, } - jobRequester := NewJobRequester(clusterId, eventReporter, leaseRequester, stateStore, utilisationService, podDefaults, defaultMaxLeasedJobs) + jobRequester := NewJobRequester( + clusterId, + eventReporter, + leaseRequester, + stateStore, + utilisationService, + podDefaults, + defaultMaxLeasedJobs, + defaultMaxRequestDuration) return jobRequester, eventReporter, leaseRequester, stateStore, utilisationService } diff --git a/internal/scheduler/queue/queue_cache.go b/internal/scheduler/queue/queue_cache.go index e46badc6c11..9a07a5afb8c 100644 --- a/internal/scheduler/queue/queue_cache.go +++ b/internal/scheduler/queue/queue_cache.go @@ -31,6 +31,15 @@ func NewQueueCache(apiClient api.SubmitClient, updateFrequency time.Duration) *A } } +func (c *ApiQueueCache) Initialise(ctx *armadacontext.Context) error { + err := c.fetchQueues(ctx) + if err != nil { + ctx.Errorf("Error initialising queue cache, failed fetching queues: %v", err) + } + + return err +} + func (c *ApiQueueCache) Run(ctx *armadacontext.Context) error { if err := c.fetchQueues(ctx); err != nil { ctx.Warnf("Error fetching queues: %v", err) diff --git a/internal/scheduler/schedulerapp.go b/internal/scheduler/schedulerapp.go index a20d05a2ae1..60418bc26fd 100644 --- a/internal/scheduler/schedulerapp.go +++ b/internal/scheduler/schedulerapp.go @@ -126,6 +126,12 @@ func Run(config schedulerconfig.Configuration) error { }() armadaClient := api.NewSubmitClient(conn) queueCache := queue.NewQueueCache(armadaClient, config.QueueRefreshPeriod) + queueCacheInitTimeout, cancel := armadacontext.WithTimeout(ctx, time.Second*30) + defer cancel() + err = queueCache.Initialise(queueCacheInitTimeout) + if err != nil { + ctx.Errorf("error initialising queue cache - %v", err) + } services = append(services, func() error { return queueCache.Run(ctx) }) // //////////////////////////////////////////////////////////////////////// @@ -262,6 +268,12 @@ func Run(config schedulerconfig.Configuration) error { floatingResourceTypes, resourceListFactory, ) + submitCheckerInitTimeout, cancel := armadacontext.WithTimeout(ctx, time.Second*30) + defer cancel() + err = submitChecker.Initialise(submitCheckerInitTimeout) + if err != nil { + ctx.Errorf("error initialising submit checker - %v", err) + } services = append(services, func() error { return submitChecker.Run(ctx) }) diff --git a/internal/scheduler/submitcheck.go b/internal/scheduler/submitcheck.go index 40afd8ea91f..5a91d758de1 100644 --- a/internal/scheduler/submitcheck.go +++ b/internal/scheduler/submitcheck.go @@ -72,36 +72,44 @@ func NewSubmitChecker( } } +func (srv *SubmitChecker) Initialise(ctx *armadacontext.Context) error { + err := srv.updateExecutors(ctx) + if err != nil { + ctx.Logger().WithStacktrace(err).Errorf("Error initialising submit checker") + } + + return err +} + func (srv *SubmitChecker) Run(ctx *armadacontext.Context) error { ctx.Infof("Will refresh executor state every %s", srv.schedulingConfig.ExecutorUpdateFrequency) - srv.updateExecutors(ctx) + if err := srv.updateExecutors(ctx); err != nil { + ctx.Logger().WithStacktrace(err).Error("Failed updating executors") + } + ticker := time.NewTicker(srv.schedulingConfig.ExecutorUpdateFrequency) for { select { case <-ctx.Done(): return nil case <-ticker.C: - srv.updateExecutors(ctx) + if err := srv.updateExecutors(ctx); err != nil { + ctx.Logger().WithStacktrace(err).Error("Failed updating executors") + } } } } -func (srv *SubmitChecker) updateExecutors(ctx *armadacontext.Context) { +func (srv *SubmitChecker) updateExecutors(ctx *armadacontext.Context) error { queues, err := srv.queueCache.GetAll(ctx) if err != nil { - ctx.Logger(). - WithStacktrace(err). - Error("Error fetching queues") - return + return fmt.Errorf("failed fetching queues from queue cache - %s", err) } - executors, err := srv.executorRepository.GetExecutors(ctx) if err != nil { - ctx.Logger(). - WithStacktrace(err). - Error("Error fetching executors") - return + return fmt.Errorf("failed fetching executors from db - %s", err) } + ctx.Infof("Retrieved %d executors", len(executors)) jobSchedulingResultsCache, err := lru.New(10000) if err != nil { @@ -166,6 +174,8 @@ func (srv *SubmitChecker) updateExecutors(ctx *armadacontext.Context) { constraintsByPool: constraintsByPool, jobSchedulingResultsCache: jobSchedulingResultsCache, }) + + return nil } func (srv *SubmitChecker) Check(ctx *armadacontext.Context, jobs []*jobdb.Job) (map[string]schedulingResult, error) { diff --git a/internal/scheduler/submitcheck_test.go b/internal/scheduler/submitcheck_test.go index 71d0bf6b6cc..97b98f822b2 100644 --- a/internal/scheduler/submitcheck_test.go +++ b/internal/scheduler/submitcheck_test.go @@ -1,6 +1,7 @@ package scheduler import ( + "fmt" "testing" "time" @@ -298,7 +299,8 @@ func TestSubmitChecker_CheckJobDbJobs(t *testing.T) { floatingResources, testfixtures.TestResourceListFactory) submitCheck.clock = fakeClock - submitCheck.updateExecutors(ctx) + err := submitCheck.Initialise(ctx) + assert.NoError(t, err) results, err := submitCheck.Check(ctx, tc.jobs) require.NoError(t, err) require.Equal(t, len(tc.expectedResult), len(results)) @@ -316,6 +318,63 @@ func TestSubmitChecker_CheckJobDbJobs(t *testing.T) { } } +func TestSubmitChecker_Initialise(t *testing.T) { + tests := map[string]struct { + queueCacheErr error + executorRepoErr error + expectError bool + }{ + "Successful initialisation": { + expectError: false, + }, + "error on queue cache error": { + expectError: true, + queueCacheErr: fmt.Errorf("failed to get queues"), + }, + "error on executor repo err": { + expectError: true, + queueCacheErr: fmt.Errorf("failed to get executors"), + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), 5*time.Second) + defer cancel() + + queue := &api.Queue{Name: "queue"} + executors := []*schedulerobjects.Executor{Executor(SmallNode("cpu"))} + + ctrl := gomock.NewController(t) + mockExecutorRepo := schedulermocks.NewMockExecutorRepository(ctrl) + if tc.executorRepoErr != nil { + mockExecutorRepo.EXPECT().GetExecutors(ctx).Return(nil, tc.executorRepoErr).AnyTimes() + } else { + mockExecutorRepo.EXPECT().GetExecutors(ctx).Return(executors, nil).AnyTimes() + } + + mockQueueCache := schedulermocks.NewMockQueueCache(ctrl) + if tc.queueCacheErr != nil { + mockQueueCache.EXPECT().GetAll(ctx).Return(nil, tc.queueCacheErr).AnyTimes() + } else { + mockQueueCache.EXPECT().GetAll(ctx).Return([]*api.Queue{queue}, nil).AnyTimes() + } + + submitCheck := NewSubmitChecker(testfixtures.TestSchedulingConfig(), + mockExecutorRepo, + mockQueueCache, + testfixtures.TestFloatingResources, + testfixtures.TestResourceListFactory) + + err := submitCheck.Initialise(ctx) + if tc.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + func Executor(nodes ...*schedulerobjects.Node) *schedulerobjects.Executor { executorId := uuid.NewString() for _, node := range nodes { diff --git a/magefiles/ci.go b/magefiles/ci.go index 6a911e231ed..210e85c3685 100644 --- a/magefiles/ci.go +++ b/magefiles/ci.go @@ -52,7 +52,10 @@ func TestSuite() error { // Checks if Armada is ready to accept jobs. func CheckForArmadaRunning() error { - time.Sleep(30 * time.Second) + // This is a bit of a shonky check, it confirms the scheduler is up and receiving reports from the executor + // at which point the system should be ready + // TODO Make a good check to confirm the system is ready, such as seeing armadactl get executors return a value + mg.Deps(CheckSchedulerReady) mg.Deps(createQueue) // Set high to take compile time into account diff --git a/magefiles/developer.go b/magefiles/developer.go index 6954ab16acf..3b770870c76 100644 --- a/magefiles/developer.go +++ b/magefiles/developer.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "os/exec" + "regexp" "strings" "time" @@ -111,28 +112,58 @@ func StopComponents() error { return nil } -// Repeatedly check logs until Pulsar is ready. -func CheckForPulsarRunning() error { +func CheckPulsarRunning() error { + return CheckDockerContainerRunning("pulsar", "alive") +} + +func CheckPostgresRunning() error { + return CheckDockerContainerRunning("pulsar", "alive") +} + +func CheckServerRunning() error { + return CheckDockerContainerRunning("server", "Starting http server listening on") +} + +func CheckSchedulerRunning() error { + return CheckDockerContainerRunning("scheduler", "Starting http server listening on") +} + +func CheckExecutorRunning() error { + return CheckDockerContainerRunning("executor", "Starting http server listening on") +} + +func CheckSchedulerReady() error { + return CheckDockerContainerRunning("scheduler", "Retrieved [1-9]+ executors") +} + +// Repeatedly check logs until container is ready. +func CheckDockerContainerRunning(containerName string, expectedLogRegex string) error { timeout := time.After(1 * time.Minute) tick := time.Tick(1 * time.Second) seconds := 0 + + logMatchRegex, err := regexp.Compile(expectedLogRegex) + if err != nil { + return fmt.Errorf("invalid log regex %s - %s", expectedLogRegex, err) + } + for { select { case <-timeout: - return fmt.Errorf("timed out waiting for Pulsar to start") + return fmt.Errorf("timed out waiting for %s to start", containerName) case <-tick: - out, err := dockerOutput("compose", "logs", "pulsar") + out, err := dockerOutput("compose", "logs", containerName) if err != nil { return err } - if strings.Contains(out, "alive") { + if len(logMatchRegex.FindStringSubmatch(out)) > 0 { // if seconds is less than 1, it means that pulsar had already started if seconds < 1 { - fmt.Printf("\nPulsar had already started!\n\n") + fmt.Printf("\n%s had already started!\n\n", containerName) return nil } - fmt.Printf("\nPulsar took %d seconds to start!\n\n", seconds) + fmt.Printf("\n%s took %d seconds to start!\n\n", containerName, seconds) return nil } seconds++ diff --git a/magefiles/main.go b/magefiles/main.go index 0b8756b3201..7c77a25e564 100644 --- a/magefiles/main.go +++ b/magefiles/main.go @@ -214,12 +214,18 @@ func LocalDev(arg string) error { mg.Deps(StartDependencies) fmt.Println("Waiting for dependencies to start...") - mg.Deps(CheckForPulsarRunning) + mg.Deps(CheckPulsarRunning) + mg.Deps(CheckPostgresRunning) switch arg { case "minimal": os.Setenv("ARMADA_COMPONENTS", "executor,server,scheduler") mg.Deps(StartComponents) + // This is a naive check to confirm the containers are running, it doesn't check they are ready + // TODO Make a good check to confirm the system is ready, such as seeing armadactl get executors return a value + mg.Deps(CheckServerRunning) + mg.Deps(CheckSchedulerRunning) + mg.Deps(CheckExecutorRunning) case "debug", "no-build": fmt.Println("Dependencies started, ending localdev...") return nil From c73e840d4c4cc28bfcb72ce87636de0d02c62af5 Mon Sep 17 00:00:00 2001 From: JamesMurkin Date: Wed, 5 Feb 2025 12:08:48 +0000 Subject: [PATCH 22/23] Revert "Add average utilisation to job events (#342)" (#4190) This reverts commit 3db3eeab1ca0b88e52adb1bb631f4939be90fb86. --- internal/common/resource/resource.go | 13 - internal/executor/domain/resources.go | 87 +-- internal/executor/domain/resources_test.go | 94 +-- .../metrics/pod_metrics/cluster_context.go | 2 +- internal/executor/reporter/event.go | 5 +- .../utilisation/cluster_utilisation.go | 2 +- .../utilisation/job_utilisation_reporter.go | 20 +- .../job_utilisation_reporter_test.go | 16 +- .../executor/utilisation/pod_utilisation.go | 6 +- .../pod_utilisation_custom_metrics.go | 2 +- .../pod_utilisation_custom_metrics_test.go | 15 +- .../pod_utilisation_kubelet_metrics.go | 8 +- pkg/armadaevents/events.pb.go | 676 +++++++----------- pkg/armadaevents/events.proto | 1 - 14 files changed, 323 insertions(+), 624 deletions(-) diff --git a/internal/common/resource/resource.go b/internal/common/resource/resource.go index 32317648e95..00265f544eb 100644 --- a/internal/common/resource/resource.go +++ b/internal/common/resource/resource.go @@ -204,19 +204,6 @@ func (a ComputeResources) Sub(b ComputeResources) { } } -func (a ComputeResources) Div(b map[string]int64) { - for k, v := range b { - existing, ok := a[k] - if ok { - if existing.Format == resource.DecimalSI { - a[k] = resource.NewMilliQuantity(existing.MilliValue()/v, existing.Format).DeepCopy() - } else { - a[k] = resource.NewQuantity(existing.Value()/v, existing.Format).DeepCopy() - } - } - } -} - func (a ComputeResources) DeepCopy() ComputeResources { targetComputeResource := make(ComputeResources) diff --git a/internal/executor/domain/resources.go b/internal/executor/domain/resources.go index 1f0a8d634a7..4d00d3cd366 100644 --- a/internal/executor/domain/resources.go +++ b/internal/executor/domain/resources.go @@ -1,96 +1,31 @@ package domain -import ( - "k8s.io/apimachinery/pkg/api/resource" - - armadaresource "github.com/armadaproject/armada/internal/common/resource" -) +import armadaresource "github.com/armadaproject/armada/internal/common/resource" type UtilisationData struct { - currentUsage armadaresource.ComputeResources - maxCurrentUsage armadaresource.ComputeResources - sumCurrentUsage armadaresource.ComputeResources - numDataPoints map[string]int64 - cumulativeUsage armadaresource.ComputeResources -} - -func NewUtilisationData( - resources map[string]resource.Quantity, -) *UtilisationData { - ret := EmptyUtilisationData() - for k, v := range resources { - ret.UpdateCurrentUsage(k, v) - } - return ret + CurrentUsage armadaresource.ComputeResources + CumulativeUsage armadaresource.ComputeResources } func EmptyUtilisationData() *UtilisationData { return &UtilisationData{ - currentUsage: armadaresource.ComputeResources{}, - maxCurrentUsage: armadaresource.ComputeResources{}, - sumCurrentUsage: armadaresource.ComputeResources{}, - numDataPoints: map[string]int64{}, - cumulativeUsage: armadaresource.ComputeResources{}, - } -} - -func (a *UtilisationData) UpdateCurrentUsage(resource string, quantity resource.Quantity) { - a.currentUsage[resource] = quantity - a.maxCurrentUsage[resource] = quantity - a.sumCurrentUsage[resource] = quantity - a.numDataPoints[resource] = 1 -} - -func (a *UtilisationData) Process(b *UtilisationData) { - a.currentUsage = b.currentUsage.DeepCopy() - a.maxCurrentUsage.Max(b.maxCurrentUsage) - a.sumCurrentUsage.Add(b.sumCurrentUsage) - a.cumulativeUsage.Max(b.cumulativeUsage) - - for k, v := range b.numDataPoints { - existing, ok := a.numDataPoints[k] - if ok { - a.numDataPoints[k] = existing + v - } else { - a.numDataPoints[k] = v - } + CurrentUsage: armadaresource.ComputeResources{}, + CumulativeUsage: armadaresource.ComputeResources{}, } } -func (u *UtilisationData) GetCurrentUsage() armadaresource.ComputeResources { - return u.currentUsage -} - -func (u *UtilisationData) GetMaxUsage() armadaresource.ComputeResources { - return u.maxCurrentUsage -} - -func (u *UtilisationData) GetAvgUsage() armadaresource.ComputeResources { - avg := u.sumCurrentUsage.DeepCopy() - avg.Div(u.numDataPoints) - return avg -} - -func (u *UtilisationData) GetCumulativeUsage() armadaresource.ComputeResources { - return u.cumulativeUsage +func (a *UtilisationData) Max(b *UtilisationData) { + a.CurrentUsage.Max(b.CurrentUsage) + a.CumulativeUsage.Max(b.CumulativeUsage) } func (u *UtilisationData) DeepCopy() *UtilisationData { - numDataPoints := map[string]int64{} - - for k, v := range u.numDataPoints { - numDataPoints[k] = v - } - return &UtilisationData{ - currentUsage: u.currentUsage.DeepCopy(), - maxCurrentUsage: u.maxCurrentUsage.DeepCopy(), - sumCurrentUsage: u.sumCurrentUsage.DeepCopy(), - numDataPoints: numDataPoints, - cumulativeUsage: u.cumulativeUsage.DeepCopy(), + CurrentUsage: u.CurrentUsage.DeepCopy(), + CumulativeUsage: u.CumulativeUsage.DeepCopy(), } } func (u *UtilisationData) IsEmpty() bool { - return len(u.cumulativeUsage) == 0 && len(u.currentUsage) == 0 + return len(u.CumulativeUsage) == 0 && len(u.CurrentUsage) == 0 } diff --git a/internal/executor/domain/resources_test.go b/internal/executor/domain/resources_test.go index 004f01fa849..5ce50f7ab65 100644 --- a/internal/executor/domain/resources_test.go +++ b/internal/executor/domain/resources_test.go @@ -9,89 +9,37 @@ import ( armadaresource "github.com/armadaproject/armada/internal/common/resource" ) -func compareQuantityMaps(a, b map[string]resource.Quantity) bool { - if len(a) != len(b) { - return false - } - for key, aVal := range a { - bVal, exists := b[key] - if !exists { - return false - } - if aVal.String() != bVal.String() { - return false - } - } - return true -} - -func TestUtilisationData_Aggregations(t *testing.T) { +func TestUtilisationData_Max(t *testing.T) { data := &UtilisationData{ - currentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("10")}, - maxCurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("10")}, - sumCurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("10")}, - numDataPoints: map[string]int64{"cpu": 1, "memory": 1}, - cumulativeUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("5")}, + CurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("10")}, + CumulativeUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("5")}, } data2 := &UtilisationData{ - currentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("2"), "memory": resource.MustParse("1")}, - maxCurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("2"), "memory": resource.MustParse("1")}, - sumCurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("2"), "memory": resource.MustParse("1")}, - numDataPoints: map[string]int64{"cpu": 1, "memory": 1}, - cumulativeUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("10")}, + CurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("2"), "memory": resource.MustParse("1")}, + CumulativeUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("10")}, } expected := &UtilisationData{ - currentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("2"), "memory": resource.MustParse("1")}, - maxCurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("2"), "memory": resource.MustParse("10")}, - sumCurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("3"), "memory": resource.MustParse("11")}, - numDataPoints: map[string]int64{"cpu": 2, "memory": 2}, - cumulativeUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("10")}, + CurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("2"), "memory": resource.MustParse("10")}, + CumulativeUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("10")}, } - agg := data - agg.Process(data2) - - assert.True(t, compareQuantityMaps(expected.currentUsage, agg.currentUsage)) - assert.True(t, compareQuantityMaps(expected.maxCurrentUsage, agg.maxCurrentUsage)) - assert.True(t, compareQuantityMaps(expected.sumCurrentUsage, agg.sumCurrentUsage)) - assert.Equal(t, expected.numDataPoints, agg.numDataPoints) - assert.True(t, compareQuantityMaps(expected.cumulativeUsage, agg.cumulativeUsage)) -} - -func TestUtilisationData_GetAvg(t *testing.T) { - data := &UtilisationData{ - currentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("100m"), "memory": resource.MustParse("10Mi")}, - maxCurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("100m"), "memory": resource.MustParse("100Mi")}, - sumCurrentUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("160m"), "memory": resource.MustParse("110Mi")}, - numDataPoints: map[string]int64{"cpu": 2, "memory": 2}, - cumulativeUsage: armadaresource.ComputeResources{"cpu": resource.MustParse("160m")}, - } - - expected := armadaresource.ComputeResources{"cpu": resource.MustParse("80m"), "memory": resource.MustParse("55Mi")} - - assert.True(t, compareQuantityMaps(expected, data.GetAvgUsage())) + max := data + max.Max(data2) + assert.Equal(t, expected.CurrentUsage, max.CurrentUsage) + assert.Equal(t, expected.CumulativeUsage, max.CumulativeUsage) } func TestUtilisationData_Max_WithEmpty(t *testing.T) { currentUsage := armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} - maxCurrentUsage := armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} - sumCurrentUsage := armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} - numDataPoints := map[string]int64{"cpu": 1, "memory": 1} cumulativeUsage := armadaresource.ComputeResources{"cpu": resource.MustParse("10")} data := &UtilisationData{ - currentUsage: currentUsage.DeepCopy(), - maxCurrentUsage: maxCurrentUsage.DeepCopy(), - sumCurrentUsage: sumCurrentUsage.DeepCopy(), - numDataPoints: numDataPoints, - cumulativeUsage: cumulativeUsage.DeepCopy(), + CurrentUsage: currentUsage.DeepCopy(), + CumulativeUsage: cumulativeUsage.DeepCopy(), } - agg := EmptyUtilisationData() - agg.Process(data) - assert.Equal(t, data.currentUsage, agg.currentUsage) - assert.Equal(t, data.maxCurrentUsage, agg.maxCurrentUsage) - assert.Equal(t, data.sumCurrentUsage, agg.sumCurrentUsage) - assert.Equal(t, data.numDataPoints, agg.numDataPoints) - assert.Equal(t, data.cumulativeUsage, agg.cumulativeUsage) + max := EmptyUtilisationData() + max.Max(data) + assert.Equal(t, data.CurrentUsage, max.CurrentUsage) + assert.Equal(t, data.CumulativeUsage, max.CumulativeUsage) } func TestUtilisationData_IsEmpty(t *testing.T) { @@ -99,15 +47,15 @@ func TestUtilisationData_IsEmpty(t *testing.T) { assert.True(t, data.IsEmpty()) cumulativeUsageNotEmpty := EmptyUtilisationData() - cumulativeUsageNotEmpty.cumulativeUsage = armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} + cumulativeUsageNotEmpty.CumulativeUsage = armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} assert.False(t, cumulativeUsageNotEmpty.IsEmpty()) currentUsageNotEmpty := EmptyUtilisationData() - currentUsageNotEmpty.currentUsage = armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} + currentUsageNotEmpty.CurrentUsage = armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} assert.False(t, currentUsageNotEmpty.IsEmpty()) allNotEmpty := EmptyUtilisationData() - allNotEmpty.cumulativeUsage = armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} - allNotEmpty.currentUsage = armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} + allNotEmpty.CumulativeUsage = armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} + allNotEmpty.CurrentUsage = armadaresource.ComputeResources{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1")} assert.False(t, allNotEmpty.IsEmpty()) } diff --git a/internal/executor/metrics/pod_metrics/cluster_context.go b/internal/executor/metrics/pod_metrics/cluster_context.go index 925f2eebde8..028c5f1f86a 100644 --- a/internal/executor/metrics/pod_metrics/cluster_context.go +++ b/internal/executor/metrics/pod_metrics/cluster_context.go @@ -197,7 +197,7 @@ func (m *ClusterContextMetrics) Collect(metrics chan<- prometheus.Metric) { nodeTypeMetric[phase].count++ nodeTypeMetric[phase].resourceRequest.Add(request) - nodeTypeMetric[phase].resourceUsage.Add(usage.GetCurrentUsage()) + nodeTypeMetric[phase].resourceUsage.Add(usage.CurrentUsage) } m.setEmptyMetrics(podMetrics) diff --git a/internal/executor/reporter/event.go b/internal/executor/reporter/event.go index 2fd1d3618af..a11e954f47c 100644 --- a/internal/executor/reporter/event.go +++ b/internal/executor/reporter/event.go @@ -345,9 +345,8 @@ func CreateJobUtilisationEvent(pod *v1.Pod, utilisationData *domain.UtilisationD }, }, }, - MaxResourcesForPeriod: utilisationData.GetMaxUsage().ToProtoMap(), - AvgResourcesForPeriod: utilisationData.GetAvgUsage().ToProtoMap(), - TotalCumulativeUsage: utilisationData.GetCumulativeUsage().ToProtoMap(), + MaxResourcesForPeriod: utilisationData.CurrentUsage.ToProtoMap(), + TotalCumulativeUsage: utilisationData.CumulativeUsage.ToProtoMap(), }, }, }) diff --git a/internal/executor/utilisation/cluster_utilisation.go b/internal/executor/utilisation/cluster_utilisation.go index 3d87d80e7b3..e2549731796 100644 --- a/internal/executor/utilisation/cluster_utilisation.go +++ b/internal/executor/utilisation/cluster_utilisation.go @@ -376,7 +376,7 @@ func (clusterUtilisationService *ClusterUtilisationService) getPodUtilisationByQ if _, ok := result[queue][pool]; !ok { result[queue][pool] = armadaresource.ComputeResources{} } - result[queue][pool].Add(podUsage.GetCurrentUsage()) + result[queue][pool].Add(podUsage.CurrentUsage) } return result } diff --git a/internal/executor/utilisation/job_utilisation_reporter.go b/internal/executor/utilisation/job_utilisation_reporter.go index 2c6b58585ea..b4387db22bf 100644 --- a/internal/executor/utilisation/job_utilisation_reporter.go +++ b/internal/executor/utilisation/job_utilisation_reporter.go @@ -25,9 +25,9 @@ type UtilisationEventReporter struct { } type podUtilisationInfo struct { - lastReported time.Time - pod *v1.Pod - utilisationData *domain.UtilisationData + lastReported time.Time + pod *v1.Pod + utilisationMax *domain.UtilisationData } func NewUtilisationEventReporter( @@ -84,12 +84,12 @@ func (r *UtilisationEventReporter) ReportUtilisationEvents() { reportingTime := now.Add(-r.reportingInterval) for _, info := range r.podInfo { currentUtilisation := r.podUtilisation.GetPodUtilisation(info.pod) - info.utilisationData.Process(currentUtilisation) + info.utilisationMax.Max(currentUtilisation) if info.lastReported.Before(reportingTime) { reported := r.reportUsage(info) if reported { info.lastReported = now - info.utilisationData = domain.EmptyUtilisationData() + info.utilisationMax = domain.EmptyUtilisationData() } } } @@ -107,9 +107,9 @@ func (r *UtilisationEventReporter) updatePod(pod *v1.Pod) { _, exists := r.podInfo[pod.Name] if !exists { r.podInfo[pod.Name] = &podUtilisationInfo{ - lastReported: time.Now(), - pod: pod, - utilisationData: r.podUtilisation.GetPodUtilisation(pod), + lastReported: time.Now(), + pod: pod, + utilisationMax: r.podUtilisation.GetPodUtilisation(pod), } } } @@ -132,10 +132,10 @@ func (r *UtilisationEventReporter) deletePod(pod *v1.Pod) { } func (r *UtilisationEventReporter) reportUsage(info *podUtilisationInfo) bool { - if info.utilisationData.IsEmpty() { + if info.utilisationMax.IsEmpty() { return false } - event, err := reporter.CreateJobUtilisationEvent(info.pod, info.utilisationData, r.clusterContext.GetClusterId()) + event, err := reporter.CreateJobUtilisationEvent(info.pod, info.utilisationMax, r.clusterContext.GetClusterId()) if err != nil { log.Errorf("failed to create utilisation event %s", err) return false diff --git a/internal/executor/utilisation/job_utilisation_reporter_test.go b/internal/executor/utilisation/job_utilisation_reporter_test.go index 3dcb15cedc2..39c23d5463d 100644 --- a/internal/executor/utilisation/job_utilisation_reporter_test.go +++ b/internal/executor/utilisation/job_utilisation_reporter_test.go @@ -22,18 +22,21 @@ import ( "github.com/armadaproject/armada/pkg/armadaevents" ) -var testPodResources = domain.NewUtilisationData( - map[string]resource.Quantity{ +var testPodResources = domain.UtilisationData{ + CurrentUsage: armadaresource.ComputeResources{ "cpu": resource.MustParse("1"), "memory": resource.MustParse("640Ki"), }, -) + CumulativeUsage: armadaresource.ComputeResources{ + "cpu": resource.MustParse("10"), + }, +} func TestUtilisationEventReporter_ReportUtilisationEvents(t *testing.T) { reportingPeriod := 100 * time.Millisecond clusterContext := fakeContext.NewFakeClusterContext(configuration.ApplicationConfiguration{ClusterId: "test", Pool: "pool"}, "kubernetes.io/hostname", nil) fakeEventReporter := &mocks.FakeEventReporter{} - fakeUtilisationService := &fakePodUtilisationService{data: testPodResources} + fakeUtilisationService := &fakePodUtilisationService{data: &testPodResources} eventReporter, err := NewUtilisationEventReporter(clusterContext, fakeUtilisationService, fakeEventReporter, reportingPeriod) require.NoError(t, err) @@ -58,9 +61,8 @@ func TestUtilisationEventReporter_ReportUtilisationEvents(t *testing.T) { _, ok = fakeEventReporter.ReceivedEvents[1].Event.Events[0].Event.(*armadaevents.EventSequence_Event_ResourceUtilisation) assert.True(t, ok) - assert.Equal(t, testPodResources.GetMaxUsage(), armadaresource.FromProtoMap(event1.ResourceUtilisation.MaxResourcesForPeriod)) - assert.Equal(t, testPodResources.GetAvgUsage(), armadaresource.FromProtoMap(event1.ResourceUtilisation.AvgResourcesForPeriod)) - assert.Equal(t, testPodResources.GetCumulativeUsage(), armadaresource.FromProtoMap(event1.ResourceUtilisation.TotalCumulativeUsage)) + assert.Equal(t, testPodResources.CurrentUsage, armadaresource.FromProtoMap(event1.ResourceUtilisation.MaxResourcesForPeriod)) + assert.Equal(t, testPodResources.CumulativeUsage, armadaresource.FromProtoMap(event1.ResourceUtilisation.TotalCumulativeUsage)) event1CreatedTime := fakeEventReporter.ReceivedEvents[0].Event.Events[0].Created event2CreatedTime := fakeEventReporter.ReceivedEvents[1].Event.Events[0].Created diff --git a/internal/executor/utilisation/pod_utilisation.go b/internal/executor/utilisation/pod_utilisation.go index a95c4dccdaf..ec075b198e8 100644 --- a/internal/executor/utilisation/pod_utilisation.go +++ b/internal/executor/utilisation/pod_utilisation.go @@ -8,6 +8,7 @@ import ( v1 "k8s.io/api/core/v1" log "github.com/armadaproject/armada/internal/common/logging" + armadaresource "github.com/armadaproject/armada/internal/common/resource" commonUtil "github.com/armadaproject/armada/internal/common/util" "github.com/armadaproject/armada/internal/executor/configuration" clusterContext "github.com/armadaproject/armada/internal/executor/context" @@ -114,7 +115,10 @@ func (q *PodUtilisationServiceImpl) RefreshUtilisationData() { podNameToUtilisationData := map[string]*domain.UtilisationData{} for _, podName := range podNames { - podNameToUtilisationData[podName] = domain.EmptyUtilisationData() + podNameToUtilisationData[podName] = &domain.UtilisationData{ + CurrentUsage: armadaresource.ComputeResources{}, + CumulativeUsage: armadaresource.ComputeResources{}, + } } for _, fetcher := range q.fetchers { diff --git a/internal/executor/utilisation/pod_utilisation_custom_metrics.go b/internal/executor/utilisation/pod_utilisation_custom_metrics.go index f5a335adb0b..bf6102dc1f8 100644 --- a/internal/executor/utilisation/pod_utilisation_custom_metrics.go +++ b/internal/executor/utilisation/pod_utilisation_custom_metrics.go @@ -65,7 +65,7 @@ func updateMetric(metricSamples model.Vector, metric configuration.CustomUsageMe if metric.Multiplier > 0 { val *= metric.Multiplier } - podData.UpdateCurrentUsage(metric.Name, toQuantity(val)) + podData.CurrentUsage[metric.Name] = toQuantity(val) } } } diff --git a/internal/executor/utilisation/pod_utilisation_custom_metrics_test.go b/internal/executor/utilisation/pod_utilisation_custom_metrics_test.go index 0d3ce631331..dfd3da85448 100644 --- a/internal/executor/utilisation/pod_utilisation_custom_metrics_test.go +++ b/internal/executor/utilisation/pod_utilisation_custom_metrics_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/api/resource" + armadaresource "github.com/armadaproject/armada/internal/common/resource" "github.com/armadaproject/armada/internal/executor/configuration" "github.com/armadaproject/armada/internal/executor/domain" ) @@ -42,18 +43,20 @@ func TestUpdateMetrics(t *testing.T) { updateMetrics(samples, config, podNameToUtilisationData) expectedUtilisationData := map[string]*domain.UtilisationData{ - "pod1": domain.NewUtilisationData( - map[string]resource.Quantity{ + "pod1": { + CurrentUsage: armadaresource.ComputeResources{ "gpu": makeQuantity(1), "accelerator-memory-copy-util": makeQuantity(4), }, - ), - "pod2": domain.NewUtilisationData( - map[string]resource.Quantity{ + CumulativeUsage: armadaresource.ComputeResources{}, + }, + "pod2": { + CurrentUsage: armadaresource.ComputeResources{ "gpu": makeMilliQuantity(550), "accelerator-memory-copy-util": makeQuantity(5), }, - ), + CumulativeUsage: armadaresource.ComputeResources{}, + }, "pod3": domain.EmptyUtilisationData(), } diff --git a/internal/executor/utilisation/pod_utilisation_kubelet_metrics.go b/internal/executor/utilisation/pod_utilisation_kubelet_metrics.go index 6fe31369c37..4a7aef1ac11 100644 --- a/internal/executor/utilisation/pod_utilisation_kubelet_metrics.go +++ b/internal/executor/utilisation/pod_utilisation_kubelet_metrics.go @@ -54,15 +54,15 @@ func (m *podUtilisationKubeletMetrics) fetch(nodes []*v1.Node, podNameToUtilisat func updatePodStats(podStats *v1alpha1.PodStats, utilisationData *domain.UtilisationData) { if podStats.CPU != nil && podStats.CPU.UsageNanoCores != nil { - utilisationData.UpdateCurrentUsage("cpu", *resource.NewScaledQuantity(int64(*podStats.CPU.UsageNanoCores), -9)) + utilisationData.CurrentUsage["cpu"] = *resource.NewScaledQuantity(int64(*podStats.CPU.UsageNanoCores), -9) } if podStats.CPU != nil && podStats.CPU.UsageCoreNanoSeconds != nil { - utilisationData.UpdateCurrentUsage("cpu", *resource.NewScaledQuantity(int64(*podStats.CPU.UsageCoreNanoSeconds), -9)) + utilisationData.CumulativeUsage["cpu"] = *resource.NewScaledQuantity(int64(*podStats.CPU.UsageCoreNanoSeconds), -9) } if podStats.Memory != nil && podStats.Memory.WorkingSetBytes != nil { - utilisationData.UpdateCurrentUsage("memory", *resource.NewQuantity(int64(*podStats.Memory.WorkingSetBytes), resource.BinarySI)) + utilisationData.CurrentUsage["memory"] = *resource.NewQuantity(int64(*podStats.Memory.WorkingSetBytes), resource.BinarySI) } if podStats.EphemeralStorage != nil && podStats.EphemeralStorage.UsedBytes != nil { - utilisationData.UpdateCurrentUsage("ephemeral-storage", *resource.NewQuantity(int64(*podStats.EphemeralStorage.UsedBytes), resource.BinarySI)) + utilisationData.CurrentUsage["ephemeral-storage"] = *resource.NewQuantity(int64(*podStats.EphemeralStorage.UsedBytes), resource.BinarySI) } } diff --git a/pkg/armadaevents/events.pb.go b/pkg/armadaevents/events.pb.go index f148a8a56bc..9a306a32f64 100644 --- a/pkg/armadaevents/events.pb.go +++ b/pkg/armadaevents/events.pb.go @@ -549,7 +549,6 @@ type ResourceUtilisation struct { TotalCumulativeUsage map[string]*resource.Quantity `protobuf:"bytes,5,rep,name=total_cumulative_usage,json=totalCumulativeUsage,proto3" json:"totalCumulativeUsage,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` JobId string `protobuf:"bytes,6,opt,name=job_id,json=jobId,proto3" json:"jobId,omitempty"` RunId string `protobuf:"bytes,7,opt,name=run_id,json=runId,proto3" json:"runId,omitempty"` - AvgResourcesForPeriod map[string]*resource.Quantity `protobuf:"bytes,8,rep,name=avg_resources_for_period,json=avgResourcesForPeriod,proto3" json:"avgResourcesForPeriod,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (m *ResourceUtilisation) Reset() { *m = ResourceUtilisation{} } @@ -620,13 +619,6 @@ func (m *ResourceUtilisation) GetRunId() string { return "" } -func (m *ResourceUtilisation) GetAvgResourcesForPeriod() map[string]*resource.Quantity { - if m != nil { - return m.AvgResourcesForPeriod - } - return nil -} - // A request to run an Armada job. Each job consists of a set of Kubernetes objects, // one of which is the main object (typically a pod spec.) and has a priority associated with it. // When the main object exits, all other objects are cleaned up. @@ -3652,7 +3644,6 @@ func init() { proto.RegisterType((*EventSequence)(nil), "armadaevents.EventSequence") proto.RegisterType((*EventSequence_Event)(nil), "armadaevents.EventSequence.Event") proto.RegisterType((*ResourceUtilisation)(nil), "armadaevents.ResourceUtilisation") - proto.RegisterMapType((map[string]*resource.Quantity)(nil), "armadaevents.ResourceUtilisation.AvgResourcesForPeriodEntry") proto.RegisterMapType((map[string]*resource.Quantity)(nil), "armadaevents.ResourceUtilisation.MaxResourcesForPeriodEntry") proto.RegisterMapType((map[string]*resource.Quantity)(nil), "armadaevents.ResourceUtilisation.TotalCumulativeUsageEntry") proto.RegisterType((*SubmitJob)(nil), "armadaevents.SubmitJob") @@ -3708,238 +3699,237 @@ func init() { func init() { proto.RegisterFile("pkg/armadaevents/events.proto", fileDescriptor_6aab92ca59e015f8) } var fileDescriptor_6aab92ca59e015f8 = []byte{ - // 3695 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x3b, 0x4b, 0x6f, 0x1b, 0xd7, - 0xd5, 0x1e, 0xbe, 0x79, 0xa8, 0x07, 0x7d, 0xf5, 0x30, 0xad, 0xc4, 0xa2, 0x4c, 0xe7, 0x4b, 0xec, - 0x20, 0xa1, 0x1c, 0xe7, 0xcb, 0x87, 0x3c, 0x3e, 0x24, 0x10, 0x6d, 0xc5, 0xb6, 0x6c, 0xd9, 0x0a, - 0x65, 0xa7, 0x6e, 0x11, 0x80, 0x19, 0x72, 0xae, 0xa8, 0xb1, 0xc8, 0x99, 0xc9, 0x3c, 0x14, 0x09, - 0x08, 0xda, 0xa4, 0x48, 0xbb, 0xce, 0xa6, 0x40, 0x91, 0x4d, 0xb3, 0xe9, 0x22, 0x45, 0xbb, 0x6c, - 0xb7, 0xdd, 0x76, 0x51, 0x14, 0xd9, 0x14, 0xe8, 0xa2, 0x21, 0x8a, 0x04, 0xdd, 0x70, 0xd1, 0xdf, - 0x50, 0xdc, 0xc7, 0xcc, 0xdc, 0x3b, 0xbc, 0xb4, 0x24, 0xc7, 0x32, 0xdc, 0x74, 0x25, 0xcd, 0x79, - 0xde, 0x39, 0xe7, 0xdc, 0x33, 0xe7, 0x9c, 0x7b, 0x09, 0x67, 0x9c, 0x9d, 0xee, 0xb2, 0xee, 0xf6, - 0x75, 0x43, 0xc7, 0xbb, 0xd8, 0xf2, 0xbd, 0x65, 0xf6, 0xa7, 0xee, 0xb8, 0xb6, 0x6f, 0xa3, 0x09, - 0x11, 0xb5, 0x50, 0xdb, 0x79, 0xd5, 0xab, 0x9b, 0xf6, 0xb2, 0xee, 0x98, 0xcb, 0x1d, 0xdb, 0xc5, - 0xcb, 0xbb, 0x2f, 0x2d, 0x77, 0xb1, 0x85, 0x5d, 0xdd, 0xc7, 0x06, 0xe3, 0x58, 0x38, 0x2f, 0xd0, - 0x58, 0xd8, 0xff, 0xd0, 0x76, 0x77, 0x4c, 0xab, 0xab, 0xa2, 0xac, 0x76, 0x6d, 0xbb, 0xdb, 0xc3, - 0xcb, 0xf4, 0xa9, 0x1d, 0x6c, 0x2d, 0xfb, 0x66, 0x1f, 0x7b, 0xbe, 0xde, 0x77, 0x38, 0xc1, 0xff, - 0xc6, 0xa2, 0xfa, 0x7a, 0x67, 0xdb, 0xb4, 0xb0, 0xbb, 0xbf, 0x4c, 0xd7, 0xeb, 0x98, 0xcb, 0x2e, - 0xf6, 0xec, 0xc0, 0xed, 0xe0, 0x11, 0xb1, 0xaf, 0x9b, 0x96, 0x8f, 0x5d, 0x4b, 0xef, 0x2d, 0x7b, - 0x9d, 0x6d, 0x6c, 0x04, 0x3d, 0xec, 0xc6, 0xff, 0xd9, 0xed, 0xfb, 0xb8, 0xe3, 0x7b, 0x23, 0x00, - 0xc6, 0x5b, 0xfb, 0x7a, 0x0e, 0x26, 0x57, 0xc9, 0xbb, 0x6e, 0xe2, 0x0f, 0x02, 0x6c, 0x75, 0x30, - 0xba, 0x00, 0xd9, 0x0f, 0x02, 0x1c, 0xe0, 0x8a, 0xb6, 0xa4, 0x9d, 0x2f, 0x36, 0x66, 0x86, 0x83, - 0xea, 0x34, 0x05, 0xbc, 0x60, 0xf7, 0x4d, 0x1f, 0xf7, 0x1d, 0x7f, 0xbf, 0xc9, 0x28, 0xd0, 0xeb, - 0x30, 0x71, 0xdf, 0x6e, 0xb7, 0x3c, 0xec, 0xb7, 0x2c, 0xbd, 0x8f, 0x2b, 0x29, 0xca, 0x51, 0x19, - 0x0e, 0xaa, 0xb3, 0xf7, 0xed, 0xf6, 0x26, 0xf6, 0x6f, 0xe9, 0x7d, 0x91, 0x0d, 0x62, 0x28, 0x7a, - 0x11, 0xf2, 0x81, 0x87, 0xdd, 0x96, 0x69, 0x54, 0xd2, 0x94, 0x6d, 0x76, 0x38, 0xa8, 0x96, 0x09, - 0xe8, 0xba, 0x21, 0xb0, 0xe4, 0x18, 0x04, 0xbd, 0x00, 0xb9, 0xae, 0x6b, 0x07, 0x8e, 0x57, 0xc9, - 0x2c, 0xa5, 0x43, 0x6a, 0x06, 0x11, 0xa9, 0x19, 0x04, 0xdd, 0x86, 0x1c, 0x73, 0x60, 0x25, 0xbb, - 0x94, 0x3e, 0x5f, 0xba, 0x74, 0xb6, 0x2e, 0x7a, 0xb5, 0x2e, 0xbd, 0x30, 0x7b, 0x62, 0x02, 0x19, - 0x5e, 0x14, 0xc8, 0xe3, 0xe0, 0x0f, 0x33, 0x90, 0xa5, 0x74, 0xe8, 0x06, 0xe4, 0x3b, 0x2e, 0x26, - 0xd6, 0xaf, 0xa0, 0x25, 0xed, 0x7c, 0xe9, 0xd2, 0x42, 0x9d, 0x79, 0xb5, 0x1e, 0x7a, 0xb5, 0x7e, - 0x27, 0xf4, 0x6a, 0x63, 0x6e, 0x38, 0xa8, 0x9e, 0xe4, 0xe4, 0x82, 0xd4, 0x50, 0x02, 0xda, 0x80, - 0xa2, 0x17, 0xb4, 0xfb, 0xa6, 0xbf, 0x66, 0xb7, 0xa9, 0xbd, 0x4b, 0x97, 0x4e, 0xc9, 0x4b, 0xdd, - 0x0c, 0xd1, 0x8d, 0x53, 0xc3, 0x41, 0x75, 0x26, 0xa2, 0x8e, 0xa5, 0x5d, 0x3b, 0xd1, 0x8c, 0x85, - 0xa0, 0x6d, 0x98, 0x76, 0xb1, 0xe3, 0x9a, 0xb6, 0x6b, 0xfa, 0xa6, 0x87, 0x89, 0xdc, 0x14, 0x95, - 0x7b, 0x46, 0x96, 0xdb, 0x94, 0x89, 0x1a, 0x67, 0x86, 0x83, 0xea, 0xe9, 0x04, 0xa7, 0xa4, 0x23, - 0x29, 0x16, 0xf9, 0x80, 0x12, 0xa0, 0x4d, 0xec, 0x53, 0x5f, 0x96, 0x2e, 0x2d, 0x3d, 0x50, 0xd9, - 0x26, 0xf6, 0x1b, 0x4b, 0xc3, 0x41, 0xf5, 0xe9, 0x51, 0x7e, 0x49, 0xa5, 0x42, 0x3e, 0xea, 0x41, - 0x59, 0x84, 0x1a, 0xe4, 0x05, 0x33, 0x54, 0xe7, 0xe2, 0x78, 0x9d, 0x84, 0xaa, 0xb1, 0x38, 0x1c, - 0x54, 0x17, 0x92, 0xbc, 0x92, 0xbe, 0x11, 0xc9, 0xc4, 0x3f, 0x1d, 0xdd, 0xea, 0xe0, 0x1e, 0x51, - 0x93, 0x55, 0xf9, 0xe7, 0x72, 0x88, 0x66, 0xfe, 0x89, 0xa8, 0x65, 0xff, 0x44, 0x60, 0xf4, 0x1e, - 0x4c, 0x44, 0x0f, 0xc4, 0x5e, 0x39, 0x1e, 0x43, 0x6a, 0xa1, 0xc4, 0x52, 0x0b, 0xc3, 0x41, 0x75, - 0x5e, 0xe4, 0x91, 0x44, 0x4b, 0xd2, 0x62, 0xe9, 0x3d, 0x66, 0x99, 0xfc, 0x78, 0xe9, 0x8c, 0x42, - 0x94, 0xde, 0x1b, 0xb5, 0x88, 0x24, 0x8d, 0x48, 0x27, 0x1b, 0x38, 0xe8, 0x74, 0x30, 0x36, 0xb0, - 0x51, 0x29, 0xa8, 0xa4, 0xaf, 0x09, 0x14, 0x4c, 0xba, 0xc8, 0x23, 0x4b, 0x17, 0x31, 0xc4, 0xd6, - 0xf7, 0xed, 0xf6, 0xaa, 0xeb, 0xda, 0xae, 0x57, 0x29, 0xaa, 0x6c, 0xbd, 0x16, 0xa2, 0x99, 0xad, - 0x23, 0x6a, 0xd9, 0xd6, 0x11, 0x98, 0xaf, 0xb7, 0x19, 0x58, 0x37, 0xb1, 0xee, 0x61, 0xa3, 0x02, - 0x63, 0xd6, 0x1b, 0x51, 0x44, 0xeb, 0x8d, 0x20, 0x23, 0xeb, 0x8d, 0x30, 0xc8, 0x80, 0x29, 0xf6, - 0xbc, 0xe2, 0x79, 0x66, 0xd7, 0xc2, 0x46, 0xa5, 0x44, 0xe5, 0x3f, 0xad, 0x92, 0x1f, 0xd2, 0x34, - 0x9e, 0x1e, 0x0e, 0xaa, 0x15, 0x99, 0x4f, 0xd2, 0x91, 0x90, 0x89, 0xde, 0x87, 0x49, 0x06, 0x69, - 0x06, 0x96, 0x65, 0x5a, 0xdd, 0xca, 0x04, 0x55, 0xf2, 0x94, 0x4a, 0x09, 0x27, 0x69, 0x3c, 0x35, - 0x1c, 0x54, 0x4f, 0x49, 0x5c, 0x92, 0x0a, 0x59, 0x20, 0xc9, 0x18, 0x0c, 0x10, 0x3b, 0x76, 0x52, - 0x95, 0x31, 0xd6, 0x64, 0x22, 0x96, 0x31, 0x12, 0x9c, 0x72, 0xc6, 0x48, 0x20, 0x63, 0x7f, 0x70, - 0x27, 0x4f, 0x8d, 0xf7, 0x07, 0xf7, 0xb3, 0xe0, 0x0f, 0x85, 0xab, 0x25, 0x69, 0xe8, 0x63, 0x0d, - 0xe6, 0x3c, 0x5f, 0xb7, 0x0c, 0xbd, 0x67, 0x5b, 0xf8, 0xba, 0xd5, 0x75, 0xb1, 0xe7, 0x5d, 0xb7, - 0xb6, 0xec, 0x4a, 0x99, 0xea, 0x39, 0x97, 0x48, 0xac, 0x2a, 0xd2, 0xc6, 0xb9, 0xe1, 0xa0, 0x5a, - 0x55, 0x4a, 0x91, 0x34, 0xab, 0x15, 0xa1, 0x3d, 0x98, 0x09, 0x3f, 0xd2, 0x77, 0x7d, 0xb3, 0x67, - 0x7a, 0xba, 0x6f, 0xda, 0x56, 0xe5, 0x24, 0xd5, 0x7f, 0x36, 0x99, 0x9f, 0x46, 0x08, 0x1b, 0x67, - 0x87, 0x83, 0xea, 0x19, 0x85, 0x04, 0x49, 0xb7, 0x4a, 0x45, 0xec, 0xc4, 0x0d, 0x17, 0x13, 0x42, - 0x6c, 0x54, 0x66, 0xc6, 0x3b, 0x31, 0x22, 0x12, 0x9d, 0x18, 0x01, 0x55, 0x4e, 0x8c, 0x90, 0x44, - 0x93, 0xa3, 0xbb, 0xbe, 0x49, 0xd4, 0xae, 0xeb, 0xee, 0x0e, 0x76, 0x2b, 0xb3, 0x2a, 0x4d, 0x1b, - 0x32, 0x11, 0xd3, 0x94, 0xe0, 0x94, 0x35, 0x25, 0x90, 0xe8, 0x33, 0x0d, 0xe4, 0xa5, 0x99, 0xb6, - 0xd5, 0x24, 0x1f, 0x6d, 0x8f, 0xbc, 0xde, 0x1c, 0x55, 0xfa, 0xdc, 0x03, 0x5e, 0x4f, 0x24, 0x6f, - 0x3c, 0x37, 0x1c, 0x54, 0xcf, 0x8d, 0x95, 0x26, 0x2d, 0x64, 0xbc, 0x52, 0x74, 0x0f, 0x4a, 0x04, - 0x89, 0x69, 0xf9, 0x63, 0x54, 0xe6, 0xe9, 0x1a, 0x4e, 0x8f, 0xae, 0x81, 0x13, 0x34, 0x4e, 0x0f, - 0x07, 0xd5, 0x39, 0x81, 0x43, 0xd2, 0x23, 0x8a, 0x42, 0x9f, 0x6a, 0x40, 0x02, 0x5d, 0xf5, 0xa6, - 0xa7, 0xa8, 0x96, 0x67, 0x46, 0xb4, 0xa8, 0x5e, 0xf3, 0x99, 0xe1, 0xa0, 0xba, 0xa4, 0x96, 0x23, - 0xe9, 0x1e, 0xa3, 0x2b, 0x8e, 0xa3, 0xe8, 0x23, 0x51, 0xa9, 0x8c, 0x8f, 0xa3, 0x88, 0x48, 0x8c, - 0xa3, 0x08, 0xa8, 0x8a, 0xa3, 0x08, 0xc9, 0x93, 0xc1, 0xbb, 0x7a, 0xcf, 0x34, 0x68, 0x31, 0x75, - 0x7a, 0x4c, 0x32, 0x88, 0x28, 0xa2, 0x64, 0x10, 0x41, 0x46, 0x92, 0x41, 0x4c, 0x9b, 0x87, 0x2c, - 0x15, 0x51, 0xfb, 0xbc, 0x08, 0x33, 0x8a, 0xad, 0x86, 0x30, 0x4c, 0x86, 0xfb, 0xa8, 0x65, 0x92, - 0x24, 0x91, 0x56, 0x59, 0xf9, 0x46, 0xd0, 0xc6, 0xae, 0x85, 0x7d, 0xec, 0x85, 0x32, 0x68, 0x96, - 0xa0, 0x2b, 0x71, 0x05, 0x88, 0x50, 0xdb, 0x4d, 0x88, 0x70, 0xf4, 0xb9, 0x06, 0x95, 0xbe, 0xbe, - 0xd7, 0x0a, 0x81, 0x5e, 0x6b, 0xcb, 0x76, 0x5b, 0x0e, 0x76, 0x4d, 0xdb, 0xa0, 0x95, 0x6c, 0xe9, - 0xd2, 0xff, 0x1f, 0x98, 0x17, 0xea, 0xeb, 0xfa, 0x5e, 0x08, 0xf6, 0xde, 0xb6, 0xdd, 0x0d, 0xca, - 0xbe, 0x6a, 0xf9, 0xee, 0x3e, 0x4b, 0x58, 0x7d, 0x15, 0x5e, 0x58, 0xd3, 0x9c, 0x92, 0x00, 0xfd, - 0x42, 0x83, 0x79, 0xdf, 0xf6, 0xf5, 0x5e, 0xab, 0x13, 0xf4, 0x83, 0x9e, 0xee, 0x9b, 0xbb, 0xb8, - 0x15, 0x78, 0x7a, 0x17, 0xf3, 0xb2, 0xf9, 0x8d, 0x83, 0x97, 0x76, 0x87, 0xf0, 0x5f, 0x8e, 0xd8, - 0xef, 0x12, 0x6e, 0xb6, 0xb2, 0xda, 0x70, 0x50, 0x5d, 0xf4, 0x15, 0x68, 0x61, 0x61, 0xb3, 0x2a, - 0x3c, 0x7a, 0x1e, 0x72, 0xa4, 0xad, 0x30, 0x0d, 0x5a, 0x1d, 0xf1, 0x16, 0xe4, 0xbe, 0xdd, 0x96, - 0x1a, 0x83, 0x2c, 0x05, 0x10, 0x5a, 0x37, 0xb0, 0x08, 0x6d, 0x3e, 0xa6, 0x75, 0x03, 0x4b, 0xa6, - 0xa5, 0x00, 0xea, 0x0c, 0x7d, 0xb7, 0xab, 0x76, 0x46, 0xe1, 0xb0, 0xce, 0x58, 0xd9, 0xed, 0x3e, - 0xd0, 0x19, 0xba, 0x0a, 0x2f, 0x3a, 0x43, 0x49, 0xb0, 0xf0, 0x85, 0x06, 0x0b, 0xe3, 0xfd, 0x8c, - 0xce, 0x41, 0x7a, 0x07, 0xef, 0xf3, 0x9e, 0xec, 0xe4, 0x70, 0x50, 0x9d, 0xdc, 0xc1, 0xfb, 0x82, - 0x54, 0x82, 0x45, 0x3f, 0x84, 0xec, 0xae, 0xde, 0x0b, 0x30, 0x2f, 0xf9, 0xeb, 0x75, 0xd6, 0x4e, - 0xd6, 0xc5, 0x76, 0xb2, 0xee, 0xec, 0x74, 0x09, 0xa0, 0x1e, 0x5a, 0xa1, 0xfe, 0x4e, 0xa0, 0x5b, - 0xbe, 0xe9, 0xef, 0x33, 0xdb, 0x51, 0x01, 0xa2, 0xed, 0x28, 0xe0, 0xf5, 0xd4, 0xab, 0xda, 0xc2, - 0xaf, 0x34, 0x38, 0x3d, 0xd6, 0xdf, 0x4f, 0xc4, 0x0a, 0x89, 0x11, 0xc7, 0xfb, 0xe7, 0x49, 0x58, - 0xe2, 0x5a, 0xa6, 0xa0, 0x95, 0x53, 0x6b, 0x99, 0x42, 0xaa, 0x9c, 0xae, 0xfd, 0x36, 0x0f, 0xc5, - 0xa8, 0xc1, 0x43, 0xd7, 0xa0, 0x6c, 0x60, 0x23, 0x70, 0x7a, 0x66, 0x87, 0x46, 0x1a, 0x09, 0x6a, - 0xd6, 0x51, 0xd3, 0xec, 0x2a, 0xe1, 0xa4, 0xf0, 0x9e, 0x4e, 0xa0, 0xd0, 0x25, 0x28, 0xf0, 0x46, - 0x66, 0x9f, 0xe6, 0xb5, 0xc9, 0xc6, 0xfc, 0x70, 0x50, 0x45, 0x21, 0x4c, 0x60, 0x8d, 0xe8, 0x50, - 0x13, 0x80, 0x4d, 0x06, 0xd6, 0xb1, 0xaf, 0xf3, 0x96, 0xaa, 0x22, 0xef, 0x86, 0xdb, 0x11, 0x9e, - 0xf5, 0xf8, 0x31, 0xbd, 0xd8, 0xe3, 0xc7, 0x50, 0xf4, 0x1e, 0x40, 0x5f, 0x37, 0x2d, 0xc6, 0xc7, - 0xfb, 0xa7, 0xda, 0xb8, 0x0c, 0xbb, 0x1e, 0x51, 0x32, 0xe9, 0x31, 0xa7, 0x28, 0x3d, 0x86, 0xa2, - 0xdb, 0x90, 0xe7, 0xb3, 0x8c, 0x4a, 0x8e, 0x6e, 0xde, 0xc5, 0x71, 0xa2, 0xb9, 0x58, 0xda, 0x8d, - 0x73, 0x16, 0xb1, 0x1b, 0xe7, 0x20, 0x62, 0xb6, 0x9e, 0xb9, 0x85, 0x7d, 0xb3, 0x8f, 0x69, 0x36, - 0xe1, 0x66, 0x0b, 0x61, 0xa2, 0xd9, 0x42, 0x18, 0x7a, 0x15, 0x40, 0xf7, 0xd7, 0x6d, 0xcf, 0xbf, - 0x6d, 0x75, 0x30, 0xed, 0x88, 0x0a, 0x6c, 0xf9, 0x31, 0x54, 0x5c, 0x7e, 0x0c, 0x45, 0x6f, 0x40, - 0xc9, 0xe1, 0x5f, 0xe0, 0x76, 0x0f, 0xd3, 0x8e, 0xa7, 0xc0, 0x0a, 0x06, 0x01, 0x2c, 0xf0, 0x8a, - 0xd4, 0xe8, 0x2a, 0x4c, 0x77, 0x6c, 0xab, 0x13, 0xb8, 0x2e, 0xb6, 0x3a, 0xfb, 0x9b, 0xfa, 0x16, - 0xa6, 0xdd, 0x4d, 0x81, 0x85, 0x4a, 0x02, 0x25, 0x86, 0x4a, 0x02, 0x85, 0x5e, 0x81, 0x62, 0x34, - 0x19, 0xa2, 0x0d, 0x4c, 0x91, 0x0f, 0x1a, 0x42, 0xa0, 0xc0, 0x1c, 0x53, 0x92, 0xc5, 0x9b, 0xde, - 0x15, 0x1e, 0x74, 0x98, 0x36, 0x25, 0x7c, 0xf1, 0x02, 0x58, 0x5c, 0xbc, 0x00, 0x16, 0xf2, 0xfb, - 0xd4, 0x81, 0xf9, 0xfd, 0xc7, 0x30, 0x87, 0xf7, 0x48, 0x8a, 0xee, 0x63, 0xcb, 0xd7, 0x7b, 0x1b, - 0xae, 0xc9, 0xbe, 0xac, 0x95, 0x69, 0x55, 0x51, 0xbf, 0xaa, 0x22, 0x65, 0x69, 0x59, 0x29, 0x45, - 0x4c, 0xcb, 0x4a, 0x82, 0x68, 0xbb, 0x4e, 0x96, 0xa7, 0x6a, 0x37, 0x60, 0x4e, 0xa9, 0x80, 0x04, - 0x4e, 0xdb, 0x34, 0xe8, 0x33, 0x4d, 0x2e, 0x1a, 0x0b, 0x9c, 0x10, 0x26, 0x06, 0x4e, 0x08, 0xab, - 0xfd, 0x59, 0x83, 0x59, 0x55, 0xf0, 0x27, 0x36, 0xa2, 0xf6, 0x48, 0x36, 0xe2, 0xbb, 0x50, 0x70, - 0x6c, 0xa3, 0xe5, 0x39, 0xb8, 0xc3, 0xd3, 0x5a, 0x62, 0x1b, 0x6e, 0xd8, 0xc6, 0xa6, 0x83, 0x3b, - 0x3f, 0x30, 0xfd, 0xed, 0x95, 0x5d, 0xdb, 0x34, 0x6e, 0x9a, 0x1e, 0xdf, 0x2f, 0x0e, 0xc3, 0x48, - 0xb5, 0x56, 0x9e, 0x03, 0x1b, 0x05, 0xc8, 0x31, 0x2d, 0xb5, 0xbf, 0xa4, 0xa1, 0x9c, 0xdc, 0x70, - 0xff, 0x49, 0xaf, 0x82, 0xee, 0x41, 0xde, 0x64, 0xad, 0x1c, 0x2f, 0x05, 0xff, 0x47, 0x48, 0xfc, - 0xf5, 0x78, 0xae, 0x5b, 0xdf, 0x7d, 0xa9, 0xce, 0x7b, 0x3e, 0x6a, 0x02, 0x2a, 0x99, 0x73, 0xca, - 0x92, 0x39, 0x10, 0x35, 0x21, 0xef, 0x61, 0x77, 0x97, 0x04, 0x07, 0x4b, 0xab, 0x55, 0x51, 0x72, - 0xc7, 0x76, 0x31, 0x91, 0xb9, 0xc9, 0x48, 0x62, 0x99, 0x9c, 0x47, 0x96, 0xc9, 0x81, 0xe8, 0x5d, - 0x28, 0x76, 0x6c, 0x6b, 0xcb, 0xec, 0xae, 0xeb, 0x0e, 0x4f, 0xac, 0x67, 0x54, 0x52, 0x2f, 0x87, - 0x44, 0x7c, 0x3c, 0x15, 0x3e, 0x26, 0xc6, 0x53, 0x11, 0x55, 0xec, 0xd0, 0x7f, 0x65, 0x00, 0x62, - 0xe7, 0xa0, 0xd7, 0xa0, 0x84, 0xf7, 0x70, 0x27, 0xf0, 0x6d, 0x3a, 0xb2, 0xd5, 0xe2, 0x49, 0x6f, - 0x08, 0x96, 0x76, 0x2f, 0xc4, 0x50, 0x92, 0x62, 0x2c, 0xbd, 0x8f, 0x3d, 0x47, 0xef, 0x84, 0x23, - 0x62, 0xba, 0x98, 0x08, 0x28, 0xa6, 0x98, 0x08, 0x88, 0x9e, 0x85, 0x0c, 0x1d, 0x2a, 0xb3, 0xe9, - 0x30, 0x1a, 0x0e, 0xaa, 0x53, 0x96, 0x3c, 0x4e, 0xa6, 0x78, 0xf4, 0x16, 0x4c, 0xee, 0x44, 0x81, - 0x47, 0xd6, 0x96, 0xa1, 0x0c, 0xb4, 0x46, 0x8f, 0x11, 0xd2, 0xea, 0x26, 0x44, 0x38, 0xda, 0x82, - 0x92, 0x6e, 0x59, 0xb6, 0x4f, 0xbf, 0x9e, 0xe1, 0xc4, 0xf8, 0xc2, 0xb8, 0x30, 0xad, 0xaf, 0xc4, - 0xb4, 0xac, 0xea, 0xa3, 0x69, 0x4f, 0x90, 0x20, 0xa6, 0x3d, 0x01, 0x8c, 0x9a, 0x90, 0xeb, 0xe9, - 0x6d, 0xdc, 0x0b, 0x3f, 0x57, 0xcf, 0x8c, 0x55, 0x71, 0x93, 0x92, 0x31, 0xe9, 0x74, 0x2e, 0xcd, - 0xf8, 0xc4, 0xb9, 0x34, 0x83, 0x2c, 0x6c, 0x41, 0x39, 0xb9, 0x9e, 0xc3, 0x55, 0x39, 0x17, 0xc4, - 0x2a, 0xa7, 0x78, 0x60, 0x61, 0xa5, 0x43, 0x49, 0x58, 0xd4, 0x71, 0xa8, 0xa8, 0x7d, 0xa9, 0xc1, - 0xac, 0x6a, 0xef, 0xa2, 0x75, 0x61, 0xc7, 0x6b, 0x7c, 0xfa, 0xa5, 0x08, 0x75, 0xce, 0x3b, 0x66, - 0xab, 0xc7, 0x1b, 0xbd, 0x01, 0x53, 0x96, 0x6d, 0xe0, 0x96, 0x4e, 0x14, 0xf4, 0x4c, 0xcf, 0xaf, - 0xa4, 0xe8, 0x89, 0x02, 0x9d, 0x9a, 0x11, 0xcc, 0x4a, 0x88, 0x10, 0xb8, 0x27, 0x25, 0x44, 0xed, - 0x43, 0x98, 0x4e, 0xcc, 0xb4, 0xa5, 0x9a, 0x2b, 0x75, 0xc8, 0x9a, 0x2b, 0xfe, 0x10, 0xa6, 0x0f, - 0xfa, 0x10, 0xb2, 0x0f, 0x51, 0xed, 0x67, 0x29, 0x28, 0x09, 0x03, 0x06, 0x74, 0x1f, 0xa6, 0xf9, - 0x47, 0xd9, 0xb4, 0xba, 0xac, 0x91, 0x4d, 0xf1, 0x0f, 0xe3, 0xc8, 0x81, 0xcf, 0x9a, 0xdd, 0xde, - 0x8c, 0x68, 0xe9, 0x87, 0x91, 0x0e, 0x23, 0x3d, 0x09, 0x26, 0x28, 0x9e, 0x92, 0x31, 0xe8, 0x1e, - 0xcc, 0x07, 0x0e, 0x69, 0xaf, 0x5b, 0x1e, 0x3f, 0x3a, 0x69, 0x59, 0x41, 0xbf, 0x8d, 0x5d, 0xba, - 0xfa, 0x2c, 0x6b, 0xf8, 0x18, 0x45, 0x78, 0xb6, 0x72, 0x8b, 0xe2, 0xc5, 0x86, 0x4f, 0x85, 0x17, - 0xec, 0x90, 0x39, 0xa4, 0x1d, 0xae, 0x01, 0x1a, 0x3d, 0x54, 0x90, 0x7c, 0xa0, 0x1d, 0xce, 0x07, - 0xb5, 0x3d, 0x28, 0x27, 0x8f, 0x0a, 0x1e, 0x93, 0x2f, 0x77, 0xa0, 0x18, 0x0d, 0xfa, 0xd1, 0x0b, - 0x90, 0x73, 0xb1, 0xee, 0xd9, 0x16, 0xdf, 0x2d, 0x74, 0xdb, 0x33, 0x88, 0xb8, 0xed, 0x19, 0xe4, - 0x21, 0x94, 0xdd, 0x81, 0x09, 0x66, 0xa4, 0xb7, 0xcd, 0x9e, 0x8f, 0x5d, 0x74, 0x05, 0x72, 0x9e, - 0xaf, 0xfb, 0xd8, 0xab, 0x68, 0x4b, 0xe9, 0xf3, 0x53, 0x97, 0xe6, 0x47, 0xa7, 0xf8, 0x04, 0xcd, - 0xd6, 0xc1, 0x28, 0xc5, 0x75, 0x30, 0x48, 0xed, 0xa7, 0x1a, 0x4c, 0x88, 0x87, 0x15, 0x8f, 0x46, - 0xec, 0xd1, 0x8c, 0x51, 0xb3, 0xc2, 0x35, 0xf0, 0x63, 0x8a, 0xe3, 0x36, 0xe5, 0x97, 0x1a, 0xb3, - 0x65, 0x34, 0xd7, 0xee, 0xc6, 0xb3, 0x24, 0xb2, 0x51, 0x3c, 0x9a, 0x50, 0x0e, 0x3b, 0x4b, 0xa2, - 0x69, 0x47, 0x62, 0x17, 0xd3, 0x8e, 0x84, 0x78, 0x88, 0xb5, 0x7e, 0x91, 0xa5, 0x6b, 0x8d, 0x4f, - 0x2d, 0x12, 0xdf, 0xf1, 0xf4, 0x11, 0xbe, 0xe3, 0x2f, 0x42, 0x9e, 0x26, 0xce, 0x68, 0x9b, 0x52, - 0xc3, 0x12, 0x90, 0x7c, 0x62, 0xcb, 0x20, 0x0f, 0x48, 0x17, 0xd9, 0xef, 0x98, 0x2e, 0x5a, 0x70, - 0x7a, 0x5b, 0xf7, 0x5a, 0x61, 0x82, 0x33, 0x5a, 0xba, 0xdf, 0x8a, 0xf6, 0x6b, 0x8e, 0xb6, 0x22, - 0x74, 0x0e, 0xba, 0xad, 0x7b, 0x9b, 0x21, 0xcd, 0x8a, 0xbf, 0x31, 0xba, 0x7b, 0xe7, 0xd5, 0x14, - 0xe8, 0x2e, 0xcc, 0xa9, 0x85, 0xe7, 0xe9, 0xca, 0xe9, 0x98, 0xde, 0x7b, 0xa0, 0xe4, 0x19, 0x05, - 0x1a, 0x7d, 0xa2, 0x41, 0x85, 0x7c, 0xc9, 0x5c, 0xfc, 0x41, 0x60, 0xba, 0x98, 0x74, 0x11, 0x5e, - 0xcb, 0xde, 0xc5, 0x6e, 0x4f, 0xdf, 0xe7, 0x27, 0x5e, 0x67, 0x47, 0xd3, 0xf6, 0x86, 0x6d, 0x34, - 0x05, 0x06, 0xf6, 0x6a, 0x8e, 0x0c, 0xbc, 0xcd, 0x84, 0x88, 0xaf, 0xa6, 0xa6, 0x10, 0x42, 0x08, - 0x8e, 0x30, 0x5b, 0x2b, 0x1d, 0x38, 0x5b, 0x7b, 0x16, 0x32, 0x8e, 0x6d, 0xf7, 0x68, 0x27, 0xc8, - 0xab, 0x35, 0xf2, 0x2c, 0x56, 0x6b, 0xe4, 0x59, 0x1c, 0x7f, 0xac, 0x65, 0x0a, 0x85, 0x72, 0xb1, - 0xf6, 0xb5, 0x06, 0x53, 0xf2, 0x21, 0xd9, 0xe8, 0x86, 0x4a, 0x1f, 0xfb, 0x86, 0xca, 0x1c, 0xc1, - 0x1a, 0xd9, 0x83, 0xac, 0x21, 0x0d, 0x79, 0xfe, 0xae, 0xc1, 0xa4, 0x74, 0x3e, 0xf7, 0xfd, 0x7a, - 0xbd, 0x5f, 0xa6, 0x60, 0x5e, 0xbd, 0xd4, 0x63, 0x69, 0xff, 0xae, 0x01, 0x29, 0xe4, 0xae, 0xc7, - 0x85, 0xce, 0xdc, 0x48, 0xf7, 0x47, 0xcd, 0x14, 0x56, 0x81, 0x23, 0x47, 0x77, 0x21, 0x3b, 0xba, - 0x07, 0x25, 0x53, 0x38, 0x24, 0x4c, 0xab, 0xce, 0x72, 0xc4, 0xa3, 0x41, 0x36, 0xdd, 0x18, 0x73, - 0x20, 0x28, 0x8a, 0x6a, 0xe4, 0x20, 0x43, 0x2a, 0xb1, 0xda, 0x2e, 0xe4, 0xf9, 0x72, 0xd0, 0xcb, - 0x50, 0xa4, 0xb9, 0x93, 0x76, 0x34, 0xac, 0x6c, 0xa6, 0x25, 0x05, 0x01, 0x26, 0x2e, 0xc9, 0x14, - 0x42, 0x18, 0xfa, 0x3f, 0x00, 0x92, 0x2e, 0x78, 0xd6, 0x4c, 0xd1, 0xdc, 0x43, 0x3b, 0x27, 0xc7, - 0x36, 0x46, 0x52, 0x65, 0x31, 0x02, 0xd6, 0x7e, 0x97, 0x82, 0x92, 0x78, 0x2c, 0xf9, 0x50, 0xca, - 0x3f, 0x82, 0xb0, 0xab, 0x6d, 0xe9, 0x86, 0x41, 0xfe, 0xe2, 0xf0, 0xc3, 0xb6, 0x3c, 0xd6, 0x48, - 0xe1, 0xff, 0x2b, 0x21, 0x07, 0xeb, 0x61, 0xe8, 0xd5, 0x0b, 0x33, 0x81, 0x12, 0xb4, 0x96, 0x93, - 0xb8, 0x85, 0x1d, 0x98, 0x53, 0x8a, 0x12, 0x3b, 0x8f, 0xec, 0xa3, 0xea, 0x3c, 0x7e, 0x9d, 0x85, - 0x39, 0xe5, 0x71, 0x70, 0x22, 0x82, 0xd3, 0x8f, 0x24, 0x82, 0x7f, 0xae, 0xa9, 0x2c, 0xcb, 0xce, - 0x82, 0x5e, 0x3b, 0xc4, 0x19, 0xf5, 0xa3, 0xb2, 0xb1, 0x1c, 0x16, 0xd9, 0x87, 0x8a, 0xc9, 0xdc, - 0x61, 0x63, 0x12, 0x5d, 0x64, 0x4d, 0x1c, 0xd5, 0xc5, 0x4e, 0x6a, 0xc2, 0x1d, 0x9a, 0x50, 0x95, - 0xe7, 0x20, 0xd2, 0xd7, 0x87, 0x1c, 0x6c, 0x74, 0x50, 0x88, 0xfb, 0x7a, 0x4e, 0x93, 0x9c, 0x1e, - 0x4c, 0x88, 0x70, 0x21, 0xfb, 0x15, 0x8f, 0x90, 0xfd, 0xe0, 0xa0, 0xec, 0xf7, 0x58, 0x63, 0x53, - 0x4a, 0xb5, 0x03, 0x0d, 0xa6, 0x13, 0xb7, 0x30, 0xbe, 0x5f, 0xdf, 0x92, 0x8f, 0x35, 0x28, 0x46, - 0x97, 0x7c, 0xd0, 0x0a, 0xe4, 0x30, 0xbb, 0x28, 0xc2, 0xd2, 0xce, 0x4c, 0x62, 0xd6, 0x4b, 0x70, - 0xfc, 0xda, 0x5e, 0xe2, 0x6e, 0x48, 0x93, 0x33, 0x3e, 0x44, 0xc1, 0xfc, 0x7b, 0x2d, 0x2c, 0x98, - 0x47, 0x56, 0x91, 0xfe, 0xee, 0xab, 0x38, 0x3e, 0xd3, 0xfd, 0xb1, 0x08, 0x59, 0xba, 0x16, 0xd2, - 0xbc, 0xfa, 0xd8, 0xed, 0x9b, 0x96, 0xde, 0xa3, 0xa1, 0x58, 0x60, 0xbb, 0x3a, 0x84, 0x89, 0xbb, - 0x3a, 0x84, 0xa1, 0x6d, 0x98, 0x8e, 0x47, 0x62, 0x54, 0x8c, 0xfa, 0xd6, 0xe0, 0x0d, 0x99, 0x88, - 0x9d, 0x36, 0x24, 0x38, 0xe5, 0x63, 0xff, 0x04, 0x12, 0x19, 0x30, 0xd5, 0xb1, 0x2d, 0x5f, 0x37, - 0x2d, 0xec, 0x32, 0x45, 0x69, 0xd5, 0xad, 0xa9, 0xcb, 0x12, 0x0d, 0x1b, 0x54, 0xc8, 0x7c, 0xf2, - 0xad, 0x29, 0x19, 0x87, 0xde, 0x87, 0xc9, 0xb0, 0x71, 0x61, 0x4a, 0x32, 0xaa, 0x5b, 0x53, 0xab, - 0x22, 0x09, 0xdb, 0x0c, 0x12, 0x97, 0x7c, 0x6b, 0x4a, 0x42, 0xa1, 0x1e, 0x94, 0x1d, 0xdb, 0xb8, - 0x6b, 0xf1, 0x72, 0x5d, 0x6f, 0xf7, 0x30, 0x9f, 0xc3, 0x2e, 0x8e, 0x14, 0x24, 0x12, 0x15, 0x4b, - 0xd4, 0x49, 0x5e, 0xf9, 0x1e, 0x62, 0x12, 0x8b, 0xde, 0x83, 0x89, 0x1e, 0xe9, 0xdf, 0x56, 0xf7, - 0x1c, 0xd3, 0xc5, 0x86, 0xfa, 0xd6, 0xe0, 0x4d, 0x81, 0x82, 0xa5, 0x49, 0x91, 0x47, 0xbe, 0x2c, - 0x21, 0x62, 0x88, 0xf7, 0xfb, 0xfa, 0x5e, 0x33, 0xb0, 0xbc, 0xd5, 0x3d, 0x7e, 0x03, 0x2c, 0xaf, - 0xf2, 0xfe, 0xba, 0x4c, 0xc4, 0xbc, 0x9f, 0xe0, 0x94, 0xbd, 0x9f, 0x40, 0xa2, 0x9b, 0xf4, 0x2b, - 0xc0, 0x5c, 0xc2, 0x6e, 0x0f, 0xce, 0x8f, 0x58, 0x8b, 0x79, 0x83, 0x0d, 0x5c, 0xf8, 0x93, 0x24, - 0x34, 0x92, 0xc0, 0x7d, 0x40, 0x5f, 0xbb, 0x89, 0xfd, 0xc0, 0xb5, 0xb0, 0xc1, 0xdb, 0xa8, 0x51, - 0x1f, 0x48, 0x54, 0x91, 0x0f, 0x24, 0xe8, 0x88, 0x0f, 0x24, 0x2c, 0xfa, 0x08, 0x66, 0x13, 0x77, - 0xa1, 0xd8, 0x7b, 0x94, 0x54, 0x87, 0x10, 0x6b, 0x0a, 0x4a, 0xd6, 0xf1, 0xaa, 0x64, 0x48, 0x9a, - 0x95, 0x5a, 0x88, 0xf6, 0xae, 0x6e, 0x75, 0xd7, 0xec, 0xb6, 0x1c, 0x73, 0x13, 0x2a, 0xed, 0x57, - 0x15, 0x94, 0x4c, 0xbb, 0x4a, 0x86, 0xac, 0x5d, 0x45, 0x11, 0xdd, 0x7b, 0x22, 0x45, 0x4c, 0x74, - 0x3f, 0x50, 0x75, 0xef, 0x89, 0x11, 0x08, 0xf7, 0x9e, 0x18, 0x40, 0x71, 0xef, 0x89, 0x53, 0x16, - 0xc2, 0x61, 0x4d, 0xed, 0x1d, 0x98, 0x4e, 0xa4, 0x17, 0xf4, 0x26, 0x44, 0xb7, 0x69, 0xee, 0xec, - 0x3b, 0x61, 0xed, 0x2a, 0xdd, 0xbe, 0x21, 0x70, 0xd5, 0xed, 0x1b, 0x02, 0xaf, 0x7d, 0x96, 0x81, - 0x42, 0x18, 0x51, 0xc7, 0xd2, 0x8d, 0x2c, 0x43, 0xbe, 0x8f, 0x3d, 0x7a, 0x63, 0x26, 0x15, 0x17, - 0x35, 0x1c, 0x24, 0x16, 0x35, 0x1c, 0x24, 0xd7, 0x5c, 0xe9, 0x87, 0xaa, 0xb9, 0x32, 0x87, 0xae, - 0xb9, 0x30, 0x3d, 0x24, 0x16, 0xf2, 0x62, 0x78, 0xb8, 0xf1, 0xe0, 0x64, 0x1b, 0x1e, 0x21, 0x8b, - 0x8c, 0x89, 0x23, 0x64, 0x11, 0x85, 0x76, 0xe0, 0xa4, 0x70, 0x00, 0xc3, 0x47, 0x6f, 0x24, 0x43, - 0x4d, 0x8d, 0x3f, 0x91, 0x6f, 0x52, 0x2a, 0xb6, 0x0f, 0x77, 0x12, 0x50, 0xb1, 0x68, 0x4d, 0xe2, - 0x48, 0x48, 0x18, 0xb8, 0x1d, 0x74, 0xd7, 0xb9, 0xd9, 0xf3, 0x71, 0x48, 0x88, 0x70, 0x31, 0x24, - 0x44, 0x78, 0xed, 0x9f, 0x29, 0x98, 0x92, 0xdf, 0xf7, 0x58, 0x02, 0xe3, 0x65, 0x28, 0xe2, 0x3d, - 0xd3, 0x6f, 0x75, 0x6c, 0x03, 0xf3, 0xce, 0x8d, 0xfa, 0x99, 0x00, 0x2f, 0xdb, 0x86, 0xe4, 0xe7, - 0x10, 0x26, 0x46, 0x53, 0xfa, 0x50, 0xd1, 0x14, 0x4f, 0x3a, 0x33, 0x87, 0x98, 0x74, 0x2a, 0xfd, - 0x54, 0x3c, 0x1e, 0x3f, 0xd5, 0xbe, 0x4a, 0x41, 0x39, 0x99, 0x76, 0x9f, 0x8c, 0x2d, 0x28, 0xef, - 0xa6, 0xf4, 0xa1, 0x77, 0xd3, 0x5b, 0x30, 0x49, 0x2a, 0x33, 0xdd, 0xf7, 0xf9, 0x05, 0xdb, 0x0c, - 0x2d, 0xae, 0x58, 0x36, 0x0a, 0xac, 0x95, 0x10, 0x2e, 0x65, 0x23, 0x01, 0x3e, 0x12, 0xba, 0xd9, - 0x23, 0x86, 0xee, 0x27, 0x29, 0x98, 0xdc, 0xb0, 0x8d, 0x3b, 0xac, 0x68, 0xf3, 0x9f, 0x14, 0x7b, - 0x3e, 0xce, 0x94, 0x56, 0x9b, 0x86, 0x49, 0xa9, 0x6a, 0xab, 0x7d, 0xca, 0xe2, 0x4c, 0xfe, 0x5c, - 0xfd, 0xf7, 0xd9, 0x65, 0x0a, 0x26, 0xc4, 0xf2, 0xaf, 0xd6, 0x80, 0xe9, 0x44, 0xb5, 0x26, 0xbe, - 0x80, 0x76, 0x98, 0x17, 0xa8, 0x5d, 0x81, 0x59, 0x55, 0x19, 0x23, 0x64, 0x1d, 0xed, 0x10, 0xa7, - 0x33, 0x57, 0x61, 0x56, 0x55, 0x8e, 0x1c, 0x7d, 0x39, 0x6f, 0xf2, 0x93, 0x4f, 0x56, 0x38, 0x1c, - 0x9d, 0xff, 0xaf, 0x51, 0xf7, 0x1c, 0x5f, 0x66, 0x7f, 0x1b, 0xca, 0x4e, 0xf8, 0xd0, 0xe2, 0x3d, - 0x1a, 0xdb, 0x96, 0xb4, 0xe3, 0x88, 0x70, 0x6b, 0x89, 0x66, 0x6d, 0x4a, 0xc6, 0xc8, 0x72, 0x78, - 0xff, 0x96, 0x53, 0xc8, 0x69, 0x26, 0x1a, 0xb9, 0x29, 0x19, 0x23, 0x98, 0x36, 0x7f, 0xb0, 0x69, - 0x69, 0xff, 0x97, 0x25, 0x4d, 0xf3, 0x74, 0xe2, 0xb2, 0x3d, 0xba, 0x08, 0x05, 0xfa, 0x4b, 0xb8, - 0xb8, 0xf3, 0xa5, 0xd6, 0xa1, 0x30, 0x69, 0x01, 0x79, 0x0e, 0x42, 0xaf, 0x40, 0x31, 0xba, 0x7f, - 0xcf, 0xcf, 0x3c, 0x59, 0xdc, 0x85, 0x40, 0x29, 0xee, 0x42, 0x20, 0x6f, 0x9a, 0x7f, 0x02, 0xa7, - 0xc7, 0xde, 0xbc, 0x3f, 0x4a, 0x0f, 0x2e, 0x74, 0xbf, 0x99, 0x23, 0x75, 0xbf, 0x7b, 0x30, 0xaf, - 0xbe, 0x10, 0x2f, 0x68, 0x4f, 0x1d, 0xa8, 0x3d, 0xb6, 0x7e, 0xfa, 0x90, 0xd6, 0x4f, 0xd5, 0x76, - 0xe8, 0xb8, 0x20, 0xba, 0x78, 0x8e, 0x2e, 0x40, 0xd6, 0xb1, 0xed, 0x9e, 0xc7, 0x2f, 0x15, 0x50, - 0x75, 0x14, 0x20, 0xaa, 0xa3, 0x80, 0x87, 0x18, 0x4e, 0x04, 0x61, 0x04, 0xc7, 0xd7, 0xe8, 0x1f, - 0x83, 0x75, 0x9f, 0xbf, 0x08, 0x85, 0xf0, 0xe0, 0x16, 0x01, 0xe4, 0xde, 0xb9, 0xbb, 0x7a, 0x77, - 0xf5, 0x4a, 0xf9, 0x04, 0x2a, 0x41, 0x7e, 0x63, 0xf5, 0xd6, 0x95, 0xeb, 0xb7, 0xae, 0x96, 0x35, - 0xf2, 0xd0, 0xbc, 0x7b, 0xeb, 0x16, 0x79, 0x48, 0x3d, 0x7f, 0x53, 0xbc, 0x0c, 0xc6, 0x2b, 0xb7, - 0x09, 0x28, 0xac, 0x38, 0x0e, 0x4d, 0x21, 0x8c, 0x77, 0x75, 0xd7, 0x24, 0x3b, 0xb9, 0xac, 0xa1, - 0x3c, 0xa4, 0x6f, 0xdf, 0x5e, 0x2f, 0xa7, 0xd0, 0x2c, 0x94, 0xaf, 0x60, 0xdd, 0xe8, 0x99, 0x16, - 0x0e, 0xf3, 0x56, 0x39, 0xdd, 0xb8, 0xff, 0xa7, 0x6f, 0x16, 0xb5, 0xaf, 0xbe, 0x59, 0xd4, 0xfe, - 0xf1, 0xcd, 0xa2, 0xf6, 0xd9, 0xb7, 0x8b, 0x27, 0xbe, 0xfa, 0x76, 0xf1, 0xc4, 0xdf, 0xbe, 0x5d, - 0x3c, 0xf1, 0xa3, 0x8b, 0x5d, 0xd3, 0xdf, 0x0e, 0xda, 0xf5, 0x8e, 0xdd, 0xe7, 0x3f, 0xe9, 0x75, - 0x5c, 0x9b, 0x24, 0x08, 0xfe, 0xb4, 0x9c, 0xfc, 0xad, 0xef, 0x6f, 0x52, 0x67, 0x56, 0xe8, 0xe3, - 0x06, 0xa3, 0xab, 0x5f, 0xb7, 0xeb, 0x0c, 0x40, 0x7f, 0xdd, 0xe9, 0xb5, 0x73, 0xf4, 0x57, 0x9c, - 0x2f, 0xff, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x5a, 0x65, 0xa8, 0x22, 0x26, 0x3c, 0x00, 0x00, + // 3666 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x3b, 0x4d, 0x6f, 0x1c, 0xc7, + 0x95, 0xea, 0xf9, 0x9e, 0x37, 0xe4, 0x70, 0x54, 0xfc, 0xd0, 0x88, 0xb6, 0x38, 0xd4, 0xc8, 0xbb, + 0x96, 0x0c, 0x7b, 0x28, 0xcb, 0xeb, 0x85, 0x3f, 0x16, 0x36, 0x38, 0x12, 0x2d, 0x89, 0x12, 0x25, + 0x7a, 0x28, 0x79, 0xb5, 0x0b, 0x03, 0xe3, 0x9e, 0xe9, 0xe2, 0xb0, 0xc5, 0x99, 0xee, 0x76, 0x7f, + 0xd0, 0x24, 0x60, 0xec, 0xda, 0x0b, 0xef, 0x9e, 0x7d, 0x59, 0x20, 0xc8, 0x25, 0xbe, 0xe4, 0xe0, + 0x20, 0x39, 0x26, 0xd7, 0x5c, 0x73, 0x08, 0x02, 0x5f, 0x02, 0x04, 0x41, 0x3c, 0x08, 0x6c, 0xe4, + 0x32, 0x87, 0xfc, 0x86, 0xa0, 0x3e, 0xba, 0xbb, 0xaa, 0xa7, 0x46, 0x24, 0x65, 0x51, 0x70, 0x9c, + 0x13, 0xd9, 0xef, 0xbb, 0xeb, 0xbd, 0x7a, 0xfd, 0xde, 0xab, 0x1a, 0x38, 0xe7, 0xec, 0xf6, 0x56, + 0x74, 0x77, 0xa0, 0x1b, 0x3a, 0xde, 0xc3, 0x96, 0xef, 0xad, 0xb0, 0x3f, 0x0d, 0xc7, 0xb5, 0x7d, + 0x1b, 0x4d, 0x89, 0xa8, 0xc5, 0xfa, 0xee, 0x6b, 0x5e, 0xc3, 0xb4, 0x57, 0x74, 0xc7, 0x5c, 0xe9, + 0xda, 0x2e, 0x5e, 0xd9, 0x7b, 0x79, 0xa5, 0x87, 0x2d, 0xec, 0xea, 0x3e, 0x36, 0x18, 0xc7, 0xe2, + 0x45, 0x81, 0xc6, 0xc2, 0xfe, 0x47, 0xb6, 0xbb, 0x6b, 0x5a, 0x3d, 0x15, 0x65, 0xad, 0x67, 0xdb, + 0xbd, 0x3e, 0x5e, 0xa1, 0x4f, 0x9d, 0x60, 0x7b, 0xc5, 0x37, 0x07, 0xd8, 0xf3, 0xf5, 0x81, 0xc3, + 0x09, 0xfe, 0x25, 0x16, 0x35, 0xd0, 0xbb, 0x3b, 0xa6, 0x85, 0xdd, 0x83, 0x15, 0x6a, 0xaf, 0x63, + 0xae, 0xb8, 0xd8, 0xb3, 0x03, 0xb7, 0x8b, 0xc7, 0xc4, 0xbe, 0x61, 0x5a, 0x3e, 0x76, 0x2d, 0xbd, + 0xbf, 0xe2, 0x75, 0x77, 0xb0, 0x11, 0xf4, 0xb1, 0x1b, 0xff, 0x67, 0x77, 0x1e, 0xe2, 0xae, 0xef, + 0x8d, 0x01, 0x18, 0x6f, 0xfd, 0xeb, 0x79, 0x98, 0x5e, 0x23, 0xef, 0xba, 0x85, 0x3f, 0x0c, 0xb0, + 0xd5, 0xc5, 0xe8, 0x12, 0x64, 0x3f, 0x0c, 0x70, 0x80, 0xab, 0xda, 0xb2, 0x76, 0xb1, 0xd8, 0x9c, + 0x1d, 0x0d, 0x6b, 0x33, 0x14, 0xf0, 0xa2, 0x3d, 0x30, 0x7d, 0x3c, 0x70, 0xfc, 0x83, 0x16, 0xa3, + 0x40, 0x6f, 0xc0, 0xd4, 0x43, 0xbb, 0xd3, 0xf6, 0xb0, 0xdf, 0xb6, 0xf4, 0x01, 0xae, 0xa6, 0x28, + 0x47, 0x75, 0x34, 0xac, 0xcd, 0x3d, 0xb4, 0x3b, 0x5b, 0xd8, 0xbf, 0xa3, 0x0f, 0x44, 0x36, 0x88, + 0xa1, 0xe8, 0x25, 0xc8, 0x07, 0x1e, 0x76, 0xdb, 0xa6, 0x51, 0x4d, 0x53, 0xb6, 0xb9, 0xd1, 0xb0, + 0x56, 0x21, 0xa0, 0x9b, 0x86, 0xc0, 0x92, 0x63, 0x10, 0xf4, 0x22, 0xe4, 0x7a, 0xae, 0x1d, 0x38, + 0x5e, 0x35, 0xb3, 0x9c, 0x0e, 0xa9, 0x19, 0x44, 0xa4, 0x66, 0x10, 0x74, 0x17, 0x72, 0xcc, 0x81, + 0xd5, 0xec, 0x72, 0xfa, 0x62, 0xe9, 0xca, 0xf9, 0x86, 0xe8, 0xd5, 0x86, 0xf4, 0xc2, 0xec, 0x89, + 0x09, 0x64, 0x78, 0x51, 0x20, 0x8f, 0x83, 0x5f, 0xcd, 0x42, 0x96, 0xd2, 0xa1, 0x5b, 0x90, 0xef, + 0xba, 0x98, 0xac, 0x7e, 0x15, 0x2d, 0x6b, 0x17, 0x4b, 0x57, 0x16, 0x1b, 0xcc, 0xab, 0x8d, 0xd0, + 0xab, 0x8d, 0x7b, 0xa1, 0x57, 0x9b, 0xf3, 0xa3, 0x61, 0xed, 0x34, 0x27, 0x17, 0xa4, 0x86, 0x12, + 0xd0, 0x26, 0x14, 0xbd, 0xa0, 0x33, 0x30, 0xfd, 0x75, 0xbb, 0x43, 0xd7, 0xbb, 0x74, 0xe5, 0x8c, + 0x6c, 0xea, 0x56, 0x88, 0x6e, 0x9e, 0x19, 0x0d, 0x6b, 0xb3, 0x11, 0x75, 0x2c, 0xed, 0xc6, 0xa9, + 0x56, 0x2c, 0x04, 0xed, 0xc0, 0x8c, 0x8b, 0x1d, 0xd7, 0xb4, 0x5d, 0xd3, 0x37, 0x3d, 0x4c, 0xe4, + 0xa6, 0xa8, 0xdc, 0x73, 0xb2, 0xdc, 0x96, 0x4c, 0xd4, 0x3c, 0x37, 0x1a, 0xd6, 0xce, 0x26, 0x38, + 0x25, 0x1d, 0x49, 0xb1, 0xc8, 0x07, 0x94, 0x00, 0x6d, 0x61, 0x9f, 0xfa, 0xb2, 0x74, 0x65, 0xf9, + 0x91, 0xca, 0xb6, 0xb0, 0xdf, 0x5c, 0x1e, 0x0d, 0x6b, 0xcf, 0x8e, 0xf3, 0x4b, 0x2a, 0x15, 0xf2, + 0x51, 0x1f, 0x2a, 0x22, 0xd4, 0x20, 0x2f, 0x98, 0xa1, 0x3a, 0x97, 0x26, 0xeb, 0x24, 0x54, 0xcd, + 0xa5, 0xd1, 0xb0, 0xb6, 0x98, 0xe4, 0x95, 0xf4, 0x8d, 0x49, 0x26, 0xfe, 0xe9, 0xea, 0x56, 0x17, + 0xf7, 0x89, 0x9a, 0xac, 0xca, 0x3f, 0x57, 0x43, 0x34, 0xf3, 0x4f, 0x44, 0x2d, 0xfb, 0x27, 0x02, + 0xa3, 0xf7, 0x61, 0x2a, 0x7a, 0x20, 0xeb, 0x95, 0xe3, 0x31, 0xa4, 0x16, 0x4a, 0x56, 0x6a, 0x71, + 0x34, 0xac, 0x2d, 0x88, 0x3c, 0x92, 0x68, 0x49, 0x5a, 0x2c, 0xbd, 0xcf, 0x56, 0x26, 0x3f, 0x59, + 0x3a, 0xa3, 0x10, 0xa5, 0xf7, 0xc7, 0x57, 0x44, 0x92, 0x46, 0xa4, 0x93, 0x0d, 0x1c, 0x74, 0xbb, + 0x18, 0x1b, 0xd8, 0xa8, 0x16, 0x54, 0xd2, 0xd7, 0x05, 0x0a, 0x26, 0x5d, 0xe4, 0x91, 0xa5, 0x8b, + 0x18, 0xb2, 0xd6, 0x0f, 0xed, 0xce, 0x9a, 0xeb, 0xda, 0xae, 0x57, 0x2d, 0xaa, 0xd6, 0x7a, 0x3d, + 0x44, 0xb3, 0xb5, 0x8e, 0xa8, 0xe5, 0xb5, 0x8e, 0xc0, 0xdc, 0xde, 0x56, 0x60, 0xdd, 0xc6, 0xba, + 0x87, 0x8d, 0x2a, 0x4c, 0xb0, 0x37, 0xa2, 0x88, 0xec, 0x8d, 0x20, 0x63, 0xf6, 0x46, 0x18, 0x64, + 0x40, 0x99, 0x3d, 0xaf, 0x7a, 0x9e, 0xd9, 0xb3, 0xb0, 0x51, 0x2d, 0x51, 0xf9, 0xcf, 0xaa, 0xe4, + 0x87, 0x34, 0xcd, 0x67, 0x47, 0xc3, 0x5a, 0x55, 0xe6, 0x93, 0x74, 0x24, 0x64, 0xa2, 0x0f, 0x60, + 0x9a, 0x41, 0x5a, 0x81, 0x65, 0x99, 0x56, 0xaf, 0x3a, 0x45, 0x95, 0x3c, 0xa3, 0x52, 0xc2, 0x49, + 0x9a, 0xcf, 0x8c, 0x86, 0xb5, 0x33, 0x12, 0x97, 0xa4, 0x42, 0x16, 0x48, 0x32, 0x06, 0x03, 0xc4, + 0x8e, 0x9d, 0x56, 0x65, 0x8c, 0x75, 0x99, 0x88, 0x65, 0x8c, 0x04, 0xa7, 0x9c, 0x31, 0x12, 0xc8, + 0xd8, 0x1f, 0xdc, 0xc9, 0xe5, 0xc9, 0xfe, 0xe0, 0x7e, 0x16, 0xfc, 0xa1, 0x70, 0xb5, 0x24, 0x0d, + 0x7d, 0xa2, 0xc1, 0xbc, 0xe7, 0xeb, 0x96, 0xa1, 0xf7, 0x6d, 0x0b, 0xdf, 0xb4, 0x7a, 0x2e, 0xf6, + 0xbc, 0x9b, 0xd6, 0xb6, 0x5d, 0xad, 0x50, 0x3d, 0x17, 0x12, 0x89, 0x55, 0x45, 0xda, 0xbc, 0x30, + 0x1a, 0xd6, 0x6a, 0x4a, 0x29, 0x92, 0x66, 0xb5, 0x22, 0xb4, 0x0f, 0xb3, 0xe1, 0x47, 0xfa, 0xbe, + 0x6f, 0xf6, 0x4d, 0x4f, 0xf7, 0x4d, 0xdb, 0xaa, 0x9e, 0xa6, 0xfa, 0xcf, 0x27, 0xf3, 0xd3, 0x18, + 0x61, 0xf3, 0xfc, 0x68, 0x58, 0x3b, 0xa7, 0x90, 0x20, 0xe9, 0x56, 0xa9, 0x88, 0x9d, 0xb8, 0xe9, + 0x62, 0x42, 0x88, 0x8d, 0xea, 0xec, 0x64, 0x27, 0x46, 0x44, 0xa2, 0x13, 0x23, 0xa0, 0xca, 0x89, + 0x11, 0x92, 0x68, 0x72, 0x74, 0xd7, 0x37, 0x89, 0xda, 0x0d, 0xdd, 0xdd, 0xc5, 0x6e, 0x75, 0x4e, + 0xa5, 0x69, 0x53, 0x26, 0x62, 0x9a, 0x12, 0x9c, 0xb2, 0xa6, 0x04, 0x12, 0x7d, 0xae, 0x81, 0x6c, + 0x9a, 0x69, 0x5b, 0x2d, 0xf2, 0xd1, 0xf6, 0xc8, 0xeb, 0xcd, 0x53, 0xa5, 0xcf, 0x3f, 0xe2, 0xf5, + 0x44, 0xf2, 0xe6, 0xf3, 0xa3, 0x61, 0xed, 0xc2, 0x44, 0x69, 0x92, 0x21, 0x93, 0x95, 0xa2, 0x07, + 0x50, 0x22, 0x48, 0x4c, 0xcb, 0x1f, 0xa3, 0xba, 0x40, 0x6d, 0x38, 0x3b, 0x6e, 0x03, 0x27, 0x68, + 0x9e, 0x1d, 0x0d, 0x6b, 0xf3, 0x02, 0x87, 0xa4, 0x47, 0x14, 0x85, 0x3e, 0xd3, 0x80, 0x04, 0xba, + 0xea, 0x4d, 0xcf, 0x50, 0x2d, 0xcf, 0x8d, 0x69, 0x51, 0xbd, 0xe6, 0x73, 0xa3, 0x61, 0x6d, 0x59, + 0x2d, 0x47, 0xd2, 0x3d, 0x41, 0x57, 0x1c, 0x47, 0xd1, 0x47, 0xa2, 0x5a, 0x9d, 0x1c, 0x47, 0x11, + 0x91, 0x18, 0x47, 0x11, 0x50, 0x15, 0x47, 0x11, 0x92, 0x27, 0x83, 0xf7, 0xf4, 0xbe, 0x69, 0xd0, + 0x62, 0xea, 0xec, 0x84, 0x64, 0x10, 0x51, 0x44, 0xc9, 0x20, 0x82, 0x8c, 0x25, 0x83, 0x98, 0x36, + 0x0f, 0x59, 0x2a, 0xa2, 0xfe, 0xc7, 0x1c, 0xcc, 0x2a, 0xb6, 0x1a, 0xc2, 0x30, 0x1d, 0xee, 0xa3, + 0xb6, 0x49, 0x92, 0x44, 0x5a, 0xb5, 0xca, 0xb7, 0x82, 0x0e, 0x76, 0x2d, 0xec, 0x63, 0x2f, 0x94, + 0x41, 0xb3, 0x04, 0xb5, 0xc4, 0x15, 0x20, 0x42, 0x6d, 0x37, 0x25, 0xc2, 0xd1, 0x8f, 0x35, 0xa8, + 0x0e, 0xf4, 0xfd, 0x76, 0x08, 0xf4, 0xda, 0xdb, 0xb6, 0xdb, 0x76, 0xb0, 0x6b, 0xda, 0x06, 0xad, + 0x64, 0x4b, 0x57, 0xfe, 0xed, 0xd0, 0xbc, 0xd0, 0xd8, 0xd0, 0xf7, 0x43, 0xb0, 0xf7, 0x8e, 0xed, + 0x6e, 0x52, 0xf6, 0x35, 0xcb, 0x77, 0x0f, 0x58, 0xc2, 0x1a, 0xa8, 0xf0, 0x82, 0x4d, 0xf3, 0x4a, + 0x02, 0xf4, 0xff, 0x1a, 0x2c, 0xf8, 0xb6, 0xaf, 0xf7, 0xdb, 0xdd, 0x60, 0x10, 0xf4, 0x75, 0xdf, + 0xdc, 0xc3, 0xed, 0xc0, 0xd3, 0x7b, 0x98, 0x97, 0xcd, 0x6f, 0x1e, 0x6e, 0xda, 0x3d, 0xc2, 0x7f, + 0x35, 0x62, 0xbf, 0x4f, 0xb8, 0x99, 0x65, 0xf5, 0xd1, 0xb0, 0xb6, 0xe4, 0x2b, 0xd0, 0x82, 0x61, + 0x73, 0x2a, 0x3c, 0x7a, 0x01, 0x72, 0xa4, 0xad, 0x30, 0x0d, 0x5a, 0x1d, 0xf1, 0x16, 0xe4, 0xa1, + 0xdd, 0x91, 0x1a, 0x83, 0x2c, 0x05, 0x10, 0x5a, 0x37, 0xb0, 0x08, 0x6d, 0x3e, 0xa6, 0x75, 0x03, + 0x4b, 0xa6, 0xa5, 0x80, 0xc5, 0x2f, 0x34, 0x58, 0x9c, 0xbc, 0x94, 0xe8, 0x02, 0xa4, 0x77, 0xf1, + 0x01, 0x6f, 0x7b, 0x4e, 0x8f, 0x86, 0xb5, 0xe9, 0x5d, 0x7c, 0x20, 0x48, 0x21, 0x58, 0xf4, 0x1f, + 0x90, 0xdd, 0xd3, 0xfb, 0x01, 0xe6, 0x55, 0x75, 0xa3, 0xc1, 0x3a, 0xb6, 0x86, 0xd8, 0xb1, 0x35, + 0x9c, 0xdd, 0x1e, 0x01, 0x34, 0x42, 0xaf, 0x37, 0xde, 0x0d, 0x74, 0xcb, 0x37, 0xfd, 0x03, 0x66, + 0x1e, 0x15, 0x20, 0x9a, 0x47, 0x01, 0x6f, 0xa4, 0x5e, 0xd3, 0x16, 0x7f, 0xa2, 0xc1, 0xd9, 0x89, + 0x4b, 0xfa, 0x7d, 0xb0, 0x70, 0x3d, 0x53, 0xd0, 0x2a, 0xa9, 0xf5, 0x4c, 0x21, 0x55, 0x49, 0xd7, + 0x7f, 0x9e, 0x87, 0x62, 0xd4, 0xa0, 0xa0, 0x1b, 0x50, 0x31, 0xb0, 0x11, 0x38, 0x7d, 0xb3, 0x4b, + 0x63, 0x83, 0x38, 0x85, 0x75, 0x84, 0x34, 0x3b, 0x48, 0x38, 0xc9, 0x3d, 0x33, 0x09, 0x14, 0xba, + 0x02, 0x05, 0x5e, 0x88, 0x1f, 0xd0, 0x7d, 0x39, 0xdd, 0x5c, 0x18, 0x0d, 0x6b, 0x28, 0x84, 0x09, + 0xac, 0x11, 0x1d, 0x6a, 0x01, 0xb0, 0xce, 0x76, 0x03, 0xfb, 0x3a, 0x6f, 0x09, 0xaa, 0x72, 0xfc, + 0xde, 0x8d, 0xf0, 0xac, 0x47, 0x8d, 0xe9, 0xc5, 0x1e, 0x35, 0x86, 0xa2, 0xf7, 0x01, 0x06, 0xba, + 0x69, 0x31, 0x3e, 0x5e, 0xff, 0xd7, 0x27, 0x65, 0x88, 0x8d, 0x88, 0x92, 0x49, 0x8f, 0x39, 0x45, + 0xe9, 0x31, 0x14, 0xdd, 0x85, 0x3c, 0xef, 0xc5, 0xab, 0x39, 0xba, 0xdd, 0x96, 0x26, 0x89, 0xe6, + 0x62, 0x69, 0x37, 0xc9, 0x59, 0xc4, 0x6e, 0x92, 0x83, 0xc8, 0xb2, 0xf5, 0xcd, 0x6d, 0xec, 0x9b, + 0x03, 0x4c, 0x77, 0x03, 0x5f, 0xb6, 0x10, 0x26, 0x2e, 0x5b, 0x08, 0x43, 0xaf, 0x01, 0xe8, 0xfe, + 0x86, 0xed, 0xf9, 0x77, 0xad, 0x2e, 0xa6, 0x15, 0x7d, 0x81, 0x99, 0x1f, 0x43, 0x45, 0xf3, 0x63, + 0x28, 0x7a, 0x13, 0x4a, 0x0e, 0xff, 0x82, 0x74, 0xfa, 0x98, 0x56, 0xec, 0x05, 0xf6, 0xc1, 0x13, + 0xc0, 0x02, 0xaf, 0x48, 0x8d, 0xae, 0xc3, 0x4c, 0xd7, 0xb6, 0xba, 0x81, 0xeb, 0x62, 0xab, 0x7b, + 0xb0, 0xa5, 0x6f, 0x63, 0x5a, 0x9d, 0x17, 0x58, 0xa8, 0x24, 0x50, 0x62, 0xa8, 0x24, 0x50, 0xe8, + 0x55, 0x28, 0x46, 0x93, 0x0d, 0x5a, 0x80, 0x17, 0x79, 0xa3, 0x1c, 0x02, 0x05, 0xe6, 0x98, 0x92, + 0x18, 0x6f, 0x7a, 0xd7, 0x78, 0xd0, 0x61, 0x5a, 0x54, 0x73, 0xe3, 0x05, 0xb0, 0x68, 0xbc, 0x00, + 0x16, 0xf2, 0x53, 0xf9, 0xd0, 0xfc, 0xf4, 0x5f, 0x30, 0x8f, 0xf7, 0x49, 0xbe, 0x1f, 0x60, 0xcb, + 0xd7, 0xfb, 0x9b, 0xae, 0xc9, 0xbe, 0x0c, 0xd5, 0x19, 0x55, 0x51, 0xba, 0xa6, 0x22, 0x65, 0x39, + 0x5e, 0x29, 0x45, 0xcc, 0xf1, 0x4a, 0x82, 0x68, 0xbb, 0x4e, 0x57, 0xca, 0xf5, 0x5b, 0x30, 0xaf, + 0x54, 0x40, 0x02, 0xa7, 0x63, 0x1a, 0xf4, 0x99, 0x26, 0x17, 0x8d, 0x05, 0x4e, 0x08, 0x13, 0x03, + 0x27, 0x84, 0xd5, 0x7f, 0xab, 0xc1, 0x9c, 0x2a, 0xf8, 0x13, 0x1b, 0x51, 0x7b, 0x22, 0x1b, 0xf1, + 0x3d, 0x28, 0x38, 0xb6, 0xd1, 0xf6, 0x1c, 0xdc, 0xe5, 0x69, 0x2d, 0xb1, 0x0d, 0x37, 0x6d, 0x63, + 0xcb, 0xc1, 0xdd, 0x7f, 0x37, 0xfd, 0x9d, 0xd5, 0x3d, 0xdb, 0x34, 0x6e, 0x9b, 0x1e, 0xdf, 0x2f, + 0x0e, 0xc3, 0x48, 0xb5, 0x42, 0x9e, 0x03, 0x9b, 0x05, 0xc8, 0x31, 0x2d, 0xf5, 0xdf, 0xa5, 0xa1, + 0x92, 0xdc, 0x70, 0x7f, 0x4f, 0xaf, 0x82, 0x1e, 0x40, 0xde, 0x64, 0xad, 0x08, 0x2f, 0x65, 0xfe, + 0x49, 0x48, 0xfc, 0x8d, 0x78, 0x2e, 0xd9, 0xd8, 0x7b, 0xb9, 0xc1, 0x7b, 0x16, 0xba, 0x04, 0x54, + 0x32, 0xe7, 0x94, 0x25, 0x73, 0x20, 0x6a, 0x41, 0xde, 0xc3, 0xee, 0x1e, 0x09, 0x0e, 0x96, 0x56, + 0x6b, 0xa2, 0xe4, 0xae, 0xed, 0x62, 0x22, 0x73, 0x8b, 0x91, 0xc4, 0x32, 0x39, 0x8f, 0x2c, 0x93, + 0x03, 0xd1, 0x7b, 0x50, 0xec, 0xda, 0xd6, 0xb6, 0xd9, 0xdb, 0xd0, 0x1d, 0x9e, 0x58, 0xcf, 0xa9, + 0xa4, 0x5e, 0x0d, 0x89, 0xf8, 0x78, 0x25, 0x7c, 0x4c, 0x8c, 0x57, 0x22, 0xaa, 0xd8, 0xa1, 0x7f, + 0xcd, 0x00, 0xc4, 0xce, 0x41, 0xaf, 0x43, 0x09, 0xef, 0xe3, 0x6e, 0xe0, 0xdb, 0x74, 0xe4, 0xa8, + 0xc5, 0x93, 0xca, 0x10, 0x2c, 0xed, 0x5e, 0x88, 0xa1, 0x24, 0xc5, 0x58, 0xfa, 0x00, 0x7b, 0x8e, + 0xde, 0x0d, 0x47, 0x9c, 0xd4, 0x98, 0x08, 0x28, 0xa6, 0x98, 0x08, 0x88, 0xfe, 0x19, 0x32, 0x74, + 0x28, 0xca, 0xa6, 0x9b, 0x68, 0x34, 0xac, 0x95, 0x2d, 0x79, 0x1c, 0x4a, 0xf1, 0xe8, 0x6d, 0x98, + 0xde, 0x8d, 0x02, 0x8f, 0xd8, 0x96, 0xa1, 0x0c, 0xb4, 0xc6, 0x8c, 0x11, 0x92, 0x75, 0x53, 0x22, + 0x1c, 0x6d, 0x43, 0x49, 0xb7, 0x2c, 0xdb, 0xa7, 0x5f, 0xcf, 0x70, 0xe2, 0x79, 0x69, 0x52, 0x98, + 0x36, 0x56, 0x63, 0x5a, 0x56, 0xa8, 0xd1, 0xb4, 0x27, 0x48, 0x10, 0xd3, 0x9e, 0x00, 0x46, 0x2d, + 0xc8, 0xf5, 0xf5, 0x0e, 0xee, 0x87, 0x9f, 0xab, 0xe7, 0x26, 0xaa, 0xb8, 0x4d, 0xc9, 0x98, 0x74, + 0x3a, 0x57, 0x65, 0x7c, 0xe2, 0x5c, 0x95, 0x41, 0x16, 0xb7, 0xa1, 0x92, 0xb4, 0xe7, 0x68, 0x55, + 0xce, 0x25, 0xb1, 0xca, 0x29, 0x1e, 0x5a, 0x57, 0xe9, 0x50, 0x12, 0x8c, 0x3a, 0x09, 0x15, 0xf5, + 0x2f, 0x35, 0x98, 0x53, 0xed, 0x5d, 0xb4, 0x21, 0xec, 0x78, 0x8d, 0x4f, 0x6f, 0x14, 0xa1, 0xce, + 0x79, 0x27, 0x6c, 0xf5, 0x78, 0xa3, 0x37, 0xa1, 0x6c, 0xd9, 0x06, 0x6e, 0xeb, 0x44, 0x41, 0xdf, + 0xf4, 0xfc, 0x6a, 0x8a, 0x4e, 0xc4, 0xe9, 0xd4, 0x87, 0x60, 0x56, 0x43, 0x84, 0xc0, 0x3d, 0x2d, + 0x21, 0xea, 0x1f, 0xc1, 0x4c, 0x62, 0x26, 0x2b, 0xd5, 0x5c, 0xa9, 0x23, 0xd6, 0x5c, 0xf1, 0x87, + 0x30, 0x7d, 0xd8, 0x87, 0x90, 0x7d, 0x88, 0xea, 0xff, 0x9b, 0x82, 0x92, 0xd0, 0x20, 0xa3, 0x87, + 0x30, 0xc3, 0x3f, 0xca, 0xa6, 0xd5, 0x63, 0x8d, 0x58, 0x8a, 0x7f, 0x18, 0xc7, 0x0e, 0x2c, 0xd6, + 0xed, 0xce, 0x56, 0x44, 0x4b, 0x3f, 0x8c, 0x74, 0x98, 0xe6, 0x49, 0x30, 0x41, 0x71, 0x59, 0xc6, + 0xa0, 0x07, 0xb0, 0x10, 0x38, 0xa4, 0x3d, 0x6c, 0x7b, 0x7c, 0xf4, 0xdf, 0xb6, 0x82, 0x41, 0x07, + 0xbb, 0xd4, 0xfa, 0x2c, 0x6b, 0x58, 0x18, 0x45, 0x78, 0x36, 0x70, 0x87, 0xe2, 0xc5, 0x86, 0x45, + 0x85, 0x17, 0xd6, 0x21, 0x73, 0xc4, 0x75, 0xb8, 0x01, 0x68, 0x7c, 0x28, 0x2e, 0xf9, 0x40, 0x3b, + 0x9a, 0x0f, 0xea, 0xfb, 0x50, 0x49, 0x8e, 0xba, 0x9f, 0x92, 0x2f, 0x77, 0xa1, 0x18, 0x0d, 0xaa, + 0xd1, 0x8b, 0x90, 0x73, 0xb1, 0xee, 0xd9, 0x16, 0xdf, 0x2d, 0x74, 0xdb, 0x33, 0x88, 0xb8, 0xed, + 0x19, 0xe4, 0x31, 0x94, 0xdd, 0x83, 0x29, 0xb6, 0x48, 0xef, 0x98, 0x7d, 0x1f, 0xbb, 0xe8, 0x1a, + 0xe4, 0x3c, 0x5f, 0xf7, 0xb1, 0x57, 0xd5, 0x96, 0xd3, 0x17, 0xcb, 0x57, 0x16, 0xc6, 0xa7, 0xd0, + 0x04, 0xcd, 0xec, 0x60, 0x94, 0xa2, 0x1d, 0x0c, 0x52, 0xff, 0x1f, 0x0d, 0xa6, 0xc4, 0x61, 0xfb, + 0x93, 0x11, 0x7b, 0xbc, 0xc5, 0xa8, 0x5b, 0xa1, 0x0d, 0x7c, 0xcc, 0x7e, 0xd2, 0x4b, 0xf9, 0xa5, + 0xc6, 0xd6, 0x32, 0x9a, 0xcb, 0xf6, 0xe2, 0x59, 0x08, 0xd9, 0x28, 0x1e, 0x4d, 0x28, 0x47, 0x9d, + 0x85, 0xd0, 0xb4, 0x23, 0xb1, 0x8b, 0x69, 0x47, 0x42, 0x3c, 0x86, 0xad, 0x5f, 0x64, 0xa9, 0xad, + 0xf1, 0xd4, 0x3d, 0xf1, 0x1d, 0x4f, 0x1f, 0xe3, 0x3b, 0xfe, 0x12, 0xe4, 0x69, 0xe2, 0x8c, 0xb6, + 0x29, 0x5d, 0x58, 0x02, 0x92, 0x4f, 0x1c, 0x19, 0xe4, 0x11, 0xe9, 0x22, 0xfb, 0x1d, 0xd3, 0x45, + 0x1b, 0xce, 0xee, 0xe8, 0x5e, 0x3b, 0x4c, 0x70, 0x46, 0x5b, 0xf7, 0xdb, 0xd1, 0x7e, 0xcd, 0xd1, + 0x56, 0x84, 0xce, 0xf1, 0x76, 0x74, 0x6f, 0x2b, 0xa4, 0x59, 0xf5, 0x37, 0xc7, 0x77, 0xef, 0x82, + 0x9a, 0x02, 0xdd, 0x87, 0x79, 0xb5, 0xf0, 0x3c, 0xb5, 0x9c, 0x8e, 0x99, 0xbd, 0x47, 0x4a, 0x9e, + 0x55, 0xa0, 0xd1, 0xa7, 0x1a, 0x54, 0xc9, 0x97, 0xcc, 0xc5, 0x1f, 0x06, 0xa6, 0x8b, 0x49, 0x17, + 0xe1, 0xb5, 0xed, 0x3d, 0xec, 0xf6, 0xf5, 0x03, 0x7e, 0x62, 0x73, 0x7e, 0x3c, 0x6d, 0x6f, 0xda, + 0x46, 0x4b, 0x60, 0x60, 0xaf, 0xe6, 0xc8, 0xc0, 0xbb, 0x4c, 0x88, 0xf8, 0x6a, 0x6a, 0x0a, 0x21, + 0x84, 0xe0, 0x18, 0xb3, 0xa1, 0xd2, 0x61, 0xb3, 0x21, 0x52, 0xad, 0x39, 0xb6, 0xdd, 0xa7, 0x9d, + 0x20, 0xaf, 0xd6, 0xc8, 0xb3, 0x58, 0xad, 0x91, 0x67, 0x71, 0xfc, 0xb1, 0x9e, 0x29, 0x14, 0x2a, + 0xc5, 0xfa, 0xd7, 0x1a, 0x94, 0xe5, 0x43, 0x9e, 0xf1, 0x0d, 0x95, 0x3e, 0xf1, 0x0d, 0x95, 0x39, + 0xc6, 0x6a, 0x64, 0x0f, 0x5b, 0x0d, 0x69, 0xc8, 0xf3, 0x27, 0x0d, 0xa6, 0xa5, 0xf3, 0xa5, 0x1f, + 0xd6, 0xeb, 0xfd, 0x28, 0x05, 0x0b, 0x6a, 0x53, 0x4f, 0xa4, 0xfd, 0xbb, 0x01, 0xa4, 0x90, 0xbb, + 0x19, 0x17, 0x3a, 0xf3, 0x63, 0xdd, 0x1f, 0x5d, 0xa6, 0xb0, 0x0a, 0x1c, 0x3b, 0x7a, 0x0a, 0xd9, + 0xd1, 0x03, 0x28, 0x99, 0xc2, 0x21, 0x57, 0x5a, 0x75, 0x16, 0x21, 0x1e, 0x6d, 0xb1, 0xe9, 0xc6, + 0x84, 0x03, 0x2d, 0x51, 0x54, 0x33, 0x07, 0x19, 0x52, 0x89, 0xd5, 0xf7, 0x20, 0xcf, 0xcd, 0x41, + 0xaf, 0x40, 0x91, 0xe6, 0x4e, 0xda, 0xd1, 0xb0, 0xb2, 0x99, 0x96, 0x14, 0x04, 0x98, 0xb8, 0xe4, + 0x51, 0x08, 0x61, 0xe8, 0x5f, 0x01, 0x48, 0xba, 0xe0, 0x59, 0x33, 0x45, 0x73, 0x0f, 0xed, 0x9c, + 0x1c, 0xdb, 0x18, 0x4b, 0x95, 0xc5, 0x08, 0x58, 0xff, 0x45, 0x0a, 0x4a, 0xe2, 0xb1, 0xda, 0x63, + 0x29, 0xff, 0x18, 0xc2, 0xae, 0xb6, 0xad, 0x1b, 0x06, 0xf9, 0x8b, 0xc3, 0x0f, 0xdb, 0xca, 0xc4, + 0x45, 0x0a, 0xff, 0x5f, 0x0d, 0x39, 0x58, 0x0f, 0x43, 0xaf, 0x0e, 0x98, 0x09, 0x94, 0xa0, 0xb5, + 0x92, 0xc4, 0x2d, 0xee, 0xc2, 0xbc, 0x52, 0x94, 0xd8, 0x79, 0x64, 0x9f, 0x54, 0xe7, 0xf1, 0xd3, + 0x2c, 0xcc, 0x2b, 0x8f, 0x33, 0x13, 0x11, 0x9c, 0x7e, 0x22, 0x11, 0xfc, 0x7f, 0x9a, 0x6a, 0x65, + 0xd9, 0x59, 0xc6, 0xeb, 0x47, 0x38, 0x63, 0x7d, 0x52, 0x6b, 0x2c, 0x87, 0x45, 0xf6, 0xb1, 0x62, + 0x32, 0x77, 0xd4, 0x98, 0x44, 0x97, 0x59, 0x13, 0x47, 0x75, 0xb1, 0x93, 0x86, 0x70, 0x87, 0x26, + 0x54, 0xe5, 0x39, 0x88, 0xf4, 0xf5, 0x21, 0x07, 0x1b, 0x1d, 0x14, 0xe2, 0xbe, 0x9e, 0xd3, 0x24, + 0xa7, 0x07, 0x53, 0x22, 0x5c, 0xc8, 0x7e, 0xc5, 0x63, 0x64, 0x3f, 0x38, 0xf4, 0x18, 0xe4, 0x69, + 0xc6, 0xa6, 0x94, 0x6a, 0x87, 0x1a, 0xcc, 0x24, 0x6e, 0x11, 0xfc, 0xb0, 0xbe, 0x25, 0x9f, 0x68, + 0x50, 0x8c, 0x2e, 0xa9, 0xa0, 0x55, 0xc8, 0x61, 0x76, 0xd1, 0x81, 0xa5, 0x9d, 0xd9, 0xc4, 0xac, + 0x97, 0xe0, 0xf8, 0xb5, 0xb3, 0xc4, 0xdd, 0x86, 0x16, 0x67, 0x7c, 0x8c, 0x82, 0xf9, 0x97, 0x5a, + 0x58, 0x30, 0x8f, 0x59, 0x91, 0xfe, 0xee, 0x56, 0x9c, 0xdc, 0xd2, 0xfd, 0xba, 0x08, 0x59, 0x6a, + 0x0b, 0x69, 0x5e, 0x7d, 0xec, 0x0e, 0x4c, 0x4b, 0xef, 0xd3, 0x50, 0x2c, 0xb0, 0x5d, 0x1d, 0xc2, + 0xc4, 0x5d, 0x1d, 0xc2, 0xd0, 0x0e, 0xcc, 0xc4, 0x23, 0x31, 0x2a, 0x46, 0x7d, 0xeb, 0xed, 0x96, + 0x4c, 0xc4, 0x4e, 0x1b, 0x12, 0x9c, 0xf2, 0xb1, 0x75, 0x02, 0x89, 0x0c, 0x28, 0x77, 0x6d, 0xcb, + 0xd7, 0x4d, 0x0b, 0xbb, 0x4c, 0x51, 0x5a, 0x75, 0xeb, 0xe7, 0xaa, 0x44, 0xc3, 0x06, 0x15, 0x32, + 0x9f, 0x7c, 0xeb, 0x47, 0xc6, 0xa1, 0x0f, 0x60, 0x3a, 0x6c, 0x5c, 0x98, 0x92, 0x8c, 0xea, 0xd6, + 0xcf, 0x9a, 0x48, 0xc2, 0x36, 0x83, 0xc4, 0x25, 0xdf, 0xfa, 0x91, 0x50, 0xa8, 0x0f, 0x15, 0xc7, + 0x36, 0xee, 0x5b, 0xbc, 0x5c, 0xd7, 0x3b, 0x7d, 0xcc, 0xe7, 0xb0, 0x4b, 0x63, 0x05, 0x89, 0x44, + 0xc5, 0x12, 0x75, 0x92, 0x57, 0xbe, 0x47, 0x97, 0xc4, 0xa2, 0xf7, 0x61, 0xaa, 0x4f, 0xfa, 0xb7, + 0xb5, 0x7d, 0xc7, 0x74, 0xb1, 0xa1, 0xbe, 0xf5, 0x76, 0x5b, 0xa0, 0x60, 0x69, 0x52, 0xe4, 0x91, + 0x0f, 0xfb, 0x45, 0x0c, 0xf1, 0xfe, 0x40, 0xdf, 0x6f, 0x05, 0x96, 0xb7, 0xb6, 0xcf, 0x6f, 0x30, + 0xe5, 0x55, 0xde, 0xdf, 0x90, 0x89, 0x98, 0xf7, 0x13, 0x9c, 0xb2, 0xf7, 0x13, 0x48, 0x74, 0x9b, + 0x7e, 0x05, 0x98, 0x4b, 0xd8, 0xed, 0xb7, 0x85, 0xb1, 0xd5, 0x62, 0xde, 0x60, 0x03, 0x17, 0xfe, + 0x24, 0x09, 0x8d, 0x24, 0x70, 0x1f, 0xd0, 0xd7, 0x6e, 0x61, 0x3f, 0x70, 0x2d, 0x6c, 0xf0, 0x36, + 0x6a, 0xdc, 0x07, 0x12, 0x55, 0xe4, 0x03, 0x09, 0x3a, 0xe6, 0x03, 0x09, 0x8b, 0x3e, 0x86, 0xb9, + 0xc4, 0x5d, 0x1e, 0xf6, 0x1e, 0x25, 0xd5, 0x21, 0xc4, 0xba, 0x82, 0x92, 0x75, 0xbc, 0x2a, 0x19, + 0x92, 0x66, 0xa5, 0x16, 0xa2, 0xbd, 0xa7, 0x5b, 0xbd, 0x75, 0xbb, 0x23, 0xc7, 0xdc, 0x94, 0x4a, + 0xfb, 0x75, 0x05, 0x25, 0xd3, 0xae, 0x92, 0x21, 0x6b, 0x57, 0x51, 0x44, 0xf7, 0x76, 0x48, 0x11, + 0x13, 0xdd, 0x6f, 0x53, 0xdd, 0xdb, 0x61, 0x04, 0xc2, 0xbd, 0x1d, 0x06, 0x50, 0xdc, 0xdb, 0xe1, + 0x94, 0x85, 0x70, 0x58, 0x53, 0x7f, 0x17, 0x66, 0x12, 0xe9, 0x05, 0xbd, 0x05, 0xd1, 0x6d, 0x90, + 0x7b, 0x07, 0x4e, 0x58, 0xbb, 0x4a, 0xb7, 0x47, 0x08, 0x5c, 0x75, 0x7b, 0x84, 0xc0, 0xeb, 0x9f, + 0x67, 0xa0, 0x10, 0x46, 0xd4, 0x89, 0x74, 0x23, 0x2b, 0x90, 0x1f, 0x60, 0x8f, 0xde, 0xf8, 0x48, + 0xc5, 0x45, 0x0d, 0x07, 0x89, 0x45, 0x0d, 0x07, 0xc9, 0x35, 0x57, 0xfa, 0xb1, 0x6a, 0xae, 0xcc, + 0x91, 0x6b, 0x2e, 0x4c, 0x0f, 0x89, 0x85, 0xbc, 0x18, 0x1e, 0x6e, 0x3c, 0x3a, 0xd9, 0x86, 0x47, + 0xc8, 0x22, 0x63, 0xe2, 0x08, 0x59, 0x44, 0xa1, 0x5d, 0x38, 0x2d, 0x1c, 0xc0, 0xf0, 0xd1, 0x1b, + 0xc9, 0x50, 0xe5, 0xc9, 0x27, 0xf2, 0x2d, 0x4a, 0xc5, 0xf6, 0xe1, 0x6e, 0x02, 0x2a, 0x16, 0xad, + 0x49, 0x1c, 0x09, 0x09, 0x03, 0x77, 0x82, 0xde, 0x06, 0x5f, 0xf6, 0x7c, 0x1c, 0x12, 0x22, 0x5c, + 0x0c, 0x09, 0x11, 0x5e, 0xff, 0x4b, 0x0a, 0xca, 0xf2, 0xfb, 0x9e, 0x48, 0x60, 0xbc, 0x02, 0x45, + 0xbc, 0x6f, 0xfa, 0xed, 0xae, 0x6d, 0x60, 0xde, 0xb9, 0x51, 0x3f, 0x13, 0xe0, 0x55, 0xdb, 0x90, + 0xfc, 0x1c, 0xc2, 0xc4, 0x68, 0x4a, 0x1f, 0x29, 0x9a, 0xe2, 0x49, 0x67, 0xe6, 0x08, 0x93, 0x4e, + 0xa5, 0x9f, 0x8a, 0x27, 0xe3, 0xa7, 0xfa, 0x57, 0x29, 0xa8, 0x24, 0xd3, 0xee, 0xf7, 0x63, 0x0b, + 0xca, 0xbb, 0x29, 0x7d, 0xe4, 0xdd, 0xf4, 0x36, 0x4c, 0x93, 0xca, 0x4c, 0xf7, 0x7d, 0x7e, 0x41, + 0x34, 0x43, 0x8b, 0x2b, 0x96, 0x8d, 0x02, 0x6b, 0x35, 0x84, 0x4b, 0xd9, 0x48, 0x80, 0x8f, 0x85, + 0x6e, 0xf6, 0x98, 0xa1, 0xfb, 0x69, 0x0a, 0xa6, 0x37, 0x6d, 0xe3, 0x1e, 0x2b, 0xda, 0xfc, 0xef, + 0xcb, 0x7a, 0x3e, 0xcd, 0x94, 0x56, 0x9f, 0x81, 0x69, 0xa9, 0x6a, 0xab, 0x7f, 0xc6, 0xe2, 0x4c, + 0xfe, 0x5c, 0xfd, 0xe3, 0xad, 0x4b, 0x19, 0xa6, 0xc4, 0xf2, 0xaf, 0xde, 0x84, 0x99, 0x44, 0xb5, + 0x26, 0xbe, 0x80, 0x76, 0x94, 0x17, 0xa8, 0x5f, 0x83, 0x39, 0x55, 0x19, 0x23, 0x64, 0x1d, 0xed, + 0x08, 0xa7, 0x33, 0xd7, 0x61, 0x4e, 0x55, 0x8e, 0x1c, 0xdf, 0x9c, 0xb7, 0xf8, 0xc9, 0x27, 0x2b, + 0x1c, 0x8e, 0xcf, 0xff, 0xfb, 0xa8, 0x7b, 0x8e, 0x2f, 0x63, 0xbf, 0x03, 0x15, 0x27, 0x7c, 0x68, + 0xf3, 0x1e, 0x8d, 0x6d, 0x4b, 0xda, 0x71, 0x44, 0xb8, 0xf5, 0x44, 0xb3, 0x56, 0x96, 0x31, 0xb2, + 0x1c, 0xde, 0xbf, 0xe5, 0x14, 0x72, 0x5a, 0x89, 0x46, 0xae, 0x2c, 0x63, 0x84, 0xa5, 0xcd, 0x1f, + 0xbe, 0xb4, 0xb4, 0xff, 0xcb, 0x92, 0xa6, 0x79, 0x26, 0x71, 0x59, 0x1c, 0x5d, 0x86, 0x02, 0xfd, + 0x25, 0x57, 0xdc, 0xf9, 0xd2, 0xd5, 0xa1, 0x30, 0xc9, 0x80, 0x3c, 0x07, 0xa1, 0x57, 0xa1, 0x18, + 0xdd, 0x1f, 0xe7, 0x67, 0x9e, 0x2c, 0xee, 0x42, 0xa0, 0x14, 0x77, 0x21, 0x90, 0x37, 0xcd, 0xff, + 0x0d, 0x67, 0x27, 0xde, 0x1c, 0x3f, 0x4e, 0x0f, 0x2e, 0x74, 0xbf, 0x99, 0x63, 0x75, 0xbf, 0xfb, + 0xb0, 0xa0, 0xbe, 0xd0, 0x2d, 0x68, 0x4f, 0x1d, 0xaa, 0x3d, 0x5e, 0xfd, 0xf4, 0x11, 0x57, 0x3f, + 0x55, 0xdf, 0xa5, 0xe3, 0x82, 0xe8, 0xe2, 0x34, 0xba, 0x04, 0x59, 0xc7, 0xb6, 0xfb, 0x1e, 0xbf, + 0x54, 0x40, 0xd5, 0x51, 0x80, 0xa8, 0x8e, 0x02, 0x1e, 0x63, 0x38, 0x11, 0x84, 0x11, 0x1c, 0x5f, + 0x03, 0x7f, 0x0a, 0xab, 0xfb, 0xc2, 0x65, 0x28, 0x84, 0x07, 0xb7, 0x08, 0x20, 0xf7, 0xee, 0xfd, + 0xb5, 0xfb, 0x6b, 0xd7, 0x2a, 0xa7, 0x50, 0x09, 0xf2, 0x9b, 0x6b, 0x77, 0xae, 0xdd, 0xbc, 0x73, + 0xbd, 0xa2, 0x91, 0x87, 0xd6, 0xfd, 0x3b, 0x77, 0xc8, 0x43, 0xea, 0x85, 0xdb, 0xe2, 0x65, 0x30, + 0x5e, 0xb9, 0x4d, 0x41, 0x61, 0xd5, 0x71, 0x68, 0x0a, 0x61, 0xbc, 0x6b, 0x7b, 0x26, 0xd9, 0xc9, + 0x15, 0x0d, 0xe5, 0x21, 0x7d, 0xf7, 0xee, 0x46, 0x25, 0x85, 0xe6, 0xa0, 0x72, 0x0d, 0xeb, 0x46, + 0xdf, 0xb4, 0x70, 0x98, 0xb7, 0x2a, 0xe9, 0xe6, 0xc3, 0xdf, 0x7c, 0xb3, 0xa4, 0x7d, 0xf5, 0xcd, + 0x92, 0xf6, 0xe7, 0x6f, 0x96, 0xb4, 0xcf, 0xbf, 0x5d, 0x3a, 0xf5, 0xd5, 0xb7, 0x4b, 0xa7, 0xfe, + 0xf0, 0xed, 0xd2, 0xa9, 0xff, 0xbc, 0xdc, 0x33, 0xfd, 0x9d, 0xa0, 0xd3, 0xe8, 0xda, 0x03, 0xfe, + 0x93, 0x54, 0xc7, 0xb5, 0x49, 0x82, 0xe0, 0x4f, 0x2b, 0xc9, 0xdf, 0xaa, 0xfe, 0x2c, 0x75, 0x6e, + 0x95, 0x3e, 0x6e, 0x32, 0xba, 0xc6, 0x4d, 0xbb, 0xc1, 0x00, 0xf4, 0xd7, 0x89, 0x5e, 0x27, 0x47, + 0x7f, 0x85, 0xf8, 0xca, 0xdf, 0x02, 0x00, 0x00, 0xff, 0xff, 0xf9, 0x44, 0xa5, 0x08, 0xe6, 0x3a, + 0x00, 0x00, } func (m *EventSequence) Marshal() (dAtA []byte, err error) { @@ -4576,32 +4566,6 @@ func (m *ResourceUtilisation) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.AvgResourcesForPeriod) > 0 { - for k := range m.AvgResourcesForPeriod { - v := m.AvgResourcesForPeriod[k] - baseI := i - if v != nil { - { - size, err := v.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintEvents(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - i -= len(k) - copy(dAtA[i:], k) - i = encodeVarintEvents(dAtA, i, uint64(len(k))) - i-- - dAtA[i] = 0xa - i = encodeVarintEvents(dAtA, i, uint64(baseI-i)) - i-- - dAtA[i] = 0x42 - } - } if len(m.RunId) > 0 { i -= len(m.RunId) copy(dAtA[i:], m.RunId) @@ -5385,20 +5349,20 @@ func (m *JobSetFilter) MarshalToSizedBuffer(dAtA []byte) (int, error) { var l int _ = l if len(m.States) > 0 { - dAtA42 := make([]byte, len(m.States)*10) - var j41 int + dAtA41 := make([]byte, len(m.States)*10) + var j40 int for _, num := range m.States { for num >= 1<<7 { - dAtA42[j41] = uint8(uint64(num)&0x7f | 0x80) + dAtA41[j40] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j41++ + j40++ } - dAtA42[j41] = uint8(num) - j41++ + dAtA41[j40] = uint8(num) + j40++ } - i -= j41 - copy(dAtA[i:], dAtA42[:j41]) - i = encodeVarintEvents(dAtA, i, uint64(j41)) + i -= j40 + copy(dAtA[i:], dAtA41[:j40]) + i = encodeVarintEvents(dAtA, i, uint64(j40)) i-- dAtA[i] = 0xa } @@ -5433,20 +5397,20 @@ func (m *CancelJobSet) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x12 } if len(m.States) > 0 { - dAtA44 := make([]byte, len(m.States)*10) - var j43 int + dAtA43 := make([]byte, len(m.States)*10) + var j42 int for _, num := range m.States { for num >= 1<<7 { - dAtA44[j43] = uint8(uint64(num)&0x7f | 0x80) + dAtA43[j42] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j43++ + j42++ } - dAtA44[j43] = uint8(num) - j43++ + dAtA43[j42] = uint8(num) + j42++ } - i -= j43 - copy(dAtA[i:], dAtA44[:j43]) - i = encodeVarintEvents(dAtA, i, uint64(j43)) + i -= j42 + copy(dAtA[i:], dAtA43[:j42]) + i = encodeVarintEvents(dAtA, i, uint64(j42)) i-- dAtA[i] = 0xa } @@ -7521,19 +7485,6 @@ func (m *ResourceUtilisation) Size() (n int) { if l > 0 { n += 1 + l + sovEvents(uint64(l)) } - if len(m.AvgResourcesForPeriod) > 0 { - for k, v := range m.AvgResourcesForPeriod { - _ = k - _ = v - l = 0 - if v != nil { - l = v.Size() - l += 1 + sovEvents(uint64(l)) - } - mapEntrySize := 1 + len(k) + sovEvents(uint64(len(k))) + l - n += mapEntrySize + 1 + sovEvents(uint64(mapEntrySize)) - } - } return n } @@ -10157,135 +10108,6 @@ func (m *ResourceUtilisation) Unmarshal(dAtA []byte) error { } m.RunId = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 8: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field AvgResourcesForPeriod", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowEvents - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthEvents - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthEvents - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.AvgResourcesForPeriod == nil { - m.AvgResourcesForPeriod = make(map[string]*resource.Quantity) - } - var mapkey string - var mapvalue *resource.Quantity - for iNdEx < postIndex { - entryPreIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowEvents - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - if fieldNum == 1 { - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowEvents - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthEvents - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey < 0 { - return ErrInvalidLengthEvents - } - if postStringIndexmapkey > l { - return io.ErrUnexpectedEOF - } - mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey - } else if fieldNum == 2 { - var mapmsglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowEvents - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - mapmsglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if mapmsglen < 0 { - return ErrInvalidLengthEvents - } - postmsgIndex := iNdEx + mapmsglen - if postmsgIndex < 0 { - return ErrInvalidLengthEvents - } - if postmsgIndex > l { - return io.ErrUnexpectedEOF - } - mapvalue = &resource.Quantity{} - if err := mapvalue.Unmarshal(dAtA[iNdEx:postmsgIndex]); err != nil { - return err - } - iNdEx = postmsgIndex - } else { - iNdEx = entryPreIndex - skippy, err := skipEvents(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthEvents - } - if (iNdEx + skippy) > postIndex { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - m.AvgResourcesForPeriod[mapkey] = mapvalue - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipEvents(dAtA[iNdEx:]) diff --git a/pkg/armadaevents/events.proto b/pkg/armadaevents/events.proto index b8e9eb50101..8905344026b 100644 --- a/pkg/armadaevents/events.proto +++ b/pkg/armadaevents/events.proto @@ -115,7 +115,6 @@ message ResourceUtilisation { map total_cumulative_usage = 5; string job_id = 6; string run_id = 7; - map avg_resources_for_period = 8; } From 6e528255ef4546ae8681d76b0ea81ef53d58a9d7 Mon Sep 17 00:00:00 2001 From: Maurice Yap Date: Thu, 6 Feb 2025 13:22:28 +0000 Subject: [PATCH 23/23] Lookout UI bugfix (shift-click select rows) (#4194) In the Lookout UI's jobs table, selecting a row, then shift-clicking another row should select or de-select that entire block of rows which are at the same depth, where grouping is applied. Due to a race condition, this doesn't work. This commit fixes that. --- .../lookoutV2/JobsTableContainer.tsx | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.tsx b/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.tsx index aa9c0beb0c1..ce01ce49aa0 100644 --- a/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.tsx +++ b/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.tsx @@ -724,19 +724,26 @@ export const JobsTableContainer = ({ setSelectedRows({}) } - const shiftSelectRow = async (row: Row) => { + const shiftSelectRow = (row: Row) => { if (lastSelectedRow === undefined || row.depth !== lastSelectedRow.depth) { return } - const sameDepthRows = table.getRowModel().rows.filter((_row) => row.depth === lastSelectedRow.depth) - const lastSelectedIdx = sameDepthRows.indexOf(lastSelectedRow) - const currentIdx = sameDepthRows.indexOf(row) - const shouldSelect = lastSelectedRow.getIsSelected() - // Race condition - if we don't wait here the rows do not get selected - await waitMillis(1) - for (let i = Math.min(lastSelectedIdx, currentIdx); i <= Math.max(lastSelectedIdx, currentIdx); i++) { - sameDepthRows[i].toggleSelected(shouldSelect) - } + + const rowIndex = table.getRowModel().rows.indexOf(row) + const lastSelectedRowIndex = table.getRowModel().rows.indexOf(lastSelectedRow) + + const selectedValue = lastSelectedRow.getIsSelected() + const rowSelectionMask: RowSelectionState = table + .getRowModel() + .rows.slice(Math.min(rowIndex, lastSelectedRowIndex), Math.max(rowIndex, lastSelectedRowIndex) + 1) + .reduce((acc, { id, depth }) => { + if (depth === row.depth) { + acc[id] = selectedValue + } + return acc + }, {}) + + table.setRowSelection((prev) => ({ ...prev, ...rowSelectionMask })) } const selectRow = async (row: Row, singleSelect: boolean) => {