Skip to content

Commit

Permalink
PENG-2457 improve typing by adding a TypeAlias for the job metric dat…
Browse files Browse the repository at this point in the history
…a structure expected by the API
  • Loading branch information
matheushent committed Dec 12, 2024
1 parent 36108aa commit cf0955b
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 10 deletions.
12 changes: 11 additions & 1 deletion jobbergate-agent/jobbergate_agent/jobbergate/schemas.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from pathlib import Path
from typing import List, Optional, TypedDict
from typing import List, Optional, TypedDict, TypeAlias

import pydantic
from pydantic import ConfigDict
Expand Down Expand Up @@ -120,3 +120,13 @@ class JobSubmissionMetricsMaxResponse(pydantic.BaseModel):

job_submission_id: int
max_times: list[JobSubmissionMetricsMaxTime]


"""
Type alias for job metric structure. It matches the following sequence of data
(time, host, step, task, CPUFrequency, CPUTime, CPUUtilization, GPUMemMB,
GPUUtilization, Pages, RSS, VMSize, ReadMB, WriteMB)
"""
JobMetricData: TypeAlias = list[
tuple[int, str, str, str, float, float, float, float, float, float, float, float, float, float]
]
7 changes: 3 additions & 4 deletions jobbergate-agent/jobbergate_agent/utils/compute.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,8 @@
from loguru import logger
from numba import njit

from jobbergate_agent.jobbergate.schemas import InfluxDBMeasure

from jobbergate_agent.jobbergate.constants import INFLUXDB_MEASUREMENT
from jobbergate_agent.jobbergate.schemas import InfluxDBMeasure, JobMetricData


def measure_memory_usage(func: Callable) -> Callable:
Expand Down Expand Up @@ -64,7 +63,7 @@ def _aggregate_with_numba(
@measure_memory_usage
def aggregate_influx_measures(
data_points: Iterator[InfluxDBMeasure],
) -> list[tuple[int, str, str, str, float, float, float, float, float, float, float, float, float, float]]:
) -> JobMetricData:
"""Aggregate the list of data points by time, host, step and task.
The output data is a list of tuples with the following format:
Expand Down Expand Up @@ -112,7 +111,7 @@ def aggregate_influx_measures(
reverse_task_mapping = {v: k for k, v in task_mapping.items()}

return cast(
list[tuple[int, str, str, str, float, float, float, float, float, float, float, float, float, float]],
JobMetricData,
[
(
int(unique_key[0]), # time
Expand Down
10 changes: 5 additions & 5 deletions jobbergate-agent/tests/jobbergate/test_compute.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import numpy as np

from jobbergate_agent.jobbergate.constants import INFLUXDB_MEASUREMENT
from jobbergate_agent.jobbergate.schemas import InfluxDBMeasure
from jobbergate_agent.jobbergate.schemas import InfluxDBMeasure, JobMetricData
from jobbergate_agent.utils.compute import (
aggregate_influx_measures,
measure_memory_usage,
Expand All @@ -26,7 +26,7 @@ def generate_and_aggregate_job_metrics_data() -> (
[int, int, int, int, int],
tuple[
list[InfluxDBMeasure],
list[tuple[int, str, str, str, float, float, float, float, float, float, float, float, float, float]],
JobMetricData,
],
]
):
Expand All @@ -40,7 +40,7 @@ def _generate_and_aggregate(
num_points_per_measurement: int, num_hosts: int, num_jobs: int, num_steps: int, num_tasks: int
) -> tuple[
list[InfluxDBMeasure],
list[tuple[int, str, str, str, float, float, float, float, float, float, float, float, float, float]],
JobMetricData,
]:
# Initialize data structures
current_time = int(datetime.now().timestamp())
Expand Down Expand Up @@ -82,7 +82,7 @@ def _generate_and_aggregate(

# Create aggregated list
aggregated_list = cast(
list[tuple[int, str, str, str, float, float, float, float, float, float, float, float, float, float]],
JobMetricData,
[
(
time,
Expand Down Expand Up @@ -119,7 +119,7 @@ async def test_aggregate_influx_measures__success(
[int, int, int, int, int],
tuple[
list[InfluxDBMeasure],
list[tuple[int, str, str, str, float, float, float, float, float, float, float, float, float, float]],
JobMetricData,
],
],
):
Expand Down

0 comments on commit cf0955b

Please sign in to comment.