Build Filters for Telco

backend/app/api/v1/commons/constants.py

@@ -1,5 +1,5 @@
 # Define the keywords for sorting.
-# numeric values take no .keyword
+# numeric fields are without .keyword
 DIRECTIONS = ("asc", "desc")
 FIELDS = (
     "ciSystem.keyword",
@@ -64,3 +64,35 @@
     "ci": "ci",
     "ec": "Engineering Candidate",
 }
+
+TELCO_FIELDS_DICT = {
+    "cpu": "CPU",
+    "benchmark": "Benchmark",
+    "releaseStream": "Release Stream",
+    "nodeName": "Node Name",
+    "startDate": "Start Date",
+    "endDate": "End Date",
+    "jobStatus": "Status",
+    "ocpVersion": "Build",
+}
+
+FILEDS_DISPLAY_NAMES = {
+    "ciSystem": "CI System",
+    "platform": "Platform",
+    "benchmark": "Benchmark",
+    "releaseStream": "Release Stream",
+    "networkType": "Network Type",
+    "workerNodesCount": "Worker Count",
+    "jobStatus": "Status",
+    "controlPlaneArch": "Control Plane Architecture",
+    "publish": "Control Plane Access",
+    "fips": "FIPS Enabled",
+    "encrypted": "Is Encrypted",
+    "ipsec": "Has IPSEC",
+    "ocpVersion": "Versions",
+    "build": "Build",
+    "computeArch": "Compute Architecture",
+    "jobStatus": "Status",
+    "startDate": "Start Date",
+    "endDate": "End Date",
+}

backend/app/api/v1/commons/ocp.py

@@ -2,7 +2,6 @@
 import pandas as pd
 import app.api.v1.commons.utils as utils
 from app.services.search import ElasticService
-from app.api.v1.commons.constants import OCP_FIELD_CONSTANT_DICT
 
 
 async def getData(
@@ -80,7 +79,7 @@ def fillEncryptionType(row):
 async def getFilterData(start_datetime: date, end_datetime: date, configpath: str):
     es = ElasticService(configpath=configpath)
 
-    aggregate = utils.buildAggregateQuery(OCP_FIELD_CONSTANT_DICT)
+    aggregate = utils.buildAggregateQuery("OCP_FIELD_CONSTANT_DICT")
 
     response = await es.filterPost(start_datetime, end_datetime, aggregate)
     await es.close()
@@ -90,8 +89,12 @@ async def getFilterData(start_datetime: date, end_datetime: date, configpath: st
     jobType = getJobType(upstreamList)
     isRehearse = getIsRehearse(upstreamList)
 
-    jobTypeObj = {"key": "jobType", "value": jobType}
-    isRehearseObj = {"key": "isRehearse", "value": isRehearse}
+    jobTypeObj = {
+        "key": "jobType",
+        "value": jobType,
+        "name": "Job Type",
+    }
+    isRehearseObj = {"key": "isRehearse", "value": isRehearse, "name": "Rehearse"}
 
     response["filterData"].append(jobTypeObj)
     response["filterData"].append(isRehearseObj)

backend/app/api/v1/commons/quay.py

@@ -2,7 +2,6 @@
 import pandas as pd
 import app.api.v1.commons.utils as utils
 from app.services.search import ElasticService
-from app.api.v1.commons.constants import QUAY_FIELD_CONSTANT_DICT
 
 
 async def getData(
@@ -55,7 +54,7 @@ async def getFilterData(start_datetime: date, end_datetime: date, configpath: st
 
     es = ElasticService(configpath=configpath)
 
-    aggregate = utils.buildAggregateQuery(QUAY_FIELD_CONSTANT_DICT)
+    aggregate = utils.buildAggregateQuery("QUAY_FIELD_CONSTANT_DICT")
 
     response = await es.filterPost(start_datetime, end_datetime, aggregate)
     await es.close()

backend/app/api/v1/commons/telco.py

@@ -6,6 +6,7 @@
 from datetime import datetime, timezone
 import app.api.v1.commons.utils as utils
 import app.api.v1.endpoints.telco.telcoGraphs as telcoGraphs
+import app.api.v1.commons.constants as constants
 
 
 async def getData(
@@ -47,7 +48,6 @@ async def getData(
         query=query, size=size, offset=offset, searchList=searchList
     )
     mapped_list = []
-
     for each_response in response["data"]:
         end_timestamp = int(each_response["timestamp"])
         test_data = each_response["data"]
@@ -89,3 +89,69 @@ async def getData(
     jobs = pd.json_normalize(mapped_list)
 
     return {"data": jobs, "total": response["total"]}
+
+
+async def getFilterData(start_datetime: date, end_datetime: date, configpath: str):
+    test_types = [
+        "oslat",
+        "cyclictest",
+        "cpu_util",
+        "deployment",
+        "ptp",
+        "reboot",
+        "rfc-2544",
+    ]
+    cfg = config.get_config()
+    try:
+        jenkins_url = cfg.get("telco.config.job_url")
+    except Exception as e:
+        print(f"Error reading telco configuration: {e}")
+
+    query = {
+        "earliest_time": "{}T00:00:00".format(start_datetime.strftime("%Y-%m-%d")),
+        "latest_time": "{}T23:59:59".format(end_datetime.strftime("%Y-%m-%d")),
+        "output_mode": "json",
+    }
+    searchList = " OR ".join(
+        ['test_type="{}"'.format(test_type) for test_type in test_types]
+    )
+    splunk = SplunkService(configpath=configpath)
+    response = await splunk.filterPost(query=query, searchList=searchList)
+    filterData = []
+    print(response["data"])
+    if len(response["data"]) > 0:
+        for item in response["data"]:
+            for field, value in item.items():
+                if field == "total_records":
+                    continue
+
+                # Determine the appropriate value transformation
+                if isinstance(value, str):
+                    v = [value] if value else []
+                elif not isinstance(value, list):
+                    v = [value]
+                else:
+                    v = value
+
+                # Build the dictionary for the current field
+                transformed_value = (
+                    utils.buildReleaseStreamFilter(value)
+                    if field == "releaseStream"
+                    else v
+                )
+
+                currDict = {
+                    "key": field,
+                    "value": transformed_value,
+                    "name": constants.TELCO_FIELDS_DICT.get(field, "Unknown Field"),
+                }
+
+                # Append the dictionary to filterData
+                filterData.append(currDict)
+
+    # can be removed once python scripts to determine success or failure are executed directly
+    # in the splunk dashboard
+    filterData.append(
+        {"key": "jobStatus", "value": ["success", "failure"], "name": "Status"}
+    )
+    return {"data": filterData, "total": response["total"]}

backend/app/api/v1/commons/utils.py

@@ -119,26 +119,6 @@ def buildAggregateQuery(constant_dict):
     return aggregate
 
 
-def removeKeys(filterDict, keys_to_remove):
-    for key in keys_to_remove:
-        if key in filterDict:
-            del filterDict[key]
-    return filterDict
-
-
-def buildPlatformFilter(upstreamList, clusterypeList):
-    filterOptions = []
-    upstreamCheck = any("rosa-hcp" in s.lower() for s in upstreamList)
-    clusterTypeCheck = any("rosa" in s.lower() for s in clusterypeList)
-
-    if upstreamCheck:
-        filterOptions.append("AWS ROSA-HCP")
-    if clusterTypeCheck:
-        filterOptions.append("AWS ROSA")
-
-    return filterOptions
-
-
 def buildReleaseStreamFilter(input_array):
     mapped_array = []
     for item in input_array:
@@ -152,9 +132,4 @@ def buildReleaseStreamFilter(input_array):
             "Stable",
         )
         mapped_array.append(match)
-    return mapped_array
-
-
-def getBuildFilter(input_list):
-    result = ["-".join(item.split("-")[-4:]) for item in input_list]
-    return result
+    return list(set(mapped_array))

backend/app/api/v1/endpoints/cpt/cptJobs.py

@@ -11,7 +11,7 @@
 from .maps.hce import hceMapper
 from .maps.telco import telcoMapper
 from .maps.ocm import ocmMapper
-from ...commons.example_responses import cpt_200_response, response_422
+from app.api.v1.commons.example_responses import cpt_200_response, response_422
 from fastapi.param_functions import Query
 from app.api.v1.commons.utils import normalize_pagination
 
@@ -30,7 +30,7 @@
     "/api/v1/cpt/jobs",
     summary="Returns a job list from all the products.",
     description="Returns a list of jobs in the specified dates of requested size \
-            If not dates are provided the API will default the values. \
+            If dates are not provided the API will default the values: \
             `startDate`: will be set to the day of the request minus 5 days.\
             `endDate`: will be set to the day of the request.",
     responses={

backend/app/api/v1/endpoints/cpt/maps/hce.py

@@ -1,4 +1,4 @@
-from ....commons.hce import getData
+from .app.api.v1.commons.hce import getData
 from datetime import date
 import pandas as pd
 

backend/app/api/v1/endpoints/cpt/maps/ocm.py

@@ -1,4 +1,4 @@
-from ....commons.ocm import getData
+from .app.api.v1.commons.ocm import getData
 from datetime import date
 import pandas as pd
 

backend/app/api/v1/endpoints/cpt/maps/ocp.py

@@ -1,5 +1,5 @@
-from ....commons.ocp import getData
-from ....commons.utils import getReleaseStream
+from .app.api.v1.commons.ocp import getData
+from .app.api.v1.commons.utils import getReleaseStream
 from datetime import date
 import pandas as pd
 

backend/app/api/v1/endpoints/cpt/maps/quay.py

@@ -1,4 +1,4 @@
-from ....commons.quay import getData
+from .app.api.v1.commons.quay import getData
 from datetime import date
 import pandas as pd
 

backend/app/api/v1/endpoints/cpt/maps/telco.py

@@ -1,5 +1,5 @@
-from ....commons.telco import getData
-from ....commons.utils import getReleaseStream
+from .app.api.v1.commons.telco import getData
+from .app.api.v1.commons.utils import getReleaseStream
 from datetime import date
 import pandas as pd
 

backend/app/api/v1/endpoints/ocm/ocmJobs.py

@@ -2,8 +2,8 @@
 from fastapi import Response
 from datetime import datetime, timedelta, date
 from fastapi import APIRouter, HTTPException
-from ...commons.ocm import getData
-from ...commons.example_responses import ocp_200_response, response_422
+from app.api.v1.commons.ocm import getData
+from app.api.v1.commons.example_responses import ocp_200_response, response_422
 from fastapi.param_functions import Query
 
 router = APIRouter()

backend/app/api/v1/endpoints/ocp/ocpJobs.py

@@ -18,7 +18,7 @@
     "/api/v1/ocp/jobs",
     summary="Returns a job list",
     description="Returns a list of jobs in the specified dates. \
-            If not dates are provided the API will default the values. \
+            If dates are not provided the API will use the following values as defaults: \
             `startDate`: will be set to the day of the request minus 5 days.\
             `endDate`: will be set to the day of the request.",
     responses={

backend/app/api/v1/endpoints/quay/quayJobs.py

@@ -18,7 +18,7 @@
     "/api/v1/quay/jobs",
     summary="Returns a job list",
     description="Returns a list of jobs in the specified dates of requested size. \
-            If not dates are provided the API will default the values. \
+            If dates are not provided the API will use the following values as defaults: \
             `startDate`: will be set to the day of the request minus 5 days.\
             `endDate`: will be set to the day of the request.",
     responses={
@@ -87,7 +87,7 @@ async def jobs(
     "/api/v1/quay/filters",
     summary="Returns the data to construct filters",
     description="Returns the data to build filters in the specified dates. \
-            If not dates are provided the API will default the values. \
+            If not dates are provided the API will use the following values as defaults. \
             `startDate`: will be set to the day of the request minus 5 days.\
             `endDate`: will be set to the day of the request.",
     responses={

backend/app/api/v1/endpoints/telco/telcoJobs.py

@@ -2,8 +2,8 @@
 from fastapi import Response
 from datetime import datetime, timedelta, date
 from fastapi import APIRouter, HTTPException
-from ...commons.telco import getData
-from ...commons.example_responses import telco_200_response, response_422
+from app.api.v1.commons.telco import getData, getFilterData
+from app.api.v1.commons.example_responses import telco_200_response, response_422
 from fastapi.param_functions import Query
 from app.api.v1.commons.utils import normalize_pagination
 
@@ -14,7 +14,7 @@
     "/api/v1/telco/jobs",
     summary="Returns a job list",
     description="Returns a list of jobs in the specified dates of requested size. \
-            If not dates are provided the API will default the values. \
+            If dates are not provided the API will use the following values as defaults: \
             `startDate`: will be set to the day of the request minus 5 days.\
             `endDate`: will be set to the day of the request.",
     responses={
@@ -55,6 +55,9 @@ async def jobs(
 
     results = await getData(start_date, end_date, size, offset, "telco.splunk")
 
+    jobs = []
+    if "data" in results and len(results["data"]) >= 1:
+        jobs = results["data"].to_dict("records")
     response = {
         "startDate": start_date.__str__(),
         "endDate": end_date.__str__(),
@@ -68,4 +71,58 @@ async def jobs(
         return Response(content=json_str, media_type="application/json")
 
     jsonstring = json.dumps(response)
-    return jsonstring
+    return response
+
+
+@router.get(
+    "/api/v1/telco/filters",
+    summary="Returns data to build the filter",
+    description="Returns a list of jobs in the specified dates of requested size. \
+            If not dates are provided the API will use the following values as defaults. \
+            `startDate`: will be set to the day of the request minus 5 days.\
+            `endDate`: will be set to the day of the request.",
+    responses={
+        200: telco_200_response(),
+        422: response_422(),
+    },
+)
+async def filters(
+    start_date: date = Query(
+        None,
+        description="Start date for searching jobs, format: 'YYYY-MM-DD'",
+        examples=["2020-11-10"],
+    ),
+    end_date: date = Query(
+        None,
+        description="End date for searching jobs, format: 'YYYY-MM-DD'",
+        examples=["2020-11-15"],
+    ),
+    pretty: bool = Query(False, description="Output content in pretty format."),
+    size: int = Query(None, description="Number of jobs to fetch"),
+    offset: int = Query(None, description="Offset Number to fetch jobs from"),
+):
+    if start_date is None:
+        start_date = datetime.utcnow().date()
+        start_date = start_date - timedelta(days=5)
+
+    if end_date is None:
+        end_date = datetime.utcnow().date()
+
+    if start_date > end_date:
+        return Response(
+            content=json.dumps(
+                {"error": "invalid date format, start_date must be less than end_date"}
+            ),
+            status_code=422,
+        )
+
+    results = await getFilterData(start_date, end_date, "telco.splunk")
+
+    response = {"filterData": results["data"], "summary": {"total": results["total"]}}
+
+    if pretty:
+        json_str = json.dumps(response, indent=4)
+        return Response(content=json_str, media_type="application/json")
+
+    jsonstring = json.dumps(response)
+    return response