From 545f6fabb8433859bccdda2e75caccbcbbfc965b Mon Sep 17 00:00:00 2001 From: Brandon <132288221+brandon-groundlight@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:49:50 -0800 Subject: [PATCH] temporary fix for how iqs can be returned (#298) Co-authored-by: Auto-format Bot --- src/groundlight/internalapi.py | 4 ++-- test/unit/conftest.py | 20 +++++++++++++++++++- test/unit/test_internalapi.py | 9 +++++++-- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/groundlight/internalapi.py b/src/groundlight/internalapi.py index bff192cd..10e1791d 100644 --- a/src/groundlight/internalapi.py +++ b/src/groundlight/internalapi.py @@ -65,7 +65,7 @@ def iq_is_confident(iq: ImageQuery, confidence_threshold: float) -> bool: The only subtlety here is that currently confidence of None means human label, which is treated as confident. """ - if not iq.result: + if not iq.result or not iq.result.confidence: return False return iq.result.confidence >= confidence_threshold # type: ignore @@ -74,7 +74,7 @@ def iq_is_answered(iq: ImageQuery) -> bool: """Returns True if the image query has a ML or human label. Placeholder and special labels (out of domain) have confidences exactly 0.5 """ - if not iq.result: + if not iq.result or not iq.result.source: return False if (iq.result.source == Source.STILL_PROCESSING) or (iq.result.source is None): # Should never be None return False diff --git a/test/unit/conftest.py b/test/unit/conftest.py index 472d0c85..96ed1aa7 100644 --- a/test/unit/conftest.py +++ b/test/unit/conftest.py @@ -2,7 +2,7 @@ import pytest from groundlight import ExperimentalApi, Groundlight -from model import Detector, ImageQuery +from model import Detector, ImageQuery, ImageQueryTypeEnum, ResultTypeEnum def pytest_configure(config): @@ -46,3 +46,21 @@ def fixture_image_query_no(gl: Groundlight, detector: Detector) -> ImageQuery: @pytest.fixture(name="gl_experimental") def _gl() -> ExperimentalApi: return ExperimentalApi() + + +@pytest.fixture(name="initial_iq") +def fixture_initial_iq() -> ImageQuery: + return ImageQuery( + metadata=None, + id="iq_fakeidhere", + type=ImageQueryTypeEnum.image_query, + created_at=datetime.utcnow(), + query="Is there a dog?", + detector_id="det_fakeidhere", + result_type=ResultTypeEnum.binary_classification, + result=None, + patience_time=30, + confidence_threshold=0.9, + rois=None, + text=None, + ) diff --git a/test/unit/test_internalapi.py b/test/unit/test_internalapi.py index e6f6fbca..5f3422de 100644 --- a/test/unit/test_internalapi.py +++ b/test/unit/test_internalapi.py @@ -1,14 +1,19 @@ from groundlight import ExperimentalApi from groundlight.internalapi import iq_is_answered, iq_is_confident +from model import ImageQuery -def test_iq_is_confident(gl_experimental: ExperimentalApi): +def test_iq_is_confident(gl_experimental: ExperimentalApi, initial_iq: ImageQuery): det = gl_experimental.get_or_create_detector("Test", "test_query") iq = gl_experimental.ask_async(det, image="test/assets/dog.jpeg") assert not iq_is_confident(iq, 0.9) + assert not iq_is_confident(initial_iq, 0.9) -def test_iq_is_answered(gl_experimental: ExperimentalApi): + +def test_iq_is_answered(gl_experimental: ExperimentalApi, initial_iq: ImageQuery): det = gl_experimental.get_or_create_detector("Test", "test_query") iq = gl_experimental.ask_async(det, image="test/assets/dog.jpeg") assert not iq_is_answered(iq) + + assert not iq_is_answered(initial_iq)