From b79c307b5d77a11242ded7990d04f8fb44356262 Mon Sep 17 00:00:00 2001 From: Brandon <132288221+brandon-groundlight@users.noreply.github.com> Date: Tue, 7 Nov 2023 16:11:57 -0800 Subject: [PATCH] Stop submitting tests to integ (#130) * temporarily disable tests * Automatically reformatting code * comment out whole test * use pytest skip rather than comment * Open the gates! Turn the tests back on! * fix ask method quality * overwrite merge overwriting fix * fix other line * Stop most SDK tests from requiring hitl input when it is not necessary * managed to be off by one in both directions --------- Co-authored-by: Auto-format Bot --- test/integration/test_groundlight.py | 52 +++++++++++++++++----------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/test/integration/test_groundlight.py b/test/integration/test_groundlight.py index b603c3f6..06acd4bd 100644 --- a/test/integration/test_groundlight.py +++ b/test/integration/test_groundlight.py @@ -52,13 +52,13 @@ def fixture_detector(gl: Groundlight) -> Detector: @pytest.fixture(name="image_query_yes") def fixture_image_query_yes(gl: Groundlight, detector: Detector) -> ImageQuery: - iq = gl.submit_image_query(detector=detector.id, image="test/assets/dog.jpeg") + iq = gl.submit_image_query(detector=detector.id, image="test/assets/dog.jpeg", human_review="NEVER") return iq @pytest.fixture(name="image_query_no") def fixture_image_query_no(gl: Groundlight, detector: Detector) -> ImageQuery: - iq = gl.submit_image_query(detector=detector.id, image="test/assets/cat.jpeg") + iq = gl.submit_image_query(detector=detector.id, image="test/assets/cat.jpeg", human_review="NEVER") return iq @@ -184,23 +184,31 @@ def validate_image_query(_image_query: ImageQuery): assert isinstance(_image_query, ImageQuery) assert is_valid_display_result(_image_query.result) - _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/dog.jpeg", wait=10) + _image_query = gl.submit_image_query( + detector=detector.id, image="test/assets/dog.jpeg", wait=10, human_review="NEVER" + ) validate_image_query(_image_query) - _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/dog.jpeg", wait=3) + _image_query = gl.submit_image_query( + detector=detector.id, image="test/assets/dog.jpeg", wait=3, human_review="NEVER" + ) validate_image_query(_image_query) - _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/dog.jpeg", wait=10, patience_time=20) + _image_query = gl.submit_image_query( + detector=detector.id, image="test/assets/dog.jpeg", wait=10, patience_time=20, human_review="NEVER" + ) validate_image_query(_image_query) _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/dog.jpeg", human_review="NEVER") validate_image_query(_image_query) _image_query = gl.submit_image_query( - detector=detector.id, image="test/assets/dog.jpeg", wait=180, confidence_threshold=0.75 + detector=detector.id, image="test/assets/dog.jpeg", wait=180, confidence_threshold=0.75, human_review="NEVER" ) validate_image_query(_image_query) assert _image_query.result.confidence >= IQ_IMPROVEMENT_THRESHOLD def test_submit_image_query_blocking(gl: Groundlight, detector: Detector): - _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/dog.jpeg", wait=10) + _image_query = gl.submit_image_query( + detector=detector.id, image="test/assets/dog.jpeg", wait=10, human_review="NEVER" + ) assert str(_image_query) assert isinstance(_image_query, ImageQuery) assert is_valid_display_result(_image_query.result) @@ -209,19 +217,19 @@ def test_submit_image_query_blocking(gl: Groundlight, detector: Detector): def test_submit_image_query_returns_yes(gl: Groundlight): # We use the "never-review" pipeline to guarantee a confident "yes" answer. detector = gl.get_or_create_detector(name="Always a dog", query="Is there a dog?", pipeline_config="never-review") - image_query = gl.submit_image_query(detector=detector, image="test/assets/dog.jpeg", wait=10) + image_query = gl.submit_image_query(detector=detector, image="test/assets/dog.jpeg", wait=10, human_review="NEVER") assert image_query.result.label == Label.YES def test_submit_image_query_filename(gl: Groundlight, detector: Detector): - _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/dog.jpeg") + _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/dog.jpeg", human_review="NEVER") assert str(_image_query) assert isinstance(_image_query, ImageQuery) assert is_valid_display_result(_image_query.result) def test_submit_image_query_png(gl: Groundlight, detector: Detector): - _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/cat.png") + _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/cat.png", human_review="NEVER") assert str(_image_query) assert isinstance(_image_query, ImageQuery) assert is_valid_display_result(_image_query.result) @@ -240,7 +248,7 @@ def test_submit_image_query_with_human_review_param(gl: Groundlight, detector: D def test_submit_image_query_jpeg_bytes(gl: Groundlight, detector: Detector): jpeg = open("test/assets/dog.jpeg", "rb").read() - _image_query = gl.submit_image_query(detector=detector.id, image=jpeg) + _image_query = gl.submit_image_query(detector=detector.id, image=jpeg, human_review="NEVER") assert str(_image_query) assert isinstance(_image_query, ImageQuery) assert is_valid_display_result(_image_query.result) @@ -252,19 +260,21 @@ def test_submit_image_query_jpeg_truncated(gl: Groundlight, detector: Detector): # This is an extra difficult test because the header is valid. # So a casual check of the image will appear valid. with pytest.raises(openapi_client.exceptions.ApiException) as exc_info: - _image_query = gl.submit_image_query(detector=detector.id, image=jpeg_truncated) + _image_query = gl.submit_image_query(detector=detector.id, image=jpeg_truncated, human_review="NEVER") exc_value = exc_info.value assert is_user_error(exc_value.status) def test_submit_image_query_bad_filename(gl: Groundlight, detector: Detector): with pytest.raises(FileNotFoundError): - _image_query = gl.submit_image_query(detector=detector.id, image="missing-file.jpeg") + _image_query = gl.submit_image_query(detector=detector.id, image="missing-file.jpeg", human_review="NEVER") def test_submit_image_query_bad_jpeg_file(gl: Groundlight, detector: Detector): with pytest.raises(ValueError) as exc_info: - _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/blankfile.jpeg") + _image_query = gl.submit_image_query( + detector=detector.id, image="test/assets/blankfile.jpeg", human_review="NEVER" + ) assert "jpeg" in str(exc_info).lower() @@ -274,10 +284,10 @@ def test_submit_image_query_pil(gl: Groundlight, detector: Detector): from PIL import Image dog = Image.open("test/assets/dog.jpeg") - _image_query = gl.submit_image_query(detector=detector.id, image=dog) + _image_query = gl.submit_image_query(detector=detector.id, image=dog, human_review="NEVER") black = Image.new("RGB", (640, 480)) - _image_query = gl.submit_image_query(detector=detector.id, image=black) + _image_query = gl.submit_image_query(detector=detector.id, image=black, human_review="NEVER") def test_submit_image_query_wait_and_want_async_causes_exception(gl: Groundlight, detector: Detector): @@ -287,7 +297,7 @@ def test_submit_image_query_wait_and_want_async_causes_exception(gl: Groundlight with pytest.raises(ValueError): _image_query = gl.submit_image_query( - detector=detector.id, image="test/assets/dog.jpeg", wait=10, want_async=True + detector=detector.id, image="test/assets/dog.jpeg", wait=10, want_async=True, human_review="NEVER" ) @@ -296,7 +306,9 @@ def test_submit_image_query_with_want_async_workflow(gl: Groundlight, detector: Tests the workflow for submitting an image query with the want_async parameter set to True. """ - _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/dog.jpeg", wait=0, want_async=True) + _image_query = gl.submit_image_query( + detector=detector.id, image="test/assets/dog.jpeg", wait=0, want_async=True, human_review="NEVER" + ) # the result should be None assert _image_query.result is None @@ -450,7 +462,7 @@ def test_enum_string_equality(): @pytest.mark.skipif(MISSING_NUMPY or MISSING_PIL, reason="Needs numpy and pillow") # type: ignore def test_submit_numpy_image(gl: Groundlight, detector: Detector): np_img = np.random.uniform(0, 255, (600, 800, 3)) # type: ignore - _image_query = gl.submit_image_query(detector=detector.id, image=np_img) + _image_query = gl.submit_image_query(detector=detector.id, image=np_img, human_review="NEVER") assert str(_image_query) assert isinstance(_image_query, ImageQuery) assert is_valid_display_result(_image_query.result) @@ -481,7 +493,7 @@ def submit_noisy_image(image, label=None): noisy_image = contrast.enhance(random.uniform(0.75, 1)) brightness = ImageEnhance.Brightness(noisy_image) noisy_image = brightness.enhance(random.uniform(0.75, 1)) - img_query = gl.submit_image_query(detector=detector.id, image=noisy_image, wait=0) + img_query = gl.submit_image_query(detector=detector.id, image=noisy_image, wait=0, human_review="NEVER") if label is not None: gl.add_label(img_query, label) return img_query