From 1df082e575e5fc50c402b3e5450fa10cfe04f63d Mon Sep 17 00:00:00 2001 From: "Joao G.A. Amorim" Date: Sun, 5 Dec 2021 19:08:17 -0300 Subject: [PATCH 1/7] update confusion matrix calc. --- lapixdl/evaluation/evaluate.py | 38 +++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/lapixdl/evaluation/evaluate.py b/lapixdl/evaluation/evaluate.py index 2041289..ecc661a 100644 --- a/lapixdl/evaluation/evaluate.py +++ b/lapixdl/evaluation/evaluate.py @@ -1,8 +1,10 @@ -from typing import List, Iterable +import warnings +import itertools import numpy as np + +from typing import List, Iterable from tqdm import tqdm -import warnings from .model import * @@ -28,17 +30,25 @@ def evaluate_segmentation(gt_masks: Iterable[Mask], SegmentationMetrics: Pixel-based classification and segmentation metrics. """ - confusion_matrix = np.zeros((len(classes), len(classes)), np.int) + qtd_classes = len(classes) + confusion_matrix = np.zeros((qtd_classes, qtd_classes), int) + dummy = confusion_matrix.copy() + r = range(qtd_classes) for (curr_gt_mask, curr_pred_mask) in tqdm(zip(gt_masks, pred_masks), unit=' masks'): - flat_gt = __flat_mask(curr_gt_mask) - flat_pred = __flat_mask(curr_pred_mask) - if len(flat_gt) != len(flat_pred): - warnings.warn( - f"The GT mask and Pred mask should have the same shape. GT length: {len(flat_gt)}. Pred length: {len(flat_pred)}.") - - for (curr_pred, curr_gt) in zip(flat_pred, flat_gt): - confusion_matrix[curr_pred, curr_gt] += 1 + curr_gt_mask = np.array(curr_gt_mask) + curr_pred_mask = np.array(curr_pred_mask) + if curr_gt_mask.shape != curr_pred_mask.shape: + warnings.warn(f"The GT mask and Pred mask should have the same shape. GT shape: {curr_gt_mask.shape}.Pred shape: {curr_pred_mask.shape}.") + + confusion_matrix_tmp = dummy.copy() + for i, j in itertools.product(r, r): + confusion_matrix_tmp[j, i] = np.sum((curr_pred_mask==j)*(curr_gt_mask==i)) + + # also can be, but shows be ~1% slow than the code above; + # confusion_matrix_tmp = np.array([[np.sum((msk==i)*(pred == j)) for i in range(qtd_cats)] for j in range(qtd_cats)]) + + confusion_matrix += confusion_matrix_tmp metrics = SegmentationMetrics(classes, confusion_matrix) @@ -65,7 +75,7 @@ def evaluate_classification(gt_classifications: Iterable[Classification], ClassificationMetrics: Classification metrics. """ - confusion_matrix = np.zeros((len(classes), len(classes)), np.int) + confusion_matrix = np.zeros((len(classes), len(classes)), int) for (curr_gt_classification, curr_pred_classification) in tqdm(zip(gt_classifications, pred_classifications), unit=' samples'): confusion_matrix[curr_pred_classification.cls, @@ -103,7 +113,7 @@ def evaluate_detection(gt_bboxes: Iterable[List[BBox]], classes_count = len(classes) cls_ious_sum = np.zeros(classes_count) - confusion_matrix = np.zeros((classes_count + 1, classes_count + 1), np.int) + confusion_matrix = np.zeros((classes_count + 1, classes_count + 1), int) undetected_idx = classes_count # Tracks detection scores to calculate the Precision x Recall curve and the Average Precision metric predictions_by_class: List[List[PredictionResult]] = [ @@ -257,7 +267,7 @@ def __calculate_binary_iou(gt_bboxes: List[BBox], def __draw_bboxes(mask_shape: Tuple[int, int], bboxes: List[BBox]) -> List[List[int]]: - mask = np.zeros(mask_shape, np.int) + mask = np.zeros(mask_shape, int) for bbox in bboxes: mask[ From 11f015c2f28a1dedff0dd698c0f0ab61812eaae2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Gustavo=20A=2E=20Amorim?= Date: Mon, 6 Dec 2021 10:43:18 -0300 Subject: [PATCH 2/7] rename vars from code review segmentation evaluation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andre Victória Matias --- lapixdl/evaluation/evaluate.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lapixdl/evaluation/evaluate.py b/lapixdl/evaluation/evaluate.py index ecc661a..cfe8fb1 100644 --- a/lapixdl/evaluation/evaluate.py +++ b/lapixdl/evaluation/evaluate.py @@ -30,10 +30,10 @@ def evaluate_segmentation(gt_masks: Iterable[Mask], SegmentationMetrics: Pixel-based classification and segmentation metrics. """ - qtd_classes = len(classes) - confusion_matrix = np.zeros((qtd_classes, qtd_classes), int) - dummy = confusion_matrix.copy() - r = range(qtd_classes) + classes_count = len(classes) + zeros_matrix = np.zeros((qtd_classes, qtd_classes), int) + confusion_matrix = zeros_matrix.copy() + classes_count_range = range(qtd_classes) for (curr_gt_mask, curr_pred_mask) in tqdm(zip(gt_masks, pred_masks), unit=' masks'): curr_gt_mask = np.array(curr_gt_mask) @@ -41,11 +41,11 @@ def evaluate_segmentation(gt_masks: Iterable[Mask], if curr_gt_mask.shape != curr_pred_mask.shape: warnings.warn(f"The GT mask and Pred mask should have the same shape. GT shape: {curr_gt_mask.shape}.Pred shape: {curr_pred_mask.shape}.") - confusion_matrix_tmp = dummy.copy() + curr_confusion_matrix = zeros_matrix.copy() for i, j in itertools.product(r, r): confusion_matrix_tmp[j, i] = np.sum((curr_pred_mask==j)*(curr_gt_mask==i)) - # also can be, but shows be ~1% slow than the code above; + # ~1% slower alternative: # confusion_matrix_tmp = np.array([[np.sum((msk==i)*(pred == j)) for i in range(qtd_cats)] for j in range(qtd_cats)]) confusion_matrix += confusion_matrix_tmp From 917486c847f02888a945aa44421e4da0ffe4c295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Vict=C3=B3ria=20Matias?= Date: Mon, 6 Dec 2021 10:59:49 -0300 Subject: [PATCH 3/7] Removing cv2 --- examples.py | 53 ++++++++++++++++++++++++++++---------------------- setup.py | 4 ++-- tests/utils.py | 15 +++++++++----- 3 files changed, 42 insertions(+), 30 deletions(-) diff --git a/examples.py b/examples.py index 83e9b5f..7bb704f 100644 --- a/examples.py +++ b/examples.py @@ -2,9 +2,10 @@ Module examples Requires: - opencv-python, numpy + numpy """ -import cv2 +from typing import Tuple + import numpy as np from matplotlib.colors import ListedColormap @@ -16,13 +17,13 @@ def main(): # Model evaluation examples evaluate_segmentation_example() - evaluate_detection_example() - evaluate_classification_example() + # evaluate_detection_example() + # evaluate_classification_example() # Results visualization examples - show_classification_example() - show_segmentation_example() - show_detection_example() + # show_classification_example() + # show_segmentation_example() + # show_detection_example() def evaluate_segmentation_example(): @@ -35,29 +36,29 @@ def evaluate_segmentation_example(): # Creating fake data # Creates a rectangle of 1s in a 0s array gt_bbox_1 = BBox(10, 10, 10, 10, 1) - mask_bin_GT_1 = cv2.rectangle(np.zeros(mask_shape, np.int), - gt_bbox_1.upper_left_point, - gt_bbox_1.bottom_right_point, - 1, -1) + mask_bin_GT_1 = draw_rectangle(np.zeros(mask_shape, np.int), + gt_bbox_1.upper_left_point, + gt_bbox_1.bottom_right_point, + 1) pred_bbox_1 = BBox(10, 10, 10, 10, 1) - mask_bin_pred_1 = cv2.rectangle(np.zeros(mask_shape, np.int), - pred_bbox_1.upper_left_point, - pred_bbox_1.bottom_right_point, - 1, -1) + mask_bin_pred_1 = draw_rectangle(np.zeros(mask_shape, np.int), + pred_bbox_1.upper_left_point, + pred_bbox_1.bottom_right_point, + 1) # Creates a rectangle of 2s in a 0s array gt_bbox_2 = BBox(110, 110, 320, 280, 2) - mask_bin_GT_2 = cv2.rectangle(np.zeros(mask_shape, np.int), - gt_bbox_2.upper_left_point, - gt_bbox_2.bottom_right_point, - 2, -1) + mask_bin_GT_2 = draw_rectangle(np.zeros(mask_shape, np.int), + gt_bbox_2.upper_left_point, + gt_bbox_2.bottom_right_point, + 2) pred_bbox_2 = BBox(70, 50, 240, 220, 2) - mask_bin_pred_2 = cv2.rectangle(np.zeros(mask_shape, np.int), - pred_bbox_2.upper_left_point, - pred_bbox_2.bottom_right_point, - 2, -1) + mask_bin_pred_2 = draw_rectangle(np.zeros(mask_shape, np.int), + pred_bbox_2.upper_left_point, + pred_bbox_2.bottom_right_point, + 2) # Merging masks mask_GT = np.maximum(mask_bin_GT_1, mask_bin_GT_2) @@ -242,4 +243,10 @@ def draw_bboxes(mask_shape, bboxes): return mask +def draw_rectangle(img: np.ndarray, pt1: Tuple[int, int], pt2: Tuple[int, int], fill: int): + cp = img.copy() + cp[slice(pt1[0], pt2[0] + 1), slice(pt1[1], pt2[1] + 1)] = fill + return cp + + main() diff --git a/setup.py b/setup.py index b06f182..1b3eb64 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='lapixdl', packages=find_packages(exclude=['tests']), - version='0.7.17', + version='0.7.18', description='Utils for Computer Vision Deep Learning research', long_description=long_description, @@ -18,7 +18,7 @@ install_requires=['numpy', 'tqdm', 'seaborn', 'pandas', 'matplotlib'], setup_requires=['pytest-runner'], - tests_require=['pytest', 'cv2'], + tests_require=['pytest'], test_suite='tests', classifiers=[ diff --git a/tests/utils.py b/tests/utils.py index 8a8deab..2df76ca 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,5 @@ from typing import Tuple -import cv2 import numpy as np from lapixdl.evaluation.model import BBox @@ -10,7 +9,13 @@ def bin_mask_from_bb(mask_shape: Tuple[int, int], bbox: BBox): ''' Draws a mask from a bounding box ''' - return cv2.rectangle(np.zeros(mask_shape, np.int), - bbox.upper_left_point, - bbox.bottom_right_point, - 1, -1) + return draw_rectangle(np.zeros(mask_shape, np.int), + bbox.upper_left_point, + bbox.bottom_right_point, + 1) + + +def draw_rectangle(img: np.ndarray, pt1: Tuple[int, int], pt2: Tuple[int, int], fill: int): + cp = img.copy() + cp[slice(pt1[0], pt2[0] + 1), slice(pt1[1], pt2[1] + 1)] = fill + return cp From 89bafbfa35eace037cde91d414bf31e8bb334696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Vict=C3=B3ria=20Matias?= Date: Mon, 6 Dec 2021 11:04:11 -0300 Subject: [PATCH 4/7] amend --- examples.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples.py b/examples.py index 7bb704f..5486b7d 100644 --- a/examples.py +++ b/examples.py @@ -17,13 +17,13 @@ def main(): # Model evaluation examples evaluate_segmentation_example() - # evaluate_detection_example() - # evaluate_classification_example() + evaluate_detection_example() + evaluate_classification_example() # Results visualization examples - # show_classification_example() - # show_segmentation_example() - # show_detection_example() + show_classification_example() + show_segmentation_example() + show_detection_example() def evaluate_segmentation_example(): From 7b8842fd416ff10273b7272ff482a9d44899a65b Mon Sep 17 00:00:00 2001 From: "Joao G.A. Amorim" Date: Mon, 6 Dec 2021 11:22:45 -0300 Subject: [PATCH 5/7] remove deprecated alias for int --- tests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index 2df76ca..38339ab 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -9,7 +9,7 @@ def bin_mask_from_bb(mask_shape: Tuple[int, int], bbox: BBox): ''' Draws a mask from a bounding box ''' - return draw_rectangle(np.zeros(mask_shape, np.int), + return draw_rectangle(np.zeros(mask_shape, int), bbox.upper_left_point, bbox.bottom_right_point, 1) From 954e8ed955b19f1f0d1aef04346133fc36381fd7 Mon Sep 17 00:00:00 2001 From: "Joao G.A. Amorim" Date: Mon, 6 Dec 2021 11:25:57 -0300 Subject: [PATCH 6/7] fix variable names --- lapixdl/evaluation/evaluate.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lapixdl/evaluation/evaluate.py b/lapixdl/evaluation/evaluate.py index cfe8fb1..25d96f3 100644 --- a/lapixdl/evaluation/evaluate.py +++ b/lapixdl/evaluation/evaluate.py @@ -31,9 +31,9 @@ def evaluate_segmentation(gt_masks: Iterable[Mask], """ classes_count = len(classes) - zeros_matrix = np.zeros((qtd_classes, qtd_classes), int) + zeros_matrix = np.zeros((classes_count, classes_count), int) confusion_matrix = zeros_matrix.copy() - classes_count_range = range(qtd_classes) + classes_count_range = range(classes_count) for (curr_gt_mask, curr_pred_mask) in tqdm(zip(gt_masks, pred_masks), unit=' masks'): curr_gt_mask = np.array(curr_gt_mask) @@ -42,13 +42,13 @@ def evaluate_segmentation(gt_masks: Iterable[Mask], warnings.warn(f"The GT mask and Pred mask should have the same shape. GT shape: {curr_gt_mask.shape}.Pred shape: {curr_pred_mask.shape}.") curr_confusion_matrix = zeros_matrix.copy() - for i, j in itertools.product(r, r): - confusion_matrix_tmp[j, i] = np.sum((curr_pred_mask==j)*(curr_gt_mask==i)) + for i, j in itertools.product(classes_count_range, classes_count_range): + curr_confusion_matrix[j, i] = np.sum((curr_pred_mask==j)*(curr_gt_mask==i)) # ~1% slower alternative: - # confusion_matrix_tmp = np.array([[np.sum((msk==i)*(pred == j)) for i in range(qtd_cats)] for j in range(qtd_cats)]) + # curr_confusion_matrix = np.array([[np.sum((msk==i)*(pred == j)) for i in range(classes_count)] for j in range(classes_count)]) - confusion_matrix += confusion_matrix_tmp + confusion_matrix += curr_confusion_matrix metrics = SegmentationMetrics(classes, confusion_matrix) From b63fbae3906f11cee044655da57f0c2b9813dce3 Mon Sep 17 00:00:00 2001 From: "Joao G.A. Amorim" Date: Mon, 6 Dec 2021 11:27:53 -0300 Subject: [PATCH 7/7] remove deprecated alias for float --- lapixdl/evaluation/evaluate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lapixdl/evaluation/evaluate.py b/lapixdl/evaluation/evaluate.py index 25d96f3..316f31e 100644 --- a/lapixdl/evaluation/evaluate.py +++ b/lapixdl/evaluation/evaluate.py @@ -191,7 +191,7 @@ def calculate_pairwise_bbox_ious(gt_bboxes: List[BBox], List[List[float]]: [gt x pred] matrix of pairwise IoUs of GT and predicted bboxes """ - ious = np.zeros((len(gt_bboxes), len(pred_bboxes)), np.float) + ious = np.zeros((len(gt_bboxes), len(pred_bboxes)), float) for i, gt_bbox in enumerate(gt_bboxes): for j, pred_bbox in enumerate(pred_bboxes): @@ -232,7 +232,7 @@ def calculate_iou_by_class(gt_bboxes: List[BBox], List[float]: IoUs of an image indexed by class """ - ious = np.zeros(classes_count, np.float) + ious = np.zeros(classes_count, float) for i in range(classes_count): ious[i] = __calculate_binary_iou([gt_bbox for gt_bbox in gt_bboxes if gt_bbox.cls == i],