[examples] Add FPS measurement

examples/attack/.gitignore

@@ -2,5 +2,4 @@ models/
 logs/
 *.pth
 *.avi
-*.npy
-*.png
+*.npy

examples/attack/yolov3_pcb_attack_evaluation.py

@@ -1,4 +1,5 @@
 import cv2
+import time
 import datetime
 import numpy as np
 
@@ -23,7 +24,8 @@
 logger = log.get_logger(__name__)
 
 # Tensorboard
-pcb_log_dir = prefix + 'logs/b/decay/0.99/' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
+pcb_log_dir = prefix + 'logs/pcb/uniform_init/decay/0.98/' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
+# pcb_log_dir = prefix + 'logs/pcb/zero_init/decay/0.98/' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
 tb = TensorBoardLogger(pcb_log_dir)
 
 # Target Model
@@ -49,18 +51,19 @@
 
     colors = np.random.uniform(0, 255, size=(len(classes), 3))
 
-    origin_cv_image = cv2.imread(prefix + 'demo.jpg')
+    origin_cv_image = cv2.imread(prefix + 'demo.png')
     origin_cv_image = cv2.cvtColor(origin_cv_image, cv2.COLOR_BGR2RGB)
 
     # Adversarial Attack
     model_path = os.path.join(WHAT_MODEL_PATH, what_yolov3_model_list[index][WHAT_MODEL_FILE_INDEX])
-    attack = PCBAttack(model_path, "multi_untargeted", classes, decay=0.99)
+    attack = PCBAttack(model_path, "multi_untargeted", classes, init="uniform", decay=0.98)
     attack.fixed = False
 
     last_outs = None
     last_boxes = None
     last_probs = None
 
+    attack_time = []
     for n in range(0, n_iteration):
         logger.info(f"Iteration: {n}")
 
@@ -69,7 +72,12 @@
         input_cv_image = np.array(input_cv_image).astype(np.float32) / 255.0
 
         # Yolo inference
+        start_time = time.time()
         input_cv_image, outs = attack.attack(input_cv_image)
+        attack_time.append(time.time() - start_time)
+
+        tb.log_scalar('attack time', 1.0 / attack_time[-1], n)
+        print("FPS:", 1.0 / attack_time[-1])
 
         if last_outs is not None:
             res_list = []
@@ -132,6 +140,7 @@
 
         if (cv2.waitKey(1) & 0xFF == ord('q')):
             break
-
+
+    print("Average FPS:", 1.0 / np.mean(attack_time))
     logger.info("Perturbation saved to noise.npy")
     np.save(prefix + 'noise/noise.npy', attack.noise)

examples/attack/yolov3_pcb_attack_universal_voc2012.py

@@ -1,6 +1,8 @@
 import cv2
+import time
 import datetime
 import numpy as np
+from tqdm import tqdm
 
 from what.models.detection.datasets.coco import COCO_CLASS_NAMES
 from what.models.detection.utils.box_utils import draw_bounding_boxes
@@ -17,16 +19,17 @@
 from what.cli.model import *
 from what.utils.file import get_file
 
-n_iteration = 50
+n_iteration = 500
 prefix = './'
 
+NUM_IMGS = 1000
 show_image = False
 
 # Logging
 logger = log.get_logger(__name__)
 
 # Tensorboard
-pcb_log_dir = prefix + 'logs/pcb-universal/' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
+pcb_log_dir = prefix + 'logs/pcb-universal/zero_init/' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
 tb = TensorBoardLogger(pcb_log_dir)
 
 # Target Model
@@ -49,7 +52,7 @@
 if __name__ == '__main__':
 
     # Load Training Dataset from FiftyOne
-    train_dataset = foz.load_zoo_dataset("voc-2012", split="validation")
+    train_dataset = foz.load_zoo_dataset("voc-2012", split="validation")[0:NUM_IMGS]
     img_paths = train_dataset.values("filepath")
 
     classes = COCO_CLASS_NAMES
@@ -63,7 +66,7 @@
 
     origin_outs = []
 
-    for i in range(len(img_paths)):
+    for i in tqdm(range(len(img_paths))):
         img_path = img_paths[i]
 
         img = cv2.imread(str(img_path))
@@ -85,7 +88,8 @@
         res_mean_list = []
         boxes_list = []
 
-        for i in range(len(img_paths)):
+        attack_time = []
+        for i in tqdm(range(len(img_paths))):
             img_path = img_paths[i]
 
             img = cv2.imread(str(img_path))
@@ -96,8 +100,10 @@
             input_cv_image = np.array(input_cv_image).astype(np.float32) / 255.0
 
             # Yolo inference
+            start_time = time.time()
             input_cv_image, outs = attack.attack(input_cv_image)
-
+            attack_time.append(time.time() - start_time)
+
             boxes, labels, probs = yolo_process_output(outs, yolov3_tiny_anchors, len(classes))
 
             boxes_list.append(len(boxes))
@@ -142,6 +148,8 @@
                 if (cv2.waitKey(0) & 0xFF == ord('q')):
                     break
 
+        print("FPS:", 1 / np.mean(attack_time))
+        tb.log_scalar('attack time', np.mean(attack_time), n)
         tb.log_scalar('mean confidence increase', np.mean(res_mean_list), n)
         tb.log_scalar('boxes', np.mean(boxes_list), n)