Merge pull request #924 from anarkiwi/yolo2

Add notes on running torchserve experimentally.

docs/README-airt.md

@@ -122,3 +122,43 @@ gr::log :DEBUG: retune_fft0 - retuning to {}
 ```
 
 while in operation - this is normal, the broken logging is because gamutrf/gr-iqtlabs are written for gnuradio 3.10, not for gnuradio 3.9 style logging.
+
+
+## inference with torchserve
+
+On a non-AIRT machine that the AIRT can reach over the network, that has an nvidia GPU and docker installed:
+
+# docker nvidia support
+
+See https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html
+
+# start torchserve
+
+From gamutRF's source directory:
+
+```
+$ mkdir /tmp/torchserve
+$ cp torchserve/config.properities /tmp/torchserve
+$ docker run --gpus all -p 8081:8081 -p 8080:8080 -v /tmp/torchserve:/torchserve -d iqtlabs/gamutrf-cuda-torchserve torchserve --start --model-store /torchserve --ts-config /torchserve/config.properties --ncs --foreground
+```
+
+# create and register model
+
+From gamutRF's source directory, and having obtained mini2_snr.pt:
+
+```
+$ pip3 install torch-model-archiver
+$ torch-model-archiver --force --model-name mini2_snr --version 1.0 --serialized-file /PATH/TO/mini2_snr.pt --handler torchserve/custom_handler.py --export-path /tmp/torchserve
+$ curl -X POST "localhost:8081/models?model_name=mini2_snr&url=mini2_snr.mar&initial_workers=4&batch_size=2"
+```
+
+Now, when starting the scanner, on the AIRT:
+
+```
+$ LD_PRELOAD=$HOME/.conda/envs/$CONDA_DEFAULT_ENV/lib/libgomp.so.1 gamutrf-scan --sdr=SoapyAIRT --freq-start=300e6 --freq-end=6e9 --tune-step-fft 256 --samp-rate=100e6 --nfft 256 --pretune --no-tag-now --inference_model_server TORCHSERVEHOSTNAME:8080 --inference_model_name mini2_snr --inference_output_dir /tmp --inference_min_db -100 --inference_min_confidence 0.5
+```
+
+# inference caveats
+
+* torchserve can run in CPU-only mode using the same container if no GPU is available, but inference performance will be poor.
+* running torchserve and gamutrf on the same host is not currently recommended (due to CPU resource competition, even when a GPU is available).

gamutrf/gryolo.py

@@ -56,7 +56,7 @@ def work(self, input_items, output_items):
         return n
 
     def draw_bounding_box(self, img, name, confidence, x, y, x_plus_w, y_plus_h):
-        label = f"{name}"
+        label = f"{name}: {confidence}"
         color = (255, 255, 255)
         cv2.rectangle(img, (x, y), (x_plus_w, y_plus_h), color, 2)
         cv2.putText(
@@ -72,21 +72,25 @@ def process_item(self, item):
         scores = []
         detections = []
 
-        for name, prediction_data in predictions.items():
-            for prediction in prediction_data:
-                conf = prediction["conf"]
-                if conf < self.confidence_threshold:
-                    continue
-                xywh = prediction["xywh"]
-                box = [
-                    xywh[0] - (0.5 * xywh[2]),
-                    xywh[1] - (0.5 * xywh[3]),
-                    xywh[2],
-                    xywh[3],
-                ]
-                detections.append({"box": box, "score": conf, "name": name})
-                boxes.append(box)
-                scores.append(conf)
+        try:
+            for name, prediction_data in predictions.items():
+                for prediction in prediction_data:
+                    conf = prediction["conf"]
+                    if conf < self.confidence_threshold:
+                        continue
+                    xywh = prediction["xywh"]
+                    box = [
+                        xywh[0] - (0.5 * xywh[2]),
+                        xywh[1] - (0.5 * xywh[3]),
+                        xywh[2],
+                        xywh[3],
+                    ]
+                    detections.append({"box": box, "score": conf, "name": name})
+                    boxes.append(box)
+                    scores.append(conf)
+        except TypeError as e:
+            print(f"invalid predictions from torchserve: {e}, {predictions}")
+            return
 
         if not detections:
             return

torchserve/config.properties

@@ -0,0 +1,3 @@
+inference_address=http://0.0.0.0:8080
+management_address=http://0.0.0.0:8081
+metrics_address=http://0.0.0.0:8082