add ncnn to rec

app/src/main/cpp/task/CVTask.h

@@ -11,7 +11,9 @@
 #include <opencv2/core.hpp>
 #include <opencv2/objdetect.hpp>
 #include <dlib/opencv/cv_image.h>
+#include "mobile_facenet.h"
 #include "DLibRecognizer.h"
+#include "NCNNRecognizer.h"
 
 using namespace std;
 using namespace cv;
@@ -21,10 +23,8 @@ class CVTask : public FrameTask {
 private:
 
     CVDetector cvDetector;
-//    Ptr<CascadeDetectorAdapter> detector;
-//    FaceRecognize *recognizer = new FaceRecognize();
-    DLibRecognizer *recognizer = new DLibRecognizer();
-    long lastRecTime = 0;
+    NCNNRecognizer *ncnnRecognizer = new NCNNRecognizer();
+    // long lastRecTime = 0;
 
     static string string_format(const string fmt, ...) {
         int size = ((int) fmt.size()) * 2 + 50;   // Use a rubric appropriate for your code
@@ -51,42 +51,31 @@ class CVTask : public FrameTask {
 
     CVTask(int fps, string path) : FrameTask(move("CVTask"), fps) {
         cvDetector.create(path);
-//        detector = makePtr<CascadeDetectorAdapter>(makePtr<CascadeClassifier>(path));
-//        detector->setMinObjectSize(Size(20, 20));
-//        recognizer->setThreadNum(4);
-//        recognizer->init("/data/user/0/com.kk.afdd/files/modules/mobilefacenet.bin",
-//                         "/data/user/0/com.kk.afdd/files/modules/mobilefacenet.param",
-//                         false);
     }
 
     ~CVTask() {
         cvDetector.destroy();
-//        delete detector;
-        delete recognizer;
+        delete ncnnRecognizer;
     }
 
     void doTask(Frame *frame) {
-        long start = TimeUtil::now();
+         long start = TimeUtil::now();
         Mat yuvImg, brgImg;
         yuvImg.create(frame->height * 3 / 2, frame->width, CV_8UC1);
         memcpy(yuvImg.data, frame->data, frame->len);
         cvtColor(yuvImg, brgImg, COLOR_YUV2GRAY_I420);
 
         std::vector<Rect> rectFaces;
         cvDetector.detect(brgImg, &rectFaces);
-//        detector->detect(brgImg, rectFaces);
 
         string json = "[";
         for (int i = 0, n = rectFaces.size(); i < n; i++) {
             Rect rect = rectFaces[i];
             string featureJson = "[";
-            if (i == 0 && (start - lastRecTime > 5000)) {
-                lastRecTime = start;
+            if (i == 0/* && (start - lastRecTime > 5000)*/) {
+                // lastRecTime = start;
                 Mat *croppedFace = new Mat(brgImg, rect);
-                Mat rgbFace, resizedRgbFace;
-                cvtColor(*croppedFace, rgbFace, COLOR_BGR2RGB);
-                resize(rgbFace, resizedRgbFace, Size(150, 150), 0, 0, INTER_AREA);
-                float *feature = recognizer->getFeature(resizedRgbFace);
+                float *feature = ncnnRecognizer->getFeature(*croppedFace);
                 delete croppedFace;
                 for (int fi = 0, fn = 128; fi < fn; fi++) {
                     featureJson.append(string_format("%f", feature[fi]));
@@ -123,8 +112,8 @@ class CVTask : public FrameTask {
     }
 
     float calculateSimilar(float *feature1, float *feature2) {
-        if (recognizer) {
-            return recognizer->calculateSimilar(feature1, feature2);
+        if (ncnnRecognizer) {
+            return ncnnRecognizer->calculateSimilar(feature1, feature2);
         }
         return 0;
     }

app/src/main/cpp/task/NCNNRecognizer.h

@@ -0,0 +1,62 @@
+//
+// Created by Will Zhang on 2020/11/14.
+//
+
+#ifndef ANDROIDNDKAICAMERA_NCNNRECOGNIZER_H
+#define ANDROIDNDKAICAMERA_NCNNRECOGNIZER_H
+
+#include <string>
+#include "net.h"
+#include <algorithm>
+#include <opencv2/core.hpp>
+
+class NCNNRecognizer {
+private:
+    ncnn::Net recognizer;
+public:
+    NCNNRecognizer() {
+        recognizer.load_param("/data/user/0/com.kk.afdd/files/modules/mobilefacenet.param");
+        recognizer.load_model("/data/user/0/com.kk.afdd/files/modules/mobilefacenet.bin");
+    }
+
+    ~NCNNRecognizer() {
+        recognizer.clear();
+    }
+
+    float *getFeature(cv::Mat croppedFace) {
+        ncnn::Extractor ex = recognizer.create_extractor();
+        ex.set_num_threads(1);
+        ex.set_light_mode(true);
+        ncnn::Mat ncnnFace = ncnn::Mat::from_pixels(croppedFace.data,
+                                                    ncnn::Mat::PIXEL_BGR2RGB,
+                                                    croppedFace.cols,
+                                                    croppedFace.rows);
+        ex.input("data", ncnnFace);
+        ncnn::Mat out;
+        ex.extract("fc1", out);
+        float *feature = new float[128];
+        for (int j = 0; j < 128; j++) {
+            // TODO not working ?
+            if (std::isnan(out[j]) || std::isinf(out[j])) {
+                feature[j] = 0;
+            } else {
+                feature[j] = out[j];
+            }
+        }
+        return feature;
+    }
+
+
+    float calculateSimilar(float *feature1, float *feature2) {
+        float ret = 0.0, mod1 = 0.0, mod2 = 0.0;
+        for (int i = 0; i < 128; ++i) {
+            ret += feature1[i] * feature2[i];
+            mod1 += feature1[i] * feature1[i];
+            mod2 += feature2[i] * feature2[i];
+        }
+        return ret / sqrt(mod1) / sqrt(mod2);
+    }
+
+};
+
+#endif //ANDROIDNDKAICAMERA_NCNNRECOGNIZER_H

app/src/main/java/com/kk/afdd/Config.java

@@ -4,6 +4,6 @@ public class Config {
     public static final int PREVIEW_WIDTH = 240;
     public static final int PREVIEW_HEIGHT = 320;
     public static final int ROTATION = 90;
-    public static final int FPS = 8;
+    public static final int FPS = 5;
     public static final boolean MIRROR = true;
 }

app/src/main/java/com/kk/afdd/MainActivity.java

@@ -234,20 +234,21 @@ public void onFaceDetected(String faceInfoJson) {
                 sFaceInfoType = new TypeToken<List<FaceInfo>>() {
                 }.getType();
             }
+            String ff = faceInfoJson;
+            int maxLogSize = 1000;
+            for(int i = 0; i <= ff.length() / maxLogSize; i++) {
+                int start = i * maxLogSize;
+                int end = (i+1) * maxLogSize;
+                end = Math.min(end, ff.length());
+                Timber.v(ff.substring(start, end));
+            }
+            faceInfoJson = faceInfoJson.replaceAll("nan", "0").replaceAll("inf", "0");
             List<FaceInfo> faceInfoList = mGson.fromJson(faceInfoJson, sFaceInfoType);
             if (faceInfoList.size() > 0) {
                 for (FaceInfo faceInfo : faceInfoList) {
                     if (null == faceInfo.feature || faceInfo.feature.isEmpty()) {
                         continue;
                     }
-                    // String ff = FeatureUtil.featureToString(FeatureUtil.getPrimitiveArray(faceInfo.feature));
-                    // int maxLogSize = 1000;
-                    // for(int i = 0; i <= ff.length() / maxLogSize; i++) {
-                    //     int start = i * maxLogSize;
-                    //     int end = (i+1) * maxLogSize;
-                    //     end = Math.min(end, ff.length());
-                    //     Timber.v(ff.substring(start, end));
-                    // }
                     mCurrentFaceInfo = faceInfoList.get(0);
                     long start = System.currentTimeMillis();
                     for (User user : mUserList) {