Switched to an OpenCV deep neural network for detecting faces

File sources:
- https://github.com/opencv/opencv/blob/master/samples/dnn/face_detector/deploy.prototxt
- https://github.com/opencv/opencv_3rdparty/blob/dnn_samples_face_detector_20170830/res10_300x300_ssd_iter_140000.caffemodel
- https://github.com/opencv/opencv/blob/master/LICENSE

Heartbeat.cpp

@@ -179,14 +179,6 @@ int main(int argc, char * argv[]) {
         downsample = DEFAULT_DOWNSAMPLE;
     }
 
-    const string CLASSIFIER_PATH = "haarcascade_frontalface_alt.xml";
-
-    std::ifstream test(CLASSIFIER_PATH);
-    if (!test) {
-        std::cout << "Face classifier xml not found!" << std::endl;
-        exit(0);
-    }
-
     bool offlineMode = input != "";
 
     VideoCapture cap;
@@ -230,8 +222,7 @@ int main(int argc, char * argv[]) {
               WIDTH, HEIGHT, TIME_BASE, downsample,
               samplingFrequency, rescanFrequency,
               minSignalSize, maxSignalSize,
-              LOG_PATH, CLASSIFIER_PATH,
-              log, gui);
+              LOG_PATH, log, gui);
 
     // Load baseline if necessary
     Baseline baseline = Baseline();

README.md

@@ -34,7 +34,7 @@ Compile the source code for your system, providing a number of required linker f
 This works with opencv 3.4.1 on macOS:
 
 ```sh
-$ g++ -std=c++11 -lopencv_core -lopencv_highgui -lopencv_imgcodecs -lopencv_imgproc -lopencv_objdetect -lopencv_video -lopencv_videoio -lavcodec Heartbeat.cpp opencv.cpp RPPG.cpp Baseline.cpp -o Heartbeat
+$ g++ -std=c++11 -lopencv_core -lopencv_dnn -lopencv_highgui -lopencv_imgcodecs -lopencv_imgproc -lopencv_objdetect -lopencv_video -lopencv_videoio -lavcodec Heartbeat.cpp opencv.cpp RPPG.cpp Baseline.cpp -o Heartbeat
 ```
 
 Alternative compilation for Ubuntu. Works with opencv 3.1:

RPPG.cpp

@@ -16,6 +16,7 @@
 #include "opencv.hpp"
 
 using namespace cv;
+using namespace dnn;
 using namespace std;
 
 #define LOW_BPM 42
@@ -31,8 +32,7 @@ bool RPPG::load(const rPPGAlgorithm algorithm,
                 const int width, const int height, const double timeBase, const int downsample,
                 const double samplingFrequency, const double rescanFrequency,
                 const int minSignalSize, const int maxSignalSize,
-                const string &logPath, const string &classifierPath,
-                const bool log, const bool gui) {
+                const string &logPath, const bool log, const bool gui) {
 
     this->algorithm = algorithm;
     this->guiMode = gui;
@@ -46,8 +46,8 @@ bool RPPG::load(const rPPGAlgorithm algorithm,
     this->samplingFrequency = samplingFrequency;
     this->timeBase = timeBase;
 
-    // Load classifiers
-    classifier.load(classifierPath);
+    // Load face detector
+    faceDetectNet = readNetFromCaffe("opencv/deploy.prototxt", "opencv/res10_300x300_ssd_iter_140000.caffemodel");
 
     // Setting up logfilepath
     ostringstream path_1;
@@ -86,14 +86,14 @@ void RPPG::processFrame(Mat &frameRGB, Mat &frameGray, int time) {
         cout << "Not valid, finding a new face" << endl;
 
         lastScanTime = time;
-        detectFace(frameGray);
+        detectFace(frameRGB, frameGray);
 
     } else if ((time - lastScanTime) * timeBase >= 1/rescanFrequency) {
 
         cout << "Valid, but rescanning face" << endl;
 
         lastScanTime = time;
-        detectFace(frameGray);
+        detectFace(frameRGB, frameGray);
         rescanFlag = true;
 
     } else {
@@ -167,13 +167,35 @@ void RPPG::processFrame(Mat &frameRGB, Mat &frameGray, int time) {
     frameGray.copyTo(lastFrameGray);
 }
 
-void RPPG::detectFace(Mat &frameGray) {
+void RPPG::detectFace(Mat &frameRGB, Mat &frameGray) {
 
     cout << "Scanning for faces…" << endl;
 
-    // Detect faces with Haar classifier
-    vector<Rect> boxes;
-    classifier.detectMultiScale(frameGray, boxes, 1.1, 2, CV_HAAR_SCALE_IMAGE, minFaceSize);
+    // Detect faces with DNN
+    Mat resize300;
+    cv::resize(frameRGB, resize300, Size(300, 300));
+    Mat blob = blobFromImage(resize300, 1.0, Size(300, 300), Scalar(104.0, 177.0, 123.0));
+    faceDetectNet.setInput(blob);
+    Mat detection = faceDetectNet.forward();
+    Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
+    float confidenceThreshold = 0.5;
+    vector<Rect> boxes = {};
+
+    for (int i = 0; i < detectionMat.rows; i++) {
+        float confidence = detectionMat.at<float>(i, 2);
+
+        if (confidence > confidenceThreshold) {
+            int xLeftBottom = static_cast<int>(detectionMat.at<float>(i, 3) * frameRGB.cols);
+            int yLeftBottom = static_cast<int>(detectionMat.at<float>(i, 4) * frameRGB.rows);
+            int xRightTop = static_cast<int>(detectionMat.at<float>(i, 5) * frameRGB.cols);
+            int yRightTop = static_cast<int>(detectionMat.at<float>(i, 6) * frameRGB.rows);
+
+            Rect object((int)xLeftBottom, (int)yLeftBottom,
+                        (int)(xRightTop - xLeftBottom),
+                        (int)(yRightTop - yLeftBottom));
+            boxes.push_back(object);
+        }
+    }
 
     if (boxes.size() > 0) {
 
@@ -625,4 +647,4 @@ void RPPG::draw(cv::Mat &frameRGB) {
         line(frameRGB, Point(corners[i].x-5,corners[i].y), Point(corners[i].x+5,corners[i].y), GREEN, 1);
         line(frameRGB, Point(corners[i].x,corners[i].y-5), Point(corners[i].x,corners[i].y+5), GREEN, 1);
     }
-}
+}

RPPG.hpp

@@ -12,10 +12,12 @@
 #include <fstream>
 #include <string>
 #include <opencv2/objdetect/objdetect.hpp>
+#include <opencv2/dnn.hpp>
 
 #include <stdio.h>
 
 using namespace cv;
+using namespace dnn;
 using namespace std;
 
 enum rPPGAlgorithm { g, pca, xminay };
@@ -32,8 +34,7 @@ class RPPG {
               const int width, const int height, const double timeBase, const int downsample,
               const double samplingFrequency, const double rescanFrequency,
               const int minSignalSize, const int maxSignalSize,
-              const string &logPath, const string &classifierPath,
-              const bool log, const bool gui);
+              const string &logPath, const bool log, const bool gui);
 
     void processFrame(Mat &frameRGB, Mat &frameGray, int time);
 
@@ -43,7 +44,7 @@ class RPPG {
 
 private:
 
-    void detectFace(Mat &frameGray);
+    void detectFace(Mat &frameRGB, Mat &frameGray);
     void setNearestBox(vector<Rect> boxes);
     void detectCorners(Mat &frameGray);
     void trackFace(Mat &frameGray);
@@ -60,8 +61,8 @@ class RPPG {
     // The algorithm
     rPPGAlgorithm algorithm;
 
-    // The classifiers
-    CascadeClassifier classifier;
+    // The face detector
+    Net faceDetectNet;
 
     // Settings
     Size minFaceSize;