Skip to content

Commit

Permalink
Option to use both Haar cascade classifier and DNN classifier for fac…
Browse files Browse the repository at this point in the history 
…e detection
  • Loading branch information
@prouast
prouast committed Sep 4, 2018
1 parent 42f48e2 commit 3a7f46f
Show file tree
Hide file tree
Showing 4 changed files with 26,268 additions and 39 deletions.
71 changes: 59 additions & 12 deletions Heartbeat.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,18 @@
#include "opencv.hpp"
#include "Baseline.hpp"

#define DEFAULT_ALGORITHM "g"
#define DEFAULT_RPPG_ALGORITHM "g"
#define DEFAULT_FACEDET_ALGORITHM "haar"
#define DEFAULT_RESCAN_FREQUENCY 1
#define DEFAULT_SAMPLING_FREQUENCY 1
#define DEFAULT_MIN_SIGNAL_SIZE 5
#define DEFAULT_MAX_SIGNAL_SIZE 5
#define DEFAULT_DOWNSAMPLE 1 // x means only every xth frame is used

#define HAAR_CLASSIFIER_PATH "haarcascade_frontalface_alt.xml"
#define DNN_PROTO_PATH "opencv/deploy.prototxt"
#define DNN_MODEL_PATH "opencv/res10_300x300_ssd_iter_140000.caffemodel"

using namespace cv;

Heartbeat::Heartbeat(int argc_, char * argv_[], bool switches_on_) {
Expand Down Expand Up @@ -79,13 +84,24 @@ bool to_bool(string s) {
return result;
}

rPPGAlgorithm to_algorithm(string s) {
rPPGAlgorithm to_rppgAlgorithm(string s) {
rPPGAlgorithm result;
if (s == "g") result = g;
else if (s == "pca") result = pca;
else if (s == "xminay") result = xminay;
else {
std::cout << "Please specify valid algorithm (g, pca, xminay)!" << std::endl;
std::cout << "Please specify valid rPPG algorithm (g, pca, xminay)!" << std::endl;
exit(0);
}
return result;
}

faceDetAlgorithm to_faceDetAlgorithm(string s) {
faceDetAlgorithm result;
if (s == "haar") result = haar;
else if (s == "deep") result = deep;
else {
std::cout << "Please specify valid face detection algorithm (haar, deep)!" << std::endl;
exit(0);
}
return result;
Expand All @@ -98,15 +114,26 @@ int main(int argc, char * argv[]) {
string input = cmd_line.get_arg("-i"); // Filepath for offline mode

// algorithm setting
rPPGAlgorithm algorithm;
string algorithmString = cmd_line.get_arg("-a");
if (algorithmString != "") {
algorithm = to_algorithm(algorithmString);
rPPGAlgorithm rPPGAlg;
string rppgAlgString = cmd_line.get_arg("-rppg");
if (rppgAlgString != "") {
rPPGAlg = to_rppgAlgorithm(rppgAlgString);
} else {
rPPGAlg = to_rppgAlgorithm(DEFAULT_RPPG_ALGORITHM);
}

cout << "Using rPPG algorithm " << rPPGAlg << "." << endl;

// face detection algorithm setting
faceDetAlgorithm faceDetAlg;
string faceDetAlgString = cmd_line.get_arg("-facedet");
if (faceDetAlgString != "") {
faceDetAlg = to_faceDetAlgorithm(faceDetAlgString);
} else {
algorithm = to_algorithm(DEFAULT_ALGORITHM);
faceDetAlg = to_faceDetAlgorithm(DEFAULT_FACEDET_ALGORITHM);
}

cout << "Using algorithm " << algorithm << "." << endl;
cout << "Using face detection algorithm " << faceDetAlg << "." << endl;

// rescanFrequency setting
double rescanFrequency;
Expand Down Expand Up @@ -179,6 +206,24 @@ int main(int argc, char * argv[]) {
downsample = DEFAULT_DOWNSAMPLE;
}

std::ifstream test1(HAAR_CLASSIFIER_PATH);
if (!test1) {
std::cout << "Face classifier xml not found!" << std::endl;
exit(0);
}

std::ifstream test2(DNN_PROTO_PATH);
if (!test2) {
std::cout << "DNN proto file not found!" << std::endl;
exit(0);
}

std::ifstream test3(DNN_MODEL_PATH);
if (!test3) {
std::cout << "DNN model file not found!" << std::endl;
exit(0);
}

bool offlineMode = input != "";

VideoCapture cap;
Expand Down Expand Up @@ -214,15 +259,17 @@ int main(int argc, char * argv[]) {
cout << "TIME BASE: " << TIME_BASE << endl;

std::ostringstream window_title;
window_title << title << " - " << WIDTH << "x" << HEIGHT << " -a " << algorithm << " -r " << rescanFrequency << " -f " << samplingFrequency << " -min " << minSignalSize << " -max " << maxSignalSize << " -ds " << downsample;
window_title << title << " - " << WIDTH << "x" << HEIGHT << " -rppg " << rPPGAlg << " -facedet " << faceDetAlg << " -r " << rescanFrequency << " -f " << samplingFrequency << " -min " << minSignalSize << " -max " << maxSignalSize << " -ds " << downsample;

// Set up rPPG
RPPG rppg = RPPG();
rppg.load(algorithm,
rppg.load(rPPGAlg, faceDetAlg,
WIDTH, HEIGHT, TIME_BASE, downsample,
samplingFrequency, rescanFrequency,
minSignalSize, maxSignalSize,
LOG_PATH, log, gui);
LOG_PATH, HAAR_CLASSIFIER_PATH,
DNN_PROTO_PATH, DNN_MODEL_PATH,
log, gui);

// Load baseline if necessary
Baseline baseline = Baseline();
Expand Down
60 changes: 38 additions & 22 deletions RPPG.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,16 @@ using namespace std;
#define QUALITY_LEVEL 0.01
#define MIN_DISTANCE 25

bool RPPG::load(const rPPGAlgorithm algorithm,
bool RPPG::load(const rPPGAlgorithm rPPGAlg, const faceDetAlgorithm faceDetAlg,
const int width, const int height, const double timeBase, const int downsample,
const double samplingFrequency, const double rescanFrequency,
const int minSignalSize, const int maxSignalSize,
const string &logPath, const bool log, const bool gui) {
const string &logPath, const string &haarPath,
const string &dnnProtoPath, const string &dnnModelPath,
const bool log, const bool gui) {

this->algorithm = algorithm;
this->rPPGAlg = rPPGAlg;
this->faceDetAlg = faceDetAlg;
this->guiMode = gui;
this->lastSamplingTime = 0;
this->logMode = log;
Expand All @@ -46,12 +49,19 @@ bool RPPG::load(const rPPGAlgorithm algorithm,
this->samplingFrequency = samplingFrequency;
this->timeBase = timeBase;

// Load face detector
faceDetectNet = readNetFromCaffe("opencv/deploy.prototxt", "opencv/res10_300x300_ssd_iter_140000.caffemodel");
// Load classifier
switch (faceDetAlg) {
case haar:
haarClassifier.load(haarPath);
break;
case deep:
dnnClassifier = readNetFromCaffe(dnnProtoPath, dnnModelPath);
break;
}

// Setting up logfilepath
ostringstream path_1;
path_1 << logPath << "_a=" << algorithm << "_min=" << minSignalSize << "_max=" << maxSignalSize << "_ds=" << downsample;
path_1 << logPath << "_rppg=" << rPPGAlg << "_facedet=" << faceDetAlg << "_min=" << minSignalSize << "_max=" << maxSignalSize << "_ds=" << downsample;
this->logfilepath = path_1.str();

// Logging bpm according to sampling frequency
Expand Down Expand Up @@ -138,7 +148,7 @@ void RPPG::processFrame(Mat &frameRGB, Mat &frameGray, int time) {
if (s.rows >= fps * minSignalSize) {

// Filtering
switch (algorithm) {
switch (rPPGAlg) {
case g:
extractSignal_g();
break;
Expand Down Expand Up @@ -170,31 +180,37 @@ void RPPG::processFrame(Mat &frameRGB, Mat &frameGray, int time) {
void RPPG::detectFace(Mat &frameRGB, Mat &frameGray) {

cout << "Scanning for faces…" << endl;

// Detect faces with DNN
Mat resize300;
cv::resize(frameRGB, resize300, Size(300, 300));
Mat blob = blobFromImage(resize300, 1.0, Size(300, 300), Scalar(104.0, 177.0, 123.0));
faceDetectNet.setInput(blob);
Mat detection = faceDetectNet.forward();
Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
float confidenceThreshold = 0.5;
vector<Rect> boxes = {};

for (int i = 0; i < detectionMat.rows; i++) {
float confidence = detectionMat.at<float>(i, 2);

if (confidence > confidenceThreshold) {
switch (faceDetAlg) {
case haar:
// Detect faces with Haar classifier
haarClassifier.detectMultiScale(frameGray, boxes, 1.1, 2, CV_HAAR_SCALE_IMAGE, minFaceSize);
break;
case deep:
// Detect faces with DNN
Mat resize300;
cv::resize(frameRGB, resize300, Size(300, 300));
Mat blob = blobFromImage(resize300, 1.0, Size(300, 300), Scalar(104.0, 177.0, 123.0));
dnnClassifier.setInput(blob);
Mat detection = dnnClassifier.forward();
Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
float confidenceThreshold = 0.5;

for (int i = 0; i < detectionMat.rows; i++) {
float confidence = detectionMat.at<float>(i, 2);
if (confidence > confidenceThreshold) {
int xLeftBottom = static_cast<int>(detectionMat.at<float>(i, 3) * frameRGB.cols);
int yLeftBottom = static_cast<int>(detectionMat.at<float>(i, 4) * frameRGB.rows);
int xRightTop = static_cast<int>(detectionMat.at<float>(i, 5) * frameRGB.cols);
int yRightTop = static_cast<int>(detectionMat.at<float>(i, 6) * frameRGB.rows);

Rect object((int)xLeftBottom, (int)yLeftBottom,
(int)(xRightTop - xLeftBottom),
(int)(yRightTop - yLeftBottom));
boxes.push_back(object);
}
}
break;
}

if (boxes.size() > 0) {
Expand Down Expand Up @@ -647,4 +663,4 @@ void RPPG::draw(cv::Mat &frameRGB) {
line(frameRGB, Point(corners[i].x-5,corners[i].y), Point(corners[i].x+5,corners[i].y), GREEN, 1);
line(frameRGB, Point(corners[i].x,corners[i].y-5), Point(corners[i].x,corners[i].y+5), GREEN, 1);
}
}
}
15 changes: 10 additions & 5 deletions RPPG.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ using namespace dnn;
using namespace std;

enum rPPGAlgorithm { g, pca, xminay };
enum faceDetAlgorithm { haar, deep };

class RPPG {

Expand All @@ -30,11 +31,13 @@ class RPPG {
RPPG() {;}

// Load Settings
bool load(const rPPGAlgorithm algorithm,
bool load(const rPPGAlgorithm rPPGAlg, const faceDetAlgorithm faceDetAlg,
const int width, const int height, const double timeBase, const int downsample,
const double samplingFrequency, const double rescanFrequency,
const int minSignalSize, const int maxSignalSize,
const string &logPath, const bool log, const bool gui);
const string &logPath, const string &haarPath,
const string &dnnProtoPath, const string &dnnModelPath,
const bool log, const bool gui);

void processFrame(Mat &frameRGB, Mat &frameGray, int time);

Expand All @@ -59,10 +62,12 @@ class RPPG {
void log();

// The algorithm
rPPGAlgorithm algorithm;
rPPGAlgorithm rPPGAlg;

// The face detector
Net faceDetectNet;
// The classifier
faceDetAlgorithm faceDetAlg;
CascadeClassifier haarClassifier;
Net dnnClassifier;

// Settings
Size minFaceSize;
Expand Down
Loading

0 comments on commit 3a7f46f

Please sign in to comment.