Big changes to the interface and API making it much easier to use:

- Separate recorder class for recording features computed by OpenFace - Separate image and sequence capture classes for easier loading of data - Capability to process features live - bug fixes with working with 16bit data - removing some code repetition
TadasBaltrusaitis · Dec 29, 2017 · c270549 · c270549
2 parents de4e351 + b30abc0
commit c270549
Show file tree

Hide file tree

Showing 202 changed files with 5,098 additions and 3,779 deletions.
diff --git a/.gitignore b/.gitignore
@@ -51,3 +51,12 @@ matlab_runners/Head Pose Experiments/experiments/ict_out/
 OpenFace\.VC\.db
 matlab_version/face_validation/vlfeat-0.9.20/
 matlab_version/face_validation/trained/intermediate/
+lib/local/GazeAnalyser/x64/
+lib/local/Utilities/x64/
+exe/FeatureExtraction/processed/
+exe/FaceLandmarkImg/processed/
+exe/FeatureExtraction/processed_features/
+matlab_runners/Demos/processed_features/
+*.db
+exe/releases/OpenFace_0.3.0_win_x64/
+exe/releases/OpenFace_0.3.0_win_x86/
diff --git a/.travis.yml b/.travis.yml
@@ -1,5 +1,8 @@
 language: cpp
 
+dist: trusty
+sudo: required
+
 branches:
   only:
     - master
@@ -9,32 +12,29 @@ compiler:
   - gcc
 
 os:
-  - osx
   - linux  
 
 before_install:
 
   # OpenCV dependencies and boost
   - if [ ${TRAVIS_OS_NAME} = linux ]; then  
+      sudo apt-get update;
+      sudo apt-get install libopenblas-dev;
       sudo apt-get install git libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev;      
       sudo apt-get install python-dev python-numpy libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libjasper-dev libdc1394-22-dev checkinstall;
-      sudo add-apt-repository -y ppa:boost-latest/ppa;
-      sudo apt-get update;
+      sudo apt-get install cmake;
       sudo apt-get install libboost1.55-all-dev;
       wget https://github.com/Itseez/opencv/archive/3.1.0.zip;
-      sudo unzip 3.1.0.zip;
+      unzip -q 3.1.0.zip;
       cd opencv-3.1.0;
-      sudo mkdir build;
+      mkdir build;
       cd build;
     fi
 
   # g++4.8.1
   - if [ "$CXX" = "g++" ]; then
       if [ ${TRAVIS_OS_NAME} = linux ]; then
-        sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; 
-        sudo apt-get update -qq;
-        sudo apt-get install -qq g++-4.8; 
-        export CXX="g++-4.8";		
+        $CXX --version;
         sudo cmake -D CMAKE_BUILD_TYPE=RELEASE -D WITH_V4L=ON -D WITH_OPENCL=OFF -D INSTALL_C_EXAMPLES=OFF -D BUILD_EXAMPLES=OFF -D BUILD_TESTS=OFF -D BUILD_PERF_TESTS=OFF -D BUILD_EXAMPLES=OFF -D INSTALL_PYTHON_EXAMPLES=OFF ..;
       fi 
     fi
@@ -43,16 +43,17 @@ before_install:
   - if [ "$CXX" = "clang++" ]; then
       if [ ${TRAVIS_OS_NAME} = linux ]; then  
         $CXX --version;
-        sudo sed -i -e 's/^Defaults\tsecure_path.*$//' /etc/sudoers;
         sudo cmake -D CMAKE_BUILD_TYPE=RELEASE -D WITH_V4L=ON -D WITH_OPENCL=OFF -D INSTALL_C_EXAMPLES=OFF -D BUILD_EXAMPLES=OFF -D BUILD_TESTS=OFF -D BUILD_PERF_TESTS=OFF -D BUILD_EXAMPLES=OFF -D INSTALL_PYTHON_EXAMPLES=OFF -D WITH_TBB=ON ..;
       fi
     fi
 
-  - if [ ${TRAVIS_OS_NAME} = osx ]; then
+  - if [ ${TRAVIS_OS_NAME} = osx ]; then  
       brew update;
       brew install tbb;
+      brew tap homebrew/science;
+      brew install vtk;
       wget https://github.com/Itseez/opencv/archive/3.1.0.zip;
-      sudo unzip 3.1.0.zip;
+      unzip -q 3.1.0.zip;
       cd opencv-3.1.0;
       sudo mkdir build;
       cd build;
@@ -70,9 +71,7 @@ script:
   - cd build
   - cmake -D CMAKE_BUILD_TYPE=RELEASE CMAKE_CXX_FLAGS="-std=c++11" -D CMAKE_EXE_LINKER_FLAGS="-std=c++11" .. 
   - make -j2
-  - ../build/bin/FaceLandmarkImg -fdir "../samples/" -ofdir "./demo_img/" -oidir "./demo_img/" -wild -q
-  - ../build/bin/FaceLandmarkImg -inroot ../samples -f sample1.jpg -outroot data -of sample1.txt -op sample1.3d -oi sample1.bmp -multi_view 1 -wild -q
-  - ../build/bin/FaceLandmarkVidMulti -inroot ../samples -f multi_face.avi -outroot output -ov multi_face.avi -q
-  - ../build/bin/FeatureExtraction -f "../samples/2015-10-15-15-14.avi" -outroot output_features -ov default.avi -of "default.txt" -simalign aligned -ov feat_test.avi -hogalign hog_test.dat -q
-  - ../build/bin/FeatureExtraction -f "../samples/2015-10-15-15-14.avi" -outroot output_features -simsize 200 -simscale 0.5 -ov default.avi -of "default.txt" -simalign aligned -ov feat_test.avi -hogalign hog_test.dat -q
-  - ../build/bin/FaceLandmarkVid -inroot ../samples -f 2015-10-15-15-14.avi -f multi_face.avi -outroot output_data -ov 1.avi -ov 2.avi -q
+  - ../build/bin/FaceLandmarkImg -fdir "../samples/" -out_dir "./demo_img"  -wild -q
+  - ../build/bin/FaceLandmarkImg -inroot ../samples -f sample1.jpg -out_dir data -of sample1.txt -multi_view 1 -wild -q
+  - ../build/bin/FeatureExtraction -f "../samples/2015-10-15-15-14.avi" -out_dir output -q
+  - ../build/bin/FeatureExtraction -f "../samples/2015-10-15-15-14.avi"  -simsize 200 -simscale 0.5 -q
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -13,7 +13,7 @@ find_package( BLAS REQUIRED )
 include_directories( ${BLAS_INCLUDE_DIRS} )
 LINK_DIRECTORIES(${BLAS_LIBRARY_DIRS})	
 
-find_package( OpenCV REQUIRED )
+find_package( OpenCV 3.1 REQUIRED )
 
 MESSAGE("OpenCV information:") 
 MESSAGE("  OpenCV_INCLUDE_DIRS: ${OpenCV_INCLUDE_DIRS}") 
@@ -191,6 +191,8 @@ add_subdirectory(lib/local/LandmarkDetector)
 add_subdirectory(lib/local/FaceAnalyser)
 # Gaze library
 add_subdirectory(lib/local/GazeAnalyser)
+# Utilities library
+add_subdirectory(lib/local/Utilities)
 # executables
 add_subdirectory(exe/FaceLandmarkImg)
 add_subdirectory(exe/FaceLandmarkVid)

diff --git a/OpenFace.sln b/OpenFace.sln
@@ -25,6 +25,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FaceLandmarkImg", "exe\Face
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GazeAnalyser", "lib\local\GazeAnalyser\GazeAnalyser.vcxproj", "{5F915541-F531-434F-9C81-79F5DB58012B}"
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Utilities", "lib\local\Utilities\Utilities.vcxproj", "{8E741EA2-9386-4CF2-815E-6F9B08991EAC}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Win32 = Debug|Win32
@@ -105,6 +107,14 @@ Global
 		{5F915541-F531-434F-9C81-79F5DB58012B}.Release|Win32.Build.0 = Release|Win32
 		{5F915541-F531-434F-9C81-79F5DB58012B}.Release|x64.ActiveCfg = Release|x64
 		{5F915541-F531-434F-9C81-79F5DB58012B}.Release|x64.Build.0 = Release|x64
+		{8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Debug|Win32.ActiveCfg = Debug|Win32
+		{8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Debug|Win32.Build.0 = Debug|Win32
+		{8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Debug|x64.ActiveCfg = Debug|x64
+		{8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Debug|x64.Build.0 = Debug|x64
+		{8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Release|Win32.ActiveCfg = Release|Win32
+		{8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Release|Win32.Build.0 = Release|Win32
+		{8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Release|x64.ActiveCfg = Release|x64
+		{8E741EA2-9386-4CF2-815E-6F9B08991EAC}.Release|x64.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -119,5 +129,6 @@ Global
 		{34032CF2-1B99-4A25-9050-E9C13DD4CD0A} = {9961DDAC-BE6E-4A6E-8EEF-FFC7D67BD631}
 		{DDC3535E-526C-44EC-9DF4-739E2D3A323B} = {9961DDAC-BE6E-4A6E-8EEF-FFC7D67BD631}
 		{5F915541-F531-434F-9C81-79F5DB58012B} = {99FEBA13-BDDF-4076-B57E-D8EF4076E20D}
+		{8E741EA2-9386-4CF2-815E-6F9B08991EAC} = {99FEBA13-BDDF-4076-B57E-D8EF4076E20D}
 	EndGlobalSection
 EndGlobal
diff --git a/README.md b/README.md
@@ -3,17 +3,19 @@
 [![Build Status](https://travis-ci.org/TadasBaltrusaitis/OpenFace.svg?branch=master)](https://travis-ci.org/TadasBaltrusaitis/OpenFace)
 [![Build status](https://ci.appveyor.com/api/projects/status/8msiklxfbhlnsmxp/branch/master?svg=true)](https://ci.appveyor.com/project/TadasBaltrusaitis/openface/branch/master)
 
-Over the past few years, there has been an increased interest in automatic facial behavior analysis and understanding. We present OpenFace – an open source tool intended for computer vision and machine learning researchers, affective computing community and people interested in building interactive applications based on facial behavior analysis. OpenFace is the ﬁrst open source tool capable of facial landmark detection, head pose estimation, facial action unit recognition, and eye-gaze estimation. The computer vision algorithms which represent the core of OpenFace demonstrate state-of-the-art results in all of the above mentioned tasks. Furthermore, our tool is capable of real-time performance and is able to run from a simple webcam without any specialist hardware.
+Over the past few years, there has been an increased interest in automatic facial behavior analysis and understanding. We present OpenFace – a tool intended for computer vision and machine learning researchers, affective computing community and people interested in building interactive applications based on facial behavior analysis. OpenFace is the ﬁrst toolkit capable of facial landmark detection, head pose estimation, facial action unit recognition, and eye-gaze estimation with available source code. The computer vision algorithms which represent the core of OpenFace demonstrate state-of-the-art results in all of the above mentioned tasks. Furthermore, our tool is capable of real-time performance and is able to run from a simple webcam without any specialist hardware.
 
-The code was written mainly by Tadas Baltrusaitis during his time at the Language Technologies Institute at the Carnegie Mellon University; Computer Laboratory, University of Cambridge; and Institute for Creative Technologies, University of Southern California.
+![Multicomp logo](https://github.com/TadasBaltrusaitis/OpenFace/blob/master/imgs/muticomp_logo_black.png)
 
-Special thanks goes to Louis-Philippe Morency and his MultiComp Lab at Institute for Creative Technologies for help in writing and testing the code, and Erroll Wood for the gaze estimation work.
+![Rainbow logo](https://github.com/TadasBaltrusaitis/OpenFace/blob/master/imgs/rainbow-logo.gif)
+
+OpenFace is an implementation of a number of research papers from the Multicomp group, Language Technologies Institute at the Carnegie Mellon University and Rainbow Group, Computer Laboratory, University of Cambridge. The founder of the project and main developer is Tadas Baltrušaitis.
+
+Special thanks goes to Louis-Philippe Morency and his MultiComp Lab at Carnegie Mellon University for help in writing and testing the code, and Erroll Wood for the gaze estimation work.
 
 ## WIKI
 **For instructions of how to install/compile/use the project please see [WIKI](https://github.com/TadasBaltrusaitis/OpenFace/wiki)**
 
-More details about the project - http://www.cl.cam.ac.uk/research/rainbow/projects/openface/
-
 ## Functionality
 
 The system is capable of performing a number of facial analysis tasks:
@@ -68,17 +70,18 @@ Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson
 in *Facial Expression Recognition and Analysis Challenge*, 
 *IEEE International Conference on Automatic Face and Gesture Recognition*, 2015 
 
-# Copyright
-
-Copyright can be found in the Copyright.txt
-
-You have to respect boost, TBB, dlib, OpenBLAS, and OpenCV licenses.
-
 # Commercial license
 
 For inquiries about the commercial licensing of the OpenFace toolkit please contact [email protected]
 
 # Final remarks
 
-I did my best to make sure that the code runs out of the box but there are always issues and I would be grateful for your understanding that this is research code and not full fledged product. However, if you encounter any problems/bugs/issues please contact me on github or by emailing me at [email protected] for any bug reports/questions/suggestions. 
+I did my best to make sure that the code runs out of the box but there are always issues and I would be grateful for your understanding that this is research code and not full fledged product. However, if you encounter any problems/bugs/issues please contact me on github or by emailing me at [email protected] for any bug reports/questions/suggestions. I prefer questions and bug reports on github as that provides visibility to others who might be encountering same issues or who have the same questions.
+
+# Copyright
+
+Copyright can be found in the Copyright.txt
+
+You have to respect boost, TBB, dlib, OpenBLAS, and OpenCV licenses.
 
+Furthermore you have to respect the licenses of the datasets used for model training - https://github.com/TadasBaltrusaitis/OpenFace/wiki/Datasets
diff --git a/appveyor.yml b/appveyor.yml
@@ -6,7 +6,6 @@ branches:
   - /^feature-.*$/
 max_jobs: 4  
 configuration:
-- Debug
 - Release
 platform:
 - x64
@@ -21,9 +20,9 @@ test_script:
   - cmd: if exist Debug (cd Debug)
   - cmd: if exist Release (cd Release)
   - cmd: dir
-  - cmd: if exist "../samples" (FaceLandmarkImg.exe -fdir "../samples/" -ofdir "./demo_img/" -oidir "./demo_img/" -wild -q) else (FaceLandmarkImg.exe -fdir "../../samples/" -ofdir "./demo_img/" -oidir "./demo_img/" -wild -q)
-  - cmd: if exist "../samples" (FaceLandmarkImg.exe -inroot ../samples -f sample1.jpg -outroot out_data -of sample1.pts -op sample1.3d -oi sample1.bmp -q) else (FaceLandmarkImg.exe -inroot ../../samples -f sample1.jpg -outroot out_data -of sample1.pts -op sample1.3d -oi sample1.bmp -q)
-  - cmd: if exist "../samples" (FaceLandmarkVidMulti.exe -inroot ../samples -f multi_face.avi -ov multi_face.avi -q) else (FaceLandmarkVidMulti.exe -inroot ../../samples -f multi_face.avi -ov multi_face.avi -q)
-  - cmd: if exist "../samples" (FeatureExtraction.exe -f "../samples/default.wmv" -outroot output_features -of "default.txt" -simalign aligned -ov feat_track.avi -hogalign hog_test.dat -q) else (FeatureExtraction.exe -f "../../samples/default.wmv" -outroot output_features -of "default.txt" -simalign aligned -ov feat_track.avi -hogalign hog_test.dat -q)
-  - cmd: if exist "../samples" (FeatureExtraction.exe -f "../samples/default.wmv" -outroot output_features -of "default.txt" -simalign aligned -simsize 200 -simscale 0.5 -ov feat_track.avi -hogalign hog_test.dat -q) else (FeatureExtraction.exe -f "../../samples/default.wmv" -outroot output_features -of "default.txt" -simalign aligned -simsize 200 -simscale 0.5 -ov feat_track.avi -hogalign hog_test.dat -q)
-  - cmd: if exist "../samples" (FaceLandmarkVid.exe -f "../samples/default.wmv" -ov track.avi -q) else (FaceLandmarkVid.exe -f "../../samples/default.wmv" -ov track.avi -q)
+  - cmd: if exist "../samples" (FaceLandmarkImg.exe -fdir "../samples/" -out_dir "./demo_img/" -wild -q) else (FaceLandmarkImg.exe -fdir "../../samples/" -out_dir "./demo_img/" -wild -q)
+  - cmd: if exist "../samples" (FaceLandmarkImg.exe -inroot ../samples -f sample1.jpg -out_dir out_data -of sample1.pts -q) else (FaceLandmarkImg.exe -inroot ../../samples -f sample1.jpg -out_dir out_data -q)
+  - cmd: if exist "../samples" (FaceLandmarkVidMulti.exe -inroot ../samples -f multi_face.avi -q) else (FaceLandmarkVidMulti.exe -inroot ../../samples -f multi_face.avi -q)
+  - cmd: if exist "../samples" (FeatureExtraction.exe -f "../samples/default.wmv" -q) else (FeatureExtraction.exe -f "../../samples/default.wmv" -q)
+  - cmd: if exist "../samples" (FeatureExtraction.exe -f "../samples/default.wmv" -simsize 200 -simscale 0.5 -q) else (FeatureExtraction.exe -f "../../samples/default.wmv" -simsize 200 -simscale 0.5 -ov feat_track.avi -q)
+  - cmd: if exist "../samples" (FaceLandmarkVid.exe -f "../samples/default.wmv"  -q) else (FaceLandmarkVid.exe -f "../../samples/default.wmv" -q)
diff --git a/exe/FaceLandmarkImg/CMakeLists.txt b/exe/FaceLandmarkImg/CMakeLists.txt
@@ -7,11 +7,13 @@ include_directories(${LandmarkDetector_SOURCE_DIR}/include)
 include_directories(../../lib/local/LandmarkDetector/include)
 include_directories(../../lib/local/FaceAnalyser/include)	
 include_directories(../../lib/local/GazeAnalyser/include)	
+include_directories(../../lib/local/Utilities/include)	
 
 add_executable(FaceLandmarkImg FaceLandmarkImg.cpp)
 target_link_libraries(FaceLandmarkImg LandmarkDetector)
 target_link_libraries(FaceLandmarkImg FaceAnalyser)
 target_link_libraries(FaceLandmarkImg GazeAnalyser)
+target_link_libraries(FaceLandmarkImg Utilities)
 target_link_libraries(FaceLandmarkImg dlib)
 
 target_link_libraries(FaceLandmarkImg ${OpenCV_LIBS} ${Boost_LIBRARIES} ${TBB_LIBRARIES}  ${BLAS_LIBRARIES})