Merge branch 'release/0.2.1'

* release/0.2.1: (111 commits) Fix hakabana path in the workshop Small fixes on hakabana workshop Add hakabana exercises to workshop Move hakabench install dest Fix code block highlighting in doc Add test for elasticsearch in cmake Fix library link private/public cmake flag Update doc with elasticsearch and kibana setup Fix private link library Add missing dependency against libhaka Add proper version to lib elasticsearch Update version number Update Kibana dashboard Improve code Remove install of removed file Fix elasticsearch issue when running in daemon Improve elasticsearch connection error message Move ips dashboard from sample to doc folder Add error handling on json object Add a kibana dashboard for alerts ...
haka-security · Sep 24, 2014 · 8c27453 · 8c27453
2 parents 2c4217e + 77e17cd
commit 8c27453
Show file tree

Hide file tree

Showing 162 changed files with 4,471 additions and 906 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -137,14 +137,16 @@ set(CMAKE_SWIG_FLAGS "-nomoduleglobal")
 
 # Install prefix
 message(STATUS "Installation prefix (CMAKE_INSTALL_PREFIX): ${CMAKE_INSTALL_PREFIX}")
+set(LOCALINSTALL "${CMAKE_BINARY_DIR}/out" CACHE STRING "Default localinstall dest")
 
-set(ENV_SH ${CMAKE_CURRENT_BINARY_DIR}/out/env.sh)
+set(ENV_SH ${LOCALINSTALL}/env.sh)
 
 add_custom_target(localinstall
-	COMMAND $(MAKE) install DESTDIR=${CMAKE_BINARY_DIR}/out
-	COMMAND echo "export LD_LIBRARY_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/out${CMAKE_INSTALL_PREFIX}/lib:${CMAKE_CURRENT_BINARY_DIR}/out${CMAKE_INSTALL_PREFIX}/lib/haka/modules/protocol\"" > ${ENV_SH}
-	COMMAND echo "export HAKA_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/out${CMAKE_INSTALL_PREFIX}\"" >> ${ENV_SH}
-	COMMAND echo "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/out${CMAKE_INSTALL_PREFIX}/bin\":\"${CMAKE_CURRENT_BINARY_DIR}/out${CMAKE_INSTALL_PREFIX}/sbin\":$PATH" >> ${ENV_SH}
+	COMMAND $(MAKE) install DESTDIR=${LOCALINSTALL}
+	COMMAND echo "export LD_LIBRARY_PATH=\"${LOCALINSTALL}${CMAKE_INSTALL_PREFIX}/lib:${LOCALINSTALL}${CMAKE_INSTALL_PREFIX}/lib/haka:${LOCALINSTALL}${CMAKE_INSTALL_PREFIX}/lib/haka/modules/protocol:${LOCALINSTALL}${CMAKE_INSTALL_PREFIX}/lib/haka/modules/packet:${LOCALINSTALL}${CMAKE_INSTALL_PREFIX}/lib/haka/modules/misc\"" > ${ENV_SH}
+	COMMAND echo "export HAKA_PATH=\"${LOCALINSTALL}${CMAKE_INSTALL_PREFIX}\"" >> ${ENV_SH}
+	COMMAND echo "export PATH=\"${LOCALINSTALL}${CMAKE_INSTALL_PREFIX}/bin\":\"${LOCALINSTALL}${CMAKE_INSTALL_PREFIX}/sbin\":$PATH" >> ${ENV_SH}
+	COMMAND echo "export LUA_PATH=\"\$(realpath \$HAKA_PATH/share/haka/lua/share/luajit* | head -n 1)/?.lua\"" >> ${ENV_SH}
 	WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
 	COMMENT "Install locally..." VERBATIM
 	VERBATIM

diff --git a/autobuild/build.sh b/autobuild/build.sh
@@ -1,4 +1,7 @@
 #! /bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 source autobuild/vars.sh
 source autobuild/includes.sh

diff --git a/autobuild/configure.sh b/autobuild/configure.sh
@@ -1,4 +1,7 @@
 #! /bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 source autobuild/vars.sh
 source autobuild/includes.sh

diff --git a/autobuild/coverage.sh b/autobuild/coverage.sh
@@ -1,4 +1,7 @@
 #! /bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 source autobuild/vars.sh
 source autobuild/includes.sh

diff --git a/autobuild/doc.sh b/autobuild/doc.sh
@@ -1,4 +1,7 @@
 #! /bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 source autobuild/vars.sh
 source autobuild/includes.sh

diff --git a/autobuild/includes.sh b/autobuild/includes.sh
@@ -1,5 +1,7 @@
 #! /bin/bash
-#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 _run() {
 	echo $*

diff --git a/autobuild/package.sh b/autobuild/package.sh
@@ -1,4 +1,7 @@
 #! /bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 source autobuild/vars.sh
 source autobuild/includes.sh

diff --git a/autobuild/staticanalysis.sh b/autobuild/staticanalysis.sh
@@ -1,4 +1,7 @@
 #! /bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 source autobuild/vars.sh
 source autobuild/includes.sh

diff --git a/autobuild/test.sh b/autobuild/test.sh
@@ -1,4 +1,7 @@
 #! /bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 source autobuild/vars.sh
 source autobuild/includes.sh

diff --git a/autobuild/vars.sh b/autobuild/vars.sh
@@ -1,4 +1,7 @@
 #! /bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 export PATH=/bin:/usr/bin:/usr/local/bin
 export ROOT=$(pwd)

diff --git a/build/FindJansson.cmake b/build/FindJansson.cmake
@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+find_path(JANSSON_INCLUDE_DIR jansson.h)
+find_library(JANSSON_LIBRARY NAMES jansson)
+
+if(JANSSON_INCLUDE_DIR AND JANSSON_LIBRARY)
+	set(JANSSON_FOUND)
+endif()
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(Jansson
+	REQUIRED_VARS JANSSON_LIBRARY JANSSON_INCLUDE_DIR)
diff --git a/build/FindLibCurl.cmake b/build/FindLibCurl.cmake
@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+find_path(LIBCURL_INCLUDE_DIR curl/curl.h)
+find_library(LIBCURL_LIBRARY NAMES curl)
+
+if(LIBCURL_INCLUDE_DIR AND LIBCURL_LIBRARY)
+	set(LIBCURL_FOUND)
+endif()
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(LibCurl
+	REQUIRED_VARS LIBCURL_LIBRARY LIBCURL_INCLUDE_DIR)
diff --git a/build/FindLibGeoIP.cmake b/build/FindLibGeoIP.cmake
@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+find_path(LIBGEOIP_INCLUDE_DIR GeoIP.h)
+find_library(LIBGEOIP_LIBRARY NAMES GeoIP)
+
+if(LIBGEOIP_INCLUDE_DIR AND LIBGEOIP_LIBRARY)
+	set(LIBGEOIP_FOUND)
+endif()
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(LibGeoIP
+	REQUIRED_VARS LIBGEOIP_LIBRARY LIBGEOIP_INCLUDE_DIR)
diff --git a/build/FindLibUuid.cmake b/build/FindLibUuid.cmake
@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+find_path(LIBUUID_INCLUDE_DIR uuid/uuid.h)
+find_library(LIBUUID_LIBRARY NAMES uuid)
+
+if(LIBUUID_INCLUDE_DIR AND LIBUUID_LIBRARY)
+	set(LIBUUID_FOUND)
+endif()
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(LibUuid
+	REQUIRED_VARS LIBUUID_LIBRARY LIBUUID_INCLUDE_DIR)
diff --git a/build/TestBenchRun.cmake b/build/TestBenchRun.cmake
@@ -0,0 +1,23 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+set(ENV{LANG} "C")
+set(ENV{LC_ALL} "C")
+set(ENV{BUILD_DIR} ${CTEST_MODULE_DIR})
+set(ENV{LUA_PATH} ${PROJECT_SOURCE_DIR}/src/lua/?.lua)
+set(ENV{HAKA_PATH} ${HAKA_PATH})
+set(ENV{PATH} $ENV{PATH}:${HAKA_PATH}/sbin:${HAKA_PATH}/bin)
+set(ENV{LD_LIBRARY_PATH} ${HAKA_PATH}/lib:${HAKA_PATH}/lib/haka/modules/protocol:${HAKA_PATH}/lib/haka/modules/packet)
+set(ENV{TZ} Europe/Paris)
+set(ENV{CONF} ${CONF})
+
+set(CMAKE_MODULE_PATH ${CTEST_MODULE_DIR} ${CMAKE_MODULE_PATH})
+
+message("Executing TZ=\"Europe/Paris\" LANG=\"C\" LC_ALL=\"C\" LUA_PATH=\"$ENV{LUA_PATH}\" HAKA_PATH=\"$ENV{HAKA_PATH}\" LD_LIBRARY_PATH=\"$ENV{LD_LIBRARY_PATH}\" CONF=\"$ENV{CONF}\" ${EXE} ${BENCH}")
+
+execute_process(COMMAND ${EXE} ${BENCH} RESULT_VARIABLE HAD_ERROR)
+
+if(HAD_ERROR)
+	message(FATAL_ERROR "Benchmark script failed")
+endif(HAD_ERROR)
diff --git a/doc/ref/hakamodule.rst b/doc/ref/hakamodule.rst
@@ -0,0 +1,11 @@
+.. This Source Code Form is subject to the terms of the Mozilla Public
+.. License, v. 2.0. If a copy of the MPL was not distributed with this
+.. file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+Haka modules
+============
+
+.. toctree::
+
+    ../../../modules/misc/geoip/doc/geoip.rst
+    ../../../modules/misc/elasticsearch/doc/elasticsearch.rst
diff --git a/doc/ref/refindex.rst b/doc/ref/refindex.rst
@@ -16,6 +16,7 @@ Welcome to Haka's User Reference Guide!
     alert.rst
     regexp.rst
     hakadissector.rst
+    hakamodule.rst
     rule.rst
     grammar.rst
     state_machine.rst

diff --git a/doc/theme/haka/static/haka.css_t b/doc/theme/haka/static/haka.css_t
@@ -220,11 +220,16 @@ li.toctree-l2 {
 	padding-left: 20px;
 }
 
+/* Admonition */
+div.admonition {
+	border-radius: 5px;
+}
+
 /* Exercise */
 div.admonition-exercise {
 	background: none repeat scroll 0 0 #f2f8ec;
     border: 1px solid #99CC66;
-    border-radius: 5px 5px 5px 5px;
+    border-radius: 5px;
 }
 
 div.admonition-exercise > p,

diff --git a/doc/user/tool_suite_haka.rst b/doc/user/tool_suite_haka.rst
@@ -128,6 +128,7 @@ Example
 
 .. literalinclude:: ../../sample/gettingstarted/gettingstarted.conf
     :tab-width: 4
+    :language: ini
 
 Service
 -------

diff --git a/doc/user/workshop.rst b/doc/user/workshop.rst
@@ -21,5 +21,6 @@ full environment containing Haka with all its dependencies.
     workshop/filter.rst
     workshop/modif.rst
     workshop/smtp.rst
+    workshop/hakabana.rst
     workshop/goingfurther.rst
     workshop/appendix.rst
diff --git a/doc/user/workshop/hakabana.rst b/doc/user/workshop/hakabana.rst
@@ -0,0 +1,71 @@
+.. This Source Code Form is subject to the terms of the Mozilla Public
+.. License, v. 2.0. If a copy of the MPL was not distributed with this
+.. file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+Hakabana
+========
+
+Hakabana is a monitoring tool that uses Kibana and Elasticsearch to visualize
+traffic passing through Haka in *real-time*. The package is already installed
+in the haka-live iso but can be downloaded directly from http://www.haka-security.org.
+
+Getting started
+^^^^^^^^^^^^^^^
+
+Hakabana module is installed at ``/usr/share/haka/modules/misc/hakabana``.
+It consists of a set of security rules that export network traffic to
+Elasticsearch server. They are then displayed thanks to our Kibana dashboard.
+
+Hakabana ships with a default configuration allowing starting quickly with
+traffic monitoring. It is available in ``/usr/share/haka/hakabana``
+
+.. admonition:: Exercise
+
+    * follow the instruction below to start haka:
+
+    .. code-block:: console
+
+        cd /usr/local/share/haka/hakabana
+        haka -c haka.conf
+
+    * visit the url: http://localhost/kibana/ and load hakabana dashboard
+      from ``/usr/share/haka/hakabana/dashboard/``
+
+I want more DNS info
+^^^^^^^^^^^^^^^^^^^^
+
+Your goal here is to customize the security rules in order to export extra data.
+
+.. admonition:: Exercise
+
+    * update the ``dns.lua`` in order to export dns types.
+
+    * add a panel to hakabana dashboard to display dns types.
+
+
+Geo localization
+^^^^^^^^^^^^^^^^
+
+Hakabana features a `geoip` module allowing to get the country code associated to an ip
+address. Here is an example using it:
+
+.. code-block:: lua
+
+    local ipv4 = require('protocol/ipv4')
+
+    local geoip_module = require('misc/geoip')
+    local geoip = geoip_module.open('/usr/share/GeoIP/GeoIP.dat')
+
+    haka.rule {
+        hook = ipv4.events.receive_packet,
+        eval = function (pkt)
+            local dst = pkt.dst
+            haka.log("geoip", "ip %s from %s",dst, geoip:country(dst))
+        end
+    }
+
+.. admonition:: Exercise
+
+    * update the ``flow.lua`` file in order to exclude traffic addressed to a given
+      country.
+
diff --git a/external/luajit/luajit.cmake b/external/luajit/luajit.cmake
@@ -58,21 +58,30 @@ add_custom_target(luajit
 )
 
 set(LUA_DIR ${CMAKE_BINARY_DIR}/${LUAJIT_DIR}/${INSTALL_FULLDIR})
-set(LUA_INCLUDE_DIR ${CMAKE_BINARY_DIR}/${LUAJIT_DIR}/${INSTALL_FULLDIR}/include/luajit-2.0)
+set(LUA_INCLUDE_DIR ${CMAKE_BINARY_DIR}/${LUAJIT_DIR}/src)
 set(LUA_LIBRARY_DIR ${CMAKE_BINARY_DIR}/${LUAJIT_DIR}/${INSTALL_FULLDIR}/lib/)
-set(LUA_LIBRARIES ${CMAKE_BINARY_DIR}/${LUAJIT_DIR}/${INSTALL_FULLDIR}/lib/libluajit-5.1.a)
+set(LUA_LIBRARIES ${CMAKE_BINARY_DIR}/${LUAJIT_DIR}/src/libluajit.a)
 
-set(LUA_COMPILER ${CMAKE_SOURCE_DIR}/external/luajit/luajitc -p "${CMAKE_BINARY_DIR}/${LUAJIT_DIR}/${INSTALL_FULLDIR}/")
+set(LUA_COMPILER ${CMAKE_SOURCE_DIR}/external/luajit/luajitc -p "${CMAKE_BINARY_DIR}/${LUAJIT_DIR}/src/")
 set(LUA_FLAGS_NONE "-g")
 set(LUA_FLAGS_DEBUG "-g")
 set(LUA_FLAGS_MEMCHECK "-g")
 set(LUA_FLAGS_RELEASE "-s")
 set(LUA_FLAGS_RELWITHDEBINFO "-g")
 set(LUA_FLAGS_MINSIZEREL "-s")
 
-install(DIRECTORY ${CMAKE_BINARY_DIR}/${LUAJIT_DIR}/${INSTALL_FULLDIR}/share/lua DESTINATION share/haka/lua/share)
-install(DIRECTORY ${CMAKE_BINARY_DIR}/${LUAJIT_DIR}/${INSTALL_FULLDIR}/share/luajit-2.0.3 DESTINATION share/haka/lua/share)
-install(DIRECTORY ${LUA_INCLUDE_DIR}/ DESTINATION include/haka/lua)
+install(DIRECTORY ${CMAKE_BINARY_DIR}/${LUAJIT_DIR}/${INSTALL_FULLDIR}/share/
+	DESTINATION share/haka/lua/share
+	PATTERN man* EXCLUDE
+)
+
+install(DIRECTORY ${CMAKE_BINARY_DIR}/${LUAJIT_DIR}/src/
+	DESTINATION include/haka/lua
+	FILES_MATCHING PATTERN *.h
+	PATTERN lj_* EXCLUDE
+	PATTERN host* EXCLUDE
+	PATTERN jit EXCLUDE
+)
 
 set(HAKA_LUAJIT 1)
 set(HAKA_LUA51 1)

diff --git a/external/luajit/luajitc b/external/luajit/luajitc
@@ -18,7 +18,6 @@ done
 
 shift $((OPTIND - 1))
 
-export LD_LIBRARY_PATH="$LUAJIT_PATH/lib"
-export LUA_PATH="$LUAJIT_PATH/share/luajit-2.0.3/?.lua"
+export LUA_PATH="$LUAJIT_PATH/?.lua"
 
-$LUAJIT_PATH/bin/luajit -b $OPTIONS $1 $OUTPUT
+$LUAJIT_PATH/luajit -b $OPTIONS $1 $OUTPUT
-Original file line number
+Diff line change
@@ Expand Up / @@ -128,6 +128,7 @@ Example @@
     .. literalinclude:: ../../sample/gettingstarted/gettingstarted.conf
         :tab-width: 4
+        :language: ini
     Service
     -------
@@ Expand Down @@