Fixes skupperproject#1613: Added fuzz testing for the http2 decoder. …

…Fixes for two issues that the fuzz tester found.

CMakeLists.txt

@@ -45,6 +45,7 @@ set(CMAKE_ENABLE_EXPORTS TRUE)
 option(CMAKE_INTERPROCEDURAL_OPTIMIZATION "Perform link time optimization" ON)
 option(ENABLE_WARNING_ERROR "Consider compiler warnings to be errors" ON)
 option(ENABLE_PROFILE_GUIDED_OPTIMIZATION "Perform profile guided optimization" OFF)
+option(ENABLE_FUZZ_TESTING "Enable building fuzzers and regression testing with libFuzzer" ON)
 
 # preserve frame pointers for ease of debugging and profiling
 #  see https://fedoraproject.org/wiki/Changes/fno-omit-frame-pointer

pom.xml

@@ -108,6 +108,8 @@
                       <exclude>share/index.html</exclude>
                       <exclude>scripts/git-clang-format</exclude>
                       <exclude>tests/nginx/**</exclude>
+                      <exclude>tests/fuzz/fuzz_http2_decoder/**</exclude>
+                      <exclude>tests/fuzz/fuzz_http1_decoder/**</exclude>
                 </excludes>
         </configuration>
             </plugin>

run.py.in

@@ -70,6 +70,7 @@ env_vars = {
     'PYTHONPATH': os.pathsep.join(python_path),
     'PATH': os.pathsep.join(dedup(["${CMAKE_BINARY_DIR}",
                                    os.path.join("${CMAKE_BINARY_DIR}", 'tests'),
+                                   os.path.join("${CMAKE_BINARY_DIR}", 'tests/fuzz'),
                                    os.path.join("${CMAKE_BINARY_DIR}", 'router'),
                                    os.path.join("${CMAKE_SOURCE_DIR}", 'tools'),
                                    os.path.join("${CMAKE_BINARY_DIR}", 'tools'),

src/decoders/http2/http2_decoder.c

@@ -126,6 +126,12 @@ qd_http2_frame_type get_frame_type(const uint8_t frame_type)
     return FRAME_TYPE_OTHER;
 }
 
+uint8_t get_pad_length(const uint8_t *data)
+{
+    uint8_t pad_length = (uint8_t) ((data)[0]);
+    return pad_length;
+}
+
 uint32_t get_stream_identifier(const uint8_t *data)
 {
     uint32_t stream_id = (((uint32_t) (data)[0]) << 24) | (((uint32_t) (data)[1]) << 16) | (((uint32_t) (data)[2]) << 8) | ((uint32_t) (data)[3]);
@@ -258,6 +264,7 @@ static int decompress_headers(qd_http2_decoder_t *decoder, uint32_t stream_id, c
         nghttp2_nv nv;
         int inflate_flags = 0;
         size_t proclen;
+        fflush(stdout);
         int rv = nghttp2_hd_inflate_hd2(decoder->inflater, &nv, &inflate_flags, data, length, 1);
         if (rv < 0) {
             qd_log(LOG_HTTP2_DECODER, QD_LOG_ERROR, "[C%"PRIu64"] decompress_headers - decompression error rv=%i", decoder->conn_state->conn_id, rv);
@@ -321,15 +328,17 @@ bool get_request_headers(qd_http2_decoder_t *decoder)
     qd_log(LOG_HTTP2_DECODER, QD_LOG_DEBUG, "[C%"PRIu64"] get_request_headers - end_stream=%i, end_headers=%i, is_padded=%i, has_priority=%i, stream_id=%" PRIu32, decoder->conn_state->conn_id, end_stream, end_headers, is_padded, has_priority, stream_id);
     scratch_buffer->offset += HTTP2_FRAME_STREAM_ID_LENGTH;
     decoder->frame_length_processed += HTTP2_FRAME_STREAM_ID_LENGTH;
-
+    uint8_t pad_length = 0;
     if(is_padded) {
+        pad_length = get_pad_length(scratch_buffer->bytes + scratch_buffer->offset);
         // Move one byte to account for pad length
         scratch_buffer->offset += 1;
         decoder->frame_length_processed += 1;
     }
 
     if(has_priority) {
         // Skip the Stream Dependency field if the priority flag is set
+        // Stream Dependency field is 4 octets.
         scratch_buffer->offset += 4;
         decoder->frame_length_processed += 4;
 
@@ -338,6 +347,29 @@ bool get_request_headers(qd_http2_decoder_t *decoder)
         decoder->frame_length_processed += 1;
     }
 
+    //
+    // Before the call to decompress_headers(), we need to make sure that there is some data left in the scratch buffer before we decompress it.
+    //
+    int buffer_data_size = scratch_buffer->size - scratch_buffer->offset;
+    printf("get_request_headers buffer_data_size=%i\n", buffer_data_size);
+    int contains_pad_length = scratch_buffer->size - pad_length;
+    int pad_length_offset = scratch_buffer->size - pad_length - scratch_buffer->offset;
+    bool valid_pad_length = contains_pad_length > 0;
+    bool valid_buffer_data = buffer_data_size > 0;
+    bool valid_pad_length_offset = pad_length_offset > 0;
+    if(decoder->frame_payload_length == 0 || !valid_buffer_data || !valid_pad_length || !valid_pad_length_offset) {
+        qd_log(LOG_HTTP2_DECODER, QD_LOG_DEBUG, "[C%"PRIu64"] get_request_headers - failure, moving decoder state to HTTP2_DECODE_ERROR", decoder->conn_state->conn_id);
+        static char error[130];
+        snprintf(error, sizeof(error), "get_request_headers - either request or response header was received with zero payload or contains bogus data, stopping decoder");
+        reset_decoder_frame_info(decoder);
+        reset_scratch_buffer(&decoder->scratch_buffer);
+        parser_error(decoder, error);
+        return false;
+    }
+
+    // Take out the padding bytes from the end of the scratch buffer
+    scratch_buffer->size = scratch_buffer->size - pad_length;
+
     // We are now finally at a place which matters to us - The Header block fragment. We will look thru and decompress it so we can get the request/response headers.
     int rv = decompress_headers(decoder, stream_id, scratch_buffer->bytes + scratch_buffer->offset, scratch_buffer->size - scratch_buffer->offset);
     if(rv < 0) {
@@ -380,11 +412,13 @@ static bool parse_request_header(qd_http2_decoder_t *decoder, const uint8_t **da
             parser_error(decoder, "scratch buffer size exceeded 65535 bytes, stopping decoder");
             return false;
         }
-        decoder->frame_length_processed += bytes_to_copy;
+        //decoder->frame_length_processed += bytes_to_copy;
         *data += bytes_to_copy;
         *length -= bytes_to_copy;
-        if(decoder->frame_length_processed == decoder->frame_length) {
-            get_request_headers(decoder);
+        if((decoder->frame_length_processed + bytes_to_copy) == decoder->frame_length) {
+            bool header_success = get_request_headers(decoder);
+            if(!header_success)
+                return false;
         }
         if (*length > 0) {
             return true; // More bytes remain to be processed, continue processing.
@@ -420,7 +454,7 @@ static bool parse_frame_header(qd_http2_decoder_t *decoder, const uint8_t **data
 {
     if (*length == 0)
         return false;
-    //
+
     // First check to see if there is anything in the scratch buffer.
     // The strategy is to copy ONLY the first 4 bytes of every frame header to the scratch buffer.
     //

tests/CMakeLists.txt

@@ -40,7 +40,6 @@ set(unit_test_SOURCES
     hash_test.c
     thread_test.c
     platform_test.c
-    static_assert_test.c
     )
 
 add_executable(unit_tests ${unit_test_SOURCES})
@@ -253,3 +252,7 @@ add_subdirectory(cpp)
 if(BUILD_BENCHMARKS)
   add_subdirectory(c_benchmarks)
 endif()
+
+if (ENABLE_FUZZ_TESTING)
+  add_subdirectory(fuzz)
+endif (ENABLE_FUZZ_TESTING)

tests/fuzz/CMakeLists.txt

@@ -0,0 +1,56 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+add_definitions(${C_STANDARD_FLAGS} ${COMPILE_WARNING_FLAGS})
+
+option(FUZZ_REGRESSION_TESTS "Run fuzz tests with regression test driver" ON)
+option(FUZZ_LONG_TESTS "Run fuzz tests that take a long time" OFF)
+set(FUZZER LibFuzzer CACHE STRING "Fuzzing engine to use")
+set(FUZZING_LIB_LibFuzzer FuzzingEngine)
+set(FUZZING_LIB_AFL -fsanitize=fuzzer)
+
+add_library(StandaloneFuzzTargetMain STATIC StandaloneFuzzTargetMain.c StandaloneFuzzTargetInit.c)
+
+if (FUZZ_REGRESSION_TESTS)
+  message(STATUS "FUZZ_REGRESSION_TESTS")
+  set(FUZZING_LIBRARY StandaloneFuzzTargetMain)
+else ()
+  message(STATUS "NO FUZZ_REGRESSION_TESTS")
+  set(FUZZING_LIBRARY ${FUZZING_LIB_${FUZZER}})
+endif ()
+
+macro(add_fuzz_test test)
+  add_executable (${test} ${ARGN})
+  target_link_libraries (${test} ${FUZZING_LIBRARY} skupper-router)
+  set_target_properties(fuzz_http2_decoder PROPERTIES LINKER_LANGUAGE CXX)
+
+  if(FUZZ_REGRESSION_TESTS)
+    file(GLOB_RECURSE files ${CMAKE_CURRENT_SOURCE_DIR}/${test}/*)
+    unset(file_lines)
+    foreach(f IN LISTS files)
+      set(file_lines "${file_lines}${f}\n")
+    endforeach()
+    file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${test}-files" "${file_lines}")
+    add_test(${test} ${TEST_WRAP} ${test} "@${CMAKE_CURRENT_BINARY_DIR}/${test}-files")
+  else(FUZZ_REGRESSION_TESTS)
+    add_test(${test} ${TEST_WRAP} ${test} "-runs=1 ${CMAKE_CURRENT_SOURCE_DIR}/${test}")
+  endif(FUZZ_REGRESSION_TESTS)
+endmacro(add_fuzz_test test)
+
+add_fuzz_test(fuzz_http2_decoder fuzz_http2_decoder.c)
+#add_fuzz_test(fuzz_http1_decoder fuzz_http1_decoder.c)

tests/fuzz/Containerfile

@@ -0,0 +1,54 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+FROM gcr.io/oss-fuzz-base/base-builder
+RUN apt-get update 
+
+# Ensure we work from right python version
+# Minimum python version required by qpid-proton and skupper-router is Python 3.9
+RUN apt-get install -y python3.9 python3.9-dev && \
+    ln --force -s /usr/bin/python3.9 /usr/local/bin/python3 && \
+    apt-get install -y python3-pip
+RUN apt-get install -y libuv1-dev wget cmake emacs python3-dev libwebsockets-dev libtool zlib1g-dev cmake libsasl2-dev libssl-dev sasl2-bin libnghttp2-dev
+
+# LibwebSockets library is required by skupper-router
+RUN git clone https://github.com/warmcat/libwebsockets.git --branch v4.3-stable
+WORKDIR /src
+RUN mkdir libwebsockets/build && cd libwebsockets/build && cmake .. -DLWS_LINK_TESTAPPS_DYNAMIC=ON -DLWS_WITH_LIBUV=OFF -DLWS_WITHOUT_BUILTIN_GETIFADDRS=ON -DLWS_WITHOUT_BUILTIN_SHA1=ON -DLWS_WITH_STATIC=OFF -DLWS_IPV6=ON -DLWS_WITH_HTTP2=OFF -DLWS_WITHOUT_CLIENT=OFF -DLWS_WITHOUT_SERVER=OFF -DLWS_WITHOUT_TESTAPPS=ON -DLWS_WITHOUT_TEST_SERVER=ON -DLWS_WITHOUT_TEST_SERVER_EXTPOLL=ON -DLWS_WITHOUT_TEST_PING=ON -DLWS_WITHOUT_TEST_CLIENT=ON  && make install
+
+RUN git clone https://github.com/apache/qpid-proton.git
+WORKDIR /src/qpid-proton
+RUN mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=OFF -DENABLE_LINKTIME_OPTIMIZATION=OFF -DBUILD_TLS=ON -DSSL_IMPL=openssl -DBUILD_TOOLS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_TESTING=OFF && make install
+
+WORKDIR /src
+RUN git clone https://github.com/ganeshmurthy/skupper-router.git --branch FUZZ-TESTING
+
+WORKDIR /src/skupper-router
+
+# refresh the build directory if it exists already
+RUN rm build -rf || true
+
+# /usr/local/bin/compile compiles libFuzzer or AmericanFuzzyLop(afl), then calls /src/build.sh and sets correct environment variables for it
+RUN echo cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=OFF -DFUZZ_REGRESSION_TESTS=OFF -DCMAKE_C_FLAGS=-DQD_MEMORY_DEBUG -DRUNTIME_CHECK=asan > /src/build.sh
+
+# build and  run the test. Choose AFL for fuzzer
+RUN mkdir build
+WORKDIR /src/skupper-router/build
+RUN export FUZZING_LANGUAGE='' && export FUZZING_ENGINE=afl && /usr/local/bin/compile
+WORKDIR /src/skupper-router/build/tests/fuzz
+RUN make
+ENTRYPOINT LD_LIBRARY_PATH=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/ AFL_MAP_SIZE=10000000 AFL_DEBUG=1 AFL_SKIP_CPUFREQ=1 AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 make test; bash
+#CMD ["/bin/bash"]

tests/fuzz/StandaloneFuzzTargetInit.c

@@ -0,0 +1,36 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+#include <qpid/dispatch/alloc_pool.h>
+#include <qpid/dispatch/log.h>
+
+void qd_log_initialize(void);
+void qd_error_initialize(void);
+
+#include "libFuzzingEngine.h"
+
+int LLVMFuzzerInitialize(int *argc, char ***argv)
+{
+    qd_alloc_initialize();
+    qd_log_initialize();
+    qd_error_initialize();
+    return 0;
+}
+