Debug 2025

common/fuzzer_utils.py

@@ -72,6 +72,12 @@ def dockerfiles(self):
 def get_fuzz_target_binary(search_directory: str,
                            fuzz_target_name: str) -> Optional[str]:
     """Return target binary path."""
+    logs.info(f'Searching for fuzz target binary named {fuzz_target_name} under'
+              f' directory {search_directory}')
+    logs.info(f'Search diretory {os.path.abspath(search_directory)} exists: '
+              f'{os.path.exists(os.path.abspath(search_directory))}')
+    logs.info(f'list Search diretory {search_directory}: '
+              f'{os.listdir(search_directory)}')
     if fuzz_target_name:
         fuzz_target_binary = os.path.join(search_directory, fuzz_target_name)
         if os.path.exists(fuzz_target_binary):
@@ -83,7 +89,11 @@ def get_fuzz_target_binary(search_directory: str,
     if os.path.exists(default_fuzz_target_binary):
         return default_fuzz_target_binary
 
+    logs.info('Searching for possible fuzz target in search directory: '
+              f'{search_directory}')
     for root, _, files in os.walk(search_directory):
+        logs.info(f'Searching for possible fuzz target under subdir {root}: '
+                  f'{files}')
         if root == 'uninstrumented':
             continue
         for filename in files:

experiment/build/gcb_build.py

@@ -105,7 +105,8 @@ def _build(
         # TODO(metzman): Refactor code so that local_build stores logs as well.
         build_utils.store_build_logs(config_name, result)
         if result.retcode != 0:
-            logs.error('%s failed.', command)
+            logs.error('%s failed. Return code: %d. Output: %s. Timedout: %s',
+                       command, result.retcode, result.output, result.timed_out)
             raise subprocess.CalledProcessError(result.retcode, command)
     return result
 

experiment/measurer/coverage_utils.py

@@ -133,7 +133,9 @@ def merge_profdata_files(self):
 
         result = merge_profdata_files(files_to_merge, self.merged_profdata_file)
         if result.retcode != 0:
-            logger.error('Profdata files merging failed.')
+            logger.error(
+                f'Profdata files merging failed for (fuzzer, benchmark): '
+                f'({self.fuzzer}, {self.benchmark}).')
 
     def generate_coverage_summary_json(self):
         """Generates the coverage summary json from merged profdata file."""
@@ -224,6 +226,10 @@ def get_trial_ids(experiment: str, fuzzer: str, benchmark: str):
 
 def merge_profdata_files(src_files, dst_file):
     """Uses llvm-profdata to merge |src_files| to |dst_files|."""
+    if src_files:
+        logger.info('llvm-profdata src files are: %s', src_files)
+    else:
+        logger.error('llvm-profdata has no src files: %s', src_files)
     command = ['llvm-profdata', 'merge', '-sparse']
     command.extend(src_files)
     command.extend(['-o', dst_file])

experiment/measurer/measure_manager.py

@@ -436,11 +436,22 @@ def get_current_coverage(self) -> int:
 
     def generate_profdata(self, cycle: int):
         """Generate .profdata file from .profraw file."""
+        self.logger.info('Listing all files under %s to find profraw files: %s',
+                         self.coverage_dir, os.listdir(self.coverage_dir))
+        candiates = {
+            f: os.path.getsize(f)
+            for f in glob.glob(self.profraw_file_pattern.replace('%m', '*'))
+        }
+        self.logger.info('Candidate profraw files are: %s', candiates)
         files_to_merge = self.get_profraw_files()
         if os.path.isfile(self.profdata_file):
             # If coverage profdata exists, then merge it with
             # existing available data.
             files_to_merge += [self.profdata_file]
+            self.logger.info('profdata_file does exist: %s', self.profdata_file)
+        else:
+            self.logger.warning('profdata_file does not exist: %s',
+                                self.profdata_file)
 
         result = coverage_utils.merge_profdata_files(files_to_merge,
                                                      self.profdata_file)

experiment/measurer/run_coverage.py

@@ -44,6 +44,7 @@ def do_coverage_run(  # pylint: disable=too-many-locals
     """Does a coverage run of |coverage_binary| on |new_units_dir|. Writes
     the result to |profraw_file_pattern|."""
     with tempfile.TemporaryDirectory() as merge_dir:
+        logger.info("coverage binary exists?: %s", os.path.exists(os.path.abspath(coverage_binary)))
         command = [
             coverage_binary, '-merge=1', '-dump_coverage=1',
             f'-artifact_prefix={crashes_dir}/', f'-timeout={UNIT_TIMEOUT}',
@@ -61,6 +62,8 @@ def do_coverage_run(  # pylint: disable=too-many-locals
                                      expect_zero=False,
                                      kill_children=True,
                                      timeout=MAX_TOTAL_TIME)
+
+        logger.info("coverage dir after coverage run %s", os.listdir(coverage_binary_dir))
 
     if result.retcode != 0:
         logger.error('Coverage run failed.',

experiment/runner.py

@@ -177,10 +177,13 @@ def run_fuzzer(max_total_time, log_filename):
     input_corpus = environment.get('SEED_CORPUS_DIR')
     output_corpus = os.environ['OUTPUT_CORPUS_DIR']
     fuzz_target_name = environment.get('FUZZ_TARGET')
+    logs.info('all ENV VAR '
+              f'{[f"{key}: {value}" for key, value in os.environ.items()]}')
     target_binary = fuzzer_utils.get_fuzz_target_binary(FUZZ_TARGET_DIR,
                                                         fuzz_target_name)
     if not target_binary:
-        logs.error('Fuzz target binary not found.')
+        logs.error(f'Fuzz target binary {fuzz_target_name} not found under '
+                   f'{FUZZ_TARGET_DIR}')
         return
 
     if max_total_time is None:

fuzzers/afl_empty_seeds/builder.Dockerfile

@@ -0,0 +1,33 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG parent_image
+FROM $parent_image
+
+# Download and compile AFL v2.57b.
+# Set AFL_NO_X86 to skip flaky tests.
+RUN git clone \
+        --depth 1 \
+        --branch v2.57b \
+        https://github.com/google/AFL.git /afl && \
+    cd /afl && \
+    CFLAGS= CXXFLAGS= AFL_NO_X86=1 make
+
+# Use afl_driver.cpp from LLVM as our fuzzing library.
+RUN apt-get update && \
+    apt-get install wget -y && \
+    wget https://raw.githubusercontent.com/llvm/llvm-project/5feb80e748924606531ba28c97fe65145c65372e/compiler-rt/lib/fuzzer/afl/afl_driver.cpp -O /afl/afl_driver.cpp && \
+    clang -Wno-pointer-sign -c /afl/llvm_mode/afl-llvm-rt.o.c -I/afl && \
+    clang++ -stdlib=libc++ -std=c++11 -O2 -c /afl/afl_driver.cpp && \
+    ar r /libAFL.a *.o

fuzzers/afl_empty_seeds/fuzzer.py

@@ -0,0 +1,152 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Integration code for AFL fuzzer."""
+
+import json
+import os
+import shutil
+import subprocess
+
+from fuzzers import utils
+
+
+def prepare_build_environment():
+    """Set environment variables used to build targets for AFL-based
+    fuzzers."""
+    cflags = ['-fsanitize-coverage=trace-pc-guard']
+    utils.append_flags('CFLAGS', cflags)
+    utils.append_flags('CXXFLAGS', cflags)
+
+    os.environ['CC'] = 'clang'
+    os.environ['CXX'] = 'clang++'
+    os.environ['FUZZER_LIB'] = '/libAFL.a'
+
+
+def build():
+    """Build benchmark."""
+    prepare_build_environment()
+
+    utils.build_benchmark()
+
+    print('[post_build] Copying afl-fuzz to $OUT directory')
+    # Copy out the afl-fuzz binary as a build artifact.
+    shutil.copy('/afl/afl-fuzz', os.environ['OUT'])
+
+
+def get_stats(output_corpus, fuzzer_log):  # pylint: disable=unused-argument
+    """Gets fuzzer stats for AFL."""
+    # Get a dictionary containing the stats AFL reports.
+    stats_file = os.path.join(output_corpus, 'fuzzer_stats')
+    if not os.path.exists(stats_file):
+        print('Can\'t find fuzzer_stats')
+        return '{}'
+    with open(stats_file, encoding='utf-8') as file_handle:
+        stats_file_lines = file_handle.read().splitlines()
+    stats_file_dict = {}
+    for stats_line in stats_file_lines:
+        key, value = stats_line.split(': ')
+        stats_file_dict[key.strip()] = value.strip()
+
+    # Report to FuzzBench the stats it accepts.
+    stats = {'execs_per_sec': float(stats_file_dict['execs_per_sec'])}
+    return json.dumps(stats)
+
+
+def prepare_fuzz_environment(input_corpus):
+    """Prepare to fuzz with AFL or another AFL-based fuzzer."""
+    # Tell AFL to not use its terminal UI so we get usable logs.
+    os.environ['AFL_NO_UI'] = '1'
+    # Skip AFL's CPU frequency check (fails on Docker).
+    os.environ['AFL_SKIP_CPUFREQ'] = '1'
+    # No need to bind affinity to one core, Docker enforces 1 core usage.
+    os.environ['AFL_NO_AFFINITY'] = '1'
+    # AFL will abort on startup if the core pattern sends notifications to
+    # external programs. We don't care about this.
+    os.environ['AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES'] = '1'
+    # Don't exit when crashes are found. This can happen when corpus from
+    # OSS-Fuzz is used.
+    os.environ['AFL_SKIP_CRASHES'] = '1'
+    # Shuffle the queue
+    os.environ['AFL_SHUFFLE_QUEUE'] = '1'
+
+    # AFL needs at least one non-empty seed to start.
+    utils.create_seed_file_for_empty_corpus(input_corpus)
+
+
+def check_skip_det_compatible(additional_flags):
+    """ Checks if additional flags are compatible with '-d' option"""
+    # AFL refuses to take in '-d' with '-M' or '-S' options for parallel mode.
+    # (cf. https://github.com/google/AFL/blob/8da80951/afl-fuzz.c#L7477)
+    if '-M' in additional_flags or '-S' in additional_flags:
+        return False
+    return True
+
+
+def prepare_empty_corpus(input_corpus):
+    if os.path.exists(input_corpus):
+        shutil.rmtree(input_corpus)
+    os.makedirs(input_corpus)
+    with open(os.path.join(input_corpus, 'a'), 'wb') as f:
+        f.write(b'a')
+
+
+def run_afl_fuzz(input_corpus,
+                 output_corpus,
+                 target_binary,
+                 additional_flags=None,
+                 hide_output=False):
+    """Run afl-fuzz."""
+
+    prepare_empty_corpus(input_corpus)
+    # Spawn the afl fuzzing process.
+    print('[run_afl_fuzz] Running target with afl-fuzz')
+    command = [
+        './afl-fuzz',
+        '-i',
+        input_corpus,
+        '-o',
+        output_corpus,
+        # Use no memory limit as ASAN doesn't play nicely with one.
+        '-m',
+        'none',
+        '-t',
+        '1000+',  # Use same default 1 sec timeout, but add '+' to skip hangs.
+    ]
+    # Use '-d' to skip deterministic mode, as long as it it compatible with
+    # additional flags.
+    if not additional_flags or check_skip_det_compatible(additional_flags):
+        command.append('-d')
+    if additional_flags:
+        command.extend(additional_flags)
+    dictionary_path = utils.get_dictionary_path(target_binary)
+
+    # if dictionary_path:
+    #     command.extend(['-x', dictionary_path])
+    command += [
+        '--',
+        target_binary,
+        # Pass INT_MAX to afl the maximize the number of persistent loops it
+        # performs.
+        '2147483647'
+    ]
+    print('[run_afl_fuzz] Running command: ' + ' '.join(command))
+    output_stream = subprocess.DEVNULL if hide_output else None
+    subprocess.check_call(command, stdout=output_stream, stderr=output_stream)
+
+
+def fuzz(input_corpus, output_corpus, target_binary):
+    """Run afl-fuzz on target."""
+    prepare_fuzz_environment(input_corpus)
+
+    run_afl_fuzz(input_corpus, output_corpus, target_binary)

fuzzers/afl_empty_seeds/runner.Dockerfile

@@ -0,0 +1,15 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/fuzzbench/base-image

fuzzers/aflplusplus_empty_seeds/builder.Dockerfile

@@ -0,0 +1,49 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG parent_image
+FROM $parent_image
+
+RUN apt-get update && \
+    apt-get install -y \
+        build-essential \
+        python3-dev \
+        python3-setuptools \
+        automake \
+        cmake \
+        git \
+        flex \
+        bison \
+        libglib2.0-dev \
+        libpixman-1-dev \
+        cargo \
+        libgtk-3-dev \
+        # for QEMU mode
+        ninja-build \
+        gcc-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* //')-plugin-dev \
+        libstdc++-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* //')-dev
+
+# Download afl++.
+RUN git clone -b dev https://github.com/AFLplusplus/AFLplusplus /afl && \
+    cd /afl && \
+    git checkout 56d5aa3101945e81519a3fac8783d0d8fad82779 || \
+    true
+
+# Build without Python support as we don't need it.
+# Set AFL_NO_X86 to skip flaky tests.
+RUN cd /afl && \
+    unset CFLAGS CXXFLAGS && \
+    export CC=clang AFL_NO_X86=1 && \
+    PYTHON_INCLUDE=/ make && \
+    cp utils/aflpp_driver/libAFLDriver.a /

fuzzers/aflplusplus_empty_seeds/description.md

@@ -0,0 +1,14 @@
+# aflplusplus
+
+AFL++ fuzzer instance that has the following config active for all benchmarks:
+  - PCGUARD instrumentation 
+  - cmplog feature
+  - dict2file feature
+  - "fast" power schedule
+  - persistent mode + shared memory test cases
+
+Repository: [https://github.com/AFLplusplus/AFLplusplus/](https://github.com/AFLplusplus/AFLplusplus/)
+
+[builder.Dockerfile](builder.Dockerfile)
+[fuzzer.py](fuzzer.py)
+[runner.Dockerfile](runner.Dockerfile)