diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 229eb1b63..f528df717 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -324,4 +324,95 @@ jobs: exit ${exit_code} +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + experiment: + # the ml train test is placed here to use cached data set + needs: [ download_data ] + + runs-on: ubuntu-latest + + steps: + + - name: Checkout CredData + uses: actions/checkout@v3 + with: + repository: Samsung/CredData + + - name: Cache data + id: cache-data + uses: actions/cache@v3 + with: + path: data + key: cred-data-${{ hashFiles('snapshot.yaml') }} + + - name: Failure in case when cache missed + if: steps.cache-data.outputs.cache-hit != 'true' + run: exit 1 + + - name: Exclude some sets for speed-up + run: | + rm -rf data/2* data/8* data/b* + rm -rf meta/2* meta/8* meta/b* + mkdir -vp ${{ github.workspace }}/CredData + mv data ${{ github.workspace }}/CredData/ + mv meta ${{ github.workspace }}/CredData/ + + - name: Set up Python 3.8 + if: steps.cache-data.outputs.cache-hit != 'true' + uses: actions/setup-python@v3 + with: + python-version: "3.8" + + - name: Update PIP + run: python -m pip install --upgrade pip + + - name: Checkout current CredSweeper + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: CredSweeper.head + + - name: Install development packages + run: python -m pip install --requirement CredSweeper.head/requirements.txt + + - name: Install experimental packages + # some versions will be changed for compatibility + run: python -m pip install --requirement CredSweeper.head/experiment/requirements.txt + + - name: dbg + run: echo ${{ github.workspace }} && ls -al ${{ github.workspace }} && tree ${{ github.workspace }} + + - name: Lighten spit.json + run: | + mv -vf ${{ github.workspace }}/CredSweeper.head/experiment/src/split.json ${{ github.workspace }}/CredSweeper.head/experiment/src/split.json.bak + cat ${{ github.workspace }}/CredSweeper.head/experiment/src/split.json.bak + grep -v '"[28b][0-9a-f]\+' ${{ github.workspace }}/CredSweeper.head/experiment/src/split.json.bak >${{ github.workspace }}/CredSweeper.head/experiment/src/split.json + cat ${{ github.workspace }}/CredSweeper.head/experiment/src/split.json + + - name: Run the experiment + run: | + cd CredSweeper.head + ls -al #dbg + pwd #dbg + export PYTHONPATH=$(pwd):${PYTHONPATH} + cd experiment + # check whether credsweeper is available as module + python -m credsweeper --banner + # use only 2 epochs for the test + sed -i 's/epochs=42,/epochs=2,/' main.py + python main.py --data ${{ github.workspace }}/CredData -j $(( 2 * $(nproc) )) + ls -al results #dbg + python -m tf2onnx.convert --saved-model $(find results -mindepth 1 -maxdepth 1 -type d) --output ../credsweeper/ml_model/ml_model.onnx --verbose + # dbg + git diff + # crc32 should be changed + python -m credsweeper --banner + # run quick scan + python -m credsweeper --log debug --path ../tests/samples --save-json + NEW_MODEL_FOUND_SAMPLES=$(jq '.|length' output.json) + if [ 100 -gt ${NEW_MODEL_FOUND_SAMPLES} ]; then + echo "Failure: found ${NEW_MODEL_FOUND_SAMPLES} credentials" + exit 1 + fi + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 701a2943e..2402f703a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -58,7 +58,7 @@ jobs: - name: Check ml_model.onnx integrity if: ${{ always() && steps.code_checkout.conclusion == 'success' }} run: | - md5sum --binary credsweeper/ml_model/ml_model.onnx | grep 8cb870a200d7bc07893aacec38f54033 + md5sum --binary credsweeper/ml_model/ml_model.onnx | grep 4774fdce802e940023316c32f14a68df # # # Python setup diff --git a/cicd/benchmark.txt b/cicd/benchmark.txt index 56e895d13..f8f142c3f 100644 --- a/cicd/benchmark.txt +++ b/cicd/benchmark.txt @@ -232,16 +232,16 @@ FileType FileNumber ValidLines Positives Negatives Templat .zsh 7 1109 13 .zsh-theme 1 121 1 TOTAL: 10188 19071512 5188 53756 5484 -Detected Credentials: 6082 -credsweeper result_cnt : 5232, lost_cnt : 0, true_cnt : 4388, false_cnt : 844 +Detected Credentials: 5792 +credsweeper result_cnt : 5038, lost_cnt : 0, true_cnt : 4606, false_cnt : 432 Category TP FP TN FN FPR FNR ACC PRC RCL F1 -------------------------- ---- ---- -------- ---- -------- -------- -------- -------- -------- -------- -Authentication Credentials 76 78 2607 15 0.029050 0.164835 0.966499 0.493506 0.835165 0.620408 -Cryptographic Primitives 47 14 158 7 0.081395 0.129630 0.907080 0.770492 0.870370 0.817391 -Generic Secret 1061 71 30139 143 0.002350 0.118771 0.993188 0.937279 0.881229 0.908390 -Generic Token 295 33 4242 41 0.007719 0.122024 0.983951 0.899390 0.877976 0.888554 -Other 496 329 3417 188 0.087827 0.274854 0.883296 0.601212 0.725146 0.657389 -Password 1049 248 11123 381 0.021810 0.266434 0.950863 0.808790 0.733566 0.769344 -Predefined Pattern 353 71 5233 25 0.013386 0.066138 0.983105 0.832547 0.933862 0.880299 +Authentication Credentials 74 27 2658 17 0.010056 0.186813 0.984150 0.732673 0.813187 0.770833 +Cryptographic Primitives 43 3 169 11 0.017442 0.203704 0.938053 0.934783 0.796296 0.860000 +Generic Secret 1112 25 30185 92 0.000828 0.076412 0.996276 0.978012 0.923588 0.950021 +Generic Token 304 7 4268 32 0.001637 0.095238 0.991542 0.977492 0.904762 0.939722 +Other 503 330 3416 181 0.088094 0.264620 0.884650 0.603842 0.735380 0.663151 +Password 1200 34 11337 230 0.002990 0.160839 0.979377 0.972447 0.839161 0.900901 +Predefined Pattern 359 6 5298 19 0.001131 0.050265 0.995600 0.983562 0.949735 0.966353 Private Key 1011 0 1477 0 1.000000 1.000000 1.000000 1.000000 - 4388 844 19065480 800 0.000044 0.154202 0.999914 0.838685 0.845798 0.842226 + 4606 432 19065892 582 0.000023 0.112182 0.999947 0.914252 0.887818 0.900841 diff --git a/credsweeper/ml_model/features.py b/credsweeper/ml_model/features.py index 80a2c4567..7f7183a39 100644 --- a/credsweeper/ml_model/features.py +++ b/credsweeper/ml_model/features.py @@ -49,6 +49,26 @@ def any_word_in_(self, lower_case_line: str) -> bool: return False +class WordInVariable(Feature): + """Feature returns true if candidate value contains at least one word from predefined list.""" + + def __init__(self, words: List[str]) -> None: + """Feature is true if candidate value contains at least one predefined word. + + Args: + words: list of predefined words - MUST BE IN LOWER CASE + + """ + super().__init__() + self.words = words + + def extract(self, candidate: Candidate) -> bool: + """Returns true if any words in first line""" + if candidate.line_data_list[0].variable: + return self.any_word_in_(candidate.line_data_list[0].variable.lower()) + return False + + class WordInSecret(Feature): """Feature returns true if candidate value contains at least one word from predefined list.""" diff --git a/credsweeper/ml_model/ml_model.onnx b/credsweeper/ml_model/ml_model.onnx index bd64a7734..7dd3fcbd9 100644 Binary files a/credsweeper/ml_model/ml_model.onnx and b/credsweeper/ml_model/ml_model.onnx differ diff --git a/credsweeper/ml_model/ml_validator.py b/credsweeper/ml_model/ml_validator.py index 9c456825c..877117ad5 100644 --- a/credsweeper/ml_model/ml_validator.py +++ b/credsweeper/ml_model/ml_validator.py @@ -44,7 +44,7 @@ def __init__(self, threshold: Union[float, ThresholdPreset], azure: bool = False self.threshold = model_details["thresholds"][threshold.value] else: self.threshold = 0.5 - self.maxlen = int(model_details.get("max_len", 50)) + self.maxlen = int(model_details.get("max_len", 160)) self.common_feature_list = [] self.unique_feature_list = [] logger.info("Init ML validator, model file path: %s", model_file_path) diff --git a/credsweeper/ml_model/model_config.json b/credsweeper/ml_model/model_config.json index 749121076..4097fe1ed 100644 --- a/credsweeper/ml_model/model_config.json +++ b/credsweeper/ml_model/model_config.json @@ -6,8 +6,16 @@ "high": 0.7979102, "highest": 0.9299587 }, - "max_len": 50, + "max_len": 160, "features": [ + { + "type": "WordInVariable", + "kwargs": { + "words": [ + "test" + ] + } + }, { "type": "WordInSecret", "kwargs": { @@ -77,11 +85,10 @@ "null", "nil", "undefined", - "None", + "none", "true", "false", - "True", - "False" + "example" ] } }, @@ -117,6 +124,22 @@ ] } }, + { + "type": "WordInLine", + "kwargs": { + "words": [ + "://" + ] + } + }, + { + "type": "WordInLine", + "kwargs": { + "words": [ + "@" + ] + } + }, { "type": "WordInLine", "kwargs": { @@ -154,11 +177,9 @@ "null", "nil", "undefined", - "None", + "none", "true", - "false", - "True", - "False" + "false" ] } }, @@ -181,6 +202,14 @@ ] } }, + { + "type": "WordInPath", + "kwargs": { + "words": [ + "src" + ] + } + }, { "type": "ShannonEntropy", "kwargs": { @@ -272,56 +301,89 @@ "type": "FileExtension", "kwargs": { "extensions": [ - ".ps1", - ".hs", - ".graphql", - ".storyboard", - ".strings", - ".template", - ".scss", - ".mjs", - ".lock", + "", + ".adoc", + ".asciidoc", + ".bash", + ".bat", + ".bats", + ".c", + ".cc", + ".cfg", + ".cnf", + ".conf", + ".config", + ".cpp", + ".cs", + ".diff", + ".env", + ".ex", ".example", ".ex", ".tf", ".exs", + ".ext", + ".go", + ".golden", + ".gradle", ".groovy", - ".sql", - ".cc", + ".h", + ".hpp", + ".hs", + ".html", + ".in", + ".inc", + ".ini", ".j2", + ".java", + ".js", + ".json", + ".jsp", + ".kt", + ".lua", + ".m", + ".markdown", + ".markerb", + ".md", + ".ndjson", + ".php", ".pl", - ".rs", - ".rrc", - ".gradle", - ".scala", - ".sum", - ".pem", + ".pod", + ".postinst", + ".pp", ".properties", - ".xml", - ".vue", - ".tsx", - ".html", - ".conf", - ".key", - ".toml", + ".proto", + ".ps1", + ".pxd", + ".py", + ".r", + ".rb", + ".rnh", + ".rs", + ".rsp", ".rst", - ".kt", - ".asciidoc", - ".yaml", + ".sample", + ".sbt", + ".scala", + ".scss", ".sh", - ".c", - ".cs", - ".php", - ".txt", - ".yml", - ".java", + ".slim", + ".snap", + ".sql", + ".t", + ".template", + ".tf", + ".tfstate", + ".tl", + ".tmpl", + ".toml", + ".travis", ".ts", - ".md", - ".js", - ".json", - ".rb", - ".py", - ".go" + ".tsx", + ".txt", + ".vue", + ".yaml", + ".yml" ] } }, @@ -331,12 +393,17 @@ "rule_names": [ "Token", "Secret", - "AWS Client ID", + "Github Old Token", "API", "Credential", "Password", "Key", - "Auth" + "Auth", + "JSON Web Token", + "URL Credentials", + "Nonce", + "Salt", + "Certificate" ] } } diff --git a/experiment/__init__.py b/experiment/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/experiment/augmentation/main.py b/experiment/augmentation/main.py index e5c98b10e..b1d2e6143 100644 --- a/experiment/augmentation/main.py +++ b/experiment/augmentation/main.py @@ -1,13 +1,16 @@ +import logging import os -from shutil import rmtree -from multiprocessing import Pool -import pandas as pd import random import sys from pathlib import Path -import tqdm +from shutil import rmtree + +import pandas as pd + +from .obfuscation import get_obfuscated_value, generate_value, SecretCreds -from obfuscation import get_obfuscated_value, generate_value, SecretCreds +logging.basicConfig(format="%(asctime)s | %(levelname)s | %(filename)s:%(lineno)s | %(message)s", level="DEBUG") +logger = logging.getLogger(__name__) BASE_PATH = ["test", "src", "other"] COLUMN_TYPES = { @@ -118,10 +121,10 @@ def add_raw_lines(meta_df, filepath, content): def write2aug_file(repo_local_path, meta_df, aug_file): fls_path = list(set(meta_df.FilePath)) for filepath in fls_path: - with open(repo_local_path / filepath, "r") as reader: + with open(repo_local_path / filepath, "r", encoding="utf8") as reader: content = reader.readlines() add_raw_lines(meta_df, filepath, content) - with open(repo_local_path / aug_file, "w") as writer: + with open(repo_local_path / aug_file, "w", encoding="utf8") as writer: Rows = meta_df.RawLine writer.writelines(Rows) @@ -150,7 +153,7 @@ def get_linage(repo_local_path, df): files_length = {} overall_linage = 0 for filepath in fls_path: - with open(repo_local_path / filepath, "r") as reader: + with open(repo_local_path / filepath, "r", encoding="utf8") as reader: content = reader.readlines() overall_linage += len(content) files_length[filepath] = len(content) @@ -162,7 +165,7 @@ def get_extentions(meta_df): exts = set() for file_path in file_paths: if "." in file_path: - exts.update(["." + file_path.split(".")[-1]]) + exts.update(["." + file_path.split(".")[-1].lower()]) return list(exts) @@ -184,37 +187,30 @@ def get_true_row(df, idx, aug_file): "RawLine": "" # }) idx += line_diff - t_df = t_df.append(add_series) + t_df = pd.concat([t_df, add_series]) return t_df, idx -def get_false_row(row_numb, aug_filename, files_length, fl_true_lines): - fl_path = list(files_length.keys()) +def get_false_row(df, idx, aug_file): + temp_df = df[df["GroundTruth"] == "F"] + fl_path = list(temp_df["FilePath"]) + if len(fl_path) == 0: + return None, idx + lines = list(temp_df["LineStart:LineEnd"]) rand = random.randint(0, len(fl_path) - 1) - fl_name = fl_path[rand] - fl_length = files_length[fl_name] - # Filter true lines - true_lines = fl_true_lines[fl_name] - if fl_length == len(true_lines): - return None - - t_df = None - while t_df is None: - rand_row = random.randint(1, fl_length) - if rand_row in true_lines: - continue - orig_linenumb = str(rand_row) + ":" + str(rand_row) - new_linenumb = str(row_numb) + ":" + str(row_numb) - t_df = pd.Series({ - "FilePath": fl_name, - "LineStart:LineEnd": orig_linenumb, - "GroundTruth": "F", - "New_LineNumb": new_linenumb, - "New_FilePath": aug_filename, - "RawLine": "" - }) - return t_df + line_numb = lines[rand].split(":") + t_df = temp_df.iloc[rand].copy() + line_diff = int(line_numb[1]) - int(line_numb[0]) + new_linenumb = str(idx) + ":" + str(idx + line_diff) + add_series = pd.Series({ + "New_LineNumb": new_linenumb, # + "New_FilePath": aug_file, # + "RawLine": "" # + }) + idx += line_diff + t_df = pd.concat([t_df, add_series]) + return t_df, idx def get_true_lines(df): @@ -249,7 +245,7 @@ def generate_rows(repo_local_path, aug_filename, df, true_stake, scale): ground_trues, idx = get_true_row(df, row_numb, aug_filename) row_numb = idx else: - ground_trues = get_false_row(row_numb, aug_filename, files_length, fl_true_lines) + ground_trues, idx = get_false_row(df, row_numb, aug_filename) if ground_trues is None: row_numb -= 1 continue @@ -257,46 +253,40 @@ def generate_rows(repo_local_path, aug_filename, df, true_stake, scale): return new_series -def aug_dir(arg): - repo_local_path, meta_path, dir_name, true_stake, scale = arg - - meta_data = load_meta(meta_path, dir_name) - aug_meta = str(repo_local_path / "aug_data" / "meta" / dir_name) + ".csv" - new_meta = [] +def aug_data(repo_local_path, meta_data, true_stake, scale): + augument_list = [ + "Authentication Credentials" # + "Cryptographic Primitives", # + "Generic Secret", # + "Generic Token", # + "Password", # + "Predefined Pattern", # + ] for base in BASE_PATH: - aug_filetemp = repo_local_path / "aug_data" / "data" / dir_name / base - os.makedirs(aug_filetemp) - aug_filetemp = aug_filetemp / dir_name + new_meta = [] + aug_meta = str(repo_local_path / "aug_data" / "meta" / base) + ".csv" + aug_file_template = repo_local_path / "aug_data" / "data" / base meta_df = meta_data[meta_data["FilePath"].str.contains(base)] - augument_list = [ - "Password", # - "Generic Secret", # - "Predefined Pattern", # - "Cryptographic Primitives", # - "Generic Token", # - "Authentication Credentials" # - ] meta_df = meta_df[meta_df["Category"].isin(augument_list)] exts = get_extentions(meta_df) for extension in exts: - bool_series = meta_df["FilePath"].str.endswith(extension) - ext_df = meta_df[bool_series] - aug_filename = str(aug_filetemp) + extension + ext_df = meta_df[meta_df["FilePath"].str.endswith(extension)] + aug_filename = str(aug_file_template) + extension new_series = generate_rows(repo_local_path, aug_filename, ext_df, true_stake, scale) if new_series: new_meta_df = join_series(new_series) write2aug_file(repo_local_path, new_meta_df, aug_filename) new_meta.append(new_meta_df) - if new_meta: - write_meta(new_meta, aug_meta) + if new_meta: + write_meta(new_meta, aug_meta) -def build_corpus(repo_local_path, meta_path, repos_paths, true_stake: float, scale: float): +def build_corpus(repo_local_path: Path, meta_path: Path, repos_paths, true_stake: float, scale: float): """ Build the corpus for this repo. Parameters ---------- - repo_path: str + repo_local_path: str Path to the CredPosDataset repository meta_path: str Path to the metadata @@ -304,6 +294,8 @@ def build_corpus(repo_local_path, meta_path, repos_paths, true_stake: float, sca List of repos directory names true_stake: Part of the rows with "True" cases in the aggregated data + scale: + scale Returns ------- @@ -316,30 +308,23 @@ def build_corpus(repo_local_path, meta_path, repos_paths, true_stake: float, sca pass os.makedirs(repo_local_path / "aug_data") os.makedirs(repo_local_path / "aug_data" / "meta") + os.makedirs(repo_local_path / "aug_data" / "data") print(f"Start augmentation for {len(repos_paths)} repos, " f"Generated data will be saved to {repo_local_path / 'aug_data'}") - args = [] + meta_data = pd.DataFrame() for rep_name in repos_paths: - args.append((repo_local_path, meta_path, rep_name, true_stake, scale)) - - with Pool(processes=get_pool_count()) as p: - max_ = len(repos_paths) - with tqdm.tqdm(total=max_) as pbar: - for i, _ in enumerate(p.imap_unordered(aug_dir, args)): - pbar.update() + _meta_data = load_meta(meta_path, rep_name) + meta_data = pd.concat([_meta_data, meta_data]) + aug_data(repo_local_path, meta_data, true_stake, scale) print(f"Augmentation finished") -if __name__ == "__main__": - CredDataDirectory = sys.argv[1] - true_stake = sys.argv[2] - scale = sys.argv[3] - +def main(cred_data_dir, true_stake, scale): try: - CredDataDirectory = os.path.abspath(CredDataDirectory) + cred_data_dir = os.path.abspath(cred_data_dir) except: raise ValueError("Please set a valid CredData directory") - if not os.path.isdir(CredDataDirectory): + if not os.path.isdir(cred_data_dir): raise ValueError("Please set a valid CredData. It should be a valid path") try: @@ -354,9 +339,16 @@ def build_corpus(repo_local_path, meta_path, repos_paths, true_stake: float, sca except: raise ValueError("Please set a valid scale. It cannot contain commas, spaces, or characters.") - repo_path = Path(CredDataDirectory) + repo_path = Path(cred_data_dir) data_path = repo_path / "data" - meta_path = repo_path / "meta" - repos_paths = os.listdir(data_path) + _meta_path = repo_path / "meta" + _repos_paths = os.listdir(data_path) - build_corpus(repo_path, meta_path, repos_paths, true_stake, scale) + build_corpus(repo_path, _meta_path, _repos_paths, true_stake, scale) + + +if __name__ == "__main__": + _cred_data_dir = sys.argv[1] + _true_stake = sys.argv[2] + _scale = sys.argv[3] + main(_cred_data_dir, _true_stake, _scale) diff --git a/experiment/main.py b/experiment/main.py index 52caf76f2..f320964df 100644 --- a/experiment/main.py +++ b/experiment/main.py @@ -2,17 +2,18 @@ import random from argparse import ArgumentParser from copy import deepcopy -from time import time +from datetime import datetime from typing import Tuple, List -import tensorflow as tf + import numpy as np +import tensorflow as tf from tensorflow.python.keras import Model from experiment.src.data_loader import read_detected_data, read_metadata, join_label, get_missing, eval_no_model, \ get_y_labels, eval_with_model -from experiment.src.prepare_data import prepare_train_data from experiment.src.features import prepare_data from experiment.src.lstm_model import get_model_string_features +from experiment.src.prepare_data import prepare_train_data from experiment.src.split import load_fixed_split @@ -43,8 +44,10 @@ def main(cred_data_location: str) -> str: meta_data_copy = deepcopy(meta_data) # Combine original and augmented data together - detected_data.update(read_detected_data("data/result_aug_data.json", "aug_data/")) - meta_data.update(read_metadata(f"{cred_data_location}/aug_data/meta", "aug_data/")) + aug_detected_data = read_detected_data("data/result_aug_data.json", "aug_data/") + detected_data.update(aug_detected_data) + aug_metadata = read_metadata(f"{cred_data_location}/aug_data/meta", "aug_data/") + meta_data.update(aug_metadata) df = join_label(detected_data, meta_data) @@ -68,17 +71,17 @@ def main(cred_data_location: str) -> str: fit_history = keras_model.fit( [X_train_value, X_train_features], y_train, - batch_size=64, - epochs=10, + batch_size=128, + epochs=42, # Class 1 in train data is roughly ~4 times more abundant than 0. As can be seen from the log class_weight={ - 0: 4, - 1: 1 + 0: 2, + 1: 3 }) os.makedirs("results/", exist_ok=True) - current_time = int(time()) - model_file_name = f"results/ml_model_at-{current_time}.h5" + current_time = datetime.now().strftime("%Y%m%d_%H%M%S") + model_file_name = f"results/ml_model_at-{current_time}" keras_model.save(model_file_name, include_optimizer=False) print('-' * 40) @@ -98,7 +101,7 @@ def main(cred_data_location: str) -> str: print("Results on test without model:") eval_no_model(df_test, df_missing_test) print("Results on test with model:") - eval_with_model(df_test, df_missing_test, test_predictions) + eval_with_model(df_test.copy(), df_missing_test, test_predictions) return model_file_name @@ -118,15 +121,17 @@ def main(cred_data_location: str) -> str: metavar="POSITIVE_INT") args = parser.parse_args() - fixed_seed = 42 + fixed_seed = 42 # int(datetime.now().timestamp()) + # print(f"Random seed:{fixed_seed}") if fixed_seed is not None: tf.random.set_seed(fixed_seed) np.random.seed(fixed_seed) random.seed(fixed_seed) - cred_data_location = args.cred_data_location + _cred_data_location = args.cred_data_location j = int(args.jobs) - prepare_train_data(cred_data_location, j) - model_file_name = main(cred_data_location) - print(f"You can find your model in: {model_file_name}") + prepare_train_data(_cred_data_location, j) + _model_file_name = main(_cred_data_location) + print(f"You can find your model in: {_model_file_name}") + # python -m tf2onnx.convert --saved-model results/ml_model_at-20240201_073238 --output ../credsweeper/ml_model/ml_model.onnx --verbose diff --git a/experiment/requirements.txt b/experiment/requirements.txt new file mode 100644 index 000000000..e83f5bc8f --- /dev/null +++ b/experiment/requirements.txt @@ -0,0 +1,8 @@ +h5py==3.10.0 +keras==2.13.1 +numpy==1.23.5 +onnx==1.15.0 +protobuf==3.20.3 +tensorflow==2.13.1 +tf2onnx==1.16.0 +wrapt==1.14.1 diff --git a/experiment/src/data_loader.py b/experiment/src/data_loader.py index 583ecec18..14cdf2e8a 100644 --- a/experiment/src/data_loader.py +++ b/experiment/src/data_loader.py @@ -1,15 +1,17 @@ import json import os -from typing import Tuple, Dict +import pathlib from copy import deepcopy -import pandas as pd +from typing import Tuple, Dict + import numpy as np +import pandas as pd identifier = Tuple[str, int] def strip_data_path(file_path, split="CredData/"): - file_path = file_path.replace("//", "/") + file_path = pathlib.Path(file_path).as_posix() return file_path.split(split, 1)[-1] @@ -21,17 +23,19 @@ def read_detected_data(file_path: str, split="CredData/") -> Dict[identifier, Di detected_lines = {} for detection in detections: + if 1 != len(detection["line_data_list"]): + continue for line_data in detection["line_data_list"]: relative_path = strip_data_path(line_data["path"], split) index = relative_path, line_data["line_num"] data_to_save = deepcopy(line_data) data_to_save["path"] = relative_path - data_to_save["RuleNames"] = [detection["rule"]] + data_to_save["RuleName"] = [detection["rule"]] if index not in detected_lines: detected_lines[index] = data_to_save else: - detected_lines[index]["RuleNames"].append(detection["rule"]) + detected_lines[index]["RuleName"].append(detection["rule"]) print(f"Detected {len(detected_lines)} unique lines!") print(f"{len(detections)} detections in total") @@ -121,7 +125,8 @@ def eval_no_model(df: pd.DataFrame, df_missing: pd.DataFrame): def eval_with_model(df: pd.DataFrame, df_missing: pd.DataFrame, predictions: np.ndarray): - df["Correct"] = df["GroundTruth"] == predictions + df["Correct"] = False + df.loc[df["GroundTruth"] == predictions, "Correct"] = True tp = len(df[df["GroundTruth"] & df["Correct"]]) fp = len(df[~df["GroundTruth"] & ~df["Correct"]]) tn = len(df[~df["GroundTruth"] & df["Correct"]]) @@ -153,5 +158,5 @@ def eval_with_model(df: pd.DataFrame, df_missing: pd.DataFrame, predictions: np. def get_y_labels(df: pd.DataFrame) -> np.ndarray: - true_cases = np.array(df["GroundTruth"]) + true_cases = np.array(df["GroundTruth"], dtype=np.int8) return true_cases diff --git a/experiment/src/features.py b/experiment/src/features.py index 4bef7fb1e..ee8caf834 100644 --- a/experiment/src/features.py +++ b/experiment/src/features.py @@ -1,31 +1,36 @@ from typing import Tuple, Union +import numpy as np +import pandas as pd + from credsweeper.common.constants import Severity from credsweeper.credentials import Candidate from credsweeper.credentials import LineData from credsweeper.ml_model import MlValidator -import numpy as np -import pandas as pd +from credsweeper.utils import Util -MlValidator() # Initialize global MLValidator object +ml_validator = MlValidator(0.5) # Initialize global MLValidator object class CustomLineData(LineData): """Object that allows to create LineData from scanner results""" - def __init__(self, line: str, value: str, line_num: int, path: str) -> None: + def __init__(self, line: str, value: str, line_num: int, path: str, variable: str) -> None: self.line: str = line self.line_num: int = line_num self.path: str = path self.value = value + self.file_type = Util.get_extension(path) + self.variable = variable def get_candidates(line_data: dict): """Get list of candidates. 1 candidate for each rule that detected this line""" - ld = CustomLineData(line_data["line"], line_data["value"], line_data["line_num"], line_data["path"]) + ld = CustomLineData(line_data["line"], line_data["value"], line_data["line_num"], line_data["path"], + line_data["variable"]) candidates = [] - for rule in line_data["RuleNames"]: - candidates.append(Candidate([ld], [], rule, Severity.MEDIUM, config=None, use_ml=True)) + for rule in line_data["RuleName"]: + candidates.append(Candidate([ld], [], rule, Severity.MEDIUM, None, None, True)) return candidates @@ -35,10 +40,10 @@ def get_features(line_data: Union[dict, pd.Series]): value = line_data["value"] candidates = get_candidates(line_data) - line_input = MlValidator.encode(value, MlValidator.char_to_index) + line_input = ml_validator.encode(value, ml_validator.char_to_index) - common_features = MlValidator.extract_common_features(candidates) - unique_features = MlValidator.extract_unique_features(candidates) + common_features = ml_validator.extract_common_features(candidates) + unique_features = ml_validator.extract_unique_features(candidates) extracted_features = np.hstack([common_features, unique_features]) @@ -51,6 +56,7 @@ def prepare_data(df: pd.DataFrame) -> Tuple[np.ndarray, np.ndarray]: X_features = [] for i, row in df.iterrows(): + assert row["line"] is not None, row line_input, extracted_features = get_features(row) X_values.append(line_input) X_features.append(extracted_features) diff --git a/experiment/src/lstm_model.py b/experiment/src/lstm_model.py index f4cbb79cc..4d332ebf0 100644 --- a/experiment/src/lstm_model.py +++ b/experiment/src/lstm_model.py @@ -15,14 +15,14 @@ def get_model_string_features(vocab_size: int, feature_size: int) -> Model: Return: Keras model """ - lstm_input = Input(shape=(None, vocab_size), name="Value_input") - lstm_branch = Bidirectional(LSTM(30))(lstm_input) + lstm_input = Input(shape=(None, vocab_size), name="line_input") + lstm_branch = Bidirectional(LSTM(160))(lstm_input) - feature_input = Input(shape=(feature_size, ), name="Features_input") + feature_input = Input(shape=(feature_size, ), name="feature_input") joined_features = Concatenate()([lstm_branch, feature_input]) - x = Dense(100, activation='relu', name="Dense_1")(joined_features) - x = Dense(1, activation='sigmoid', name="Prediction")(x) + x = Dense(160, activation='relu', name="Dense_1")(joined_features) + x = Dense(1, activation='sigmoid', name="prediction")(x) model = Model([lstm_input, feature_input], x) diff --git a/experiment/src/prepare_data.py b/experiment/src/prepare_data.py index d748b60f1..400acfdcc 100644 --- a/experiment/src/prepare_data.py +++ b/experiment/src/prepare_data.py @@ -1,30 +1,33 @@ import os -import sys import subprocess +import sys + +from credsweeper.utils import Util +from ..augmentation.main import main as aug_main def execute_scanner(dataset_location: str, result_location_str, j, use_ml=False): """Execute CredSweeper as a separate process to make sure no global states is shared with training script""" dir_path = os.path.dirname(os.path.realpath(__file__)) + "/.." - command = f"{sys.executable} -m credsweeper --path {dataset_location}/data --save-json {result_location_str} -j {j}" + command = f"{sys.executable} -m credsweeper --path {dataset_location}/data" \ + f" --save-json {result_location_str} -j {j} --sort --rules train_config.yaml" if not use_ml: command += " --ml_threshold 0" - subprocess.call(command, shell=True, cwd=dir_path, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) - - -def get_aug_data(dataset_location: str): - """Execute CredSweeper as a separate process to make sure no global states is shared with training script""" - dir_path = os.path.dirname(os.path.realpath(__file__)) + "/.." - command = f"{sys.executable} main.py {dataset_location} 0.1 5" - subprocess.call(command, shell=True, cwd=dir_path + "/augmentation") + subprocess.check_call(command, shell=True, cwd=dir_path, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) def prepare_train_data(cred_data_location: str, j: int): os.makedirs("data", exist_ok=True) + if not os.path.exists("train_config.yaml"): + # use only rules which marked as use_ml may be valuable + rules = Util.yaml_load("../credsweeper/rules/config.yaml") + new_rules = [x for x in rules if x.get("use_ml")] + Util.yaml_dump(new_rules, "train_config.yaml") + if not os.path.exists("data/result_aug_data.json"): print(f"Augment data from {cred_data_location}") - get_aug_data(cred_data_location) + aug_main(cred_data_location, 0.5, 5) execute_scanner(cred_data_location + "/aug_data", "data/result_aug_data.json", j) if not os.path.exists("data/result.json"): diff --git a/experiment/src/split.json b/experiment/src/split.json index a49a6b723..f4f8dbd8c 100644 --- a/experiment/src/split.json +++ b/experiment/src/split.json @@ -54,11 +54,11 @@ "8cda00f3", "78e5819e", "50b38ef0", - "369d561d", "c9b945fa", "81a41883", "8c3c80e8", - "bd7bcb9d" + "bd7bcb9d", + "369d561d" ], "train_repo_list": [ "e3377359", diff --git a/tests/__init__.py b/tests/__init__.py index 96a342b4b..2a8415539 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -8,14 +8,14 @@ SAMPLES_CRED_LINE_COUNT: int = 412 # credentials count after post-processing -SAMPLES_POST_CRED_COUNT: int = 314 +SAMPLES_POST_CRED_COUNT: int = 355 # with option --doc SAMPLES_IN_DOC = 396 # archived credentials that are not found without --depth -SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 23 -SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 16 +SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 18 +SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 11 SAMPLES_IN_DEEP_3 = SAMPLES_IN_DEEP_2 + 1 # well known string with all latin letters diff --git a/tests/data/depth_3.json b/tests/data/depth_3.json index 804b5ad89..5c257bd17 100644 --- a/tests/data/depth_3.json +++ b/tests/data/depth_3.json @@ -112,7 +112,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9844, + "ml_probability": 0.9832, "rule": "API", "severity": "medium", "confidence": "moderate", @@ -187,7 +187,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99679, + "ml_probability": 1.0, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -212,7 +212,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99687, + "ml_probability": 0.99595, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -237,7 +237,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.98984, + "ml_probability": 0.99904, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -567,7 +567,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99983, + "ml_probability": 0.99998, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -717,24 +717,24 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99804, + "ml_probability": 0.99566, "rule": "Certificate", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "cert := 'LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMrekND'", + "line": "const certificatePEM := 'LS0tLS1CRUdJTiBDRVJUSUZ0VLQW9JQkFRRFkwYzFUS0I1b1pQd1EKN3QxQ3dNSXJ2cUI2R0lVM3RQeTZSaGNrWlhUa09COFllQldKN1VLZkN6OEhHZJQ0FURS0tLS0tCk1JSUMrekND'", "line_num": 1, "path": "tests/samples/certificate.hs", "info": "tests/samples/certificate.hs|RAW", - "value": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMrekND", - "value_start": 9, - "value_end": 57, - "variable": "cert", + "value": "LS0tLS1CRUdJTiBDRVJUSUZ0VLQW9JQkFRRFkwYzFUS0I1b1pQd1EKN3QxQ3dNSXJ2cUI2R0lVM3RQeTZSaGNrWlhUa09COFllQldKN1VLZkN6OEhHZJQ0FURS0tLS0tCk1JSUMrekND", + "value_start": 25, + "value_end": 165, + "variable": "certificatePEM", "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.222794209361695, - "valid": false + "entropy": 5.133473310626378, + "valid": true } } ] @@ -817,7 +817,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92279, + "ml_probability": 0.99999, "rule": "Credential", "severity": "medium", "confidence": "moderate", @@ -892,7 +892,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.70902, + "ml_probability": 0.99855, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ID:master,PW:dipPr10Gg!", + "line_num": 1, + "path": "tests/samples/doc_id_pair_passwd_pair", + "info": "tests/samples/doc_id_pair_passwd_pair|RAW", + "value": "dipPr10Gg!", + "value_start": 13, + "value_end": 23, + "variable": "master,PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.989735285398626, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.97706, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -917,7 +942,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8653, + "ml_probability": 0.82782, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ANYID:master PW:dipPr12Gg!", + "line_num": 3, + "path": "tests/samples/doc_id_pair_passwd_pair", + "info": "tests/samples/doc_id_pair_passwd_pair|RAW", + "value": "dipPr12Gg!", + "value_start": 16, + "value_end": 26, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.989735285398626, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99433, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -942,7 +992,57 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87845, + "ml_probability": 0.99879, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "id:master,password:dipPr14Gg!", + "line_num": 5, + "path": "tests/samples/doc_id_pair_passwd_pair", + "info": "tests/samples/doc_id_pair_passwd_pair|RAW", + "value": "dipPr14Gg!", + "value_start": 19, + "value_end": 29, + "variable": "master,password", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.989735285398626, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99914, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ID:master/PW:dipPr15Gg!", + "line_num": 6, + "path": "tests/samples/doc_id_pair_passwd_pair", + "info": "tests/samples/doc_id_pair_passwd_pair|RAW", + "value": "dipPr15Gg!", + "value_start": 13, + "value_end": 23, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.989735285398626, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99822, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -967,7 +1067,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9126, + "ml_probability": 0.99801, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -992,7 +1092,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.70976, + "ml_probability": 0.99973, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1017,7 +1117,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.90224, + "ml_probability": 0.9934, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1042,7 +1142,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.75512, + "ml_probability": 0.99777, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1067,7 +1167,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.88595, + "ml_probability": 0.89618, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1092,7 +1192,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89187, + "ml_probability": 0.96988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1117,7 +1217,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.69226, + "ml_probability": 0.99464, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1142,7 +1242,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.736, + "ml_probability": 0.9994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1167,7 +1267,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89888, + "ml_probability": 0.99486, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1192,7 +1292,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92214, + "ml_probability": 0.99329, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1217,7 +1317,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.73325, + "ml_probability": 0.99954, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1242,7 +1342,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91456, + "ml_probability": 0.9947, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1267,7 +1367,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86461, + "ml_probability": 0.95669, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1292,7 +1392,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.88128, + "ml_probability": 0.93823, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1317,23 +1417,23 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.85087, + "ml_probability": 0.99761, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "id:master pass:dipPr122Gg!", - "line_num": 23, + "line": "Username:master/Password:dipPr125Gg!", + "line_num": 26, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "dipPr122Gg!", - "value_start": 15, - "value_end": 26, - "variable": "pass", + "value": "dipPr125Gg!", + "value_start": 25, + "value_end": 36, + "variable": "Password", "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.9631196533066344, + "entropy": 3.1449378351248165, "valid": false } } @@ -1342,20 +1442,20 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.66385, + "ml_probability": 0.9952, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "user:master pw:dipPr124Gg!", - "line_num": 25, + "line": "userId:master,password:dipPr126Gg!", + "line_num": 27, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "dipPr124Gg!", - "value_start": 15, - "value_end": 26, - "variable": "pw", + "value": "dipPr126Gg!", + "value_start": 23, + "value_end": 34, + "variable": "master,password", "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.1449378351248165, @@ -1367,19 +1467,19 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.63818, + "ml_probability": 0.98699, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "Username:master/Password:dipPr125Gg!", - "line_num": 26, + "line": "dipPr128Gg! ID:master dipPr128Gg! Password:dipPr128Gg!", + "line_num": 29, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "dipPr125Gg!", - "value_start": 25, - "value_end": 36, + "value": "dipPr128Gg!", + "value_start": 43, + "value_end": 54, "variable": "Password", "entropy_validation": { "iterator": "BASE64_CHARS", @@ -1392,20 +1492,20 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8757, + "ml_probability": 0.98715, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "dipPr128Gg! ID:master dipPr128Gg! Password:dipPr128Gg!", - "line_num": 29, + "line": "ANYid:master,pw:dipPr129Gg!", + "line_num": 30, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "tests/samples/doc_id_pair_passwd_pair|RAW", - "value": "dipPr128Gg!", - "value_start": 43, - "value_end": 54, - "variable": "Password", + "value": "dipPr129Gg!", + "value_start": 16, + "value_end": 27, + "variable": "master,pw", "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.1449378351248165, @@ -1417,7 +1517,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87015, + "ml_probability": 0.9925, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1442,7 +1542,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89149, + "ml_probability": 0.98809, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1467,7 +1567,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.85143, + "ml_probability": 0.82063, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1492,7 +1592,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87582, + "ml_probability": 0.92741, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1517,7 +1617,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.6929, + "ml_probability": 0.70792, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1542,7 +1642,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72186, + "ml_probability": 0.97576, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1567,7 +1667,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86912, + "ml_probability": 0.99015, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1592,7 +1692,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.6589, + "ml_probability": 0.99939, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1617,7 +1717,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89053, + "ml_probability": 0.99121, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1642,7 +1742,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.66151, + "ml_probability": 0.9987, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1667,7 +1767,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.71129, + "ml_probability": 0.99779, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1692,7 +1792,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86641, + "ml_probability": 0.77806, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1717,7 +1817,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87283, + "ml_probability": 0.91752, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1742,7 +1842,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.67565, + "ml_probability": 0.98801, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1767,7 +1867,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89277, + "ml_probability": 0.9925, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1792,7 +1892,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87973, + "ml_probability": 0.9868, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1817,7 +1917,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72488, + "ml_probability": 0.99822, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1842,7 +1942,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.7479, + "ml_probability": 0.97112, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1892,7 +1992,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.70815, + "ml_probability": 0.80507, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1917,7 +2017,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.79195, + "ml_probability": 0.98411, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1942,7 +2042,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72954, + "ml_probability": 0.90348, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1967,7 +2067,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89053, + "ml_probability": 0.98951, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1992,7 +2092,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.75145, + "ml_probability": 0.99974, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2017,7 +2117,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89373, + "ml_probability": 0.99776, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2042,7 +2142,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91183, + "ml_probability": 0.99742, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2067,7 +2167,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.71787, + "ml_probability": 0.99982, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2092,7 +2192,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72872, + "ml_probability": 0.99966, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2117,7 +2217,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.662, + "ml_probability": 0.99972, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2142,7 +2242,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.90253, + "ml_probability": 0.99631, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2167,7 +2267,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86741, + "ml_probability": 0.96932, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2192,7 +2292,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87403, + "ml_probability": 0.98897, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2217,7 +2317,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87872, + "ml_probability": 0.98662, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2242,7 +2342,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8957, + "ml_probability": 0.99816, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2267,7 +2367,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89389, + "ml_probability": 0.99701, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2292,7 +2392,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89507, + "ml_probability": 0.9985, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2317,7 +2417,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89883, + "ml_probability": 0.9986, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "Name:master,PW:dipPr169Gg!", + "line_num": 70, + "path": "tests/samples/doc_id_pair_passwd_pair", + "info": "tests/samples/doc_id_pair_passwd_pair|RAW", + "value": "dipPr169Gg!", + "value_start": 15, + "value_end": 26, + "variable": "master,PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.1449378351248165, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.9626, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2342,7 +2467,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.90411, + "ml_probability": 0.98081, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2367,7 +2492,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91314, + "ml_probability": 0.99749, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2392,7 +2517,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72298, + "ml_probability": 0.99946, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2417,7 +2542,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.83405, + "ml_probability": 0.96707, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2442,7 +2567,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74746, + "ml_probability": 0.99972, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2467,7 +2592,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.7562, + "ml_probability": 0.9994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2492,7 +2617,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89481, + "ml_probability": 0.99855, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2517,7 +2642,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91281, + "ml_probability": 0.99737, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2542,7 +2667,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72739, + "ml_probability": 0.93882, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2567,7 +2692,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.76693, + "ml_probability": 0.99424, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2592,7 +2717,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.76693, + "ml_probability": 0.99424, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2617,7 +2742,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.68589, + "ml_probability": 0.99975, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2642,7 +2767,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.78269, + "ml_probability": 0.99929, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "username:master/pw:dipPr188Gg!", + "line_num": 89, + "path": "tests/samples/doc_id_pair_passwd_pair", + "info": "tests/samples/doc_id_pair_passwd_pair|RAW", + "value": "dipPr188Gg!", + "value_start": 19, + "value_end": 30, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.9631196533066344, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.98968, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2667,7 +2817,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.77617, + "ml_probability": 0.98985, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2717,7 +2867,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74917, + "ml_probability": 0.79989, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2767,7 +2917,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.81033, + "ml_probability": 0.97325, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2792,23 +2942,23 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.65668, + "ml_probability": 0.99968, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "ID/Password:master/iPp2@GRq", - "line_num": 3, - "path": "tests/samples/doc_id_passwd_pair", - "info": "tests/samples/doc_id_passwd_pair|RAW", - "value": "master/iPp2@GRq", - "value_start": 12, - "value_end": 27, - "variable": "Password", + "line": "id:master@example.com,pw:dipPr198Gg!", + "line_num": 99, + "path": "tests/samples/doc_id_pair_passwd_pair", + "info": "tests/samples/doc_id_pair_passwd_pair|RAW", + "value": "dipPr198Gg!", + "value_start": 25, + "value_end": 36, + "variable": "master@example.com,pw", "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.64643122256795, + "entropy": 3.1449378351248165, "valid": false } } @@ -2817,23 +2967,23 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.73294, + "ml_probability": 0.99997, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "ID/Pass:master/iPp3@GRq", - "line_num": 4, - "path": "tests/samples/doc_id_passwd_pair", - "info": "tests/samples/doc_id_passwd_pair|RAW", - "value": "master/iPp3@GRq", - "value_start": 8, - "value_end": 23, - "variable": "Pass", + "line": "id:master@example.com,pw:IHQSB1GG!", + "line_num": 102, + "path": "tests/samples/doc_id_pair_passwd_pair", + "info": "tests/samples/doc_id_pair_passwd_pair|RAW", + "value": "IHQSB1GG!", + "value_start": 25, + "value_end": 34, + "variable": "master@example.com,pw", "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.64643122256795, + "entropy": 2.595488890170944, "valid": false } } @@ -2842,7 +2992,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.65287, + "ml_probability": 0.92675, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2864,56 +3014,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.81151, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "username/password:master/iPp7@GRq", - "line_num": 8, - "path": "tests/samples/doc_id_passwd_pair", - "info": "tests/samples/doc_id_passwd_pair|RAW", - "value": "master/iPp7@GRq", - "value_start": 18, - "value_end": 33, - "variable": "password", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.64643122256795, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.77054, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "id/passwd:master/iPp8@GRq", - "line_num": 9, - "path": "tests/samples/doc_id_passwd_pair", - "info": "tests/samples/doc_id_passwd_pair|RAW", - "value": "master/iPp8@GRq", - "value_start": 10, - "value_end": 25, - "variable": "passwd", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.64643122256795, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -2967,20 +3067,20 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.83017, + "ml_probability": 0.68001, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32 id/pw:master/iPp19@GRq", - "line_num": 20, + "line": "ID/PWD:master/iPp21@GRq", + "line_num": 22, "path": "tests/samples/doc_id_passwd_pair", "info": "tests/samples/doc_id_passwd_pair|RAW", - "value": "master/iPp19@GRq", - "value_start": 18, - "value_end": 34, - "variable": "pw", + "value": "master/iPp21@GRq", + "value_start": 7, + "value_end": 23, + "variable": "PWD", "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.75, @@ -2992,23 +3092,23 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72484, + "ml_probability": 0.81183, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "id/pass:master,iPp20@GRq", - "line_num": 21, + "line": "user/pwd:master/iPp22@GRq", + "line_num": 23, "path": "tests/samples/doc_id_passwd_pair", "info": "tests/samples/doc_id_passwd_pair|RAW", - "value": "master,iPp20@GRq", - "value_start": 8, - "value_end": 24, - "variable": "pass", + "value": "master/iPp22@GRq", + "value_start": 9, + "value_end": 25, + "variable": "pwd", "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.5, + "entropy": 3.625, "valid": false } } @@ -3017,20 +3117,20 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.63765, + "ml_probability": 0.76211, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "ID/PWD:master/iPp21@GRq", - "line_num": 22, + "line": "user/pass:master/iPp25@GRq", + "line_num": 26, "path": "tests/samples/doc_id_passwd_pair", "info": "tests/samples/doc_id_passwd_pair|RAW", - "value": "master/iPp21@GRq", - "value_start": 7, - "value_end": 23, - "variable": "PWD", + "value": "master/iPp25@GRq", + "value_start": 10, + "value_end": 26, + "variable": "pass", "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.75, @@ -3042,20 +3142,20 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.67153, + "ml_probability": 0.7549, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "user/pass:master/iPp25@GRq", - "line_num": 26, + "line": "ID/Password=master/iPp27@GRq", + "line_num": 28, "path": "tests/samples/doc_id_passwd_pair", "info": "tests/samples/doc_id_passwd_pair|RAW", - "value": "master/iPp25@GRq", - "value_start": 10, - "value_end": 26, - "variable": "pass", + "value": "master/iPp27@GRq", + "value_start": 12, + "value_end": 28, + "variable": "Password", "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.75, @@ -3067,20 +3167,20 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.73262, + "ml_probability": 0.63024, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "ID/Password=master/iPp27@GRq", - "line_num": 28, + "line": "ID/PW:master/iPp28@GRq", + "line_num": 29, "path": "tests/samples/doc_id_passwd_pair", "info": "tests/samples/doc_id_passwd_pair|RAW", - "value": "master/iPp27@GRq", - "value_start": 12, - "value_end": 28, - "variable": "Password", + "value": "master/iPp28@GRq", + "value_start": 6, + "value_end": 22, + "variable": "PW", "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.75, @@ -3089,6 +3189,156 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "Password:Prl23Db#@", + "line_num": 1, + "path": "tests/samples/doc_passwd_pair", + "info": "tests/samples/doc_passwd_pair|RAW", + "value": "Prl23Db#@", + "value_start": 9, + "value_end": 18, + "variable": "Password", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "Password=Prl23Db#@", + "line_num": 4, + "path": "tests/samples/doc_passwd_pair", + "info": "tests/samples/doc_passwd_pair|RAW", + "value": "Prl23Db#@", + "value_start": 9, + "value_end": 18, + "variable": "Password", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "pwd:Prl23Db#@", + "line_num": 5, + "path": "tests/samples/doc_passwd_pair", + "info": "tests/samples/doc_passwd_pair|RAW", + "value": "Prl23Db#@", + "value_start": 4, + "value_end": 13, + "variable": "pwd", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ANY_password=Prl23Db#@", + "line_num": 8, + "path": "tests/samples/doc_passwd_pair", + "info": "tests/samples/doc_passwd_pair|RAW", + "value": "Prl23Db#@", + "value_start": 13, + "value_end": 22, + "variable": "ANY_password", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "pass:Prl23Db#@", + "line_num": 10, + "path": "tests/samples/doc_passwd_pair", + "info": "tests/samples/doc_passwd_pair|RAW", + "value": "Prl23Db#@", + "value_start": 5, + "value_end": 14, + "variable": "pass", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ANY-password=Prl23Db#@", + "line_num": 11, + "path": "tests/samples/doc_passwd_pair", + "info": "tests/samples/doc_passwd_pair|RAW", + "value": "Prl23Db#@", + "value_start": 13, + "value_end": 22, + "variable": "ANY-password", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -3117,19 +3367,19 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.83165, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "master@98.76.54.32 password:Prl23Db#@", - "line_num": 14, + "line": "\"password\":\"Prl23Db#@\"", + "line_num": 21, "path": "tests/samples/doc_passwd_pair", "info": "tests/samples/doc_passwd_pair|RAW", "value": "Prl23Db#@", - "value_start": 28, - "value_end": 37, + "value_start": 12, + "value_end": 21, "variable": "password", "entropy_validation": { "iterator": "BASE64_CHARS", @@ -3142,20 +3392,20 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86952, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "default password:Prl23Db#@", - "line_num": 18, + "line": "passwd=Prl23Db#@", + "line_num": 26, "path": "tests/samples/doc_passwd_pair", "info": "tests/samples/doc_passwd_pair|RAW", "value": "Prl23Db#@", - "value_start": 17, - "value_end": 26, - "variable": "password", + "value_start": 7, + "value_end": 16, + "variable": "passwd", "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 2.4654972233440207, @@ -3167,20 +3417,20 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86952, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "Passwd:Prl23Db#@ Prl23Db#@", - "line_num": 23, + "line": "ANY_password:Prl23Db#@", + "line_num": 34, "path": "tests/samples/doc_passwd_pair", "info": "tests/samples/doc_passwd_pair|RAW", "value": "Prl23Db#@", - "value_start": 7, - "value_end": 16, - "variable": "Passwd", + "value_start": 13, + "value_end": 22, + "variable": "ANY_password", "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 2.4654972233440207, @@ -3192,19 +3442,19 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86952, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "root/Prl23Db#@,root password:Prl23Db#@", - "line_num": 38, + "line": "--password=Prl23Db#@", + "line_num": 37, "path": "tests/samples/doc_passwd_pair", "info": "tests/samples/doc_passwd_pair|RAW", "value": "Prl23Db#@", - "value_start": 29, - "value_end": 38, + "value_start": 11, + "value_end": 20, "variable": "password", "entropy_validation": { "iterator": "BASE64_CHARS", @@ -3217,19 +3467,94 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86952, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "Prl23Db#@ username:Prl23Db#@,Prl23Db#@ password:Prl23Db#@", - "line_num": 40, + "line": "Prl23Db#@:password:Prl23Db#@", + "line_num": 41, "path": "tests/samples/doc_passwd_pair", "info": "tests/samples/doc_passwd_pair|RAW", "value": "Prl23Db#@", - "value_start": 48, - "value_end": 57, + "value_start": 19, + "value_end": 28, + "variable": "password", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ANYpassword=Prl23Db#@", + "line_num": 45, + "path": "tests/samples/doc_passwd_pair", + "info": "tests/samples/doc_passwd_pair|RAW", + "value": "Prl23Db#@", + "value_start": 12, + "value_end": 21, + "variable": "ANYpassword", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "passwords:Prl23Db#@", + "line_num": 46, + "path": "tests/samples/doc_passwd_pair", + "info": "tests/samples/doc_passwd_pair|RAW", + "value": "Prl23Db#@", + "value_start": 10, + "value_end": 19, + "variable": "passwords", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "password=>Prl23Db#@", + "line_num": 48, + "path": "tests/samples/doc_passwd_pair", + "info": "tests/samples/doc_passwd_pair|RAW", + "value": "Prl23Db#@", + "value_start": 10, + "value_end": 19, "variable": "password", "entropy_validation": { "iterator": "BASE64_CHARS", @@ -3242,7 +3567,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.67487, + "ml_probability": 0.99998, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -3267,7 +3592,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.66829, + "ml_probability": 0.99997, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -3292,7 +3617,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93611, + "ml_probability": 0.99978, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -3317,7 +3642,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93611, + "ml_probability": 0.99978, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3342,7 +3667,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8848, + "ml_probability": 1.0, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3367,7 +3692,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8842, + "ml_probability": 0.99999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3392,7 +3717,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.68275, + "ml_probability": 0.99999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -3417,7 +3742,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.76487, + "ml_probability": 0.99998, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3442,7 +3767,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89228, + "ml_probability": 0.99999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3467,7 +3792,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.88738, + "ml_probability": 0.99998, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3492,7 +3817,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.65062, + "ml_probability": 0.99998, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -3517,7 +3842,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.71847, + "ml_probability": 0.99994, "rule": "API", "severity": "medium", "confidence": "moderate", @@ -3542,7 +3867,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.71847, + "ml_probability": 0.99994, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3567,7 +3892,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93888, + "ml_probability": 0.96761, + "rule": "Key", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "access key:AIhq5Xyb1Gga9Q26", + "line_num": 27, + "path": "tests/samples/doc_secret_pair", + "info": "tests/samples/doc_secret_pair|RAW", + "value": "AIhq5Xyb1Gga9Q26", + "value_start": 11, + "value_end": 27, + "variable": "key", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 4.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99983, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -3592,7 +3942,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93888, + "ml_probability": 0.99983, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3617,7 +3967,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9423, + "ml_probability": 0.9984, + "rule": "Key", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ANY_key=AIhq5Xyb1Gga9Q29", + "line_num": 30, + "path": "tests/samples/doc_secret_pair", + "info": "tests/samples/doc_secret_pair|RAW", + "value": "AIhq5Xyb1Gga9Q29", + "value_start": 8, + "value_end": 24, + "variable": "ANY_key", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.875, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99998, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -3642,7 +4017,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9423, + "ml_probability": 0.99998, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3667,7 +4042,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.77193, + "ml_probability": 0.99987, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -3692,7 +4067,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.65814, + "ml_probability": 0.99999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -3714,6 +4089,56 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.96933, + "rule": "Auth", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "Authentication key:AIhq5Xyb1Gga9Q35", + "line_num": 36, + "path": "tests/samples/doc_secret_pair", + "info": "tests/samples/doc_secret_pair|RAW", + "value": "AIhq5Xyb1Gga9Q35", + "value_start": 19, + "value_end": 35, + "variable": "Authentication key", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.875, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.96933, + "rule": "Key", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "Authentication key:AIhq5Xyb1Gga9Q35", + "line_num": 36, + "path": "tests/samples/doc_secret_pair", + "info": "tests/samples/doc_secret_pair|RAW", + "value": "AIhq5Xyb1Gga9Q35", + "value_start": 19, + "value_end": 35, + "variable": "key", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.875, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -3789,6 +4214,56 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.9974, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "master@98.76.54.32(pw:IhqSb1Gg)", + "line_num": 3, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg)", + "value_start": 22, + "value_end": 31, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ID:gildong.hong@example.com mailto:{1} PW:IhqSb1Gg", + "line_num": 4, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 42, + "value_end": 50, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -3814,6 +4289,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.97165, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (PW:IhqSb1Gg)", + "line_num": 6, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg)", + "value_start": 44, + "value_end": 53, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -3842,7 +4342,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3892,7 +4392,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3967,7 +4467,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3989,6 +4489,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.98501, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "\uacc4\uc815:master(PW:IhqSb1Gg)", + "line_num": 16, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg)", + "value_start": 13, + "value_end": 22, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4014,6 +4539,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.96719, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "98.76.54.32(pw:IhqSb1Gg)", + "line_num": 17, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg)", + "value_start": 15, + "value_end": 24, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4042,7 +4592,57 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99942, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "98.76.54.32/pw:IhqSb1Gg", + "line_num": 19, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 15, + "value_end": 23, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ID:gildong.hong@example.com mailto:{1}/pw:IhqSb1Gg", + "line_num": 20, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 42, + "value_end": 50, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4089,6 +4689,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.97165, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "sftp gildong.hong@98.76.54.32 mailto:{1} (pw:IhqSb1Gg)", + "line_num": 22, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg)", + "value_start": 45, + "value_end": 54, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4117,7 +4742,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.97165, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "gildong.hong@98.76.54.32 mailto:{1} (pw:IhqSb1Gg)", + "line_num": 23, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg)", + "value_start": 40, + "value_end": 49, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4167,7 +4817,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4217,7 +4867,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4242,7 +4892,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86154, + "ml_probability": 0.99997, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ANY_password,default:IhqSb1Gg", + "line_num": 31, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 21, + "value_end": 29, + "variable": "ANY_password,default", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99652, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4267,7 +4942,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86154, + "ml_probability": 0.99652, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -4314,6 +4989,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99169, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "98.76.54.32 ANY_PW:IhqSb1Gg", + "line_num": 34, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 19, + "value_end": 27, + "variable": "ANY_PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4339,6 +5039,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.96719, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "98.76.54.32(ID/PW:IhqSb1Gg)", + "line_num": 36, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg)", + "value_start": 18, + "value_end": 27, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4367,7 +5092,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4392,7 +5117,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.80989, + "ml_probability": 0.99967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4439,6 +5164,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.98501, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "id:xxxx(ANYpw:IhqSb1Gg)", + "line_num": 46, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg)", + "value_start": 14, + "value_end": 23, + "variable": "ANYpw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4464,6 +5214,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99995, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "gildong.hong@98.76.54.32,pw:IhqSb1Gg", + "line_num": 47, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 28, + "value_end": 36, + "variable": "gildong.hong@98.76.54.32,pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4564,6 +5339,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.96719, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "98.76.54.32:xxxx(PW:IhqSb1Gg)", + "line_num": 51, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg)", + "value_start": 20, + "value_end": 29, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4589,6 +5389,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "gildong.hong@98.76.54.32 PW:IhqSb1Gg", + "line_num": 56, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 28, + "value_end": 36, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4617,7 +5442,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.80989, + "ml_probability": 0.99967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4642,7 +5467,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.80989, + "ml_probability": 0.99967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4667,7 +5492,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.80989, + "ml_probability": 0.85093, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "Access wifi:xxxx(PW:IhqSb1Gg)", + "line_num": 63, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg)", + "value_start": 20, + "value_end": 29, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4692,7 +5542,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72739, + "ml_probability": 0.99678, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4764,6 +5614,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.9974, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "master@98.76.54.32(pw:IhqSb1Gg)", + "line_num": 73, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg)", + "value_start": 22, + "value_end": 31, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4789,6 +5664,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99995, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "master@98.76.54.32,PW:IhqSb1Gg", + "line_num": 74, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 22, + "value_end": 30, + "variable": "master@98.76.54.32,PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4814,6 +5714,56 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99169, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "98.76.54.32 pw:IhqSb1Gg", + "line_num": 75, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 15, + "value_end": 23, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99976, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "config:xxxx,PW:IhqSb1Gg", + "line_num": 78, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 15, + "value_end": 23, + "variable": "xxxx,PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4864,6 +5814,56 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "scp gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", + "line_num": 82, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 43, + "value_end": 51, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "id:gildong.hong@xxx.com mailto:{1} pw:IhqSb1Gg", + "line_num": 83, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 38, + "value_end": 46, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4889,6 +5889,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", + "line_num": 84, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 39, + "value_end": 47, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4914,6 +5939,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ssh gildong.hong@98.76.54.32 mailto:{1},pw:IhqSb1Gg", + "line_num": 85, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 43, + "value_end": 51, + "variable": ",pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4964,6 +6014,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99948, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", + "line_num": 87, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 45, + "value_end": 53, + "variable": "pwd", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -5039,6 +6114,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", + "line_num": 90, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 44, + "value_end": 52, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -5117,7 +6217,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5167,7 +6267,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5217,7 +6317,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5267,7 +6367,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5339,6 +6439,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", + "line_num": 100, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 43, + "value_end": 51, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -5367,7 +6492,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5392,7 +6517,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5442,7 +6567,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5464,6 +6589,56 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "-ANYID:gildong.hong@example.com mailto:{1} -pw:IhqSb1Gg", + "line_num": 105, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 47, + "value_end": 55, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ID:gildong.hong@xxxx.net mailto:{1} pw:IhqSb1Gg", + "line_num": 106, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg", + "value_start": 39, + "value_end": 47, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -5514,6 +6689,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99733, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "http://98.76.54.32:xxx(pw:IhqSb1Gg)", + "line_num": 108, + "path": "tests/samples/doc_various", + "info": "tests/samples/doc_various|RAW", + "value": "IhqSb1Gg)", + "value_start": 26, + "value_end": 35, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -5614,56 +6814,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92689, - "rule": "Github Old Token", - "severity": "high", - "confidence": "moderate", - "line_data_list": [ - { - "line": "\\ngit_token = \"gireogicracklecrackle1231567190113413981\"\\n\\n", - "line_num": 1, - "path": "tests/samples/encoded_data", - "info": "tests/samples/encoded_data|ENCODED|RAW", - "value": "gireogicracklecrackle1231567190113413981", - "value_start": 15, - "value_end": 55, - "variable": null, - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.97402442086502, - "valid": true - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92689, - "rule": "Token", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "\\ngit_token = \"gireogicracklecrackle1231567190113413981\"\\n\\n", - "line_num": 1, - "path": "tests/samples/encoded_data", - "info": "tests/samples/encoded_data|ENCODED|RAW", - "value": "gireogicracklecrackle1231567190113413981", - "value_start": 15, - "value_end": 55, - "variable": "ngit_token", - "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.97402442086502, - "valid": true - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -5773,8 +6923,8 @@ "confidence": "strong", "line_data_list": [ { - "line": "ghp_00000000000000000000000000000004WZ4EQ", - "line_num": 2, + "line": "ghp_00000000000000000000000000000004WZ4EQ # classic", + "line_num": 1, "path": "tests/samples/github_classic_token", "info": "tests/samples/github_classic_token|RAW", "value": "ghp_00000000000000000000000000000004WZ4EQ", @@ -5817,23 +6967,23 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99921, + "ml_probability": 1.0, "rule": "Github Old Token", "severity": "high", "confidence": "moderate", "line_data_list": [ { - "line": "git_token = \"gireogicracklecrackle1231567190113416781\"", + "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", "line_num": 1, "path": "tests/samples/github_key.groovy", "info": "tests/samples/github_key.groovy|RAW", - "value": "gireogicracklecrackle1231567190113416781", - "value_start": 13, - "value_end": 53, + "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", + "value_start": 23, + "value_end": 63, "variable": null, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 3.992896608419107, + "entropy": 4.246439344671015, "valid": true } } @@ -5842,23 +6992,23 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99921, + "ml_probability": 1.0, "rule": "Token", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "git_token = \"gireogicracklecrackle1231567190113416781\"", + "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", "line_num": 1, "path": "tests/samples/github_key.groovy", "info": "tests/samples/github_key.groovy|RAW", - "value": "gireogicracklecrackle1231567190113416781", - "value_start": 13, - "value_end": 53, - "variable": "git_token", + "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", + "value_start": 23, + "value_end": 63, + "variable": "GITHUB_ACCESS_TOKEN", "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 3.992896608419107, + "entropy": 4.246439344671015, "valid": true } } @@ -6497,7 +7647,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92706, + "ml_probability": 0.99989, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6622,7 +7772,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99868, + "ml_probability": 0.99454, "rule": "JSON Web Token", "severity": "medium", "confidence": "moderate", @@ -6647,7 +7797,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99322, + "ml_probability": 0.99905, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -6672,7 +7822,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99799, + "ml_probability": 1.0, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -6697,7 +7847,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99801, + "ml_probability": 1.0, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -6722,7 +7872,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99794, + "ml_probability": 1.0, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -6747,7 +7897,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.98351, + "ml_probability": 0.99999, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -6769,56 +7919,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.94397, - "rule": "API", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "api_key = \"QMEFk2TZGlkGcOkG-R9UwV1ZmlNkWEYvQzF\"", - "line_num": 1, - "path": "tests/samples/key_value.json", - "info": "tests/samples/key_value.json|STRUCT|KEY_VALUE:`api_key = \"QMEFk2TZGlkGcOkG-R9UwV1ZmlNkWEYvQzF\"`", - "value": "QMEFk2TZGlkGcOkG-R9UwV1ZmlNkWEYvQzF", - "value_start": 11, - "value_end": 46, - "variable": "api_key", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.332592430684724, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.94397, - "rule": "Key", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "api_key = \"QMEFk2TZGlkGcOkG-R9UwV1ZmlNkWEYvQzF\"", - "line_num": 1, - "path": "tests/samples/key_value.json", - "info": "tests/samples/key_value.json|STRUCT|KEY_VALUE:`api_key = \"QMEFk2TZGlkGcOkG-R9UwV1ZmlNkWEYvQzF\"`", - "value": "QMEFk2TZGlkGcOkG-R9UwV1ZmlNkWEYvQzF", - "value_start": 11, - "value_end": 46, - "variable": "api_key", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 4.332592430684724, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -6937,7 +8037,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91803, + "ml_probability": 0.99999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -7027,7 +8127,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91803, + "ml_probability": 0.99999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -7052,24 +8152,24 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.72707, "rule": "Nonce", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": " \"nonce\": \"qPRjfoZWaBPH0KbXMCicm5v1VdG5Hj0DUFMHdSxPOiA\"", - "line_num": 2, + "line": "noncestr = 'fCx4RSNk0mPRxyvZ'", + "line_num": 1, "path": "tests/samples/nonce.hs", "info": "tests/samples/nonce.hs|RAW", - "value": "qPRjfoZWaBPH0KbXMCicm5v1VdG5Hj0DUFMHdSxPOiA", - "value_start": 14, - "value_end": 57, - "variable": "nonce", + "value": "fCx4RSNk0mPRxyvZ", + "value_start": 12, + "value_end": 28, + "variable": "noncestr", "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.9260374290200755, - "valid": true + "entropy": 3.75, + "valid": false } } ] @@ -7102,7 +8202,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.84216, + "ml_probability": 0.9973, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7115,36 +8215,11 @@ "value": "nCzx8A8#!", "value_start": 7, "value_end": 16, - "variable": "my_pw", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.2432750011217983, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.69168, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "val password: String = \"exord13Paw64\", // scala", - "line_num": 3, - "path": "tests/samples/pass_valid", - "info": "tests/samples/pass_valid|RAW", - "value": "exord13Paw64", - "value_start": 24, - "value_end": 36, - "variable": "password", + "variable": "my_pw", "entropy_validation": { - "iterator": "BASE36_CHARS", - "entropy": 3.2862156256610597, - "valid": true + "iterator": "BASE64_CHARS", + "entropy": 2.2432750011217983, + "valid": false } } ] @@ -7152,24 +8227,24 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.95099, + "ml_probability": 0.85188, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "def connect(passwd: str = \"cq2tPr1a2\"): # python default arg", - "line_num": 4, + "line": "val password: String = \"exord13Paw64\", // scala", + "line_num": 3, "path": "tests/samples/pass_valid", "info": "tests/samples/pass_valid|RAW", - "value": "cq2tPr1a2", - "value_start": 27, + "value": "exord13Paw64", + "value_start": 24, "value_end": 36, - "variable": "passwd", + "variable": "password", "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.9477027792200903, - "valid": false + "iterator": "BASE36_CHARS", + "entropy": 3.2862156256610597, + "valid": true } } ] @@ -7177,7 +8252,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.6719, + "ml_probability": 0.98729, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7202,7 +8277,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89322, + "ml_probability": 0.99663, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7227,7 +8302,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9988, + "ml_probability": 0.99962, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7252,7 +8327,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9988, + "ml_probability": 0.99962, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7277,32 +8352,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99755, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "password = \"cackle!\"", - "line_num": 1, - "path": "tests/samples/password.gradle", - "info": "tests/samples/password.gradle|RAW", - "value": "cackle!", - "value_start": 12, - "value_end": 19, - "variable": "password", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.120589933192232, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92706, + "ml_probability": 0.99989, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7327,7 +8377,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92706, + "ml_probability": 0.99989, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7352,7 +8402,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92054, + "ml_probability": 0.99798, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7374,81 +8424,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.84714, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "password = \"cackle_!\"", - "line_num": 1, - "path": "tests/samples/passwords.tar.bz2", - "info": "tests/samples/passwords.tar.bz2|BZIP2|tests/samples/passwords.tar|TAR|./password|RAW", - "value": "cackle_!", - "value_start": 12, - "value_end": 20, - "variable": "password", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.0, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93164, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "password = \"cackle_1!\"", - "line_num": 1, - "path": "tests/samples/passwords.tar.bz2", - "info": "tests/samples/passwords.tar.bz2|BZIP2|tests/samples/passwords.tar|TAR|./1/password|RAW", - "value": "cackle_1!", - "value_start": 12, - "value_end": 21, - "variable": "password", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.2432750011217983, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91005, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "password = \"cackle_2!\"", - "line_num": 1, - "path": "tests/samples/passwords.tar.bz2", - "info": "tests/samples/passwords.tar.bz2|BZIP2|tests/samples/passwords.tar|TAR|./1/2/password|RAW", - "value": "cackle_2!", - "value_start": 12, - "value_end": 21, - "variable": "password", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.2432750011217983, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -8609,31 +9584,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99755, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "pwd = \"cackle!\"", - "line_num": 1, - "path": "tests/samples/pwd.gradle", - "info": "tests/samples/pwd.gradle|RAW", - "value": "cackle!", - "value_start": 7, - "value_end": 14, - "variable": "pwd", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.120589933192232, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -8662,7 +9612,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.98903, + "ml_probability": 0.99994, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -8687,7 +9637,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8243, + "ml_probability": 0.98631, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -8712,7 +9662,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.84325, + "ml_probability": 0.9858, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -8737,7 +9687,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.66606, + "ml_probability": 0.99831, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -8762,32 +9712,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9294, - "rule": "Salt", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "salt4 = {\"salt4\": \"my124%#$@s\\x04clt\\0\"}", - "line_num": 4, - "path": "tests/samples/salt.py", - "info": "tests/samples/salt.py|RAW", - "value": "my124%#$@s\\x04clt\\0", - "value_start": 19, - "value_end": 38, - "variable": "salt4", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.6959504039350857, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.94412, + "ml_probability": 0.9009, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8837,7 +9762,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.94412, + "ml_probability": 0.9009, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8887,7 +9812,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.96866, + "ml_probability": 0.99998, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8934,56 +9859,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99477, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "password = \"WeR15tr0n6\"; ", - "line_num": 1, - "path": "tests/samples/sample.py", - "info": "tests/samples/sample.py|STRUCT|STRUCT:0|KEYWORD:`password = \"WeR15tr0n6\"; `", - "value": "WeR15tr0n6", - "value_start": 12, - "value_end": 22, - "variable": "password", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.321928094887362, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99531, - "rule": "Secret", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "secret = \"cackle!\"", - "line_num": 1, - "path": "tests/samples/secret.gradle", - "info": "tests/samples/secret.gradle|RAW", - "value": "cackle!", - "value_start": 10, - "value_end": 17, - "variable": "secret", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.120589933192232, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -9312,7 +10187,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.98869, + "ml_probability": 0.99771, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -9362,32 +10237,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.81641, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "td : Password: MU$T6Ef09#D!", - "line_num": 1, - "path": "tests/samples/test.html", - "info": "tests/samples/test.html|XML", - "value": "MU$T6Ef09#D!", - "value_start": 26, - "value_end": 38, - "variable": "Password", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.688721875540867, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74042, + "ml_probability": 0.99924, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -9412,7 +10262,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93558, + "ml_probability": 0.99982, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -9437,32 +10287,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.77313, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "password = Cr3DeHTbIal", - "line_num": 42, - "path": "tests/samples/test.html", - "info": "tests/samples/test.html|HTML", - "value": "Cr3DeHTbIal", - "value_start": 11, - "value_end": 22, - "variable": "password", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.459431618637298, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93274, + "ml_probability": 0.86648, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -9487,7 +10312,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87852, + "ml_probability": 0.99997, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -9512,7 +10337,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.7796, + "ml_probability": 0.99967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -9712,7 +10537,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99708, + "ml_probability": 0.98586, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -9737,7 +10562,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99931, + "ml_probability": 0.90001, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -9787,7 +10612,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99672, + "ml_probability": 0.99999, "rule": "URL Credentials", "severity": "high", "confidence": "moderate", @@ -9812,7 +10637,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.78217, + "ml_probability": 0.91551, "rule": "URL Credentials", "severity": "high", "confidence": "moderate", @@ -9833,55 +10658,5 @@ } } ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.95248, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "password = \"cackle!\"", - "line_num": 2, - "path": "tests/samples/xml_data.xml", - "info": "tests/samples/xml_data.xml|RAW", - "value": "cackle!", - "value_start": 18, - "value_end": 25, - "variable": "password", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.120589933192232, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.95248, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "password : cackle!", - "line_num": 1, - "path": "tests/samples/xml_password.xml", - "info": "tests/samples/xml_password.xml|XML", - "value": "cackle!", - "value_start": 11, - "value_end": 18, - "variable": "password", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.120589933192232, - "valid": false - } - } - ] } ] diff --git a/tests/data/doc.json b/tests/data/doc.json index e545ce17a..16024b5a2 100644 --- a/tests/data/doc.json +++ b/tests/data/doc.json @@ -9748,8 +9748,8 @@ "confidence": "strong", "line_data_list": [ { - "line": "ghp_00000000000000000000000000000004WZ4EQ", - "line_num": 2, + "line": "ghp_00000000000000000000000000000004WZ4EQ # classic", + "line_num": 1, "path": "tests/samples/github_classic_token", "info": "tests/samples/github_classic_token|RAW", "value": "ghp_00000000000000000000000000000004WZ4EQ", @@ -9792,23 +9792,23 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99919, + "ml_probability": 1.0, "rule": "Github Old Token", "severity": "high", "confidence": "moderate", "line_data_list": [ { - "line": "git_token = \"gireogicracklecrackle1231567190113416781\"", + "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", "line_num": 1, "path": "tests/samples/github_key.groovy", "info": "tests/samples/github_key.groovy|RAW", - "value": "gireogicracklecrackle1231567190113416781", - "value_start": 13, - "value_end": 53, + "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", + "value_start": 23, + "value_end": 63, "variable": null, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 3.992896608419107, + "entropy": 4.246439344671015, "valid": true } } diff --git a/tests/data/ml_threshold.json b/tests/data/ml_threshold.json index 19aca6da4..3c9181cc0 100644 --- a/tests/data/ml_threshold.json +++ b/tests/data/ml_threshold.json @@ -27,7 +27,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9844, + "ml_probability": 0.9832, "rule": "API", "severity": "medium", "confidence": "moderate", @@ -102,7 +102,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99679, + "ml_probability": 1.0, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -127,7 +127,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99687, + "ml_probability": 0.99595, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -152,7 +152,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.98984, + "ml_probability": 0.99904, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -607,24 +607,24 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99804, + "ml_probability": 0.99566, "rule": "Certificate", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "cert := 'LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMrekND'", + "line": "const certificatePEM := 'LS0tLS1CRUdJTiBDRVJUSUZ0VLQW9JQkFRRFkwYzFUS0I1b1pQd1EKN3QxQ3dNSXJ2cUI2R0lVM3RQeTZSaGNrWlhUa09COFllQldKN1VLZkN6OEhHZJQ0FURS0tLS0tCk1JSUMrekND'", "line_num": 1, "path": "tests/samples/certificate.hs", "info": "", - "value": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMrekND", - "value_start": 9, - "value_end": 57, - "variable": "cert", + "value": "LS0tLS1CRUdJTiBDRVJUSUZ0VLQW9JQkFRRFkwYzFUS0I1b1pQd1EKN3QxQ3dNSXJ2cUI2R0lVM3RQeTZSaGNrWlhUa09COFllQldKN1VLZkN6OEhHZJQ0FURS0tLS0tCk1JSUMrekND", + "value_start": 25, + "value_end": 165, + "variable": "certificatePEM", "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.222794209361695, - "valid": false + "entropy": 5.133473310626378, + "valid": true } } ] @@ -632,7 +632,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92279, + "ml_probability": 0.99999, "rule": "Credential", "severity": "medium", "confidence": "moderate", @@ -707,7 +707,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.43362, + "ml_probability": 0.99855, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -732,7 +732,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.70902, + "ml_probability": 0.97706, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -757,7 +757,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.60745, + "ml_probability": 0.82782, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -782,7 +782,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8653, + "ml_probability": 0.99433, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -807,7 +807,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.62068, + "ml_probability": 0.99879, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -832,7 +832,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.4865, + "ml_probability": 0.99914, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -857,7 +857,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87845, + "ml_probability": 0.99822, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -882,7 +882,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9126, + "ml_probability": 0.99801, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -907,7 +907,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.70976, + "ml_probability": 0.99973, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -932,7 +932,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.90224, + "ml_probability": 0.9934, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -957,7 +957,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.75512, + "ml_probability": 0.99777, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -982,7 +982,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.88595, + "ml_probability": 0.89618, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1007,7 +1007,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89187, + "ml_probability": 0.96988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1032,7 +1032,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.69226, + "ml_probability": 0.99464, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1057,7 +1057,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.736, + "ml_probability": 0.9994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1082,7 +1082,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89888, + "ml_probability": 0.99486, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1107,7 +1107,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92214, + "ml_probability": 0.99329, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1132,7 +1132,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.73325, + "ml_probability": 0.99954, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1157,7 +1157,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91456, + "ml_probability": 0.9947, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1182,7 +1182,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86461, + "ml_probability": 0.95669, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1207,7 +1207,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.88128, + "ml_probability": 0.93823, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1232,7 +1232,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.85087, + "ml_probability": 0.49432, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1257,7 +1257,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.66385, + "ml_probability": 0.29685, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1282,7 +1282,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.63818, + "ml_probability": 0.99761, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1307,7 +1307,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.5829, + "ml_probability": 0.9952, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1332,7 +1332,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8757, + "ml_probability": 0.98699, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1357,7 +1357,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.46538, + "ml_probability": 0.98715, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1382,7 +1382,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87015, + "ml_probability": 0.9925, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1407,7 +1407,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89149, + "ml_probability": 0.98809, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1432,7 +1432,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.85143, + "ml_probability": 0.82063, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1457,7 +1457,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87582, + "ml_probability": 0.92741, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1482,7 +1482,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.6929, + "ml_probability": 0.70792, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1507,7 +1507,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72186, + "ml_probability": 0.97576, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1532,7 +1532,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86912, + "ml_probability": 0.99015, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1557,7 +1557,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.6589, + "ml_probability": 0.99939, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1582,7 +1582,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89053, + "ml_probability": 0.99121, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1607,7 +1607,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.66151, + "ml_probability": 0.9987, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1632,7 +1632,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.71129, + "ml_probability": 0.99779, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1657,7 +1657,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86641, + "ml_probability": 0.77806, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1682,7 +1682,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87283, + "ml_probability": 0.91752, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1707,7 +1707,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.67565, + "ml_probability": 0.98801, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1732,7 +1732,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89277, + "ml_probability": 0.9925, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1757,7 +1757,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87973, + "ml_probability": 0.9868, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1782,7 +1782,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72488, + "ml_probability": 0.99822, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1807,7 +1807,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.7479, + "ml_probability": 0.97112, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1857,7 +1857,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.70815, + "ml_probability": 0.80507, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1882,7 +1882,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.79195, + "ml_probability": 0.98411, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1907,7 +1907,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72954, + "ml_probability": 0.90348, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1932,7 +1932,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89053, + "ml_probability": 0.98951, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1957,7 +1957,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.75145, + "ml_probability": 0.99974, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1982,7 +1982,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89373, + "ml_probability": 0.99776, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2007,7 +2007,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91183, + "ml_probability": 0.99742, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2032,7 +2032,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.71787, + "ml_probability": 0.99982, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2057,7 +2057,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72872, + "ml_probability": 0.99966, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2082,7 +2082,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.662, + "ml_probability": 0.99972, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2107,7 +2107,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.90253, + "ml_probability": 0.99631, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2132,7 +2132,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86741, + "ml_probability": 0.96932, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2157,7 +2157,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87403, + "ml_probability": 0.98897, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2182,7 +2182,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87872, + "ml_probability": 0.98662, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2207,7 +2207,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8957, + "ml_probability": 0.99816, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2232,7 +2232,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89389, + "ml_probability": 0.99701, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2257,7 +2257,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89507, + "ml_probability": 0.9985, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2282,7 +2282,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.52663, + "ml_probability": 0.9986, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2307,7 +2307,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89883, + "ml_probability": 0.9626, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2332,7 +2332,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.90411, + "ml_probability": 0.98081, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2357,7 +2357,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91314, + "ml_probability": 0.99749, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2382,7 +2382,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72298, + "ml_probability": 0.99946, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2407,7 +2407,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.83405, + "ml_probability": 0.96707, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2432,7 +2432,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74746, + "ml_probability": 0.99972, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2457,7 +2457,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.7562, + "ml_probability": 0.9994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2482,7 +2482,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89481, + "ml_probability": 0.99855, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2507,7 +2507,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91281, + "ml_probability": 0.99737, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2532,7 +2532,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72739, + "ml_probability": 0.93882, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2557,7 +2557,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.76693, + "ml_probability": 0.99424, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2582,7 +2582,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.76693, + "ml_probability": 0.99424, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2607,7 +2607,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.68589, + "ml_probability": 0.99975, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2632,7 +2632,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.57198, + "ml_probability": 0.99929, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2657,7 +2657,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.78269, + "ml_probability": 0.98968, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2682,7 +2682,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.77617, + "ml_probability": 0.98985, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2732,7 +2732,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74917, + "ml_probability": 0.79989, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2782,7 +2782,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.81033, + "ml_probability": 0.97325, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2807,7 +2807,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.46557, + "ml_probability": 0.99968, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2832,7 +2832,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.62141, + "ml_probability": 0.99997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2857,7 +2857,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.5786, + "ml_probability": 0.17523, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2882,7 +2882,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.65668, + "ml_probability": 0.35836, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2907,7 +2907,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.73294, + "ml_probability": 0.19959, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2932,7 +2932,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.65287, + "ml_probability": 0.92675, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2957,7 +2957,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.57001, + "ml_probability": 0.3053, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2982,7 +2982,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.81151, + "ml_probability": 0.41885, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3007,7 +3007,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.77054, + "ml_probability": 0.49747, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3057,7 +3057,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.60169, + "ml_probability": 0.07085, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3082,7 +3082,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.51094, + "ml_probability": 0.49332, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3107,7 +3107,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.60001, + "ml_probability": 0.46771, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3157,7 +3157,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.83017, + "ml_probability": 0.04733, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3182,7 +3182,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72484, + "ml_probability": 0.41925, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3207,7 +3207,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.63765, + "ml_probability": 0.68001, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3232,7 +3232,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.56157, + "ml_probability": 0.81183, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3257,7 +3257,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.67153, + "ml_probability": 0.76211, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3282,7 +3282,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.73262, + "ml_probability": 0.7549, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3307,7 +3307,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.47441, + "ml_probability": 0.63024, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3332,7 +3332,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.604, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3357,7 +3357,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.35778, + "ml_probability": 0.5273, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3382,7 +3382,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.604, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3407,7 +3407,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.604, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3432,7 +3432,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.604, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3457,7 +3457,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.604, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3482,7 +3482,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.604, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3532,7 +3532,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.83165, + "ml_probability": 0.05644, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3557,7 +3557,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.35778, + "ml_probability": 0.5273, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3582,7 +3582,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86952, + "ml_probability": 0.16241, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3607,7 +3607,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.604, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3632,7 +3632,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86952, + "ml_probability": 0.16241, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3657,7 +3657,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.0149, + "ml_probability": 1e-05, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3682,7 +3682,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.23927, + "ml_probability": 0.00082, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3707,7 +3707,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.604, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3732,7 +3732,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.01547, + "ml_probability": 0.00049, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3757,7 +3757,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.03533, + "ml_probability": 2e-05, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3782,7 +3782,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.604, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3807,7 +3807,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.604, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3832,7 +3832,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86952, + "ml_probability": 0.16241, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3857,7 +3857,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86952, + "ml_probability": 0.16241, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3882,7 +3882,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.604, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3907,7 +3907,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.604, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3932,7 +3932,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.604, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3957,7 +3957,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.604, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3982,7 +3982,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.45687, + "ml_probability": 0.00223, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4007,7 +4007,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.67487, + "ml_probability": 0.99998, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -4032,7 +4032,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.66829, + "ml_probability": 0.99997, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -4057,7 +4057,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93611, + "ml_probability": 0.99978, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4082,7 +4082,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93611, + "ml_probability": 0.99978, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -4107,7 +4107,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8848, + "ml_probability": 1.0, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -4132,7 +4132,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8842, + "ml_probability": 0.99999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -4157,7 +4157,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.68275, + "ml_probability": 0.99999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -4182,7 +4182,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.76487, + "ml_probability": 0.99998, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -4207,7 +4207,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89228, + "ml_probability": 0.99999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -4232,7 +4232,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.88738, + "ml_probability": 0.99998, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -4257,7 +4257,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.65062, + "ml_probability": 0.99998, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -4282,7 +4282,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.71847, + "ml_probability": 0.99994, "rule": "API", "severity": "medium", "confidence": "moderate", @@ -4307,7 +4307,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.71847, + "ml_probability": 0.99994, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -4332,7 +4332,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.26944, + "ml_probability": 0.96761, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4357,7 +4357,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93888, + "ml_probability": 0.99983, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4382,7 +4382,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93888, + "ml_probability": 0.99983, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -4407,7 +4407,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.17669, + "ml_probability": 0.9984, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4432,7 +4432,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9423, + "ml_probability": 0.99998, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4457,7 +4457,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9423, + "ml_probability": 0.99998, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -4482,7 +4482,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.77193, + "ml_probability": 0.99987, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -4507,7 +4507,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.65814, + "ml_probability": 0.99999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -4532,7 +4532,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.56064, + "ml_probability": 0.96933, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -4557,7 +4557,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.56064, + "ml_probability": 0.96933, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4582,7 +4582,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.04611, + "ml_probability": 0.0, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -4607,7 +4607,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.0, + "ml_probability": 1e-05, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4707,7 +4707,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.41155, + "ml_probability": 0.9974, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4732,7 +4732,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.55291, + "ml_probability": 0.99926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4757,7 +4757,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.25294, + "ml_probability": 0.60267, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4807,7 +4807,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.44517, + "ml_probability": 0.97165, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4857,7 +4857,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4907,7 +4907,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4982,7 +4982,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5007,7 +5007,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.4718, + "ml_probability": 0.98501, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5057,7 +5057,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.41155, + "ml_probability": 0.96719, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5107,7 +5107,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.33717, + "ml_probability": 0.99942, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5132,7 +5132,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.55291, + "ml_probability": 0.99926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5157,7 +5157,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5207,7 +5207,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.44517, + "ml_probability": 0.97165, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5257,7 +5257,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.44517, + "ml_probability": 0.97165, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5282,7 +5282,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5332,7 +5332,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5382,7 +5382,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5407,7 +5407,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.62039, + "ml_probability": 0.99997, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5432,7 +5432,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86154, + "ml_probability": 0.99652, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -5457,7 +5457,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86154, + "ml_probability": 0.99652, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -5507,7 +5507,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.55291, + "ml_probability": 0.99169, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5557,7 +5557,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.41155, + "ml_probability": 0.96719, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5607,7 +5607,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5632,7 +5632,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.80989, + "ml_probability": 0.99967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5682,7 +5682,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.4718, + "ml_probability": 0.98501, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5732,7 +5732,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.33717, + "ml_probability": 0.99995, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5857,7 +5857,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.41155, + "ml_probability": 0.96719, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5882,7 +5882,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.25294, + "ml_probability": 0.60267, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5932,7 +5932,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.55291, + "ml_probability": 0.99926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5982,7 +5982,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.80989, + "ml_probability": 0.99967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6007,7 +6007,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.80989, + "ml_probability": 0.99967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6032,7 +6032,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.52054, + "ml_probability": 0.85093, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6057,7 +6057,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.80989, + "ml_probability": 0.99967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6082,7 +6082,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72739, + "ml_probability": 0.99678, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6132,7 +6132,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.17685, + "ml_probability": 0.02208, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6157,7 +6157,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.16891, + "ml_probability": 0.19502, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6207,7 +6207,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.41155, + "ml_probability": 0.9974, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6257,7 +6257,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.33717, + "ml_probability": 0.99995, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6307,7 +6307,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.55291, + "ml_probability": 0.99169, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6332,7 +6332,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.43105, + "ml_probability": 0.99976, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6407,7 +6407,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.55291, + "ml_probability": 0.99926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6432,7 +6432,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.55291, + "ml_probability": 0.99926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6482,7 +6482,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.55291, + "ml_probability": 0.99926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6532,7 +6532,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.55291, + "ml_probability": 0.99926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6607,7 +6607,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.54815, + "ml_probability": 0.99948, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6707,7 +6707,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.55291, + "ml_probability": 0.99926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6807,7 +6807,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6857,7 +6857,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6907,7 +6907,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6957,7 +6957,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7032,7 +7032,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.55291, + "ml_probability": 0.99926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7082,7 +7082,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7107,7 +7107,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7157,7 +7157,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7182,7 +7182,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.55291, + "ml_probability": 0.99926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7207,7 +7207,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.55291, + "ml_probability": 0.99926, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7282,7 +7282,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.41155, + "ml_probability": 0.99733, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7513,8 +7513,8 @@ "confidence": "strong", "line_data_list": [ { - "line": "ghp_00000000000000000000000000000004WZ4EQ", - "line_num": 2, + "line": "ghp_00000000000000000000000000000004WZ4EQ # classic", + "line_num": 1, "path": "tests/samples/github_classic_token", "info": "", "value": "ghp_00000000000000000000000000000004WZ4EQ", @@ -7557,23 +7557,23 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99921, + "ml_probability": 1.0, "rule": "Github Old Token", "severity": "high", "confidence": "moderate", "line_data_list": [ { - "line": "git_token = \"gireogicracklecrackle1231567190113416781\"", + "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", "line_num": 1, "path": "tests/samples/github_key.groovy", "info": "", - "value": "gireogicracklecrackle1231567190113416781", - "value_start": 13, - "value_end": 53, + "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", + "value_start": 23, + "value_end": 63, "variable": null, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 3.992896608419107, + "entropy": 4.246439344671015, "valid": true } } @@ -7582,23 +7582,23 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99921, + "ml_probability": 1.0, "rule": "Token", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "git_token = \"gireogicracklecrackle1231567190113416781\"", + "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", "line_num": 1, "path": "tests/samples/github_key.groovy", "info": "", - "value": "gireogicracklecrackle1231567190113416781", - "value_start": 13, - "value_end": 53, - "variable": "git_token", + "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", + "value_start": 23, + "value_end": 63, + "variable": "GITHUB_ACCESS_TOKEN", "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 3.992896608419107, + "entropy": 4.246439344671015, "valid": true } } @@ -8237,7 +8237,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92706, + "ml_probability": 0.99989, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8362,7 +8362,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99868, + "ml_probability": 0.99454, "rule": "JSON Web Token", "severity": "medium", "confidence": "moderate", @@ -8387,7 +8387,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99322, + "ml_probability": 0.99905, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -8412,7 +8412,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99799, + "ml_probability": 1.0, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -8437,7 +8437,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99801, + "ml_probability": 1.0, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -8462,7 +8462,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99794, + "ml_probability": 1.0, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -8487,7 +8487,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.98351, + "ml_probability": 0.99999, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -8627,7 +8627,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91803, + "ml_probability": 0.99999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -8717,7 +8717,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91803, + "ml_probability": 0.99999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -8742,24 +8742,24 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.72707, "rule": "Nonce", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": " \"nonce\": \"qPRjfoZWaBPH0KbXMCicm5v1VdG5Hj0DUFMHdSxPOiA\"", - "line_num": 2, + "line": "noncestr = 'fCx4RSNk0mPRxyvZ'", + "line_num": 1, "path": "tests/samples/nonce.hs", "info": "", - "value": "qPRjfoZWaBPH0KbXMCicm5v1VdG5Hj0DUFMHdSxPOiA", - "value_start": 14, - "value_end": 57, - "variable": "nonce", + "value": "fCx4RSNk0mPRxyvZ", + "value_start": 12, + "value_end": 28, + "variable": "noncestr", "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.9260374290200755, - "valid": true + "entropy": 3.75, + "valid": false } } ] @@ -8792,7 +8792,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.01147, + "ml_probability": 1e-05, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8817,7 +8817,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.84216, + "ml_probability": 0.9973, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8842,7 +8842,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.69168, + "ml_probability": 0.85188, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8867,7 +8867,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.95099, + "ml_probability": 0.54187, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8892,7 +8892,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.6719, + "ml_probability": 0.98729, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8917,7 +8917,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89322, + "ml_probability": 0.99663, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8942,7 +8942,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9988, + "ml_probability": 0.99962, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8967,7 +8967,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9988, + "ml_probability": 0.99962, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -8992,7 +8992,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99755, + "ml_probability": 0.00053, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -9017,7 +9017,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92706, + "ml_probability": 0.99989, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -9042,7 +9042,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92706, + "ml_probability": 0.99989, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -9067,7 +9067,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92054, + "ml_probability": 0.99798, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -9382,7 +9382,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99755, + "ml_probability": 0.00053, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -9432,7 +9432,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.98903, + "ml_probability": 0.99994, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -9457,7 +9457,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8243, + "ml_probability": 0.98631, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -9482,7 +9482,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.84325, + "ml_probability": 0.9858, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -9507,7 +9507,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.66606, + "ml_probability": 0.99831, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -9532,7 +9532,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9294, + "ml_probability": 0.40922, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -9557,7 +9557,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99531, + "ml_probability": 0.00016, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -9907,7 +9907,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.85879, + "ml_probability": 1.0, "rule": "JSON Web Token", "severity": "medium", "confidence": "moderate", @@ -9932,7 +9932,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99708, + "ml_probability": 0.98586, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -9957,7 +9957,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99931, + "ml_probability": 0.90001, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -10007,7 +10007,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99672, + "ml_probability": 0.99999, "rule": "URL Credentials", "severity": "high", "confidence": "moderate", @@ -10032,7 +10032,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.78217, + "ml_probability": 0.91551, "rule": "URL Credentials", "severity": "high", "confidence": "moderate", @@ -10057,7 +10057,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.95248, + "ml_probability": 0.28875, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -10082,7 +10082,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.95248, + "ml_probability": 0.28875, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -10107,7 +10107,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.59548, + "ml_probability": 0.10282, "rule": "Password", "severity": "medium", "confidence": "moderate", diff --git a/tests/data/output.json b/tests/data/output.json index e98d7d384..e41cd4f56 100644 --- a/tests/data/output.json +++ b/tests/data/output.json @@ -27,7 +27,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9844, + "ml_probability": 0.9832, "rule": "API", "severity": "medium", "confidence": "moderate", @@ -102,7 +102,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99679, + "ml_probability": 1.0, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -127,7 +127,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99687, + "ml_probability": 0.99595, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -152,7 +152,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.98984, + "ml_probability": 0.99904, "rule": "Auth", "severity": "medium", "confidence": "moderate", @@ -607,24 +607,24 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99804, + "ml_probability": 0.99566, "rule": "Certificate", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "cert := 'LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMrekND'", + "line": "const certificatePEM := 'LS0tLS1CRUdJTiBDRVJUSUZ0VLQW9JQkFRRFkwYzFUS0I1b1pQd1EKN3QxQ3dNSXJ2cUI2R0lVM3RQeTZSaGNrWlhUa09COFllQldKN1VLZkN6OEhHZJQ0FURS0tLS0tCk1JSUMrekND'", "line_num": 1, "path": "tests/samples/certificate.hs", "info": "", - "value": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMrekND", - "value_start": 9, - "value_end": 57, - "variable": "cert", + "value": "LS0tLS1CRUdJTiBDRVJUSUZ0VLQW9JQkFRRFkwYzFUS0I1b1pQd1EKN3QxQ3dNSXJ2cUI2R0lVM3RQeTZSaGNrWlhUa09COFllQldKN1VLZkN6OEhHZJQ0FURS0tLS0tCk1JSUMrekND", + "value_start": 25, + "value_end": 165, + "variable": "certificatePEM", "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.222794209361695, - "valid": false + "entropy": 5.133473310626378, + "valid": true } } ] @@ -632,7 +632,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92279, + "ml_probability": 0.99999, "rule": "Credential", "severity": "medium", "confidence": "moderate", @@ -707,7 +707,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.70902, + "ml_probability": 0.99855, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ID:master,PW:dipPr10Gg!", + "line_num": 1, + "path": "tests/samples/doc_id_pair_passwd_pair", + "info": "", + "value": "dipPr10Gg!", + "value_start": 13, + "value_end": 23, + "variable": "master,PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.989735285398626, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.97706, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -732,7 +757,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8653, + "ml_probability": 0.82782, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ANYID:master PW:dipPr12Gg!", + "line_num": 3, + "path": "tests/samples/doc_id_pair_passwd_pair", + "info": "", + "value": "dipPr12Gg!", + "value_start": 16, + "value_end": 26, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.989735285398626, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99433, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -757,7 +807,57 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87845, + "ml_probability": 0.99879, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "id:master,password:dipPr14Gg!", + "line_num": 5, + "path": "tests/samples/doc_id_pair_passwd_pair", + "info": "", + "value": "dipPr14Gg!", + "value_start": 19, + "value_end": 29, + "variable": "master,password", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.989735285398626, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99914, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ID:master/PW:dipPr15Gg!", + "line_num": 6, + "path": "tests/samples/doc_id_pair_passwd_pair", + "info": "", + "value": "dipPr15Gg!", + "value_start": 13, + "value_end": 23, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.989735285398626, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99822, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -782,7 +882,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9126, + "ml_probability": 0.99801, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -807,7 +907,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.70976, + "ml_probability": 0.99973, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -832,7 +932,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.90224, + "ml_probability": 0.9934, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -857,7 +957,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.75512, + "ml_probability": 0.99777, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -882,7 +982,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.88595, + "ml_probability": 0.89618, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -907,7 +1007,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89187, + "ml_probability": 0.96988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -932,7 +1032,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.69226, + "ml_probability": 0.99464, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -957,7 +1057,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.736, + "ml_probability": 0.9994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -982,7 +1082,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89888, + "ml_probability": 0.99486, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1007,7 +1107,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92214, + "ml_probability": 0.99329, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1032,7 +1132,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.73325, + "ml_probability": 0.99954, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1057,7 +1157,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91456, + "ml_probability": 0.9947, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1082,7 +1182,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86461, + "ml_probability": 0.95669, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1107,7 +1207,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.88128, + "ml_probability": 0.93823, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1132,23 +1232,23 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.85087, + "ml_probability": 0.99761, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "id:master pass:dipPr122Gg!", - "line_num": 23, + "line": "Username:master/Password:dipPr125Gg!", + "line_num": 26, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "", - "value": "dipPr122Gg!", - "value_start": 15, - "value_end": 26, - "variable": "pass", + "value": "dipPr125Gg!", + "value_start": 25, + "value_end": 36, + "variable": "Password", "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.9631196533066344, + "entropy": 3.1449378351248165, "valid": false } } @@ -1157,20 +1257,20 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.66385, + "ml_probability": 0.9952, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "user:master pw:dipPr124Gg!", - "line_num": 25, + "line": "userId:master,password:dipPr126Gg!", + "line_num": 27, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "", - "value": "dipPr124Gg!", - "value_start": 15, - "value_end": 26, - "variable": "pw", + "value": "dipPr126Gg!", + "value_start": 23, + "value_end": 34, + "variable": "master,password", "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.1449378351248165, @@ -1182,19 +1282,19 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.63818, + "ml_probability": 0.98699, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "Username:master/Password:dipPr125Gg!", - "line_num": 26, + "line": "dipPr128Gg! ID:master dipPr128Gg! Password:dipPr128Gg!", + "line_num": 29, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "", - "value": "dipPr125Gg!", - "value_start": 25, - "value_end": 36, + "value": "dipPr128Gg!", + "value_start": 43, + "value_end": 54, "variable": "Password", "entropy_validation": { "iterator": "BASE64_CHARS", @@ -1207,20 +1307,20 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8757, + "ml_probability": 0.98715, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "dipPr128Gg! ID:master dipPr128Gg! Password:dipPr128Gg!", - "line_num": 29, + "line": "ANYid:master,pw:dipPr129Gg!", + "line_num": 30, "path": "tests/samples/doc_id_pair_passwd_pair", "info": "", - "value": "dipPr128Gg!", - "value_start": 43, - "value_end": 54, - "variable": "Password", + "value": "dipPr129Gg!", + "value_start": 16, + "value_end": 27, + "variable": "master,pw", "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.1449378351248165, @@ -1232,7 +1332,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87015, + "ml_probability": 0.9925, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1257,7 +1357,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89149, + "ml_probability": 0.98809, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1282,7 +1382,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.85143, + "ml_probability": 0.82063, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1307,7 +1407,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87582, + "ml_probability": 0.92741, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1332,7 +1432,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.6929, + "ml_probability": 0.70792, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1357,7 +1457,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72186, + "ml_probability": 0.97576, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1382,7 +1482,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86912, + "ml_probability": 0.99015, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1407,7 +1507,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.6589, + "ml_probability": 0.99939, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1432,7 +1532,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89053, + "ml_probability": 0.99121, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1457,7 +1557,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.66151, + "ml_probability": 0.9987, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1482,7 +1582,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.71129, + "ml_probability": 0.99779, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1507,7 +1607,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86641, + "ml_probability": 0.77806, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1532,7 +1632,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87283, + "ml_probability": 0.91752, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1557,7 +1657,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.67565, + "ml_probability": 0.98801, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1582,7 +1682,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89277, + "ml_probability": 0.9925, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1607,7 +1707,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87973, + "ml_probability": 0.9868, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1632,7 +1732,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72488, + "ml_probability": 0.99822, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1657,7 +1757,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.7479, + "ml_probability": 0.97112, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1707,7 +1807,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.70815, + "ml_probability": 0.80507, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1732,7 +1832,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.79195, + "ml_probability": 0.98411, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1757,7 +1857,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72954, + "ml_probability": 0.90348, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1782,7 +1882,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89053, + "ml_probability": 0.98951, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1807,7 +1907,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.75145, + "ml_probability": 0.99974, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1832,7 +1932,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89373, + "ml_probability": 0.99776, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1857,7 +1957,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91183, + "ml_probability": 0.99742, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1882,7 +1982,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.71787, + "ml_probability": 0.99982, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1907,7 +2007,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72872, + "ml_probability": 0.99966, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1932,7 +2032,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.662, + "ml_probability": 0.99972, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1957,7 +2057,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.90253, + "ml_probability": 0.99631, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -1982,7 +2082,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86741, + "ml_probability": 0.96932, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2007,7 +2107,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87403, + "ml_probability": 0.98897, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2032,7 +2132,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.87872, + "ml_probability": 0.98662, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2057,7 +2157,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8957, + "ml_probability": 0.99816, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2082,7 +2182,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89389, + "ml_probability": 0.99701, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2107,7 +2207,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89507, + "ml_probability": 0.9985, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2132,7 +2232,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89883, + "ml_probability": 0.9986, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "Name:master,PW:dipPr169Gg!", + "line_num": 70, + "path": "tests/samples/doc_id_pair_passwd_pair", + "info": "", + "value": "dipPr169Gg!", + "value_start": 15, + "value_end": 26, + "variable": "master,PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.1449378351248165, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.9626, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2157,7 +2282,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.90411, + "ml_probability": 0.98081, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2182,7 +2307,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91314, + "ml_probability": 0.99749, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2207,7 +2332,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72298, + "ml_probability": 0.99946, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2232,7 +2357,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.83405, + "ml_probability": 0.96707, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2257,7 +2382,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74746, + "ml_probability": 0.99972, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2282,7 +2407,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.7562, + "ml_probability": 0.9994, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2307,7 +2432,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89481, + "ml_probability": 0.99855, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2332,7 +2457,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91281, + "ml_probability": 0.99737, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2357,7 +2482,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72739, + "ml_probability": 0.93882, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2382,7 +2507,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.76693, + "ml_probability": 0.99424, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2407,7 +2532,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.76693, + "ml_probability": 0.99424, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2432,7 +2557,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.68589, + "ml_probability": 0.99975, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2457,7 +2582,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.78269, + "ml_probability": 0.99929, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "username:master/pw:dipPr188Gg!", + "line_num": 89, + "path": "tests/samples/doc_id_pair_passwd_pair", + "info": "", + "value": "dipPr188Gg!", + "value_start": 19, + "value_end": 30, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.9631196533066344, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.98968, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2482,7 +2632,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.77617, + "ml_probability": 0.98985, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2532,7 +2682,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74917, + "ml_probability": 0.79989, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2582,7 +2732,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.81033, + "ml_probability": 0.97325, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2607,23 +2757,23 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.65668, + "ml_probability": 0.99968, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "ID/Password:master/iPp2@GRq", - "line_num": 3, - "path": "tests/samples/doc_id_passwd_pair", + "line": "id:master@example.com,pw:dipPr198Gg!", + "line_num": 99, + "path": "tests/samples/doc_id_pair_passwd_pair", "info": "", - "value": "master/iPp2@GRq", - "value_start": 12, - "value_end": 27, - "variable": "Password", + "value": "dipPr198Gg!", + "value_start": 25, + "value_end": 36, + "variable": "master@example.com,pw", "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.64643122256795, + "entropy": 3.1449378351248165, "valid": false } } @@ -2632,23 +2782,23 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.73294, + "ml_probability": 0.99997, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "ID/Pass:master/iPp3@GRq", - "line_num": 4, - "path": "tests/samples/doc_id_passwd_pair", + "line": "id:master@example.com,pw:IHQSB1GG!", + "line_num": 102, + "path": "tests/samples/doc_id_pair_passwd_pair", "info": "", - "value": "master/iPp3@GRq", - "value_start": 8, - "value_end": 23, - "variable": "Pass", + "value": "IHQSB1GG!", + "value_start": 25, + "value_end": 34, + "variable": "master@example.com,pw", "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.64643122256795, + "entropy": 2.595488890170944, "valid": false } } @@ -2657,7 +2807,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.65287, + "ml_probability": 0.92675, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -2679,56 +2829,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.81151, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "username/password:master/iPp7@GRq", - "line_num": 8, - "path": "tests/samples/doc_id_passwd_pair", - "info": "", - "value": "master/iPp7@GRq", - "value_start": 18, - "value_end": 33, - "variable": "password", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.64643122256795, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.77054, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "id/passwd:master/iPp8@GRq", - "line_num": 9, - "path": "tests/samples/doc_id_passwd_pair", - "info": "", - "value": "master/iPp8@GRq", - "value_start": 10, - "value_end": 25, - "variable": "passwd", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 3.64643122256795, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -2782,20 +2882,20 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.83017, + "ml_probability": 0.68001, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "98.76.54.32 id/pw:master/iPp19@GRq", - "line_num": 20, + "line": "ID/PWD:master/iPp21@GRq", + "line_num": 22, "path": "tests/samples/doc_id_passwd_pair", "info": "", - "value": "master/iPp19@GRq", - "value_start": 18, - "value_end": 34, - "variable": "pw", + "value": "master/iPp21@GRq", + "value_start": 7, + "value_end": 23, + "variable": "PWD", "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.75, @@ -2807,23 +2907,23 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72484, + "ml_probability": 0.81183, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "id/pass:master,iPp20@GRq", - "line_num": 21, + "line": "user/pwd:master/iPp22@GRq", + "line_num": 23, "path": "tests/samples/doc_id_passwd_pair", "info": "", - "value": "master,iPp20@GRq", - "value_start": 8, - "value_end": 24, - "variable": "pass", + "value": "master/iPp22@GRq", + "value_start": 9, + "value_end": 25, + "variable": "pwd", "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 3.5, + "entropy": 3.625, "valid": false } } @@ -2832,20 +2932,20 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.63765, + "ml_probability": 0.76211, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "ID/PWD:master/iPp21@GRq", - "line_num": 22, + "line": "user/pass:master/iPp25@GRq", + "line_num": 26, "path": "tests/samples/doc_id_passwd_pair", "info": "", - "value": "master/iPp21@GRq", - "value_start": 7, - "value_end": 23, - "variable": "PWD", + "value": "master/iPp25@GRq", + "value_start": 10, + "value_end": 26, + "variable": "pass", "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.75, @@ -2857,20 +2957,20 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.67153, + "ml_probability": 0.7549, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "user/pass:master/iPp25@GRq", - "line_num": 26, + "line": "ID/Password=master/iPp27@GRq", + "line_num": 28, "path": "tests/samples/doc_id_passwd_pair", "info": "", - "value": "master/iPp25@GRq", - "value_start": 10, - "value_end": 26, - "variable": "pass", + "value": "master/iPp27@GRq", + "value_start": 12, + "value_end": 28, + "variable": "Password", "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.75, @@ -2882,20 +2982,20 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.73262, + "ml_probability": 0.63024, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "ID/Password=master/iPp27@GRq", - "line_num": 28, + "line": "ID/PW:master/iPp28@GRq", + "line_num": 29, "path": "tests/samples/doc_id_passwd_pair", "info": "", - "value": "master/iPp27@GRq", - "value_start": 12, - "value_end": 28, - "variable": "Password", + "value": "master/iPp28@GRq", + "value_start": 6, + "value_end": 22, + "variable": "PW", "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 3.75, @@ -2904,6 +3004,156 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "Password:Prl23Db#@", + "line_num": 1, + "path": "tests/samples/doc_passwd_pair", + "info": "", + "value": "Prl23Db#@", + "value_start": 9, + "value_end": 18, + "variable": "Password", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "Password=Prl23Db#@", + "line_num": 4, + "path": "tests/samples/doc_passwd_pair", + "info": "", + "value": "Prl23Db#@", + "value_start": 9, + "value_end": 18, + "variable": "Password", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "pwd:Prl23Db#@", + "line_num": 5, + "path": "tests/samples/doc_passwd_pair", + "info": "", + "value": "Prl23Db#@", + "value_start": 4, + "value_end": 13, + "variable": "pwd", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ANY_password=Prl23Db#@", + "line_num": 8, + "path": "tests/samples/doc_passwd_pair", + "info": "", + "value": "Prl23Db#@", + "value_start": 13, + "value_end": 22, + "variable": "ANY_password", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "pass:Prl23Db#@", + "line_num": 10, + "path": "tests/samples/doc_passwd_pair", + "info": "", + "value": "Prl23Db#@", + "value_start": 5, + "value_end": 14, + "variable": "pass", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ANY-password=Prl23Db#@", + "line_num": 11, + "path": "tests/samples/doc_passwd_pair", + "info": "", + "value": "Prl23Db#@", + "value_start": 13, + "value_end": 22, + "variable": "ANY-password", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -2932,19 +3182,19 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.83165, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "master@98.76.54.32 password:Prl23Db#@", - "line_num": 14, + "line": "\"password\":\"Prl23Db#@\"", + "line_num": 21, "path": "tests/samples/doc_passwd_pair", "info": "", "value": "Prl23Db#@", - "value_start": 28, - "value_end": 37, + "value_start": 12, + "value_end": 21, "variable": "password", "entropy_validation": { "iterator": "BASE64_CHARS", @@ -2957,20 +3207,20 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86952, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "default password:Prl23Db#@", - "line_num": 18, + "line": "passwd=Prl23Db#@", + "line_num": 26, "path": "tests/samples/doc_passwd_pair", "info": "", "value": "Prl23Db#@", - "value_start": 17, - "value_end": 26, - "variable": "password", + "value_start": 7, + "value_end": 16, + "variable": "passwd", "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 2.4654972233440207, @@ -2982,20 +3232,20 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86952, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "Passwd:Prl23Db#@ Prl23Db#@", - "line_num": 23, + "line": "ANY_password:Prl23Db#@", + "line_num": 34, "path": "tests/samples/doc_passwd_pair", "info": "", "value": "Prl23Db#@", - "value_start": 7, - "value_end": 16, - "variable": "Passwd", + "value_start": 13, + "value_end": 22, + "variable": "ANY_password", "entropy_validation": { "iterator": "BASE64_CHARS", "entropy": 2.4654972233440207, @@ -3007,19 +3257,19 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86952, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "root/Prl23Db#@,root password:Prl23Db#@", - "line_num": 38, + "line": "--password=Prl23Db#@", + "line_num": 37, "path": "tests/samples/doc_passwd_pair", "info": "", "value": "Prl23Db#@", - "value_start": 29, - "value_end": 38, + "value_start": 11, + "value_end": 20, "variable": "password", "entropy_validation": { "iterator": "BASE64_CHARS", @@ -3032,19 +3282,94 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86952, + "ml_probability": 0.63602, "rule": "Password", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "Prl23Db#@ username:Prl23Db#@,Prl23Db#@ password:Prl23Db#@", - "line_num": 40, + "line": "Prl23Db#@:password:Prl23Db#@", + "line_num": 41, "path": "tests/samples/doc_passwd_pair", "info": "", "value": "Prl23Db#@", - "value_start": 48, - "value_end": 57, + "value_start": 19, + "value_end": 28, + "variable": "password", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ANYpassword=Prl23Db#@", + "line_num": 45, + "path": "tests/samples/doc_passwd_pair", + "info": "", + "value": "Prl23Db#@", + "value_start": 12, + "value_end": 21, + "variable": "ANYpassword", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "passwords:Prl23Db#@", + "line_num": 46, + "path": "tests/samples/doc_passwd_pair", + "info": "", + "value": "Prl23Db#@", + "value_start": 10, + "value_end": 19, + "variable": "passwords", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.4654972233440207, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.63602, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "password=>Prl23Db#@", + "line_num": 48, + "path": "tests/samples/doc_passwd_pair", + "info": "", + "value": "Prl23Db#@", + "value_start": 10, + "value_end": 19, "variable": "password", "entropy_validation": { "iterator": "BASE64_CHARS", @@ -3057,7 +3382,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.67487, + "ml_probability": 0.99998, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -3082,7 +3407,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.66829, + "ml_probability": 0.99997, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -3107,7 +3432,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93611, + "ml_probability": 0.99978, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -3132,7 +3457,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93611, + "ml_probability": 0.99978, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3157,7 +3482,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8848, + "ml_probability": 1.0, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3182,7 +3507,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8842, + "ml_probability": 0.99999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3207,7 +3532,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.68275, + "ml_probability": 0.99999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -3232,7 +3557,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.76487, + "ml_probability": 0.99998, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3257,7 +3582,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89228, + "ml_probability": 0.99999, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3282,7 +3607,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.88738, + "ml_probability": 0.99998, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3307,7 +3632,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.65062, + "ml_probability": 0.99998, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -3332,7 +3657,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.71847, + "ml_probability": 0.99994, "rule": "API", "severity": "medium", "confidence": "moderate", @@ -3357,7 +3682,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.71847, + "ml_probability": 0.99994, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3382,7 +3707,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93888, + "ml_probability": 0.96761, + "rule": "Key", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "access key:AIhq5Xyb1Gga9Q26", + "line_num": 27, + "path": "tests/samples/doc_secret_pair", + "info": "", + "value": "AIhq5Xyb1Gga9Q26", + "value_start": 11, + "value_end": 27, + "variable": "key", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 4.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99983, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -3407,7 +3757,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.93888, + "ml_probability": 0.99983, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3432,7 +3782,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9423, + "ml_probability": 0.9984, + "rule": "Key", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ANY_key=AIhq5Xyb1Gga9Q29", + "line_num": 30, + "path": "tests/samples/doc_secret_pair", + "info": "", + "value": "AIhq5Xyb1Gga9Q29", + "value_start": 8, + "value_end": 24, + "variable": "ANY_key", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.875, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99998, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -3457,7 +3832,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9423, + "ml_probability": 0.99998, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -3482,7 +3857,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.77193, + "ml_probability": 0.99987, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -3507,7 +3882,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.65814, + "ml_probability": 0.99999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -3529,6 +3904,56 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.96933, + "rule": "Auth", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "Authentication key:AIhq5Xyb1Gga9Q35", + "line_num": 36, + "path": "tests/samples/doc_secret_pair", + "info": "", + "value": "AIhq5Xyb1Gga9Q35", + "value_start": 19, + "value_end": 35, + "variable": "Authentication key", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.875, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.96933, + "rule": "Key", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "Authentication key:AIhq5Xyb1Gga9Q35", + "line_num": 36, + "path": "tests/samples/doc_secret_pair", + "info": "", + "value": "AIhq5Xyb1Gga9Q35", + "value_start": 19, + "value_end": 35, + "variable": "key", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.875, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -3604,6 +4029,56 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.9974, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "master@98.76.54.32(pw:IhqSb1Gg)", + "line_num": 3, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg)", + "value_start": 22, + "value_end": 31, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ID:gildong.hong@example.com mailto:{1} PW:IhqSb1Gg", + "line_num": 4, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 42, + "value_end": 50, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -3629,6 +4104,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.97165, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ssh gildong.hong@98.76.54.32 mailto:{1} (PW:IhqSb1Gg)", + "line_num": 6, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg)", + "value_start": 44, + "value_end": 53, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -3657,7 +4157,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3707,7 +4207,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3782,7 +4282,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3804,6 +4304,81 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.98501, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "\uacc4\uc815:master(PW:IhqSb1Gg)", + "line_num": 16, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg)", + "value_start": 13, + "value_end": 22, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "IPv4", + "severity": "info", + "confidence": "weak", + "line_data_list": [ + { + "line": "98.76.54.32(pw:IhqSb1Gg)", + "line_num": 17, + "path": "tests/samples/doc_various", + "info": "", + "value": "98.76.54.32", + "value_start": 0, + "value_end": 11, + "variable": null, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.515950268099853, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.96719, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "98.76.54.32(pw:IhqSb1Gg)", + "line_num": 17, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg)", + "value_start": 15, + "value_end": 24, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -3813,8 +4388,8 @@ "confidence": "weak", "line_data_list": [ { - "line": "98.76.54.32(pw:IhqSb1Gg)", - "line_num": 17, + "line": "98.76.54.32/pw:IhqSb1Gg", + "line_num": 19, "path": "tests/samples/doc_various", "info": "", "value": "98.76.54.32", @@ -3831,24 +4406,49 @@ }, { "api_validation": "NOT_AVAILABLE", - "ml_validation": "NOT_AVAILABLE", - "ml_probability": null, - "rule": "IPv4", - "severity": "info", - "confidence": "weak", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99942, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", "line_data_list": [ { "line": "98.76.54.32/pw:IhqSb1Gg", "line_num": 19, "path": "tests/samples/doc_various", "info": "", - "value": "98.76.54.32", - "value_start": 0, - "value_end": 11, - "variable": null, + "value": "IhqSb1Gg", + "value_start": 15, + "value_end": 23, + "variable": "pw", "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 3.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ID:gildong.hong@example.com mailto:{1}/pw:IhqSb1Gg", + "line_num": 20, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 42, + "value_end": 50, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, "valid": false } } @@ -3857,7 +4457,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3904,6 +4504,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.97165, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "sftp gildong.hong@98.76.54.32 mailto:{1} (pw:IhqSb1Gg)", + "line_num": 22, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg)", + "value_start": 45, + "value_end": 54, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -3932,7 +4557,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.97165, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "gildong.hong@98.76.54.32 mailto:{1} (pw:IhqSb1Gg)", + "line_num": 23, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg)", + "value_start": 40, + "value_end": 49, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -3982,7 +4632,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4032,7 +4682,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4057,7 +4707,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86154, + "ml_probability": 0.99997, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ANY_password,default:IhqSb1Gg", + "line_num": 31, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 21, + "value_end": 29, + "variable": "ANY_password,default", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99652, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -4082,7 +4757,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.86154, + "ml_probability": 0.99652, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -4129,6 +4804,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99169, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "98.76.54.32 ANY_PW:IhqSb1Gg", + "line_num": 34, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 19, + "value_end": 27, + "variable": "ANY_PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4154,6 +4854,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.96719, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "98.76.54.32(ID/PW:IhqSb1Gg)", + "line_num": 36, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg)", + "value_start": 18, + "value_end": 27, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4182,7 +4907,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4207,7 +4932,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.80989, + "ml_probability": 0.99967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4254,6 +4979,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.98501, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "id:xxxx(ANYpw:IhqSb1Gg)", + "line_num": 46, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg)", + "value_start": 14, + "value_end": 23, + "variable": "ANYpw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4279,6 +5029,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99995, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "gildong.hong@98.76.54.32,pw:IhqSb1Gg", + "line_num": 47, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 28, + "value_end": 36, + "variable": "gildong.hong@98.76.54.32,pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4379,6 +5154,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.96719, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "98.76.54.32:xxxx(PW:IhqSb1Gg)", + "line_num": 51, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg)", + "value_start": 20, + "value_end": 29, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4398,7 +5198,32 @@ "variable": null, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.515950268099853, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "gildong.hong@98.76.54.32 PW:IhqSb1Gg", + "line_num": 56, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 28, + "value_end": 36, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, "valid": false } } @@ -4432,7 +5257,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.80989, + "ml_probability": 0.99967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4457,7 +5282,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.80989, + "ml_probability": 0.99967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4482,7 +5307,32 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.80989, + "ml_probability": 0.85093, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "Access wifi:xxxx(PW:IhqSb1Gg)", + "line_num": 63, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg)", + "value_start": 20, + "value_end": 29, + "variable": "PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99967, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4507,7 +5357,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.72739, + "ml_probability": 0.99678, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4579,6 +5429,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.9974, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "master@98.76.54.32(pw:IhqSb1Gg)", + "line_num": 73, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg)", + "value_start": 22, + "value_end": 31, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4604,6 +5479,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99995, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "master@98.76.54.32,PW:IhqSb1Gg", + "line_num": 74, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 22, + "value_end": 30, + "variable": "master@98.76.54.32,PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4629,6 +5529,56 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99169, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "98.76.54.32 pw:IhqSb1Gg", + "line_num": 75, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 15, + "value_end": 23, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99976, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "config:xxxx,PW:IhqSb1Gg", + "line_num": 78, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 15, + "value_end": 23, + "variable": "xxxx,PW", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4679,6 +5629,56 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "scp gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", + "line_num": 82, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 43, + "value_end": 51, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "id:gildong.hong@xxx.com mailto:{1} pw:IhqSb1Gg", + "line_num": 83, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 38, + "value_end": 46, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4704,6 +5704,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", + "line_num": 84, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 39, + "value_end": 47, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4729,6 +5754,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ssh gildong.hong@98.76.54.32 mailto:{1},pw:IhqSb1Gg", + "line_num": 85, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 43, + "value_end": 51, + "variable": ",pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4779,6 +5829,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99948, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "(ssh gildong.hong@98.76.54.32 mailto{1}) pwd:IhqSb1Gg", + "line_num": 87, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 45, + "value_end": 53, + "variable": "pwd", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -4848,7 +5923,32 @@ "variable": null, "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 2.515950268099853, + "entropy": 2.515950268099853, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ssh gildong.hong@98.76.54.32 mailto:{1}, pw:IhqSb1Gg", + "line_num": 90, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 44, + "value_end": 52, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, "valid": false } } @@ -4932,7 +6032,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -4982,7 +6082,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5032,7 +6132,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5082,7 +6182,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74047, + "ml_probability": 0.99156, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5154,6 +6254,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ssh gildong.hong@98.76.54.32 mailto:{1} pw:IhqSb1Gg", + "line_num": 100, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 43, + "value_end": 51, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -5182,7 +6307,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5207,7 +6332,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5257,7 +6382,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.74398, + "ml_probability": 0.99988, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -5279,6 +6404,56 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "-ANYID:gildong.hong@example.com mailto:{1} -pw:IhqSb1Gg", + "line_num": 105, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 47, + "value_end": 55, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99926, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "ID:gildong.hong@xxxx.net mailto:{1} pw:IhqSb1Gg", + "line_num": 106, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg", + "value_start": 39, + "value_end": 47, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 3.0, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -5329,6 +6504,31 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "VALIDATED_KEY", + "ml_probability": 0.99733, + "rule": "Password", + "severity": "medium", + "confidence": "moderate", + "line_data_list": [ + { + "line": "http://98.76.54.32:xxx(pw:IhqSb1Gg)", + "line_num": 108, + "path": "tests/samples/doc_various", + "info": "", + "value": "IhqSb1Gg)", + "value_start": 26, + "value_end": 35, + "variable": "pw", + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 2.8177111123931664, + "valid": false + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -5538,8 +6738,8 @@ "confidence": "strong", "line_data_list": [ { - "line": "ghp_00000000000000000000000000000004WZ4EQ", - "line_num": 2, + "line": "ghp_00000000000000000000000000000004WZ4EQ # classic", + "line_num": 1, "path": "tests/samples/github_classic_token", "info": "", "value": "ghp_00000000000000000000000000000004WZ4EQ", @@ -5582,23 +6782,23 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99921, + "ml_probability": 1.0, "rule": "Github Old Token", "severity": "high", "confidence": "moderate", "line_data_list": [ { - "line": "git_token = \"gireogicracklecrackle1231567190113416781\"", + "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", "line_num": 1, "path": "tests/samples/github_key.groovy", "info": "", - "value": "gireogicracklecrackle1231567190113416781", - "value_start": 13, - "value_end": 53, + "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", + "value_start": 23, + "value_end": 63, "variable": null, "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 3.992896608419107, + "entropy": 4.246439344671015, "valid": true } } @@ -5607,23 +6807,23 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99921, + "ml_probability": 1.0, "rule": "Token", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": "git_token = \"gireogicracklecrackle1231567190113416781\"", + "line": "GITHUB_ACCESS_TOKEN = \"lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy\"", "line_num": 1, "path": "tests/samples/github_key.groovy", "info": "", - "value": "gireogicracklecrackle1231567190113416781", - "value_start": 13, - "value_end": 53, - "variable": "git_token", + "value": "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy", + "value_start": 23, + "value_end": 63, + "variable": "GITHUB_ACCESS_TOKEN", "entropy_validation": { "iterator": "BASE36_CHARS", - "entropy": 3.992896608419107, + "entropy": 4.246439344671015, "valid": true } } @@ -6262,7 +7462,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92706, + "ml_probability": 0.99989, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6387,7 +7587,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99868, + "ml_probability": 0.99454, "rule": "JSON Web Token", "severity": "medium", "confidence": "moderate", @@ -6412,7 +7612,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99322, + "ml_probability": 0.99905, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -6437,7 +7637,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99799, + "ml_probability": 1.0, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -6462,7 +7662,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99801, + "ml_probability": 1.0, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -6487,7 +7687,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99794, + "ml_probability": 1.0, "rule": "Secret", "severity": "medium", "confidence": "moderate", @@ -6512,7 +7712,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.98351, + "ml_probability": 0.99999, "rule": "Key", "severity": "medium", "confidence": "moderate", @@ -6652,7 +7852,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91803, + "ml_probability": 0.99999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -6742,7 +7942,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.91803, + "ml_probability": 0.99999, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -6767,24 +7967,24 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.995, + "ml_probability": 0.72707, "rule": "Nonce", "severity": "medium", "confidence": "moderate", "line_data_list": [ { - "line": " \"nonce\": \"qPRjfoZWaBPH0KbXMCicm5v1VdG5Hj0DUFMHdSxPOiA\"", - "line_num": 2, + "line": "noncestr = 'fCx4RSNk0mPRxyvZ'", + "line_num": 1, "path": "tests/samples/nonce.hs", "info": "", - "value": "qPRjfoZWaBPH0KbXMCicm5v1VdG5Hj0DUFMHdSxPOiA", - "value_start": 14, - "value_end": 57, - "variable": "nonce", + "value": "fCx4RSNk0mPRxyvZ", + "value_start": 12, + "value_end": 28, + "variable": "noncestr", "entropy_validation": { "iterator": "BASE64_CHARS", - "entropy": 4.9260374290200755, - "valid": true + "entropy": 3.75, + "valid": false } } ] @@ -6817,7 +8017,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.84216, + "ml_probability": 0.9973, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6842,7 +8042,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.69168, + "ml_probability": 0.85188, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6867,32 +8067,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.95099, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "def connect(passwd: str = \"cq2tPr1a2\"): # python default arg", - "line_num": 4, - "path": "tests/samples/pass_valid", - "info": "", - "value": "cq2tPr1a2", - "value_start": 27, - "value_end": 36, - "variable": "passwd", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.9477027792200903, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.6719, + "ml_probability": 0.98729, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6917,7 +8092,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.89322, + "ml_probability": 0.99663, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6942,7 +8117,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9988, + "ml_probability": 0.99962, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6967,7 +8142,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9988, + "ml_probability": 0.99962, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -6992,32 +8167,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99755, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "password = \"cackle!\"", - "line_num": 1, - "path": "tests/samples/password.gradle", - "info": "", - "value": "cackle!", - "value_start": 12, - "value_end": 19, - "variable": "password", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.120589933192232, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92706, + "ml_probability": 0.99989, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7042,7 +8192,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92706, + "ml_probability": 0.99989, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7067,7 +8217,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.92054, + "ml_probability": 0.99798, "rule": "Password", "severity": "medium", "confidence": "moderate", @@ -7379,31 +8529,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99755, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "pwd = \"cackle!\"", - "line_num": 1, - "path": "tests/samples/pwd.gradle", - "info": "", - "value": "cackle!", - "value_start": 7, - "value_end": 14, - "variable": "pwd", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.120589933192232, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -7432,7 +8557,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.98903, + "ml_probability": 0.99994, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -7457,7 +8582,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.8243, + "ml_probability": 0.98631, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -7482,7 +8607,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.84325, + "ml_probability": 0.9858, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -7507,7 +8632,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.66606, + "ml_probability": 0.99831, "rule": "Salt", "severity": "medium", "confidence": "moderate", @@ -7529,56 +8654,6 @@ } ] }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.9294, - "rule": "Salt", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "salt4 = {\"salt4\": \"my124%#$@s\\x04clt\\0\"}", - "line_num": 4, - "path": "tests/samples/salt.py", - "info": "", - "value": "my124%#$@s\\x04clt\\0", - "value_start": 19, - "value_end": 38, - "variable": "salt4", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.6959504039350857, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99531, - "rule": "Secret", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "secret = \"cackle!\"", - "line_num": 1, - "path": "tests/samples/secret.gradle", - "info": "", - "value": "cackle!", - "value_start": 10, - "value_end": 17, - "variable": "secret", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.120589933192232, - "valid": false - } - } - ] - }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -7907,7 +8982,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.85879, + "ml_probability": 1.0, "rule": "JSON Web Token", "severity": "medium", "confidence": "moderate", @@ -7932,7 +9007,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99708, + "ml_probability": 0.98586, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -7957,7 +9032,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99931, + "ml_probability": 0.90001, "rule": "Token", "severity": "medium", "confidence": "moderate", @@ -8007,7 +9082,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.99672, + "ml_probability": 0.99999, "rule": "URL Credentials", "severity": "high", "confidence": "moderate", @@ -8032,7 +9107,7 @@ { "api_validation": "NOT_AVAILABLE", "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.78217, + "ml_probability": 0.91551, "rule": "URL Credentials", "severity": "high", "confidence": "moderate", @@ -8053,55 +9128,5 @@ } } ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.95248, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "data : password = \"cackle!\"", - "line_num": 2, - "path": "tests/samples/xml_data.xml", - "info": "", - "value": "cackle!", - "value_start": 19, - "value_end": 26, - "variable": "password", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.120589933192232, - "valid": false - } - } - ] - }, - { - "api_validation": "NOT_AVAILABLE", - "ml_validation": "VALIDATED_KEY", - "ml_probability": 0.95248, - "rule": "Password", - "severity": "medium", - "confidence": "moderate", - "line_data_list": [ - { - "line": "password : cackle!", - "line_num": 5, - "path": "tests/samples/xml_password.xml", - "info": "", - "value": "cackle!", - "value_start": 11, - "value_end": 18, - "variable": "password", - "entropy_validation": { - "iterator": "BASE64_CHARS", - "entropy": 2.120589933192232, - "valid": false - } - } - ] } ] diff --git a/tests/ml_model/test_ml_validator.py b/tests/ml_model/test_ml_validator.py index 97a6f9cea..1173144a2 100644 --- a/tests/ml_model/test_ml_validator.py +++ b/tests/ml_model/test_ml_validator.py @@ -1,49 +1,48 @@ +import unittest + from credsweeper import ThresholdPreset from credsweeper.app import APP_PATH - from credsweeper.config import Config from credsweeper.credentials import Candidate from credsweeper.ml_model import MlValidator from credsweeper.utils import Util -def test_ml_validator_simple_p(): - ml_validator = MlValidator(threshold=ThresholdPreset.medium) - assert ml_validator is not None - file_name = APP_PATH / "secret" / "config.json" - config_dict = Util.json_load(file_name) - config_dict["validation"] = {} - config_dict["validation"]["api_validation"] = False - config_dict["use_filters"] = True - config_dict["find_by_ext"] = False - config_dict["depth"] = 0 - config_dict["doc"] = False - config_dict["find_by_ext_list"] = [] - config_dict["size_limit"] = None - config = Config(config_dict) - candidate = Candidate.get_dummy_candidate(config, "test.py", ".py", "test_info") - candidate.line_data_list[0].line = '"geheimnis" : "Jhd2gH5634"' - candidate.line_data_list[0].variable = "geheimnis" - candidate.line_data_list[0].value = "Jhd2gH5634" - - decision, probability = ml_validator.validate(candidate) - assert 0.919577 < probability < 0.919578 - assert decision - - candidate.line_data_list[0].path = "test.yaml" - candidate.line_data_list[0].file_type = ".yaml" - decision, probability = ml_validator.validate(candidate) - assert 0.240346 < probability < 0.240347 - assert not decision - - candidate.line_data_list[0].path = "test.zip" - candidate.line_data_list[0].file_type = ".zip" - decision, probability = ml_validator.validate(candidate) - assert 0.51862 < probability < 0.51864 - assert not decision - - candidate.line_data_list[0].path = "test.zip bla bla bla" - candidate.line_data_list[0].file_type = ".py" - decision, probability = ml_validator.validate(candidate) - assert 0.919577 < probability < 0.919578 - assert decision +class TestMlValidator(unittest.TestCase): + + def test_ml_validator_simple_p(self): + ml_validator = MlValidator(threshold=ThresholdPreset.medium) + assert ml_validator is not None + file_name = APP_PATH / "secret" / "config.json" + config_dict = Util.json_load(file_name) + config_dict["validation"] = {} + config_dict["validation"]["api_validation"] = False + config_dict["use_filters"] = True + config_dict["find_by_ext"] = False + config_dict["depth"] = 0 + config_dict["doc"] = False + config_dict["find_by_ext_list"] = [] + config_dict["size_limit"] = None + config = Config(config_dict) + candidate = Candidate.get_dummy_candidate(config, "main.py", ".py", "test_info") + candidate.line_data_list[0].line = '"geheimnis" : "Jhd2gH5634"' + candidate.line_data_list[0].variable = "geheimnis" + candidate.line_data_list[0].value = "Jhd2gH5634" + + decision, probability = ml_validator.validate(candidate) + self.assertAlmostEqual(probability, 0.9926, delta=0.0001) + + candidate.line_data_list[0].path = "sample.py" + candidate.line_data_list[0].file_type = ".yaml" + decision, probability = ml_validator.validate(candidate) + self.assertAlmostEqual(probability, 0.9801, delta=0.0001) + + candidate.line_data_list[0].path = "test.zip" + candidate.line_data_list[0].file_type = ".zip" + decision, probability = ml_validator.validate(candidate) + self.assertAlmostEqual(probability, 0.9730, delta=0.0001) + + candidate.line_data_list[0].path = "other.txt" + candidate.line_data_list[0].file_type = ".txt" + decision, probability = ml_validator.validate(candidate) + self.assertAlmostEqual(probability, 0.9122, delta=0.0001) diff --git a/tests/samples/certificate.hs b/tests/samples/certificate.hs index ed8c5501c..9164cf0e7 100644 --- a/tests/samples/certificate.hs +++ b/tests/samples/certificate.hs @@ -1 +1 @@ -cert := 'LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMrekND' \ No newline at end of file +const certificatePEM := 'LS0tLS1CRUdJTiBDRVJUSUZ0VLQW9JQkFRRFkwYzFUS0I1b1pQd1EKN3QxQ3dNSXJ2cUI2R0lVM3RQeTZSaGNrWlhUa09COFllQldKN1VLZkN6OEhHZJQ0FURS0tLS0tCk1JSUMrekND' diff --git a/tests/samples/github_classic_token b/tests/samples/github_classic_token index d613bb350..3aeb50f62 100644 --- a/tests/samples/github_classic_token +++ b/tests/samples/github_classic_token @@ -1,4 +1,2 @@ -# Classic -ghp_00000000000000000000000000000004WZ4EQ -# random -ghp_1010110010001010101001010101010101011001 +ghp_00000000000000000000000000000004WZ4EQ # classic +ghp_1010110010001010101001010101010101011001 # random diff --git a/tests/samples/github_key.groovy b/tests/samples/github_key.groovy index d31e2ccac..b76d6a30a 100644 --- a/tests/samples/github_key.groovy +++ b/tests/samples/github_key.groovy @@ -1 +1 @@ -git_token = "gireogicracklecrackle1231567190113416781" +GITHUB_ACCESS_TOKEN = "lbyxnhqjfnzhlpnvcvaxjumgvdlnktgugdofmzyy" diff --git a/tests/samples/nonce.hs b/tests/samples/nonce.hs index 714598981..5587ad232 100644 --- a/tests/samples/nonce.hs +++ b/tests/samples/nonce.hs @@ -1,3 +1 @@ -{ - "nonce": "qPRjfoZWaBPH0KbXMCicm5v1VdG5Hj0DUFMHdSxPOiA" -} \ No newline at end of file +noncestr = 'fCx4RSNk0mPRxyvZ' diff --git a/tests/test_app.py b/tests/test_app.py index c849825dd..cc02d5027 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -40,7 +40,7 @@ def transform(x: AnyStr) -> str: return transform(_stdout), transform(_stderr) def test_it_works_p(self) -> None: - target_path = str(SAMPLES_PATH / "password.gradle") + target_path = str(SAMPLES_PATH / "passwd.groovy") _stdout, _stderr = self._m_credsweeper(["--path", target_path, "--log", "silence"]) output = " ".join(_stdout.split()[:-1]) @@ -49,14 +49,25 @@ def test_it_works_p(self) -> None: | severity: medium | confidence: moderate | line_data_list: - [line: 'password = \"cackle!\"' + [line: 'gi_reo_gi_passwd = "cAc48k1Zd7"; password_confirmation = "cAc48k1Zd7";' | line_num: 1 | path: {target_path} - | value: 'cackle!' - | entropy_validation: BASE64_CHARS 2.120590 False] + | value: 'cAc48k1Zd7' + | entropy_validation: BASE64_CHARS 3.121928 False] | api_validation: NOT_AVAILABLE | ml_validation: VALIDATED_KEY\n - Detected Credentials: 1\n + rule: Password + | severity: medium + | confidence: moderate + | line_data_list: + [line: 'gi_reo_gi_passwd = "cAc48k1Zd7"; password_confirmation = "cAc48k1Zd7";' + | line_num: 1 + | path: {target_path} + | value: 'cAc48k1Zd7' + | entropy_validation: BASE64_CHARS 3.121928 False] + | api_validation: NOT_AVAILABLE + | ml_validation: VALIDATED_KEY\n + Detected Credentials: 2\n Time Elapsed: """ expected = " ".join(expected.split()) @@ -530,45 +541,21 @@ def test_depth_n(self) -> None: # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # - def test_denylist_value_p(self) -> None: - target_path = str(SAMPLES_PATH / "password.gradle") + def test_denylist_p(self) -> None: + target_path = str(SAMPLES_PATH / "github_classic_token") with tempfile.TemporaryDirectory() as tmp_dir: json_filename = os.path.join(tmp_dir, f"{__name__}.json") denylist_filename = os.path.join(tmp_dir, "list.txt") with open(denylist_filename, "w") as f: - f.write("cackle!") + f.write('ghp_00000000000000000000000000000004WZ4EQ # classic') # full line _stdout, _stderr = self._m_credsweeper([ "--path", target_path, "--denylist", denylist_filename, "--save-json", json_filename, "--log", "silence" ]) with open(json_filename, "r") as json_file: report = json.load(json_file) self.assertEqual(0, len(report)) - - # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # - - def test_denylist_value_n(self) -> None: - target_path = str(SAMPLES_PATH / "password.gradle") - with tempfile.TemporaryDirectory() as tmp_dir: - json_filename = os.path.join(tmp_dir, f"{__name__}.json") - denylist_filename = os.path.join(tmp_dir, "list.txt") - with open(denylist_filename, "w") as f: - f.write("abc") - _stdout, _stderr = self._m_credsweeper([ - "--path", target_path, "--denylist", denylist_filename, "--save-json", json_filename, "--log", "silence" - ]) - with open(json_filename, "r") as json_file: - report = json.load(json_file) - self.assertEqual(1, len(report)) - - # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # - - def test_denylist_line_p(self) -> None: - target_path = str(SAMPLES_PATH / "password.gradle") - with tempfile.TemporaryDirectory() as tmp_dir: - json_filename = os.path.join(tmp_dir, f"{__name__}.json") - denylist_filename = os.path.join(tmp_dir, "list.txt") with open(denylist_filename, "w") as f: - f.write(' password = "cackle!" ') + f.write('ghp_00000000000000000000000000000004WZ4EQ') # value only _stdout, _stderr = self._m_credsweeper([ "--path", target_path, "--denylist", denylist_filename, "--save-json", json_filename, "--log", "silence" ]) @@ -578,13 +565,13 @@ def test_denylist_line_p(self) -> None: # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # - def test_denylist_line_n(self) -> None: - target_path = str(SAMPLES_PATH / "password.gradle") + def test_denylist_n(self) -> None: + target_path = str(SAMPLES_PATH / "github_classic_token") with tempfile.TemporaryDirectory() as tmp_dir: json_filename = os.path.join(tmp_dir, f"{__name__}.json") denylist_filename = os.path.join(tmp_dir, "list.txt") with open(denylist_filename, "w") as f: - f.write("abc") + f.write('4WZ4EQ # classic') # part of line - will not exclude _stdout, _stderr = self._m_credsweeper([ "--path", target_path, "--denylist", denylist_filename, "--save-json", json_filename, "--log", "silence" ]) diff --git a/tests/test_main.py b/tests/test_main.py index b801798cb..0bab77b2e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -399,7 +399,7 @@ def test_find_by_ext_n(self) -> None: def test_tar_p(self) -> None: # deep scan in tar file. First level is bz2 archive to hide credentials with inflate content_provider: FilesProvider = TextProvider([SAMPLES_PATH / "passwords.tar.bz2"]) - cred_sweeper = CredSweeper(depth=2) + cred_sweeper = CredSweeper(depth=2, ml_threshold=0) cred_sweeper.run(content_provider=content_provider) self.assertEqual(3, len(cred_sweeper.credential_manager.get_credentials())) @@ -499,7 +499,7 @@ def test_pdf_n(self) -> None: def test_py_p(self) -> None: content_provider: FilesProvider = TextProvider([SAMPLES_PATH / "sample.py"]) - cred_sweeper = CredSweeper(depth=33) + cred_sweeper = CredSweeper(depth=3, ml_threshold=ThresholdPreset.lowest) cred_sweeper.run(content_provider=content_provider) found_credentials = cred_sweeper.credential_manager.get_credentials() self.assertEqual(1, len(found_credentials)) @@ -564,7 +564,7 @@ def test_yaml_n(self) -> None: def test_encoded_p(self) -> None: # test for finding credentials in ENCODED data content_provider: FilesProvider = TextProvider([SAMPLES_PATH / "encoded_data"]) - cred_sweeper = CredSweeper(depth=5) + cred_sweeper = CredSweeper(depth=5, ml_threshold=0) cred_sweeper.run(content_provider=content_provider) found_credentials = cred_sweeper.credential_manager.get_credentials() self.assertEqual(2, len(found_credentials)) @@ -598,7 +598,7 @@ def test_docx_n(self) -> None: def test_html_p(self) -> None: # test for finding credentials in html content_provider: FilesProvider = TextProvider([SAMPLES_PATH / "test.html"]) - cred_sweeper = CredSweeper(depth=5) + cred_sweeper = CredSweeper(depth=5, ml_threshold=0) cred_sweeper.run(content_provider=content_provider) found_credentials = cred_sweeper.credential_manager.get_credentials() expected_credential_lines = [ @@ -622,7 +622,7 @@ def test_html_p(self) -> None: self.assertEqual(1, len(cred.line_data_list)) self.assertIn(cred.line_data_list[0].line, expected_credential_lines) expected_credential_lines.remove(cred.line_data_list[0].line) - self.assertEqual(0, len(expected_credential_lines)) + self.assertEqual(0, len(expected_credential_lines), expected_credential_lines) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #