2.3.1 (#177)

* Fix env file parsing

* Test env file parsing

* Add Terraform detection

* Improve password detection

* Improve keys detection

* Update tests

* Update config

* Bump version

* Bump coverage

* Update Terraform parser

* Typing for older Python

.gitignore

@@ -146,7 +146,7 @@ celerybeat-schedule
 *.sage.py
 
 # Environments
-.env
+#.env
 .venv
 env/
 venv/

tests/fixtures/.env

@@ -0,0 +1,13 @@
+# Compliant
+export DANGER_GITHUB_API_TOKEN=
+export DANGER_GITHUB_API_TOKEN=""
+export DANGER_GITHUB_API_TOKEN=<DANGER_GITHUB_API_TOKEN>
+export DANGER_GITHUB_API_TOKEN=$token
+export DANGER_GITHUB_API_TOKEN="${token}"
+# export COMMENTED_API_TOKEN="${token}"
+
+# Noncompliant
+API_TOKEN=${API_TOKEN:-YXNkZmZmZmZm_HARDcoded01}
+export DANGER_GITHUB_API_TOKEN=YXNkZmZmZmZm_HARDcoded02
+export DANGER_GITHUB_API_TOKEN="YXNkZmZmZmZm_HARDcoded03"
+# export COMMENTED_API_TOKEN="YXNkZmZmZmZm_HARDcoded04"

tests/fixtures/.pypirc

@@ -1,12 +1,12 @@
 [compliant]
-repository: https://pypi.example.com
+repository: https://pypi.fqdn.tld
 username: username
 password: $${PYPI_PASSWORD}
-repository: https://pypi.example.com
+repository: https://pypi.fqdn.tld
 username: username
 password:
 
 [noncompliant]
-repository: https://pypi.example.com
+repository: https://pypi.fqdn.tld
 username: username
 password: hardcoded

tests/fixtures/ast/fixture.py

@@ -21,6 +21,7 @@ def compliant():
     config['db_password'] = ''
     secrets = get_secrets(config["secret_key"])
     login(password="")
+    login(access_token=get_secret_value("ENV_CLIENT_SECRET"))
     data = {"login": login, "password": new_password, "previousPassword": password}
     worker_class = "aiohttp.worker.GunicornWebWorker"
     if 1 == 1:

tests/fixtures/ast/fixture.tf

@@ -0,0 +1,43 @@
+# Compliant
+
+variable "ONE_API_TOKEN" {
+    type    = string
+    default = ""
+}
+
+variable "TWO_API_TOKEN" {
+    type    = string
+    default = ENV_VAR_TOKEN
+}
+
+variable "PASSWORDS" {
+    type    = list(string)
+    default = []
+}
+
+variable "PASSWORDS" {
+    type    = list(string)
+    default = ["", password]
+}
+
+
+# Noncompliant
+
+variable "ONE_API_TOKEN" {
+    type    = string
+    default = "23d968ff-10b9-4e6f-a33a-hardcoded01"
+}
+
+variable "TWO_API_TOKEN" {
+    type        = string
+    description = "Service API Key"
+    default     = "23d968ff-10b9-4e6f-a33a-hardcoded02"
+}
+
+variable "PASSWORDS" {
+    type    = list(string)
+    default = [
+        "23d968ff-10b9-4e6f-a33a-hardcoded03",
+        "23d968ff-10b9-4e6f-a33a-hardcoded04"
+    ]
+}

tests/fixtures/falsepositive/keys.yml

@@ -7,3 +7,4 @@ compliant:
   columnkey: h4rdc0dedOkay06
   uniq_key: h4rdc0dedOkay07
   uniqueKey: h4rdc0dedOkay08
+  Ref: Check40charsLengthRefRuleh4rdc0dedOkay09

tests/fixtures/hardcoded.json

@@ -7,7 +7,8 @@
     "05_variable_password": "{{ password }}",
     "06_variable_password": "{{ THIS_IS_A_VERY_LONG_A_PLACEHOLDER_FOR_PASSWORD }}",
     "07_variable_password": "{password}",
-    "08_variable_password": "{ password }"
+    "08_variable_password": "{ password }",
+    "09_variable_password": "This is not a password"
   },
   "noncompliant": {
     "01_static_password": "hardcoded0",

tests/fixtures/hardcoded.xml

@@ -9,6 +9,7 @@
     <o6_variable_password>{{ THIS_IS_A_VERY_LONG_A_PLACEHOLDER_FOR_PASSWORD }}</o6_variable_password>
     <o7_variable_password>{password}</o7_variable_password>
     <o8_variable_password>{ password }</o8_variable_password>
+    <o9_variable_password>This is not a password</o9_variable_password>
   </compliant>
   <noncompliant>
     <o1_static_password>hardcoded0</o1_static_password>

tests/fixtures/hardcoded.yml

@@ -7,6 +7,7 @@
 06_variable_password: "{{ THIS_IS_A_VERY_LONG_A_PLACEHOLDER_FOR_PASSWORD }}"
 07_variable_password: "{password}"
 08_variable_password: "{ password }"
+09_variable_password: 'This is not a password'
 
 # Noncompliant
 01_static_password: hardcoded0

tests/fixtures/pip.conf

@@ -1,8 +1,8 @@
 [global]
 # Compliant
-index = https://pypi.example.com/pypi
-index-url = https://username:@pypi.example.com/simple
+index = https://pypi.fqdn.tld/pypi
+index-url = https://username:@pypi.fqdn.tld/simple
 
 # Noncompliant
-index = https://username:hardcoded1@pypi.example.com/pypi
-index-url = https://username:hardcoded2@pypi.example.com/simple
+index = https://username:hardcoded1@pypi.fqdn.tld/pypi
+index-url = https://username:hardcoded2@pypi.fqdn.tld/simple

tests/unit/core/test_pairs.py

@@ -105,6 +105,7 @@ def test_filter_static(key, value, expected):
         ("File.404", False, None),
         (".aws/credentials", False, Config),
         (".dockercfg", False, Dockercfg),
+        (".env", False, Shell),
         (".htpasswd", False, Htpasswd),
         (".npmrc", False, Npmrc),
         (".pypirc", False, Pypirc),
@@ -132,7 +133,7 @@ def test_filter_static(key, value, expected):
         ("settings01.ini", False, Config),
         ("settings02.ini", False, Config),
         ("settings.cfg", False, Config),
-        ("settings.env", False, Config),
+        ("settings.env", False, Shell),
     ],
 )
 def test_load_plugin(filename, ast, expected):

tests/unit/core/test_secrets.py

@@ -48,6 +48,7 @@ def test_detect_secrets_by_key(src, expected):
     [
         (".aws/credentials", "Critical", 3),
         (".dockercfg", "High", 1),
+        (".env", "Medium", 4),
         (".htpasswd", "Medium", 2),
         (".npmrc", "High", 5),
         (".pypirc", "High", 1),
@@ -133,7 +134,7 @@ def test_detect_secrets_by_key(src, expected):
     ],
 )
 def test_detect_secrets_by_value(src, severity, expected):
-    args = parse_args(["--ast", "--severity", severity, fixture_path(src)])
+    args = parse_args(["--severity", severity, fixture_path(src)])
     config = load_config(args)
     rules = load_rules(args, config)
     pairs = make_pairs(config, FIXTURE_PATH.joinpath(src))

tests/unit/core/test_utils.py

@@ -105,6 +105,7 @@ def test_similar_strings(str1, str2, expected):
         ("apikeys.yml", "GITHUBKEY", "YXNkZmZmZmZm_HARDcoded", 19),
         ("pip.conf", "username", "hardcoded1", 7),
         ("java.properties", "sonar.jdbc.password", "hardcoded02", 10),
+        ("invalid.sh", "pwd", "hardcoded", 0),
         ("404", "password", "hardcoded", 0),
     ],
 )
@@ -116,7 +117,7 @@ def test_find_line_number_single(src, key, value, expected):
 @pytest.mark.parametrize(
     ("src", "linenumbers"),
     [
-        ("hardcoded.yml", [12, 14, 15, 16, 19]),
+        ("hardcoded.yml", [13, 15, 16, 17, 20]),
         ("privatekeys.yml", [5, 7, 11, 12, 13, 14]),
         ("java.properties", [9, 10, 11]),
     ],

tests/unit/plugins/test_semgrep.py

@@ -26,6 +26,7 @@
         ("fixture.py", 10),
         ("fixture.rb", 9),
         ("fixture.scala", 8),
+        ("fixture.tf", 4),
         ("fixture.ts", 11),
         ("fixture.vue", 11),
     ],

tests/unit/test_main.py

@@ -15,13 +15,13 @@ def test_main():
 @pytest.mark.parametrize(
     ("ast", "expected"),
     [
-        ("--ast", 421),
-        ("", 313),
+        ("--ast", 430),
+        ("", 318),
     ],
 )
 def test_run(ast, expected):
     if platform.startswith("win"):
-        expected = 313
+        expected = 318
 
     argv = ["-F", "None"]
     if ast:

whispers/__version__.py

@@ -1,4 +1,4 @@
-VERSION = (2, 3, 0)
+VERSION = (2, 3, 1)
 
 __version__ = ".".join(map(str, VERSION))
 

whispers/config.yml

@@ -7,7 +7,7 @@ include:
 exclude:
   files:
     - .*(__pycache__|\.eggs|build|dev|\.vscode|\.git)
-    - .*(locale|spec|test|mock|dummy|fixture)s?
+    - .*(locale|lang(uage)?|spec|test|mock(ing)?|synthetic|dummy|fixture|example|e2e)s?
     - .*(integration|node_modules)
     - .*(package(-lock)?|npm-shrinkwrap)\.json
 
@@ -18,4 +18,4 @@ exclude:
     - ^(true|false|yes|no|on|off|(en|dis)able|1|0)$
     - ^((cn?trl|alt|shift|del|ins|esc|tab|f[\d]+) ?[\+_\-\\/] ?)+[\w]+$
     - .*_(user|password|token|key|placeholder|name)$
-    - ^(aws_)?(access_key_id|secret_access_key|session_token)$
+    - ^(aws_)?(access_key_id|secret_access_key|session_token|secretsmanager)$

whispers/core/pairs.py

@@ -105,10 +105,12 @@ def load_plugin(file: Path, ast: bool = False) -> Optional[object]:
     Optional `ast` param enables/disables Semgrep.
     Returns None if no plugin found.
     """
-    if file.suffix in [".dist", ".template"]:
-        filetype = file.stem.split(".")[-1]
+    file_name = file.name.lower()
+
+    if file.suffix.lower() in [".dist", ".template"]:
+        filetype = file.stem.split(".")[-1].lower()
     else:
-        filetype = file.name.split(".")[-1]
+        filetype = file_name.split(".")[-1]
 
     if filetype in ["yaml", "yml"]:
         return Yml
@@ -125,22 +127,22 @@ def load_plugin(file: Path, ast: bool = False) -> Optional[object]:
     elif filetype.startswith("pypirc"):
         return Pypirc
 
-    elif file.name == "pip.conf":
+    elif file_name == "pip.conf":
         return Pip
 
-    elif file.name == "build.gradle":
+    elif file_name == "build.gradle":
         return Gradle
 
-    elif filetype in ["conf", "cfg", "cnf", "config", "ini", "env", "credentials", "s3cfg"]:
+    elif filetype in ["conf", "cfg", "cnf", "config", "ini", "credentials", "s3cfg"]:
         return Config
 
     elif filetype == "properties":
         return Jproperties
 
-    elif filetype.startswith(("sh", "bash", "zsh", "env")):
+    elif filetype.startswith(("sh", "bash", "zsh", "env")) or file_name == "environment":
         return Shell
 
-    elif "dockerfile" in file.name.lower():
+    elif "dockerfile" in file_name:
         return Dockerfile
 
     elif filetype == "dockercfg":

whispers/plugins/semgrep.py

@@ -24,6 +24,7 @@ class AST:
     DOTACCESS = "DotAccess"
     EN = "EN"
     EQ = "Eq"
+    F = "F"
     FIELDDEFCOLON = "FieldDefColon"
     FN = "FN"
     ID = "Id"
@@ -34,6 +35,7 @@ class AST:
     NOTEQ = "NotEq"
     OP = "Op"
     OR = "Or"
+    RECORD = "Record"
     SOME = "some"
     STRING = "String"
     TUPLE = "Tuple"
@@ -135,6 +137,14 @@ def call(ast: List, key: str = "", value: str = "") -> Iterable[Tuple[str, str]]
 
             key = AST.literal(call_arg_1)
 
+            if AST.RECORD in call_arg_2:
+                for record in call_arg_2[AST.RECORD]:
+                    statement = record.get(AST.F, {}).get(AST.DEFSTMT, [{}, {}])
+                    record_key, record_value = AST.defstmt(statement)
+
+                    if record_key == "default" and record_value:
+                        return key, record_value  # Terraform default variable value
+
             if AST.CONDITIONAL in call_arg_2:
                 value = AST.literal(call_arg_2[AST.CONDITIONAL][-1])
 
@@ -149,7 +159,7 @@ def dotaccess(ast: List) -> str:
         return AST.name(tree)
 
     @staticmethod
-    def defstmt(ast: List) -> Tuple[str, str]:
+    def defstmt(ast: List) -> Tuple[str, Any]:
         name = ast[0].get(AST.NAME, {})
         key = AST.name(name)
 
@@ -161,13 +171,16 @@ def defstmt(ast: List) -> Tuple[str, str]:
             if AST.DOTACCESS in some[AST.CALL][0]:
                 key = AST.dotaccess(some[AST.CALL])
 
-            elif AST.IDSPECIAL in some[AST.CALL][0]:
+            elif AST.IDSPECIAL in some[AST.CALL][0] and some[AST.CALL][1]:
                 value = AST.literal(some[AST.CALL][1][-1].get(AST.ARG, {}))
                 return key, value
 
             key = AST.name(some[AST.CALL][0])
             value = AST.call_args(some[AST.CALL][1])[1]
 
+        elif AST.CONTAINER in some:
+            value = list(map(lambda item: AST.literal(item), some[AST.CONTAINER][1]))
+
         else:
             value = AST.literal(some)
 
@@ -243,15 +256,19 @@ def traverse(self, ast: Any) -> Iterable[KeyValuePair]:
                     key, value = AST.defstmt(ast_values)
                     yield KeyValuePair(key, value)
 
-                elif ast_key == AST.CONTAINER and AST.TUPLE in ast_values:
+                elif ast_key == AST.CONTAINER and AST.TUPLE in ast_values and len(ast_values[1]) > 1:
                     key = AST.literal(ast_values[1][0])
                     value = AST.literal(ast_values[1][1])
                     yield KeyValuePair(key, value)
                     return StopIteration
 
                 elif ast_key == AST.CALL:
                     key, value = AST.call(ast_values)
-                    yield KeyValuePair(key, value)
+
+                    if isinstance(value, list):
+                        yield from map(lambda v: KeyValuePair(key, v), value)
+                    else:
+                        yield KeyValuePair(key, value)
 
                     if AST.call_op(ast_values) in [AST.EQ, AST.NOTEQ]:
                         key, value = AST.call_args(ast_values[1])