2.4.0 (#178)

* Add Gitlab CI tests * Improve URI credentials detection * Update severity * Update tests * Update config * Update severity * Remove semgrep telemetry * Bump version * Trim deps * Update AWS Account ID rule * Update README * Lazy-load parsers * Update README * Lazy-load parsers * Update README * Update RELEASE NOTES
adeptex · Oct 5, 2024 · 24ee0f0 · 24ee0f0
1 parent 879dcab
commit 24ee0f0
Show file tree

Hide file tree

Showing 23 changed files with 216 additions and 148 deletions.
diff --git a/Makefile b/Makefile
@@ -1,8 +1,8 @@
 install:
-	pip3 install -r requirements.txt -e .
+	pip3 install -r requirements.txt --no-deps -e .
 
-install-dev:
-	pip3 install -r requirements.txt -e ".[dev]"
+install-dev: install
+	pip3 install -r requirements-dev.txt
 
 flake8-lint:
 	flake8 whispers/ tests/unit/ setup.py

diff --git a/README.md b/README.md
@@ -9,7 +9,7 @@
 
 > "My little birds are everywhere, even in the North, they whisper to me the strangest stories." - _Varys_
 
-Whispers is an information security analysis tool designed for identifying <u>**hardcoded secrets in structured text and static code**</u> ([CWE-798](https://cwe.mitre.org/data/definitions/798.html)). Whispers can be used as a [standalone binary](https://github.com/adeptex/whispers#download), or as a [Python module](https://github.com/adeptex/whispers#install), which is meant to facilitate its usage individually and as part of automated processes and pipelines at scale.
+Whispers is an information security analysis tool designed for identifying <u>**hardcoded secrets in structured text and static code**</u> ([CWE-798](https://cwe.mitre.org/data/definitions/798.html)). Whispers can be used as a [standalone binary](https://github.com/adeptex/whispers#download), or as a [Python module](https://github.com/adeptex/whispers#install), which is meant to facilitate its usage individually, and in automated processes and pipelines at scale.
 
 * :clipboard: [Release notes](https://github.com/adeptex/whispers/blob/master/RELEASE_NOTES.md)
 * :gear: [Request a feature](https://github.com/adeptex/whispers/issues/new?assignees=&labels=&template=feature_request.md&title=)
@@ -252,28 +252,26 @@ Simple filtering based on rules and severity can also be done with CLI arguments
 
 | Group                | Rule ID              | Severity        |
 |----------------------|----------------------|-----------------|
-| files                | file-known           | Low             |
+| keys                 | aws-secret           | Critical        |
+| keys                 | aws-token            | Critical        |
+| keys                 | privatekey           | Critical        |
+| keys                 | apikey-known         | Critical        |
+| keys                 | apikey               | High            |
+| keys                 | aws-id               | Medium          |
+| keys                 | aws-account          | Low             |
+| keys                 | apikey-maybe         | Low             |
+| passwords            | password             | High            |
+| passwords            | uri                  | High            |
 | infra                | dockercfg            | High            |
-| infra                | htpasswd             | Medium          |
 | infra                | npmrc                | High            |
 | infra                | pip                  | High            |
 | infra                | pypirc               | High            |
-| keys                 | apikey               | Medium          |
-| keys                 | apikey-known         | High            |
-| keys                 | apikey-maybe         | Low             |
-| keys                 | aws-id               | Critical        |
-| keys                 | aws-secret           | Critical        |
-| keys                 | aws-token            | Critical        |
-| keys                 | privatekey           | High            |
-| misc                 | comment              | Info            |
+| infra                | htpasswd             | Medium          |
+| misc                 | webhook              | Medium          |
 | misc                 | creditcard           | Low             |
 | misc                 | secret               | Low             |
-| misc                 | webhook              | Low             |
-| passwords            | password             | High            |
-| passwords            | uri                  | High            |
-| python               | cors                 | Low             |
-| python               | system               | Low             |
-
+| misc                 | comment              | Info            |
+| files                | file-known           | Low             |
 
 ### Custom rules
 

diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
@@ -1,74 +1,99 @@
-# Whispers 2.3.0 release notes
+# Whispers 2.4.0 release notes
 
-* **New Feature:** 💫 <u>Static Code Analysis</u> 💫 is now supported!
-    * The present release complements classic Whispers' structured text analysis with [Semgrep](https://semgrep.dev)'s AST generator for [common programming languages](https://semgrep.dev/docs/supported-languages) like Python, PHP, Java/Scala/Kotlin, JavaScript/TypeScript, Go, etc etc.
-    * New argument `--ast` for enabling this feature via the CLI (it is disabled by default)
-    * New setting `ast: true` for enabling this feature via a custom config file (set to `ast: false` by default)
-    * Replaced [`astroid`](https://github.com/adeptex/whispers/blob/8f17f77e2199c55458ff125e3fb477a2a9349593/whispers/plugins/python.py) Python AST generator with [`semgrep`](https://github.com/adeptex/whispers/blob/master/whispers/plugins/semgrep.py)
+* 💫 **Remove Semgrep telemetry** 💫
+* Lazy-load parsers
+* Severity levels reassignment
+* Detection rule improvements
+    * URI credentials
+    * AWS Account ID
+* Generalize default config
 
-* [Detection rule](https://github.com/adeptex/whispers/blob/master/whispers/rules) improvements
-    * Known API keys
-    * AWS account ID
-    * Passwords
-    * Creditcards
 
-* Drop end-of-life Python support
-    * Versions 3.6 and 3.7 are no longer supported. Oldest supported version is Python 3.8.
-    * Last release that supports Python 3.6 and 3.7 is [Whispers 2.2.1](https://github.com/adeptex/whispers/releases/tag/2.2.1)
+## 💫 Remove Semgrep telemetry 💫
 
-* Dependency tracking improvements
-    * New [`requirements-dev.txt`](https://github.com/adeptex/whispers/blob/master/requirements-dev.txt) file allows Dependabot updates for dev dependencies
-    * Modified [`setup.py`](https://github.com/adeptex/whispers/blob/master/setup.py) to read from `requirements.txt` and `requirements-dev.txt`
-    * Updated build CI to use Python 3.12.3
+It's a better world now that corporations build telemetry into every single piece of software... **not really** 😒... It was shoking to see telemetry packages installed as part of Whispers. But how did this happen? 
 
-* Debugging and troubleshooting
-    * Modified [`config.yml`](https://github.com/adeptex/whispers/blob/master/whispers/config.yml) to exclude known false positives
-    * Fixed [`Dockerfile`](https://github.com/adeptex/whispers/blob/master/Dockerfile) to work with `docker build -t whispers .` or the same `make image`
-    * New arg `--dump` for generating an AST of a file: `whispers --dump src/example.ts`
+As it turns out, Semgrep includes A LOT of code to support exfiltrating metadata and usage information from your machine. This Whispers release is largely dedicated to stripping out all unnecessary spyware garbage "required" by Semgrep. Apart from privacy, a nice side effect of this is that now Semgrep runs a lot faster when parsing ASTs! Win-win. 
 
+The following are 24 (out of 32.. wtf??) "required" Semgrep dependencies that are now excluded:
 
-## 💫 New Feature: Static Code Analysis 💫
-
-With the release of Whispers 2.3, it is now possible to accurately apply Whispers' secret detection techniques for structured text to static code. Before this release, Whispers only supported structured text formats, such as JSON or XML. [Semgrep](https://semgrep.dev) is an open source SAST tool, which has a built-in feature for generating Abstract Structure Trees (ASTs) for [many common programming languages](https://semgrep.dev/docs/supported-languages). Generating an AST for static code yields an accurate structured text representation, which can be checked for secrets with Whispers' rules and plugins. As such, generating ASTs requires an additional "format conversion" step, which naturally affects runtime speed. When AST is enabled it will take longer to scan the same scope if any source code files are present. The increased amount of runtime time would be however long it takes to run the following command on all static code files in scope:
-
-```sh
-semgrep scan --metrics=off --quiet --dump-ast --json --lang $LANG $SRCFILE
 ```
-
-Consider the following benchmarks:
-
-```sh
-time whispers -F " " tests/fixtures
-# 313 detected secrets
-# 0,51s user 0,03s system 99% cpu 0,540 total
-# 0,60s user 0,04s system 99% cpu 0,642 total
-
-time whispers -a -F " " tests/fixtures
-# 421 detected secrets
-# 2,20s user 0,40s system 100% cpu 2,589 total
-# 2,32s user 0,46s system 100% cpu 2,772 total
+certifi==2024.8.30
+charset-normalizer==3.3.2
+Deprecated==1.2.14
+googleapis-common-protos==1.65.0
+idna==3.10
+importlib_metadata==7.1.0
+markdown-it-py==3.0.0
+mdurl==0.1.2
+opentelemetry-api==1.25.0
+opentelemetry-exporter-otlp-proto-common==1.25.0
+opentelemetry-exporter-otlp-proto-http==1.25.0
+opentelemetry-instrumentation==0.46b0
+opentelemetry-instrumentation-requests==0.46b0
+opentelemetry-proto==1.25.0
+opentelemetry-sdk==1.25.0
+opentelemetry-semantic-conventions==0.46b0
+opentelemetry-util-http==0.46b0
+protobuf==4.25.5
+Pygments==2.18.0
+requests==2.32.3
+rich==13.9.1
+setuptools==75.1.0
+urllib3==2.2.3
+zipp==3.20.2
 ```
 
-AST conversion is **disabled by default** - `semgrep` will **not** execute at all unless explicitly enabled. Custom config files that are missing `ast: false` or `ast: true` will default to `false`.
+The confirmation of this great success can be seen in every `pip3 install whispers` log in the form of these amazing error messages:
 
-```yaml
-ast: true  # enable AST in config.yml
 ```
-
-```sh
-whispers --ast target/dir/or/file  # enable AST in CLI
+ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
+semgrep 1.85.0 requires attrs>=21.3, which is not installed.
+semgrep 1.85.0 requires boltons~=21.0, which is not installed.
+semgrep 1.85.0 requires click-option-group~=0.5, which is not installed.
+semgrep 1.85.0 requires colorama~=0.4.0, which is not installed.
+semgrep 1.85.0 requires defusedxml~=0.7.1, which is not installed.
+semgrep 1.85.0 requires exceptiongroup~=1.2.0, which is not installed.
+semgrep 1.85.0 requires glom~=22.1, which is not installed.
+semgrep 1.85.0 requires opentelemetry-api~=1.25.0, which is not installed.
+semgrep 1.85.0 requires opentelemetry-exporter-otlp-proto-http~=1.25.0, which is not installed.
+semgrep 1.85.0 requires opentelemetry-instrumentation-requests~=0.46b0, which is not installed.
+semgrep 1.85.0 requires opentelemetry-sdk~=1.25.0, which is not installed.
+semgrep 1.85.0 requires peewee~=3.14, which is not installed.
+semgrep 1.85.0 requires ruamel.yaml<0.18,>=0.16.0, which is not installed.
+semgrep 1.85.0 requires tomli~=2.0.1, which is not installed.
+semgrep 1.85.0 requires wcmatch~=8.3, which is not installed.
 ```
 
 
 ## ❌ Breaking changes ❌
 
-### ❌ Replaced `astroid` with `semgrep` ❌
-
-Before Whispers 2.3, only Python AST scanning was natively supported by `astroid`, and integrated via [`plugins/python.py`](https://github.com/adeptex/whispers/blob/8f17f77e2199c55458ff125e3fb477a2a9349593/whispers/plugins/python.py). With the release of Whispers 2.3, this functionality is superseded by `semgrep`, and integrated via [`plugins/semgrep.py`](https://github.com/adeptex/whispers/blob/master/whispers/plugins/semgrep.py). As a base line, the new `semgrep` plugin detects the same findings as the `astroid` plugin, but supports more programming languages.
-
-Unfortunately `semgrep` has telemetry enabled by default, but can be turned off via [`--metrics=off`](https://github.com/adeptex/whispers/blob/master/whispers/plugins/semgrep.py#L57). In any case, `semgrep` will not execute unless explicitly enabled via args or config.
-
-⚠️ **NOTE:** At the time of writing, `semgrep` [does not support Windows OS natively](https://github.com/semgrep/semgrep/issues/1330), and can only be installed through WSL. As such, compiled Whispers PE32+ executable comes without Static Code Analysis support. Installing Whispers on Windows via WSL with `pip3 install whispers` *does* have Static Code Analysis support.
+### ❌ Severity levels reassignment ❌
+
+Adjusted rule severity levels to add structure. New severity levels are the following:
+
+| Group                | Rule ID              | Severity Before | Severity Now |
+|----------------------|----------------------|-----------------|--------------|
+| keys                 | aws-secret           | Critical        | Critical     |
+| keys                 | aws-token            | Critical        | Critical     |
+| keys                 | privatekey           | High            | Critical     |
+| keys                 | apikey-known         | High            | Critical     |
+| keys                 | apikey               | Medium          | High         |
+| keys                 | aws-id               | Critical        | Medium       |
+| keys                 | aws-account          | Low             | Low          |
+| keys                 | apikey-maybe         | Low             | Low          |
+| passwords            | password             | High            | High         |
+| passwords            | uri                  | High            | High         |
+| infra                | dockercfg            | High            | High         |
+| infra                | npmrc                | High            | High         |
+| infra                | pip                  | High            | High         |
+| infra                | pypirc               | High            | High         |
+| infra                | htpasswd             | Medium          | Medium       |
+| misc                 | webhook              | Low             | Medium       |
+| misc                 | creditcard           | Low             | Low          |
+| misc                 | secret               | Low             | Low          |
+| misc                 | comment              | Info            | Info         |
+| files                | file-known           | Low             | Low          |
 
 
 # Changelog
@@ -79,3 +104,4 @@ Unfortunately `semgrep` has telemetry enabled by default, but can be turned off
 |2022-07-29|2.1.0|https://github.com/adeptex/whispers/releases/tag/2.1.0|
 |2023-10-23|2.2.0|https://github.com/adeptex/whispers/releases/tag/2.2.0|
 |2024-06-16|2.3.0|https://github.com/adeptex/whispers/releases/tag/2.3.0|
+|2024-10-05|2.4.0|https://github.com/adeptex/whispers/releases/tag/2.4.0|
diff --git a/requirements.txt b/requirements.txt
@@ -2,9 +2,13 @@ beautifulsoup4==4.12.3
 crossplane==0.5.8
 jellyfish==1.1.0
 jproperties==2.1.1
+jsonschema==4.23.0
+jsonschema-specifications==2023.12.1
 luhn==0.2.0
 lxml==5.3.0
-pyyaml==6.0.2
+PyYAML==6.0.2
 semgrep==1.85.0; sys_platform != 'win32'
 six==1.16.0
+soupsieve==2.6
+typing_extensions==4.12.2
 wrapt==1.16.0
diff --git a/tests/fixtures/.gitlab-ci.yml b/tests/fixtures/.gitlab-ci.yml
@@ -0,0 +1,16 @@
+variables:
+  USERNAME: admin
+  PASSWORD: HardcodedAdminPassword01
+  APIKEY: 23d968ff-10b9-4e6f-a33a-hardcoded02
+
+compliant:
+  stage: test
+  script:
+    - curl https://$USERNAME:[email protected]
+    - curl https://git.hosting-name.com/?APIKEY=$APIKEY
+
+noncompliant:
+  stage: test
+  script:
+    - curl https://admin:[email protected]
+    - curl https://git.hosting-name.com/?APIKEY=23d968ff-10b9-4e6f-a33a-hardcoded04
diff --git a/tests/fixtures/arn.xml b/tests/fixtures/arn.xml
@@ -5,8 +5,8 @@
       <ok02>arn:aws:kms:{REGION}:{ACCOUNT}:key/{KEY_ID}</ok02>
   </compliant>
   <noncompliant>
-    <arn01>arn:aws:kms:eu-central-1:123456123456:key/hardcoded</arn01>
-    <arn02>arn:aws:kms:ap-southeast-1:123456123456:key/hardcoded</arn02>
-    <arn03>arn:aws:iam::123456123456:oidc-provider/auth-dev.mozilla.auth0.com</arn03>
+    <arn01>arn:aws:kms:eu-central-1:111122223333:key/hardcoded</arn01>
+    <arn02>arn:aws:kms:ap-southeast-1:111122223333:key/hardcoded</arn02>
+    <arn03>arn:aws:iam::111122223333:oidc-provider/auth-dev.mozilla.auth0.com</arn03>
   </noncompliant>
 </tests>
diff --git a/tests/fixtures/arn.yml b/tests/fixtures/arn.yml
@@ -4,8 +4,8 @@ compliant:
 
 
 noncompliant:
-  arn01: arn:aws:kms:eu-central-1:123456123456:key/hardcoded
-  arn02: arn:aws:kms:ap-southeast-1:123456123456:key/hardcoded
-  arn03: arn:aws:iam::123456123456:oidc-provider/auth-dev.mozilla.auth0.com
+  arn01: arn:aws:kms:eu-central-1:111122223333:key/hardcoded
+  arn02: arn:aws:kms:ap-southeast-1:111122223333:key/hardcoded
+  arn03: arn:aws:iam::111122223333:oidc-provider/auth-dev.mozilla.auth0.com
   arn_list:
-    - arn:aws:kms:eu-central-1:123456123456:key/hardcoded
+    - arn:aws:kms:eu-central-1:111122223333:key/hardcoded
diff --git a/tests/fixtures/aws.json b/tests/fixtures/aws.json
@@ -4,7 +4,7 @@
       "aws_id": "{{ AWS ID }}",
       "aws_key": "${AWS_KEY}",
       "commit_id": "912ec803b2ce49e4a541068d495ab57000000000",
-      "role": "arn:aws:iam::123456789000:role/role-name"
+      "role": "arn:aws:iam::111122223333:role/role-name"
     },
     {
       "aws_account01": "000000000000",
@@ -18,7 +18,7 @@
       "aws_token": "FakeYXdzELv//////////wEldj3948yOJRO84jgpoip239232hEOHhfkjhefkwue97jorhfiuh+XjFC9Je/YG7JCqKjrspab2lB+7/Fb1NJFjgwur47Dbhs/L7nh+/VGnwLoAo8CIqoPBLRmXItaoiuuofZnr+ktihZk1Yi55sYZ12hfRMPVbDmhf9Ke683+e9bJirhUEghw9424JOhgwrgqq99MvzCEFe4eXPOSgAcQcD2xqnnKO738tjhoh23HFqjflhefibWegfqefgqUF12hvgfwegqf"
     },
     {
-      "aws_account01": "123456789123"
+      "aws_account01": "111122223333"
     }
   ]
 }
diff --git a/tests/fixtures/aws.xml b/tests/fixtures/aws.xml
@@ -4,14 +4,14 @@
     <aws_id>{{ AWS ID }}</aws_id>
     <aws_key>${AWS_KEY}</aws_key>
     <commit_id>912ec803b2ce49e4a541068d495ab57000000000</commit_id>
-    <role>arn:aws:iam::123456789000:role/role-name</role>
+    <role>arn:aws:iam::111122223333:role/role-name</role>
     <aws_account>000000000000</aws_account>
     <aws_account>111111111111</aws_account>
   </compliant>
   <noncompliant>
     <aws_id>AKIAHI38FAKE1IWUQEEN</aws_id>
     <aws_key>PA3XsxZ8d8cPQLmnZzFAKEdzC6ND2a8vhbyXU/Dw</aws_key>
     <aws_token>FakeYXdzELv//////////wEldj3948yOJRO84jgpoip239232hEOHhfkjhefkwue97jorhfiuh+XjFC9Je/YG7JCqKjrspab2lB+7/Fb1NJFjgwur47Dbhs/L7nh+/VGnwLoAo8CIqoPBLRmXItaoiuuofZnr+ktihZk1Yi55sYZ12hfRMPVbDmhf9Ke683+e9bJirhUEghw9424JOhgwrgqq99MvzCEFe4eXPOSgAcQcD2xqnnKO738tjhoh23HFqjflhefibWegfqefgqUF12hvgfwegqf</aws_token>
-    <aws_account>123456789123</aws_account>
+    <aws_account>111122223333</aws_account>
   </noncompliant>
 </tests>
diff --git a/tests/fixtures/aws.yml b/tests/fixtures/aws.yml
@@ -2,12 +2,12 @@ compliant:
   aws_id: "{{ AWS ID }}"
   aws_key: "${AWS_KEY}"
   commit_id: 912ec803b2ce49e4a541068d495ab57000000000
-  role: arn:aws:iam::123456789000:role/role-name
+  role: arn:aws:iam::111122223333:role/role-name
   aws_account01: '000000000000'
   aws_account02: '111111111111'
 
 noncompliant:
   aws_id: AKIAHI38FAKE1IWUQEEN
   aws_key: PA3XsxZ8d8cPQLmnZzFAKEdzC6ND2a8vhbyXU/Dw
   aws_token: FakeYXdzELv//////////wEldj3948yOJRO84jgpoip239232hEOHhfkjhefkwue97jorhfiuh+XjFC9Je/YG7JCqKjrspab2lB+7/Fb1NJFjgwur47Dbhs/L7nh+/VGnwLoAo8CIqoPBLRmXItaoiuuofZnr+ktihZk1Yi55sYZ12hfRMPVbDmhf9Ke683+e9bJirhUEghw9424JOhgwrgqq99MvzCEFe4eXPOSgAcQcD2xqnnKO738tjhoh23HFqjflhefibWegfqefgqUF12hvgfwegqf
-  aws_account: '123456789123'
+  aws_account: '111122223333'
diff --git a/tests/fixtures/nginx.conf b/tests/fixtures/nginx.conf
@@ -15,6 +15,6 @@ http {
 	server {
 		set $apikey 23d968ff-10b9-4e6f-a33a-hardcoded01;
 		proxy_set_header x-apikey 23d968ff-10b9-4e6f-a33a-hardcoded02;
-		proxy_set_header x-url https://admin:[email protected];
+		proxy_set_header x-url https://admin:23d968ff-10b9-4e6f-a33a-[email protected];
 	}
 }
diff --git a/tests/fixtures/severity.yml b/tests/fixtures/severity.yml
@@ -1,13 +1,11 @@
-blocker:
-  aws_id: AKIAHI38FAKHARDCODED
-
 critical:
   rsa: "-----BEGIN RSA PRIVATE KEY-----\nQyNTUxOQAAACCtrF27B/zd9DEpd38IbVBy93wSeYXKU0AGXMyO8ePu2QAAAKBSzpYEUs6W\n-----END RSA PRIVATE KEY-----"
 
-major:
+high:
   apikey: YXNkZmZmZmZm_HARDcoded
 
-minor:
+medium:
+  aws_id: AKIAHI38FAKHARDCODED
   slack-webhook: https://hooks.slack.com/services/HARDC0D3D/HARDC0D3D/hardcodedWebhook01
 
 info: