Clean up thog scan

.github/workflows/thog_scan_commit.yml

@@ -41,22 +41,6 @@ jobs:
         run: |
           gh auth setup-git
 
-      # - name: Run TruffleHog scan
-      #   id: trufflehog_scan
-      #   uses: trufflesecurity/[email protected]
-      #   with:
-      #     base: ""
-      #     head: ${{ github.ref_name }}
-      #     extra_args: --only-verified --json --regex --max-depth=50
-      #   continue-on-error: true
-
-      # - name: Install and Run Trufflehog Scan
-      #   id: trufflehog_scan
-      #   run: |
-      #     curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin
-      #     trufflehog filesystem --directory="/home/runner/work/px-benefit-finder/px-benefit-finder" --debug --fail --only-verified --json > truffleHogResults.json
-      #   continue-on-error: true
-
       - name: Install TruffleHog3
         run: |
           pip install trufflehog3
@@ -70,7 +54,7 @@ jobs:
           echo "branch=$BRANCH_NAME" >> $GITHUB_OUTPUT
           echo "emoji=:exclamation:" >> $GITHUB_OUTPUT
           echo "actor=$ACTOR" >> $GITHUB_OUTPUT
-          trufflehog3 --branch $BRANCH_NAME --no-entropy --severity MEDIUM -vv --format json --output truffleHogResults.json
+          trufflehog3 --branch $BRANCE_NAME --depth 100 --no-entropy -vv -r rules.yml --format json --output truffleHogResults.json
           trufflehog3 -R report.json --output truffleHogReport.html
 
       - name: Check TruffleHog Results
@@ -82,12 +66,6 @@ jobs:
             echo "file_exists=false" >> $GITHUB_ENV
           fi
 
-      # - name: Convert JSON to Readable Report
-      #   if: always() && env.file_exists == 'true'
-      #   run: |
-      #     jq -r '.results[] | "File: \(.path)\nCommit: \(.commit)\nDate: \(.date)\nReason: \(.reason)\n---------------------------"' truffleHogResults.json > truffleHogReport.txt
-      #
-
       - name: Cloud.gov login
         env:
           CF_USER: "${{ secrets.CF_USER }}"

.trufflehog3.yml

@@ -0,0 +1,5 @@
+exclude: # exclude matching issues
+  - message: Build directories
+    paths:
+      - rules.yml
+      - .trufflehog3.yml

rules.yml

@@ -0,0 +1,224 @@
+#
+# entropy-based rules
+#
+- id: high-entropy
+  message: High Entropy
+  minlen: 20
+  alphabet: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/="
+  threshold: 4.5
+  severity: MEDIUM
+- id: high-entropy
+  message: High Entropy
+  minlen: 20
+  alphabet: "0123456789abcdefABCDEF"
+  threshold: 3.0
+  severity: MEDIUM
+#
+# regexes-based rules
+#
+- id: private.pgp-key
+  message: PGP Private Key Block
+  pattern: "-----BEGIN PGP PRIVATE KEY BLOCK-----"
+  severity: HIGH
+- id: private.key
+  message: Private Key
+  pattern: "-----BEGIN (RSA|DSA|EC|OPENSSH) PRIVATE KEY-----"
+  severity: HIGH
+
+- id: amazon.aws-api-key
+  message: AWS API Key
+  pattern: "AKIA[0-9A-Z]{16}"
+  severity: MEDIUM
+- id: amazon.mws-auth-token
+  message: Amazon MWS Auth Token
+  pattern: "amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"
+  severity: MEDIUM
+
+- id: facebook.access-token
+  message: Facebook Access Token
+  pattern: "EAACEdEose0cBA[0-9A-Za-z]+"
+  severity: MEDIUM
+- id: facebook.oauth-token
+  message: Facebook OAuth Token
+  pattern: '[f|F][a|A][c|C][e|E][b|B][o|O][o|O][k|K].*[''|"][0-9a-f]{32}[''|"]'
+  severity: MEDIUM
+
+# TODO add new github token formats
+- id: github.token
+  message: GitHub Token
+  pattern: '[g|G][i|I][t|T][h|H][u|U][b|B].*[''|"][0-9a-zA-Z]{35,40}[''|"]'
+  severity: MEDIUM
+
+- id: google.access-token
+  message: Google Access Token
+  pattern: "ya29\\.[0-9A-Za-z\\-_]+"
+  severity: MEDIUM
+- id: google.api-key
+  message: Google API Key
+  pattern: "AIza[0-9A-Za-z\\-_]{35}"
+  severity: MEDIUM
+- id: google.oauth-token
+  message: Google OAuth
+  pattern: "[0-9]+-[0-9A-Za-z_]{32}\\.apps\\.googleusercontent\\.com"
+  severity: MEDIUM
+- id: google.gcp-service-account
+  message: Google Cloud Platform Service Account
+  pattern: '"type": "service_account"'
+  severity: MEDIUM
+
+- id: heroku.api-key
+  message: Heroku API Key
+  pattern: "[h|H][e|E][r|R][o|O][k|K][u|U].*[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}"
+  severity: MEDIUM
+
+- id: mailchimp.api-key
+  message: MailChimp API Key
+  pattern: "[0-9a-f]{32}-us[0-9]{1,2}"
+  severity: MEDIUM
+
+- id: mailgun.api-key
+  message: Mailgun API Key
+  pattern: "key-[0-9a-zA-Z]{32}"
+  severity: MEDIUM
+
+- id: paypal.braintree-access-token
+  message: PayPal Braintree Access Token
+  pattern: "access_token\\$production\\$[0-9a-z]{16}\\$[0-9a-f]{32}"
+  severity: MEDIUM
+
+- id: picatic.api-key
+  message: Picatic API Key
+  pattern: "sk_live_[0-9a-z]{32}"
+  severity: MEDIUM
+
+- id: slack.token
+  message: Slack Token
+  pattern: "xox[p|b|o|a]-[0-9]{12}-[0-9]{12}-[0-9]{12}-[a-z0-9]{32}"
+  severity: MEDIUM
+- id: slack.bot-token
+  message: Slack Bot Token
+  pattern: "xox[p|b|o|a]-[0-9]{12}-[0-9]{12}-[a-z0-9]{24}"
+  severity: MEDIUM
+- id: slack.webhook
+  message: Slack Webhook
+  pattern: "https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}"
+  severity: MEDIUM
+
+- id: stripe.api-key
+  message: Stripe API Key
+  pattern: "sk_live_[0-9a-zA-Z]{24}"
+  severity: MEDIUM
+- id: stripe.restricted-api-key
+  message: Stripe Restricted API Key
+  pattern: "rk_live_[0-9a-zA-Z]{24}"
+  severity: MEDIUM
+
+- id: square.access-token
+  message: Square Access Token
+  pattern: "sq0atp-[0-9A-Za-z\\-_]{22}"
+  severity: MEDIUM
+- id: square.oauth-secret
+  message: Square OAuth Secret
+  pattern: "sq0csp-[0-9A-Za-z\\-_]{43}"
+  severity: MEDIUM
+
+- id: twilio.api-key
+  message: Twilio API Key
+  pattern: "SK[0-9a-fA-F]{32}"
+  severity: MEDIUM
+
+- id: twitter.access-token
+  message: Twitter Access Token
+  pattern: "[t|T][w|W][i|I][t|T][t|T][e|E][r|R].*[1-9][0-9]+-[0-9a-zA-Z]{40}"
+  severity: MEDIUM
+- id: twitter.oauth-token
+  message: Twitter OAuth Token
+  pattern: '[t|T][w|W][i|I][t|T][t|T][e|E][r|R].*[''|"][0-9a-zA-Z]{35,44}[''|"]'
+  severity: MEDIUM
+
+- id: generic.api-key
+  message: Generic API Key
+  pattern: '[a|A][p|P][i|I][_]?[k|K][e|E][y|Y].*[''|"][0-9a-zA-Z]{32,45}[''|"]'
+  severity: LOW
+- id: generic.secret
+  message: Generic Secret
+  pattern: '[s|S][e|E][c|C][r|R][e|E][t|T].*[''|"][0-9a-zA-Z]{32,45}[''|"]'
+  severity: LOW
+- id: generic.password-in-url
+  message: Password in URL
+  pattern: "[a-zA-Z]{3,10}://[^/\\s:@]{3,20}:[^/\\s:@]{3,20}@.{1,100}[\"'\\s]"
+  severity: LOW
+
+- id: sql.connection-string
+  message: SQL Connection String
+  pattern: "(?:postgresql|mysql):\\/\\/([^:\\/\\s]+):([^@\\/\\s]+)@([^:\\/\\s]+):(\\d+)\\/([^\\/\\s]+)"
+  severity: HIGH
+
+- id: redis.connection-string
+  message: Redis Connection String
+  pattern: "redis:\\/\\/([^:\/\\s]+):([^@\/\\s]+)@([^:\/\\s]+):(\\d+)\\/([^\\/\\s]+)"
+  severity: HIGH
+
+- id: mongodb.connection-string
+  message: MongoDB Connection String
+  pattern: "mongodb:\\/\\/([^:\/\\s]+):([^@\/\\s]+)@([^:\/\\s]+):(\\d+)\\/([^\\/\\s]+)"
+  severity: HIGH
+
+- id: ftp.connection-string
+  message: FTP Connection String
+  pattern: "ftp:\\/\\/([^:\/\\s]+):([^@\/\\s]+)@([^:\/\\s]+):(\\d+)\\/([^\\/\\s]+)"
+  severity: HIGH
+
+- id: memcached.connection-string
+  message: Memcached Connection String
+  pattern: "memcached:\\/\\/([^:\/\\s]+):([^@\/\\s]+)@([^:\/\\s]+):(\\d+)\\/([^\\/\\s]+)"
+  severity: HIGH
+
+- id: elasticsearch.connection-string
+  message: Elasticsearch Connection String
+  pattern: "elasticsearch:\\/\\/([^:\/\\s]+):([^@\/\\s]+)@([^:\/\\s]+):(\\d+)\\/([^\\/\\s]+)"
+  severity: HIGH
+
+- id: cloudfoundry.vcap-services
+  message: Cloud Foundry VCAP Services Credentials
+  pattern: '"VCAP_SERVICES":\s*{[^}]*"credentials":\s*{[^}]*"[^"]+":\s*"[^"]+"[^}]*}[^}]*}'
+  severity: HIGH
+
+- id: drupal.database-connection
+  message: Drupal Database Connection String
+  pattern: "\\$databases\\['default'\\]\\['default'\\] = array\\('driver' => '([^']+)', 'database' => '([^']+)', 'username' => '([^']+)', 'password' => '([^']+)', 'host' => '([^']+)', 'port' => '([^']+)'\\);"
+  severity: HIGH
+
+- id: drupal.database-connection
+  message: Drupal Database Connection String
+  pattern: "\\$databases\\['default'\\]\\['default'\\] = array\\('driver' => '([^']+)', 'database' => '([^']+)', 'username' => '([^']+)', 'password' => '([^']+)', 'host' => '([^']+)', 'port' => '([^']+)'\\);"
+  severity: HIGH
+
+- id: drupal.api-keys
+  message: Drupal API Keys
+  pattern: "\\$config\\['([^']+)\\.settings'\\]\\['api_key'\\] = '([^']+)';"
+  severity: HIGH
+
+- id: json.sql-hostname
+  message: MySQL Hostname
+  pattern: '"hostname":'
+  severity: MEDIUM
+
+- id: json.sql-port
+  message: MySQL Port
+  pattern: '"port":'
+  severity: MEDIUM
+
+- id: json.sql-username
+  message: MySQL Username
+  pattern: '"username":'
+  severity: HIGH
+
+- id: json.sql-password
+  message: MySQL Password
+  pattern: '"password":'
+  severity: HIGH
+
+- id: json.sql-database
+  message: MySQL Database
+  pattern: '"database":'