From 3f6bcd211744cc820c616c804a081df1793cdac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C5=A1per=20Oblak?= Date: Sat, 3 Aug 2024 14:30:33 +0200 Subject: [PATCH] Initial commit. --- .github/workflows/check_links.yml | 25 ++++++ .github/workflows/ci.yml | 89 +++++++++++++++++++ .github/workflows/gitleaks.yml | 16 ++++ .github/workflows/release.yml | 24 +++++ .github/workflows/spellcheck.yml | 11 +++ .gitignore | 27 ++++++ .golangci.yml | 80 +++++++++++++++++ .pre-commit-config.yaml | 29 ++++++ .releaserc | 10 +++ LICENSE | 21 +++++ README.md | 79 +++++++++++++++++ certstream.go | 142 ++++++++++++++++++++++++++++++ cspell.json | 23 +++++ example/main.go | 21 +++++ go.mod | 5 ++ go.sum | 2 + project-words.txt | 1 + renovate.json | 59 +++++++++++++ 18 files changed, 664 insertions(+) create mode 100644 .github/workflows/check_links.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/gitleaks.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/spellcheck.yml create mode 100644 .gitignore create mode 100644 .golangci.yml create mode 100644 .pre-commit-config.yaml create mode 100644 .releaserc create mode 100644 LICENSE create mode 100644 README.md create mode 100644 certstream.go create mode 100644 cspell.json create mode 100644 example/main.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 project-words.txt create mode 100644 renovate.json diff --git a/.github/workflows/check_links.yml b/.github/workflows/check_links.yml new file mode 100644 index 0000000..6472d17 --- /dev/null +++ b/.github/workflows/check_links.yml @@ -0,0 +1,25 @@ +name: Check for empty links + +on: + repository_dispatch: + workflow_dispatch: + schedule: + - cron: "00 18 * * *" + +jobs: + linkChecker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Link Checker + id: lychee + uses: lycheeverse/lychee-action@2b973e86fc7b1f6b36a93795fe2c9c6ae1118621 # v1.10.0 + + - name: Create Issue From File + if: env.lychee_exit_code != 0 + uses: peter-evans/create-issue-from-file@24452a72d85239eacf1468b0f1982a9f3fec4c94 # v5 + with: + title: Link Checker Report + content-filepath: ./lychee/out.md + labels: report, automated issue diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..156b0d2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,89 @@ +name: Continious integration + +on: + pull_request: + branches: + - main + +jobs: + tests: + name: Run tests and upload results + runs-on: ubuntu-latest + strategy: + matrix: + go-version: ["1.21", "1.22"] + + steps: + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + # This is currently workaround for checking if gofiles have changed, + # Because paths filter doesn't work with required checks + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@c65cd883420fd2eb864698a825fc4162dd94482c # v44 + with: + files: | + cmd/** + internal/** + .golangci.yml + go.mod + go.sum + + - name: Setup Go + if: steps.changed-files.outputs.any_modified == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5 + with: + go-version: ${{ matrix.go-version }} + + - name: golangci-lint + if: steps.changed-files.outputs.any_modified == 'true' + uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86 # v6 + with: + version: latest + args: --timeout=5m + + - name: Install dependencies + if: steps.changed-files.outputs.any_modified == 'true' + run: go mod download + + - name: Test with Go + if: steps.changed-files.outputs.any_modified == 'true' + run: go test -race -coverprofile=coverage.txt -covermode=atomic ./... + + - name: Upload coverage reports to Codecov + if: steps.changed-files.outputs.any_modified == 'true' + uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: cover.txt + slug: bl4ko/netbox-ssot + + vulnerabilities: + name: Check for vulnerabilities + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@c65cd883420fd2eb864698a825fc4162dd94482c # v44 + with: + files: | + cmd/** + internal/** + .golangci.yml + go.mod + go.sum + .dockerignore + Dockerfile + + - name: Run Trivy vulnerability scanner + if: steps.changed-files.outputs.any_modified == 'true' + uses: aquasecurity/trivy-action@master + with: + scan-type: 'fs' + scan-ref: '.' + vuln-type: 'os,library' + severity: 'CRITICAL,HIGH' diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml new file mode 100644 index 0000000..88bb585 --- /dev/null +++ b/.github/workflows/gitleaks.yml @@ -0,0 +1,16 @@ +name: Check for git secrets with Gitleaks +on: + pull_request: + branches: + - main +jobs: + scan: + name: gitleaks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + with: + fetch-depth: 0 + - uses: gitleaks/gitleaks-action@44c470ffc35caa8b1eb3e8012ca53c2f9bea4eb5 # v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..485578c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,24 @@ +name: Create semantic release from commit messages +on: + push: + branches: [main] + workflow_dispatch: + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 + with: + node-version: "lts/*" + + - name: Release + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + run: npx semantic-release@24.0.0 diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml new file mode 100644 index 0000000..998f983 --- /dev/null +++ b/.github/workflows/spellcheck.yml @@ -0,0 +1,11 @@ +name: Check spelling with spellcheck +on: + pull_request: + branches: + - main +jobs: + spellcheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + - uses: streetsidesoftware/cspell-action@0e63b882c2ef0e24d78b8b1fbb132b42c0a0d0cb # v6 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d19c362 --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +# Created by https://www.toptal.com/developers/gitignore/api/go +# Edit at https://www.toptal.com/developers/gitignore?templates=go + +### Go ### +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work + +# End of https://www.toptal.com/developers/gitignore/api/go diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..0c0d671 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,80 @@ +linters-settings: + misspell: + locale: US + +issues: + max-issues-per-linter: 50 + max-same-issues: 20 + +linters: + enable: + - revive + - gocyclo + - dupl + - errorlint + - gocognit + - cyclop + - paralleltest + - lll + - nestif + - maintidx + - exhaustive + - nilnil + - gochecknoglobals + - goerr113 + - ireturn + - containedctx + - gosec + - gocritic + - asasalint + - asciicheck + - bidichk + - bodyclose + - decorder + - dogsled + - dupword + - durationcheck + - errchkjson + - errname + - execinquery + - exportloopref + - forcetypeassert + - gci + - gocheckcompilerdirectives + - gochecknoinits + - goconst + - godot + - goheader + - goimports + - gomnd + - gomoddirectives + - goprintffuncname + - gosmopolitan + - grouper + - importas + - interfacebloat + - makezero + - mirror + - misspell + - musttag + - nakedret + - nilerr + - noctx + - nolintlint + - nosprintfhostport + - prealloc + - predeclared + - promlinter + - reassign + - rowserrcheck + - sqlclosecheck + - tagalign + - tenv + - thelper + - tparallel + - unconvert + - unparam + - usestdlibvars + - wastedassign + - whitespace + - zerologlint diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..17b9322 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,29 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + - id: check-json + - id: check-xml + - id: end-of-file-fixer + - id: trailing-whitespace + - id: pretty-format-json + args: ["--autofix"] + - id: check-case-conflict + + - repo: https://github.com/dnephin/pre-commit-golang + rev: v0.5.1 + hooks: + - id: golangci-lint + - id: go-unit-tests + - id: go-mod-tidy + + - repo: https://github.com/gitleaks/gitleaks + rev: v8.18.4 + hooks: + - id: gitleaks + + - repo: https://github.com/streetsidesoftware/cspell-cli + rev: v8.13.1 + hooks: + - id: cspell diff --git a/.releaserc b/.releaserc new file mode 100644 index 0000000..548e644 --- /dev/null +++ b/.releaserc @@ -0,0 +1,10 @@ +{ + "branches": [ + "main" + ], + "plugins": [ + "@semantic-release/commit-analyzer", + "@semantic-release/release-notes-generator", + "@semantic-release/github" + ] +} diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..097176b --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 bl4ko + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e5c84b9 --- /dev/null +++ b/README.md @@ -0,0 +1,79 @@ +# Certstream-go + +Simple go client library for interacting with the cerstream logs, inspired by [CaliDog](https://github.com/CaliDog/certstream-go). + +# Usage + +```go +package main + +import ( + "fmt" + + "github.com/bl4ko/certstream-go" +) + +func main() { + certstreamServerURL := "wss://certstream.calidog.io" + timeout := 15 + stream, errStream := certstream.EventStream(true, certstreamServerURL, timeout) + for { + select { + case message := <-stream: + fmt.Printf("Received stream: %+v\n\n", message) + case err := <-errStream: + fmt.Printf("Received error: %s\n\n", err) + } + } +} +``` + +# Example certstream.Message + +Certstream-go returns data in the following format: + +```go +type Message struct { + MessageType string `json:"message_type"` + Data struct { + CertIndex int `json:"cert_index"` + CertLink string `json:"cert_link"` + LeafCert struct { + AllDomains []string `json:"all_domains"` + Extensions struct { + AuthorityInfoAccess string `json:"authorityInfoAccess"` + AuthorityKeyIdentifier string `json:"authorityKeyIdentifier"` + BasicConstraints string `json:"basicConstraints"` + CertificatePolicies string `json:"certificatePolicies"` + CtlPoisonByte bool `json:"ctlPoisonByte"` + ExtendedKeyUsage string `json:"extendedKeyUsage"` + KeyUsage string `json:"keyUsage"` + SubjectAltName string `json:"subjectAltName"` + SubjectKeyIdentifier string `json:"subjectKeyIdentifier"` + } `json:"extensions"` + Fingerprint string `json:"fingerprint"` + NotAfter int `json:"not_after"` + NotBefore int `json:"not_before"` + SerialNumber string `json:"serial_number"` + SignatureAlgorithm string `json:"signature_algorithm"` + Subject Name `json:"subject"` + Issuer Name `json:"issuer"` + IsCA bool `json:"is_ca"` + } `json:"leaf_cert"` + Seen float64 `json:"seen"` + Source Source `json:"source"` + UpdateType string `json:"update_type"` + } `json:"data"` +} + +type Name struct { + C string `json:"C"` + CN string `json:"CN"` + L string `json:"L"` + O string `json:"O"` + OU string `json:"OU"` + ST string `json:"ST"` + Aggregated string `json:"aggregated"` + EmailAddress string `json:"email_address"` +} +``` diff --git a/certstream.go b/certstream.go new file mode 100644 index 0000000..180c647 --- /dev/null +++ b/certstream.go @@ -0,0 +1,142 @@ +package certstream + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/gorilla/websocket" +) + +const ( + PingPeriod time.Duration = 15 * time.Second + DefaultTimeout = 15 + DefaultSleep = 5 +) + +type Message struct { + MessageType string `json:"message_type"` + Data struct { + CertIndex int `json:"cert_index"` + CertLink string `json:"cert_link"` + LeafCert struct { + AllDomains []string `json:"all_domains"` + Extensions struct { + AuthorityInfoAccess string `json:"authorityInfoAccess"` + AuthorityKeyIdentifier string `json:"authorityKeyIdentifier"` + BasicConstraints string `json:"basicConstraints"` + CertificatePolicies string `json:"certificatePolicies"` + CtlPoisonByte bool `json:"ctlPoisonByte"` + ExtendedKeyUsage string `json:"extendedKeyUsage"` + KeyUsage string `json:"keyUsage"` + SubjectAltName string `json:"subjectAltName"` + SubjectKeyIdentifier string `json:"subjectKeyIdentifier"` + } `json:"extensions"` + Fingerprint string `json:"fingerprint"` + NotAfter int `json:"not_after"` + NotBefore int `json:"not_before"` + SerialNumber string `json:"serial_number"` + SignatureAlgorithm string `json:"signature_algorithm"` + Subject Name `json:"subject"` + Issuer Name `json:"issuer"` + IsCA bool `json:"is_ca"` + } `json:"leaf_cert"` + Seen float64 `json:"seen"` + Source Source `json:"source"` + UpdateType string `json:"update_type"` + } `json:"data"` +} + +type Name struct { + C string `json:"C"` + CN string `json:"CN"` + L string `json:"L"` + O string `json:"O"` + OU string `json:"OU"` + ST string `json:"ST"` + Aggregated string `json:"aggregated"` + EmailAddress string `json:"email_address"` +} + +type Source struct { + Name string `json:"name"` + URL string `json:"url"` +} + +func EventStream(skipHeartbeats bool, certstreamServerURL string, timeout int) (chan Message, chan error) { + if timeout == 0 { + timeout = DefaultTimeout + } + + outputStream := make(chan Message) + errStream := make(chan error) + + go connectAndListen(certstreamServerURL, timeout, skipHeartbeats, outputStream, errStream) + + return outputStream, errStream +} + +func connectAndListen(url string, timeout int, skipHeartbeats bool, outputStream chan Message, errStream chan error) { + for { + c, _, err := websocket.DefaultDialer.Dial(url, nil) + if err != nil { + errStream <- fmt.Errorf("Error connecting to certstream: %w", err) + time.Sleep(DefaultSleep * time.Second) + continue + } + done := make(chan struct{}) + go sendPingMessages(c, done, errStream) + + if err := readMessages(c, timeout, skipHeartbeats, outputStream); err != nil { + errStream <- fmt.Errorf("Error reading messages: %w", err) + close(done) + c.Close() + time.Sleep(DefaultSleep * time.Second) + continue + } + + close(done) + c.Close() + } +} + +func sendPingMessages(c *websocket.Conn, done chan struct{}, errStream chan error) { + ticker := time.NewTicker(PingPeriod) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if err := c.WriteMessage(websocket.PingMessage, nil); err != nil { + errStream <- fmt.Errorf("Error sending ping message: %w", err) + return + } + case <-done: + return + } + } +} + +func readMessages(c *websocket.Conn, timeout int, skipHeartbeats bool, outputStream chan Message) error { + for { + if err := c.SetReadDeadline(time.Now().Add(time.Duration(timeout) * time.Second)); err != nil { + return fmt.Errorf("Error creating wss deadline: %w", err) + } + + _, rawMessage, err := c.ReadMessage() + if err != nil { + return fmt.Errorf("Error reading message: %w", err) + } + + var message Message + if err := json.Unmarshal(rawMessage, &message); err != nil { + return fmt.Errorf("Error unmarshalling certstream message: %w", err) + } + + if skipHeartbeats && message.MessageType == "heartbeat" { + continue + } + + outputStream <- message + } +} diff --git a/cspell.json b/cspell.json new file mode 100644 index 0000000..8a59efd --- /dev/null +++ b/cspell.json @@ -0,0 +1,23 @@ +{ + "dictionaries": [ + "project-words" + ], + "dictionaryDefinitions": [ + { + "name": "project-words", + "path": "./project-words.txt" + } + ], + "enabled": false, + "ignorePaths": [ + ".git/*", + ".git/!{COMMIT_EDITMSG,EDITMSG}", + ".git/*/**", + ".gitignore", + "action/lib/**", + "cspell.json", + "go.mod", + "go.sum" + ], + "language": "en" +} diff --git a/example/main.go b/example/main.go new file mode 100644 index 0000000..663f3ab --- /dev/null +++ b/example/main.go @@ -0,0 +1,21 @@ +package main + +import ( + "fmt" + + "github.com/bl4ko/certstream-go" +) + +func main() { + certstreamServerURL := "wss://certstream.calidog.io" + timeout := 15 + stream, errStream := certstream.EventStream(true, certstreamServerURL, timeout) + for { + select { + case message := <-stream: + fmt.Printf("Received stream: %+v\n\n", message) + case err := <-errStream: + fmt.Printf("Received error: %s\n\n", err) + } + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..9e5f1b2 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/bl4ko/certstream-go + +go 1.21.4 + +require github.com/gorilla/websocket v1.5.3 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..25a9fc4 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/project-words.txt b/project-words.txt new file mode 100644 index 0000000..e634d0e --- /dev/null +++ b/project-words.txt @@ -0,0 +1 @@ +certstream diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..59f909a --- /dev/null +++ b/renovate.json @@ -0,0 +1,59 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "baseBranches": [ + "main" + ], + "extends": [ + "config:recommended", + "docker:pinDigests", + "helpers:pinGitHubActionDigests", + ":pinDevDependencies" + ], + "packageRules": [ + { + "automerge": true, + "groupName": "go dependencies", + "matchManagers": [ + "gomod" + ] + }, + { + "automerge": true, + "groupName": "github actions", + "matchManagers": [ + "github-actions" + ] + }, + { + "automerge": true, + "groupName": "dockerfile dependencies", + "matchManagers": [ + "dockerfile" + ] + }, + { + "automerge": true, + "automergeStrategy": "rebase", + "groupName": "semantic-release", + "matchManagers": [ + "regex" + ] + } + ], + "pre-commit": { + "enabled": true + }, + "regexManagers": [ + { + "datasourceTemplate": "npm", + "depNameTemplate": "semantic-release", + "description": "Update semantic-release version used by npx", + "fileMatch": [ + "^\\.github/workflows/[^/]+\\.ya?ml$" + ], + "matchStrings": [ + "\\srun: npx semantic-release@(?.*?)\\s" + ] + } + ] +}