diff --git a/tools/vulndb/internal/db/createTablesIfNotExist.sql b/tools/vulndb/internal/db/createTablesIfNotExist.sql new file mode 100644 index 0000000..1f175b6 --- /dev/null +++ b/tools/vulndb/internal/db/createTablesIfNotExist.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS github_advisories ( + id TEXT PRIMARY KEY NOT NULL, + repository NOT NULL, + published DATETIME NOT NULL, + severity TEXT NOT NULL +); diff --git a/tools/vulndb/internal/db/database.go b/tools/vulndb/internal/db/database.go new file mode 100644 index 0000000..0978ca1 --- /dev/null +++ b/tools/vulndb/internal/db/database.go @@ -0,0 +1,76 @@ +package db + +import ( + "context" + "database/sql" + _ "embed" // Embed SQL files + "net/url" + "strings" + + "github.com/AlexGustafsson/cupdate/tools/vulndb/internal/ossf" + _ "modernc.org/sqlite" +) + +//go:embed createTablesIfNotExist.sql +var createTablesIfNotExist string + +type Conn struct { + db *sql.DB +} + +func Open(path string) (*Conn, error) { + db, err := sql.Open("sqlite", path) + if err != nil { + return nil, err + } + + _, err = db.Exec(createTablesIfNotExist) + if err != nil { + _ = db.Close() + return nil, err + } + + return &Conn{db: db}, nil +} + +func (c *Conn) Insert(ctx context.Context, vuln ossf.OpenSourceVulnerability) error { + statement, err := c.db.PrepareContext(ctx, `INSERT INTO github_advisories (id, repository, published, severity) VALUES (?, ?, ?, ?) ON CONFLICT DO NOTHING;`) + if err != nil { + return err + } + + repository := "" + for _, reference := range vuln.References { + u, err := url.Parse(reference.Url) + if err == nil { + segments := len(u.Path) - len(strings.ReplaceAll(u.Path, "/", "")) + if u.Host == "github.com" && segments == 2 { + repository = reference.Url + break + } + } + } + + // No repository found + if repository == "" { + return nil + } + + severity := "" + if value, ok := vuln.DatabaseSpecific["severity"]; ok { + severity = value.(string) + } + + // TODO: Insert ranges, or duplicate for each range + + _, err = statement.ExecContext(ctx, vuln.ID, repository, vuln.Published, severity) + if err != nil { + return err + } + + return nil +} + +func (c *Conn) Close() error { + return c.db.Close() +} diff --git a/tools/vulndb/internal/git/clone.go b/tools/vulndb/internal/git/clone.go new file mode 100644 index 0000000..ecfdf10 --- /dev/null +++ b/tools/vulndb/internal/git/clone.go @@ -0,0 +1,37 @@ +package git + +import ( + "context" + "os/exec" +) + +func ShallowClone(ctx context.Context, repository string, output string, directories ...string) error { + cloneCmd := exec.CommandContext(ctx, "git", "clone", "--filter=tree:0", "--depth=1", "--no-checkout", "--sparse", repository, output) + if err := cloneCmd.Run(); err != nil { + return err + } + + sparseInitCmd := exec.CommandContext(ctx, "git", "sparse-checkout", "init", "--sparse-index", "--cone") + sparseInitCmd.Dir = output + if err := sparseInitCmd.Run(); err != nil { + return err + } + + sparseInitOptions := append([]string{ + "sparse-checkout", + "add", + }, directories...) + spareInitCheckoutCmd := exec.CommandContext(ctx, "git", sparseInitOptions...) + spareInitCheckoutCmd.Dir = output + if err := spareInitCheckoutCmd.Run(); err != nil { + return err + } + + checkoutCmd := exec.CommandContext(ctx, "git", "checkout") + checkoutCmd.Dir = output + if err := checkoutCmd.Run(); err != nil { + return err + } + + return nil +} diff --git a/tools/vulndb/internal/ossf/ossf.go b/tools/vulndb/internal/ossf/ossf.go new file mode 100644 index 0000000..f916f66 --- /dev/null +++ b/tools/vulndb/internal/ossf/ossf.go @@ -0,0 +1,67 @@ +// Code generated by github.com/atombender/go-jsonschema, DO NOT EDIT. + +package ossf + +import "time" + + +// A schema for describing a vulnerability in an open source package. See also +// https://ossf.github.io/osv-schema/. +type OpenSourceVulnerability struct { + Affected []Affected `json:"affected,omitempty"` + Aliases []string `json:"aliases,omitempty"` + Credits []Credit `json:"credits,omitempty"` + DatabaseSpecific map[string]any `json:"database_specific,omitempty"` + Details *string `json:"details,omitempty"` + ID Prefix `json:"id"` + Modified time.Time `json:"modified"` + Published time.Time `json:"published,omitempty"` + References []Reference `json:"references,omitempty"` + Related []string `json:"related,omitempty"` + SchemaVersion *string `json:"schema_version,omitempty"` + Severity Severity `json:"severity,omitempty"` + Summary *string `json:"summary,omitempty"` + Withdrawn time.Time `json:"withdrawn,omitempty"` +} + +type Affected struct { + DatabaseSpecific map[string]any `json:"database_specific,omitempty"` + EcosystemSpecific map[string]any `json:"ecosystem_specific,omitempty"` + Package *AffectedPackage `json:"package,omitempty"` + Ranges []AffactedRange `json:"ranges,omitempty"` + Severity Severity `json:"severity,omitempty"` + Versions []string `json:"versions,omitempty"` +} + +type AffectedPackage struct { + Ecosystem string `json:"ecosystem"` + Name string `json:"name"` + Purl *string `json:"purl,omitempty"` +} + +type AffactedRange struct { + DatabaseSpecific map[string]any `json:"database_specific,omitempty"` + Events []map[string]any `json:"events"` + Repo *string `json:"repo,omitempty"` + Type string `json:"type"` +} + +type Credit struct { + Contact []string `json:"contact,omitempty"` + Name string `json:"name"` + Type string `json:"type,omitempty"` +} + +type Reference struct { + Type string `json:"type"` + Url string `json:"url"` +} + +// These home databases are also documented at +// https://ossf.github.io/osv-schema/#id-modified-fields. +type Prefix string + +type Severity []struct { + Score string `json:"score"` + Type string `json:"type"` +} diff --git a/tools/vulndb/main.go b/tools/vulndb/main.go new file mode 100644 index 0000000..76eb145 --- /dev/null +++ b/tools/vulndb/main.go @@ -0,0 +1,78 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io/fs" + "log/slog" + "os" + "os/signal" + "path/filepath" + + "github.com/AlexGustafsson/cupdate/tools/vulndb/internal/db" + "github.com/AlexGustafsson/cupdate/tools/vulndb/internal/git" + "github.com/AlexGustafsson/cupdate/tools/vulndb/internal/ossf" +) + +func main() { + ctx, cancel := context.WithCancel(context.Background()) + + go func() { + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt) + + <-signals + slog.Info("Caught signal, exiting gracefully") + cancel() + }() + + if err := run(ctx); err != nil { + slog.Error("Fatal error", slog.Any("error", err)) + os.Exit(1) + } +} + +func run(ctx context.Context) error { + workdir, err := os.MkdirTemp(os.TempDir(), "cupdate-vulndb-*") + if err != nil { + return err + } + + workdir = filepath.Join(workdir, "advisory-database") + + err = git.ShallowClone(context.Background(), "https://github.com/github/advisory-database", workdir, "advisories/github-reviewed/2024") + if err != nil { + return fmt.Errorf("failed to clone repository: %w", err) + } + + db, err := db.Open("vulndb.sqlite") + if err != nil { + return err + } + defer db.Close() + + err = filepath.WalkDir(workdir, func(path string, d fs.DirEntry, err error) error { + if filepath.Ext(path) == ".json" { + file, err := os.Open(path) + if err != nil { + return err + } + + var vuln ossf.OpenSourceVulnerability + if err := json.NewDecoder(file).Decode(&vuln); err != nil { + return err + } + + return db.Insert(ctx, vuln) + } + + return nil + }) + + if err := db.Close(); err != nil { + slog.Error("Failed to close database", slog.Any("error", db)) + } + + return err +} diff --git a/tools/vulndb/vulndb.sqlite b/tools/vulndb/vulndb.sqlite new file mode 100644 index 0000000..cf7b80e Binary files /dev/null and b/tools/vulndb/vulndb.sqlite differ