Add support for GitHub advisories

tools/vulndb/internal/db/createTablesIfNotExist.sql

@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS github_advisories (
+  id TEXT PRIMARY KEY NOT NULL,
+  repository NOT NULL,
+  published DATETIME NOT NULL,
+  severity TEXT NOT NULL
+);

tools/vulndb/internal/db/database.go

@@ -0,0 +1,76 @@
+package db
+
+import (
+	"context"
+	"database/sql"
+	_ "embed" // Embed SQL files
+	"net/url"
+	"strings"
+
+	"github.com/AlexGustafsson/cupdate/tools/vulndb/internal/ossf"
+	_ "modernc.org/sqlite"
+)
+
+//go:embed createTablesIfNotExist.sql
+var createTablesIfNotExist string
+
+type Conn struct {
+	db *sql.DB
+}
+
+func Open(path string) (*Conn, error) {
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = db.Exec(createTablesIfNotExist)
+	if err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+
+	return &Conn{db: db}, nil
+}
+
+func (c *Conn) Insert(ctx context.Context, vuln ossf.OpenSourceVulnerability) error {
+	statement, err := c.db.PrepareContext(ctx, `INSERT INTO github_advisories (id, repository, published, severity) VALUES (?, ?, ?, ?) ON CONFLICT DO NOTHING;`)
+	if err != nil {
+		return err
+	}
+
+	repository := ""
+	for _, reference := range vuln.References {
+		u, err := url.Parse(reference.Url)
+		if err == nil {
+			segments := len(u.Path) - len(strings.ReplaceAll(u.Path, "/", ""))
+			if u.Host == "github.com" && segments == 2 {
+				repository = reference.Url
+				break
+			}
+		}
+	}
+
+	// No repository found
+	if repository == "" {
+		return nil
+	}
+
+	severity := ""
+	if value, ok := vuln.DatabaseSpecific["severity"]; ok {
+		severity = value.(string)
+	}
+
+	// TODO: Insert ranges, or duplicate for each range
+
+	_, err = statement.ExecContext(ctx, vuln.ID, repository, vuln.Published, severity)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Conn) Close() error {
+	return c.db.Close()
+}

tools/vulndb/internal/git/clone.go

@@ -0,0 +1,37 @@
+package git
+
+import (
+	"context"
+	"os/exec"
+)
+
+func ShallowClone(ctx context.Context, repository string, output string, directories ...string) error {
+	cloneCmd := exec.CommandContext(ctx, "git", "clone", "--filter=tree:0", "--depth=1", "--no-checkout", "--sparse", repository, output)
+	if err := cloneCmd.Run(); err != nil {
+		return err
+	}
+
+	sparseInitCmd := exec.CommandContext(ctx, "git", "sparse-checkout", "init", "--sparse-index", "--cone")
+	sparseInitCmd.Dir = output
+	if err := sparseInitCmd.Run(); err != nil {
+		return err
+	}
+
+	sparseInitOptions := append([]string{
+		"sparse-checkout",
+		"add",
+	}, directories...)
+	spareInitCheckoutCmd := exec.CommandContext(ctx, "git", sparseInitOptions...)
+	spareInitCheckoutCmd.Dir = output
+	if err := spareInitCheckoutCmd.Run(); err != nil {
+		return err
+	}
+
+	checkoutCmd := exec.CommandContext(ctx, "git", "checkout")
+	checkoutCmd.Dir = output
+	if err := checkoutCmd.Run(); err != nil {
+		return err
+	}
+
+	return nil
+}

tools/vulndb/main.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"os"
+	"os/signal"
+	"path/filepath"
+
+	"github.com/AlexGustafsson/cupdate/tools/vulndb/internal/db"
+	"github.com/AlexGustafsson/cupdate/tools/vulndb/internal/git"
+	"github.com/AlexGustafsson/cupdate/tools/vulndb/internal/ossf"
+)
+
+func main() {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt)
+
+		<-signals
+		slog.Info("Caught signal, exiting gracefully")
+		cancel()
+	}()
+
+	if err := run(ctx); err != nil {
+		slog.Error("Fatal error", slog.Any("error", err))
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context) error {
+	workdir, err := os.MkdirTemp(os.TempDir(), "cupdate-vulndb-*")
+	if err != nil {
+		return err
+	}
+
+	workdir = filepath.Join(workdir, "advisory-database")
+
+	err = git.ShallowClone(context.Background(), "https://github.com/github/advisory-database", workdir, "advisories/github-reviewed/2024")
+	if err != nil {
+		return fmt.Errorf("failed to clone repository: %w", err)
+	}
+
+	db, err := db.Open("vulndb.sqlite")
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	err = filepath.WalkDir(workdir, func(path string, d fs.DirEntry, err error) error {
+		if filepath.Ext(path) == ".json" {
+			file, err := os.Open(path)
+			if err != nil {
+				return err
+			}
+
+			var vuln ossf.OpenSourceVulnerability
+			if err := json.NewDecoder(file).Decode(&vuln); err != nil {
+				return err
+			}
+
+			return db.Insert(ctx, vuln)
+		}
+
+		return nil
+	})
+
+	if err := db.Close(); err != nil {
+		slog.Error("Failed to close database", slog.Any("error", db))
+	}
+
+	return err
+}