Implementation of python setup.py extractor.

docs/supported_inventory_types.md

@@ -58,6 +58,7 @@ SCALIBR supports extracting software package information from a variety of OS an
   * Installed PyPI packages (global and venv)
   * Lockfiles: requirements.txt, poetry.lock, Pipfile.lock, pdm.lock
   * Conda packages
+  * setup.py
 * R
   * Lockfiles: renv.lock
 * Ruby

extractor/filesystem/language/python/setup/setup.go

@@ -0,0 +1,198 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package setup extracts packages from setup.py.
+package setup
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/google/osv-scalibr/extractor"
+	"github.com/google/osv-scalibr/extractor/filesystem"
+	"github.com/google/osv-scalibr/extractor/filesystem/internal/units"
+	"github.com/google/osv-scalibr/extractor/filesystem/language/python/internal/pypipurl"
+	"github.com/google/osv-scalibr/plugin"
+	"github.com/google/osv-scalibr/purl"
+	"github.com/google/osv-scalibr/stats"
+)
+
+const (
+	// Name is the unique name of this extractor.
+	Name = "python/setup"
+
+	// defaultMaxFileSizeBytes is the maximum file size an extractor will unmarshal.
+	// If Extract gets a bigger file, it will return an error.
+	defaultMaxFileSizeBytes = 100 * units.MiB
+)
+
+// Config is the configuration for the Extractor.
+type Config struct {
+	// Stats is a stats collector for reporting metrics.
+	Stats stats.Collector
+	// MaxFileSizeBytes is the maximum file size this extractor will unmarshal. If
+	// `FileRequired` gets a bigger file, it will return false,
+	MaxFileSizeBytes int64
+}
+
+// DefaultConfig returns the default configuration for the setup.py extractor.
+func DefaultConfig() Config {
+	return Config{
+		Stats:            nil,
+		MaxFileSizeBytes: defaultMaxFileSizeBytes,
+	}
+}
+
+// Extractor extracts python packages from setup.py.
+type Extractor struct {
+	stats            stats.Collector
+	maxFileSizeBytes int64
+}
+
+// New returns a setup.py extractor.
+func New(cfg Config) *Extractor {
+	return &Extractor{
+		stats:            cfg.Stats,
+		maxFileSizeBytes: cfg.MaxFileSizeBytes,
+	}
+}
+
+// Config returns the configuration of the extractor.
+func (e Extractor) Config() Config {
+	return Config{
+		Stats:            e.stats,
+		MaxFileSizeBytes: e.maxFileSizeBytes,
+	}
+}
+
+// Name of the extractor.
+func (e Extractor) Name() string { return Name }
+
+// Version of the extractor.
+func (e Extractor) Version() int { return 0 }
+
+// Requirements of the extractor.
+func (e Extractor) Requirements() *plugin.Capabilities { return &plugin.Capabilities{} }
+
+// FileRequired returns true if the specified file matches python setup.py file pattern.
+func (e Extractor) FileRequired(api filesystem.FileAPI) bool {
+	path := api.Path()
+
+	if filepath.Base(path) != "setup.py" {
+		return false
+	}
+
+	fileinfo, err := api.Stat()
+	if err != nil {
+		return false
+	}
+	if e.maxFileSizeBytes > 0 && fileinfo.Size() > e.maxFileSizeBytes {
+		e.reportFileRequired(path, fileinfo.Size(), stats.FileRequiredResultSizeLimitExceeded)
+		return false
+	}
+
+	e.reportFileRequired(path, fileinfo.Size(), stats.FileRequiredResultOK)
+	return true
+}
+
+func (e Extractor) reportFileRequired(path string, fileSizeBytes int64, result stats.FileRequiredResult) {
+	if e.stats == nil {
+		return
+	}
+	e.stats.AfterFileRequired(e.Name(), &stats.FileRequiredStats{
+		Path:          path,
+		Result:        result,
+		FileSizeBytes: fileSizeBytes,
+	})
+}
+
+// Extract extracts packages from setup.py files passed through the scan input.
+func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]*extractor.Inventory, error) {
+	inventory, err := e.extractFromInput(ctx, input)
+
+	if e.stats != nil {
+		var fileSizeBytes int64
+		if input.Info != nil {
+			fileSizeBytes = input.Info.Size()
+		}
+		e.stats.AfterFileExtracted(e.Name(), &stats.FileExtractedStats{
+			Path:          input.Path,
+			Result:        filesystem.ExtractorErrorToFileExtractedResult(err),
+			FileSizeBytes: fileSizeBytes,
+		})
+	}
+	return inventory, err
+}
+
+var packageVersionRe = regexp.MustCompile(`['"]\W?(\w+\W?==\W?[\w.]*)`)
+
+func (e Extractor) extractFromInput(ctx context.Context, input *filesystem.ScanInput) ([]*extractor.Inventory, error) {
+	s := bufio.NewScanner(input.Reader)
+	pkgs := []*extractor.Inventory{}
+
+	for s.Scan() {
+		// Return if canceled or exceeding deadline.
+		if err := ctx.Err(); err != nil {
+			return pkgs, fmt.Errorf("%s halted at %q because of context error: %v", e.Name(), input.Path, err)
+		}
+
+		line := s.Text()
+		line = strings.TrimSpace(line)
+
+		matches := packageVersionRe.FindAllString(line, -1)
+
+		for _, match := range matches {
+			pkg := strings.Split(match, "==")
+
+			if len(pkg) == 2 {
+				pkgName := strings.TrimSpace(strings.Trim(pkg[0], `"'`))
+				pkgVersion := strings.TrimSpace(strings.Trim(pkg[1], `"'`))
+
+				if containsTemplate(pkgName, pkgVersion) {
+					continue
+				}
+
+				i := &extractor.Inventory{
+					Name:      pkgName,
+					Version:   pkgVersion,
+					Locations: []string{input.Path},
+				}
+
+				pkgs = append(pkgs, i)
+			}
+		}
+
+		if s.Err() != nil {
+			return pkgs, fmt.Errorf("error while scanning setup.py file from %v: %w", input.Path, s.Err())
+		}
+	}
+
+	return pkgs, nil
+}
+
+func containsTemplate(pkgName, pkgVersion string) bool {
+	return strings.ContainsAny(pkgName, "%{}") || strings.ContainsAny(pkgVersion, "%{}")
+}
+
+// Ecosystem returns the OSV Ecosystem of the software extracted by this extractor.
+func (Extractor) Ecosystem(i *extractor.Inventory) string { return "PyPI" }
+
+// ToPURL converts an inventory created by this extractor into a PURL.
+func (e Extractor) ToPURL(i *extractor.Inventory) *purl.PackageURL {
+	return pypipurl.MakePackageURL(i)
+}