Support offline validation (no network) using only built-in, local schema

.vscode/launch.json

@@ -5,6 +5,27 @@
     "version": "0.2.0",
     "configurations": [
 
+        {
+            "showGlobalVariables": true,
+            "name": "Debug: validate",
+            "type": "go",
+            "request": "launch",
+            "mode": "debug",
+            "program": "main.go", // "program": "${file}",
+            "args": ["validate", "-i", "examples/cyclonedx/SBOM/protonmail-webclient-v4-0912dff/bom.json"],
+            "dlvFlags": ["--check-go-version=false"]
+        },
+        {
+            "showGlobalVariables": true,
+            "name": "Debug: validate (offline)",
+            "type": "go",
+            "request": "launch",
+            "mode": "debug",
+            "program": "main.go", // "program": "${file}",
+            "args": ["validate", "-i", "test/cyclonedx/cdx-1-5-mature-example-1.json"],
+            "dlvFlags": ["--check-go-version=false"]
+        },
+
         {
             "showGlobalVariables": true,
             "name": "Debug: validate",

README.md

@@ -318,11 +318,17 @@ See each command's section for contextual examples of the `--where` flag filter
 
 ### Validate
 
-This command will parse standardized SBOMs and validate it against its declared format and version (e.g., SPDX 2.3, CycloneDX 1.6). Custom  variants of standard JSON schemas can be used for validation by supplying the `--variant` name as a flag. Explicit JSON schemas can be specified using the `--force` flag.
+This command will parse standardized SBOMs and validate it against its declared format and version (e.g., SPDX 2.3, CycloneDX 1.6). 
+
+- Custom  variants of standard JSON schemas can be used for validation by supplying the `--variant` name as a flag. 
+- Explicit JSON schemas can be specified using the `--force` flag.
 
 #### Validating using supported schemas
 
-Use the [schema](#schema) command to list supported schemas formats, versions and variants.
+Use the [schema](#schema) command to list supported schemas formats, versions and variants.  
+
+- A "supported" schema is already **"built-in"** to the utility resources along with any dependent schemas it imports.
+- This means that BOM files **can be validated when there is no network connection** to load the schemas from remote locations (a.k.a., *"off-line"* mode).
 
 #### Validating using "custom" schemas
 

cmd/validate.go

@@ -181,6 +181,71 @@ func validationError(document *schema.BOM, valid bool, err error) {
 	getLogger().Info(message)
 }
 
+func LoadSchemaDependencies(depSchemaLoader *gojsonschema.SchemaLoader, schemas []schema.FormatSchemaInstance, schemaNames []string) (err error) {
+	for _, schemaName := range schemaNames {
+		formatSchemaInstance, errMatch := schema.FindMatchingFormatSchemaInstance(
+			schemas, schemaName)
+		if errMatch != nil {
+			return fmt.Errorf("schema '%s' match not found in resources: '%s'", schemaName, schema.SCHEMA_FORMAT_COMMON)
+		}
+		getLogger().Debugf("Found: '%s': %v", schemaName, formatSchemaInstance)
+
+		getLogger().Infof("Added schema '%s' to loader:...", formatSchemaInstance.File)
+		err = AddDependencySchemaToLoader(depSchemaLoader, formatSchemaInstance)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func AddDependencySchemaToLoader(depSchemaLoader *gojsonschema.SchemaLoader, formatSchemaInstance schema.FormatSchemaInstance) (err error) {
+	getLogger().Debugf("Reading schema: '%s'...", formatSchemaInstance.File)
+	bSchema, errRead := resources.BOMSchemaFiles.ReadFile(formatSchemaInstance.File)
+
+	if errRead != nil {
+		return errRead
+	}
+	getLogger().Tracef("schema: %s", bSchema)
+	sharedSchemaLoader := gojsonschema.NewBytesLoader(bSchema)
+	depSchemaLoader.AddSchema(formatSchemaInstance.Url, sharedSchemaLoader)
+	return
+}
+
+func LoadCompileSchemaDependencies(
+	bomSchemaLoader gojsonschema.JSONLoader,
+	bomSchemaInstance schema.FormatSchemaInstance,
+	bomSchemaDependencies []string,
+) (jsonBOMSchema *gojsonschema.Schema, err error) {
+
+	if len(bomSchemaDependencies) > 0 {
+		getLogger().Infof("Found schema dependencies: %v", bomSchemaDependencies)
+		// Create a schema loader we will add all dep. schemas into
+		depSchemaLoader := gojsonschema.NewSchemaLoader()
+
+		// Load common schema from application resources
+		var commonSchemas schema.FormatSchema
+		commonSchemas, err = SupportedFormatConfig.FindMatchingFormatSchema(schema.SCHEMA_FORMAT_COMMON)
+		if err != nil {
+			return
+		}
+		getLogger().Tracef("Found '%s' schemas: %v", schema.SCHEMA_FORMAT_COMMON, commonSchemas)
+
+		err = LoadSchemaDependencies(depSchemaLoader, commonSchemas.Schemas, bomSchemaDependencies)
+		if err != nil {
+			return
+		}
+
+		// Compile BOM schema (JSON) with the dependency schemas and return it
+		getLogger().Infof("Compiling schema: '%s'...", bomSchemaInstance.File)
+		jsonBOMSchema, err = depSchemaLoader.Compile(bomSchemaLoader)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
 func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, validateFlags utils.ValidateCommandFlags) (valid bool, bom *schema.BOM, schemaErrors []gojsonschema.ResultError, err error) {
 	getLogger().Enter()
 	defer getLogger().Exit()
@@ -213,8 +278,8 @@ func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, va
 
 	// Create a loader for the BOM (JSON) document
 	var documentLoader gojsonschema.JSONLoader
-	var schemaLoader gojsonschema.JSONLoader
-	var errRead error
+	var jsonBOMSchemaLoader gojsonschema.JSONLoader
+	var errRead, errLoadCompile error
 	var bSchema, bDocument []byte
 
 	if bDocument = bom.GetRawBytes(); len(bDocument) > 0 {
@@ -233,14 +298,15 @@ func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, va
 		return INVALID, bom, schemaErrors, fmt.Errorf("unable to load document: '%s'", bom.GetFilename())
 	}
 
+	// Regardless of how or where we load JSON schemas from the final
+	// we define the overall Schema object used to validate the BOM document
+	var jsonBOMSchema *gojsonschema.Schema
 	schemaName := bom.SchemaInfo.File
 
 	// If caller "forced" a specific schema file (version), load it instead of
 	// any SchemaInfo found in config.json
-	// TODO: support remote schema load (via URL) with a flag (default should always be local file for security)
 	forcedSchemaFile := validateFlags.ForcedJsonSchemaFile
 	if forcedSchemaFile != "" {
-
 		if !isValidURIPrefix(forcedSchemaFile) {
 			// attempt to load as a local file
 			forcedSchemaFile = "file://" + forcedSchemaFile
@@ -254,7 +320,7 @@ func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, va
 		}
 
 		getLogger().Infof("Loading schema from '--force' flag: '%s'...", forcedSchemaFile)
-		schemaLoader = gojsonschema.NewReferenceLoader(forcedSchemaFile)
+		jsonBOMSchemaLoader = gojsonschema.NewReferenceLoader(forcedSchemaFile)
 		getLogger().Infof("Validating document using forced schema (i.e., '--force %s')", forcedSchemaFile)
 	} else {
 		// Load the matching JSON schema (format, version and variant) from embedded resources
@@ -268,10 +334,19 @@ func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, va
 			return INVALID, bom, schemaErrors, errRead
 		}
 
-		schemaLoader = gojsonschema.NewBytesLoader(bSchema)
+		// Create a schema loader for the primary BOM schema file
+		jsonBOMSchemaLoader = gojsonschema.NewBytesLoader(bSchema)
+
+		// If the BOM schema has $refs to other schemas, attempt to load and compile
+		// them from those included as built-in resources
+		jsonBOMSchema, errLoadCompile = LoadCompileSchemaDependencies(jsonBOMSchemaLoader, bom.SchemaInfo, bom.SchemaInfo.Dependencies)
+		if err != nil {
+			return INVALID, bom, schemaErrors, errLoadCompile
+		}
 	}
 
-	if schemaLoader == nil {
+	// At this point we should have a BOM schema loader
+	if jsonBOMSchemaLoader == nil {
 		// we force result to INVALID as any errors from the library means
 		// we could NOT actually confirm the input documents validity
 		return INVALID, bom, schemaErrors, fmt.Errorf("unable to read schema: '%s'", schemaName)
@@ -280,27 +355,28 @@ func Validate(writer io.Writer, persistentFlags utils.PersistentCommandFlags, va
 	// create a reusable schema object (TODO: validate multiple documents)
 	var errLoad error = nil
 	const RETRY int = 3
-	var jsonBOMSchema *gojsonschema.Schema
 
 	// we force result to INVALID as any errors from the library means
 	// we could NOT actually confirm the input documents validity
 	// WARNING: if schemas reference "remote" schemas which are loaded
 	// over http... then there is a chance of 503 errors (as the pkg. loads
 	// externally referenced schemas over network)... attempt fixed retry...
-	for i := 0; i < RETRY; i++ {
-		jsonBOMSchema, errLoad = gojsonschema.NewSchema(schemaLoader)
+	if jsonBOMSchema == nil {
+		for i := 0; i < RETRY; i++ {
+			jsonBOMSchema, errLoad = gojsonschema.NewSchema(jsonBOMSchemaLoader)
 
-		if errLoad == nil {
-			break
+			if errLoad == nil {
+				break
+			}
+			getLogger().Warningf("unable to load referenced schema over HTTP: \"%v\"\n retrying...", errLoad)
 		}
-		getLogger().Warningf("unable to load referenced schema over HTTP: \"%v\"\n retrying...", errLoad)
-	}
 
-	if errLoad != nil {
-		return INVALID, bom, schemaErrors, fmt.Errorf("unable to load schema: '%s'", schemaName)
+		if errLoad != nil {
+			return INVALID, bom, schemaErrors, fmt.Errorf("unable to load schema: `%s`", schemaName)
+		}
 	}
 
-	getLogger().Infof("Schema '%s' loaded.", schemaName)
+	getLogger().Infof("Schema '%s' loaded", schemaName)
 
 	// Validate against the schema and save result determination
 	getLogger().Infof("Validating '%s'...", bom.GetFilenameInterpolated())

resources/config/config.json

@@ -1,5 +1,31 @@
 {
     "formats": [
+        {
+            "canonicalName": "common",
+            "propertyKeyFormat": "",
+            "propertyKeyVersion": "version",
+            "propertyValueFormat": "latest",
+            "schemas": [
+                {
+                    "version": "",
+                    "variant": "",
+                    "name": "jsf-0.82.schema.json",
+                    "file": "schema/cyclonedx/common/jsf-0.82.schema.json",
+                    "development": "",
+                    "url": "http://cyclonedx.org/schema/jsf-0.82.schema.json",
+                    "default": false
+                },
+                {
+                    "version": "",
+                    "variant": "",
+                    "name": "spdx.schema.json",
+                    "file": "schema/cyclonedx/common/spdx.schema.json",
+                    "development": "",
+                    "url": "http://cyclonedx.org/schema/spdx.schema.json",
+                    "default": false
+                }                
+            ]
+        },        
         {
             "canonicalName": "SPDX",
             "propertyKeyFormat": "SPDXID",
@@ -57,7 +83,8 @@
                     "file": "schema/cyclonedx/1.2/bom-1.2.schema.json",
                     "development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.2.schema.json",
                     "url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.2.schema.json",
-                    "default": false
+                    "default": false,
+                    "dependencies": ["spdx.schema.json"]
                 },
                 {
                     "version": "1.2",
@@ -66,7 +93,8 @@
                     "file": "schema/cyclonedx/1.2/bom-1.2-strict.schema.json",
                     "development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.2-strict.schema.json",
                     "url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.2-strict.schema.json",
-                    "default": false
+                    "default": false,
+                    "dependencies": ["spdx.schema.json"]
                 },
                 {
                     "version": "1.3",
@@ -75,7 +103,8 @@
                     "file": "schema/cyclonedx/1.3/bom-1.3.schema.json",
                     "development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.3.schema.json",
                     "url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.3.schema.json",
-                    "default": false
+                    "default": false,
+                    "dependencies": ["spdx.schema.json"]
                 },
                 {
                     "version": "1.3",
@@ -84,7 +113,8 @@
                     "file": "schema/cyclonedx/1.3/bom-1.3-strict.schema.json",
                     "development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.3-strict.schema.json",
                     "url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.3-strict.schema.json",
-                    "default": false
+                    "default": false,
+                    "dependencies": ["spdx.schema.json"]
                 },
                 {
                     "version": "1.4",
@@ -93,7 +123,8 @@
                     "file": "schema/cyclonedx/1.4/bom-1.4.schema.json",
                     "development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.4.schema.json",
                     "url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.4.schema.json",
-                    "default": false
+                    "default": false,
+                    "dependencies": ["jsf-0.82.schema.json", "spdx.schema.json"]                    
                 },
                 {
                     "version": "1.5",
@@ -102,7 +133,8 @@
                     "file": "schema/cyclonedx/1.5/bom-1.5.schema.json",
                     "development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.5.schema.json",
                     "url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.5.schema.json",
-                    "default": false
+                    "default": false,
+                    "dependencies": ["jsf-0.82.schema.json", "spdx.schema.json"]                    
                 },
                 {
                     "version": "1.6",
@@ -111,7 +143,8 @@
                     "file": "schema/cyclonedx/1.6/bom-1.6.schema.json",
                     "development": "https://github.com/CycloneDX/specification/blob/master/schema/bom-1.6.schema.json",
                     "url": "https://raw.githubusercontent.com/CycloneDX/specification/master/schema/bom-1.6.schema.json",
-                    "default": true
+                    "default": true,
+                    "dependencies": ["jsf-0.82.schema.json", "spdx.schema.json"]
                 },
                 {
                     "version": "1.3",
@@ -120,7 +153,8 @@
                     "file": "schema/test/bom-1.3-custom.schema.json",
                     "development":"https://github.com/CycloneDX/sbom-utility/blob/main/resources/schema/test/bom-1.3-custom.schema.json",
                     "url": "",
-                    "default": false
+                    "default": false,
+                    "dependencies": ["spdx.schema.json"]
                 },
                 {
                     "version": "1.4",
@@ -129,7 +163,8 @@
                     "file": "schema/test/bom-1.4-custom.schema.json",
                     "development":"https://github.com/CycloneDX/sbom-utility/blob/main/resources/schema/test/bom-1.4-custom.schema.json",
                     "url": "",
-                    "default": false
+                    "default": false,
+                    "dependencies": ["jsf-0.82.schema.json", "spdx.schema.json"]
                 }
             ]
         }

resources/resources.go

@@ -37,7 +37,7 @@ func LoadConfigFile(baseFilename string) (bData []byte, err error) {
 	return
 }
 
-func LoadSchemaFile(baseFilename string) (bData []byte, err error) {
-	bData, err = ConfigFiles.ReadFile(RESOURCES_SCHEMA_DIR + baseFilename)
-	return
-}
+// func LoadSchemaFile(baseFilename string) (bData []byte, err error) {
+// 	bData, err = BOMSchemaFiles.ReadFile(RESOURCES_SCHEMA_DIR + baseFilename)
+// 	return
+// }