Skip to content

Commit

Permalink
CLI commands allow files or URIs for usnistgov#297
Browse files Browse the repository at this point in the history 
This PR adjusts the commands and supporting databind classes and ports
them to use URIs. The commands will use the new UriUtils class and the
toUri() function to use a remote URI supported as a URL and load the
model, schema, or target file. If a local file, it will convert the file
path to valid URI.

Integration tests have been added to validate this functionality works
with the update CLI commands.
  • Loading branch information
@aj-stein-nist
aj-stein-nist committed Jan 26, 2024
1 parent d9004e6 commit 1e28414
Show file tree
Hide file tree
Showing 8 changed files with 163 additions and 118 deletions.
7 changes: 4 additions & 3 deletions databind/src/main/java/gov/nist/secauto/metaschema/databind/IBindingContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@

import java.io.IOException;
import java.math.BigInteger;
import java.net.URI;
import java.nio.file.Path;
import java.time.ZonedDateTime;
import java.util.List;
Expand Down Expand Up @@ -346,7 +347,7 @@ default IValidationResult validate(@NonNull INodeItem nodeItem) {
* if an error occurred when parsing the target as XML
*/
default IValidationResult validate(
@NonNull Path target,
@NonNull URI target,
@NonNull Format asFormat,
@NonNull IValidationSchemaProvider schemaProvider) throws IOException, SAXException {
IValidationResult retval;
Expand All @@ -362,7 +363,7 @@ default IValidationResult validate(
JSONObject json = YamlOperations.yamlToJson(YamlOperations.parseYaml(target));
assert json != null;
retval = new JsonSchemaContentValidator(schemaProvider.getJsonSchema())
.validate(json, ObjectUtils.notNull(target.toUri()));
.validate(json, ObjectUtils.notNull(target));
break;
default:
throw new UnsupportedOperationException("Unsupported format: " + asFormat.name());
Expand All @@ -385,7 +386,7 @@ default IValidationResult validate(
* @throws IOException
* if an error occurred while loading the document
*/
default IValidationResult validateWithConstraints(@NonNull Path target) throws IOException {
default IValidationResult validateWithConstraints(@NonNull URI target) throws IOException {
IBoundLoader loader = newBoundLoader();
loader.disableFeature(DeserializationFeature.DESERIALIZE_VALIDATE_CONSTRAINTS);

Expand Down
14 changes: 7 additions & 7 deletions databind/src/main/java/gov/nist/secauto/metaschema/databind/io/yaml/YamlOperations.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@

package gov.nist.secauto.metaschema.databind.io.yaml;

import gov.nist.secauto.metaschema.core.util.ObjectUtils;

import org.json.JSONException;
import org.json.JSONObject;
import org.yaml.snakeyaml.DumperOptions;
Expand All @@ -36,11 +38,9 @@
import org.yaml.snakeyaml.representer.Representer;
import org.yaml.snakeyaml.resolver.Resolver;

import java.io.BufferedReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.io.InputStream;
import java.net.URI;
import java.util.Map;

import edu.umd.cs.findbugs.annotations.NonNull;
Expand Down Expand Up @@ -84,9 +84,9 @@ private YamlOperations() {
*/
@SuppressWarnings({ "unchecked", "null" })
@NonNull
public static Map<String, Object> parseYaml(Path target) throws IOException {
try (BufferedReader reader = Files.newBufferedReader(target.toAbsolutePath(), StandardCharsets.UTF_8)) {
return (Map<String, Object>) YAML_PARSER.load(reader);
public static Map<String, Object> parseYaml(URI target) throws IOException {
try (InputStream is = ObjectUtils.notNull(target.toURL().openStream())) {
return (Map<String, Object>) YAML_PARSER.load(is);
}
}

Expand Down
3 changes: 1 addition & 2 deletions .../gov/nist/secauto/metaschema/databind/model/impl/AbstractBoundDefinitionModelComplex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,7 @@ public IBindingContext getBindingContext() {
public <CLASS> CLASS newInstance() {
Class<?> clazz = getBoundClass();
try {
@SuppressWarnings("unchecked")
Constructor<CLASS> constructor
@SuppressWarnings("unchecked") Constructor<CLASS> constructor
= (Constructor<CLASS>) clazz.getDeclaredConstructor();
return ObjectUtils.notNull(constructor.newInstance());
} catch (NoSuchMethodException ex) {
Expand Down
59 changes: 18 additions & 41 deletions ...rc/main/java/gov/nist/secauto/metaschema/cli/commands/AbstractValidateContentCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import gov.nist.secauto.metaschema.core.util.CollectionUtil;
import gov.nist.secauto.metaschema.core.util.CustomCollectors;
import gov.nist.secauto.metaschema.core.util.ObjectUtils;
import gov.nist.secauto.metaschema.core.util.UriUtils;
import gov.nist.secauto.metaschema.databind.IBindingContext;
import gov.nist.secauto.metaschema.databind.IBindingContext.IValidationSchemaProvider;
import gov.nist.secauto.metaschema.databind.io.Format;
Expand All @@ -58,8 +59,8 @@

import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collection;
Expand Down Expand Up @@ -91,7 +92,7 @@ public abstract class AbstractValidateContentCommand
private static final Option CONSTRAINTS_OPTION = ObjectUtils.notNull(
Option.builder("c")
.hasArg()
.argName("FILE")
.argName("URI")
.desc("additional constraint definitions")
.build());

Expand All @@ -116,38 +117,6 @@ public List<ExtraArgument> getExtraArguments() {
@SuppressWarnings("PMD.PreserveStackTrace") // intended
@Override
public void validateOptions(CallingContext callingContext, CommandLine cmdLine) throws InvalidArgumentException {
if (cmdLine.hasOption(CONSTRAINTS_OPTION)) {
String[] args = cmdLine.getOptionValues(CONSTRAINTS_OPTION);
for (String arg : args) {
Path constraint = Paths.get(arg);
if (!Files.exists(constraint)) {
throw new InvalidArgumentException(
"The provided external constraint file '" + constraint + "' does not exist.");
}
if (!Files.isRegularFile(constraint)) {
throw new InvalidArgumentException(
"The provided external constraint file '" + constraint + "' is not a file.");
}
if (!Files.isReadable(constraint)) {
throw new InvalidArgumentException(
"The provided external constraint file '" + constraint + "' is not readable.");
}
}
}

List<String> extraArgs = cmdLine.getArgList();
if (extraArgs.size() != 1) {
throw new InvalidArgumentException("The source to validate must be provided.");
}

Path source = Paths.get(extraArgs.get(0));
if (!Files.exists(source)) {
throw new InvalidArgumentException("The provided source file '" + source + "' does not exist.");
}
if (!Files.isReadable(source)) {
throw new InvalidArgumentException("The provided source file '" + source + "' is not readable.");
}

if (cmdLine.hasOption(AS_OPTION)) {
try {
String toFormatText = cmdLine.getOptionValue(AS_OPTION);
Expand Down Expand Up @@ -182,6 +151,7 @@ protected abstract IBindingContext getBindingContext(@NonNull Set<IConstraintSet
@SuppressWarnings("PMD.OnlyOneReturn") // readability
@Override
public ExitStatus execute() {
URI cwd = Paths.get("").toUri();
CommandLine cmdLine = getCommandLine();

Set<IConstraintSet> constraintSets;
Expand All @@ -190,11 +160,10 @@ public ExitStatus execute() {
constraintSets = new LinkedHashSet<>();
String[] args = cmdLine.getOptionValues(CONSTRAINTS_OPTION);
for (String arg : args) {
Path constraintPath = Paths.get(arg);
assert constraintPath != null;
try {
constraintSets.add(constraintLoader.load(constraintPath));
} catch (IOException | MetaschemaException ex) {
URI constraintUri = ObjectUtils.requireNonNull(UriUtils.toUri(arg, cwd));
constraintSets.add(constraintLoader.load(constraintUri));
} catch (IOException | MetaschemaException | URISyntaxException ex) {
return ExitCode.IO_ERROR.exitMessage("Unable to load constraint set '" + arg + "'.").withThrowable(ex);
}
}
Expand All @@ -213,8 +182,16 @@ public ExitStatus execute() {
IBoundLoader loader = bindingContext.newBoundLoader();

List<String> extraArgs = cmdLine.getArgList();
@SuppressWarnings("null") Path source = resolvePathAgainstCWD(Paths.get(extraArgs.get(0)));
assert source != null;
// @SuppressWarnings("null")
String sourceName = extraArgs.get(0);
URI source;

try {
source = UriUtils.toUri(sourceName, cwd);
} catch (URISyntaxException ex) {
return ExitCode.IO_ERROR.exitMessage("Cannot load source '%s' as it is not a valid file or URI.")
.withThrowable(ex);
}

Format asFormat;
if (cmdLine.hasOption(AS_OPTION)) {
Expand Down
22 changes: 13 additions & 9 deletions ...ema-cli/src/main/java/gov/nist/secauto/metaschema/cli/commands/GenerateSchemaCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import gov.nist.secauto.metaschema.core.model.xml.ModuleLoader;
import gov.nist.secauto.metaschema.core.util.CustomCollectors;
import gov.nist.secauto.metaschema.core.util.ObjectUtils;
import gov.nist.secauto.metaschema.core.util.UriUtils;
import gov.nist.secauto.metaschema.databind.io.Format;
import gov.nist.secauto.metaschema.schemagen.ISchemaGenerator;
import gov.nist.secauto.metaschema.schemagen.ISchemaGenerator.SchemaFormat;
Expand All @@ -54,6 +55,8 @@

import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
Expand Down Expand Up @@ -148,14 +151,6 @@ public void validateOptions(CallingContext callingContext, CommandLine cmdLine)
if (extraArgs.isEmpty() || extraArgs.size() > 2) {
throw new InvalidArgumentException("Illegal number of arguments.");
}

Path module = Paths.get(extraArgs.get(0));
if (!Files.exists(module)) {
throw new InvalidArgumentException("The provided metaschema module '" + module + "' does not exist.");
}
if (!Files.isReadable(module)) {
throw new InvalidArgumentException("The provided metaschema module '" + module + "' is not readable.");
}
}

@Override
Expand All @@ -171,6 +166,7 @@ protected ExitStatus executeCommand(
@NonNull CallingContext callingContext,
@NonNull CommandLine cmdLine) {
List<String> extraArgs = cmdLine.getArgList();
URI cwd = Paths.get("").toUri();

Path destination = null;
if (extraArgs.size() > 1) {
Expand Down Expand Up @@ -214,7 +210,15 @@ protected ExitStatus executeCommand(
}
}

Path input = Paths.get(extraArgs.get(0));
URI input;
String inputName = extraArgs.get(0);

try {
input = UriUtils.toUri(extraArgs.get(0), cwd);
} catch (URISyntaxException ex) {
return ExitCode.IO_ERROR.exitMessage(
String.format("Unable to load '%s' as it is not a valid file or URI.", inputName)).withThrowable(ex);
}
assert input != null;
try {
ModuleLoader loader = new ModuleLoader();
Expand Down
47 changes: 33 additions & 14 deletions ...main/java/gov/nist/secauto/metaschema/cli/commands/ValidateContentUsingModuleCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import gov.nist.secauto.metaschema.core.model.xml.ModuleLoader;
import gov.nist.secauto.metaschema.core.util.CollectionUtil;
import gov.nist.secauto.metaschema.core.util.ObjectUtils;
import gov.nist.secauto.metaschema.core.util.UriUtils;
import gov.nist.secauto.metaschema.databind.IBindingContext;
import gov.nist.secauto.metaschema.schemagen.ISchemaGenerator;
import gov.nist.secauto.metaschema.schemagen.ISchemaGenerator.SchemaFormat;
Expand All @@ -51,6 +52,8 @@

import java.io.BufferedReader;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
Expand Down Expand Up @@ -92,16 +95,19 @@ public Collection<? extends Option> gatherOptions() {

@Override
public void validateOptions(CallingContext callingContext, CommandLine cmdLine) throws InvalidArgumentException {
super.validateOptions(callingContext, cmdLine);

String metaschemaName = cmdLine.getOptionValue(MetaschemaCommandSupport.METASCHEMA_OPTION);
Path metaschema = Paths.get(metaschemaName);
if (!Files.exists(metaschema)) {
throw new InvalidArgumentException("The provided module '" + metaschema + "' does not exist.");
}
if (!Files.isReadable(metaschema)) {
throw new InvalidArgumentException("The provided module '" + metaschema + "' is not readable.");
}
// super.validateOptions(callingContext, cmdLine);
//
// String metaschemaName =
// cmdLine.getOptionValue(MetaschemaCommandSupport.METASCHEMA_OPTION);
// Path metaschema = Paths.get(metaschemaName);
// if (!Files.exists(metaschema)) {
// throw new InvalidArgumentException("The provided module '" + metaschema + "'
// does not exist.");
// }
// if (!Files.isReadable(metaschema)) {
// throw new InvalidArgumentException("The provided module '" + metaschema + "'
// is not readable.");
// }
}

@Override
Expand Down Expand Up @@ -132,17 +138,30 @@ private Path getTempDir() throws IOException {
@NonNull
private IModule getModule(@NonNull Set<IConstraintSet> constraintSets)
throws MetaschemaException, IOException {
URI cwd = Paths.get("").toUri();

if (module == null) {
String moduleName = getCommandLine().getOptionValue(MetaschemaCommandSupport.METASCHEMA_OPTION);
Path modulePath = Paths.get(moduleName);
assert modulePath != null;
String moduleName
= ObjectUtils.requireNonNull(getCommandLine().getOptionValue(MetaschemaCommandSupport.METASCHEMA_OPTION));
URI moduleUri;

try {
moduleUri = UriUtils.toUri(moduleName, cwd);
} catch (URISyntaxException ex) {
IOException newEx = new IOException( // NOPMD - intentional
String.format("Cannot load module as '%s' is not a valid file or URL.", moduleName));
newEx.addSuppressed(ex);
throw newEx;
}

assert moduleUri != null;

ExternalConstraintsModulePostProcessor postProcessor
= new ExternalConstraintsModulePostProcessor(constraintSets);

ModuleLoader loader = new ModuleLoader(CollectionUtil.singletonList(postProcessor));
loader.allowEntityResolution();
module = loader.load(modulePath);
module = loader.load(moduleUri);
}
assert module != null;
return module;
Expand Down
27 changes: 15 additions & 12 deletions ...ema-cli/src/main/java/gov/nist/secauto/metaschema/cli/commands/ValidateModuleCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,16 @@
import gov.nist.secauto.metaschema.core.model.xml.ModuleLoader;
import gov.nist.secauto.metaschema.core.util.CollectionUtil;
import gov.nist.secauto.metaschema.core.util.ObjectUtils;
import gov.nist.secauto.metaschema.core.util.UriUtils;

import org.apache.commons.cli.CommandLine;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.xml.sax.SAXException;

import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Paths;
import java.util.LinkedList;
import java.util.List;
Expand Down Expand Up @@ -103,14 +104,6 @@ public void validateOptions(CallingContext callingContext, CommandLine cmdLine)
if (extraArgs.size() != 1) {
throw new InvalidArgumentException("The source to validate must be provided.");
}

File target = new File(extraArgs.get(0));
if (!target.exists()) {
throw new InvalidArgumentException("The provided source file '" + target.getPath() + "' does not exist.");
}
if (!target.canRead()) {
throw new InvalidArgumentException("The provided source file '" + target.getPath() + "' is not readable.");
}
}

@Override
Expand All @@ -121,15 +114,25 @@ public ICommandExecutor newExecutor(CallingContext callingContext, CommandLine c
@SuppressWarnings({ "PMD.OnlyOneReturn", "unused" }) // readability
@NonNull
protected ExitStatus executeCommand(CallingContext callingContext, CommandLine cmdLine) {
URI cwd = Paths.get("").toUri();
List<String> extraArgs = cmdLine.getArgList();
Path target = Paths.get(extraArgs.get(0));
String targetName = extraArgs.get(0);
URI target;

try {
target = UriUtils.toUri(targetName, cwd);
} catch (URISyntaxException ex) {
return ExitCode.PROCESSING_ERROR
.exitMessage(String.format("The target '%s' cannot be loaded as it is not a valid file or URL.", targetName));
}

assert target != null;

IValidationResult schemaValidationResult;
try {
List<Source> schemaSources = getXmlSchemaSources();
schemaValidationResult = IContentValidator.validateWithXmlSchema(
ObjectUtils.notNull(target.toUri()),
ObjectUtils.notNull(target),
schemaSources);
} catch (IOException | SAXException ex) {
return ExitCode.PROCESSING_ERROR.exit().withThrowable(ex);
Expand Down
Loading

0 comments on commit 1e28414

Please sign in to comment.