From 3b00ce149d58093243d820ed869bba8798637fd0 Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Wed, 21 Aug 2024 10:16:49 -0400 Subject: [PATCH 1/2] Apply pre-commit effects and recommendations from Black, flake8, and isort Disclaimer: Participation by NIST in the creation of the documentation of mentioned software is not intended to imply a recommendation or endorsement by the National Institute of Standards and Technology, nor is it intended to imply that any specific software is necessarily the best available for the purpose. Signed-off-by: Alex Nelson --- generate.py | 304 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 187 insertions(+), 117 deletions(-) diff --git a/generate.py b/generate.py index 0c2e872..aee818e 100644 --- a/generate.py +++ b/generate.py @@ -1,38 +1,53 @@ -import os -import rdflib -from copy import deepcopy import argparse +import importlib.resources import json +import os from typing import Union -from case_utils.namespace import * + import case_utils.ontology +import rdflib +from case_utils.namespace import ( + NS_CASE_INVESTIGATION, + NS_CASE_VOCABULARY, + NS_UCO_ACTION, + NS_UCO_CORE, + NS_UCO_IDENTITY, + NS_UCO_LOCATION, + NS_UCO_MARKING, + NS_UCO_OBSERVABLE, + NS_UCO_PATTERN, + NS_UCO_ROLE, + NS_UCO_TOOL, + NS_UCO_TYPES, + NS_UCO_VICTIM, + NS_UCO_VOCABULARY, +) from case_utils.ontology.version_info import CURRENT_CASE_VERSION -import importlib.resources - -#NOTICE +# NOTICE # This software was produced for the U.S. Government under contract FA8702-22-C-0001, # and is subject to the Rights in Data-General Clause 52.227-14, Alt. IV (DEC 2007) # ©2022 The MITRE Corporation. All Rights Reserved. # Released under PRS 18-4297. -__version__ = '0.0.2' +__version__ = "0.0.2" NS_SH = rdflib.SH NS_RDF = rdflib.RDF NS_XSD = rdflib.XSD -caseutils_version = case_utils.__version__ -#uco_version = '0.9.0' - don't know how uco denotes their version +caseutils_version = case_utils.__version__ +# uco_version = '0.9.0' - don't know how uco denotes their version -ignore_keys = ['http://www.w3.org/2000/01/rdf-schema#range', #rdf:range - 'http://www.w3.org/2000/01/rdf-schema#label', #rdf:label - 'http://www.w3.org/2000/01/rdf-schema#comment', #rdf:comment - 'http://www.w3.org/ns/shacl#targetClass', #sh:targetClass - ] +ignore_keys = [ + "http://www.w3.org/2000/01/rdf-schema#range", # rdf:range + "http://www.w3.org/2000/01/rdf-schema#label", # rdf:label + "http://www.w3.org/2000/01/rdf-schema#comment", # rdf:comment + "http://www.w3.org/ns/shacl#targetClass", # sh:targetClass +] -#direct uco vocabulary +# direct uco vocabulary # obs_prefix = {str(NS_UCO_OBSERVABLE):'observable:', # str(NS_UCO_CORE):'core:', # str(NS_UCO_TOOL):'tool:', @@ -51,53 +66,62 @@ # str(NS_UCO_TYPES):"types:", # } -#case-uco vocabulary -obs_prefix = { #uco vocabulary - str(NS_UCO_MARKING):'uco-marking:', - str(NS_UCO_TOOL):'uco-tool:', - str(NS_SH):'sh:', - str(NS_RDF):'rdfs:', - str(NS_XSD):'xsd:', - str(NS_RDF):'rdf:', - str(NS_UCO_OBSERVABLE):'uco-observable:', - str(NS_UCO_MARKING):'uco-marking:', - str(NS_UCO_IDENTITY):'uco-identity:', - str(NS_UCO_VICTIM):"uco-victim:", - str(NS_UCO_VOCABULARY):'uco-vocabulary:', - str(NS_UCO_PATTERN):'uco-pattern:', - str(NS_UCO_CORE):'uco-core:', - str(NS_UCO_TOOL):'uco-tool:', - str(NS_UCO_ACTION):'uco-action:', - str(NS_UCO_LOCATION):'uco-location:', - 'http://www.w3.org/2000/01/rdf-schema#':'rdfs:', - str(NS_UCO_ROLE):"uco-role:", - str(NS_UCO_TYPES):"uco-types:", - #case vocab - str(NS_CASE_INVESTIGATION): "investigation:", - str(NS_CASE_VOCABULARY):'vocabulary:'} +# case-uco vocabulary +obs_prefix = { # uco vocabulary + str(NS_UCO_MARKING): "uco-marking:", + str(NS_UCO_TOOL): "uco-tool:", + str(NS_SH): "sh:", + str(NS_RDF): "rdfs:", + str(NS_XSD): "xsd:", + str(NS_RDF): "rdf:", + str(NS_UCO_OBSERVABLE): "uco-observable:", + str(NS_UCO_MARKING): "uco-marking:", + str(NS_UCO_IDENTITY): "uco-identity:", + str(NS_UCO_VICTIM): "uco-victim:", + str(NS_UCO_VOCABULARY): "uco-vocabulary:", + str(NS_UCO_PATTERN): "uco-pattern:", + str(NS_UCO_CORE): "uco-core:", + str(NS_UCO_TOOL): "uco-tool:", + str(NS_UCO_ACTION): "uco-action:", + str(NS_UCO_LOCATION): "uco-location:", + "http://www.w3.org/2000/01/rdf-schema#": "rdfs:", + str(NS_UCO_ROLE): "uco-role:", + str(NS_UCO_TYPES): "uco-types:", + # case vocab + str(NS_CASE_INVESTIGATION): "investigation:", + str(NS_CASE_VOCABULARY): "vocabulary:", +} reverse_obs_prefix = {} -for k,v in obs_prefix.items(): +for k, v in obs_prefix.items(): reverse_obs_prefix[v] = k -def reducestring(string:str, returnChange=False): - for k,v in obs_prefix.items(): + +def reducestring(string: str, returnChange=False): + for k, v in obs_prefix.items(): if k in string: if returnChange: - return string.replace(k,v),True + return string.replace(k, v), True else: - return string.replace(k,v) + return string.replace(k, v) if returnChange: - return str(string),False + return str(string), False else: return str(string) def makedirs(directory): - os.makedirs(f'{directory}',exist_ok = True) + os.makedirs(f"{directory}", exist_ok=True) + class main: - def __init__(self,ontology_dir:str = None,directory:str = "templates", useCaseUtils:bool = False, short:bool = False ): + def __init__( + self, + ontology_dir: str = None, + directory: str = "templates", + useCaseUtils: bool = False, + short: bool = False, + ): makedirs(directory) self.switch = useCaseUtils @@ -115,26 +139,27 @@ def __init__(self,ontology_dir:str = None,directory:str = "templates", useCaseUt for onto in ontology_dir: if os.path.isdir(onto): for onto in self.onto_dir: - for root, dirs, files in os.walk(onto, topdown = False): - for name in files: - if name.endswith(".ttl"): - adir = os.path.join(root, name) - if adir not in self.files_dir: - self.files_dir.append(adir) + for root, dirs, files in os.walk(onto, topdown=False): + for name in files: + if name.endswith(".ttl"): + adir = os.path.join(root, name) + if adir not in self.files_dir: + self.files_dir.append(adir) elif os.path.isfile(onto): - if adir not in self.files_dir: - self.files_dir.append(adir) + if adir not in self.files_dir: + self.files_dir.append(adir) self.directory = directory - def paduco(self,string): - v = string.split(":")[0]+":" + def paduco(self, string): + v = string.split(":")[0] + ":" if v in reverse_obs_prefix: - return string #don't pad eg. investigation + return string # don't pad eg. investigation else: - return self.prepad+"-"+string #if not defined, pad it. - def removepad(self,string): + return self.prepad + "-" + string # if not defined, pad it. + + def removepad(self, string): if string.startswith(self.prepad): - return string.replace(self.prepad + "-","") + return string.replace(self.prepad + "-", "") else: return string @@ -144,48 +169,71 @@ def load_graph(self): for file in self.files_dir: self.g.parse(file) if self.switch: - case_utils.ontology.load_subclass_hierarchy(g) + case_utils.ontology.load_subclass_hierarchy(self.g) else: - ttl_filename = "case-"+CURRENT_CASE_VERSION+".ttl" + ttl_filename = "case-" + CURRENT_CASE_VERSION + ".ttl" ttl_data = importlib.resources.read_text(case_utils.ontology, ttl_filename) self.g.parse(data=ttl_data) case_utils.ontology.load_subclass_hierarchy(self.g) def load_case_version(self): try: - self.case_version = str([list(i) for i in self.g.query('SELECT ?s ?p ?o WHERE{?s owl:versionInfo ?o}')][0][2]) - except: + self.case_version = str( + [ + list(i) + for i in self.g.query( + "SELECT ?s ?p ?o WHERE{?s owl:versionInfo ?o}" + ) + ][0][2] + ) + except Exception: self.case_version = CURRENT_CASE_VERSION - - def getSubClassOf(self,name:str): - res = [i for i in self.g.query("SELECT ?o WHERE {{ {} rdfs:subClassOf ?o }}".format(self.removepad(name)))] + def getSubClassOf(self, name: str): + res = [ + i + for i in self.g.query( + "SELECT ?o WHERE {{ {} rdfs:subClassOf ?o }}".format( + self.removepad(name) + ) + ) + ] if res: return res[0] - return res #comes back as list of triples - - def getProperty(self,name:str): - res = [i for i in self.g.query("SELECT ?o WHERE {{ {} sh:property ?o }}".format(self.removepad(name)))] - return res #comes back as list of triples - - def getPath(self,name:str): - res = [i for i in self.g.query("SELECT ?o WHERE {{ {} sh:path ?o }}".format(self.removepad(name)))] - return res #comes back as list of triples - - def getUCOname(self,name): + return res # comes back as list of triples + + def getProperty(self, name: str): + res = [ + i + for i in self.g.query( + "SELECT ?o WHERE {{ {} sh:property ?o }}".format(self.removepad(name)) + ) + ] + return res # comes back as list of triples + + def getPath(self, name: str): + res = [ + i + for i in self.g.query( + "SELECT ?o WHERE {{ {} sh:path ?o }}".format(self.removepad(name)) + ) + ] + return res # comes back as list of triples + + def getUCOname(self, name): return self.removepad(reducestring(name)) - def getParents(self,name:str,hist: list = [] ): + def getParents(self, name: str, hist: list = []): parents = self.getSubClassOf(name) if not parents: return [] else: - for parent in parents: #DFS + for parent in parents: # DFS p = self.getUCOname(parent) if p not in hist: hist.append(p) - #check if each parent has parents that isn't part of hist - for i in self.getParents(p,hist): + # check if each parent has parents that isn't part of hist + for i in self.getParents(p, hist): if i not in hist: hist.append(i) return hist @@ -209,49 +257,48 @@ def generate_classes(self): s, p = triple self.class_names.append(self.removepad(reducestring(s))) - def findContext(self,dict_graph:dict): + def findContext(self, dict_graph: dict): c = {} for t in dict_graph.keys(): - for k,v in obs_prefix.items(): + for k, v in obs_prefix.items(): if self.paduco(t).startswith(v): c[self.paduco(v.strip(":"))] = k.strip("/") + "#" return c - def load_single(self,name:str): - single = {'@context':{}, - '@graph':[{}] - } - single['@context']['kb'] = 'http://example.org/kb/' + def load_single(self, name: str): + single = {"@context": {}, "@graph": [{}]} + single["@context"]["kb"] = "http://example.org/kb/" n = "".join(name.split(":")[1:]) - single['@graph'][0]['@id'] = "kb:"+n.lower()+"1" - single['@graph'][0]['@type'] = self.paduco(name) + single["@graph"][0]["@id"] = "kb:" + n.lower() + "1" + single["@graph"][0]["@type"] = self.paduco(name) - #add the properties of the object + # add the properties of the object props = self.getProperty(name) for prop in props: for p in self.bnode_dict[str(prop[0])]: if p: - single['@graph'][0][self.paduco(p)] = None - - single['@context'].update(self.findContext({name:None})) - single['@context'].update(self.findContext(single['@graph'][0])) - single['@version']={'case_util':caseutils_version,'ontology_version':self.case_version} + single["@graph"][0][self.paduco(p)] = None + single["@context"].update(self.findContext({name: None})) + single["@context"].update(self.findContext(single["@graph"][0])) + single["@version"] = { + "case_util": caseutils_version, + "ontology_version": self.case_version, + } if self.generate_short: pass else: - #add the parent's properties - parents = self.getParents(name,[]) + # add the parent's properties + parents = self.getParents(name, []) for parent in parents: props = self.getProperty(parent) for prop in props: for node in self.bnode_dict[str(prop[0])]: - single['@graph'][0][self.paduco(node)] = None - single['@context'].update(self.findContext(single['@graph'][0])) + single["@graph"][0][self.paduco(node)] = None + single["@context"].update(self.findContext(single["@graph"][0])) return single - def generate(self): self.load_graph() self.load_case_version() @@ -260,8 +307,7 @@ def generate(self): self.generate_classes() return - - def convertToJson(self,obj_name:str): + def convertToJson(self, obj_name: str): vocab, newname = obj_name.split(":") obj = self.load_single(obj_name) @@ -271,12 +317,12 @@ def convertToJson(self,obj_name:str): nextdir = f"{self.directory}/{self.paduco(vocab)}" makedirs(nextdir) - with open(f"{nextdir}/{newname}.json", 'w') as fl: - json.dump(obj, fl,indent=2) + with open(f"{nextdir}/{newname}.json", "w") as fl: + json.dump(obj, fl, indent=2) fl.close() print(f"Success:{self.paduco(obj_name)}") - def run(self,name:Union[list,str] = None): + def run(self, name: Union[list, str] = None): if not name: name = self.class_names else: @@ -285,18 +331,42 @@ def run(self,name:Union[list,str] = None): self.convertToJson(k) return + if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-o',"--ontology", help="ontology ttl file.",type=str,required=False) - parser.add_argument("--output", help="default output folder for studs",type=str,required=False,default = "templates") - - parser.add_argument("-s", "--specific", help="specific single object name",type=str,required=False) - parser.add_argument("-a", "--caseutil", help="[T/F] allow case_utils to load uco ontology.",type=bool,required=False,default=False) - parser.add_argument("-t", "--short", help="[T/F] generate short stub (no superclass properties) or full stub.",type=bool,required=False,default=False) + parser.add_argument( + "-o", "--ontology", help="ontology ttl file.", type=str, required=False + ) + parser.add_argument( + "--output", + help="default output folder for studs", + type=str, + required=False, + default="templates", + ) + + parser.add_argument( + "-s", "--specific", help="specific single object name", type=str, required=False + ) + parser.add_argument( + "-a", + "--caseutil", + help="[T/F] allow case_utils to load uco ontology.", + type=bool, + required=False, + default=False, + ) + parser.add_argument( + "-t", + "--short", + help="[T/F] generate short stub (no superclass properties) or full stub.", + type=bool, + required=False, + default=False, + ) args = parser.parse_args() - obj = main(args.ontology,args.output,args.caseutil,args.short) - + obj = main(args.ontology, args.output, args.caseutil, args.short) makedirs(args.output) obj.generate() From 9994788426b2adfcff36b4e7d79d3ee4920342e4 Mon Sep 17 00:00:00 2001 From: Alex Nelson Date: Wed, 21 Aug 2024 10:20:34 -0400 Subject: [PATCH 2/2] Add pre-commit and mechanisms for pre-commit review and refreshing Disclaimer: Participation by NIST in the creation of the documentation of mentioned software is not intended to imply a recommendation or endorsement by the National Institute of Standards and Technology, nor is it intended to imply that any specific software is necessarily the best available for the purpose. Signed-off-by: Alex Nelson --- .flake8 | 3 + .github/workflows/ci.yml | 44 +++++++++++++++ .github/workflows/supply-chain.yml | 43 +++++++++++++++ .gitignore | 1 + .pre-commit-config.yaml | 15 +++++ Makefile | 88 ++++++++++++++++++++++++++++++ 6 files changed, 194 insertions(+) create mode 100644 .flake8 create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/supply-chain.yml create mode 100644 .pre-commit-config.yaml create mode 100644 Makefile diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..cb50e3a --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 88 +extend-ignore = E501 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a336193 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,44 @@ +# Portions of this file contributed by NIST are governed by the +# following statement: +# +# This software was developed at the National Institute of Standards +# and Technology by employees of the Federal Government in the course +# of their official duties. Pursuant to title 17 Section 105 of the +# United States Code this software is not subject to copyright +# protection and is in the public domain. NIST assumes no +# responsibility whatsoever for its use by other parties, and makes +# no guarantees, expressed or implied, about its quality, +# reliability, or any other characteristic. +# +# We would appreciate acknowledgement if the software is used. + +name: Continuous Integration + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + schedule: + - cron: '15 5 * * TUE' + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - '3.9' + - '3.12' + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Pre-commit Checks + run: | + pip -q install pre-commit + pre-commit run --all-files diff --git a/.github/workflows/supply-chain.yml b/.github/workflows/supply-chain.yml new file mode 100644 index 0000000..d301c93 --- /dev/null +++ b/.github/workflows/supply-chain.yml @@ -0,0 +1,43 @@ +# Portions of this file contributed by NIST are governed by the +# following statement: +# +# This software was developed at the National Institute of Standards +# and Technology by employees of the Federal Government in the course +# of their official duties. Pursuant to title 17 Section 105 of the +# United States Code this software is not subject to copyright +# protection and is in the public domain. NIST assumes no +# responsibility whatsoever for its use by other parties, and makes +# no guarantees, expressed or implied, about its quality, +# reliability, or any other characteristic. +# +# We would appreciate acknowledgement if the software is used. + +# This workflow uses Make to review direct dependencies of this +# repository. + +name: Supply Chain + +on: + schedule: + - cron: '15 5 * * 1,2,3,4,5' + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - '3.9' + - '3.12' + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Review dependencies + run: make check-supply-chain diff --git a/.gitignore b/.gitignore index d3e0f0b..2600c9f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .idea __pycache__ .ipynb_checkpoints +.venv-pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..439fa3f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,15 @@ +repos: + - repo: https://github.com/psf/black + rev: 23.12.1 + hooks: + - id: black + - repo: https://github.com/pycqa/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + name: isort (python) + args: ["--profile", "black", "--filter-files"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9aa19f8 --- /dev/null +++ b/Makefile @@ -0,0 +1,88 @@ +#!/usr/bin/make -f + +# Portions of this file contributed by NIST are governed by the +# following statement: +# +# This software was developed at the National Institute of Standards +# and Technology by employees of the Federal Government in the course +# of their official duties. Pursuant to title 17 Section 105 of the +# United States Code this software is not subject to copyright +# protection and is in the public domain. NIST assumes no +# responsibility whatsoever for its use by other parties, and makes +# no guarantees, expressed or implied, about its quality, +# reliability, or any other characteristic. +# +# We would appreciate acknowledgement if the software is used. + +SHELL := /bin/bash + +PYTHON3 ?= python3 + +all: \ + .venv-pre-commit/var/.pre-commit-built.log + +.PHONY: \ + check-supply-chain \ + check-supply-chain-pre-commit + +# This virtual environment is meant to be built once and then persist, even through 'make clean'. +# If a recipe is written to remove this flag file, it should first run `pre-commit uninstall`. +.venv-pre-commit/var/.pre-commit-built.log: + rm -rf .venv-pre-commit + test -r .pre-commit-config.yaml \ + || (echo "ERROR:Makefile:pre-commit is expected to install for this repository, but .pre-commit-config.yaml does not seem to exist." >&2 ; exit 1) + $(PYTHON3) -m venv \ + .venv-pre-commit + source .venv-pre-commit/bin/activate \ + && pip install \ + --upgrade \ + pip \ + setuptools \ + wheel + source .venv-pre-commit/bin/activate \ + && pip install \ + pre-commit + source .venv-pre-commit/bin/activate \ + && pre-commit install + mkdir -p \ + .venv-pre-commit/var + touch $@ + +check: \ + .venv-pre-commit/var/.pre-commit-built.log + +check-supply-chain: \ + check-supply-chain-pre-commit + +# Update pre-commit configuration and use the updated config file to +# review code. Only have Make exit if 'pre-commit run' modifies files. +check-supply-chain-pre-commit: \ + .venv-pre-commit/var/.pre-commit-built.log + source .venv-pre-commit/bin/activate \ + && pre-commit autoupdate + git diff \ + --exit-code \ + .pre-commit-config.yaml \ + || ( \ + source .venv-pre-commit/bin/activate \ + && pre-commit run \ + --all-files \ + --config .pre-commit-config.yaml \ + ) \ + || git diff \ + --stat \ + --exit-code \ + || ( \ + echo \ + "WARNING:Makefile:pre-commit configuration can be updated. It appears the updated would change file formatting." \ + >&2 \ + ; exit 1 \ + ) + @git diff \ + --exit-code \ + .pre-commit-config.yaml \ + || echo \ + "INFO:Makefile:pre-commit configuration can be updated. It appears the update would not change file formatting." \ + >&2 + +clean: