From d718029dde09cc20fab597dac15364dd372c77df Mon Sep 17 00:00:00 2001 From: mereolog Date: Thu, 8 Feb 2024 12:16:32 +0100 Subject: [PATCH] revert to 21e562fe7e0627508fc9727d0e6c0efbb9601dfb Signed-off-by: mereolog --- publisher/product/datadictionary/build.sh | 23 ++-- publisher/product/ontology/build-about.sh | 19 ++- publisher/product/ontology/build.sh | 136 ++++++++++++++++++---- 3 files changed, 132 insertions(+), 46 deletions(-) diff --git a/publisher/product/datadictionary/build.sh b/publisher/product/datadictionary/build.sh index f3098001..11c27363 100644 --- a/publisher/product/datadictionary/build.sh +++ b/publisher/product/datadictionary/build.sh @@ -58,38 +58,35 @@ function publishProductDataDictionaryContent() { local extractDataColumn="${DATADICTIONARY_COLUMNS:+ --extract-data-column $(echo "${DATADICTIONARY_COLUMNS}" | sed 's/|/ --extract-data-column /g')}" - local dev_suffix=${DEV_SPEC/.rdf/} - local prod_suffix=${PROD_SPEC/.rdf/} - logRule "Running OntoViewer Toolkit to generate CSV files containing data from ontologies - see logs \"$(logFileName datadictionary.DEV.log)\"" - logItem "$(basename "${DEV_SPEC}")" "$(logFileName "${tag_root_url}/$(basename "${datadictionaryBaseName}-${dev_suffix}.csv")")" + logItem "$(basename "${DEV_SPEC}")" "$(logFileName "${tag_root_url}/$(basename "${datadictionaryBaseName}-dev.csv")")" debug=false ${ONTOVIEWER_TOOLKIT_JAVA} --goal extract-data ${maturityLevel} ${maturityLevelProperty} ${extractDataColumn} --data "${source_family_root}/${DEV_SPEC}" \ - --output "${datadictionaryBaseName}-${dev_suffix}.csv" \ + --output "${datadictionaryBaseName}-dev.csv" \ --filter-pattern "${HYGIENE_TEST_PARAMETER_VALUE}" \ --ontology-mapping "${source_family_root}/catalog-v001.xml" > "${datadictionary_product_tag_root}/datadictionary.DEV.log" 2>&1 logRule "Running OntoViewer Toolkit to generate CSV files containing data from ontologies - see logs \"$(logFileName datadictionary.PROD.log)\"" - logItem "$(basename "${PROD_SPEC}")" "$(logFileName "${tag_root_url}/$(basename "${datadictionaryBaseName}-${prod_suffix}.csv")")" + logItem "$(basename "${PROD_SPEC}")" "$(logFileName "${tag_root_url}/$(basename "${datadictionaryBaseName}-prod.csv")")" debug=false ${ONTOVIEWER_TOOLKIT_JAVA} --goal extract-data ${maturityLevel} ${maturityLevelProperty} ${extractDataColumn} --data "${source_family_root}/${PROD_SPEC}" \ - --output "${datadictionaryBaseName}-${prod_suffix}.csv" \ + --output "${datadictionaryBaseName}-prod.csv" \ --filter-pattern "${HYGIENE_TEST_PARAMETER_VALUE}" \ --ontology-mapping "${source_family_root}/catalog-v001.xml" > "${datadictionary_product_tag_root}/datadictionary.PROD.log" 2>&1 logRule "Writing from csv files to xlsx files for prod" - echo -e "==== $(basename "${datadictionaryBaseName}-${prod_suffix}.xlsx")" > "${datadictionary_product_tag_root}/datadictionary.log" + echo -e "==== $(basename "${datadictionaryBaseName}-prod.xlsx")" > "${datadictionary_product_tag_root}/datadictionary.log" ${PYTHON3} ${SCRIPT_DIR}/lib/csv-to-xlsx.py \ - "${datadictionaryBaseName}-${prod_suffix}.csv" \ - "${datadictionaryBaseName}-${prod_suffix}.xlsx" \ + "${datadictionaryBaseName}-prod.csv" \ + "${datadictionaryBaseName}-prod.xlsx" \ "${datadictionary_script_dir}/csvconfig" 2>&1 | tee -a "${datadictionary_product_tag_root}/datadictionary.log" logRule "Writing from csv files to xlsx files for dev" - echo -e "\n==== $(basename "${datadictionaryBaseName}-${dev_suffix}.xlsx")" >> "${datadictionary_product_tag_root}/datadictionary.log" + echo -e "\n==== $(basename "${datadictionaryBaseName}-dev.xlsx")" >> "${datadictionary_product_tag_root}/datadictionary.log" ${PYTHON3} ${SCRIPT_DIR}/lib/csv-to-xlsx.py \ - "${datadictionaryBaseName}-${dev_suffix}.csv" \ - "${datadictionaryBaseName}-${dev_suffix}.xlsx" \ + "${datadictionaryBaseName}-dev.csv" \ + "${datadictionaryBaseName}-dev.xlsx" \ "${datadictionary_script_dir}/csvconfig" 2>&1 | tee -a "${datadictionary_product_tag_root}/datadictionary.log" return 0 diff --git a/publisher/product/ontology/build-about.sh b/publisher/product/ontology/build-about.sh index 601d275e..0587b9dc 100644 --- a/publisher/product/ontology/build-about.sh +++ b/publisher/product/ontology/build-about.sh @@ -22,21 +22,18 @@ function ontologyCreateAboutFiles () { rm -f "${TMPDIR}/err.tmp" - local dev_suffix=${DEV_SPEC/.rdf/} - local prod_suffix=${PROD_SPEC/.rdf/} - # use "owl:imports" from ${PROD_SPEC} if exists test -n "${PROD_SPEC}" && test -r "${tag_root:?}/${PROD_SPEC}" && ( cd "${tag_root:?}" || return $? - logItem " from \"${PROD_SPEC}\"$(echo -en '\t')" "$(logFileName "${tag_root}/Load${prod_suffix}.rdf")" + logItem " from \"${PROD_SPEC}\"$(echo -en '\t')" "$(logFileName "${tag_root}/Load${ONTPUB_FAMILY^^}Prod.rdf")" cat > "${tmpAboutFileProd}" << __HERE__ @prefix rdf: @prefix rdfs: @prefix owl: @prefix xsd: -<${product_root_url}/Load${prod_suffix}/> a owl:Ontology; +<${product_root_url}/Load${ONTPUB_FAMILY^^}Prod/> a owl:Ontology; __HERE__ getOwlImports < "${tag_root}/${PROD_SPEC}" 2>/dev/null | \ @@ -46,10 +43,10 @@ __HERE__ "${JENA_ARQ}" \ --data="${tmpAboutFileProd}" \ --query="${SCRIPT_DIR}/lib/echo.sparql" \ - --results=RDF > "${tag_root}/Load${prod_suffix}.rdf" 2> "${TMPDIR}/err.tmp" + --results=RDF > "${tag_root}/Load${ONTPUB_FAMILY^^}Prod.rdf" 2> "${TMPDIR}/err.tmp" if [ -s "${TMPDIR}/err.tmp" ] ; then - warning "no RDF XML output generated. Use Load${prod_suffix}.ttl file instead" + warning "no RDF XML output generated. Use Load${ONTPUB_FAMILY^^}Prod.ttl file instead" fi rm -f "${TMPDIR}/err.tmp" ) @@ -58,14 +55,14 @@ __HERE__ test -n "${DEV_SPEC}" && test -r "${tag_root:?}/${PROD_SPEC}" && ( cd "${tag_root}" || return $? - logItem " from \"${DEV_SPEC}\"$(echo -en '\t')" "$(logFileName "${tag_root}/Load${dev_suffix}.rdf")" + logItem " from \"${DEV_SPEC}\"$(echo -en '\t')" "$(logFileName "${tag_root}/Load${ONTPUB_FAMILY^^}Dev.rdf")" cat > "${tmpAboutFileDev}" << __HERE__ @prefix rdf: @prefix rdfs: @prefix owl: @prefix xsd: -<${product_root_url}/Load${dev_suffix}/> a owl:Ontology; +<${product_root_url}/Load${ONTPUB_FAMILY^^}Dev/> a owl:Ontology; __HERE__ getOwlImports < "${tag_root}/${DEV_SPEC}" 2>/dev/null | \ @@ -75,10 +72,10 @@ __HERE__ "${JENA_ARQ}" \ --data="${tmpAboutFileDev}" \ --query="${SCRIPT_DIR}/lib/echo.sparql" \ - --results=RDF > "${tag_root}/Load${dev_suffix}.rdf" 2> "${TMPDIR}/err.tmp" + --results=RDF > "${tag_root}/Load${ONTPUB_FAMILY^^}Dev.rdf" 2> "${TMPDIR}/err.tmp" if [ -s "${TMPDIR}/err.tmp" ] ; then - warning "no RDF XML output generated. Use Load${dev_suffix}.ttl file instead" + warning "no RDF XML output generated. Use Load${ONTPUB_FAMILY^^}Dev.ttl file instead" fi rm -f "${TMPDIR}/err.tmp" ) diff --git a/publisher/product/ontology/build.sh b/publisher/product/ontology/build.sh index a37cc540..74827778 100644 --- a/publisher/product/ontology/build.sh +++ b/publisher/product/ontology/build.sh @@ -48,6 +48,13 @@ function publishProductOntology() { test -z "${ONTPUB_SHACL_INFIX}" || ontologyCreateSHACLFiles || return $? ontologyZipFiles > "${tag_root}/ontology-zips.log" || return $? + + if ((speedy)) ; then + log "speedy=true -> Not doing quads because they are slow" + else + buildquads || return $? + fi + return 0 } @@ -107,7 +114,7 @@ function runHygieneTests() { logItem "${SPEC}" "$(getOntologyIRI < "${source_family_root}/${SPEC}")" if ${ONTOVIEWER_TOOLKIT_JAVA} --data "${source_family_root}/${SPEC}" \ --output ${TMPDIR}/output.json $(test -s "${source_family_root}/catalog-v001.xml" && echo "--ontology-mapping ${source_family_root}/catalog-v001.xml") \ - --goal consistency-check &> "${hygiene_product_tag_root}/consistency-check.log" && jq -e "." &>/dev/null < "${TMPDIR}/output.json" ; then + --goal consistency-check &> "${hygiene_product_tag_root}/consistency-check.log" && jq -e "" &>/dev/null < "${TMPDIR}/output.json" ; then displayMissingImports "${TMPDIR}/output.json" if [ "$(jq -r ".consistent" < "${TMPDIR}/output.json")" = "true" ] ; then echo -e "\t\x1b\x5b\x33\x32\x6d$(echo "Ontology \"${SPEC}\" is consistent." | tee -a "${hygiene_product_tag_root}/consistency-check.log")\x1b\x5b\x30\x6d" @@ -129,7 +136,7 @@ function runHygieneTests() { logItem "${SPEC}" "$(getOntologyIRI < "${source_family_root}/${SPEC}")" if ${ONTOVIEWER_TOOLKIT_JAVA} --data "${source_family_root}/${SPEC}" \ --output ${TMPDIR}/output.json $(test -s "${source_family_root}/catalog-v001.xml" && echo "--ontology-mapping ${source_family_root}/catalog-v001.xml") \ - --goal consistency-check &>> "${hygiene_product_tag_root}/consistency-check.log" && jq -e "." &>/dev/null < "${TMPDIR}/output.json" ; then + --goal consistency-check &>> "${hygiene_product_tag_root}/consistency-check.log" && jq -e "" &>/dev/null < "${TMPDIR}/output.json" ; then displayMissingImports "${TMPDIR}/output.json" if [ "$(jq -r ".consistent" < "${TMPDIR}/output.json")" = "true" ] ; then echo -e "\t\x1b\x5b\x33\x32\x6d$(echo "Ontology \"${SPEC}\" is consistent." | tee -a "${hygiene_product_tag_root}/consistency-check.log")\x1b\x5b\x30\x6d" @@ -698,6 +705,97 @@ function ontologyZipFiles () { return 0 } +function buildquads () { + + local ProdQuadsFile="${tag_root}/prod.${ONTPUB_FAMILY}.nq" + local DevQuadsFile="${tag_root}/dev.${ONTPUB_FAMILY}.nq" + + local ProdFlatNT="${tag_root}/old_prod.${ONTPUB_FAMILY}-quickstart.nt" + local DevFlatNT="${tag_root}/old_dev.${ONTPUB_FAMILY}-quickstart.nt" + + local ProdFlatTTL="${tag_root}/old_prod.${ONTPUB_FAMILY}-quickstart.ttl" + local DevFlatTTL="${tag_root}/old_dev.${ONTPUB_FAMILY}-quickstart.ttl" + + local ProdTMPTTL="$(mktemp ${TMPDIR}/prod.temp.XXXXXX.ttl)" + local DevTMPTTL="$(mktemp ${TMPDIR}/dev.temp.XXXXXX.ttl)" + + local CSVPrefixes="${tag_root}/prefixes.${ONTPUB_FAMILY}.csv" + local TTLPrefixes="${tag_root}/prefixes.${ONTPUB_FAMILY}.ttl" + local SPARQLPrefixes="${tag_root}/prefixes.${ONTPUB_FAMILY}.sq" + + + local tmpflat="$(mktemp ${TMPDIR}/flatten.XXXXXX.sq)" + cat >"${tmpflat}" << __HERE__ +PREFIX owl: + +CONSTRUCT {?s ?p ?o} +WHERE {GRAPH ?g {?s ?p ?o +FILTER NOT EXISTS {?s a owl:Ontology} +} +} +__HERE__ + + local tmpflatecho="$(mktemp ${TMPDIR}/flattecho.XXXXXX.sq)" + cat >"${tmpflatecho}" << __HERE__ +PREFIX owl: + +CONSTRUCT {?s ?p ?o} +WHERE {?s ?p ?o +FILTER NOT EXISTS {?s a owl:Ontology} +} +__HERE__ + + + local tmppx="$(mktemp ${TMPDIR}/px.XXXXXX.sq)" + cat >"${tmppx}" << __HERE__ +prefix sm: +prefix owl: +prefix xsd: + +SELECT ?line +WHERE {graph ?g {?o a owl:Ontology ; +sm:fileAbbreviation ?px +BIND (CONCAT ("prefix ", ?px, ": <", xsd:string(?o), ">") AS ?line) +} +} +__HERE__ + + local tmpecho="$(mktemp ${TMPDIR}/echo.XXXXXX.sq)" + cat >"${tmpecho}" << __HERE__ +CONSTRUCT {?s ?p ?o} +WHERE {?s ?p ?o} +__HERE__ + + + local prefixes="$(mktemp ${TMPDIR}/prefixes.XXXXXX)" + + local tmpmodule="$(mktemp ${TMPDIR}/module.XXXXXX.nt)" + + + log "starting buildquads with the new quadify" + + + + ( + cd ${ontology_product_tag_root} + echo "starting dev" + + ${FIND} . -mindepth 2 -name '*.ttl' -print | while read file; do quadify "$file"; done > "${DevQuadsFile}" + echo "starting prod" + ${GREP} -rl 'fibo-fnd-utl-av:hasMaturityLevel fibo-fnd-utl-av:Release' | \ + while read file ; do quadify $file ; done > ${ProdQuadsFile} + set -x + ${FIND} ${INPUT} -name "Metadata*.rdf" -exec \ + ${JENA_RIOT} \ + --syntax=RDF/XML {} \; \ + > ${tmpmodule} + + ) + + log "finished buildquads" + + return 0 + } function createQuickVersions() { require source_family_root || return $? @@ -707,22 +805,16 @@ function createQuickVersions() { setProduct ontology || return $? - logStep "createQuickVersions" - # # Get ontologies for Dev # - log "Merging all dev ontologies into one RDF file" - local quick_dev_spec=${DEV_SPEC/.rdf/}-quickstart - local quick_prod_spec=${PROD_SPEC/.rdf/}-quickstart - ${ONTOVIEWER_TOOLKIT_JAVA} \ --goal merge-imports \ --data "${source_family_root}/${DEV_SPEC}" $(test -s "${source_family_root}/catalog-v001.xml" && echo "--ontology-mapping \"${source_family_root}/catalog-v001.xml\"") \ - --ontology-iri "${product_root_url}/${quick_dev_spec}/" --ontology-version-iri "${tag_root_url}/${quick_dev_spec}/" \ - --output "${tag_root}/${quick_dev_spec}.rdf" &>/dev/null + --ontology-iri "${product_root_url}/Quick${ONTPUB_FAMILY^^}Dev/" --ontology-version-iri "${tag_root_url}/Quick${ONTPUB_FAMILY^^}Dev/" \ + --output "${tag_root}/Quick${ONTPUB_FAMILY^^}Dev.rdf" &>/dev/null # # Get ontologies for Prod @@ -731,20 +823,20 @@ function createQuickVersions() { ${ONTOVIEWER_TOOLKIT_JAVA} \ --goal merge-imports \ --data "${source_family_root}/${PROD_SPEC}" $(test -s "${source_family_root}/catalog-v001.xml" && echo "--ontology-mapping \"${source_family_root}/catalog-v001.xml\"") \ - --ontology-iri "${product_root_url}/${quick_prod_spec}/" --ontology-version-iri "${tag_root_url}/${quick_prod_spec}/" \ - --output "${tag_root}/${quick_prod_spec}.rdf" &>/dev/null - - log "Converting quick ontologies to other formats" + --ontology-iri "${product_root_url}/Quick${ONTPUB_FAMILY^^}Prod/" --ontology-version-iri "${tag_root_url}/Quick${ONTPUB_FAMILY^^}Prod/" \ + --output "${tag_root}/Quick${ONTPUB_FAMILY^^}Prod.rdf" &>/dev/null - ${SCRIPT_DIR}/utils/convertRdfFile.sh rdf-xml "${tag_root}/${quick_dev_spec}.rdf" "turtle" - ${SCRIPT_DIR}/utils/convertRdfFile.sh rdf-xml "${tag_root}/${quick_prod_spec}.rdf" "turtle" + ${SCRIPT_DIR}/utils/convertRdfFile.sh rdf-xml "${tag_root}/Quick${ONTPUB_FAMILY^^}Dev.rdf" "turtle" && \ + mv "${tag_root}/Quick${ONTPUB_FAMILY^^}Dev.ttl" ${tag_root}/dev.${ONTPUB_FAMILY}-quickstart.ttl + ${SCRIPT_DIR}/utils/convertRdfFile.sh rdf-xml "${tag_root}/Quick${ONTPUB_FAMILY^^}Prod.rdf" "turtle" && \ + mv "${tag_root}/Quick${ONTPUB_FAMILY^^}Prod.ttl" ${tag_root}/prod.${ONTPUB_FAMILY}-quickstart.ttl - ${JENA_ARQ} --data=${tag_root}/${quick_dev_spec}.ttl --query=/publisher/lib/echo.sparql --results=NT > ${tag_root}/${quick_dev_spec}.nt - ${JENA_ARQ} --data=${tag_root}/${quick_prod_spec}.ttl --query=/publisher/lib/echo.sparql --results=NT > ${tag_root}/${quick_prod_spec}.nt + ${JENA_ARQ} --data=${tag_root}/dev.${ONTPUB_FAMILY}-quickstart.ttl --query=/publisher/lib/echo.sparql --results=NT > ${tag_root}/dev.${ONTPUB_FAMILY}-quickstart.nt + ${JENA_ARQ} --data=${tag_root}/prod.${ONTPUB_FAMILY}-quickstart.ttl --query=/publisher/lib/echo.sparql --results=NT > ${tag_root}/prod.${ONTPUB_FAMILY}-quickstart.nt - zip ${tag_root}/${quick_dev_spec}.ttl.zip ${tag_root}/${quick_dev_spec}.ttl - zip ${tag_root}/${quick_prod_spec}.ttl.zip ${tag_root}/${quick_prod_spec}.ttl - zip ${tag_root}/${quick_dev_spec}.nt.zip ${tag_root}/${quick_dev_spec}.nt - zip ${tag_root}/${quick_prod_spec}.nt.zip ${tag_root}/${quick_prod_spec}.nt + zip ${tag_root}/dev.${ONTPUB_FAMILY}-quickstart.ttl.zip ${tag_root}/dev.${ONTPUB_FAMILY}-quickstart.ttl + zip ${tag_root}/prod.${ONTPUB_FAMILY}-quickstart.ttl.zip ${tag_root}/prod.${ONTPUB_FAMILY}-quickstart.ttl + zip ${tag_root}/dev.${ONTPUB_FAMILY}-quickstart.nt.zip ${tag_root}/dev.${ONTPUB_FAMILY}-quickstart.nt + zip ${tag_root}/prod.${ONTPUB_FAMILY}-quickstart.nt.zip ${tag_root}/prod.${ONTPUB_FAMILY}-quickstart.nt }