build.gradle

import java.time.*
import java.time.format.*
import java.time.temporal.*
import eu.esdihumboldt.util.config.*

buildscript {
  repositories {
    // mavenLocal()
    maven {
      url 'https://artifactory.wetransform.to/artifactory/local'
    }
    mavenCentral()
  }
  dependencies {
    def haleVersion = '6.0.0'
    // for Config class
    classpath "eu.esdihumboldt.hale:eu.esdihumboldt.util.config:$haleVersion"
    // for XMLSchemaUpdater (depends on https://github.com/halestudio/hale/pull/731)
    classpath "eu.esdihumboldt.hale:eu.esdihumboldt.hale.common.core:$haleVersion"

    classpath 'org.ajoberstar.grgit:grgit-core:5.2.2'
  }
}

plugins {
  id 'de.undercouch.download' version '5.6.0'
}

ext {
  resourcesFolder = file('resources')
  hostsFolder = new File(resourcesFolder, 'hosts')
  jarFolder = new File(buildDir, 'jars')
  now = LocalDate.now()
  time = LocalTime.now().format(DateTimeFormatter.ofPattern('HHmmss'))
  grgit = org.ajoberstar.grgit.Grgit.open(dir: projectDir)
}

wrapper {
  gradleVersion = '8.10.2'
}

/*
 * Helper classes and task definitions
 */

class MetadataHelper {
  static final METADATA_DIR_NAME = '.metadata'

  private final Project project
  private final File parentDir
  private final File metadataDir
  MetadataHelper(Project project, File parentDir) {
    this.project = project
    this.parentDir = parentDir
    this.metadataDir = new File(parentDir, METADATA_DIR_NAME)
  }

  def withConfig(Closure configure) {
    def configFile = new File(metadataDir, 'metadata.yml')
    configFile.parentFile.mkdirs()
    def config = configFile.exists() ? ConfigYaml.load(configFile) : new Config()
    // unexpectedly, when accessing config, instead of getAt get is used
    // when accessing config -> thus we interact with the map instead
    def map = config.asMap()
    def result = configure(map)
    result
  }

  def updateConfig(Closure configure) {
    def configFile = new File(metadataDir, 'metadata.yml')
    configFile.parentFile.mkdirs()
    def config = configFile.exists() ? ConfigYaml.load(configFile) : new Config()
    // unexpectedly, when accessing config, instead of getAt get is used
    // when accessing config -> thus we interact with the map instead
    def map = config.asMap()
    def result = configure(map)
    ConfigYaml.save(config, configFile)
    result
  }

  private boolean changesInFolder() {
    File checkForDir = parentDir
    File repoRoot = project.projectDir

    def relativePath = repoRoot.relativePath(checkForDir)
    //XXX do we need to replace \ with / for Windows?

    // add changes on relative path to index, so something like line-ending changes is not registered
    project.ext.grgit.add(patterns: [relativePath])

    def status = project.ext.grgit.status()

    // check for any changes and conflicts
    return [
      status.conflicts,
      status.staged.added,
      status.staged.modified,
      status.staged.removed,
      status.unstaged.added,
      status.unstaged.modified,
      status.unstaged.removed
    ].any { setOfPaths ->
      def example = setOfPaths.find { it.startsWith(relativePath) && !it.startsWith(relativePath + '/.metadata') }
      if (example) {
        println "Changes detected for path $relativePath (at least $example changed)"
        true
      }
      else {
        false
      }
    }
  }

  void setVersionIfChanged() {
    // only set version if there are changes to parentDir
    if (!changesInFolder()) {
      return
    }

    int year = project.ext.now.getYear();
    int month = project.ext.now.getMonthValue();
    int day = project.ext.now.getDayOfMonth();

    def version = "$year.$month.$day" as String

    updateConfig {
      it.year = year
      if (it.version == version) {
        // same version -> add time as qualifier (OSGi style version)
        version = ("${version}.${project.ext.time}" as String)
        if (it.version == version) {
          // still the same -> fail
          throw new IllegalStateException('Change detected but same version is already set')
        }
      }
      it.version = version
    }

    println "New version is $version"
  }

  void cleanFolderKeepMetadata() {
    if (parentDir.exists()) {
      parentDir.eachFile { file ->
        if (file.isDirectory()) {
          if (!file.name in ['.metadata']) {
            project.delete(file)
          }
        } else {
          file.delete()
        }
      }
    }

    parentDir.mkdirs()
  }

  String getVersion() {
    def version = withConfig {
      it.version
    }
    if (!version) {
      // default to year snapshot
      int year = project.ext.now.getYear();
      "${year}-SNAPSHOT"
    }
    else {
      version
    }
  }

  String getSnapshot() {
    def version = withConfig {
      it.year
    }
    if (!version) {
      // default to current year
      int year = project.ext.now.getYear();
      "${year}-SNAPSHOT"
    }
    else {
      "${version}-SNAPSHOT"
    }
  }

  void generateJar(File sourceFolder, File targetFolder, String name) {
    def version = getVersion()

    // assemble different publication versions
    def fileVersions = new HashSet<>()
    fileVersions << version
    fileVersions << getSnapshot()
    fileVersions << 'CURRENT-SNAPSHOT'

    fileVersions.each { fileVersion ->

      def jarFile = new File(targetFolder, "${name}_${fileVersion}.jar")

      // OSGi symbolic name
      def symbolicName = "to.wetransform.offline-resources.${name}"

      //TODO include OSGi/hale related stuff?

      // build jar
      project.ant.jar(destfile: jarFile) {
        fileset(dir: sourceFolder, includes: '**/*')
        manifest {
          attribute(name: 'Implementation-Version', value: version)
          // OSGi related metadata
          attribute(name: 'Bundle-Version', value: fileVersion)
          attribute(name: 'Bundle-Name', value: "Offline Resource bundle ${name}")
          attribute(name: 'Bundle-ManifestVersion', value: 2)
          attribute(name: 'Bundle-SymbolicName', value: "$symbolicName;singleton:=true")
        }
      }

    }
  }
}

class ResourcesArchiveTask extends DefaultTask {
  // the host that the resource bundle mirrors resources from
  @Input
  def host
  // the remote URL to retrieve the resource ZIP archive from
  @Input
  def archiveUrl
  // list of patterns for resources to exclude
  @Input
  List<String> excludes = []

  @TaskAction
  def downloadAndExtract() {
    assert host
    assert archiveUrl

    def that = this

    // download archive
    File tmpSchemas = new File(temporaryDir, 'schemas.zip')
    project.download.run {
      src archiveUrl
      dest tmpSchemas
      onlyIfModified true
    }

    File targetDir = project.file("${project.ext.hostsFolder}/$host")

    // clean up before download to remove any obsolete files
    def md = new MetadataHelper(project, targetDir)
    md.cleanFolderKeepMetadata()

    project.copy {
      from(project.zipTree(tmpSchemas)) {
        that.excludes.each { ex ->
          exclude(ex)
        }
      }
      into targetDir
    }

    // update version on change
    md.setVersionIfChanged()
  }
}

class INSPIREDownloadTask extends DefaultTask {
 // the host that the resource bundle mirrors resources from
  @Input
  def host
  // the remote URL to retrieve the resource ZIP archive from
  @Input
  def archiveUrl

  // create application schema index file
  def inspireSchemaIndex(def project, File target, List<File> folders) {
    // get application schemas from INSPIRE registry
    def registry
    new URL('https://inspire.ec.europa.eu/applicationschema/applicationschema.en.json').withReader { reader ->
      registry = new groovy.json.JsonSlurper().parse(reader)
    }
    def appSchemas = []
    registry.register.containeditems*.applicationschema.each {
      def theme = it.theme
      if (!theme) {
        theme = it.themes?.getAt(0)?.theme
      }

      appSchemas << [id: it.id, name: it.label.text, theme: theme?.label?.text, themeId: theme?.id]
    }

    def inspireShortIdRegex = '^http://inspire.ec.europa.eu/applicationschema/([^/]*)$'

    // sort application schemas by ID to have a consistent representation
    appSchemas = appSchemas.sort { it.id }

    target.withWriter {
      it << '<?xml version="1.0"?>\n'
      def xml = new groovy.xml.MarkupBuilder(it)
      xml.applicationSchemas {
        appSchemas.each { appSchema ->
          xml.applicationSchema(id: appSchema.id, name: appSchema.name, theme: appSchema.theme,
              themeId: appSchema.themeId) {
            // find corresponding schema files

            // extract short identifier
            def matcher = ( appSchema.id =~ inspireShortIdRegex )
            def shortId = matcher[0][1]

            // replace abandoned(?) Geology schema with Geology core
            if (shortId == 'ge') shortId = 'ge-core'

            folders.each { File folder ->
              File asDir = new File(folder, shortId)
              if (asDir.exists()) {
                // find version subfolders and sort to have consistent represention
                def versionDirs = asDir.listFiles().findAll { it.isDirectory() }.sort { it.name }
                versionDirs.each { File versionDir ->
                  // find XSD files and sort to have a consistent representation
                  def xsdFiles = versionDir.listFiles().findAll { it.isFile() && it.name =~ ~/.*\.xsd/ }.sort { it.name }
                  xsdFiles.each { File xsd ->
                    xml.schema(
                      version: versionDir.name,
                      url: "https://inspire.ec.europa.eu/${folder.name}/${asDir.name}/${versionDir.name}/${xsd.name}"
                    )
                  }
                }
              }
            }
          }
        }
      }
    }
  }

  @TaskAction
  def downloadAndExtract() {
    assert host
    assert archiveUrl

    // need a clean tmp dir
    temporaryDir.deleteDir()
    temporaryDir.mkdir()

    // download archive
    File tmpSchemas = new File(temporaryDir, 'schemas.zip')
    project.download.run {
      src archiveUrl
      dest tmpSchemas
      onlyIfModified true
    }
    File targetDir = project.file("${project.ext.hostsFolder}/$host")

    def md = new MetadataHelper(project, targetDir)
    md.cleanFolderKeepMetadata()

    project.copy {
      // Exclude Zip files, 'governance' folder, and 'readme.md' file
      exclude(
        '**/*.zip',
        'application-schemas-main/governance-release-process/**',
        'application-schemas-main/README.md',
        'application-schemas-main/.github/**',
        '**/*.groovy' // Groovy scripts used for validation
      )

      from project.zipTree(tmpSchemas)
      into temporaryDir
    }

    /*
     * The website does redirect for some schemas to a subfolder with older versions.
     * For example `hy-p/3.0/HydroPhysicalWaters.xsd` is redirected to `2021.1/hy-p/3.0/HydroPhysicalWaters.xsd`
     * The resource bundles don't know about these redirects, the file from the original folder is accessed.
     * Loading the full schema may fail though, because some schema have been removed from the root and are only
     * available in folders representing older versions.
     * Examples:
     * - `lc/0.0/LandCover.xsd`
     * - `wfd/0.0/WaterFrameworkDirective.xsd`
     *
     * Since we don't know the exact redirection rules we attempt to have a copy of al schemas in the root, in
     * their respective newest version, by copying newer versions over older versions.
     */

    File applicationtempschemas = new File(temporaryDir, 'application-schemas-main')
    File tempRoot = new File(applicationtempschemas, 'tmp-schemas')
    def versions = new File(applicationtempschemas, "schemas").listFiles().findAll { file ->
      file.isDirectory() && file.name =~ /^[2-9][0-9]{3}\.\d+$/
    }.collect { it.name }.sort()
    println "Identified versions in 'schemas': ${versions.join(', ')}"
    // safety check
    assert versions.contains('2023.1')
    assert versions[0] == '2021.1'

    // copy all versions in order
    versions.each { v ->
      project.copy {
        from new File(applicationtempschemas, "schemas/$v")
        into tempRoot
        include '**/*'
      }
    }

    // copy all files from main folder
    project.copy {
      from new File(applicationtempschemas, "schemas")
      into tempRoot
      include '**/*'
    }

    // delete schemas folder and move schemas aggregated in temp folder to its location
    File schemasFolder = new File(applicationtempschemas, 'schemas')
    schemasFolder.deleteDir()
    tempRoot.renameTo(schemasFolder)

    // copy files to target folder
    project.copy {
      from applicationtempschemas
      into targetDir
      include '**/*'
    }

    // create application schema index file (used by hale-studio specific bundle)
    inspireSchemaIndex(project, new File(targetDir, 'applicationSchemas.xml'),
      [new File(targetDir, 'schemas')]
    )

    // update version on change
    md.setVersionIfChanged()
  }
}

class WgetHostIndexDownloadTask extends DefaultTask {
  // the list of URLs pointing to a web server index
  @Input
  List<String> indexUrls
  // the list of extensions to accept
  @Input
  List<String> fileExtensions

  @TaskAction
  def download() {
    assert indexUrls

    def hosts = indexUrls.collect { URI.create(it).host }.unique()
    assert hosts.size() == 1 : 'Index URLs must be from same host'

    def host = hosts[0]

    File targetDir = project.file("${project.ext.hostsFolder}/$host")

    // clean up before download to remove any obsolete files
    def md = new MetadataHelper(project, targetDir)
    md.cleanFolderKeepMetadata()

    for (String indexUrl : indexUrls) {
      // download files
      project.exec {
        workingDir = targetDir
        executable = 'wget' // requires wget
        def runArgs = ['-e', 'robots=off', '-r', '--no-parent', '--no-host-directories']
        if (fileExtensions) {
          runArgs << '-A'
          runArgs << fileExtensions.join(',')
        }
        runArgs << indexUrl
        args = runArgs
      }
    }

    // update version on change
    md.setVersionIfChanged()
  }
}

class HostPathsDownloadTask extends DefaultTask {
  // the host that the resource bundle mirrors resources from
  @Input
  def hostUrl
  // the paths of the files that should be retrieved from the host
  @Input
  List<String> paths

  @TaskAction
  def downloadPaths() {
    assert hostUrl
    assert paths

    // determine host
    def host = URI.create(hostUrl).host
    assert host

    File targetDir = project.file("${project.ext.hostsFolder}/$host")

    // clean up before download to remove any obsolete files
    def md = new MetadataHelper(project, targetDir)
    md.cleanFolderKeepMetadata()

    for (String path : paths) {
      File localFile = new File(targetDir, path)
      def fileUrl = hostUrl + "/" + path

      // download file
      project.download.run {
        src fileUrl
        dest localFile
        overwrite true
      }
    }

    // update version on change
    md.setVersionIfChanged()
  }
}

class WgetHostPathsDownloadTask extends DefaultTask {
  // the host that the resource bundle mirrors resources from
  @Input
  def hostUrl
  // the paths of the files that should be retrieved from the host
  @Input
  List<String> paths

  @TaskAction
  def downloadPaths() {
    assert hostUrl
    assert paths

    // determine host
    def host = URI.create(hostUrl).host
    assert host

    File targetDir = project.file("${project.ext.hostsFolder}/$host")

    // clean up before download to remove any obsolete files
    def md = new MetadataHelper(project, targetDir)
    md.cleanFolderKeepMetadata()

    for (String path : paths) {
      def fileUrl = hostUrl + "/" + path

      // download file
      project.exec {
        workingDir = targetDir
        executable = 'wget' // requires wget
        args '--force-directories', '--no-host-directories', fileUrl
      }
    }

    // update version on change
    md.setVersionIfChanged()
  }
}

class XmlSchemaDownloadTask extends DefaultTask {
  // the location of the remote schema
  @Input
  def schemaUrl
  // the name if the schema
  @Input
  def schemaName
  // the name of the resource group
  @Input
  def resourceGroup

  @TaskAction
  def download() {
    assert schemaUrl
    assert schemaName
    assert resourceGroup

    File schemaTargetDir = project.file("${project.ext.resourcesFolder}/$resourceGroup/$schemaName")

    // clean up before download to remove any obsolete files
    def md = new MetadataHelper(project, schemaTargetDir)
    md.cleanFolderKeepMetadata()

    File schemaFile = new File(schemaTargetDir, 'schema.xsd')

    // download archive
    project.download.run {
      src schemaUrl
      dest schemaFile
      overwrite true
    }

    // update schema file and download references
    eu.esdihumboldt.hale.common.core.io.project.util.XMLSchemaUpdater.update(
      schemaFile, URI.create(schemaUrl), true, null)

    // update version on change
    md.setVersionIfChanged()
    md.withConfig {
      it.originalLocation = schemaUrl as String
    }
  }
}


/**
 * Dummy task for testing (changes result on every run)
 */
task('dummy') {
  group 'Download'
  description 'Dummy download task for testing'
}.doFirst {
  def host = 'example.com'
  File targetDir = project.file("${project.ext.hostsFolder}/$host")

  // clean up before download to remove any obsolete files
  def md = new MetadataHelper(project, targetDir)
  md.cleanFolderKeepMetadata()

  def content = now.toString()

  new File(targetDir, 'root.txt').text = content
  def subfolder = new File(targetDir, 'sub')
  subfolder.mkdirs()
  new File(subfolder, 'sub.txt').text = content

  // update version on change
  md.setVersionIfChanged()
}

/*
 * Download tasks
 */

/**
 * Aggregation task for downloads
 */
task('downloads') {
  group 'Download'
  description 'Runs all downloads'
}

/**
 * Schemas from schemas.opengis.net
 */
task opengisSchemas(type: ResourcesArchiveTask) {
  group 'Download'
  host = "schemas.opengis.net"
  archiveUrl = "http://schemas.opengis.net/SCHEMAS_OPENGIS_NET.zip"
  //FIXME some excludes like .html and .txt seem not to work in subfolders, while .zip does
  // reason unclear
  excludes = ['**/*.html', '**/*.zip', '.timestamp', '**/*.txt', '*.rss', '*.ico',
    '.LISTING', '**/*.xmi', '**/*.yaml', '**/*.yml', '**/*.json', '**/.validaterc']
}
tasks.downloads.dependsOn(opengisSchemas)

/**
 * Schemas from www.w3.org
 */
task w3Schemas(type: HostPathsDownloadTask) {
  group 'Download'
  hostUrl = "https://www.w3.org"
  paths = [
    '1999/xlink.xsd',
    '2001/xml.xsd',
    '2001/XMLSchema.xsd',
    '2005/08/addressing/ws-addr.xsd'
  ]
}
tasks.downloads.dependsOn(w3Schemas)

/**
 * Schemas from schemas.geosciml.org
 */
task geoscimlSchemas(type: HostPathsDownloadTask) {
  // this schema seems to used in deegree in some context
  // most other schemas related to GeoSciML seem to be available via OGC
  group 'Download'
  hostUrl = "http://schemas.geosciml.org"
  paths = [
    'cgiutilities/3.0/cgiUtilities.xsd'
  ]
}
tasks.downloads.dependsOn(geoscimlSchemas)

/**
 * Schemas from portele.de
 */
task porteleSchemas(type: HostPathsDownloadTask) {
  group 'Download'
  hostUrl = "http://portele.de"
  paths = [
    'ShapeChangeAppinfo.xsd'
  ]
}
tasks.downloads.dependsOn(porteleSchemas)

/**
 * INSPIRE schemas
 */
task inspireSchemas(type: INSPIREDownloadTask) {
  group 'Download'
  host = "inspire.ec.europa.eu"
  archiveUrl = "https://github.com/INSPIRE-MIF/application-schemas/archive/refs/heads/main.zip"
}
tasks.downloads.dependsOn(inspireSchemas)

/**
 * AAA and XPlanung schemas
 */
def aaaVersions = ['6.0', '7.1']
def wfsExtVersions = [ '1.0', '2.0' ]
def shapeChangeVersions = [ '1.0' ]
// Map version number used in URLs to version number displayed to user
def xplanungVersions = [
  '4.0.2',
  '4.1-Kernmodell',
  '5.0',
  '5.1',
  '5.2',
  '5.3',
  '5.4',
  '6.0'
]

def getXPlanungSchemaBaseUrl = { version ->
  "https://repository.gdi-de.org/schemas/de.xleitstelle.xplanung/${version}/"
}

def getXPlanungSchemaUrl = { version ->
  "${getXPlanungSchemaBaseUrl(version)}XPlanung-Operationen.xsd"
}

def determineDisplayVersion = { url, element ->
  def xsd = new XmlSlurper().parse(url)
  xsd."${element}"
}

task gdiRepositorySchemas(type: WgetHostIndexDownloadTask) {
  group 'Download'
  indexUrls = aaaVersions.collect { version -> "http://repository.gdi-de.org/schemas/adv/nas/${version}/" } +
    wfsExtVersions.collect { version -> "http://repository.gdi-de.org/schemas/adv/wfsext/${version}/" } +
    shapeChangeVersions.collect { version -> "http://repository.gdi-de.org/schemas/adv/sc/${version}/" } +
    xplanungVersions.collect { urlVersion -> getXPlanungSchemaBaseUrl(urlVersion) }
  fileExtensions = ['xml','xsd', 'txt']
}
tasks.downloads.dependsOn(gdiRepositorySchemas)

/**
 * Schemas for metadata validation:
 *
 * http://build-artifacts.wetransform.to/schemas/iso19139/2012-07-13/gmd/gmd.xsd (gmd version "2012-07-13")
 *     - mirrored version of the schema at https://standards.iso.org/iso/19139/Schemas/gmd/gmd.xsd
 *       with relativized schema references to the ISO 19139 and ISO 19136 (GML) XSDs
 * https://www.isotc211.org/2005/gmd/gmd.xsd (gmd version "0.1")
 * https://www.isotc211.org/2005/gmx/gmx.xsd
 * http://schemas.opengis.net/csw/2.0.2/profiles/apiso/1.0.0/apiso.xsd
 */

/* FIXME for now disabled because at least on of the resources is not longer accessible like that
   TODO recreate missing resources in file system and re-enable tasks that still work
task metadataIso(type: XmlSchemaDownloadTask) {
  group 'Download'
  schemaUrl = 'http://build-artifacts.wetransform.to/schemas/iso19139/2012-07-13/gmd/gmd.xsd'
  schemaName = 'ISO_19139_GMD'
  resourceGroup = 'metadata-schemas'
}
tasks.downloads.dependsOn(metadataIso)

task metadataIsoTC211(type: XmlSchemaDownloadTask) {
  group 'Download'
  schemaUrl = 'https://www.isotc211.org/2005/gmd/gmd.xsd'
  schemaName = 'ISO_TC211_GMD'
  resourceGroup = 'metadata-schemas'
}
tasks.downloads.dependsOn(metadataIsoTC211)

task metadataExtensionsIsoTC211(type: XmlSchemaDownloadTask) {
  group 'Download'
  schemaUrl = 'https://www.isotc211.org/2005/gmx/gmx.xsd'
  schemaName = 'ISO_TC211_GMX'
  resourceGroup = 'metadata-schemas'
}
tasks.downloads.dependsOn(metadataExtensionsIsoTC211)

task metadataCSW2(type: XmlSchemaDownloadTask) {
  group 'Download'
  schemaUrl = 'http://schemas.opengis.net/csw/2.0.2/profiles/apiso/1.0.0/apiso.xsd'
  schemaName = 'CSW_2_APISO'
  resourceGroup = 'metadata-schemas'
}
tasks.downloads.dependsOn(metadataCSW2)
*/


/*
 * Download the FeatureConcept information from the INSPIRE registry, which is
 * used in HALE when generating an INSPIRE Dataset Feed.
 */
task downloadInspireFeatureConcepts {
  doLast {
    File targetDir = project.file("${project.ext.resourcesFolder}/inspire-feature-concepts")

    // clean up before download to remove any obsolete files
    def md = new MetadataHelper(project, targetDir)
    md.cleanFolderKeepMetadata()

    def paths = ['featureconcept/featureconcept.en.json']
    for (String path : paths) {
      File localFile = new File(targetDir, path)
      def fileUrl = "https://inspire.ec.europa.eu/" + path

      // download file
      project.download.run {
        src fileUrl
        dest localFile
        overwrite true
      }
    }

    // update version on change
    md.setVersionIfChanged()
  }
}
tasks.downloads.dependsOn('downloadInspireFeatureConcepts')


/*
 * Download code list files based on registry (requires wget, XML content type)
 */
task downloadInspireCodeLists {
  doLast {

    // determine resources to retrieve from registry
    def registry
    new URL('https://inspire.ec.europa.eu/codelist/codelist.en.json').withReader { reader ->
      registry = new groovy.json.JsonSlurper().parse(reader)
    }

    def languages = ['en', 'bg', 'cs', 'hr', 'da', 'de', 'el', 'es', 'et', 'fi', 'fr',
      'hu', 'it', 'lt', 'lv', 'mt', 'nl', 'pl', 'pt', 'ro', 'sk', 'sl', 'sv']

    def genericUrls = []
    def fixedUrls = []
    registry.register.containeditems*.codelist.id.each { id ->
      // the ID is the URI to retrieve the code list by default
      genericUrls << id
      // extract code list name
      def regex = '/([^/]*)$'
      def matcher = ( id =~ regex )
      def name = matcher[0][1]
      // add fixed urls TODO in all languages?
      fixedUrls << "${id}/${name}.en.xml"
      fixedUrls << "${id}/${name}.en.json"
    }

    // problem is, we can't download both generic and fixed,
    // because the fixed URLs require a directory with the name
    // of the generic URL -> so we create two archives/bundles

    File genericDir = project.file("${project.ext.resourcesFolder}/inspire-code-lists-generic")

    // clean up before download to remove any obsolete files
    def genericMd = new MetadataHelper(project, genericDir)
    genericMd.cleanFolderKeepMetadata()

    genericUrls.each { url ->
      exec {
        workingDir = genericDir
        executable = 'wget'
        args '--force-directories', '--no-host-directories', "--header=Accept: application/xml", "--header=Accept-Language: en", url
      }
    }

    // update version on change
    genericMd.setVersionIfChanged()

    File fixedDir = project.file("${project.ext.resourcesFolder}/inspire-code-lists-fixed")

    // clean up before download to remove any obsolete files
    def fixedMd = new MetadataHelper(project, fixedDir)
    fixedMd.cleanFolderKeepMetadata()

    fixedUrls.each { url ->
      exec {
        workingDir = fixedDir
        executable = 'wget'
        args '--force-directories', '--no-host-directories', url
      }
    }

    // update version on change
    fixedMd.setVersionIfChanged()

  }
}
tasks.downloads.dependsOn('downloadInspireCodeLists')


/*
 * Processing tasks
 */

/**
 * Task for creating resource JARs for use in Java applications
 */
task(jars) {

}.doLast {

  /*
   * JARs for host folders
   */
  def files = project.ext.hostsFolder.listFiles()
  project.logger.info("Identified ${files.size()} potential host JARs")

  jarFolder.deleteDir()
  jarFolder.mkdirs()

  def tmpJarFolder = new File(temporaryDir, 'jar')

  files.each { File file ->
    // for each host folder
    if (file.isDirectory()) {
      println "Creating resources JARs for host ${file.name}"

      // prepare structure for inside JAR
      // use sub-directories to avoid confusion w/ other resources
      tmpJarFolder.deleteDir()
      tmpJarFolder.mkdirs()

      def resFolder = new File(tmpJarFolder, "to/wetransform/offline-resources/hosts/${file.name}")
      resFolder.mkdirs()

      copy {
        from file
        into resFolder

        exclude "${MetadataHelper.METADATA_DIR_NAME}/**"
      }

      def md = new MetadataHelper(project, file)
      md.generateJar(tmpJarFolder, jarFolder, file.name)
    }
  }

  /*
   * JARs for other folders
   */
  files = project.ext.resourcesFolder.listFiles()

  /* XXX currently not used - TODO control where a Jar should be created?
  files.each { File file ->
    if (file.isDirectory() && !file.name.equals('hosts')) {
      println "Creating resources JARs for ${file.name}"

      // prepare structure for inside JAR
      // use sub-directories to avoid confusion w/ other resources
      tmpJarFolder.deleteDir()
      tmpJarFolder.mkdirs()

      def resFolder = new File(tmpJarFolder, "to/wetransform/offline-resources/${file.name}")
      resFolder.mkdirs()

      copy {
        from file
        into resFolder

        exclude "${MetadataHelper.METADATA_DIR_NAME}/**"
      }

      def md = new MetadataHelper(project, file)
      md.generateJar(tmpJarFolder, jarFolder, file.name)
    }
  }
  */

}

/**
 * Task that publishes all Jars to artifactory.
 */
task('publishJars', type: GradleBuild) {
  buildFile = file('publish.gradle')
  buildName = 'publishJars'

  startParameter.projectProperties.publications = jarFolder.getAbsolutePath()

  tasks = ['publish']
}

/**
 * Task that publishes all Jars to the local maven repository.
 */
task('publishJarsToMavenLocal', type: GradleBuild) {
  buildFile = file('publish.gradle')
  buildName = 'publishJarsToMavenLocal'

  startParameter.projectProperties.publications = jarFolder.getAbsolutePath()

  tasks = ['publishToMavenLocal']
}

/**
 * Task that publishes current versions to an update site.
 * Requires Jars to be created and artifacts published to Maven Local beforehand.
 */
task('updateSite', type: GradleBuild) {
  buildFile = file('updatesite/build.gradle')
  buildName = 'updateSite'

  startParameter.projectProperties.publications = jarFolder.getAbsolutePath()

  tasks = ['clean', 'updateSite']
}

/*
 * Hale studio specific tasks
 */

class ResourceBundleTask extends DefaultTask {
  // the host that the resource bundle mirrors resources from
  @Input
  def host
  // the local folder with source files
  @Input
  def sourceDir
  // optional additional sources
  @Input
  @Optional
  def additionalSources
  // the prefix for the symbolic name
  @Input
  def symbolicNamePrefix = 'eu.esdihumboldt.util.resource'
  // the bundle identifier (is appended to prefix sepearate by a dot)
  @Input
  def bundleId
  // the served resource type
  @Input
  def resourceType = 'xmlSchema'
  // the destination dir
  @Input
  def destDir = project.buildDir
  // extra content for the plugin.xml - must be a closure taking a markup builder as argument
  @Internal
  java.util.function.Consumer<groovy.xml.MarkupBuilder> pluginExtra

  ResourceBundleTask() {
    getOutputs().upToDateWhen {
      // we don't support proper up-to-date checking here
      false
    }
  }

  @TaskAction
  def createBundle() {
    assert host
    assert sourceDir
    assert bundleId
    assert symbolicNamePrefix
    assert resourceType // the primary resource type
    assert destDir

    def md = new MetadataHelper(project, sourceDir)

    def version = md.version
    def symbolicName = "$symbolicNamePrefix.$bundleId"

    File tmpDir = temporaryDir //File.createTempFile('resources', 'bundle')
    tmpDir.delete()
    tmpDir.mkdir()

    // create plugin.xml
    File tmpPluginXml = new File(tmpDir, 'plugin.xml')
    tmpPluginXml.withWriter {
      it << '<?xml version="1.0"?>\n'
      it << '<?eclipse version="3.4"?>\n'
      def xml = new groovy.xml.MarkupBuilder(it)
      xml.plugin {
        extension(point: 'eu.esdihumboldt.util.resource') {
          resolver(id: bundleId, resourceType: resourceType) {
            xml.host(name: host)
            bundleResolver()
          }
        }
        if (pluginExtra != null) {
          pluginExtra.accept(xml)
        }
      }
    }

    def variants = [version, md.snapshot, 'CURRENT-SNAPSHOT']
    variants.each { variant ->
      File bundle = new File(destDir, "${symbolicName}_${variant}.jar")

      // build jar
      ant.jar(destfile: bundle) {
        fileset (dir: sourceDir, excludes: '**/.metadata/**')
        if (additionalSources) {
          fileset (dir: additionalSources, excludes: '**/.metadata/**')
        }
        fileset (dir: tmpPluginXml.parentFile, includes: tmpPluginXml.name)
        manifest {
          attribute(name: 'Bundle-Version', value: version)
          attribute(name: 'Bundle-Name', value: "Resources from $host")
          attribute(name: 'Bundle-ManifestVersion', value: 2)
          attribute(name: 'Bundle-SymbolicName', value: "$symbolicName;singleton:=true")
        }
      }
    }
  }
}

/**
 * Task for creating resource bundles used by hale
 */
task haleResourceBundles() {
}

task porteleSchemasBundle(type: ResourceBundleTask) {
  bundleId = 'schemas.portele.de'
  host = 'portele.de'
  sourceDir = new File(hostsFolder, 'portele.de')
  destDir = jarFolder
}
haleResourceBundles.dependsOn('porteleSchemasBundle')

task inspireFeatureConceptsBundle(type: ResourceBundleTask) {
  bundleId = 'featureconcepts.inspire'
  host = 'inspire.ec.europa.eu'
  sourceDir = new File(resourcesFolder, 'inspire-feature-concepts')
  destDir = jarFolder
}
haleResourceBundles.dependsOn('inspireFeatureConceptsBundle')

task w3SchemasBundle(type: ResourceBundleTask) {
  bundleId = 'w3.org'
  host = 'www.w3.org'
  sourceDir = new File(hostsFolder, 'www.w3.org')
  destDir = jarFolder
}
haleResourceBundles.dependsOn('w3SchemasBundle')

task inspireCodeListsBundleGeneric(type: ResourceBundleTask) {
  bundleId = 'codelists.inspire.accept-xml'
  host = 'inspire.ec.europa.eu'
  sourceDir = new File(resourcesFolder, 'inspire-code-lists-generic')
  destDir = jarFolder
  resourceType = 'xmlCodeList'
}
haleResourceBundles.dependsOn('inspireCodeListsBundleGeneric')

task inspireCodeListsBundleFixed(type: ResourceBundleTask) {
  bundleId = 'codelists.inspire'
  host = 'inspire.ec.europa.eu'
  sourceDir = new File(resourcesFolder, 'inspire-code-lists-fixed')
  destDir = jarFolder
  resourceType = 'xmlCodeList'
}
haleResourceBundles.dependsOn('inspireCodeListsBundleFixed')

task opengisBundle(type: ResourceBundleTask) {
  bundleId = 'schemas.opengis.net'
  host = 'schemas.opengis.net'
  sourceDir = new File(hostsFolder, 'schemas.opengis.net')
  destDir = jarFolder
}
haleResourceBundles.dependsOn('opengisBundle')

task inspireSchemasBundle(type: ResourceBundleTask) {
  bundleId = 'schemas.inspire'
  host = 'inspire.ec.europa.eu'
  sourceDir = new File(hostsFolder, 'inspire.ec.europa.eu')
  additionalSources = new File(resourcesFolder, 'inspire-fixup')
  destDir = jarFolder
  pluginExtra = { xml ->
    // read application schemas file
    def applicationSchemasFile = new File(sourceDir, 'applicationSchemas.xml')
    def appSchemas = new groovy.xml.XmlSlurper().parse(applicationSchemasFile)

    def themeShortIdRegex = '^http://inspire.ec.europa.eu/theme/([^/]*)$'

    xml.extension(point: 'eu.esdihumboldt.hale.ui.schema.presets') {
      println appSchemas
      appSchemas.applicationSchema.each { appSchema ->
        def id = appSchema.@id

        appSchema.schema.each { schema ->
          def theme = appSchema.@theme as String
          def descr = theme ? "GML Application Schema from the INSPIRE Data Specification on ${theme}." : "Application Schema from the INSPIRE Data Specifications (not associated to a theme)."

          def themeId = appSchema.@themeId as String
          def iconPath = 'icons/inspire.gif'
          if (themeId) {
            // extract short identifier
            def matcher = ( themeId =~ themeShortIdRegex )
            def shortId = matcher[0][1]

            iconPath = "icons/themes/16_${shortId}.png"
          }

          xml.schema(
            description: descr,
            icon: iconPath,
            id: id,
            name: "INSPIRE ${appSchema.'@name'}",
//            tag: 'Bundled',
            version: schema.@version
          ) {
            xml.uri(value: schema.@url)
          }
        }
      }
    }
  }
}
haleResourceBundles.dependsOn('inspireSchemasBundle')

task gdiRepositorySchemasBundle(type: ResourceBundleTask) {
  bundleId = 'schemas.repository.gdi-de.org'
  host = 'repository.gdi-de.org'
  sourceDir = new File(hostsFolder, 'repository.gdi-de.org')
  additionalSources = new File(resourcesFolder, 'xplanung-fixup')
  destDir = jarFolder
  pluginExtra = { xml ->
    xml.extension(point: 'eu.esdihumboldt.hale.ui.schema.presets') {
      aaaVersions.each { version ->
        xml.schema(
          category: 'de',
          description: "AAA (ATKIS, ALKIS, AFIS) Modell der Arbeitsgemeinschaft der Vermessungsverwaltungen der Länder der Bundesrepublik Deutschland (AdV)",
          id: "de.adv-online.aaa.${version.replace('.', '_')}",
          name: "AAA XML Schema",
          version: "${version}"
        ) {
          xml.uri(value: "http://repository.gdi-de.org/schemas/adv/nas/${version}/aaa.xsd")
        }
      }

      xplanungVersions.each { urlVersion ->
        xml.schema(
          category: 'de',
          description: "XPlanung GML Application Schema",
          icon: 'icons/xplanung.png',
          id: "de.xplanung.xplangml.${urlVersion.replace('.', '_')}",
          name: "XPlanGML",
          version: determineDisplayVersion(getXPlanungSchemaUrl(urlVersion), '@version')
        ) {
          xml.uri(value: getXPlanungSchemaUrl(urlVersion))
        }
      }
    }
  }
}
haleResourceBundles.dependsOn('gdiRepositorySchemasBundle')