♻️ Advance Scala and Spark versions. Refactor the code to align with …

…the breaking changes of onFHIR.

pom.xml

@@ -91,7 +91,7 @@
         <!-- This is a special placeholder to manage the version from a single property throughout the parent and child modules. No other property name can be used. -->
         <!-- https://maven.apache.org/maven-ci-friendly.html -->
         <revision>1.1-SNAPSHOT</revision>
-        <scala.version>2.13.8</scala.version>
+        <scala.version>2.13.16</scala.version>
         <scala.binary.version>2.13</scala.binary.version>
 
         <!--Dependency versions-->
@@ -104,7 +104,7 @@
         <fluent-logger.version>0.3.4</fluent-logger.version>
         <com.fasterxml.version>2.15.1</com.fasterxml.version>
         <scalatest.version>3.2.17</scalatest.version>
-        <spark.version>3.5.1</spark.version>
+        <spark.version>3.5.4</spark.version>
         <jackson.version>2.15.1</jackson.version>
         <logback.version>1.2.11</logback.version>
         <logstash-logback-encoder.version>7.2</logstash-logback-encoder.version>

tofhir-engine/src/main/scala/io/tofhir/engine/mapping/schema/SchemaFolderLoader.scala

@@ -59,7 +59,7 @@ class SchemaFolderLoader(folderUri: URI, majorFhirVersion: String = MajorFhirVer
   private def getListOfSchemas: Seq[File] = {
     val folder = new File(folderUri)
     try {
-      IOUtil.getFilesFromFolder(folder, withExtension = Some(FileExtensions.JSON.toString), recursively = Some(true))
+      IOUtil.getFilesFromFolder(folder, recursively = true, ignoreHidden = true, withExtension = Some(FileExtensions.JSON.toString))
     } catch {
       case e: Throwable => throw FhirMappingException(s"Given folder for the schema repository is not valid.", e)
     }

tofhir-engine/src/main/scala/io/tofhir/engine/mapping/service/LocalTerminologyService.scala

@@ -25,7 +25,7 @@ class LocalTerminologyService(settings:LocalFhirTerminologyServiceSettings) exte
   //All csv files given in the configured folder
   val relatedFiles:Map[String, File] =
     IOUtil
-      .getFilesFromFolder(new File(FileUtils.getPath(settings.folderPath).toUri), withExtension = Some(FileUtils.FileExtensions.CSV.toString), recursively = Some(true))
+      .getFilesFromFolder(new File(FileUtils.getPath(settings.folderPath).toUri), recursively = true, ignoreHidden = true, withExtension = Some(FileUtils.FileExtensions.CSV.toString))
       .map(f => f.getName -> f)
       .toMap
 

tofhir-engine/src/main/scala/io/tofhir/engine/repository/mapping/FhirMappingFolderRepository.scala

@@ -34,7 +34,7 @@ class FhirMappingFolderRepository(folderUri: URI) extends IFhirMappingRepository
     val folder = new File(folderUri)
     var files = Seq.empty[File]
     try {
-      files = IOUtil.getFilesFromFolder(folder, withExtension = Some(FileExtensions.JSON.toString), recursively = Some(true))
+      files = IOUtil.getFilesFromFolder(folder, recursively = true, ignoreHidden = true, withExtension = Some(FileExtensions.JSON.toString))
     } catch {
       case e: Throwable => throw FhirMappingException(s"Given folder for the mapping repository is not valid.", e)
     }

tofhir-engine/src/main/scala/io/tofhir/engine/util/FileUtils.scala

@@ -41,7 +41,7 @@ object FileUtils {
    */
   def findFileByName(repoPath: String, name: String): Option[File] = {
     val repoFile = FileUtils.getPath(repoPath).toFile
-    val allFiles = IOUtil.getFilesFromFolder(repoFile, withExtension = Some(FileExtensions.JSON.toString), recursively = Some(true))
+    val allFiles = IOUtil.getFilesFromFolder(repoFile, recursively = true, ignoreHidden = true, withExtension = Some(FileExtensions.JSON.toString))
     val filteredFiles = allFiles.filter(f => {
       f.getName
         .toLowerCase.equals(name.toLowerCase)

tofhir-server/src/main/scala/io/tofhir/server/repository/job/JobFolderRepository.scala

@@ -192,7 +192,7 @@ class JobFolderRepository(jobRepositoryFolderPath: String, projectRepository: IP
     projectDirectories.foreach { projectDirectory =>
       // job-id -> FhirMappingJob
       val fhirJobMap: mutable.Map[String, FhirMappingJob] = mutable.Map.empty
-      val files = IOUtil.getFilesFromFolder(projectDirectory, withExtension = Some(FileExtensions.JSON.toString), recursively = Some(true))
+      val files = IOUtil.getFilesFromFolder(projectDirectory, recursively = true, ignoreHidden = true, withExtension = Some(FileExtensions.JSON.toString))
       files.map { file =>
         val source = Source.fromFile(file, StandardCharsets.UTF_8.name()) // read the JSON file
         val fileContent = try source.mkString finally source.close()

tofhir-server/src/main/scala/io/tofhir/server/repository/mapping/ProjectMappingFolderRepository.scala

@@ -213,7 +213,7 @@ class ProjectMappingFolderRepository(mappingRepositoryFolderPath: String, projec
     projectDirectories.foreach { projectDirectory =>
       // mapping-id -> FhirMapping
       val fhirMappingMap: mutable.Map[String, FhirMapping] = mutable.Map.empty
-      val files = IOUtil.getFilesFromFolder(projectDirectory, withExtension = Some(FileExtensions.JSON.toString), recursively = Some(true))
+      val files = IOUtil.getFilesFromFolder(projectDirectory, recursively = true, ignoreHidden = true, withExtension = Some(FileExtensions.JSON.toString))
       files.foreach { file =>
         val source = Source.fromFile(file, StandardCharsets.UTF_8.name()) // read the JSON file
         val fileContent = try source.mkString finally source.close()

tofhir-server/src/main/scala/io/tofhir/server/repository/mappingContext/MappingContextFolderRepository.scala

@@ -233,7 +233,7 @@ class MappingContextFolderRepository(mappingContextRepositoryFolderPath: String,
     var directories = Seq.empty[File]
     directories = folder.listFiles.filter(_.isDirectory).toSeq
     directories.foreach { projectDirectory =>
-      val files = IOUtil.getFilesFromFolder(projectDirectory, withExtension = None, recursively = Some(true))
+      val files = IOUtil.getFilesFromFolder(projectDirectory, recursively = true, ignoreHidden = true, withExtension = None)
       val fileNameList = files.map(_.getName)
       this.mappingContextDefinitions.put(projectDirectory.getName, fileNameList)
     }

tofhir-server/src/main/scala/io/tofhir/server/repository/schema/SchemaFolderRepository.scala

@@ -246,7 +246,7 @@ class SchemaFolderRepository(schemaRepositoryFolderPath: String, projectReposito
       var schemaFiles = Seq.empty[File]
       try {
         // We may need to give a warning if there are non-json files or other directories inside the project folders.
-        schemaFiles = IOUtil.getFilesFromFolder(projectFolder, withExtension = Some(FileExtensions.JSON.toString), recursively = Some(false))
+        schemaFiles = IOUtil.getFilesFromFolder(projectFolder, recursively = true, ignoreHidden = true, withExtension = Some(FileExtensions.JSON.toString))
       } catch {
         case e: Throwable => throw FhirMappingException(s"Given folder for the schema repository is not valid at path ${projectFolder.getAbsolutePath}", e)
       }