add continuables.jl for loading paths

Project.toml

@@ -7,6 +7,7 @@ version = "0.1.0"
 Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 CSVFiles = "5d742f6a-9f54-50ce-8119-2520741973ca"
+Continuables = "79afa230-ca09-11e8-120b-5decf7bf5e25"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DataFramesMeta = "1313f7d8-7da2-5740-9ea0-a2ca25f37964"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"

src/UnfoldBIDS.jl

@@ -2,7 +2,7 @@ module UnfoldBIDS
 
 # basics
 using StatsModels, DataFrames, DataFramesMeta, Statistics, Printf
-using ProgressBars
+using ProgressBars, Continuables
 # file loading
 using PyMNE, CSV
 # unfold

src/load.jl

@@ -1,85 +1,71 @@
 function bids_layout(bidsPath::AbstractString;
-    derivative::Bool=true,
+    derivatives::Bool=true,
     specificFolder::Union{Nothing,AbstractString}=nothing,
     excludeFolder::Union{Nothing,AbstractString}=nothing,
     ses::Union{Nothing,AbstractString}=nothing,
     task::Union{Nothing,AbstractString}=nothing,
     run::Union{Nothing,AbstractString}=nothing)
 
     # Any files with these endings will be returned
-    file_pattern = ["eeg", "set", "fif", "vhdr", "edf"]
-    nPattern = 2
+    file_ending = [".eeg", ".set", ".fif", ".vhdr", ".edf"]
 
+    file_pattern = [""]
     # Extend file pattern
     if ses !== nothing
         file_pattern = push!(file_pattern, "ses-" * ses)
-        nPattern += 1
     end
 
     if task !== nothing
         file_pattern = push!(file_pattern, "task-" * task)
-        nPattern += 1
     end
 
     if run !== nothing
         file_pattern = push!(file_pattern, "run-" * run)
-        nPattern += 1
     end
 
-    # Choose a specific folder in either ./ or ./derivatives
-    if derivative && specificFolder !== nothing
-        sPath = joinpath(bidsPath, "derivatives", specificFolder)
-        #@show sPath
-    elseif specificFolder !== nothing
-        sPath = joinpath(bidsPath, specificFolder)
-        #@show sPath
-    end
-
-    # Exclude these folders when using raw data
-    if derivative && excludeFolder !== nothing
-        exclude = excludeFolder
-    elseif !derivative && excludeFolder !== nothing
-        exclude = ["derivatives", excludeFolder]
-    elseif !derivative
-        exclude = "derivatives"
+    # Choose what to ignore and check if derivatives should be used
+    exclude = [] 
+    if derivatives
+        # TODO: Bug in loading only derivatives
+        bidsPath = joinpath(bidsPath, "derivatives")
     else
-        exclude = ""
+        push!(exclude, "derivatives")
     end
 
+    if excludeFolder !== nothing
+        exclude = push!(exclude, excludeFolder)
+    end
 
-    files_df = DataFrame(subject=[], ses=[], task=[], run=[], file=[], path=[])  # Initialize an empty DataFrame to hold results
-
-    # regular expression for additional information
-    # regex = r"sub-(.+)|_task-(.+)|_run-(.+)_eeg"
+    # Choose a specific folder in either ./ or ./derivatives
+    if specificFolder !== nothing
+        bidsPath = joinpath(bidsPath, specificFolder)
+    end
 
 
-    # Search for files matching file pattern
-    if specificFolder !== nothing
-        for (root, dirs, files) in walkdir(sPath)
-            for file in files
 
-                if sum(occursin.(file_pattern, file)) >= nPattern
+    files_df = DataFrame(subject=[], ses=[], task=[], run=[], file=[])  # Initialize an empty DataFrame to hold results
 
-                    get_info!(files_df, root, file)
-                end
+    list_all_eegpaths(path) = @cont begin
+        if isfile(path)
+            (any(endswith.(path, file_ending)) & all(occursin.(file_pattern, path))) && cont(path)
+        elseif isdir(path)
+            startswith(basename(path), ".") && return # skip all hidden files/ paths
+            if exclude !== nothing
+                basename(path) in (exclude...,) && return
             end
-        end
-
-        # When no specific folder is given look up whole Path    
-    else
-        for (root, dirs, files) in walkdir(bidsPath)
-            #filter(x->!startswith(x, '.'), files)
-            for file in files
-                if sum(occursin.(file_pattern, file)) >= nPattern &&
-                   (derivative && (exclude == "" || !any(occursin.(exclude, root))) ||
-                    (!derivative && !any(occursin.(exclude, root))))
-
-                    get_info!(files_df, root, file)
-                end
+            for file in readdir(path)
+                foreach(cont, list_all_eegpaths(joinpath(path, file)))
             end
         end
     end
 
+    all_paths = collect(list_all_eegpaths(abspath(bidsPath)))
+
+    # Add additional information
+    for path in all_paths
+        get_info!(files_df, path)
+    end
+
     # Check for multiple session/tasks/runs
     check_df(files_df, ses, task, run)
 
@@ -95,7 +81,7 @@
 
 #
 # get subject and file information
-function get_info!(files_df, root, file)
+function get_info!(files_df, file)
 
     # Make regex for parts
     regex_sub = r"sub-(\d+)"
@@ -113,10 +99,16 @@
         !isnothing(ses) ? ses.captures[1] : missing,
         !isnothing(task) ? task.captures[1] : missing,
         !isnothing(run) ? run.captures[1] : missing,
-        file, root))
+        file))
     return files_df
 end
 
+"""
+check_df(files_df, ses, task, run)
+
+Internal; Checks if the multiple sessions/task/runs are found if none of these are provided
+"""
+
 function check_df(files_df, ses, task, run)
     if ses === nothing && files_df.ses !== missing && length(unique(files_df.ses)) > 1
         @warn "You provided no session, however I found multiple sessions so I loaded all of them! Please check if that was intended."
@@ -209,12 +201,12 @@
     allFiles = []
     # Do some stuff @byrow, i.e. find the tsv files
     for s in eachrow(layoutDF)
-        eegFile = s.file
+        eegFile = basename(s.file)
         subStr = findlast("eeg", eegFile)[1]
         tmpFile = eegFile[begin:subStr-1] * "events.tsv"
 
         # Check if file exists
-        files = readdir(s.path) # Gives all files as Vector of strings
+        files = readdir(replace(s.file, basename(eegFile) => "")) # Gives all files as Vector of strings
 
         tmpIdx = occursin.(tmpFile, files)