From fb7e66265b8f90cc72243bd52e074e3f17fefb6f Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 24 Oct 2020 09:20:03 +0300 Subject: [PATCH 01/11] add filewalks --- .gitignore | 2 + src/fusion/filewalks.nim | 126 ++++++++++++++++++++++++++++++++++ tests/config.nims | 1 + tests/lib/tfusion/osutils.nim | 16 +++++ tests/lib/tfusion/paths.nim | 11 +++ tests/tfilewalks.nim | 75 ++++++++++++++++++++ 6 files changed, 231 insertions(+) create mode 100644 src/fusion/filewalks.nim create mode 100644 tests/lib/tfusion/osutils.nim create mode 100644 tests/lib/tfusion/paths.nim create mode 100644 tests/tfilewalks.nim diff --git a/.gitignore b/.gitignore index 0bdc3221..f2fe9304 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ nimcache/ nimblecache/ htmldocs/ + +/build diff --git a/src/fusion/filewalks.nim b/src/fusion/filewalks.nim new file mode 100644 index 00000000..89d7bd34 --- /dev/null +++ b/src/fusion/filewalks.nim @@ -0,0 +1,126 @@ +#[ +## Design rationale +* an intermediate `GlobOpt` is used to allow easier proc forwarding +* a yieldFilter, regex match etc isn't needed because caller can filter at + call site, without loss of generality, unlike `follow`; this simplifies the API. + +## Future work: +* provide a way to do error reporting, which is tricky because iteration cannot be resumed +]# + +import std/[os, algorithm, deques, macros] + +type + PathEntrySub* = object + kind*: PathComponent + path*: string + PathEntry* = object + kind*: PathComponent + path*: string + ## absolute or relative path wrt globbed dir + depth*: int + ## depth wrt GlobOpt.dir (which is at depth 0) + epilogue*: bool + GlobMode* = enum + gDfs ## depth first search + gBfs ## breadth first search + FollowCallback* = proc(entry: PathEntry): bool + SortCmpCallback* = proc (x, y: PathEntrySub): int + GlobOpt* = object + dir*: string ## root of glob + relative: bool ## when true, paths are are returned relative to `dir`, else they start with `dir` + checkDir: bool ## if true, raises `OSError` when `dir` can't be listed. Deeper + ## directories do not cause `OSError`, and currently no error reporting is done for those. + globMode: GlobMode ## controls how paths are returned + includeRoot: bool ## whether to include root `dir` + includeEpilogue: bool + ## when false, yields: someDir, + ## when true, yields: someDir, , someDir: each dir is + ## yielded a 2nd time. This is useful in applications that aggregate data over dirs. + followSymlinks: bool ## whether to follow symlinks + follow: FollowCallback + ## if not `nil`, `glob` visits `entry` if `follow(entry) == true`. + sortCmp: SortCmpCallback + ## if not `nil`, immediate children of a dir are sorted using `sortCmp` + +macro ctor(obj: untyped, a: varargs[untyped]): untyped = + ##[ + generates an object constructor call from a list of fields + ]## + # xxx expose in some `fusion/macros` + runnableExamples: + type Foo = object + a, b: int + doAssert Foo.ctor(a,b) == Foo(a: a, b: b) + result = nnkObjConstr.newTree(obj) + for ai in a: result.add nnkExprColonExpr.newTree(ai, ai) + +proc initGlobOpt*( + dir: string, relative = false, checkDir = true, globMode = gDfs, + includeRoot = false, includeEpilogue = false, followSymlinks = false, + follow: FollowCallback = nil, sortCmp: SortCmpCallback = nil): GlobOpt = + GlobOpt.ctor(dir, relative, checkDir, globMode, includeRoot, includeEpilogue, followSymlinks, follow, sortCmp) +import timn/dbgs +iterator globOpt*(opt: GlobOpt): PathEntry = + ##[ + Recursively walks `dir`. + This is more flexible than `os.walkDirRec`. + ]## + runnableExamples: + import os,sugar + if false: # see also `tfilewalks.nim` + # list hidden files of depth <= 2 + 1 in your home. + for e in glob(getHomeDir(), follow = a=>a.path.isHidden and a.depth <= 2): + if e.kind in {pcFile, pcLinkToFile}: echo e.path + + var entry = PathEntry(depth: 0, path: ".") + when nimvm: entry.kind = pcDir + else: # xxx support `symlinkExists` in nimvm + entry.kind = if symlinkExists(opt.dir): pcLinkToDir else: pcDir + var stack = initDeque[PathEntry]() + + var checkDir = opt.checkDir + if dirExists(opt.dir): + stack.addLast entry + elif checkDir: + dbg dirExists(opt.dir) + raise newException(OSError, "invalid root dir: " & opt.dir) + + var dirsLevel: seq[PathEntrySub] + while stack.len > 0: + let current = if opt.globMode == gDfs: stack.popLast() else: stack.popFirst() + entry.epilogue = current.epilogue + entry.depth = current.depth + entry.kind = current.kind + entry.path = if opt.relative: current.path else: opt.dir / current.path + normalizePath(entry.path) # pending https://github.com/timotheecour/Nim/issues/343 + + if opt.includeRoot or current.depth > 0: + yield entry + + if (current.kind == pcDir or current.kind == pcLinkToDir and opt.followSymlinks) and not current.epilogue: + if opt.follow == nil or opt.follow(current): + if opt.sortCmp != nil: + dirsLevel.setLen 0 + if opt.includeEpilogue: + stack.addLast PathEntry(depth: current.depth, path: current.path, kind: current.kind, epilogue: true) + # checkDir is still needed here in first iteration because things could + # fail for reasons other than `not dirExists`. + for k, p in walkDir(opt.dir / current.path, relative = true, checkDir = checkDir): + if opt.sortCmp != nil: + dirsLevel.add PathEntrySub(kind: k, path: p) + else: + stack.addLast PathEntry(depth: current.depth + 1, path: current.path / p, kind: k) + checkDir = false + # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong + # permissions), it'll abort iteration and there would be no way to resume iteration. + if opt.sortCmp != nil: + sort(dirsLevel, opt.sortCmp) + for i in 0.. 0 + let a2 = dir / a + if a.endsWith("/"): + createDir(a2) + dbg a2 + else: + createDir(a2.parentDir) + writeFile(a2, "") diff --git a/tests/lib/tfusion/paths.nim b/tests/lib/tfusion/paths.nim new file mode 100644 index 00000000..07877249 --- /dev/null +++ b/tests/lib/tfusion/paths.nim @@ -0,0 +1,11 @@ +import std/os + +const + fusionRoot* = currentSourcePath.parentDir.parentDir.parentDir.parentDir + buildDir* = fusionRoot / "build" + nimbleFile = fusionRoot / "fusion.nimble" + +static: doAssert nimbleFile.fileExists # sanity check + +when isMainModule: + echo buildDir diff --git a/tests/tfilewalks.nim b/tests/tfilewalks.nim new file mode 100644 index 00000000..c6cc79d8 --- /dev/null +++ b/tests/tfilewalks.nim @@ -0,0 +1,75 @@ +import std/[sugar,os,strutils,sequtils,algorithm] +from std/private/globs import nativeToUnixPath +from tfusion/paths import buildDir +from tfusion/osutils import genTestPaths + +import fusion/filewalks + +proc processAux[T](a: T): seq[string] = + a.mapIt(it.path.nativeToUnixPath) + +proc process[T](a: T): seq[string] = + a.processAux.sorted + +const dir = buildDir/"tfilewalks" + +proc main() = + # when nimvm: + # discard + # else: + # defer: removeDir(dir) + let paths = """ +d1/f1.txt +d1/d1a/f2.txt +d1/d1a/f3 +d1/d1a/d1a1/ +d1/d1b/d1b1/f4 +d2/ +f5 +""".splitLines.filter(a=>a.len>0) + + when nimvm: + discard + else: + genTestPaths(dir, paths) + + block: # follow + # filter by pcFile + doAssert toSeq(glob(dir, follow = a=>a.path.lastPathPart != "d1b", relative = true)) + .filterIt(it.kind == pcFile).process == @["d1/d1a/f2.txt", "d1/d1a/f3", "d1/f1.txt", "f5"] + # filter by pcDir + doAssert toSeq(glob(dir, relative = true)) + .filterIt(it.kind == pcDir).process == @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] + + block: # includeRoot + doAssert toSeq(glob(dir, relative = true, includeRoot = true)) + .filterIt(it.kind == pcDir).process == @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] + + block: # checkDir + doAssertRaises(OSError): discard toSeq(glob("nonexistant")) + doAssertRaises(OSError): discard toSeq(glob("f5")) + doAssert toSeq(glob("nonexistant", checkDir = false)) == @[] + + # sortCmp + proc mySort(a, b: PathEntrySub): int = cmp(a.path, b.path) + doAssert toSeq(glob(dir, relative = true, sortCmp = mySort)).processAux == + @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/f1.txt", "d2", "f5"] + + # gBfs + doAssert toSeq(glob(dir, relative = true, sortCmp = mySort, globMode = gBfs)).processAux == + @["d1", "d2", "f5", "d1/d1a", "d1/d1b", "d1/f1.txt", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1b/d1b1", "d1/d1b/d1b1/f4"] + + # includeEpilogue + doAssert toSeq(glob(dir, relative = true, sortCmp = mySort, includeEpilogue = true, includeRoot = true)).processAux == + @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1a", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/d1b/d1b1", "d1/d1b", "d1/f1.txt", "d1", "d2", "d2", "f5", "."] + echo toSeq(glob(dir)) + +# when false: +when true: + #[ + PRTEMP sort this out + pending https://github.com/nim-lang/Nim/issues/15597 and https://github.com/nim-lang/Nim/issues/15595 + ]# + static: main() + +main() From dba2b7a3daa6293753c8584c49b28e65353698a8 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 25 Oct 2020 00:52:27 +0300 Subject: [PATCH 02/11] _ --- src/fusion/filewalks.nim | 20 +++++++++++++------- tests/lib/tfusion/osutils.nim | 4 ++-- tests/tfilewalks.nim | 7 +++---- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/fusion/filewalks.nim b/src/fusion/filewalks.nim index 89d7bd34..49747e1c 100644 --- a/src/fusion/filewalks.nim +++ b/src/fusion/filewalks.nim @@ -60,7 +60,7 @@ proc initGlobOpt*( includeRoot = false, includeEpilogue = false, followSymlinks = false, follow: FollowCallback = nil, sortCmp: SortCmpCallback = nil): GlobOpt = GlobOpt.ctor(dir, relative, checkDir, globMode, includeRoot, includeEpilogue, followSymlinks, follow, sortCmp) -import timn/dbgs +# import timn/dbgs # PRTEMP iterator globOpt*(opt: GlobOpt): PathEntry = ##[ Recursively walks `dir`. @@ -83,7 +83,7 @@ iterator globOpt*(opt: GlobOpt): PathEntry = if dirExists(opt.dir): stack.addLast entry elif checkDir: - dbg dirExists(opt.dir) + # dbg dirExists(opt.dir) raise newException(OSError, "invalid root dir: " & opt.dir) var dirsLevel: seq[PathEntrySub] @@ -97,24 +97,28 @@ iterator globOpt*(opt: GlobOpt): PathEntry = if opt.includeRoot or current.depth > 0: yield entry + # let isSort = opt.sortCmp != nil # hits: bug #15595 + let isSort = not opt.sortCmp.isNil if (current.kind == pcDir or current.kind == pcLinkToDir and opt.followSymlinks) and not current.epilogue: - if opt.follow == nil or opt.follow(current): - if opt.sortCmp != nil: + # if opt.follow == nil or opt.follow(current): + if opt.follow.isNil or opt.follow(current): + if isSort: dirsLevel.setLen 0 if opt.includeEpilogue: stack.addLast PathEntry(depth: current.depth, path: current.path, kind: current.kind, epilogue: true) # checkDir is still needed here in first iteration because things could # fail for reasons other than `not dirExists`. - for k, p in walkDir(opt.dir / current.path, relative = true, checkDir = checkDir): - if opt.sortCmp != nil: + # for k, p in walkDir(opt.dir / current.path, relative = true, checkDir = checkDir): + for k, p in walkDir(opt.dir / current.path, relative = true): + if isSort: dirsLevel.add PathEntrySub(kind: k, path: p) else: stack.addLast PathEntry(depth: current.depth + 1, path: current.path / p, kind: k) checkDir = false # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong # permissions), it'll abort iteration and there would be no way to resume iteration. - if opt.sortCmp != nil: + if isSort: sort(dirsLevel, opt.sortCmp) for i in 0.. Date: Sun, 25 Oct 2020 13:30:50 +0200 Subject: [PATCH 03/11] make test robust for both vm and runtime --- src/fusion/filewalks.nim | 16 ++++------- tests/tfilewalks.nim | 62 +++++++++++++++++++--------------------- 2 files changed, 35 insertions(+), 43 deletions(-) diff --git a/src/fusion/filewalks.nim b/src/fusion/filewalks.nim index 49747e1c..652c2588 100644 --- a/src/fusion/filewalks.nim +++ b/src/fusion/filewalks.nim @@ -60,7 +60,7 @@ proc initGlobOpt*( includeRoot = false, includeEpilogue = false, followSymlinks = false, follow: FollowCallback = nil, sortCmp: SortCmpCallback = nil): GlobOpt = GlobOpt.ctor(dir, relative, checkDir, globMode, includeRoot, includeEpilogue, followSymlinks, follow, sortCmp) -# import timn/dbgs # PRTEMP + iterator globOpt*(opt: GlobOpt): PathEntry = ##[ Recursively walks `dir`. @@ -83,7 +83,6 @@ iterator globOpt*(opt: GlobOpt): PathEntry = if dirExists(opt.dir): stack.addLast entry elif checkDir: - # dbg dirExists(opt.dir) raise newException(OSError, "invalid root dir: " & opt.dir) var dirsLevel: seq[PathEntrySub] @@ -96,12 +95,10 @@ iterator globOpt*(opt: GlobOpt): PathEntry = normalizePath(entry.path) # pending https://github.com/timotheecour/Nim/issues/343 if opt.includeRoot or current.depth > 0: - yield entry - # let isSort = opt.sortCmp != nil # hits: bug #15595 - let isSort = not opt.sortCmp.isNil + yield entry # single `yield` to avoid code bloat + let isSort = not opt.sortCmp.isNil # != nil # hits: bug #15595 if (current.kind == pcDir or current.kind == pcLinkToDir and opt.followSymlinks) and not current.epilogue: - # if opt.follow == nil or opt.follow(current): if opt.follow.isNil or opt.follow(current): if isSort: dirsLevel.setLen 0 @@ -109,8 +106,7 @@ iterator globOpt*(opt: GlobOpt): PathEntry = stack.addLast PathEntry(depth: current.depth, path: current.path, kind: current.kind, epilogue: true) # checkDir is still needed here in first iteration because things could # fail for reasons other than `not dirExists`. - # for k, p in walkDir(opt.dir / current.path, relative = true, checkDir = checkDir): - for k, p in walkDir(opt.dir / current.path, relative = true): + for k, p in walkDir(opt.dir / current.path, relative = true, checkDir = checkDir): if isSort: dirsLevel.add PathEntrySub(kind: k, path: p) else: @@ -126,7 +122,5 @@ iterator globOpt*(opt: GlobOpt): PathEntry = stack.addLast PathEntry(depth: current.depth + 1, path: current.path / ai.path, kind: ai.kind) template glob*(args: varargs[untyped]): untyped = - ## convenience wrapper + ## convenience wrapper around `globOpt` globOpt(initGlobOpt(args)) - # let opt = initGlobOpt(args) - # for ai in diff --git a/tests/tfilewalks.nim b/tests/tfilewalks.nim index 2d8c43e1..334b2e9c 100644 --- a/tests/tfilewalks.nim +++ b/tests/tfilewalks.nim @@ -1,39 +1,19 @@ import std/[sugar,os,strutils,sequtils,algorithm] -# from std/private/globs import nativeToUnixPath +from std/private/globs import nativeToUnixPath + from tfusion/paths import buildDir -from tfusion/osutils import genTestPaths import fusion/filewalks proc processAux[T](a: T): seq[string] = - # a.mapIt(it.path.nativeToUnixPath) - a.mapIt(it.path) + a.mapIt(it.path.nativeToUnixPath) proc process[T](a: T): seq[string] = a.processAux.sorted const dir = buildDir/"tfilewalks" -proc main() = - # when nimvm: - # discard - # else: - # defer: removeDir(dir) - let paths = """ -d1/f1.txt -d1/d1a/f2.txt -d1/d1a/f3 -d1/d1a/d1a1/ -d1/d1b/d1b1/f4 -d2/ -f5 -""".splitLines.filter(a=>a.len>0) - - when nimvm: - discard - else: - genTestPaths(dir, paths) - +proc test() = block: # follow # filter by pcFile doAssert toSeq(glob(dir, follow = a=>a.path.lastPathPart != "d1b", relative = true)) @@ -63,12 +43,30 @@ f5 # includeEpilogue doAssert toSeq(glob(dir, relative = true, sortCmp = mySort, includeEpilogue = true, includeRoot = true)).processAux == @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1a", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/d1b/d1b1", "d1/d1b", "d1/f1.txt", "d1", "d2", "d2", "f5", "."] - echo toSeq(glob(dir)) - -when true: - #[ - pending https://github.com/nim-lang/Nim/issues/15597 and https://github.com/nim-lang/Nim/issues/15595 - ]# - static: main() -main() +when defined(fusionTfilewalksTesting): + when (NimMajor, NimMinor, NimPatch) >= (1, 5, 3): + # pending https://github.com/nim-lang/Nim/pull/15705 + static: test() + test() +else: + from tfusion/osutils import genTestPaths + import std/strformat + proc main() = + defer: removeDir(dir) + let paths = """ +d1/f1.txt +d1/d1a/f2.txt +d1/d1a/f3 +d1/d1a/d1a1/ +d1/d1b/d1b1/f4 +d2/ +f5 +""".splitLines.filter(a=>a.len>0) + genTestPaths(dir, paths) + const nim = getCurrentCompilerExe() + const input = currentSourcePath() + let cmd = &"{nim} c -r -d:fusionTfilewalksTesting {input}" + let status = execShellCmd(cmd) + doAssert status == 0 + main() From 56b628e0aef77380ee5f874607ca3e4f2f44eba3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 25 Oct 2020 14:09:22 +0200 Subject: [PATCH 04/11] fix test --- tests/tfilewalks.nim | 78 +++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/tests/tfilewalks.nim b/tests/tfilewalks.nim index 334b2e9c..4550918f 100644 --- a/tests/tfilewalks.nim +++ b/tests/tfilewalks.nim @@ -1,57 +1,57 @@ -import std/[sugar,os,strutils,sequtils,algorithm] -from std/private/globs import nativeToUnixPath - +import std/[os,sequtils,sugar] from tfusion/paths import buildDir -import fusion/filewalks +const dir = buildDir/"tfilewalks" -proc processAux[T](a: T): seq[string] = - a.mapIt(it.path.nativeToUnixPath) +when defined(fusionTfilewalksTesting): + import std/[sugar,os,strutils,sequtils,algorithm] + from std/private/globs import nativeToUnixPath + import fusion/filewalks -proc process[T](a: T): seq[string] = - a.processAux.sorted + proc processAux[T](a: T): seq[string] = + a.mapIt(it.path.nativeToUnixPath) -const dir = buildDir/"tfilewalks" + proc process[T](a: T): seq[string] = + a.processAux.sorted -proc test() = - block: # follow - # filter by pcFile - doAssert toSeq(glob(dir, follow = a=>a.path.lastPathPart != "d1b", relative = true)) - .filterIt(it.kind == pcFile).process == @["d1/d1a/f2.txt", "d1/d1a/f3", "d1/f1.txt", "f5"] - # filter by pcDir - doAssert toSeq(glob(dir, relative = true)) - .filterIt(it.kind == pcDir).process == @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] + proc test() = + block: # follow + # filter by pcFile + doAssert toSeq(glob(dir, follow = a=>a.path.lastPathPart != "d1b", relative = true)) + .filterIt(it.kind == pcFile).process == @["d1/d1a/f2.txt", "d1/d1a/f3", "d1/f1.txt", "f5"] + # filter by pcDir + doAssert toSeq(glob(dir, relative = true)) + .filterIt(it.kind == pcDir).process == @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] - block: # includeRoot - doAssert toSeq(glob(dir, relative = true, includeRoot = true)) - .filterIt(it.kind == pcDir).process == @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] + block: # includeRoot + doAssert toSeq(glob(dir, relative = true, includeRoot = true)) + .filterIt(it.kind == pcDir).process == @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] - block: # checkDir - doAssertRaises(OSError): discard toSeq(glob("nonexistant")) - doAssertRaises(OSError): discard toSeq(glob("f5")) - doAssert toSeq(glob("nonexistant", checkDir = false)) == @[] + block: # checkDir + doAssertRaises(OSError): discard toSeq(glob("nonexistant")) + doAssertRaises(OSError): discard toSeq(glob("f5")) + doAssert toSeq(glob("nonexistant", checkDir = false)) == @[] - # sortCmp - proc mySort(a, b: PathEntrySub): int = cmp(a.path, b.path) - doAssert toSeq(glob(dir, relative = true, sortCmp = mySort)).processAux == - @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/f1.txt", "d2", "f5"] + # sortCmp + proc mySort(a, b: PathEntrySub): int = cmp(a.path, b.path) + doAssert toSeq(glob(dir, relative = true, sortCmp = mySort)).processAux == + @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/f1.txt", "d2", "f5"] - # gBfs - doAssert toSeq(glob(dir, relative = true, sortCmp = mySort, globMode = gBfs)).processAux == - @["d1", "d2", "f5", "d1/d1a", "d1/d1b", "d1/f1.txt", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1b/d1b1", "d1/d1b/d1b1/f4"] + # gBfs + doAssert toSeq(glob(dir, relative = true, sortCmp = mySort, globMode = gBfs)).processAux == + @["d1", "d2", "f5", "d1/d1a", "d1/d1b", "d1/f1.txt", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1b/d1b1", "d1/d1b/d1b1/f4"] - # includeEpilogue - doAssert toSeq(glob(dir, relative = true, sortCmp = mySort, includeEpilogue = true, includeRoot = true)).processAux == - @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1a", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/d1b/d1b1", "d1/d1b", "d1/f1.txt", "d1", "d2", "d2", "f5", "."] + # includeEpilogue + doAssert toSeq(glob(dir, relative = true, sortCmp = mySort, includeEpilogue = true, includeRoot = true)).processAux == + @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1a", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/d1b/d1b1", "d1/d1b", "d1/f1.txt", "d1", "d2", "d2", "f5", "."] -when defined(fusionTfilewalksTesting): when (NimMajor, NimMinor, NimPatch) >= (1, 5, 3): # pending https://github.com/nim-lang/Nim/pull/15705 static: test() test() else: from tfusion/osutils import genTestPaths - import std/strformat + import std/[strformat,strutils] proc main() = defer: removeDir(dir) let paths = """ @@ -67,6 +67,8 @@ f5 const nim = getCurrentCompilerExe() const input = currentSourcePath() let cmd = &"{nim} c -r -d:fusionTfilewalksTesting {input}" - let status = execShellCmd(cmd) - doAssert status == 0 + when (NimMajor, NimMinor, NimPatch) >= (1, 4, 0): + # for `nativeToUnixPath` + let status = execShellCmd(cmd) + doAssert status == 0 main() From 089b74818b18a41b64924ce936495ad537b507ce Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 25 Oct 2020 14:21:03 +0200 Subject: [PATCH 05/11] rename glob to walkPaths --- src/fusion/filewalks.nim | 42 ++++++++++++++++++++-------------------- tests/tfilewalks.nim | 20 +++++++++---------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/fusion/filewalks.nim b/src/fusion/filewalks.nim index 652c2588..c12cbb38 100644 --- a/src/fusion/filewalks.nim +++ b/src/fusion/filewalks.nim @@ -1,6 +1,6 @@ #[ ## Design rationale -* an intermediate `GlobOpt` is used to allow easier proc forwarding +* an intermediate `WalkOpt` is used to allow easier proc forwarding * a yieldFilter, regex match etc isn't needed because caller can filter at call site, without loss of generality, unlike `follow`; this simplifies the API. @@ -17,21 +17,21 @@ type PathEntry* = object kind*: PathComponent path*: string - ## absolute or relative path wrt globbed dir + ## absolute or relative path wrt walked dir depth*: int - ## depth wrt GlobOpt.dir (which is at depth 0) + ## depth wrt WalkOpt.dir (which is at depth 0) epilogue*: bool - GlobMode* = enum - gDfs ## depth first search - gBfs ## breadth first search + WalkMode* = enum + dfs ## depth first search + bfs ## breadth first search FollowCallback* = proc(entry: PathEntry): bool SortCmpCallback* = proc (x, y: PathEntrySub): int - GlobOpt* = object - dir*: string ## root of glob + WalkOpt* = object + dir*: string ## root of walk relative: bool ## when true, paths are are returned relative to `dir`, else they start with `dir` checkDir: bool ## if true, raises `OSError` when `dir` can't be listed. Deeper ## directories do not cause `OSError`, and currently no error reporting is done for those. - globMode: GlobMode ## controls how paths are returned + walkMode: WalkMode ## controls how paths are returned includeRoot: bool ## whether to include root `dir` includeEpilogue: bool ## when false, yields: someDir, @@ -39,7 +39,7 @@ type ## yielded a 2nd time. This is useful in applications that aggregate data over dirs. followSymlinks: bool ## whether to follow symlinks follow: FollowCallback - ## if not `nil`, `glob` visits `entry` if `follow(entry) == true`. + ## if not `nil`, `walkPath` visits `entry` if `follow(entry) == true`. sortCmp: SortCmpCallback ## if not `nil`, immediate children of a dir are sorted using `sortCmp` @@ -55,13 +55,13 @@ macro ctor(obj: untyped, a: varargs[untyped]): untyped = result = nnkObjConstr.newTree(obj) for ai in a: result.add nnkExprColonExpr.newTree(ai, ai) -proc initGlobOpt*( - dir: string, relative = false, checkDir = true, globMode = gDfs, +proc initWalkOpt*( + dir: string, relative = false, checkDir = true, walkMode = dfs, includeRoot = false, includeEpilogue = false, followSymlinks = false, - follow: FollowCallback = nil, sortCmp: SortCmpCallback = nil): GlobOpt = - GlobOpt.ctor(dir, relative, checkDir, globMode, includeRoot, includeEpilogue, followSymlinks, follow, sortCmp) + follow: FollowCallback = nil, sortCmp: SortCmpCallback = nil): WalkOpt = + WalkOpt.ctor(dir, relative, checkDir, walkMode, includeRoot, includeEpilogue, followSymlinks, follow, sortCmp) -iterator globOpt*(opt: GlobOpt): PathEntry = +iterator walkPathsOpt*(opt: WalkOpt): PathEntry = ##[ Recursively walks `dir`. This is more flexible than `os.walkDirRec`. @@ -70,7 +70,7 @@ iterator globOpt*(opt: GlobOpt): PathEntry = import os,sugar if false: # see also `tfilewalks.nim` # list hidden files of depth <= 2 + 1 in your home. - for e in glob(getHomeDir(), follow = a=>a.path.isHidden and a.depth <= 2): + for e in walkPaths(getHomeDir(), follow = a=>a.path.isHidden and a.depth <= 2): if e.kind in {pcFile, pcLinkToFile}: echo e.path var entry = PathEntry(depth: 0, path: ".") @@ -87,7 +87,7 @@ iterator globOpt*(opt: GlobOpt): PathEntry = var dirsLevel: seq[PathEntrySub] while stack.len > 0: - let current = if opt.globMode == gDfs: stack.popLast() else: stack.popFirst() + let current = if opt.walkMode == dfs: stack.popLast() else: stack.popFirst() entry.epilogue = current.epilogue entry.depth = current.depth entry.kind = current.kind @@ -117,10 +117,10 @@ iterator globOpt*(opt: GlobOpt): PathEntry = if isSort: sort(dirsLevel, opt.sortCmp) for i in 0..a.path.lastPathPart != "d1b", relative = true)) + doAssert toSeq(walkPaths(dir, follow = a=>a.path.lastPathPart != "d1b", relative = true)) .filterIt(it.kind == pcFile).process == @["d1/d1a/f2.txt", "d1/d1a/f3", "d1/f1.txt", "f5"] # filter by pcDir - doAssert toSeq(glob(dir, relative = true)) + doAssert toSeq(walkPaths(dir, relative = true)) .filterIt(it.kind == pcDir).process == @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] block: # includeRoot - doAssert toSeq(glob(dir, relative = true, includeRoot = true)) + doAssert toSeq(walkPaths(dir, relative = true, includeRoot = true)) .filterIt(it.kind == pcDir).process == @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] block: # checkDir - doAssertRaises(OSError): discard toSeq(glob("nonexistant")) - doAssertRaises(OSError): discard toSeq(glob("f5")) - doAssert toSeq(glob("nonexistant", checkDir = false)) == @[] + doAssertRaises(OSError): discard toSeq(walkPaths("nonexistant")) + doAssertRaises(OSError): discard toSeq(walkPaths("f5")) + doAssert toSeq(walkPaths("nonexistant", checkDir = false)) == @[] # sortCmp proc mySort(a, b: PathEntrySub): int = cmp(a.path, b.path) - doAssert toSeq(glob(dir, relative = true, sortCmp = mySort)).processAux == + doAssert toSeq(walkPaths(dir, relative = true, sortCmp = mySort)).processAux == @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/f1.txt", "d2", "f5"] - # gBfs - doAssert toSeq(glob(dir, relative = true, sortCmp = mySort, globMode = gBfs)).processAux == + # bfs + doAssert toSeq(walkPaths(dir, relative = true, sortCmp = mySort, walkMode = bfs)).processAux == @["d1", "d2", "f5", "d1/d1a", "d1/d1b", "d1/f1.txt", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1b/d1b1", "d1/d1b/d1b1/f4"] # includeEpilogue - doAssert toSeq(glob(dir, relative = true, sortCmp = mySort, includeEpilogue = true, includeRoot = true)).processAux == + doAssert toSeq(walkPaths(dir, relative = true, sortCmp = mySort, includeEpilogue = true, includeRoot = true)).processAux == @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1a", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/d1b/d1b1", "d1/d1b", "d1/f1.txt", "d1", "d2", "d2", "f5", "."] when (NimMajor, NimMinor, NimPatch) >= (1, 5, 3): From 80a34f726ab50038304782c8f2ed0bbeb4817470 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 25 Oct 2020 14:22:20 +0200 Subject: [PATCH 06/11] fix windows test --- tests/tfilewalks.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tfilewalks.nim b/tests/tfilewalks.nim index 74cab10f..944c46b0 100644 --- a/tests/tfilewalks.nim +++ b/tests/tfilewalks.nim @@ -66,7 +66,7 @@ f5 genTestPaths(dir, paths) const nim = getCurrentCompilerExe() const input = currentSourcePath() - let cmd = &"{nim} c -r -d:fusionTfilewalksTesting {input}" + let cmd = &"{nim} r -d:fusionTfilewalksTesting {input}" when (NimMajor, NimMinor, NimPatch) >= (1, 4, 0): # for `nativeToUnixPath` let status = execShellCmd(cmd) From 6fdc5f5a55e8c3ad7558e296832ce76c0e075b9b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 25 Oct 2020 17:34:46 +0200 Subject: [PATCH 07/11] cleanup --- tests/lib/tfusion/osutils.nim | 3 +-- tests/lib/tfusion/paths.nim | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/lib/tfusion/osutils.nim b/tests/lib/tfusion/osutils.nim index 2bb2b9e4..66eed75a 100644 --- a/tests/lib/tfusion/osutils.nim +++ b/tests/lib/tfusion/osutils.nim @@ -1,5 +1,5 @@ import std/[os,strutils] -# import timn/dbgs + proc genTestPaths*(dir: string, paths: seq[string]) = ## generates a filesystem rooted under `dir` from given relative `paths`. ## `paths` ending in `/` are treated as directories. @@ -10,7 +10,6 @@ proc genTestPaths*(dir: string, paths: seq[string]) = let a2 = dir / a if a.endsWith("/"): createDir(a2) - # dbg a2 else: createDir(a2.parentDir) writeFile(a2, "") diff --git a/tests/lib/tfusion/paths.nim b/tests/lib/tfusion/paths.nim index 07877249..4894e82d 100644 --- a/tests/lib/tfusion/paths.nim +++ b/tests/lib/tfusion/paths.nim @@ -3,9 +3,6 @@ import std/os const fusionRoot* = currentSourcePath.parentDir.parentDir.parentDir.parentDir buildDir* = fusionRoot / "build" - nimbleFile = fusionRoot / "fusion.nimble" + nimbleFile* = fusionRoot / "fusion.nimble" static: doAssert nimbleFile.fileExists # sanity check - -when isMainModule: - echo buildDir From 8520e1f48834401a0b8c62c58f12468619ba1f13 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 26 Oct 2020 11:18:12 +0200 Subject: [PATCH 08/11] update now that https://github.com/nim-lang/Nim/pull/15705 was merged --- tests/tfilewalks.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/tfilewalks.nim b/tests/tfilewalks.nim index 944c46b0..12ebbcc6 100644 --- a/tests/tfilewalks.nim +++ b/tests/tfilewalks.nim @@ -45,8 +45,7 @@ when defined(fusionTfilewalksTesting): doAssert toSeq(walkPaths(dir, relative = true, sortCmp = mySort, includeEpilogue = true, includeRoot = true)).processAux == @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1a", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/d1b/d1b1", "d1/d1b", "d1/f1.txt", "d1", "d2", "d2", "f5", "."] - when (NimMajor, NimMinor, NimPatch) >= (1, 5, 3): - # pending https://github.com/nim-lang/Nim/pull/15705 + when (NimMajor, NimMinor, NimPatch) >= (1, 5, 1): static: test() test() else: From b129155dccff80a6ccd821c4d33302ef44d1c1d9 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 26 Oct 2020 22:32:17 +0200 Subject: [PATCH 09/11] fixup --- src/fusion/filewalks.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fusion/filewalks.nim b/src/fusion/filewalks.nim index c12cbb38..91150b6f 100644 --- a/src/fusion/filewalks.nim +++ b/src/fusion/filewalks.nim @@ -97,9 +97,9 @@ iterator walkPathsOpt*(opt: WalkOpt): PathEntry = if opt.includeRoot or current.depth > 0: yield entry # single `yield` to avoid code bloat - let isSort = not opt.sortCmp.isNil # != nil # hits: bug #15595 + let isSort = opt.sortCmp != nil if (current.kind == pcDir or current.kind == pcLinkToDir and opt.followSymlinks) and not current.epilogue: - if opt.follow.isNil or opt.follow(current): + if opt.follow == nil or opt.follow(current): if isSort: dirsLevel.setLen 0 if opt.includeEpilogue: From 69f4375dfc0bc3f7c401e3715cb356b0a7bd817e Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 15 Jan 2021 21:50:53 -0800 Subject: [PATCH 10/11] fixup --- tests/tfilewalks.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tfilewalks.nim b/tests/tfilewalks.nim index 12ebbcc6..b4d76552 100644 --- a/tests/tfilewalks.nim +++ b/tests/tfilewalks.nim @@ -4,7 +4,7 @@ from tfusion/paths import buildDir const dir = buildDir/"tfilewalks" when defined(fusionTfilewalksTesting): - import std/[sugar,os,strutils,sequtils,algorithm] + import std/[sugar,os,sequtils,algorithm] from std/private/globs import nativeToUnixPath import fusion/filewalks From a404a7842fd4d2692c0885019a3bfec7afb32428 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 16 Jan 2021 14:18:13 -0800 Subject: [PATCH 11/11] address comments --- src/fusion/filewalks.nim | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/fusion/filewalks.nim b/src/fusion/filewalks.nim index 91150b6f..128b97af 100644 --- a/src/fusion/filewalks.nim +++ b/src/fusion/filewalks.nim @@ -17,18 +17,18 @@ type PathEntry* = object kind*: PathComponent path*: string - ## absolute or relative path wrt walked dir + ## absolute or relative path with respect to walked dir depth*: int - ## depth wrt WalkOpt.dir (which is at depth 0) + ## depth with respect to WalkOpt.dir (which is at depth 0) epilogue*: bool WalkMode* = enum dfs ## depth first search bfs ## breadth first search FollowCallback* = proc(entry: PathEntry): bool - SortCmpCallback* = proc (x, y: PathEntrySub): int + SortCmpCallback* = proc(x, y: PathEntrySub): int WalkOpt* = object dir*: string ## root of walk - relative: bool ## when true, paths are are returned relative to `dir`, else they start with `dir` + relative: bool ## when true, paths are returned relative to `dir`. Otherwise they start with `dir` checkDir: bool ## if true, raises `OSError` when `dir` can't be listed. Deeper ## directories do not cause `OSError`, and currently no error reporting is done for those. walkMode: WalkMode ## controls how paths are returned @@ -45,7 +45,7 @@ type macro ctor(obj: untyped, a: varargs[untyped]): untyped = ##[ - generates an object constructor call from a list of fields + Generates an object constructor call from a list of fields. ]## # xxx expose in some `fusion/macros` runnableExamples: @@ -104,7 +104,7 @@ iterator walkPathsOpt*(opt: WalkOpt): PathEntry = dirsLevel.setLen 0 if opt.includeEpilogue: stack.addLast PathEntry(depth: current.depth, path: current.path, kind: current.kind, epilogue: true) - # checkDir is still needed here in first iteration because things could + # `checkDir` is still needed here in first iteration because things could # fail for reasons other than `not dirExists`. for k, p in walkDir(opt.dir / current.path, relative = true, checkDir = checkDir): if isSort: @@ -122,5 +122,5 @@ iterator walkPathsOpt*(opt: WalkOpt): PathEntry = stack.addLast PathEntry(depth: current.depth + 1, path: current.path / ai.path, kind: ai.kind) template walkPaths*(args: varargs[untyped]): untyped = - ## convenience wrapper around `walkPathsOpt` + ## Convenience wrapper around `walkPathsOpt`. walkPathsOpt(initWalkOpt(args))