From 939d04e1fbe70ec75558cee640b058533abd34ca Mon Sep 17 00:00:00 2001 From: Justin Espedal Date: Wed, 1 Dec 2021 17:31:43 +0900 Subject: [PATCH 1/2] Track System file operations for safe deletion of untouched files We avoid unnecessary file operations where possible, because avoiding file IO is faster, and it also helps build systems that depend on file modification times to detect modification. Unfortunately, this means we can't just delete an entire folder and start from scratch again. Stale files may still exist after re-copying a set of files where some of the original files have since been deleted. By tracking all files in a folder before starting file operations, and determining which files are still live based on the file operations performed, we can then safely delete all files which are determined to be stale. markAllFilesInFolderAsUntouched is to be called before a batch of file operations targeting a specific folder. If any file operations are performed by third-party libraries, those should be accompanied by markFileAsTouched. Finally, deleteUntouchedFiles can be called to remove stale files from the destination folder. --- src/hxp/System.hx | 69 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/hxp/System.hx b/src/hxp/System.hx index 0e43daf..3bf5647 100644 --- a/src/hxp/System.hx +++ b/src/hxp/System.hx @@ -22,6 +22,7 @@ class System private static var _hostPlatform:HostPlatform; private static var _isText:Map; private static var _processorCores:Int = 0; + private static var _untouchedFiles:Map; private static function __init__():Void { @@ -220,6 +221,8 @@ class System if (textFile) { + markFileAsTouched(destination); + // Log.info ("", " - \x1b[1mProcessing template file:\x1b[0m " + source + " \x1b[3;37m->\x1b[0m " + destination); var fileContents:String = File.getContent(source); @@ -276,6 +279,8 @@ class System { // allFiles.push (destination); + markFileAsTouched(destination); + if (!isNewer(source, destination)) { return; @@ -306,6 +311,18 @@ class System } } + public static function deleteUntouchedFiles():Void + { + var paths = [for (key in _untouchedFiles.keys()) key]; + paths.sort((a, b) -> a < b ? -1 : a > b ? 1 : 0); + for (path in paths) + { + Log.info("", " - \x1b[1mDeleting file:\x1b[0m " + path); + FileSystem.deleteFile(path); + } + _untouchedFiles = null; + } + public static function findTemplate(templatePaths:Array, path:String, warnIfNotFound:Bool = true):String { var matches = findTemplates(templatePaths, path, warnIfNotFound); @@ -351,6 +368,8 @@ class System public static function deleteFile(path:String) { + markFileAsTouched(path); + try { if (FileSystem.exists(path) && !FileSystem.isDirectory(path)) @@ -551,6 +570,8 @@ class System public static function linkFile(source:String, destination:String, symbolic:Bool = true, overwrite:Bool = false) { + markFileAsTouched(destination); + if (!isNewer(source, destination)) { return; @@ -592,6 +613,52 @@ class System mkdir(directory); } + public static function markAllFilesInFolderAsUntouched(path:String, pathsToIgnore:Array = null):Void + { + _untouchedFiles = []; + var ignoredPathsMap:Map = null; + if (pathsToIgnore != null) ignoredPathsMap = [for (pathToIgnore in pathsToIgnore) path + "/" + pathToIgnore => true]; + if (FileSystem.exists(path)) + { + _scanFilesRecursively(path, _untouchedFiles, ignoredPathsMap); + } + } + + private static function _scanFilesRecursively(path:String, map:Map, ignorePaths:Map):Void + { + for (file in FileSystem.readDirectory(path)) + { + if (file.charAt(0) == ".") + { + continue; + } + + var fullPath = Path.combine(path, file); + + if (ignorePaths.exists(fullPath)) + { + continue; + } + + if (FileSystem.isDirectory(fullPath)) + { + _scanFilesRecursively(fullPath, map, ignorePaths); + } + else + { + map.set(fullPath, true); + } + } + } + + public static function markFileAsTouched(path:String):Void + { + if (_untouchedFiles != null) + { + _untouchedFiles.remove(path); + } + } + public static function mkdir(directory:String):Void { try @@ -889,6 +956,7 @@ class System } else { + markFileAsTouched(path); FileSystem.deleteFile(path); } } @@ -945,6 +1013,7 @@ class System if (index > -1) { output = output.substr(0, index) + replacement + output.substr(index + replaceString.length); + markFileAsTouched(sourcePath); File.saveContent(sourcePath, output); } } From 8a34063574ee5c045586a12a41523a1a8da122bf Mon Sep 17 00:00:00 2001 From: Justin Espedal Date: Wed, 1 Dec 2021 20:29:33 +0900 Subject: [PATCH 2/2] Allow system file operation tracking to be put in an unknown state. If certain external file operations cannot be tracked, markAllFilesStateAsUnknown should be used. This will prevent any files from being deleted. The project should be manually cleaned by the user in this case. --- src/hxp/System.hx | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/hxp/System.hx b/src/hxp/System.hx index 3bf5647..d3a78e8 100644 --- a/src/hxp/System.hx +++ b/src/hxp/System.hx @@ -22,6 +22,7 @@ class System private static var _hostPlatform:HostPlatform; private static var _isText:Map; private static var _processorCores:Int = 0; + private static var _unknownFileState:Bool = false; private static var _untouchedFiles:Map; private static function __init__():Void @@ -313,6 +314,13 @@ class System public static function deleteUntouchedFiles():Void { + if (_unknownFileState) + { + _unknownFileState = false; + _untouchedFiles = null; + return; + } + var paths = [for (key in _untouchedFiles.keys()) key]; paths.sort((a, b) -> a < b ? -1 : a > b ? 1 : 0); for (path in paths) @@ -616,6 +624,7 @@ class System public static function markAllFilesInFolderAsUntouched(path:String, pathsToIgnore:Array = null):Void { _untouchedFiles = []; + _unknownFileState = false; var ignoredPathsMap:Map = null; if (pathsToIgnore != null) ignoredPathsMap = [for (pathToIgnore in pathsToIgnore) path + "/" + pathToIgnore => true]; if (FileSystem.exists(path)) @@ -651,6 +660,11 @@ class System } } + public static function markAllFilesStateAsUnknown():Void + { + _unknownFileState = true; + } + public static function markFileAsTouched(path:String):Void { if (_untouchedFiles != null)