From 5010f5d03e604569fc964764146b6c56d5276df8 Mon Sep 17 00:00:00 2001 From: "Ilia (Elias) Motornyi" Date: Sun, 17 Nov 2024 00:57:41 +0200 Subject: [PATCH] NOVA project upload algorithm. Fixes #338 --- .gitignore | 3 +- gradle.properties | 2 +- .../com/jetbrains/micropython/nova/actions.kt | 19 ++- .../run/MicroPythonRunConfiguration.kt | 124 +++++++++++------- test-projects/.gitignore | 6 + test-projects/layout-a/.gitignore | 2 + test-projects/layout-a/.idea/layout-a.iml | 23 ++++ test-projects/layout-a/.idea/modules.xml | 8 ++ .../.idea/runConfigurations/Flash_project.xml | 6 + .../.ignored-as-dot/not-uploadable-c.txt | 0 .../ignored-as-excluded/not-uploadable-a.txt | 0 .../ignored-as-test/not-uploadable-b.txt | 0 .../layout-a/lib/inner/uploadable-c.py | 0 test-projects/layout-a/lib/uploadable-b.py | 0 test-projects/layout-a/main.py | 16 +++ test-projects/layout-a/uploadable-a.py | 0 test-projects/layout-b/.gitignore | 1 + test-projects/layout-b/.idea/.gitignore | 3 + test-projects/layout-b/.idea/layout-b.iml | 23 ++++ test-projects/layout-b/.idea/modules.xml | 8 ++ .../.ignored-as-dot/not-uploadable-c.txt | 0 .../ignored-as-excluded/not-uploadable-a.txt | 0 .../ignored-as-test/not-uploadable-b.txt | 0 .../lib-ignored/inner/not-uploadable-e.py | 0 .../layout-b/lib-ignored/not-uploadable-d.py | 0 test-projects/layout-b/not-uploadable-root.py | 16 +++ test-projects/layout-b/src-2/uploadable-d.py | 1 + .../layout-b/src/lib/inner/uploadable-c.py | 0 .../layout-b/src/lib/uploadable-b.py | 0 test-projects/layout-b/src/main.py | 16 +++ test-projects/layout-b/src/uploadable-a.py | 0 31 files changed, 217 insertions(+), 60 deletions(-) create mode 100644 test-projects/.gitignore create mode 100644 test-projects/layout-a/.gitignore create mode 100644 test-projects/layout-a/.idea/layout-a.iml create mode 100644 test-projects/layout-a/.idea/modules.xml create mode 100644 test-projects/layout-a/.idea/runConfigurations/Flash_project.xml create mode 100644 test-projects/layout-a/.ignored-as-dot/not-uploadable-c.txt create mode 100644 test-projects/layout-a/ignored-as-excluded/not-uploadable-a.txt create mode 100644 test-projects/layout-a/ignored-as-test/not-uploadable-b.txt create mode 100644 test-projects/layout-a/lib/inner/uploadable-c.py create mode 100644 test-projects/layout-a/lib/uploadable-b.py create mode 100644 test-projects/layout-a/main.py create mode 100644 test-projects/layout-a/uploadable-a.py create mode 100644 test-projects/layout-b/.gitignore create mode 100644 test-projects/layout-b/.idea/.gitignore create mode 100644 test-projects/layout-b/.idea/layout-b.iml create mode 100644 test-projects/layout-b/.idea/modules.xml create mode 100644 test-projects/layout-b/.ignored-as-dot/not-uploadable-c.txt create mode 100644 test-projects/layout-b/ignored-as-excluded/not-uploadable-a.txt create mode 100644 test-projects/layout-b/ignored-as-test/not-uploadable-b.txt create mode 100644 test-projects/layout-b/lib-ignored/inner/not-uploadable-e.py create mode 100644 test-projects/layout-b/lib-ignored/not-uploadable-d.py create mode 100644 test-projects/layout-b/not-uploadable-root.py create mode 100644 test-projects/layout-b/src-2/uploadable-d.py create mode 100644 test-projects/layout-b/src/lib/inner/uploadable-c.py create mode 100644 test-projects/layout-b/src/lib/uploadable-b.py create mode 100644 test-projects/layout-b/src/main.py create mode 100644 test-projects/layout-b/src/uploadable-a.py diff --git a/.gitignore b/.gitignore index 609939eb..86e6ccd7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ -*.iml -.idea/ +/.idea/ !/.idea/runConfigurations/ .gradle/ build/ diff --git a/gradle.properties b/gradle.properties index eef3c0f1..b7feeccc 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=2.0.0.pre-alpha11-2024.3 +version=2.0.0.pre-alpha12-2024.3 platformType=PC platformVersion=243-EAP-SNAPSHOT pythonPlugin=PythonCore:EAP-SNAPSHOT diff --git a/src/main/kotlin/com/jetbrains/micropython/nova/actions.kt b/src/main/kotlin/com/jetbrains/micropython/nova/actions.kt index 35741c08..f00955c6 100644 --- a/src/main/kotlin/com/jetbrains/micropython/nova/actions.kt +++ b/src/main/kotlin/com/jetbrains/micropython/nova/actions.kt @@ -265,18 +265,23 @@ open class UploadFile() : DumbAwareAction("Upload File(s) to Micropython device" override fun update(e: AnActionEvent) { val project = e.project val file = e.getData(CommonDataKeys.VIRTUAL_FILE) - var enabled = false - if (project != null && file?.isInLocalFileSystem == true) { - enabled = ModuleUtil.findModuleForFile(file, project)?.microPythonFacet != null + if (project != null + && file?.isInLocalFileSystem == true + && ModuleUtil.findModuleForFile(file, project)?.microPythonFacet != null + ) { + e.presentation.text = + if (file.isDirectory) "Upload Directory to Micropython device" else "Upload File to Micropython device" + } else { + e.presentation.isEnabledAndVisible = false } - e.presentation.isEnabledAndVisible = enabled } override fun actionPerformed(e: AnActionEvent) { FileDocumentManager.getInstance().saveAllDocuments() - val files = e.getData(CommonDataKeys.VIRTUAL_FILE_ARRAY) - if (files.isNullOrEmpty()) return - MicroPythonRunConfiguration.uploadMultipleFiles(e.project ?: return, null, files.toList()) + val file = e.getData(CommonDataKeys.VIRTUAL_FILE) + if (file != null) { + MicroPythonRunConfiguration.uploadFileOrFolder(e.project ?: return, file) + } } } diff --git a/src/main/kotlin/com/jetbrains/micropython/run/MicroPythonRunConfiguration.kt b/src/main/kotlin/com/jetbrains/micropython/run/MicroPythonRunConfiguration.kt index 8fd2963a..9e078c1b 100644 --- a/src/main/kotlin/com/jetbrains/micropython/run/MicroPythonRunConfiguration.kt +++ b/src/main/kotlin/com/jetbrains/micropython/run/MicroPythonRunConfiguration.kt @@ -25,21 +25,21 @@ import com.intellij.execution.configurations.RunProfileState import com.intellij.execution.configurations.RuntimeConfigurationError import com.intellij.execution.runners.ExecutionEnvironment import com.intellij.facet.ui.ValidationResult -import com.intellij.openapi.actionSystem.LangDataKeys +import com.intellij.openapi.application.EDT import com.intellij.openapi.fileEditor.FileDocumentManager +import com.intellij.openapi.fileTypes.FileTypeRegistry import com.intellij.openapi.module.Module import com.intellij.openapi.module.ModuleUtil import com.intellij.openapi.options.ShowSettingsUtil +import com.intellij.openapi.progress.checkCanceled import com.intellij.openapi.project.Project -import com.intellij.openapi.project.guessModuleDir import com.intellij.openapi.project.guessProjectDir +import com.intellij.openapi.project.modules import com.intellij.openapi.project.rootManager -import com.intellij.openapi.roots.ModuleRootManager import com.intellij.openapi.roots.ui.configuration.ProjectSettingsService import com.intellij.openapi.vfs.StandardFileSystems import com.intellij.openapi.vfs.VfsUtil import com.intellij.openapi.vfs.VirtualFile -import com.intellij.openapi.vfs.isFile import com.intellij.platform.util.progress.reportSequentialProgress import com.intellij.project.stateStore import com.intellij.util.PathUtil @@ -49,9 +49,12 @@ import com.jetbrains.micropython.nova.performReplAction import com.jetbrains.micropython.settings.MicroPythonProjectConfigurable import com.jetbrains.micropython.settings.microPythonFacet import com.jetbrains.python.sdk.PythonSdkUtil +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import org.jdom.Element import java.nio.file.Path + /** * @author Mikhail Golubev */ @@ -69,11 +72,14 @@ class MicroPythonRunConfiguration(project: Project, factory: ConfigurationFactor override fun getState(executor: Executor, environment: ExecutionEnvironment): RunProfileState? { - val uploadPath = if (!path.isEmpty()) path else project.basePath ?: return null - val toUpload = listOf(VfsUtil.findFile(Path.of(uploadPath), true) ?: return null) - val currentModule = environment.dataContext?.getData(LangDataKeys.MODULE) - - if (uploadMultipleFiles(project, currentModule, toUpload)) { + val success: Boolean + if (path.isBlank()) { + success = uploadProject(project) + } else { + val toUpload = VfsUtil.findFile(Path.of(project.basePath ?: return null), true) ?: return null + success = uploadFileOrFolder(project, toUpload) + } + if (success) { val fileSystemWidget = fileSystemWidget(project) if(resetOnSuccess)fileSystemWidget?.reset() if(runReplOnSuccess) fileSystemWidget?.activateRepl() @@ -145,56 +151,74 @@ class MicroPythonRunConfiguration(project: Project, factory: ConfigurationFactor } companion object { - private fun getClosestRoot(file: VirtualFile, roots: Set, module: Module): VirtualFile? { - var parent: VirtualFile? = file - while (parent != null) { - if (parent in roots) { - break + + private fun VirtualFile.leadingDot() = this.name.startsWith(".") + + fun uploadFileOrFolder(project: Project, toUpload: VirtualFile): Boolean { + FileDocumentManager.getInstance().saveAllDocuments() + performUpload(project,listOf(toUpload.name to toUpload)) + return false + } + + private fun collectUploadables(project: Project): Set { + return project.modules.flatMap { module -> + val moduleRoots = module.rootManager + .contentEntries + .flatMap { it.sourceFolders.asSequence() } + .mapNotNull { if (!it.isTestSource) it.file else null } + .filter { !it.leadingDot() } + .toMutableList() + + if (moduleRoots.isEmpty()) { + module.rootManager.contentRoots.filterTo(moduleRoots) { it.isDirectory && !it.leadingDot() } + } + moduleRoots + }.toSet() + } + + private fun collectExcluded(project: Project): Set { + val ideaDir = project.stateStore.directoryStorePath?.let { VfsUtil.findFile(it, false) } + val excludes = if (ideaDir == null) mutableSetOf() else mutableSetOf(ideaDir) + project.modules.forEach { module -> + PythonSdkUtil.findPythonSdk(module)?.homeDirectory?.apply { excludes.add(this) } + module.rootManager.contentEntries.forEach { entry -> + excludes.addAll(entry.excludeFolderFiles) } - parent = parent.parent } - return parent ?: module.guessModuleDir() + return excludes } - fun uploadMultipleFiles(project: Project, currentModule: Module?, toUpload: List): Boolean { - FileDocumentManager.getInstance().saveAllDocuments() - performReplAction(project, true, "Upload files") {fileSystemWidget -> - val filesToUpload = mutableListOf>() - for (uploadFile in toUpload) { - val roots = mutableSetOf() - val module = - currentModule ?: ModuleUtil.findModuleForFile(uploadFile, project) ?: continue - val rootManager = module.rootManager - roots.addAll(rootManager.sourceRoots) - if (roots.isEmpty()) { - roots.addAll(rootManager.contentRoots) - } + fun uploadProject(project: Project): Boolean { + val filesToUpload = collectUploadables(project).map { file -> "" to file }.toMutableList() + return performUpload(project, filesToUpload) + } - val pythonPath = PythonSdkUtil.findPythonSdk(module)?.homeDirectory - val ideaDir = project.stateStore.directoryStorePath?.let { VfsUtil.findFile(it, false) } - val excludeRoots = listOfNotNull( - pythonPath, - ideaDir, - *ModuleRootManager.getInstance(module).excludeRoots - ) - - - VfsUtil.processFileRecursivelyWithoutIgnored(uploadFile) { file -> - if ( - file.isFile && file.isValid && - excludeRoots.none { VfsUtil.isAncestor(it, file, false) } - ) { - getClosestRoot(file, roots, module)?.apply { - val shortPath = VfsUtil.getRelativePath(file, this) - if (shortPath != null) filesToUpload.add(shortPath to file)//todo low priority optimize - } - } - true + private fun performUpload(project: Project, filesToUpload: List>): Boolean { + val flatListToUpload = filesToUpload.toMutableList() + val ignorableFolders = collectExcluded(project) + performReplAction(project, true, "Upload files") { fileSystemWidget -> + withContext(Dispatchers.EDT) { + FileDocumentManager.getInstance().saveAllDocuments() + } + val fileTypeRegistry = FileTypeRegistry.getInstance() + var index = 0 + while (index < flatListToUpload.size) { + val file = flatListToUpload[index].second + if (!file.isValid || file.leadingDot() || fileTypeRegistry.isFileIgnored(file)) { + flatListToUpload.removeAt(index) + } else if (ignorableFolders.any { VfsUtil.isAncestor(it, file, true) }) { + flatListToUpload.removeAt(index) + } else if (file.isDirectory) { + file.children.forEach { flatListToUpload.add("${flatListToUpload[index].first}/${it.name}" to it) } + flatListToUpload.removeAt(index) + } else { + index++ } + checkCanceled() } //todo low priority create empty folders reportSequentialProgress(filesToUpload.size) { reporter -> - filesToUpload.forEach { (path, file) -> + flatListToUpload.forEach { (path, file) -> reporter.itemStep(path) fileSystemWidget.upload(path, file.contentsToByteArray()) } diff --git a/test-projects/.gitignore b/test-projects/.gitignore new file mode 100644 index 00000000..aba7dc84 --- /dev/null +++ b/test-projects/.gitignore @@ -0,0 +1,6 @@ +/.venv/ +/*/.idea/libraries/ +/*/.idea/inspectionProfiles +/*/.idea/workspace.xml +/*/.idea/misc.xml +vcs.xml \ No newline at end of file diff --git a/test-projects/layout-a/.gitignore b/test-projects/layout-a/.gitignore new file mode 100644 index 00000000..c523909d --- /dev/null +++ b/test-projects/layout-a/.gitignore @@ -0,0 +1,2 @@ +# don't upload me +/venv-dont-upload/ \ No newline at end of file diff --git a/test-projects/layout-a/.idea/layout-a.iml b/test-projects/layout-a/.idea/layout-a.iml new file mode 100644 index 00000000..48a35498 --- /dev/null +++ b/test-projects/layout-a/.idea/layout-a.iml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test-projects/layout-a/.idea/modules.xml b/test-projects/layout-a/.idea/modules.xml new file mode 100644 index 00000000..17f71199 --- /dev/null +++ b/test-projects/layout-a/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/test-projects/layout-a/.idea/runConfigurations/Flash_project.xml b/test-projects/layout-a/.idea/runConfigurations/Flash_project.xml new file mode 100644 index 00000000..a1b4d3e4 --- /dev/null +++ b/test-projects/layout-a/.idea/runConfigurations/Flash_project.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/test-projects/layout-a/.ignored-as-dot/not-uploadable-c.txt b/test-projects/layout-a/.ignored-as-dot/not-uploadable-c.txt new file mode 100644 index 00000000..e69de29b diff --git a/test-projects/layout-a/ignored-as-excluded/not-uploadable-a.txt b/test-projects/layout-a/ignored-as-excluded/not-uploadable-a.txt new file mode 100644 index 00000000..e69de29b diff --git a/test-projects/layout-a/ignored-as-test/not-uploadable-b.txt b/test-projects/layout-a/ignored-as-test/not-uploadable-b.txt new file mode 100644 index 00000000..e69de29b diff --git a/test-projects/layout-a/lib/inner/uploadable-c.py b/test-projects/layout-a/lib/inner/uploadable-c.py new file mode 100644 index 00000000..e69de29b diff --git a/test-projects/layout-a/lib/uploadable-b.py b/test-projects/layout-a/lib/uploadable-b.py new file mode 100644 index 00000000..e69de29b diff --git a/test-projects/layout-a/main.py b/test-projects/layout-a/main.py new file mode 100644 index 00000000..5596b447 --- /dev/null +++ b/test-projects/layout-a/main.py @@ -0,0 +1,16 @@ +# This is a sample Python script. + +# Press Shift+F10 to execute it or replace it with your code. +# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings. + + +def print_hi(name): + # Use a breakpoint in the code line below to debug your script. + print(f'Hi, {name}') # Press Ctrl+F8 to toggle the breakpoint. + + +# Press the green button in the gutter to run the script. +if __name__ == '__main__': + print_hi('PyCharm') + +# See PyCharm help at https://www.jetbrains.com/help/pycharm/ diff --git a/test-projects/layout-a/uploadable-a.py b/test-projects/layout-a/uploadable-a.py new file mode 100644 index 00000000..e69de29b diff --git a/test-projects/layout-b/.gitignore b/test-projects/layout-b/.gitignore new file mode 100644 index 00000000..bd462cca --- /dev/null +++ b/test-projects/layout-b/.gitignore @@ -0,0 +1 @@ +# don't upload me diff --git a/test-projects/layout-b/.idea/.gitignore b/test-projects/layout-b/.idea/.gitignore new file mode 100644 index 00000000..26d33521 --- /dev/null +++ b/test-projects/layout-b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/test-projects/layout-b/.idea/layout-b.iml b/test-projects/layout-b/.idea/layout-b.iml new file mode 100644 index 00000000..d16a8898 --- /dev/null +++ b/test-projects/layout-b/.idea/layout-b.iml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test-projects/layout-b/.idea/modules.xml b/test-projects/layout-b/.idea/modules.xml new file mode 100644 index 00000000..841c1913 --- /dev/null +++ b/test-projects/layout-b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/test-projects/layout-b/.ignored-as-dot/not-uploadable-c.txt b/test-projects/layout-b/.ignored-as-dot/not-uploadable-c.txt new file mode 100644 index 00000000..e69de29b diff --git a/test-projects/layout-b/ignored-as-excluded/not-uploadable-a.txt b/test-projects/layout-b/ignored-as-excluded/not-uploadable-a.txt new file mode 100644 index 00000000..e69de29b diff --git a/test-projects/layout-b/ignored-as-test/not-uploadable-b.txt b/test-projects/layout-b/ignored-as-test/not-uploadable-b.txt new file mode 100644 index 00000000..e69de29b diff --git a/test-projects/layout-b/lib-ignored/inner/not-uploadable-e.py b/test-projects/layout-b/lib-ignored/inner/not-uploadable-e.py new file mode 100644 index 00000000..e69de29b diff --git a/test-projects/layout-b/lib-ignored/not-uploadable-d.py b/test-projects/layout-b/lib-ignored/not-uploadable-d.py new file mode 100644 index 00000000..e69de29b diff --git a/test-projects/layout-b/not-uploadable-root.py b/test-projects/layout-b/not-uploadable-root.py new file mode 100644 index 00000000..5596b447 --- /dev/null +++ b/test-projects/layout-b/not-uploadable-root.py @@ -0,0 +1,16 @@ +# This is a sample Python script. + +# Press Shift+F10 to execute it or replace it with your code. +# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings. + + +def print_hi(name): + # Use a breakpoint in the code line below to debug your script. + print(f'Hi, {name}') # Press Ctrl+F8 to toggle the breakpoint. + + +# Press the green button in the gutter to run the script. +if __name__ == '__main__': + print_hi('PyCharm') + +# See PyCharm help at https://www.jetbrains.com/help/pycharm/ diff --git a/test-projects/layout-b/src-2/uploadable-d.py b/test-projects/layout-b/src-2/uploadable-d.py new file mode 100644 index 00000000..fc80254b --- /dev/null +++ b/test-projects/layout-b/src-2/uploadable-d.py @@ -0,0 +1 @@ +pass \ No newline at end of file diff --git a/test-projects/layout-b/src/lib/inner/uploadable-c.py b/test-projects/layout-b/src/lib/inner/uploadable-c.py new file mode 100644 index 00000000..e69de29b diff --git a/test-projects/layout-b/src/lib/uploadable-b.py b/test-projects/layout-b/src/lib/uploadable-b.py new file mode 100644 index 00000000..e69de29b diff --git a/test-projects/layout-b/src/main.py b/test-projects/layout-b/src/main.py new file mode 100644 index 00000000..5596b447 --- /dev/null +++ b/test-projects/layout-b/src/main.py @@ -0,0 +1,16 @@ +# This is a sample Python script. + +# Press Shift+F10 to execute it or replace it with your code. +# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings. + + +def print_hi(name): + # Use a breakpoint in the code line below to debug your script. + print(f'Hi, {name}') # Press Ctrl+F8 to toggle the breakpoint. + + +# Press the green button in the gutter to run the script. +if __name__ == '__main__': + print_hi('PyCharm') + +# See PyCharm help at https://www.jetbrains.com/help/pycharm/ diff --git a/test-projects/layout-b/src/uploadable-a.py b/test-projects/layout-b/src/uploadable-a.py new file mode 100644 index 00000000..e69de29b