引入export-db.xdef和import-db.xdef来生成导入导出模型对象

entropy-cloud · Sep 1, 2024 · 2045c51 · 2045c51
1 parent 63406ce
commit 2045c51
Show file tree

Hide file tree

Showing 32 changed files with 2,089 additions and 346 deletions.
diff --git a/nop-batch/nop-batch-core/src/main/java/io/nop/batch/core/BatchTaskBuilder.java b/nop-batch/nop-batch-core/src/main/java/io/nop/batch/core/BatchTaskBuilder.java
@@ -210,8 +210,8 @@ public BatchTaskBuilder consumer(IBatchConsumer<?, IBatchChunkContext> consumer)
                 return this;
             if (list.size() == 1) {
                 consumer = list.first();
+                return consumer(consumer);
             }
-            return consumer(consumer);
         }
 
         this.consumer = consumer;

diff --git a/nop-cli-core/src/main/java/io/nop/cli/commands/CliExportDbCommand.java b/nop-cli-core/src/main/java/io/nop/cli/commands/CliExportDbCommand.java
@@ -7,9 +7,9 @@
  */
 package io.nop.cli.commands;
 
-import io.nop.core.lang.json.JsonTool;
 import io.nop.core.resource.IResource;
 import io.nop.core.resource.ResourceHelper;
+import io.nop.core.resource.component.ResourceComponentManager;
 import io.nop.dbtool.exp.ExportDbTool;
 import io.nop.dbtool.exp.config.ExportDbConfig;
 import picocli.CommandLine;
@@ -29,18 +29,23 @@ public class CliExportDbCommand implements Callable<Integer> {
 
     @CommandLine.Parameters(description = "配置文件路径")
     String configPath;
+
     @Override
     public Integer call() {
         IResource resource = ResourceHelper.resolveRelativePathResource(configPath);
-        ExportDbConfig config = JsonTool.parseBeanFromResource(resource, ExportDbConfig.class);
+        ExportDbTool tool = new ExportDbTool();
+        ExportDbConfig config = (ExportDbConfig) ResourceComponentManager.instance().loadComponentModel(resource.getPath());
+        // 有可能会修改config的属性，所以需要复制一份
+        config = config.cloneInstance();
+
+        tool.setConfig(config);
+
         if (outputDir != null)
             config.setOutputDir(outputDir.getAbsolutePath());
 
         if (config.getOutputDir() == null)
             config.setOutputDir(new File("data").getAbsolutePath());
 
-        ExportDbTool tool = new ExportDbTool();
-        tool.setConfig(config);
         tool.execute();
         return 0;
     }

diff --git a/nop-cli-core/src/main/java/io/nop/cli/commands/CliImportDbCommand.java b/nop-cli-core/src/main/java/io/nop/cli/commands/CliImportDbCommand.java
@@ -7,9 +7,9 @@
  */
 package io.nop.cli.commands;
 
-import io.nop.core.lang.json.JsonTool;
 import io.nop.core.resource.IResource;
 import io.nop.core.resource.ResourceHelper;
+import io.nop.core.resource.component.ResourceComponentManager;
 import io.nop.dbtool.exp.ImportDbTool;
 import io.nop.dbtool.exp.config.ImportDbConfig;
 import picocli.CommandLine;
@@ -32,7 +32,10 @@ public class CliImportDbCommand implements Callable<Integer> {
     @Override
     public Integer call() {
         IResource resource = ResourceHelper.resolveRelativePathResource(configPath);
-        ImportDbConfig config = JsonTool.parseBeanFromResource(resource, ImportDbConfig.class);
+        ImportDbConfig config = (ImportDbConfig) ResourceComponentManager.instance().loadComponentModel(resource.getPath());
+        // 有可能会修改config的属性，所以需要复制一份
+        config = config.cloneInstance();
+
         if (inputDir != null)
             config.setInputDir(inputDir.getAbsolutePath());
 

diff --git a/nop-cli-core/src/test/java/io/nop/cli/NopCliTasks.java b/nop-cli-core/src/test/java/io/nop/cli/NopCliTasks.java
@@ -102,7 +102,7 @@ public void testGenDao() {
 
     @Test
     public void testExportDb() {
-        String[] args = new String[]{"export-db", "../nop-cli/demo/export.json",
+        String[] args = new String[]{"export-db", "../nop-cli/demo/test.export-db.xml",
                 "-o", "target/data"
         };
         NopCliApplication app = new NopCliApplication();
@@ -112,7 +112,7 @@ public void testExportDb() {
 
     @Test
     public void testImportDb() {
-        String[] args = new String[]{"import-db", "../nop-cli/demo/import.json",
+        String[] args = new String[]{"import-db", "../nop-cli/demo/test.import-db.xml",
                 "-i", "target/data"
         };
         NopCliApplication app = new NopCliApplication();
@@ -157,4 +157,5 @@ public void testGenOrmFromXDef() {
         app.setFactory(factory);
         assertEquals(0, app.run(args));
     }
+
 }
diff --git a/nop-cli/demo/jdbc-connection.xml b/nop-cli/demo/jdbc-connection.xml
@@ -0,0 +1,9 @@
+<jdbc-connection x:schema="/nop/schema/db/jdbc-connection.xdef"
+                 username="nop" password="nop-test"
+                 catalog="dev"
+>
+    <driverClassName>com.mysql.cj.jdbc.Driver</driverClassName>
+    <jdbcUrl><![CDATA[
+        jdbc:mysql://127.0.0.1:3306/dev?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC
+    ]]></jdbcUrl>
+</jdbc-connection>
diff --git a/nop-cli/demo/test.export-db.xml b/nop-cli/demo/test.export-db.xml
@@ -0,0 +1,19 @@
+<export-db x:schema="/nop/schema/db/export-db.xdef" xmlns:x="/nop/schema/xdsl.xdef"
+           threadCount="4" exportFormats="csv,sql">
+
+    <jdbc-connection x:extends="jdbc-connection.xml"/>
+
+    <tables>
+        <table name="nop_auth_user" to="user">
+            <fields>
+                <field name="sid"/>
+            </fields>
+        </table>
+
+        <table name="nop_auth_session" to="session" exportAllFields="false">
+            <fields>
+                <field name="sid" to="id"/>
+            </fields>
+        </table>
+    </tables>
+</export-db>
diff --git a/nop-cli/demo/test.import-db.xml b/nop-cli/demo/test.import-db.xml
@@ -0,0 +1,19 @@
+<import-db x:schema="/nop/schema/db/import-db.xdef" xmlns:x="/nop/schema/xdsl.xdef"
+           threadCount="4" >
+
+    <jdbc-connection x:extends="jdbc-connection.xml"/>
+
+    <tables>
+        <table name="nop_auth_user" from="user">
+            <fields>
+                <field name="sid"/>
+            </fields>
+        </table>
+
+        <table name="nop_auth_session" from="session" >
+            <fields>
+                <field name="sid" from="id"/>
+            </fields>
+        </table>
+    </tables>
+</import-db>
diff --git a/nop-codegen/src/test/java/io/nop/codegen/CodeGen.java b/nop-codegen/src/test/java/io/nop/codegen/CodeGen.java
@@ -52,6 +52,8 @@ public static void main(String[] args) {
             XCodeGenerator.runPrecompile(new File(projectDir, "../nop-rule/nop-rule-core"), "", false);
             XCodeGenerator.runPrecompile(new File(projectDir, "../nop-stream/nop-stream-cep"), "", false);
             XCodeGenerator.runPrecompile(new File(projectDir, "../nop-auth/nop-auth-core"), "", false);
+
+            XCodeGenerator.runPrecompile(new File(projectDir, "../nop-dbtool/nop-dbtool-exp"), "", false);
             // System.setProperty("dryRun","true");
             // CodeGenTask.main(new String[]{new File("projectDir","../nop-graphql").getAbsolutePath()});
         } finally {

diff --git a/...ol/nop-dbtool-core/src/main/java/io/nop/dbtool/core/discovery/jdbc/JdbcMetaDiscovery.java b/...ol/nop-dbtool-core/src/main/java/io/nop/dbtool/core/discovery/jdbc/JdbcMetaDiscovery.java
@@ -267,7 +267,7 @@ private void discoverColumns(DataBaseMeta meta, DatabaseMetaData metaData, Strin
                 if(defaultValue != null) {
                     if (StringHelper.isNumber(defaultValue)) {
                         col.setDefaultValue(defaultValue);
-                    } else if (defaultValue.startsWith("'") && defaultValue.endsWith("'")) {
+                    } else if (defaultValue.startsWith("'") && defaultValue.endsWith("'") && defaultValue.length() > 2) {
                         defaultValue = defaultValue.substring(1, defaultValue.length() - 1);
                         col.setDefaultValue(defaultValue);
                     }

diff --git a/nop-dbtool/nop-dbtool-exp/precompile/gen-dbtool-dsl.xgen b/nop-dbtool/nop-dbtool-exp/precompile/gen-dbtool-dsl.xgen
@@ -0,0 +1,5 @@
+<c:script>
+    codeGenerator.renderModel('/nop/schema/db/jdbc-connection.xdef','/nop/templates/xdsl', '/',$scope);
+    codeGenerator.renderModel('/nop/schema/db/export-db.xdef','/nop/templates/xdsl', '/',$scope);
+    codeGenerator.renderModel('/nop/schema/db/import-db.xdef','/nop/templates/xdsl', '/',$scope);
+</c:script>
diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/ExportDbTool.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/ExportDbTool.java
@@ -25,6 +25,7 @@
 import io.nop.commons.concurrent.executor.SyncThreadPoolExecutor;
 import io.nop.commons.util.StringHelper;
 import io.nop.core.resource.IResourceLoader;
+import io.nop.core.resource.component.ResourceComponentManager;
 import io.nop.core.resource.impl.FileResource;
 import io.nop.core.resource.record.IResourceRecordIO;
 import io.nop.core.resource.record.csv.CsvResourceRecordIO;
@@ -34,6 +35,7 @@
 import io.nop.dbtool.core.discovery.jdbc.JdbcMetaDiscovery;
 import io.nop.dbtool.exp.config.ExportDbConfig;
 import io.nop.dbtool.exp.config.ExportTableConfig;
+import io.nop.dbtool.exp.config.JdbcConnectionConfig;
 import io.nop.orm.model.OrmEntityModel;
 
 import javax.sql.DataSource;
@@ -55,8 +57,16 @@ public void setConfig(ExportDbConfig config) {
         this.config = config;
     }
 
+    public ExportDbConfig getConfig() {
+        return config;
+    }
+
+    public void setConfigPath(String configPath) {
+        this.setConfig((ExportDbConfig) ResourceComponentManager.instance().loadComponentModel(configPath));
+    }
+
     public void execute() {
-        Guard.notEmpty(config.getCatalog(),"catalog");
+        Guard.notEmpty(config.getJdbcConnection(), "jdbc-connection");
 
         this.outputResourceLoader = new FileResource(new File(config.getOutputDir()));
 
@@ -67,16 +77,17 @@ public void execute() {
             executor = SyncThreadPoolExecutor.INSTANCE;
         }
 
+        JdbcConnectionConfig conn = config.getJdbcConnection();
         try {
-            DataSource ds = config.buildDataSource();
-            if (config.getDialect() == null) {
+            DataSource ds = conn.buildDataSource();
+            if (conn.getDialect() == null) {
                 this.dialect = DialectManager.instance().getDialectForDataSource(ds);
             } else {
-                this.dialect = DialectManager.instance().getDialect(config.getDialect());
+                this.dialect = DialectManager.instance().getDialect(conn.getDialect());
             }
 
             List<CompletableFuture<?>> futures = new ArrayList<>();
-            for (ExportTableConfig tableConfig : getAllTables().values()) {
+            for (ExportTableConfig tableConfig : getAllTables(ds).values()) {
                 futures.add(executor.submit(() -> runTask(tableConfig, ds), null));
             }
             FutureHelper.getFromFuture(FutureHelper.waitAll(futures));
@@ -85,7 +96,7 @@ public void execute() {
         }
     }
 
-    private Map<String, ExportTableConfig> getAllTables() {
+    private Map<String, ExportTableConfig> getAllTables(DataSource dataSource) {
         Map<String, ExportTableConfig> map = new LinkedHashMap<>();
         if (config.getTables() != null) {
             config.getTables().forEach(table -> map.put(StringHelper.lowerCase(table.getName()), table));
@@ -94,8 +105,10 @@ private Map<String, ExportTableConfig> getAllTables() {
         if (config.isExportAllTables() || !StringHelper.isEmpty(config.getTableNamePrefix())) {
             String tableNamePattern = config.isExportAllTables() ? null : config.getTableNamePrefix() + "%";
 
-            DataBaseMeta meta = JdbcMetaDiscovery.forDataSource(config.buildDataSource())
-                                                 .discover(config.getCatalog(), null, tableNamePattern);
+            JdbcConnectionConfig conn = config.getJdbcConnection();
+
+            DataBaseMeta meta = JdbcMetaDiscovery.forDataSource(dataSource)
+                    .discover(conn.getCatalog(), null, tableNamePattern);
 
             for (OrmEntityModel table : meta.getTables().values()) {
                 String name = StringHelper.lowerCase(table.getTableName());
@@ -172,4 +185,4 @@ private <S, C> ResourceRecordConsumer<S, C> newResourceConsumer(IResourceRecordI
         consumer.setResourceLoader(outputResourceLoader);
         return consumer;
     }
-}
+}
diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/ImportDbTool.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/ImportDbTool.java
@@ -29,6 +29,7 @@
 import io.nop.core.model.query.QueryBeanHelper;
 import io.nop.core.resource.IResource;
 import io.nop.core.resource.IResourceLoader;
+import io.nop.core.resource.component.ResourceComponentManager;
 import io.nop.core.resource.impl.FileResource;
 import io.nop.core.resource.record.IResourceRecordIO;
 import io.nop.core.resource.record.csv.CsvResourceRecordIO;
@@ -41,6 +42,7 @@
 import io.nop.dbtool.core.discovery.jdbc.JdbcMetaDiscovery;
 import io.nop.dbtool.exp.config.ImportDbConfig;
 import io.nop.dbtool.exp.config.ImportTableConfig;
+import io.nop.dbtool.exp.config.JdbcConnectionConfig;
 import io.nop.orm.eql.utils.OrmDialectHelper;
 import io.nop.orm.model.OrmEntityModel;
 
@@ -67,8 +69,16 @@ public void setConfig(ImportDbConfig config) {
         this.config = config;
     }
 
+    public ImportDbConfig getConfig() {
+        return config;
+    }
+
+    public void setConfigPath(String configPath) {
+        this.setConfig((ImportDbConfig) ResourceComponentManager.instance().loadComponentModel(configPath));
+    }
+
     public void execute() {
-        Guard.notEmpty(config.getCatalog(), "catalog");
+        Guard.notEmpty(config.getJdbcConnection(), "jdbc-connection");
 
         this.inputResourceLoader = new FileResource(new File(config.getInputDir()));
 
@@ -79,11 +89,12 @@ public void execute() {
             executor = SyncThreadPoolExecutor.INSTANCE;
         }
 
+        JdbcConnectionConfig conn = config.getJdbcConnection();
         try {
-            DataSource ds = config.buildDataSource();
+            DataSource ds = conn.buildDataSource();
             this.dialect = DialectManager.instance().getDialectForDataSource(ds);
 
-            DataBaseMeta meta = JdbcMetaDiscovery.forDataSource(ds).discover(config.getCatalog(), null, null);
+            DataBaseMeta meta = JdbcMetaDiscovery.forDataSource(ds).discover(conn.getCatalog(), null, null);
 
             List<CompletableFuture<?>> futures = new ArrayList<>();
             for (ImportTableConfig tableConfig : getAllTables().values()) {
@@ -155,23 +166,23 @@ private void runTask(ImportTableConfig tableConfig, DataSource ds, DataBaseMeta
     }
 
     private IBatchLoader<Map<String, Object>, IBatchChunkContext> newLoader(ImportTableConfig tableConfig) {
-        String from = tableConfig.getFrom();
-        if (from == null)
-            from = tableConfig.getName();
+        String from = tableConfig.getSourceName();
         tableConfig.setFrom(from);
         String format = tableConfig.getFormat();
         if (format == null) {
             format = "csv";
             tableConfig.setFormat(format);
         }
 
-        IResource resource = inputResourceLoader.getResource(from + '.' + format);
+        String resourcePath = from + '.' + format;
+
+        IResource resource = inputResourceLoader.getResource(resourcePath);
         if (!resource.exists())
             return null;
 
         CsvResourceRecordIO<Map<String, Object>> recordIO = new CsvResourceRecordIO<>();
         recordIO.setRecordType(Map.class);
-        return newResourceLoader(recordIO, tableConfig);
+        return newResourceLoader(recordIO, tableConfig, resourcePath);
     }
 
     private IBatchConsumer<Map<String, Object>, IBatchChunkContext> newConsumer(OrmEntityModel tableModel,
@@ -184,13 +195,11 @@ private IBatchConsumer<Map<String, Object>, IBatchChunkContext> newConsumer(OrmE
     }
 
     private <S, C> ResourceRecordLoader<S, C> newResourceLoader(IResourceRecordIO<S> recordIO,
-                                                                ImportTableConfig tableConfig) {
+                                                                ImportTableConfig tableConfig,
+                                                                String resourcePath) {
         ResourceRecordLoader<S, C> loader = new ResourceRecordLoader<>();
         loader.setRecordIO(recordIO);
-        String from = tableConfig.getName();
-        if (from == null)
-            from = tableConfig.getName();
-        loader.setResourcePath(from + "." + tableConfig.getFormat());
+        loader.setResourcePath(resourcePath);
         loader.setResourceLoader(inputResourceLoader);
 
         if (tableConfig.getFilter() != null) {