From 4206ee8548effa7b362808fd272853a4fe452a01 Mon Sep 17 00:00:00 2001 From: canonical Date: Sun, 1 Sep 2024 23:11:32 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AF=BC=E5=87=BA=E6=95=B0=E6=8D=AE=E8=A1=A8?= =?UTF-8?q?=E6=97=B6=E5=A2=9E=E5=8A=A0=E5=AD=97=E6=AE=B5=E6=98=A0=E5=B0=84?= =?UTF-8?q?=E6=9C=BA=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jdbc/consumer/GenInsertSqlRecordIO.java | 59 ++++++-- nop-cli/demo/test.export-db.xml | 8 +- nop-cli/demo/test.import-db.xml | 4 +- .../discovery/jdbc/JdbcMetaDiscovery.java | 22 ++- .../io/nop/dbtool/exp/DbToolExpConstants.java | 8 + .../io/nop/dbtool/exp/DbToolExpErrors.java | 2 + .../java/io/nop/dbtool/exp/ExportDbTool.java | 141 +++++++++++++----- .../io/nop/dbtool/exp/FieldsProcessor.java | 45 ++++++ .../nop/dbtool/exp/config/ExportDbConfig.java | 15 +- .../dbtool/exp/config/ExportTableConfig.java | 46 +++++- .../exp/config/ExportTableFieldConfig.java | 4 +- .../nop/dbtool/exp/config/IFieldConfig.java | 29 ++++ .../nop/dbtool/exp/config/ImportDbConfig.java | 1 + .../dbtool/exp/config/ImportTableConfig.java | 33 ++++ .../exp/config/ImportTableFieldConfig.java | 5 +- .../exp/config/_gen/_ExportDbConfig.java | 46 ++++-- .../exp/config/_gen/_ExportTableConfig.java | 46 +++--- .../config/_gen/_ExportTableFieldConfig.java | 72 ++++++--- .../exp/config/_gen/_ImportDbConfig.java | 2 +- .../config/_gen/_ImportTableFieldConfig.java | 34 ++++- .../_vfs/nop/schema/db/export-db.xdef | 7 +- .../_vfs/nop/schema/db/import-db.xdef | 3 +- 22 files changed, 502 insertions(+), 130 deletions(-) create mode 100644 nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/DbToolExpConstants.java create mode 100644 nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/FieldsProcessor.java create mode 100644 nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/IFieldConfig.java diff --git a/nop-batch/nop-batch-jdbc/src/main/java/io/nop/batch/jdbc/consumer/GenInsertSqlRecordIO.java b/nop-batch/nop-batch-jdbc/src/main/java/io/nop/batch/jdbc/consumer/GenInsertSqlRecordIO.java index 5f5474553..afdca477f 100644 --- a/nop-batch/nop-batch-jdbc/src/main/java/io/nop/batch/jdbc/consumer/GenInsertSqlRecordIO.java +++ b/nop-batch/nop-batch-jdbc/src/main/java/io/nop/batch/jdbc/consumer/GenInsertSqlRecordIO.java @@ -19,15 +19,21 @@ import java.io.IOException; import java.io.Writer; +import java.util.List; import java.util.Map; public class GenInsertSqlRecordIO implements IResourceRecordIO> { private String dialect = DaoConstants.DIALECT_MYSQL; + private List fields; public void setDialect(String dialect) { this.dialect = dialect; } + public void setFields(List fields) { + this.fields = fields; + } + @Override public IRecordInput> openInput(IResource resource, String encoding) { throw new UnsupportedOperationException("openInput"); @@ -38,17 +44,19 @@ public IRecordOutput> openOutput(IResource resource, String String tableName = StringHelper.fileNameNoExt(resource.getName()); IDialect dialect = DialectManager.instance().getDialect(this.dialect); - return new GenInsertSqlOutput(dialect, tableName, resource.getWriter(null)); + return new GenInsertSqlOutput(dialect, tableName, fields, resource.getWriter(null)); } static class GenInsertSqlOutput implements IRecordOutput> { private final IDialect dialect; private final String tableName; + private final List fields; private final Writer out; - public GenInsertSqlOutput(IDialect dialect, String tableName, Writer out) { + public GenInsertSqlOutput(IDialect dialect, String tableName, List fields, Writer out) { this.dialect = dialect; this.tableName = tableName; + this.fields = fields; this.out = out; } @@ -74,23 +82,46 @@ String buildInsertSql(Map record) { sb.append("insert into ").append(tableName); sb.append("("); boolean first = true; - for (String colName : record.keySet()) { - if (first) { - first = false; - } else { - sb.append(','); + if (fields != null) { + for (String colName : fields) { + if (first) { + first = false; + } else { + sb.append(','); + } + sb.append(colName); + } + } else { + for (String colName : record.keySet()) { + if (first) { + first = false; + } else { + sb.append(','); + } + sb.append(colName); } - sb.append(colName); } sb.append(") values ("); first = true; - for (Object value : record.values()) { - if (first) { - first = false; - } else { - sb.append(','); + if (fields != null) { + for (String colName : fields) { + Object value = record.get(colName); + if (first) { + first = false; + } else { + sb.append(','); + } + sb.append(encode(value)); + } + } else { + for (Object value : record.values()) { + if (first) { + first = false; + } else { + sb.append(','); + } + sb.append(encode(value)); } - sb.append(encode(value)); } sb.append(")"); sb.append(";\n"); diff --git a/nop-cli/demo/test.export-db.xml b/nop-cli/demo/test.export-db.xml index c2dc6b629..3eafc6d18 100644 --- a/nop-cli/demo/test.export-db.xml +++ b/nop-cli/demo/test.export-db.xml @@ -4,15 +4,15 @@ - +
- +
- +
- +
diff --git a/nop-cli/demo/test.import-db.xml b/nop-cli/demo/test.import-db.xml index a015d200c..36097c2eb 100644 --- a/nop-cli/demo/test.import-db.xml +++ b/nop-cli/demo/test.import-db.xml @@ -6,13 +6,13 @@ - +
- +
diff --git a/nop-dbtool/nop-dbtool-core/src/main/java/io/nop/dbtool/core/discovery/jdbc/JdbcMetaDiscovery.java b/nop-dbtool/nop-dbtool-core/src/main/java/io/nop/dbtool/core/discovery/jdbc/JdbcMetaDiscovery.java index 46b3dbc76..bf524e88e 100644 --- a/nop-dbtool/nop-dbtool-core/src/main/java/io/nop/dbtool/core/discovery/jdbc/JdbcMetaDiscovery.java +++ b/nop-dbtool/nop-dbtool-core/src/main/java/io/nop/dbtool/core/discovery/jdbc/JdbcMetaDiscovery.java @@ -256,7 +256,7 @@ private void discoverColumns(DataBaseMeta meta, DatabaseMetaData metaData, Strin String remarks = columns.getString("REMARKS"); String generated = columns.getString("IS_GENERATEDCOLUMN"); String defaultValue = columns.getString("COLUMN_DEF"); - int ordinal = columns.getInt("ORDINAL_POSITION"); + //int ordinal = columns.getInt("ORDINAL_POSITION"); columnName = normalizeColName(columnName); @@ -264,7 +264,7 @@ private void discoverColumns(DataBaseMeta meta, DatabaseMetaData metaData, Strin col.setCode(columnName); col.setName(StringHelper.colCodeToPropName(columnName)); // Note：获取到的默认值可能是包含引号的转义值（若值为函数，则不会被转义），在使用时需注意 - if(defaultValue != null) { + if (defaultValue != null) { if (StringHelper.isNumber(defaultValue)) { col.setDefaultValue(defaultValue); } else if (defaultValue.startsWith("'") && defaultValue.endsWith("'") && defaultValue.length() > 2) { @@ -298,7 +298,7 @@ private void discoverColumns(DataBaseMeta meta, DatabaseMetaData metaData, Strin } else { col.setComment(remarks); } - col.setPropId(ordinal); + // col.setPropId(ordinal); if ("Yes".equalsIgnoreCase(generated)) { col.setInsertable(false); col.setUpdatable(false); @@ -323,7 +323,10 @@ private void discoverColumns(DataBaseMeta meta, DatabaseMetaData metaData, Strin } table.setDbPkName(pk.getConstraint()); - cols.sort(Comparator.comparing(OrmColumnModel::getPropId)); + // ordinal存在重复的情况 + for (int i = 0, n = cols.size(); i < n; i++) { + cols.get(i).setPropId(i + 1); + } // 没有主键，则这里强制设置第一个字段为主键 if (pk.getColumns().isEmpty()) { @@ -335,7 +338,16 @@ private void discoverColumns(DataBaseMeta meta, DatabaseMetaData metaData, Strin } } - // ordinal存在重复的情况 + // 确保主键排在前面 + cols.sort((c1, c2) -> { + if (c1.isPrimary() && !c2.isPrimary()) + return -1; + if (!c1.isPrimary() && c2.isPrimary()) + return 1; + return Integer.compare(c1.getPropId(), c2.getPropId()); + }); + + // 重新设置propId for (int i = 0, n = cols.size(); i < n; i++) { cols.get(i).setPropId(i + 1); } diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/DbToolExpConstants.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/DbToolExpConstants.java new file mode 100644 index 000000000..77435ae92 --- /dev/null +++ b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/DbToolExpConstants.java @@ -0,0 +1,8 @@ +package io.nop.dbtool.exp; + +public interface DbToolExpConstants { + String VAR_INPUT = "input"; + String VAR_OUTPUT = "output"; + + String VAR_VALUE = "value"; +} diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/DbToolExpErrors.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/DbToolExpErrors.java index ae16773d2..e679c8b09 100644 --- a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/DbToolExpErrors.java +++ b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/DbToolExpErrors.java @@ -13,6 +13,8 @@ public interface DbToolExpErrors { String ARG_TABLE_NAME = "tableName"; + String ARG_FIELD_NAME = "fieldName"; + ErrorCode ERR_EXP_UNDEFINED_TABLE = define("nop.err.exp.undefined-table", "未定义的数据库表:{}", ARG_TABLE_NAME); } diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/ExportDbTool.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/ExportDbTool.java index 2f4eb5bf2..1404de58c 100644 --- a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/ExportDbTool.java +++ b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/ExportDbTool.java @@ -13,6 +13,7 @@ import io.nop.batch.core.IBatchChunkContext; import io.nop.batch.core.IBatchConsumer; import io.nop.batch.core.IBatchLoader; +import io.nop.batch.core.IBatchProcessor; import io.nop.batch.core.IBatchTask; import io.nop.batch.core.IBatchTaskContext; import io.nop.batch.core.consumer.MultiBatchConsumer; @@ -35,28 +36,42 @@ import io.nop.dbtool.core.discovery.jdbc.JdbcMetaDiscovery; import io.nop.dbtool.exp.config.ExportDbConfig; import io.nop.dbtool.exp.config.ExportTableConfig; +import io.nop.dbtool.exp.config.ExportTableFieldConfig; import io.nop.dbtool.exp.config.JdbcConnectionConfig; +import io.nop.orm.model.IColumnModel; import io.nop.orm.model.OrmEntityModel; import javax.sql.DataSource; import java.io.File; import java.util.ArrayList; -import java.util.LinkedHashMap; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; public class ExportDbTool { private ExportDbConfig config; + private Map args; private IResourceLoader outputResourceLoader; private IDialect dialect; + private DataSource dataSource; + public void setConfig(ExportDbConfig config) { this.config = config; } + public Map getArgs() { + return args; + } + + public void setArgs(Map args) { + this.args = args; + } + public ExportDbConfig getConfig() { return config; } @@ -65,9 +80,22 @@ public void setConfigPath(String configPath) { this.setConfig((ExportDbConfig) ResourceComponentManager.instance().loadComponentModel(configPath)); } + public void syncConfigWithDb() { + JdbcConnectionConfig conn = config.getJdbcConnection(); + this.dataSource = conn.buildDataSource(); + if (conn.getDialect() == null) { + this.dialect = DialectManager.instance().getDialectForDataSource(dataSource); + } else { + this.dialect = DialectManager.instance().getDialect(conn.getDialect()); + } + readTableMetas(); + } + public void execute() { Guard.notEmpty(config.getJdbcConnection(), "jdbc-connection"); + syncConfigWithDb(); + this.outputResourceLoader = new FileResource(new File(config.getOutputDir())); IThreadPoolExecutor executor; @@ -77,18 +105,11 @@ public void execute() { executor = SyncThreadPoolExecutor.INSTANCE; } - JdbcConnectionConfig conn = config.getJdbcConnection(); try { - DataSource ds = conn.buildDataSource(); - if (conn.getDialect() == null) { - this.dialect = DialectManager.instance().getDialectForDataSource(ds); - } else { - this.dialect = DialectManager.instance().getDialect(conn.getDialect()); - } - List> futures = new ArrayList<>(); - for (ExportTableConfig tableConfig : getAllTables(ds).values()) { - futures.add(executor.submit(() -> runTask(tableConfig, ds), null)); + + for (ExportTableConfig tableConfig : config.getTables()) { + futures.add(executor.submit(() -> runTask(tableConfig, dataSource), null)); } FutureHelper.getFromFuture(FutureHelper.waitAll(futures)); } finally { @@ -96,40 +117,78 @@ public void execute() { } } - private Map getAllTables(DataSource dataSource) { - Map map = new LinkedHashMap<>(); + private void readTableMetas() { + Map> map = new HashMap<>(); if (config.getTables() != null) { - config.getTables().forEach(table -> map.put(StringHelper.lowerCase(table.getName()), table)); + config.getTables().forEach(table -> { + String name = StringHelper.lowerCase(table.getSourceTableName()); + map.computeIfAbsent(name, k -> new ArrayList<>()).add(table); + }); } - if (config.isExportAllTables() || !StringHelper.isEmpty(config.getTableNamePrefix())) { - String tableNamePattern = config.isExportAllTables() ? null : config.getTableNamePrefix() + "%"; + if (config.getExcludeTableNames() != null) { + map.keySet().removeAll(config.getExcludeTableNames()); + } - JdbcConnectionConfig conn = config.getJdbcConnection(); + JdbcConnectionConfig conn = config.getJdbcConnection(); + + String tableNamePattern = config.getTableNamePattern(); - DataBaseMeta meta = JdbcMetaDiscovery.forDataSource(dataSource) - .discover(conn.getCatalog(), null, tableNamePattern); + DataBaseMeta meta = JdbcMetaDiscovery.forDataSource(dataSource) + .discover(conn.getCatalog(), config.getSchemaPattern(), tableNamePattern); - for (OrmEntityModel table : meta.getTables().values()) { - String name = StringHelper.lowerCase(table.getTableName()); + for (OrmEntityModel table : meta.getTables().values()) { + String name = StringHelper.lowerCase(table.getTableName()); - if (config.getExcludeTableNames() != null && config.getExcludeTableNames().contains(name)) + if (config.getExcludeTableNames() != null && config.getExcludeTableNames().contains(name)) + continue; + + if (!config.isExportAllTables()) { + if (!map.containsKey(name)) continue; + } - if (!map.containsKey(name)) { - ExportTableConfig tableConfig = new ExportTableConfig(); - tableConfig.setName(name); - map.put(name, tableConfig); + mergeTableConfig(map, name, table); + } + } + + private void mergeTableConfig(Map> map, String name, OrmEntityModel table) { + List list = map.get(name); + if (list == null) { + ExportTableConfig tableConfig = new ExportTableConfig(); + tableConfig.setName(name); + for (IColumnModel col : table.getColumns()) { + ExportTableFieldConfig field = new ExportTableFieldConfig(); + field.setName(col.getCode()); + field.setStdDataType(col.getStdDataType()); + tableConfig.addField(field); + } + config.addTable(tableConfig); + map.put(name, Collections.singletonList(tableConfig)); + } else { + for (ExportTableConfig old : list) { + if (!old.isExportAllFields()) + continue; + + List sourceNames = old.getSourceFieldNames(); + for (IColumnModel col : table.getColumns()) { + if (sourceNames.contains(col.getCode())) + continue; + + ExportTableFieldConfig field = new ExportTableFieldConfig(); + field.setName(col.getCode()); + field.setStdDataType(col.getStdDataType()); + old.addField(field); } } } - return map; } private void runTask(ExportTableConfig tableConfig, DataSource ds) { BatchTaskBuilder builder = new BatchTaskBuilder(); builder.loader(newLoader(tableConfig, ds)); builder.batchSize(config.getBatchSize()); + builder.addProcessor(newProcessor(tableConfig)); builder.consumer(newConsumer(tableConfig)); IBatchTask task = builder.build(); @@ -137,6 +196,10 @@ private void runTask(ExportTableConfig tableConfig, DataSource ds) { task.execute(context); } + private IBatchProcessor, Map, IBatchChunkContext> newProcessor(ExportTableConfig tableConfig) { + return new FieldsProcessor(tableConfig.getFields()); + } + private IBatchLoader, IBatchChunkContext> newLoader(ExportTableConfig tableConfig, DataSource ds) { JdbcBatchLoader> loader = new JdbcBatchLoader<>(); @@ -147,41 +210,47 @@ private IBatchLoader, IBatchChunkContext> newLoader(ExportTa private IBatchConsumer, IBatchChunkContext> newConsumer(ExportTableConfig tableConfig) { List, IBatchChunkContext>> list = new ArrayList<>(); + List fields = tableConfig.getTargetFieldNames(); + if (config.getExportFormats() != null) { for (String format : config.getExportFormats()) { + String fileName = tableConfig.getExportFileName(format); if ("sql".equals(format)) { - list.add(newGenSqlConsumer(tableConfig, format)); + list.add(newGenSqlConsumer(fileName, fields)); } else if ("csv".equals(format) || "csv.gz".equals(format)) { - list.add(newCsvConsumer(tableConfig, format)); + list.add(newCsvConsumer(fileName, fields)); } else { throw new IllegalArgumentException("nop.err.dbtool.invalid-exp-format:" + format); } } } if (list.isEmpty()) - return newCsvConsumer(tableConfig, "csv"); + return newCsvConsumer(tableConfig.getExportFileName("csv"), fields); return new MultiBatchConsumer<>(list); } - private IBatchConsumer, IBatchChunkContext> newCsvConsumer(ExportTableConfig tableConfig, String format) { + private IBatchConsumer, IBatchChunkContext> newCsvConsumer(String resourcePath, List fields) { CsvResourceRecordIO> recordIO = new CsvResourceRecordIO<>(); recordIO.setRecordType(Map.class); recordIO.setSupportZip(true); + recordIO.setHeaders(fields); - return newResourceConsumer(recordIO, tableConfig, format); + return newResourceConsumer(recordIO, resourcePath); } - private IBatchConsumer, IBatchChunkContext> newGenSqlConsumer(ExportTableConfig tableConfig, String format) { + private IBatchConsumer, IBatchChunkContext> newGenSqlConsumer(String resourcePath, + List fields) { GenInsertSqlRecordIO recordIO = new GenInsertSqlRecordIO(); recordIO.setDialect(dialect.getName()); - return newResourceConsumer(recordIO, tableConfig, format); + recordIO.setFields(fields); + return newResourceConsumer(recordIO, resourcePath); } private ResourceRecordConsumer newResourceConsumer(IResourceRecordIO recordIO, - ExportTableConfig tableConfig, String format) { + String resourcePath) { ResourceRecordConsumer consumer = new ResourceRecordConsumer<>(); consumer.setRecordIO(recordIO); - consumer.setResourcePath(tableConfig.getName() + "." + format); + consumer.setResourcePath(resourcePath); consumer.setResourceLoader(outputResourceLoader); return consumer; } diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/FieldsProcessor.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/FieldsProcessor.java new file mode 100644 index 000000000..b1c5c2ea3 --- /dev/null +++ b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/FieldsProcessor.java @@ -0,0 +1,45 @@ +package io.nop.dbtool.exp; + +import io.nop.api.core.exceptions.NopException; +import io.nop.batch.core.IBatchChunkContext; +import io.nop.batch.core.IBatchProcessor; +import io.nop.core.lang.eval.IEvalScope; +import io.nop.dbtool.exp.config.IFieldConfig; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +import static io.nop.dbtool.exp.DbToolExpErrors.ARG_FIELD_NAME; + +public class FieldsProcessor implements IBatchProcessor, Map, IBatchChunkContext> { + private final List fields; + + public FieldsProcessor(List fields) { + this.fields = fields; + } + + @Override + public void process(Map item, Consumer> consumer, IBatchChunkContext context) { + Map ret = new LinkedHashMap<>(); + IEvalScope scope = context.getEvalScope(); + scope.setLocalValue(DbToolExpConstants.VAR_INPUT, item); + + for (IFieldConfig field : fields) { + Object value = item.get(field.getSourceFieldName()); + if (field.getStdDataType() != null) + value = field.getStdDataType().convert(value, + err -> new NopException(err).source(field).param(ARG_FIELD_NAME, field.getName())); + + if (field.getTransformExpr() != null) { + scope.setLocalValue(DbToolExpConstants.VAR_VALUE, value); + value = field.getTransformExpr().invoke(scope); + } + + value = field.validate(value, scope); + ret.put(field.getName(), value); + } + consumer.accept(ret); + } +} diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ExportDbConfig.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ExportDbConfig.java index 08bd2c5f7..0d7ef9f6d 100644 --- a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ExportDbConfig.java +++ b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ExportDbConfig.java @@ -2,8 +2,19 @@ import io.nop.dbtool.exp.config._gen._ExportDbConfig; -public class ExportDbConfig extends _ExportDbConfig{ - public ExportDbConfig(){ +import java.util.stream.Collectors; +public class ExportDbConfig extends _ExportDbConfig { + public ExportDbConfig() { + + } + + @Override + public ExportDbConfig cloneInstance() { + ExportDbConfig ret = super.cloneInstance(); + if (ret.getTables() != null) { + ret.setTables(ret.getTables().stream().map(ExportTableConfig::cloneInstance).collect(Collectors.toList())); + } + return ret; } } diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ExportTableConfig.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ExportTableConfig.java index 525e96be3..3f8564329 100644 --- a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ExportTableConfig.java +++ b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ExportTableConfig.java @@ -5,17 +5,41 @@ import io.nop.dbtool.exp.config._gen._ExportTableConfig; import io.nop.orm.dao.DaoQueryHelper; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + public class ExportTableConfig extends _ExportTableConfig { public ExportTableConfig() { } + @Override + public ExportTableConfig cloneInstance() { + ExportTableConfig ret = super.cloneInstance(); + if (ret.getFields() != null) { + ret.setFields(ret.getFields().stream().map(ExportTableFieldConfig::cloneInstance).collect(Collectors.toList())); + } + return ret; + } + + public String getExportFileName(String format) { + return getName() + "." + format; + } + + public String getSourceTableName() { + String from = getFrom(); + if (from == null) + from = getName(); + return from; + } + public SQL buildSQL() { SQL.SqlBuilder sb = SQL.begin().name(getName()); if (!StringHelper.isEmpty(getSql())) { sb.append(getSql()); } else { - sb.append("select * from ").append(getName()); + sb.append("select * from ").append(getFrom()); if (getFilter() != null) { sb.where(); @@ -24,4 +48,24 @@ public SQL buildSQL() { } return sb.end(); } + + public List getTargetFieldNames() { + List ret = new ArrayList<>(getFields().size()); + for (ExportTableFieldConfig field : getFields()) { + if (field.isIgnore()) + continue; + ret.add(field.getName()); + } + return ret; + } + + public List getSourceFieldNames() { + List ret = new ArrayList<>(getFields().size()); + for (ExportTableFieldConfig field : getFields()) { + if (field.isIgnore()) + continue; + ret.add(field.getSourceFieldName()); + } + return ret; + } } diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ExportTableFieldConfig.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ExportTableFieldConfig.java index 0828d9445..e7237b13f 100644 --- a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ExportTableFieldConfig.java +++ b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ExportTableFieldConfig.java @@ -2,8 +2,8 @@ import io.nop.dbtool.exp.config._gen._ExportTableFieldConfig; -public class ExportTableFieldConfig extends _ExportTableFieldConfig{ - public ExportTableFieldConfig(){ +public class ExportTableFieldConfig extends _ExportTableFieldConfig implements IFieldConfig { + public ExportTableFieldConfig() { } } diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/IFieldConfig.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/IFieldConfig.java new file mode 100644 index 000000000..14765242f --- /dev/null +++ b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/IFieldConfig.java @@ -0,0 +1,29 @@ +package io.nop.dbtool.exp.config; + +import io.nop.api.core.util.ISourceLocationGetter; +import io.nop.commons.type.StdDataType; +import io.nop.core.lang.eval.IEvalAction; +import io.nop.core.lang.eval.IEvalScope; + +public interface IFieldConfig extends ISourceLocationGetter { + String getName(); + + String getFrom(); + + default String getSourceFieldName() { + String from = getFrom(); + if (from == null) + from = getName(); + return from; + } + + boolean isIgnore(); + + StdDataType getStdDataType(); + + IEvalAction getTransformExpr(); + + default Object validate(Object value, IEvalScope scope) { + return value; + } +} diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ImportDbConfig.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ImportDbConfig.java index 5cf656401..9809ee84b 100644 --- a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ImportDbConfig.java +++ b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ImportDbConfig.java @@ -9,6 +9,7 @@ public ImportDbConfig() { } + @Override public ImportDbConfig cloneInstance() { ImportDbConfig ret = super.cloneInstance(); if (ret.getTables() != null) { diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ImportTableConfig.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ImportTableConfig.java index 2dcb90c18..d6009192c 100644 --- a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ImportTableConfig.java +++ b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ImportTableConfig.java @@ -2,15 +2,48 @@ import io.nop.dbtool.exp.config._gen._ImportTableConfig; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + public class ImportTableConfig extends _ImportTableConfig { public ImportTableConfig() { } + @Override + public ImportTableConfig cloneInstance() { + ImportTableConfig ret = super.cloneInstance(); + if (ret.getFields() != null) { + ret.setFields(ret.getFields().stream().map(ImportTableFieldConfig::cloneInstance).collect(Collectors.toList())); + } + return ret; + } + public String getSourceName() { String from = getFrom(); if (from == null) from = getName(); return from; } + + public List getTargetFieldNames() { + List ret = new ArrayList<>(getFields().size()); + for (ImportTableFieldConfig field : getFields()) { + if (field.isIgnore()) + continue; + ret.add(field.getName()); + } + return ret; + } + + public List getSourceFieldNames() { + List ret = new ArrayList<>(getFields().size()); + for (ImportTableFieldConfig field : getFields()) { + if (field.isIgnore()) + continue; + ret.add(field.getSourceFieldName()); + } + return ret; + } } diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ImportTableFieldConfig.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ImportTableFieldConfig.java index 02117fa66..1c870b779 100644 --- a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ImportTableFieldConfig.java +++ b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/ImportTableFieldConfig.java @@ -2,8 +2,9 @@ import io.nop.dbtool.exp.config._gen._ImportTableFieldConfig; -public class ImportTableFieldConfig extends _ImportTableFieldConfig{ - public ImportTableFieldConfig(){ +public class ImportTableFieldConfig extends _ImportTableFieldConfig implements IFieldConfig { + public ImportTableFieldConfig() { } + } diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ExportDbConfig.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ExportDbConfig.java index 7acfc210a..5190a4f71 100644 --- a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ExportDbConfig.java +++ b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ExportDbConfig.java @@ -60,10 +60,17 @@ public abstract class _ExportDbConfig extends io.nop.core.resource.component.Abs /** * - * xml name: tableNamePrefix + * xml name: schemaPattern * */ - private java.lang.String _tableNamePrefix ; + private java.lang.String _schemaPattern ; + + /** + * + * xml name: tableNamePattern + * + */ + private java.lang.String _tableNamePattern ; /** * @@ -195,19 +202,38 @@ public void setOutputDir(java.lang.String value){ /** * - * xml name: tableNamePrefix + * xml name: schemaPattern + * + */ + + public java.lang.String getSchemaPattern(){ + return _schemaPattern; + } + + + public void setSchemaPattern(java.lang.String value){ + checkAllowChange(); + + this._schemaPattern = value; + + } + + + /** + * + * xml name: tableNamePattern * */ - public java.lang.String getTableNamePrefix(){ - return _tableNamePrefix; + public java.lang.String getTableNamePattern(){ + return _tableNamePattern; } - public void setTableNamePrefix(java.lang.String value){ + public void setTableNamePattern(java.lang.String value){ checkAllowChange(); - this._tableNamePrefix = value; + this._tableNamePattern = value; } @@ -301,7 +327,8 @@ protected void outputJson(IJsonHandler out){ out.putNotNull("exportFormats",this.getExportFormats()); out.putNotNull("jdbcConnection",this.getJdbcConnection()); out.putNotNull("outputDir",this.getOutputDir()); - out.putNotNull("tableNamePrefix",this.getTableNamePrefix()); + out.putNotNull("schemaPattern",this.getSchemaPattern()); + out.putNotNull("tableNamePattern",this.getTableNamePattern()); out.putNotNull("tables",this.getTables()); out.putNotNull("threadCount",this.getThreadCount()); } @@ -321,7 +348,8 @@ protected void copyTo(ExportDbConfig instance){ instance.setExportFormats(this.getExportFormats()); instance.setJdbcConnection(this.getJdbcConnection()); instance.setOutputDir(this.getOutputDir()); - instance.setTableNamePrefix(this.getTableNamePrefix()); + instance.setSchemaPattern(this.getSchemaPattern()); + instance.setTableNamePattern(this.getTableNamePattern()); instance.setTables(this.getTables()); instance.setThreadCount(this.getThreadCount()); } diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ExportTableConfig.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ExportTableConfig.java index 40e8698ff..77c1200c3 100644 --- a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ExportTableConfig.java +++ b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ExportTableConfig.java @@ -39,24 +39,24 @@ public abstract class _ExportTableConfig extends io.nop.core.resource.component. /** * - * xml name: name + * xml name: from * */ - private java.lang.String _name ; + private java.lang.String _from ; /** * - * xml name: sql + * xml name: name * */ - private java.lang.String _sql ; + private java.lang.String _name ; /** * - * xml name: to + * xml name: sql * */ - private java.lang.String _to ; + private java.lang.String _sql ; /** * @@ -150,57 +150,57 @@ public void setFilter(io.nop.api.core.beans.TreeBean value){ /** * - * xml name: name + * xml name: from * */ - public java.lang.String getName(){ - return _name; + public java.lang.String getFrom(){ + return _from; } - public void setName(java.lang.String value){ + public void setFrom(java.lang.String value){ checkAllowChange(); - this._name = value; + this._from = value; } /** * - * xml name: sql + * xml name: name * */ - public java.lang.String getSql(){ - return _sql; + public java.lang.String getName(){ + return _name; } - public void setSql(java.lang.String value){ + public void setName(java.lang.String value){ checkAllowChange(); - this._sql = value; + this._name = value; } /** * - * xml name: to + * xml name: sql * */ - public java.lang.String getTo(){ - return _to; + public java.lang.String getSql(){ + return _sql; } - public void setTo(java.lang.String value){ + public void setSql(java.lang.String value){ checkAllowChange(); - this._to = value; + this._sql = value; } @@ -246,9 +246,9 @@ protected void outputJson(IJsonHandler out){ out.putNotNull("exportAllFields",this.isExportAllFields()); out.putNotNull("fields",this.getFields()); out.putNotNull("filter",this.getFilter()); + out.putNotNull("from",this.getFrom()); out.putNotNull("name",this.getName()); out.putNotNull("sql",this.getSql()); - out.putNotNull("to",this.getTo()); out.putNotNull("transformExpr",this.getTransformExpr()); } @@ -264,9 +264,9 @@ protected void copyTo(ExportTableConfig instance){ instance.setExportAllFields(this.isExportAllFields()); instance.setFields(this.getFields()); instance.setFilter(this.getFilter()); + instance.setFrom(this.getFrom()); instance.setName(this.getName()); instance.setSql(this.getSql()); - instance.setTo(this.getTo()); instance.setTransformExpr(this.getTransformExpr()); } diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ExportTableFieldConfig.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ExportTableFieldConfig.java index d0499799a..39bfc4d32 100644 --- a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ExportTableFieldConfig.java +++ b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ExportTableFieldConfig.java @@ -16,6 +16,20 @@ "PMD.UnnecessaryFullyQualifiedName","PMD.EmptyControlStatement","java:S116","java:S101","java:S1128","java:S1161"}) public abstract class _ExportTableFieldConfig extends io.nop.core.resource.component.AbstractComponentModel { + /** + * + * xml name: from + * + */ + private java.lang.String _from ; + + /** + * + * xml name: ignore + * + */ + private boolean _ignore = false; + /** * * xml name: name @@ -32,71 +46,83 @@ public abstract class _ExportTableFieldConfig extends io.nop.core.resource.compo /** * - * xml name: to + * xml name: transformExpr * */ - private java.lang.String _to ; + private io.nop.core.lang.eval.IEvalAction _transformExpr ; /** - * - * xml name: transformExpr * + * xml name: from + * */ - private io.nop.core.lang.eval.IEvalAction _transformExpr ; + + public java.lang.String getFrom(){ + return _from; + } + + + public void setFrom(java.lang.String value){ + checkAllowChange(); + + this._from = value; + + } + /** * - * xml name: name + * xml name: ignore * */ - public java.lang.String getName(){ - return _name; + public boolean isIgnore(){ + return _ignore; } - public void setName(java.lang.String value){ + public void setIgnore(boolean value){ checkAllowChange(); - this._name = value; + this._ignore = value; } /** * - * xml name: stdDataType + * xml name: name * */ - public io.nop.commons.type.StdDataType getStdDataType(){ - return _stdDataType; + public java.lang.String getName(){ + return _name; } - public void setStdDataType(io.nop.commons.type.StdDataType value){ + public void setName(java.lang.String value){ checkAllowChange(); - this._stdDataType = value; + this._name = value; } /** * - * xml name: to + * xml name: stdDataType * */ - public java.lang.String getTo(){ - return _to; + public io.nop.commons.type.StdDataType getStdDataType(){ + return _stdDataType; } - public void setTo(java.lang.String value){ + public void setStdDataType(io.nop.commons.type.StdDataType value){ checkAllowChange(); - this._to = value; + this._stdDataType = value; } @@ -135,9 +161,10 @@ public void freeze(boolean cascade){ protected void outputJson(IJsonHandler out){ super.outputJson(out); + out.putNotNull("from",this.getFrom()); + out.putNotNull("ignore",this.isIgnore()); out.putNotNull("name",this.getName()); out.putNotNull("stdDataType",this.getStdDataType()); - out.putNotNull("to",this.getTo()); out.putNotNull("transformExpr",this.getTransformExpr()); } @@ -150,9 +177,10 @@ public ExportTableFieldConfig cloneInstance(){ protected void copyTo(ExportTableFieldConfig instance){ super.copyTo(instance); + instance.setFrom(this.getFrom()); + instance.setIgnore(this.isIgnore()); instance.setName(this.getName()); instance.setStdDataType(this.getStdDataType()); - instance.setTo(this.getTo()); instance.setTransformExpr(this.getTransformExpr()); } diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ImportDbConfig.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ImportDbConfig.java index 93fc4598f..57ed7c8fc 100644 --- a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ImportDbConfig.java +++ b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ImportDbConfig.java @@ -35,7 +35,7 @@ public abstract class _ImportDbConfig extends io.nop.core.resource.component.Abs * xml name: ignoreDuplicate * */ - private boolean _ignoreDuplicate = false; + private boolean _ignoreDuplicate = true; /** * diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ImportTableFieldConfig.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ImportTableFieldConfig.java index df90d9938..f5d009f06 100644 --- a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ImportTableFieldConfig.java +++ b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/config/_gen/_ImportTableFieldConfig.java @@ -30,6 +30,13 @@ public abstract class _ImportTableFieldConfig extends io.nop.core.resource.compo */ private java.lang.String _from ; + /** + * + * xml name: ignore + * + */ + private boolean _ignore = false; + /** * * xml name: name @@ -42,7 +49,7 @@ public abstract class _ImportTableFieldConfig extends io.nop.core.resource.compo * xml name: stdDataType * */ - private java.lang.String _stdDataType ; + private io.nop.commons.type.StdDataType _stdDataType ; /** * @@ -89,6 +96,25 @@ public void setFrom(java.lang.String value){ } + /** + * + * xml name: ignore + * + */ + + public boolean isIgnore(){ + return _ignore; + } + + + public void setIgnore(boolean value){ + checkAllowChange(); + + this._ignore = value; + + } + + /** * * xml name: name @@ -114,12 +140,12 @@ public void setName(java.lang.String value){ * */ - public java.lang.String getStdDataType(){ + public io.nop.commons.type.StdDataType getStdDataType(){ return _stdDataType; } - public void setStdDataType(java.lang.String value){ + public void setStdDataType(io.nop.commons.type.StdDataType value){ checkAllowChange(); this._stdDataType = value; @@ -163,6 +189,7 @@ protected void outputJson(IJsonHandler out){ out.putNotNull("dictName",this.getDictName()); out.putNotNull("from",this.getFrom()); + out.putNotNull("ignore",this.isIgnore()); out.putNotNull("name",this.getName()); out.putNotNull("stdDataType",this.getStdDataType()); out.putNotNull("transformExpr",this.getTransformExpr()); @@ -179,6 +206,7 @@ protected void copyTo(ImportTableFieldConfig instance){ instance.setDictName(this.getDictName()); instance.setFrom(this.getFrom()); + instance.setIgnore(this.isIgnore()); instance.setName(this.getName()); instance.setStdDataType(this.getStdDataType()); instance.setTransformExpr(this.getTransformExpr()); diff --git a/nop-xdefs/src/main/resources/_vfs/nop/schema/db/export-db.xdef b/nop-xdefs/src/main/resources/_vfs/nop/schema/db/export-db.xdef index c3f3afd96..aa0d25442 100644 --- a/nop-xdefs/src/main/resources/_vfs/nop/schema/db/export-db.xdef +++ b/nop-xdefs/src/main/resources/_vfs/nop/schema/db/export-db.xdef @@ -2,7 +2,7 @@ xmlns:xdef="/nop/schema/xdef.xdef" threadCount="!int=0" batchSize="!int=0" - tableNamePrefix="string" exportAllTables="!boolean=false" + schemaPattern="string" tableNamePattern="string" exportAllTables="!boolean=false" xdef:name="ExportDbConfig" xdef:bean-package="io.nop.dbtool.exp.config" exportFormats="csv-set" > @@ -12,13 +12,14 @@ - - diff --git a/nop-xdefs/src/main/resources/_vfs/nop/schema/db/import-db.xdef b/nop-xdefs/src/main/resources/_vfs/nop/schema/db/import-db.xdef index 22964dd19..1490cdde4 100644 --- a/nop-xdefs/src/main/resources/_vfs/nop/schema/db/import-db.xdef +++ b/nop-xdefs/src/main/resources/_vfs/nop/schema/db/import-db.xdef @@ -21,7 +21,8 @@ -