导出数据表时增加字段映射机制

entropy-cloud · Sep 1, 2024 · 4206ee8 · 4206ee8
1 parent 2045c51
commit 4206ee8
Show file tree

Hide file tree

Showing 22 changed files with 502 additions and 130 deletions.
diff --git a/nop-batch/nop-batch-jdbc/src/main/java/io/nop/batch/jdbc/consumer/GenInsertSqlRecordIO.java b/nop-batch/nop-batch-jdbc/src/main/java/io/nop/batch/jdbc/consumer/GenInsertSqlRecordIO.java
@@ -19,15 +19,21 @@
 
 import java.io.IOException;
 import java.io.Writer;
+import java.util.List;
 import java.util.Map;
 
 public class GenInsertSqlRecordIO implements IResourceRecordIO<Map<String, Object>> {
     private String dialect = DaoConstants.DIALECT_MYSQL;
+    private List<String> fields;
 
     public void setDialect(String dialect) {
         this.dialect = dialect;
     }
 
+    public void setFields(List<String> fields) {
+        this.fields = fields;
+    }
+
     @Override
     public IRecordInput<Map<String, Object>> openInput(IResource resource, String encoding) {
         throw new UnsupportedOperationException("openInput");
@@ -38,17 +44,19 @@ public IRecordOutput<Map<String, Object>> openOutput(IResource resource, String
         String tableName = StringHelper.fileNameNoExt(resource.getName());
 
         IDialect dialect = DialectManager.instance().getDialect(this.dialect);
-        return new GenInsertSqlOutput(dialect, tableName, resource.getWriter(null));
+        return new GenInsertSqlOutput(dialect, tableName, fields, resource.getWriter(null));
     }
 
     static class GenInsertSqlOutput implements IRecordOutput<Map<String, Object>> {
         private final IDialect dialect;
         private final String tableName;
+        private final List<String> fields;
         private final Writer out;
 
-        public GenInsertSqlOutput(IDialect dialect, String tableName, Writer out) {
+        public GenInsertSqlOutput(IDialect dialect, String tableName, List<String> fields, Writer out) {
             this.dialect = dialect;
             this.tableName = tableName;
+            this.fields = fields;
             this.out = out;
         }
 
@@ -74,23 +82,46 @@ String buildInsertSql(Map<String, Object> record) {
             sb.append("insert into ").append(tableName);
             sb.append("(");
             boolean first = true;
-            for (String colName : record.keySet()) {
-                if (first) {
-                    first = false;
-                } else {
-                    sb.append(',');
+            if (fields != null) {
+                for (String colName : fields) {
+                    if (first) {
+                        first = false;
+                    } else {
+                        sb.append(',');
+                    }
+                    sb.append(colName);
+                }
+            } else {
+                for (String colName : record.keySet()) {
+                    if (first) {
+                        first = false;
+                    } else {
+                        sb.append(',');
+                    }
+                    sb.append(colName);
                 }
-                sb.append(colName);
             }
             sb.append(") values (");
             first = true;
-            for (Object value : record.values()) {
-                if (first) {
-                    first = false;
-                } else {
-                    sb.append(',');
+            if (fields != null) {
+                for (String colName : fields) {
+                    Object value = record.get(colName);
+                    if (first) {
+                        first = false;
+                    } else {
+                        sb.append(',');
+                    }
+                    sb.append(encode(value));
+                }
+            } else {
+                for (Object value : record.values()) {
+                    if (first) {
+                        first = false;
+                    } else {
+                        sb.append(',');
+                    }
+                    sb.append(encode(value));
                 }
-                sb.append(encode(value));
             }
             sb.append(")");
             sb.append(";\n");

diff --git a/nop-cli/demo/test.export-db.xml b/nop-cli/demo/test.export-db.xml
@@ -4,15 +4,15 @@
     <jdbc-connection x:extends="jdbc-connection.xml"/>
 
     <tables>
-        <table name="nop_auth_user" to="user">
+        <table name="user" from="nop_auth_user">
             <fields>
-                <field name="sid"/>
+                <field name="ID" from="USER_ID"/>
             </fields>
         </table>
 
-        <table name="nop_auth_session" to="session" exportAllFields="false">
+        <table name="session" from="nop_auth_session" exportAllFields="false">
             <fields>
-                <field name="sid" to="id"/>
+                <field name="ID" from="SID"/>
             </fields>
         </table>
     </tables>

diff --git a/nop-cli/demo/test.import-db.xml b/nop-cli/demo/test.import-db.xml
@@ -6,13 +6,13 @@
     <tables>
         <table name="nop_auth_user" from="user">
             <fields>
-                <field name="sid"/>
+                <field name="USER_ID"/>
             </fields>
         </table>
 
         <table name="nop_auth_session" from="session" >
             <fields>
-                <field name="sid" from="id"/>
+                <field name="SID" from="ID"/>
             </fields>
         </table>
     </tables>

diff --git a/...ol/nop-dbtool-core/src/main/java/io/nop/dbtool/core/discovery/jdbc/JdbcMetaDiscovery.java b/...ol/nop-dbtool-core/src/main/java/io/nop/dbtool/core/discovery/jdbc/JdbcMetaDiscovery.java
@@ -256,15 +256,15 @@ private void discoverColumns(DataBaseMeta meta, DatabaseMetaData metaData, Strin
                 String remarks = columns.getString("REMARKS");
                 String generated = columns.getString("IS_GENERATEDCOLUMN");
                 String defaultValue = columns.getString("COLUMN_DEF");
-                int ordinal = columns.getInt("ORDINAL_POSITION");
+                //int ordinal = columns.getInt("ORDINAL_POSITION");
 
                 columnName = normalizeColName(columnName);
 
                 OrmColumnModel col = new OrmColumnModel();
                 col.setCode(columnName);
                 col.setName(StringHelper.colCodeToPropName(columnName));
                 // Note：获取到的默认值可能是包含引号的转义值（若值为函数，则不会被转义），在使用时需注意
-                if(defaultValue != null) {
+                if (defaultValue != null) {
                     if (StringHelper.isNumber(defaultValue)) {
                         col.setDefaultValue(defaultValue);
                     } else if (defaultValue.startsWith("'") && defaultValue.endsWith("'") && defaultValue.length() > 2) {
@@ -298,7 +298,7 @@ private void discoverColumns(DataBaseMeta meta, DatabaseMetaData metaData, Strin
                 } else {
                     col.setComment(remarks);
                 }
-                col.setPropId(ordinal);
+                // col.setPropId(ordinal);
                 if ("Yes".equalsIgnoreCase(generated)) {
                     col.setInsertable(false);
                     col.setUpdatable(false);
@@ -323,7 +323,10 @@ private void discoverColumns(DataBaseMeta meta, DatabaseMetaData metaData, Strin
             }
             table.setDbPkName(pk.getConstraint());
 
-            cols.sort(Comparator.comparing(OrmColumnModel::getPropId));
+            // ordinal存在重复的情况
+            for (int i = 0, n = cols.size(); i < n; i++) {
+                cols.get(i).setPropId(i + 1);
+            }
 
             // 没有主键，则这里强制设置第一个字段为主键
             if (pk.getColumns().isEmpty()) {
@@ -335,7 +338,16 @@ private void discoverColumns(DataBaseMeta meta, DatabaseMetaData metaData, Strin
                 }
             }
 
-            // ordinal存在重复的情况
+            // 确保主键排在前面
+            cols.sort((c1, c2) -> {
+                if (c1.isPrimary() && !c2.isPrimary())
+                    return -1;
+                if (!c1.isPrimary() && c2.isPrimary())
+                    return 1;
+                return Integer.compare(c1.getPropId(), c2.getPropId());
+            });
+
+            // 重新设置propId
             for (int i = 0, n = cols.size(); i < n; i++) {
                 cols.get(i).setPropId(i + 1);
             }

diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/DbToolExpConstants.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/DbToolExpConstants.java
@@ -0,0 +1,8 @@
+package io.nop.dbtool.exp;
+
+public interface DbToolExpConstants {
+    String VAR_INPUT = "input";
+    String VAR_OUTPUT = "output";
+
+    String VAR_VALUE = "value";
+}
diff --git a/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/DbToolExpErrors.java b/nop-dbtool/nop-dbtool-exp/src/main/java/io/nop/dbtool/exp/DbToolExpErrors.java
@@ -13,6 +13,8 @@
 
 public interface DbToolExpErrors {
     String ARG_TABLE_NAME = "tableName";
+    String ARG_FIELD_NAME = "fieldName";
+
     ErrorCode ERR_EXP_UNDEFINED_TABLE = define("nop.err.exp.undefined-table",
             "未定义的数据库表:{}", ARG_TABLE_NAME);
 }