diff --git a/docs/dev-guide/report/excel-import.md b/docs/dev-guide/report/excel-import.md index e9c84a925..1ca4b4fcd 100644 --- a/docs/dev-guide/report/excel-import.md +++ b/docs/dev-guide/report/excel-import.md @@ -14,7 +14,7 @@ ## 配置说明 -## 如果解析列表 +## 如何解析列表 * sheet或者field上标注list=true，表示将会解析得到一个列表 * 列表的第一列必须是数字列，不要求编号唯一，也不要求编号列在字段列表中定义。它仅仅被用于确定列表行的范围。 @@ -23,7 +23,7 @@ ```xml - + ``` @@ -45,7 +45,7 @@ ExcelReportHelper中提供了根据导入的业务数据自动生成html或者xlsx文件的方法。 ```javascript - Object bean = ExcelHelper.loadXlsxObject("/nop/test/imp/test5.imp.xml", resource); + Object bean = ExcelHelper.loadXlsxObject("/nop/test/imp/test5.imp.xml", resource); String html = ExcelReportHelper.getHtmlForXlsxObject(impModelPath, bean, scope); ExcelReportHelper.saveXlsxObject(impModelPath, resource, bean); ``` @@ -66,7 +66,7 @@ ExcelReportHelper中提供了根据导入的业务数据自动生成html或者xl - + @@ -88,7 +88,7 @@ ExcelReportHelper中提供了根据导入的业务数据自动生成html或者xl _.findWhere(cell.rp.ev.indexValues,'year',cell.cp.ev.$toInt()).value - + @@ -201,17 +201,20 @@ ExcelReportHelper中提供了根据导入的业务数据自动生成html或者xl 如果要使用Nop平台的Excel导入导出功能，只需要在pom文件中引入如下模块 ```xml - - + + + + io.github.entropy-cloud nop-spring-core-starter - + - - -io.github.entropy-cloud -nop-report-core - + + + io.github.entropy-cloud + nop-report-core + + ``` diff --git a/nop-batch/nop-batch-core/src/main/java/io/nop/batch/core/BatchTaskBuilder.java b/nop-batch/nop-batch-core/src/main/java/io/nop/batch/core/BatchTaskBuilder.java index fe986e472..202da2b8b 100644 --- a/nop-batch/nop-batch-core/src/main/java/io/nop/batch/core/BatchTaskBuilder.java +++ b/nop-batch/nop-batch-core/src/main/java/io/nop/batch/core/BatchTaskBuilder.java @@ -23,6 +23,7 @@ import io.nop.batch.core.loader.ChunkSortBatchLoader; import io.nop.batch.core.loader.RetryBatchLoader; import io.nop.batch.core.processor.BatchChunkProcessor; +import io.nop.batch.core.processor.BatchSequentialProcessor; import io.nop.batch.core.processor.InvokerBatchChunkProcessor; import io.nop.commons.concurrent.executor.ExecutorHelper; import io.nop.commons.concurrent.ratelimit.DefaultRateLimiter; @@ -44,6 +45,7 @@ public class BatchTaskBuilder implements IBatchTaskBuilder { private Long taskVersion; private IBatchLoaderProvider loader; private IBatchConsumerProvider consumer; + private boolean useBatchRequestGenerator; private IBatchProcessorProvider processor; private int batchSize = 100; @@ -133,6 +135,12 @@ public BatchTaskBuilder taskVersion(Long taskVersion) { return this; } + @PropertySetter + public BatchTaskBuilder useBatchRequestGenerator(boolean b) { + this.useBatchRequestGenerator = b; + return this; + } + @PropertySetter public BatchTaskBuilder taskKeyExpr(IEvalFunction expr) { this.taskKeyExpr = expr; @@ -328,6 +336,9 @@ protected IBatchChunkProcessor buildChunkProcessor(IBatchTaskContext context) { if (this.processor != null) { // 如果设置了processor,则先执行processor再调用consumer，否则直接调用consumer IBatchProcessor processor = this.processor.setup(context); + if (useBatchRequestGenerator) { + processor = new BatchSequentialProcessor(processor); + } consumer = new BatchProcessorConsumer<>(processor, (IBatchConsumer) consumer); } diff --git a/nop-batch/nop-batch-core/src/main/java/io/nop/batch/core/loader/ResourceRecordLoaderProvider.java b/nop-batch/nop-batch-core/src/main/java/io/nop/batch/core/loader/ResourceRecordLoaderProvider.java index a7a0c8741..8ce67cd96 100644 --- a/nop-batch/nop-batch-core/src/main/java/io/nop/batch/core/loader/ResourceRecordLoaderProvider.java +++ b/nop-batch/nop-batch-core/src/main/java/io/nop/batch/core/loader/ResourceRecordLoaderProvider.java @@ -7,6 +7,7 @@ */ package io.nop.batch.core.loader; +import io.nop.api.core.convert.ConvertHelper; import io.nop.api.core.exceptions.NopException; import io.nop.batch.core.IBatchAggregator; import io.nop.batch.core.IBatchChunkContext; @@ -15,6 +16,7 @@ import io.nop.batch.core.IBatchTaskContext; import io.nop.batch.core.common.AbstractBatchResourceHandler; import io.nop.commons.util.IoHelper; +import io.nop.core.lang.eval.IEvalAction; import io.nop.core.resource.IResource; import io.nop.core.resource.record.IResourceRecordInputProvider; import io.nop.dataset.record.IRecordInput; @@ -50,7 +52,7 @@ public class ResourceRecordLoaderProvider extends AbstractBatchResourceHandle /** * 最多读取多少行数据（包含跳过的记录） */ - long maxCount; + IEvalAction maxCountExpr; /** * 跳过起始的多少行数据 @@ -111,12 +113,8 @@ public boolean isSaveState() { return saveState; } - public long getMaxCount() { - return maxCount; - } - - public void setMaxCount(long maxCount) { - this.maxCount = maxCount; + public void setMaxCountExpr(IEvalAction maxCountExpr) { + this.maxCountExpr = maxCountExpr; } public long getSkipCount() { @@ -164,8 +162,10 @@ LoaderState newLoaderState(IBatchTaskContext context) { skip(input, skipCount, state); } - if (maxCount > 0) { - input = input.limit(maxCount); + if (maxCountExpr != null) { + Long maxCount = ConvertHelper.toLong(maxCountExpr.invoke(context)); + if (maxCount != null) + input = input.limit(maxCount); } if (recordRowNumber) { diff --git a/nop-batch/nop-batch-dsl/pom.xml b/nop-batch/nop-batch-dsl/pom.xml index bb805fc74..ccd786d2f 100644 --- a/nop-batch/nop-batch-dsl/pom.xml +++ b/nop-batch/nop-batch-dsl/pom.xml @@ -36,6 +36,11 @@ nop-batch-dao + + io.github.entropy-cloud + nop-batch-gen + + io.github.entropy-cloud nop-task-core diff --git a/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/manager/FileBatchSupport.java b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/manager/FileBatchSupport.java index 152bd9cb0..5c880a90f 100644 --- a/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/manager/FileBatchSupport.java +++ b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/manager/FileBatchSupport.java @@ -34,8 +34,8 @@ public static IBatchLoaderProvider newFileReader(BatchFileReaderModel lo loader.setResourceLoader(resourceLoader); loader.setSaveState(Boolean.TRUE.equals(saveState)); - if (loaderModel.getMaxCount() != null) - loader.setMaxCount(loaderModel.getMaxCount()); + if (loaderModel.getMaxCountExpr() != null) + loader.setMaxCountExpr(loaderModel.getMaxCountExpr()); loader.setPathExpr(loaderModel.getFilePath()); loader.setEncoding(loaderModel.getEncoding()); loader.setAggregator(aggregator); diff --git a/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/manager/ModelBasedBatchTaskBuilderFactory.java b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/manager/ModelBasedBatchTaskBuilderFactory.java index bfd661b46..06bac678b 100644 --- a/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/manager/ModelBasedBatchTaskBuilderFactory.java +++ b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/manager/ModelBasedBatchTaskBuilderFactory.java @@ -22,17 +22,21 @@ import io.nop.batch.core.processor.MultiBatchProcessorProvider; import io.nop.batch.dsl.model.BatchChunkProcessorBuilderModel; import io.nop.batch.dsl.model.BatchConsumerModel; +import io.nop.batch.dsl.model.BatchGeneratorModel; import io.nop.batch.dsl.model.BatchListenersModel; import io.nop.batch.dsl.model.BatchLoaderModel; import io.nop.batch.dsl.model.BatchOrmWriterModel; import io.nop.batch.dsl.model.BatchProcessorModel; import io.nop.batch.dsl.model.BatchTaggerModel; import io.nop.batch.dsl.model.BatchTaskModel; +import io.nop.batch.gen.loader.BatchGenLoaderProvider; import io.nop.commons.collections.OrderByComparator; import io.nop.commons.util.CollectionHelper; import io.nop.commons.util.retry.IRetryPolicy; +import io.nop.core.lang.eval.IEvalAction; import io.nop.core.lang.eval.IEvalFunction; import io.nop.core.reflect.bean.BeanTool; +import io.nop.core.resource.VirtualFileSystem; import io.nop.dao.api.IDaoProvider; import io.nop.dao.api.INamedSqlBuilder; import io.nop.dao.jdbc.IJdbcTemplate; @@ -93,6 +97,7 @@ public IBatchTaskBuilder newTaskBuilder(IBeanProvider beanContainer) { builder.taskKeyExpr(batchTaskModel.getTaskKeyExpr()); builder.allowStartIfComplete(batchTaskModel.getAllowStartIfComplete()); builder.startLimit(batchTaskModel.getStartLimit()); + builder.useBatchRequestGenerator(batchTaskModel.isUseBatchRequestGenerator()); builder.batchSize(batchTaskModel.getBatchSize()); if (batchTaskModel.getJitterRatio() != null) @@ -257,6 +262,8 @@ private IBatchLoaderProvider buildLoader0(BatchLoaderModel loaderModel, return newJdbcReader(loaderModel.getJdbcReader(), beanProvider, jdbcTemplate, sqlLibManager); } else if (loaderModel.getOrmReader() != null) { return newOrmReader(loaderModel.getOrmReader(), daoProvider); + } else if (loaderModel.getGenerator() != null) { + return newGenerator(loaderModel.getGenerator()); } else if (loaderModel.getSource() != null) { return context -> (batchSize, ctx) -> (List) loaderModel.getSource().call2(null, batchSize, ctx, ctx.getEvalScope()); @@ -265,6 +272,16 @@ private IBatchLoaderProvider buildLoader0(BatchLoaderModel loaderModel, } } + private IBatchLoaderProvider newGenerator(BatchGeneratorModel generatorModel) { + String genModelPath = generatorModel.getGenModelPath(); + IEvalAction totalCountExpr = generatorModel.getTotalCountExpr(); + BatchGenLoaderProvider provider = new BatchGenLoaderProvider<>(); + provider.setTotalCount(totalCountExpr); + provider.setResourcePath(genModelPath); + provider.setResourceLoader(VirtualFileSystem.instance()); + return provider; + } + private IBatchAggregator> loadAggregator(String beanName, IBeanProvider beanContainer) { if (beanName == null) return null; diff --git a/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/BatchGeneratorModel.java b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/BatchGeneratorModel.java new file mode 100644 index 000000000..39eadbc14 --- /dev/null +++ b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/BatchGeneratorModel.java @@ -0,0 +1,9 @@ +package io.nop.batch.dsl.model; + +import io.nop.batch.dsl.model._gen._BatchGeneratorModel; + +public class BatchGeneratorModel extends _BatchGeneratorModel{ + public BatchGeneratorModel(){ + + } +} diff --git a/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchFileReaderModel.java b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchFileReaderModel.java index ec7dfbd4e..41c955a38 100644 --- a/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchFileReaderModel.java +++ b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchFileReaderModel.java @@ -53,10 +53,10 @@ public abstract class _BatchFileReaderModel extends io.nop.core.resource.compone /** * - * xml name: maxCount - * 读取的最大记录数，缺省值为-1，表示不限制 + * xml name: maxCountExpr + * */ - private java.lang.Long _maxCount ; + private io.nop.core.lang.eval.IEvalAction _maxCountExpr ; /** * @@ -176,19 +176,19 @@ public void setHeaders(java.util.Set value){ /** * - * xml name: maxCount - * 读取的最大记录数，缺省值为-1，表示不限制 + * xml name: maxCountExpr + * */ - public java.lang.Long getMaxCount(){ - return _maxCount; + public io.nop.core.lang.eval.IEvalAction getMaxCountExpr(){ + return _maxCountExpr; } - public void setMaxCount(java.lang.Long value){ + public void setMaxCountExpr(io.nop.core.lang.eval.IEvalAction value){ checkAllowChange(); - this._maxCount = value; + this._maxCountExpr = value; } @@ -270,7 +270,7 @@ protected void outputJson(IJsonHandler out){ out.putNotNull("filePath",this.getFilePath()); out.putNotNull("filter",this.getFilter()); out.putNotNull("headers",this.getHeaders()); - out.putNotNull("maxCount",this.getMaxCount()); + out.putNotNull("maxCountExpr",this.getMaxCountExpr()); out.putNotNull("newRecordInputProvider",this.getNewRecordInputProvider()); out.putNotNull("resourceIO",this.getResourceIO()); out.putNotNull("resourceLoader",this.getResourceLoader()); @@ -290,7 +290,7 @@ protected void copyTo(BatchFileReaderModel instance){ instance.setFilePath(this.getFilePath()); instance.setFilter(this.getFilter()); instance.setHeaders(this.getHeaders()); - instance.setMaxCount(this.getMaxCount()); + instance.setMaxCountExpr(this.getMaxCountExpr()); instance.setNewRecordInputProvider(this.getNewRecordInputProvider()); instance.setResourceIO(this.getResourceIO()); instance.setResourceLoader(this.getResourceLoader()); diff --git a/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchGeneratorModel.java b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchGeneratorModel.java new file mode 100644 index 000000000..603b122ab --- /dev/null +++ b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchGeneratorModel.java @@ -0,0 +1,107 @@ +package io.nop.batch.dsl.model._gen; + +import io.nop.commons.collections.KeyedList; //NOPMD NOSONAR - suppressed UnusedImports - Used for List Prop +import io.nop.core.lang.json.IJsonHandler; +import io.nop.batch.dsl.model.BatchGeneratorModel; +import io.nop.commons.util.ClassHelper; + + + +// tell cpd to start ignoring code - CPD-OFF +/** + * generate from /nop/schema/task/batch.xdef

+ * + */ +@SuppressWarnings({"PMD.UselessOverridingMethod","PMD.UnusedLocalVariable", + "PMD.UnnecessaryFullyQualifiedName","PMD.EmptyControlStatement","java:S116","java:S101","java:S1128","java:S1161"}) +public abstract class _BatchGeneratorModel extends io.nop.core.resource.component.AbstractComponentModel { + + /** + * + * xml name: genModelPath + * + */ + private java.lang.String _genModelPath ; + + /** + * + * xml name: totalCountExpr + * + */ + private io.nop.core.lang.eval.IEvalAction _totalCountExpr ; + + /** + * + * xml name: genModelPath + * + */ + + public java.lang.String getGenModelPath(){ + return _genModelPath; + } + + + public void setGenModelPath(java.lang.String value){ + checkAllowChange(); + + this._genModelPath = value; + + } + + + /** + * + * xml name: totalCountExpr + * + */ + + public io.nop.core.lang.eval.IEvalAction getTotalCountExpr(){ + return _totalCountExpr; + } + + + public void setTotalCountExpr(io.nop.core.lang.eval.IEvalAction value){ + checkAllowChange(); + + this._totalCountExpr = value; + + } + + + + @Override + public void freeze(boolean cascade){ + if(frozen()) return; + super.freeze(cascade); + + if(cascade){ //NOPMD - suppressed EmptyControlStatement - Auto Gen Code + + } + } + + @Override + protected void outputJson(IJsonHandler out){ + super.outputJson(out); + + out.putNotNull("genModelPath",this.getGenModelPath()); + out.putNotNull("totalCountExpr",this.getTotalCountExpr()); + } + + public BatchGeneratorModel cloneInstance(){ + BatchGeneratorModel instance = newInstance(); + this.copyTo(instance); + return instance; + } + + protected void copyTo(BatchGeneratorModel instance){ + super.copyTo(instance); + + instance.setGenModelPath(this.getGenModelPath()); + instance.setTotalCountExpr(this.getTotalCountExpr()); + } + + protected BatchGeneratorModel newInstance(){ + return (BatchGeneratorModel) ClassHelper.newInstance(getClass()); + } +} + // resume CPD analysis - CPD-ON diff --git a/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchLoaderModel.java b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchLoaderModel.java index 62c7119dc..b6af4eb95 100644 --- a/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchLoaderModel.java +++ b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchLoaderModel.java @@ -44,6 +44,13 @@ public abstract class _BatchLoaderModel extends io.nop.batch.dsl.model.BatchList */ private io.nop.batch.dsl.model.BatchFileReaderModel _fileReader ; + /** + * + * xml name: generator + * + */ + private io.nop.batch.dsl.model.BatchGeneratorModel _generator ; + /** * * xml name: jdbc-reader @@ -148,6 +155,25 @@ public void setFileReader(io.nop.batch.dsl.model.BatchFileReaderModel value){ } + /** + * + * xml name: generator + * + */ + + public io.nop.batch.dsl.model.BatchGeneratorModel getGenerator(){ + return _generator; + } + + + public void setGenerator(io.nop.batch.dsl.model.BatchGeneratorModel value){ + checkAllowChange(); + + this._generator = value; + + } + + /** * * xml name: jdbc-reader @@ -234,6 +260,8 @@ public void freeze(boolean cascade){ this._fileReader = io.nop.api.core.util.FreezeHelper.deepFreeze(this._fileReader); + this._generator = io.nop.api.core.util.FreezeHelper.deepFreeze(this._generator); + this._jdbcReader = io.nop.api.core.util.FreezeHelper.deepFreeze(this._jdbcReader); this._ormReader = io.nop.api.core.util.FreezeHelper.deepFreeze(this._ormReader); @@ -249,6 +277,7 @@ protected void outputJson(IJsonHandler out){ out.putNotNull("aggregator",this.getAggregator()); out.putNotNull("bean",this.getBean()); out.putNotNull("fileReader",this.getFileReader()); + out.putNotNull("generator",this.getGenerator()); out.putNotNull("jdbcReader",this.getJdbcReader()); out.putNotNull("ormReader",this.getOrmReader()); out.putNotNull("saveState",this.getSaveState()); @@ -268,6 +297,7 @@ protected void copyTo(BatchLoaderModel instance){ instance.setAggregator(this.getAggregator()); instance.setBean(this.getBean()); instance.setFileReader(this.getFileReader()); + instance.setGenerator(this.getGenerator()); instance.setJdbcReader(this.getJdbcReader()); instance.setOrmReader(this.getOrmReader()); instance.setSaveState(this.getSaveState()); diff --git a/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchTaskModel.java b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchTaskModel.java index cd04d7b3a..8d4aee9cd 100644 --- a/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchTaskModel.java +++ b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchTaskModel.java @@ -185,6 +185,13 @@ public abstract class _BatchTaskModel extends io.nop.core.resource.component.Abs */ private io.nop.batch.core.BatchTransactionScope _transactionScope ; + /** + * + * xml name: useBatchRequestGenerator + * + */ + private boolean _useBatchRequestGenerator = false; + /** * * xml name: allowStartIfComplete @@ -720,6 +727,25 @@ public void setTransactionScope(io.nop.batch.core.BatchTransactionScope value){ } + /** + * + * xml name: useBatchRequestGenerator + * + */ + + public boolean isUseBatchRequestGenerator(){ + return _useBatchRequestGenerator; + } + + + public void setUseBatchRequestGenerator(boolean value){ + checkAllowChange(); + + this._useBatchRequestGenerator = value; + + } + + @Override public void freeze(boolean cascade){ @@ -777,6 +803,7 @@ protected void outputJson(IJsonHandler out){ out.putNotNull("taskName",this.getTaskName()); out.putNotNull("taskVersion",this.getTaskVersion()); out.putNotNull("transactionScope",this.getTransactionScope()); + out.putNotNull("useBatchRequestGenerator",this.isUseBatchRequestGenerator()); } public BatchTaskModel cloneInstance(){ @@ -812,6 +839,7 @@ protected void copyTo(BatchTaskModel instance){ instance.setTaskName(this.getTaskName()); instance.setTaskVersion(this.getTaskVersion()); instance.setTransactionScope(this.getTransactionScope()); + instance.setUseBatchRequestGenerator(this.isUseBatchRequestGenerator()); } protected BatchTaskModel newInstance(){ diff --git a/nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/BatchGenConstants.java b/nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/BatchGenConstants.java index dfcd92dc7..7b7bfc450 100644 --- a/nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/BatchGenConstants.java +++ b/nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/BatchGenConstants.java @@ -7,12 +7,18 @@ */ package io.nop.batch.gen; +import io.nop.batch.core.BatchConstants; + public interface BatchGenConstants { String MODEL_TYPE_BATCH_GEN = "batch-gen"; String FILE_TYPE_GEN_JSON = "batch-gen.json"; String FILE_TYPE_GEN_JSON5 = "batch-gen.json5"; String FILE_TYPE_GEN_YAML = "batch-gen.yaml"; - String VAR_CHUNK_CONTEXT = "chunkContext"; + String POSTFIX_BATCH_GEN_XLSX = ".batch-gen.xlsx"; + + String XDSL_BATCH_GEN_IMP_PATH = "/nop/batch/imp/batch-gen.imp.xml"; + + String VAR_CHUNK_CONTEXT = BatchConstants.VAR_BATCH_CHUNK_CTX; String VAR_CHUNK_RESPONSE = "response"; } diff --git a/nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/loader/BatchGenLoader.java b/nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/loader/BatchGenLoaderProvider.java similarity index 80% rename from nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/loader/BatchGenLoader.java rename to nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/loader/BatchGenLoaderProvider.java index 013a857e4..988ecf6d2 100644 --- a/nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/loader/BatchGenLoader.java +++ b/nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/loader/BatchGenLoaderProvider.java @@ -7,6 +7,8 @@ */ package io.nop.batch.gen.loader; +import io.nop.api.core.convert.ConvertHelper; +import io.nop.api.core.exceptions.NopException; import io.nop.batch.core.IBatchChunkContext; import io.nop.batch.core.IBatchLoaderProvider; import io.nop.batch.core.IBatchTaskContext; @@ -16,24 +18,21 @@ import io.nop.batch.gen.generator.BatchGenState; import io.nop.batch.gen.model.BatchGenModel; import io.nop.batch.gen.model.BatchGenModelParser; +import io.nop.core.lang.eval.IEvalAction; import java.util.ArrayList; import java.util.List; -public class BatchGenLoader extends AbstractBatchResourceHandler +public class BatchGenLoaderProvider extends AbstractBatchResourceHandler implements IBatchLoaderProvider { /** * 总共生成多少条记录 */ - private long totalCount; + private IEvalAction totalCountExpr; - public long getTotalCount() { - return totalCount; - } - - public void setTotalCount(long totalCount) { - this.totalCount = totalCount; + public void setTotalCount(IEvalAction totalCountExpr) { + this.totalCountExpr = totalCountExpr; } static class LoaderState { @@ -44,6 +43,8 @@ static class LoaderState { @Override public IBatchLoader setup(IBatchTaskContext context) { + long totalCount = ConvertHelper.toPrimitiveLong(totalCountExpr.invoke(context), NopException::new); + LoaderState state = new LoaderState(); state.genModel = new BatchGenModelParser().parseFromResource(getResource(context)); state.genState = new BatchGenState(state.genModel, totalCount); diff --git a/nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/model/BatchGenModel.java b/nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/model/BatchGenModel.java index f3540cc6d..f05c85187 100644 --- a/nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/model/BatchGenModel.java +++ b/nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/model/BatchGenModel.java @@ -9,6 +9,8 @@ import io.nop.api.core.annotations.data.DataBean; import io.nop.api.core.beans.TreeBean; +import io.nop.api.core.util.INeedInit; +import io.nop.core.lang.json.delta.JsonMerger; import io.nop.core.resource.component.AbstractComponentModel; import io.nop.core.type.IGenericType; @@ -19,7 +21,7 @@ * 数据生成模型。用于按照指定的数据分布比例来批量生成一批测试数据。 */ @DataBean -public class BatchGenModel extends AbstractComponentModel implements IBatchGenCaseModel { +public class BatchGenModel extends AbstractComponentModel implements IBatchGenCaseModel, INeedInit { private String name; @@ -109,4 +111,44 @@ public List getSubCases() { public void setSubCases(List subCases) { this.subCases = subCases; } + + private boolean inited = false; + + @Override + public void init() { + if (inited) + return; + inited = true; + for (BatchGenCaseModel subCase : getSubCases()) { + mergeWithParent(subCase, this); + } + } + + void mergeWithParent(BatchGenCaseModel subCase, IBatchGenCaseModel parent) { + if (subCase.isInheritParent()) { + Map template = mergeMap(parent.getMergedTemplate(), subCase.getTemplate()); + subCase.setMergedTemplate(template); + + Map outputVars = mergeMap(parent.getMergedOutputVars(), subCase.getOutputVars()); + subCase.setMergedOutputVars(outputVars); + } else { + subCase.setMergedTemplate(subCase.getTemplate()); + subCase.setMergedOutputVars(subCase.getOutputVars()); + } + + if (subCase.getSubCases() != null) { + subCase.getSubCases().forEach(sub -> { + mergeWithParent(sub, subCase); + }); + } + } + + Map mergeMap(Map m1, Map m2) { + if (m1 == null || m1.isEmpty()) + return m2; + if (m2 == null || m1.isEmpty()) + return m1; + + return (Map) JsonMerger.instance().merge(m1, m2); + } } \ No newline at end of file diff --git a/nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/model/BatchGenModelParser.java b/nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/model/BatchGenModelParser.java index e124497c4..32dfa94cb 100644 --- a/nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/model/BatchGenModelParser.java +++ b/nop-batch/nop-batch-gen/src/main/java/io/nop/batch/gen/model/BatchGenModelParser.java @@ -7,55 +7,31 @@ */ package io.nop.batch.gen.model; +import io.nop.batch.gen.BatchGenConstants; import io.nop.core.lang.eval.EvalExprProvider; import io.nop.core.lang.json.DeltaJsonOptions; import io.nop.core.lang.json.JsonTool; -import io.nop.core.lang.json.delta.JsonMerger; import io.nop.core.resource.IResource; import io.nop.core.resource.component.parse.AbstractResourceParser; - -import java.util.Map; +import io.nop.xlang.xdsl.DslModelHelper; public class BatchGenModelParser extends AbstractResourceParser { @Override protected BatchGenModel doParseResource(IResource resource) { + if (resource.getName().endsWith(BatchGenConstants.POSTFIX_BATCH_GEN_XLSX)) { + BatchGenModel model = (BatchGenModel) DslModelHelper.newExcelModelLoader( + BatchGenConstants.XDSL_BATCH_GEN_IMP_PATH).parseFromResource(resource); + model.init(); + return model; + } + DeltaJsonOptions options = new DeltaJsonOptions(); options.setIgnoreUnknownValueResolver(false); options.setEvalContext(EvalExprProvider.newEvalScope()); BatchGenModel model = JsonTool.loadDeltaBean(resource, BatchGenModel.class, options); - - for (BatchGenCaseModel subCase : model.getSubCases()) { - mergeWithParent(subCase, model); - } + model.init(); return model; } - void mergeWithParent(BatchGenCaseModel subCase, IBatchGenCaseModel parent) { - if (subCase.isInheritParent()) { - Map template = mergeMap(parent.getMergedTemplate(), subCase.getTemplate()); - subCase.setMergedTemplate(template); - - Map outputVars = mergeMap(parent.getMergedOutputVars(), subCase.getOutputVars()); - subCase.setMergedOutputVars(outputVars); - } else { - subCase.setMergedTemplate(subCase.getTemplate()); - subCase.setMergedOutputVars(subCase.getOutputVars()); - } - - if (subCase.getSubCases() != null) { - subCase.getSubCases().forEach(sub -> { - mergeWithParent(sub, subCase); - }); - } - } - - Map mergeMap(Map m1, Map m2) { - if (m1 == null || m1.isEmpty()) - return m2; - if (m2 == null || m1.isEmpty()) - return m1; - - return (Map) JsonMerger.instance().merge(m1, m2); - } } diff --git a/nop-batch/nop-batch-gen/src/main/resources/_vfs/nop/batch/imp/batch-gen.imp.xml b/nop-batch/nop-batch-gen/src/main/resources/_vfs/nop/batch/imp/batch-gen.imp.xml index e53bb9376..c4da72817 100644 --- a/nop-batch/nop-batch-gen/src/main/resources/_vfs/nop/batch/imp/batch-gen.imp.xml +++ b/nop-batch/nop-batch-gen/src/main/resources/_vfs/nop/batch/imp/batch-gen.imp.xml @@ -1,229 +1,115 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - value == 'M' || value == 'Y' || value == true || value == 'true' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - + function normalizeCase(subCase){ + subCase.subCases = subCase.subCases?.map(caseItem=>{ + caseItem.subCases?.map(caseName=>{ + return $.notNull(subCaseMap[caseName],"invalid caseName:"+caseName); + }); + caseItem.removeProp('subCases'); + }); + } - - - + rootRecord.subCases.$('sss')?.forEach(subCase=>{ + subCaseMap[subCase.name] = subCase; + }); - - - + rootRecord.subCaseMap?.forEach(normalizeCase); - - - - - - - + rootRecord.removeProp('subCaseMap'); - - - - ]]> - + + + - - + + - - + + - - + + - - + + - - + + - + - + - - - - - value == 'M' || value == 'Y' || value == true || value == 'true' - - + + - + - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - + - - + + - - + + - - + + + + + + - - - + - - - + - - - + + + - + + diff --git a/nop-bom/pom.xml b/nop-bom/pom.xml index 430c0c7d1..d342b998d 100644 --- a/nop-bom/pom.xml +++ b/nop-bom/pom.xml @@ -564,6 +564,12 @@ ${nop-entropy.version} + + io.github.entropy-cloud + nop-batch-dsl + ${nop-entropy.version} + + io.github.entropy-cloud nop-batch-orm diff --git a/nop-cli-core/pom.xml b/nop-cli-core/pom.xml index 7420d5500..a7d449276 100644 --- a/nop-cli-core/pom.xml +++ b/nop-cli-core/pom.xml @@ -100,6 +100,11 @@ nop-task-core + + io.github.entropy-cloud + nop-batch-dsl + + diff --git a/nop-cli-core/src/test/java/io/nop/cli/TestNopCli.java b/nop-cli-core/src/test/java/io/nop/cli/TestNopCli.java index 774fa1f91..b90b53b54 100644 --- a/nop-cli-core/src/test/java/io/nop/cli/TestNopCli.java +++ b/nop-cli-core/src/test/java/io/nop/cli/TestNopCli.java @@ -8,17 +8,21 @@ package io.nop.cli; import io.nop.api.core.config.AppConfig; +import io.nop.core.CoreConfigs; import io.nop.core.initialize.CoreInitialization; +import io.nop.core.unittest.BaseTestCase; import io.quarkus.test.junit.QuarkusTest; import jakarta.inject.Inject; import org.junit.jupiter.api.Test; import picocli.CommandLine; +import java.io.File; + import static io.nop.codegen.CodeGenConfigs.CFG_CODEGEN_TRACE_ENABLED; import static org.junit.jupiter.api.Assertions.assertEquals; @QuarkusTest -public class TestNopCli { +public class TestNopCli extends BaseTestCase { @Inject CommandLine.IFactory factory; @@ -76,4 +80,17 @@ public void testRunTask(){ int ret = app.run(args); assertEquals(0, ret); } + + @Test + public void testRunBatchDemo(){ + CoreInitialization.destroy(); + File file = new File(getModuleDir(),"../nop-cli/demo/_vfs"); + System.setProperty(CoreConfigs.CFG_RESOURCE_DIR_OVERRIDE_VFS.getName(), file.getAbsolutePath()); + String[] args = new String[]{"run-task", "v:/batch/batch-demo.task.xml","-i","{totalCount:250,taskKey:'abc'}"}; + NopCliApplication app = new NopCliApplication(); + app.setFactory(factory); + int ret = app.run(args); + assertEquals(0, ret); + System.getProperties().remove(CoreConfigs.CFG_RESOURCE_DIR_OVERRIDE_VFS.getName()); + } } diff --git a/nop-cli/demo/_vfs/app/demo/_module b/nop-cli/demo/_vfs/app/demo/_module new file mode 100644 index 000000000..e69de29bb diff --git a/nop-cli/demo/_vfs/app/demo/orm/app.orm.xml b/nop-cli/demo/_vfs/app/demo/orm/app.orm.xml index c9951e75d..46b1443f6 100644 --- a/nop-cli/demo/_vfs/app/demo/orm/app.orm.xml +++ b/nop-cli/demo/_vfs/app/demo/orm/app.orm.xml @@ -1,7 +1,8 @@ + orm:forceDynamicEntity="true" + xmlns:i18n-en="i18n-en" xmlns:orm-gen="orm-gen" xmlns:xpl="xpl" xmlns:orm="orm"> - + \ No newline at end of file diff --git a/nop-cli/demo/_vfs/app/demo/orm/demo.orm.xlsx b/nop-cli/demo/_vfs/app/demo/orm/demo.orm.xlsx index 60fa43035..58570f48c 100644 Binary files a/nop-cli/demo/_vfs/app/demo/orm/demo.orm.xlsx and b/nop-cli/demo/_vfs/app/demo/orm/demo.orm.xlsx differ diff --git a/nop-cli/demo/_vfs/batch/batch-demo.task.xml b/nop-cli/demo/_vfs/batch/batch-demo.task.xml new file mode 100644 index 000000000..4c689730f --- /dev/null +++ b/nop-cli/demo/_vfs/batch/batch-demo.task.xml @@ -0,0 +1,28 @@ + + + + + + + + + taskKey + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/nop-cli/demo/_vfs/batch/create-card.batch-gen.xlsx b/nop-cli/demo/_vfs/batch/create-card.batch-gen.xlsx index 74b575e30..10eb92f64 100644 Binary files a/nop-cli/demo/_vfs/batch/create-card.batch-gen.xlsx and b/nop-cli/demo/_vfs/batch/create-card.batch-gen.xlsx differ diff --git a/nop-cli/demo/_vfs/batch/demo.orm.xlsx b/nop-cli/demo/_vfs/batch/demo.orm.xlsx deleted file mode 100644 index 9d7b38990..000000000 Binary files a/nop-cli/demo/_vfs/batch/demo.orm.xlsx and /dev/null differ diff --git a/nop-cli/demo/_vfs/nop/task/demo/create-card.task.xml b/nop-cli/demo/_vfs/nop/task/demo/create-card.task.xml new file mode 100644 index 000000000..e613a5e98 --- /dev/null +++ b/nop-cli/demo/_vfs/nop/task/demo/create-card.task.xml @@ -0,0 +1,47 @@ + + + + + + + + + + + + RESULT + + + + + + + RESULT + + + + + + \ No newline at end of file diff --git a/nop-commons/src/main/java/io/nop/commons/util/StringHelper.java b/nop-commons/src/main/java/io/nop/commons/util/StringHelper.java index 7cdfe9bac..185dd9ced 100644 --- a/nop-commons/src/main/java/io/nop/commons/util/StringHelper.java +++ b/nop-commons/src/main/java/io/nop/commons/util/StringHelper.java @@ -4456,4 +4456,12 @@ public static String filePathMd5(String filePath) { fileName = fileName.substring(0, 20); return md5Hash(filePath) + '-' + fileName; } + + @Deterministic + public static short shortHash(String str) { + if (str == null) + str = ""; + int hash = HashHelper.murmur3_32(str); + return (short) (hash % Short.MAX_VALUE); + } } \ No newline at end of file diff --git a/nop-excel/src/main/java/io/nop/excel/imp/ImportExcelParser.java b/nop-excel/src/main/java/io/nop/excel/imp/ImportExcelParser.java index fda75a10b..1f1473851 100644 --- a/nop-excel/src/main/java/io/nop/excel/imp/ImportExcelParser.java +++ b/nop-excel/src/main/java/io/nop/excel/imp/ImportExcelParser.java @@ -123,6 +123,10 @@ public Object parseFromWorkbook(ExcelWorkbook workbook) { } } + if (importModel.getNormalizeFieldsExpr() != null) { + importModel.getNormalizeFieldsExpr().invoke(scope); + } + if (importModel.getValidator() != null) { try { importModel.getValidator().invoke(scope); diff --git a/nop-excel/src/main/java/io/nop/excel/imp/model/_gen/_ImportModel.java b/nop-excel/src/main/java/io/nop/excel/imp/model/_gen/_ImportModel.java index 4e169fd18..1468da32e 100644 --- a/nop-excel/src/main/java/io/nop/excel/imp/model/_gen/_ImportModel.java +++ b/nop-excel/src/main/java/io/nop/excel/imp/model/_gen/_ImportModel.java @@ -51,6 +51,13 @@ public abstract class _ImportModel extends io.nop.core.resource.component.Abstra */ private boolean _ignoreUnknownSheet = false; + /** + * + * xml name: normalizeFieldsExpr + * + */ + private io.nop.core.lang.eval.IEvalAction _normalizeFieldsExpr ; + /** * * xml name: resultType @@ -181,6 +188,25 @@ public void setIgnoreUnknownSheet(boolean value){ } + /** + * + * xml name: normalizeFieldsExpr + * + */ + + public io.nop.core.lang.eval.IEvalAction getNormalizeFieldsExpr(){ + return _normalizeFieldsExpr; + } + + + public void setNormalizeFieldsExpr(io.nop.core.lang.eval.IEvalAction value){ + checkAllowChange(); + + this._normalizeFieldsExpr = value; + + } + + /** * * xml name: resultType @@ -324,6 +350,7 @@ protected void outputJson(IJsonHandler out){ out.putNotNull("defaultStripText",this.isDefaultStripText()); out.putNotNull("dump",this.isDump()); out.putNotNull("ignoreUnknownSheet",this.isIgnoreUnknownSheet()); + out.putNotNull("normalizeFieldsExpr",this.getNormalizeFieldsExpr()); out.putNotNull("resultType",this.getResultType()); out.putNotNull("sheets",this.getSheets()); out.putNotNull("templatePath",this.getTemplatePath()); @@ -345,6 +372,7 @@ protected void copyTo(ImportModel instance){ instance.setDefaultStripText(this.isDefaultStripText()); instance.setDump(this.isDump()); instance.setIgnoreUnknownSheet(this.isIgnoreUnknownSheet()); + instance.setNormalizeFieldsExpr(this.getNormalizeFieldsExpr()); instance.setResultType(this.getResultType()); instance.setSheets(this.getSheets()); instance.setTemplatePath(this.getTemplatePath()); diff --git a/nop-orm/src/main/java/io/nop/orm/utils/OrmDaoHelper.java b/nop-orm/src/main/java/io/nop/orm/utils/OrmDaoHelper.java new file mode 100644 index 000000000..f2072e126 --- /dev/null +++ b/nop-orm/src/main/java/io/nop/orm/utils/OrmDaoHelper.java @@ -0,0 +1,18 @@ +package io.nop.orm.utils; + +import io.nop.core.reflect.bean.BeanTool; +import io.nop.dao.api.DaoProvider; +import io.nop.dao.api.IEntityDao; +import io.nop.orm.IOrmEntity; + +import java.util.Map; + +public class OrmDaoHelper { + public static IOrmEntity saveEntity(String entityName, Map data) { + IEntityDao dao = DaoProvider.instance().dao(entityName); + IOrmEntity entity = dao.newEntity(); + BeanTool.instance().setProperties(entity, data); + dao.saveEntity(entity); + return entity; + } +} diff --git a/nop-orm/src/main/resources/_vfs/nop/orm/xlib/dao.xlib b/nop-orm/src/main/resources/_vfs/nop/orm/xlib/dao.xlib index 0ec16a7e4..2cb9981cb 100644 --- a/nop-orm/src/main/resources/_vfs/nop/orm/xlib/dao.xlib +++ b/nop-orm/src/main/resources/_vfs/nop/orm/xlib/dao.xlib @@ -112,5 +112,14 @@ ]]> + + + + + + import io.nop.orm.utils.OrmDaoHelper; + return OrmDaoHelper.saveEntity(entityName, data); + + \ No newline at end of file diff --git a/nop-orm/src/main/resources/_vfs/nop/orm/xlib/orm-gen.xlib b/nop-orm/src/main/resources/_vfs/nop/orm/xlib/orm-gen.xlib index c5e0bba11..10071c0b7 100644 --- a/nop-orm/src/main/resources/_vfs/nop/orm/xlib/orm-gen.xlib +++ b/nop-orm/src/main/resources/_vfs/nop/orm/xlib/orm-gen.xlib @@ -9,6 +9,7 @@ + @@ -18,6 +19,21 @@ + + + { + if(entityNode.attrCsvSet('tagSet')?.contains('kv-table')){ + entityNode.setAttr('className', 'io.nop.orm.support.DynamicOrmKeyValueTable'); + }else{ + entityNode.setAttr('className','io.nop.orm.support.DynamicOrmEntity'); + } + }); + } + ]]> + + diff --git a/nop-xdefs/src/main/resources/_vfs/nop/schema/excel/imp.xdef b/nop-xdefs/src/main/resources/_vfs/nop/schema/excel/imp.xdef index fcb96828d..7ca397486 100644 --- a/nop-xdefs/src/main/resources/_vfs/nop/schema/excel/imp.xdef +++ b/nop-xdefs/src/main/resources/_vfs/nop/schema/excel/imp.xdef @@ -18,6 +18,8 @@ + + diff --git a/nop-xdefs/src/main/resources/_vfs/nop/schema/task/batch.xdef b/nop-xdefs/src/main/resources/_vfs/nop/schema/task/batch.xdef index 9be2b8915..6e3342b1e 100644 --- a/nop-xdefs/src/main/resources/_vfs/nop/schema/task/batch.xdef +++ b/nop-xdefs/src/main/resources/_vfs/nop/schema/task/batch.xdef @@ -10,6 +10,7 @@ singleMode="boolean=false" singleSession="boolean" saveState="boolean" transactionScope="enum:io.nop.batch.core.BatchTransactionScope" rateLimit="double" jitterRatio="double" allowStartIfComplete="boolean" startLimit="!int=0" + useBatchRequestGenerator="!boolean=false" executor="bean-name" xdef:name="BatchTaskModel" xdef:bean-package="io.nop.batch.dsl.model" x:schema="/nop/schema/xdef.xdef" xmlns:x="/nop/schema/xdsl.xdef" xdef:model-name-prop="taskName" xdef:model-version-prop="taskVersion" @@ -61,7 +62,7 @@ @fileModelPath 文件模型路径。当没有指定resourceIO和newRecordInputProvider时，根据fileModelPath自动生成resourceIO --> + resourceLoader="bean-name" resourceIO="bean-name" maxCountExpr="expr" fileModelPath="v-path"> @@ -87,6 +88,9 @@ + + + diff --git a/nop-xlang/src/main/java/io/nop/xlang/xdef/domain/SimpleStdDomainHandlers.java b/nop-xlang/src/main/java/io/nop/xlang/xdef/domain/SimpleStdDomainHandlers.java index 6e1008805..ff3012f5e 100644 --- a/nop-xlang/src/main/java/io/nop/xlang/xdef/domain/SimpleStdDomainHandlers.java +++ b/nop-xlang/src/main/java/io/nop/xlang/xdef/domain/SimpleStdDomainHandlers.java @@ -654,6 +654,7 @@ public Object parseProp(IStdDomainOptions options, SourceLocation loc, String pr JsonParseOptions opts = new JsonParseOptions(); opts.setKeepLocation(true); + opts.setStrictMode(false); try { return JsonTool.instance().parseFromText(loc, text.toString(), opts); } catch (Exception e) { diff --git a/nop-xlang/src/main/java/io/nop/xlang/xdsl/XDslExtender.java b/nop-xlang/src/main/java/io/nop/xlang/xdsl/XDslExtender.java index 7fe7e4090..296b7acdc 100644 --- a/nop-xlang/src/main/java/io/nop/xlang/xdsl/XDslExtender.java +++ b/nop-xlang/src/main/java/io/nop/xlang/xdsl/XDslExtender.java @@ -351,7 +351,8 @@ private List genCpExtends(IXDefinition def, XNode root, XNode node, if (AppConfig.isDebugMode()) addPathRef(extendsNode, currentPath); - List genExtends = extendsNode.detachChildren(); + List genExtends = !extendsNode.isDummyNode() ? + Collections.singletonList(extendsNode) : extendsNode.detachChildren(); List ret = new ArrayList<>(genExtends.size()); for (XNode gen : genExtends) { if (XplParseHelper.getAttrBool(gen, keys.DUMP, false)) {