From dd229e531f7e31af283f3e504bdcd32f3811f4e0 Mon Sep 17 00:00:00 2001 From: inv-jishnu <31100916+inv-jishnu@users.noreply.github.com> Date: Fri, 31 Jan 2025 10:29:23 +0530 Subject: [PATCH] Add data chunk and task result enums and dtos (#2442) Co-authored-by: Peckstadt Yves --- .../dataimport/datachunk/ImportDataChunk.java | 14 +++++++ .../datachunk/ImportDataChunkStatus.java | 41 +++++++++++++++++++ .../datachunk/ImportDataChunkStatusState.java | 8 ++++ .../core/dataimport/datachunk/ImportRow.java | 11 +++++ .../dataimport/task/ImportTaskAction.java | 7 ++++ .../task/result/ImportResultStatus.java | 12 ++++++ .../task/result/ImportTargetResult.java | 19 +++++++++ .../task/result/ImportTargetResultStatus.java | 12 ++++++ .../task/result/ImportTaskResult.java | 25 +++++++++++ .../ImportTransactionBatch.java | 14 +++++++ .../ImportTransactionBatchResult.java | 32 +++++++++++++++ .../ImportTransactionBatchStatus.java | 18 ++++++++ 12 files changed, 213 insertions(+) create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunk.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatus.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatusState.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportRow.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTaskAction.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportResultStatus.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTargetResult.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTargetResultStatus.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTaskResult.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatch.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatchResult.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatchStatus.java diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunk.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunk.java new file mode 100644 index 000000000..69ed97421 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunk.java @@ -0,0 +1,14 @@ +package com.scalar.db.dataloader.core.dataimport.datachunk; + +import java.util.List; +import lombok.Builder; +import lombok.Data; + +/** * Import data chunk data */ +@Data +@Builder +public class ImportDataChunk { + + int dataChunkId; + List sourceData; +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatus.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatus.java new file mode 100644 index 000000000..d6db3e1e7 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatus.java @@ -0,0 +1,41 @@ +package com.scalar.db.dataloader.core.dataimport.datachunk; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.time.Instant; +import lombok.Builder; +import lombok.Data; + +/** * A DTO to store import data chunk details */ +@Data +@Builder +@JsonDeserialize(builder = ImportDataChunkStatus.ImportDataChunkStatusBuilder.class) +public class ImportDataChunkStatus { + + @JsonProperty("dataChunkId") + private final int dataChunkId; + + @JsonProperty("startTime") + private final Instant startTime; + + @JsonProperty("endTime") + private final Instant endTime; + + @JsonProperty("totalRecords") + private final int totalRecords; + + @JsonProperty("successCount") + private final int successCount; + + @JsonProperty("failureCount") + private final int failureCount; + + @JsonProperty("batchCount") + private final int batchCount; + + @JsonProperty("totalDurationInMilliSeconds") + private final int totalDurationInMilliSeconds; + + @JsonProperty("status") + private final ImportDataChunkStatusState status; +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatusState.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatusState.java new file mode 100644 index 000000000..9a6398a45 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatusState.java @@ -0,0 +1,8 @@ +package com.scalar.db.dataloader.core.dataimport.datachunk; + +/** * Status of the import data chunk during the import process */ +public enum ImportDataChunkStatusState { + START, + IN_PROGRESS, + COMPLETE +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportRow.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportRow.java new file mode 100644 index 000000000..824ca4ffa --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportRow.java @@ -0,0 +1,11 @@ +package com.scalar.db.dataloader.core.dataimport.datachunk; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Value; + +/** Stores data related to a single row on import file */ +@Value +public class ImportRow { + int rowNumber; + JsonNode sourceData; +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTaskAction.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTaskAction.java new file mode 100644 index 000000000..6b8c95d2a --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTaskAction.java @@ -0,0 +1,7 @@ +package com.scalar.db.dataloader.core.dataimport.task; + +/** * Import task actions */ +public enum ImportTaskAction { + INSERT, + UPDATE, +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportResultStatus.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportResultStatus.java new file mode 100644 index 000000000..70c03b9c3 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportResultStatus.java @@ -0,0 +1,12 @@ +package com.scalar.db.dataloader.core.dataimport.task.result; + +public enum ImportResultStatus { + SUCCESS, + PARTIAL_SUCCESS, + FAILURE, + VALIDATION_FAILED, + RETRIEVAL_FAILED, + MAPPING_FAILED, + TIMEOUT, + CANCELLED +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTargetResult.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTargetResult.java new file mode 100644 index 000000000..0fe4e0379 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTargetResult.java @@ -0,0 +1,19 @@ +package com.scalar.db.dataloader.core.dataimport.task.result; + +import com.fasterxml.jackson.databind.JsonNode; +import com.scalar.db.dataloader.core.dataimport.task.ImportTaskAction; +import java.util.List; +import lombok.Builder; +import lombok.Value; + +@Builder +@Value +public class ImportTargetResult { + String namespace; + String tableName; + ImportTaskAction importAction; + List errors; + boolean dataMapped; + JsonNode importedRecord; + ImportTargetResultStatus status; +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTargetResultStatus.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTargetResultStatus.java new file mode 100644 index 000000000..d774f8f82 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTargetResultStatus.java @@ -0,0 +1,12 @@ +package com.scalar.db.dataloader.core.dataimport.task.result; + +public enum ImportTargetResultStatus { + VALIDATION_FAILED, + RETRIEVAL_FAILED, + MISSING_COLUMNS, + DATA_ALREADY_EXISTS, + DATA_NOT_FOUND, + SAVE_FAILED, + SAVED, + ABORTED +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTaskResult.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTaskResult.java new file mode 100644 index 000000000..3e08cc709 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTaskResult.java @@ -0,0 +1,25 @@ +package com.scalar.db.dataloader.core.dataimport.task.result; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.util.List; +import lombok.Builder; +import lombok.Value; + +@Builder +@Value +@JsonDeserialize(builder = ImportTaskResult.ImportTaskResultBuilder.class) +public class ImportTaskResult { + @JsonProperty("rowNumber") + int rowNumber; + + @JsonProperty("targets") + List targets; + + @JsonProperty("rawRecord") + JsonNode rawRecord; + + @JsonProperty("dataChunkId") + int dataChunkId; +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatch.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatch.java new file mode 100644 index 000000000..a922fd8af --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatch.java @@ -0,0 +1,14 @@ +package com.scalar.db.dataloader.core.dataimport.transactionbatch; + +import com.scalar.db.dataloader.core.dataimport.datachunk.ImportRow; +import java.util.List; +import lombok.Builder; +import lombok.Value; + +/** Transaction batch details */ +@Builder +@Value +public class ImportTransactionBatch { + int transactionBatchId; + List sourceData; +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatchResult.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatchResult.java new file mode 100644 index 000000000..0e44b6695 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatchResult.java @@ -0,0 +1,32 @@ +package com.scalar.db.dataloader.core.dataimport.transactionbatch; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.scalar.db.dataloader.core.dataimport.task.result.ImportTaskResult; +import java.util.List; +import lombok.Builder; +import lombok.Value; + +/** Transaction batch result */ +@Builder +@Value +@JsonDeserialize(builder = ImportTransactionBatchResult.ImportTransactionBatchResultBuilder.class) +public class ImportTransactionBatchResult { + @JsonProperty("dataChunkId") + int dataChunkId; + + @JsonProperty("transactionBatchId") + int transactionBatchId; + + @JsonProperty("transactionId") + String transactionId; + + @JsonProperty("records") + List records; + + @JsonProperty("errors") + List errors; + + @JsonProperty("success") + boolean success; +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatchStatus.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatchStatus.java new file mode 100644 index 000000000..1b7bae34c --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatchStatus.java @@ -0,0 +1,18 @@ +package com.scalar.db.dataloader.core.dataimport.transactionbatch; + +import com.scalar.db.dataloader.core.dataimport.task.result.ImportTaskResult; +import java.util.List; +import lombok.Builder; +import lombok.Value; + +/** Batch status details */ +@Builder +@Value +public class ImportTransactionBatchStatus { + int dataChunkId; + int transactionBatchId; + String transactionId; + List records; + List errors; + boolean success; +}