diff --git a/core/src/main/java/com/scalar/db/common/error/CoreError.java b/core/src/main/java/com/scalar/db/common/error/CoreError.java index b18c487e6..3992cb6c2 100644 --- a/core/src/main/java/com/scalar/db/common/error/CoreError.java +++ b/core/src/main/java/com/scalar/db/common/error/CoreError.java @@ -765,6 +765,44 @@ public enum CoreError implements ScalarDbError { DATA_LOADER_TABLE_METADATA_RETRIEVAL_FAILED( Category.USER_ERROR, "0166", "Failed to retrieve table metadata. Details: %s", "", ""), + DATA_LOADER_DUPLICATE_DATA_MAPPINGS( + Category.USER_ERROR, + "0167", + "Duplicate data mappings found for table '%s' in the control file", + "", + ""), + DATA_LOADER_MISSING_COLUMN_MAPPING( + Category.USER_ERROR, + "0168", + "No mapping found for column '%s' in table '%s' in the control file. Control file validation set at 'FULL'. All columns need to be mapped.", + "", + ""), + DATA_LOADER_CONTROL_FILE_MISSING_DATA_MAPPINGS( + Category.USER_ERROR, "0169", "The control file is missing data mappings", "", ""), + DATA_LOADER_TARGET_COLUMN_NOT_FOUND( + Category.USER_ERROR, + "0170", + "The target column '%s' for source field '%s' could not be found in table '%s'", + "", + ""), + DATA_LOADER_MISSING_PARTITION_KEY( + Category.USER_ERROR, + "0171", + "The required partition key '%s' is missing in the control file mapping for table '%s'", + "", + ""), + DATA_LOADER_MISSING_CLUSTERING_KEY( + Category.USER_ERROR, + "0172", + "The required clustering key '%s' is missing in the control file mapping for table '%s'", + "", + ""), + DATA_LOADER_MULTIPLE_MAPPINGS_FOR_COLUMN_FOUND( + Category.USER_ERROR, + "0173", + "Duplicated data mappings found for column '%s' in table '%s'", + "", + ""), // // Errors for the concurrency error category // diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/controlfile/ControlFile.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/controlfile/ControlFile.java new file mode 100644 index 000000000..bfa66b345 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/controlfile/ControlFile.java @@ -0,0 +1,27 @@ +package com.scalar.db.dataloader.core.dataimport.controlfile; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +import lombok.Setter; + +/** Represents the control file */ +@Getter +@Setter +public class ControlFile { + + @JsonProperty("tables") + private final List tables; + + /** Class constructor */ + public ControlFile() { + this.tables = new ArrayList<>(); + } + + @JsonCreator + public ControlFile(@JsonProperty("tables") List tables) { + this.tables = tables; + } +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/controlfile/ControlFileValidationException.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/controlfile/ControlFileValidationException.java new file mode 100644 index 000000000..e4e032a4c --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/controlfile/ControlFileValidationException.java @@ -0,0 +1,14 @@ +package com.scalar.db.dataloader.core.dataimport.controlfile; + +/** Represents the control file */ +public class ControlFileValidationException extends Exception { + + /** + * Class constructor + * + * @param message error message + */ + public ControlFileValidationException(String message) { + super(message); + } +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/controlfile/ControlFileValidator.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/controlfile/ControlFileValidator.java new file mode 100644 index 000000000..6c31a851b --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/controlfile/ControlFileValidator.java @@ -0,0 +1,225 @@ +package com.scalar.db.dataloader.core.dataimport.controlfile; + +import com.scalar.db.api.TableMetadata; +import com.scalar.db.common.error.CoreError; +import com.scalar.db.dataloader.core.util.RuntimeUtil; +import com.scalar.db.dataloader.core.util.TableMetadataUtil; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +/** Class to validate a control file */ +public class ControlFileValidator { + + /** + * Validate a control file + * + * @param controlFile Control file instance + * @param controlFileValidationMode Defines the strictness of the control file validation + * @param tableMetadataMap Metadata for one or more ScalarDB tables + * @throws ControlFileValidationException when the control file is invalid + */ + public static void validate( + ControlFile controlFile, + ControlFileValidationLevel controlFileValidationMode, + Map tableMetadataMap) + throws ControlFileValidationException { + + // Method argument null check + RuntimeUtil.checkNotNull(controlFile, controlFileValidationMode, tableMetadataMap); + + // Make sure the control file is not empty + checkEmptyMappings(controlFile); + + // Table metadata existence and target column validation + Set uniqueTables = new HashSet<>(); + for (ControlFileTable controlFileTable : controlFile.getTables()) { + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + + // Make sure that multiple table mappings for one table do not exist + if (uniqueTables.contains(lookupKey)) { + throw new ControlFileValidationException( + CoreError.DATA_LOADER_DUPLICATE_DATA_MAPPINGS.buildMessage(lookupKey)); + } + uniqueTables.add(lookupKey); + + // Make sure no column is mapped multiple times + Set mappedTargetColumns = checkDuplicateColumnMappings(controlFileTable); + + // Make sure table metadata is provided for each table mentioned in the data mappings + checkMultiTableMetadata(tableMetadataMap, controlFileTable); + + TableMetadata tableMetadata = tableMetadataMap.get(lookupKey); + + // Make sure the specified target columns in the mappings actually exist + checkIfTargetColumnExist(tableMetadata, controlFileTable); + + // Make sure all table columns are mapped + if (controlFileValidationMode == ControlFileValidationLevel.FULL) { + checkIfAllColumnsAreMapped(tableMetadata, mappedTargetColumns, controlFileTable); + continue; + } + + // Make sure all keys (partition keys and clustering keys) are mapped + if (controlFileValidationMode == ControlFileValidationLevel.KEYS) { + checkPartitionKeys(tableMetadata, mappedTargetColumns, controlFileTable); + checkClusteringKeys(tableMetadata, mappedTargetColumns, controlFileTable); + } + } + } + + /** + * Check that all table columns are mapped in the control file. Ran only when the control file + * validation mode is set to 'FULL' + * + * @param tableMetadata Metadata for one ScalarDB table + * @param mappedTargetColumns All target columns that are mapped in the control file + * @param controlFileTable Control file entry for one ScalarDB table + * @throws ControlFileValidationException when there is a column that is not mapped in the control + * file + */ + private static void checkIfAllColumnsAreMapped( + TableMetadata tableMetadata, + Set mappedTargetColumns, + ControlFileTable controlFileTable) + throws ControlFileValidationException { + LinkedHashSet columnNames = tableMetadata.getColumnNames(); + for (String columnName : columnNames) { + if (!mappedTargetColumns.contains(columnName)) { + throw new ControlFileValidationException( + CoreError.DATA_LOADER_MISSING_COLUMN_MAPPING.buildMessage( + columnName, TableMetadataUtil.getTableLookupKey(controlFileTable))); + } + } + } + + /** + * Check that the control file has mappings for at least one table + * + * @param controlFile Control file instance + * @throws ControlFileValidationException when the control file has no mappings for any table + */ + private static void checkEmptyMappings(ControlFile controlFile) + throws ControlFileValidationException { + // Make sure data mapping for at least one table is provided + if (controlFile.getTables().isEmpty()) { + throw new ControlFileValidationException( + CoreError.DATA_LOADER_CONTROL_FILE_MISSING_DATA_MAPPINGS.buildMessage()); + } + } + + /** + * Check that metadata is provided for each table that is mapped in the control file. If the table + * metadata is missing this probably means the namespace and table combination does not exist. + * + * @param tableMetadataMap Metadata for one or more ScalarDB tables + * @param controlFileTable Control file entry for one ScalarDB table + * @throws ControlFileValidationException when metadata for a mapped table is missing + */ + private static void checkMultiTableMetadata( + Map tableMetadataMap, ControlFileTable controlFileTable) + throws ControlFileValidationException { + // Make sure table metadata is available for each table data mapping + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + if (!tableMetadataMap.containsKey(lookupKey)) { + throw new ControlFileValidationException( + CoreError.DATA_LOADER_MISSING_NAMESPACE_OR_TABLE.buildMessage( + controlFileTable.getNamespace(), controlFileTable.getTable())); + } + } + + /** + * Check that the mapped target column exists in the provided table metadata. + * + * @param tableMetadata Metadata for the table + * @param controlFileTable Control file entry for one ScalarDB table + * @throws ControlFileValidationException when the target column does not exist + */ + private static void checkIfTargetColumnExist( + TableMetadata tableMetadata, ControlFileTable controlFileTable) + throws ControlFileValidationException { + + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + LinkedHashSet columnNames = tableMetadata.getColumnNames(); + + for (ControlFileTableFieldMapping mapping : controlFileTable.getMappings()) { + // Make sure the target fields are found in the table metadata + if (!columnNames.contains(mapping.getTargetColumn())) { + throw new ControlFileValidationException( + CoreError.DATA_LOADER_TARGET_COLUMN_NOT_FOUND.buildMessage( + mapping.getTargetColumn(), mapping.getSourceField(), lookupKey)); + } + } + } + + /** + * Check that the required partition keys are mapped in the control file. Ran only for control + * file validation mode KEYS and FULL. + * + * @param tableMetadata Metadata for one ScalarDB table + * @param mappedTargetColumns Set of target columns that are mapped in the control file + * @param controlFileTable Control file entry for one ScalarDB table + * @throws ControlFileValidationException when a partition key is not mapped + */ + private static void checkPartitionKeys( + TableMetadata tableMetadata, + Set mappedTargetColumns, + ControlFileTable controlFileTable) + throws ControlFileValidationException { + LinkedHashSet partitionKeyNames = tableMetadata.getPartitionKeyNames(); + for (String partitionKeyName : partitionKeyNames) { + if (!mappedTargetColumns.contains(partitionKeyName)) { + throw new ControlFileValidationException( + CoreError.DATA_LOADER_MISSING_PARTITION_KEY.buildMessage( + partitionKeyName, TableMetadataUtil.getTableLookupKey(controlFileTable))); + } + } + } + + /** + * Check that the required clustering keys are mapped in the control file. Ran only for control + * file validation mode KEYS and FULL. + * + * @param tableMetadata Metadata for one ScalarDB table + * @param mappedTargetColumns Set of target columns that are mapped in the control file + * @param controlFileTable Control file entry for one ScalarDB table + * @throws ControlFileValidationException when a clustering key is not mapped + */ + private static void checkClusteringKeys( + TableMetadata tableMetadata, + Set mappedTargetColumns, + ControlFileTable controlFileTable) + throws ControlFileValidationException { + LinkedHashSet clusteringKeyNames = tableMetadata.getClusteringKeyNames(); + for (String clusteringKeyName : clusteringKeyNames) { + if (!mappedTargetColumns.contains(clusteringKeyName)) { + throw new ControlFileValidationException( + CoreError.DATA_LOADER_MISSING_CLUSTERING_KEY.buildMessage( + clusteringKeyName, TableMetadataUtil.getTableLookupKey(controlFileTable))); + } + } + } + + /** + * Check that a control file table mapping does not contain duplicate mappings for the same target + * column + * + * @param controlFileTable Control file entry for one ScalarDB table + * @return Set of uniquely mapped target columns + * @throws ControlFileValidationException when a duplicate mapping is found + */ + private static Set checkDuplicateColumnMappings(ControlFileTable controlFileTable) + throws ControlFileValidationException { + Set mappedTargetColumns = new HashSet<>(); + for (ControlFileTableFieldMapping mapping : controlFileTable.getMappings()) { + if (mappedTargetColumns.contains(mapping.getTargetColumn())) { + throw new ControlFileValidationException( + CoreError.DATA_LOADER_MULTIPLE_MAPPINGS_FOR_COLUMN_FOUND.buildMessage( + mapping.getTargetColumn(), TableMetadataUtil.getTableLookupKey(controlFileTable))); + } + mappedTargetColumns.add(mapping.getTargetColumn()); + } + return mappedTargetColumns; + } +} diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/controlfile/ControlFileValidatorTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/controlfile/ControlFileValidatorTest.java new file mode 100644 index 000000000..d5dbd654c --- /dev/null +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataimport/controlfile/ControlFileValidatorTest.java @@ -0,0 +1,564 @@ +package com.scalar.db.dataloader.core.dataimport.controlfile; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.scalar.db.api.TableMetadata; +import com.scalar.db.common.error.CoreError; +import com.scalar.db.dataloader.core.util.TableMetadataUtil; +import com.scalar.db.io.DataType; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class ControlFileValidatorTest { + + private static final String TABLE_NAME = "table"; + + private static final String TABLE_NAME_TWO = "table_two"; + private static final String NAMESPACE = "ns"; + private static final String COLUMN_PARTITION_KEY = "pk"; + private static final String COLUMN_CLUSTERING_KEY = "ck"; + private static final String COLUMN_ONE = "c1"; + + @Test + void validate_nullValuesGiven_shouldThrowNullPointerException() { + assertThatThrownBy(() -> ControlFileValidator.validate(null, null, null)) + .isExactlyInstanceOf(NullPointerException.class) + .hasMessage(CoreError.DATA_LOADER_ERROR_METHOD_NULL_ARGUMENT.buildMessage()); + } + + @Test + void validate_noTableMappingsGiven_shouldThrowControlFileValidationException() { + ControlFile controlFile = new ControlFile(); + Map tableMetadataMap = new HashMap<>(); + assertThatThrownBy( + () -> + ControlFileValidator.validate( + controlFile, ControlFileValidationLevel.FULL, tableMetadataMap)) + .isExactlyInstanceOf(ControlFileValidationException.class) + .hasMessage(CoreError.DATA_LOADER_CONTROL_FILE_MISSING_DATA_MAPPINGS.buildMessage()); + } + + @Test + void validate_duplicateTableMappingsGiven_shouldThrowControlFileValidationException() { + ControlFile controlFile = new ControlFile(); + ControlFileTable controlFileTable = new ControlFileTable(NAMESPACE, TABLE_NAME); + controlFile.getTables().add(controlFileTable); + controlFile.getTables().add(controlFileTable); + + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(COLUMN_PARTITION_KEY, DataType.TEXT) + .addPartitionKey(COLUMN_PARTITION_KEY) + .build(); + + Map tableMetadataMap = new HashMap<>(); + tableMetadataMap.put(lookupKey, tableMetadata); + + assertThatThrownBy( + () -> + ControlFileValidator.validate( + controlFile, ControlFileValidationLevel.MAPPED, tableMetadataMap)) + .isExactlyInstanceOf(ControlFileValidationException.class) + .hasMessage(CoreError.DATA_LOADER_DUPLICATE_DATA_MAPPINGS.buildMessage(lookupKey)); + } + + @Test + void validate_duplicateTableColumnMappingsGiven_shouldThrowControlFileValidationException() { + ControlFile controlFile = new ControlFile(); + ControlFileTable controlFileTable = new ControlFileTable(NAMESPACE, TABLE_NAME); + controlFileTable.getMappings().add(new ControlFileTableFieldMapping(COLUMN_ONE, COLUMN_ONE)); + controlFileTable.getMappings().add(new ControlFileTableFieldMapping(COLUMN_ONE, COLUMN_ONE)); + controlFile.getTables().add(controlFileTable); + + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(COLUMN_PARTITION_KEY, DataType.TEXT) + .addPartitionKey(COLUMN_PARTITION_KEY) + .build(); + + Map tableMetadataMap = new HashMap<>(); + tableMetadataMap.put(lookupKey, tableMetadata); + + assertThatThrownBy( + () -> + ControlFileValidator.validate( + controlFile, ControlFileValidationLevel.MAPPED, tableMetadataMap)) + .isExactlyInstanceOf(ControlFileValidationException.class) + .hasMessage( + CoreError.DATA_LOADER_MULTIPLE_MAPPINGS_FOR_COLUMN_FOUND.buildMessage( + COLUMN_ONE, lookupKey)); + } + + @Test + void validate_missingTableMetadataGiven_shouldThrowControlFileValidationException() { + ControlFile controlFile = new ControlFile(); + ControlFileTable controlFileTable = new ControlFileTable(NAMESPACE, TABLE_NAME); + controlFileTable.getMappings().add(new ControlFileTableFieldMapping(COLUMN_ONE, COLUMN_ONE)); + controlFile.getTables().add(controlFileTable); + + Map tableMetadataMap = new HashMap<>(); + + assertThatThrownBy( + () -> + ControlFileValidator.validate( + controlFile, ControlFileValidationLevel.MAPPED, tableMetadataMap)) + .isExactlyInstanceOf(ControlFileValidationException.class) + .hasMessage( + CoreError.DATA_LOADER_MISSING_NAMESPACE_OR_TABLE.buildMessage( + controlFileTable.getNamespace(), controlFileTable.getTable())); + } + + @Test + void validate_nonExistingTargetColumnGiven_shouldThrowControlFileValidationException() { + ControlFile controlFile = new ControlFile(); + ControlFileTable controlFileTable = new ControlFileTable(NAMESPACE, TABLE_NAME); + controlFileTable.getMappings().add(new ControlFileTableFieldMapping(COLUMN_ONE, COLUMN_ONE)); + controlFile.getTables().add(controlFileTable); + + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(COLUMN_PARTITION_KEY, DataType.TEXT) + .addPartitionKey(COLUMN_PARTITION_KEY) + .build(); + Map tableMetadataMap = new HashMap<>(); + tableMetadataMap.put(lookupKey, tableMetadata); + + assertThatThrownBy( + () -> + ControlFileValidator.validate( + controlFile, ControlFileValidationLevel.MAPPED, tableMetadataMap)) + .isExactlyInstanceOf(ControlFileValidationException.class) + .hasMessage( + CoreError.DATA_LOADER_TARGET_COLUMN_NOT_FOUND.buildMessage( + COLUMN_ONE, COLUMN_ONE, lookupKey)); + } + + @Test + void + validate_fullValidationAndHasMissingMappedColumnsGiven_shouldThrowControlFileValidationException() { + ControlFile controlFile = new ControlFile(); + ControlFileTable controlFileTable = new ControlFileTable(NAMESPACE, TABLE_NAME); + controlFileTable.getMappings().add(new ControlFileTableFieldMapping(COLUMN_ONE, COLUMN_ONE)); + controlFile.getTables().add(controlFileTable); + + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(COLUMN_PARTITION_KEY, DataType.TEXT) + .addColumn(COLUMN_ONE, DataType.TEXT) + .addPartitionKey(COLUMN_PARTITION_KEY) + .build(); + Map tableMetadataMap = new HashMap<>(); + tableMetadataMap.put(lookupKey, tableMetadata); + + assertThatThrownBy( + () -> + ControlFileValidator.validate( + controlFile, ControlFileValidationLevel.FULL, tableMetadataMap)) + .isExactlyInstanceOf(ControlFileValidationException.class) + .hasMessage( + CoreError.DATA_LOADER_MISSING_COLUMN_MAPPING.buildMessage( + COLUMN_PARTITION_KEY, lookupKey)); + } + + @Test + void + validate_keysValidationAndHasMissingMappedPartitionKeysGiven_shouldThrowControlFileValidationException() { + ControlFile controlFile = new ControlFile(); + ControlFileTable controlFileTable = new ControlFileTable(NAMESPACE, TABLE_NAME); + controlFileTable.getMappings().add(new ControlFileTableFieldMapping(COLUMN_ONE, COLUMN_ONE)); + controlFile.getTables().add(controlFileTable); + + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(COLUMN_PARTITION_KEY, DataType.TEXT) + .addColumn(COLUMN_ONE, DataType.TEXT) + .addPartitionKey(COLUMN_PARTITION_KEY) + .build(); + Map tableMetadataMap = new HashMap<>(); + tableMetadataMap.put(lookupKey, tableMetadata); + + assertThatThrownBy( + () -> + ControlFileValidator.validate( + controlFile, ControlFileValidationLevel.KEYS, tableMetadataMap)) + .isExactlyInstanceOf(ControlFileValidationException.class) + .hasMessage( + CoreError.DATA_LOADER_MISSING_PARTITION_KEY.buildMessage( + COLUMN_PARTITION_KEY, lookupKey)); + } + + @Test + void + validate_keysValidationAndHasMissingMappedClusteringKeysGiven_shouldThrowControlFileValidationException() { + ControlFile controlFile = new ControlFile(); + ControlFileTable controlFileTable = new ControlFileTable(NAMESPACE, TABLE_NAME); + controlFileTable + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_PARTITION_KEY, COLUMN_PARTITION_KEY)); + controlFileTable.getMappings().add(new ControlFileTableFieldMapping(COLUMN_ONE, COLUMN_ONE)); + controlFile.getTables().add(controlFileTable); + + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(COLUMN_PARTITION_KEY, DataType.TEXT) + .addColumn(COLUMN_CLUSTERING_KEY, DataType.TEXT) + .addColumn(COLUMN_ONE, DataType.TEXT) + .addPartitionKey(COLUMN_PARTITION_KEY) + .addClusteringKey(COLUMN_CLUSTERING_KEY) + .build(); + Map tableMetadataMap = new HashMap<>(); + tableMetadataMap.put(lookupKey, tableMetadata); + + assertThatThrownBy( + () -> + ControlFileValidator.validate( + controlFile, ControlFileValidationLevel.KEYS, tableMetadataMap)) + .isExactlyInstanceOf(ControlFileValidationException.class) + .hasMessage( + CoreError.DATA_LOADER_MISSING_CLUSTERING_KEY.buildMessage( + COLUMN_CLUSTERING_KEY, lookupKey)); + } + + @Test + void validate_mappedValidationAndValidArgumentsGiven_shouldNotThrowException() + throws ControlFileValidationException { + ControlFile controlFile = new ControlFile(); + ControlFileTable controlFileTable = new ControlFileTable(NAMESPACE, TABLE_NAME); + controlFileTable.getMappings().add(new ControlFileTableFieldMapping(COLUMN_ONE, COLUMN_ONE)); + controlFile.getTables().add(controlFileTable); + + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(COLUMN_PARTITION_KEY, DataType.TEXT) + .addColumn(COLUMN_CLUSTERING_KEY, DataType.TEXT) + .addColumn(COLUMN_ONE, DataType.TEXT) + .addPartitionKey(COLUMN_PARTITION_KEY) + .addClusteringKey(COLUMN_CLUSTERING_KEY) + .build(); + Map tableMetadataMap = new HashMap<>(); + tableMetadataMap.put(lookupKey, tableMetadata); + + ControlFileValidator.validate(controlFile, ControlFileValidationLevel.MAPPED, tableMetadataMap); + } + + @Test + void validate_keysValidationAndValidArgumentsGiven_shouldNotThrowException() + throws ControlFileValidationException { + ControlFile controlFile = new ControlFile(); + ControlFileTable controlFileTable = new ControlFileTable(NAMESPACE, TABLE_NAME); + controlFileTable + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_PARTITION_KEY, COLUMN_PARTITION_KEY)); + controlFileTable + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_CLUSTERING_KEY, COLUMN_CLUSTERING_KEY)); + controlFile.getTables().add(controlFileTable); + + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(COLUMN_PARTITION_KEY, DataType.TEXT) + .addColumn(COLUMN_CLUSTERING_KEY, DataType.TEXT) + .addColumn(COLUMN_ONE, DataType.TEXT) + .addPartitionKey(COLUMN_PARTITION_KEY) + .addClusteringKey(COLUMN_CLUSTERING_KEY) + .build(); + Map tableMetadataMap = new HashMap<>(); + tableMetadataMap.put(lookupKey, tableMetadata); + + ControlFileValidator.validate(controlFile, ControlFileValidationLevel.MAPPED, tableMetadataMap); + } + + @Test + void validate_fullValidationAndValidArgumentsGiven_shouldNotThrowException() + throws ControlFileValidationException { + ControlFile controlFile = new ControlFile(); + ControlFileTable controlFileTable = new ControlFileTable(NAMESPACE, TABLE_NAME); + controlFileTable + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_PARTITION_KEY, COLUMN_PARTITION_KEY)); + controlFileTable + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_CLUSTERING_KEY, COLUMN_CLUSTERING_KEY)); + controlFileTable.getMappings().add(new ControlFileTableFieldMapping(COLUMN_ONE, COLUMN_ONE)); + controlFile.getTables().add(controlFileTable); + + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(COLUMN_PARTITION_KEY, DataType.TEXT) + .addColumn(COLUMN_CLUSTERING_KEY, DataType.TEXT) + .addColumn(COLUMN_ONE, DataType.TEXT) + .addPartitionKey(COLUMN_PARTITION_KEY) + .addClusteringKey(COLUMN_CLUSTERING_KEY) + .build(); + Map tableMetadataMap = new HashMap<>(); + tableMetadataMap.put(lookupKey, tableMetadata); + + ControlFileValidator.validate(controlFile, ControlFileValidationLevel.FULL, tableMetadataMap); + } + + @Test + void + validate_twoControlFileTablesAndFullValidationAndHasMissingMappedColumnsGiven_shouldThrowControlFileValidationException() { + ControlFile controlFile = new ControlFile(); + ControlFileTable controlFileTable = new ControlFileTable(NAMESPACE, TABLE_NAME); + controlFileTable + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_PARTITION_KEY, COLUMN_PARTITION_KEY)); + controlFileTable + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_CLUSTERING_KEY, COLUMN_CLUSTERING_KEY)); + controlFileTable.getMappings().add(new ControlFileTableFieldMapping(COLUMN_ONE, COLUMN_ONE)); + controlFile.getTables().add(controlFileTable); + ControlFileTable controlFileTable2 = new ControlFileTable(NAMESPACE, TABLE_NAME_TWO); + controlFileTable2 + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_PARTITION_KEY, COLUMN_PARTITION_KEY)); + controlFileTable2 + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_CLUSTERING_KEY, COLUMN_CLUSTERING_KEY)); + controlFile.getTables().add(controlFileTable2); + + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + String lookupKeyTwo = TableMetadataUtil.getTableLookupKey(controlFileTable2); + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(COLUMN_PARTITION_KEY, DataType.TEXT) + .addColumn(COLUMN_CLUSTERING_KEY, DataType.TEXT) + .addColumn(COLUMN_ONE, DataType.TEXT) + .addPartitionKey(COLUMN_PARTITION_KEY) + .addClusteringKey(COLUMN_CLUSTERING_KEY) + .build(); + Map tableMetadataMap = new HashMap<>(); + tableMetadataMap.put(lookupKey, tableMetadata); + tableMetadataMap.put(lookupKeyTwo, tableMetadata); + + assertThatThrownBy( + () -> + ControlFileValidator.validate( + controlFile, ControlFileValidationLevel.FULL, tableMetadataMap)) + .isExactlyInstanceOf(ControlFileValidationException.class) + .hasMessage( + CoreError.DATA_LOADER_MISSING_COLUMN_MAPPING.buildMessage(COLUMN_ONE, lookupKeyTwo)); + } + + @Test + void + validate_twoControlFileTablesAndKeysValidationAndHasMissingMappedColumnsGiven_shouldThrowControlFileValidationException() { + ControlFile controlFile = new ControlFile(); + ControlFileTable controlFileTable = new ControlFileTable(NAMESPACE, TABLE_NAME); + controlFileTable + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_PARTITION_KEY, COLUMN_PARTITION_KEY)); + controlFileTable + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_CLUSTERING_KEY, COLUMN_CLUSTERING_KEY)); + controlFile.getTables().add(controlFileTable); + ControlFileTable controlFileTable2 = new ControlFileTable(NAMESPACE, TABLE_NAME_TWO); + controlFileTable2 + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_PARTITION_KEY, COLUMN_PARTITION_KEY)); + controlFile.getTables().add(controlFileTable2); + + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + String lookupKeyTwo = TableMetadataUtil.getTableLookupKey(controlFileTable2); + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(COLUMN_PARTITION_KEY, DataType.TEXT) + .addColumn(COLUMN_CLUSTERING_KEY, DataType.TEXT) + .addColumn(COLUMN_ONE, DataType.TEXT) + .addPartitionKey(COLUMN_PARTITION_KEY) + .addClusteringKey(COLUMN_CLUSTERING_KEY) + .build(); + Map tableMetadataMap = new HashMap<>(); + tableMetadataMap.put(lookupKey, tableMetadata); + tableMetadataMap.put(lookupKeyTwo, tableMetadata); + + assertThatThrownBy( + () -> + ControlFileValidator.validate( + controlFile, ControlFileValidationLevel.KEYS, tableMetadataMap)) + .isExactlyInstanceOf(ControlFileValidationException.class) + .hasMessage( + CoreError.DATA_LOADER_MISSING_CLUSTERING_KEY.buildMessage( + COLUMN_CLUSTERING_KEY, lookupKeyTwo)); + } + + @Test + void + validate_twoControlFileTablesAndMappedValidationAndHasMissingMappedColumnsInOneTableGiven_shouldThrowControlFileValidationException() { + ControlFile controlFile = new ControlFile(); + ControlFileTable controlFileTable = new ControlFileTable(NAMESPACE, TABLE_NAME); + controlFileTable + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_PARTITION_KEY, COLUMN_PARTITION_KEY)); + controlFileTable + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_CLUSTERING_KEY, COLUMN_CLUSTERING_KEY)); + controlFile.getTables().add(controlFileTable); + ControlFileTable controlFileTable2 = new ControlFileTable(NAMESPACE, TABLE_NAME_TWO); + controlFile.getTables().add(controlFileTable2); + + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + String lookupKeyTwo = TableMetadataUtil.getTableLookupKey(controlFileTable2); + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(COLUMN_PARTITION_KEY, DataType.TEXT) + .addColumn(COLUMN_CLUSTERING_KEY, DataType.TEXT) + .addColumn(COLUMN_ONE, DataType.TEXT) + .addPartitionKey(COLUMN_PARTITION_KEY) + .addClusteringKey(COLUMN_CLUSTERING_KEY) + .build(); + Map tableMetadataMap = new HashMap<>(); + tableMetadataMap.put(lookupKey, tableMetadata); + tableMetadataMap.put(lookupKeyTwo, tableMetadata); + + assertThatThrownBy( + () -> + ControlFileValidator.validate( + controlFile, ControlFileValidationLevel.KEYS, tableMetadataMap)) + .isExactlyInstanceOf(ControlFileValidationException.class) + .hasMessage( + CoreError.DATA_LOADER_MISSING_PARTITION_KEY.buildMessage( + COLUMN_PARTITION_KEY, lookupKeyTwo)); + } + + @Test + void + validate_twoControlFileTablesAndFullValidationAndHasValidArgumentsGiven_shouldNotThrowException() + throws ControlFileValidationException { + ControlFile controlFile = new ControlFile(); + ControlFileTable controlFileTable = new ControlFileTable(NAMESPACE, TABLE_NAME); + controlFileTable + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_PARTITION_KEY, COLUMN_PARTITION_KEY)); + controlFileTable + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_CLUSTERING_KEY, COLUMN_CLUSTERING_KEY)); + controlFileTable.getMappings().add(new ControlFileTableFieldMapping(COLUMN_ONE, COLUMN_ONE)); + controlFile.getTables().add(controlFileTable); + ControlFileTable controlFileTable2 = new ControlFileTable(NAMESPACE, TABLE_NAME_TWO); + controlFileTable2 + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_PARTITION_KEY, COLUMN_PARTITION_KEY)); + controlFileTable2 + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_CLUSTERING_KEY, COLUMN_CLUSTERING_KEY)); + controlFileTable2.getMappings().add(new ControlFileTableFieldMapping(COLUMN_ONE, COLUMN_ONE)); + controlFile.getTables().add(controlFileTable2); + + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + String lookupKeyTwo = TableMetadataUtil.getTableLookupKey(controlFileTable2); + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(COLUMN_PARTITION_KEY, DataType.TEXT) + .addColumn(COLUMN_CLUSTERING_KEY, DataType.TEXT) + .addColumn(COLUMN_ONE, DataType.TEXT) + .addPartitionKey(COLUMN_PARTITION_KEY) + .addClusteringKey(COLUMN_CLUSTERING_KEY) + .build(); + Map tableMetadataMap = new HashMap<>(); + tableMetadataMap.put(lookupKey, tableMetadata); + tableMetadataMap.put(lookupKeyTwo, tableMetadata); + + ControlFileValidator.validate(controlFile, ControlFileValidationLevel.FULL, tableMetadataMap); + } + + @Test + void + validate_twoControlFileTablesAndKeysValidationAndHasValidArgumentsGiven_shouldNotThrowException() + throws ControlFileValidationException { + ControlFile controlFile = new ControlFile(); + ControlFileTable controlFileTable = new ControlFileTable(NAMESPACE, TABLE_NAME); + controlFileTable + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_PARTITION_KEY, COLUMN_PARTITION_KEY)); + controlFileTable + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_CLUSTERING_KEY, COLUMN_CLUSTERING_KEY)); + controlFile.getTables().add(controlFileTable); + ControlFileTable controlFileTable2 = new ControlFileTable(NAMESPACE, TABLE_NAME_TWO); + controlFileTable2 + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_PARTITION_KEY, COLUMN_PARTITION_KEY)); + controlFile.getTables().add(controlFileTable2); + controlFileTable2 + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_CLUSTERING_KEY, COLUMN_CLUSTERING_KEY)); + + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + String lookupKeyTwo = TableMetadataUtil.getTableLookupKey(controlFileTable2); + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(COLUMN_PARTITION_KEY, DataType.TEXT) + .addColumn(COLUMN_CLUSTERING_KEY, DataType.TEXT) + .addColumn(COLUMN_ONE, DataType.TEXT) + .addPartitionKey(COLUMN_PARTITION_KEY) + .addClusteringKey(COLUMN_CLUSTERING_KEY) + .build(); + Map tableMetadataMap = new HashMap<>(); + tableMetadataMap.put(lookupKey, tableMetadata); + tableMetadataMap.put(lookupKeyTwo, tableMetadata); + + ControlFileValidator.validate(controlFile, ControlFileValidationLevel.KEYS, tableMetadataMap); + } + + @Test + void + validate_twoControlFileTablesAndMappedValidationAndHasValidArgumentsGiven_shouldNotThrowException() + throws ControlFileValidationException { + ControlFile controlFile = new ControlFile(); + ControlFileTable controlFileTable = new ControlFileTable(NAMESPACE, TABLE_NAME); + controlFileTable + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_PARTITION_KEY, COLUMN_PARTITION_KEY)); + controlFileTable.getMappings().add(new ControlFileTableFieldMapping(COLUMN_ONE, COLUMN_ONE)); + controlFile.getTables().add(controlFileTable); + ControlFileTable controlFileTable2 = new ControlFileTable(NAMESPACE, TABLE_NAME_TWO); + controlFileTable2 + .getMappings() + .add(new ControlFileTableFieldMapping(COLUMN_PARTITION_KEY, COLUMN_PARTITION_KEY)); + controlFileTable2.getMappings().add(new ControlFileTableFieldMapping(COLUMN_ONE, COLUMN_ONE)); + controlFile.getTables().add(controlFileTable2); + + String lookupKey = TableMetadataUtil.getTableLookupKey(controlFileTable); + String lookupKeyTwo = TableMetadataUtil.getTableLookupKey(controlFileTable2); + + TableMetadata tableMetadata = + TableMetadata.newBuilder() + .addColumn(COLUMN_PARTITION_KEY, DataType.TEXT) + .addColumn(COLUMN_CLUSTERING_KEY, DataType.TEXT) + .addColumn(COLUMN_ONE, DataType.TEXT) + .addPartitionKey(COLUMN_PARTITION_KEY) + .addClusteringKey(COLUMN_CLUSTERING_KEY) + .build(); + Map tableMetadataMap = new HashMap<>(); + tableMetadataMap.put(lookupKey, tableMetadata); + tableMetadataMap.put(lookupKeyTwo, tableMetadata); + + ControlFileValidator.validate(controlFile, ControlFileValidationLevel.MAPPED, tableMetadataMap); + } +}