From 1806edae8bde2a256a63f30359162c4d03545db3 Mon Sep 17 00:00:00 2001 From: tinker Date: Tue, 19 Nov 2024 21:18:09 +0800 Subject: [PATCH 01/50] supports create and test file system datasource --- .../odc/core/shared/constant/ConnectType.java | 4 + .../odc/core/shared/constant/DialectType.java | 8 + .../odc/core/shared/constant/ErrorCodes.java | 7 +- .../service/connection/ConnectionTesting.java | 5 +- .../connection/ConnectionValidator.java | 4 +- .../FileSystemConnectionTesting.java | 145 ++++++++++++++++++ .../connection/database/DatabaseService.java | 3 + .../client/CloudObjectStorageClient.java | 6 +- .../cloud/CloudObjectStorage.java | 4 + .../cloud/client/AlibabaCloudClient.java | 17 ++ .../cloud/client/AmazonCloudClient.java | 17 ++ .../cloud/client/NullCloudClient.java | 7 + .../odc/plugin/connect/api/TestResult.java | 16 ++ 13 files changed, 237 insertions(+), 6 deletions(-) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java index 5c336e4345..6674d8922c 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java @@ -35,6 +35,10 @@ public enum ConnectType { // reserved for future version ODP_SHARDING_OB_ORACLE(DialectType.OB_ORACLE), ORACLE(DialectType.ORACLE), + OSS(DialectType.OSS), + OBS(DialectType.OBS), + COS(DialectType.COS), + S3A(DialectType.S3A), UNKNOWN(DialectType.UNKNOWN), ; diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/DialectType.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/DialectType.java index 97b33431f8..f529ccd0c3 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/DialectType.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/DialectType.java @@ -29,6 +29,10 @@ public enum DialectType { ODP_SHARDING_OB_MYSQL, DORIS, POSTGRESQL, + OSS, + OBS, + COS, + S3A, UNKNOWN, ; @@ -67,4 +71,8 @@ public boolean isPostgreSql() { return POSTGRESQL == this; } + public boolean isFileSystem() { + return OSS == this || OBS == this || COS == this || S3A == this; + } + } diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java index 8fa9eefb6c..caaa1dc632 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java @@ -312,7 +312,12 @@ public enum ErrorCodes implements ErrorCode { * workspace */ WorkspaceDatabaseUserTypeMustBeAdmin, - ; + /** + * oss + */ + BucketNotExist, + InvalidAccessKeyId, + SignatureDoesNotMatch; @Override public String code() { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java index 666772cf20..767018e86b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java @@ -74,6 +74,8 @@ public class ConnectionTesting { private ConnectionSSLAdaptor connectionSSLAdaptor; @Autowired private CloudMetadataClient cloudMetadataClient; + @Autowired + private FileSystemConnectionTesting fileSystemConnectionTesting; @Value("${odc.sdk.test-connect.query-timeout-seconds:2}") private int queryTimeoutSeconds = 2; @@ -97,7 +99,8 @@ public ConnectionTestResult test(@NotNull @Valid TestConnectionReq req) { if (req.getAccountType() == ConnectionAccountType.SYS_READ) { connectionConfig.setDefaultSchema(null); } - return test(connectionConfig); + return connectionConfig.getDialectType().isFileSystem() ? fileSystemConnectionTesting.test(connectionConfig) + : test(connectionConfig); } public ConnectionTestResult test(@NonNull ConnectionConfig config) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionValidator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionValidator.java index b2fd42c5a5..f25f057c09 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionValidator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionValidator.java @@ -43,7 +43,9 @@ public class ConnectionValidator { void validateForUpsert(ConnectionConfig connection) { PreConditions.notNull(connection, "connection"); PreConditions.notBlank(connection.getHost(), "connection.host"); - PreConditions.notNull(connection.getPort(), "connection.port"); + if (!connection.getDialectType().isFileSystem()) { + PreConditions.notNull(connection.getPort(), "connection.port"); + } PreConditions.validNotSqlInjection(connection.getUsername(), "username"); PreConditions.validNotSqlInjection(connection.getClusterName(), "clusterName"); PreConditions.validNotSqlInjection(connection.getTenantName(), "tenantName"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java new file mode 100644 index 0000000000..5f969639ee --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.springframework.stereotype.Component; + +import com.aliyun.oss.OSSErrorCode; +import com.aliyun.oss.OSSException; +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.plugin.connect.api.TestResult; +import com.oceanbase.odc.service.cloud.model.CloudProvider; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.connection.model.ConnectionTestResult; +import com.oceanbase.odc.service.objectstorage.cloud.CloudResourceConfigurations; +import com.oceanbase.odc.service.objectstorage.cloud.client.CloudClient; +import com.oceanbase.odc.service.objectstorage.cloud.client.CloudException; +import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectsRequest; +import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectMetadata; +import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; +import com.oceanbase.tools.migrator.common.exception.UnExpectedException; + +import lombok.NonNull; + +/** + * @Author：tinker + * @Date: 2024/11/19 11:24 + * @Descripition: + */ + +@Component +public class FileSystemConnectionTesting { + + private static final String cosRegex = "cos\\.(\\w+-\\w+)\\.myqcloud\\.com"; + private static final String ossRegex = "oss-([a-zA-Z0-9-]+)\\.aliyuncs\\.com"; + private static final String obsRegex = "obs\\.([a-zA-Z0-9-]+)\\.myhuaweicloud\\.com"; + private static final String s3Regex = "s3\\.([a-zA-Z0-9-]+)\\.amazonaws\\.com(\\.cn)?"; + private static final String tempFileNamePrefix = "odc-test-object"; + private static final String tempFileData = "This is a test object to check read and write permissions."; + private static final String OSS_ERROR_CODE_INVALID_ACCESS_KEY_ID = "InvalidAccessKeyId"; + + public ConnectionTestResult test(@NonNull ConnectionConfig config) { + ObjectStorageConfiguration storageConfig = new ObjectStorageConfiguration(); + storageConfig.setAccessKeyId(config.getUsername()); + storageConfig.setAccessKeySecret(config.getPassword()); + storageConfig.setBucketName(config.getDefaultSchema().split("/")[0]); + storageConfig.setRegion(getRegion(config)); + storageConfig.setCloudProvider(getCloudProvider(config.getDialectType())); + storageConfig.setPublicEndpoint(config.getHost()); + try { + CloudClient cloudClient = + new CloudResourceConfigurations.CloudClientBuilder().generateCloudClient(storageConfig); + String tempFileName = generateTempFileName(); + String objectKey = config.getDefaultSchema() + tempFileName; + cloudClient.putObject(storageConfig.getBucketName(), objectKey, + new ByteArrayInputStream(tempFileData.getBytes(StandardCharsets.UTF_8)), new ObjectMetadata()); + DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(); + deleteObjectsRequest.setBucketName(storageConfig.getBucketName()); + deleteObjectsRequest.setKey(objectKey); + cloudClient.deleteObjects(deleteObjectsRequest); + return ConnectionTestResult.success(config.getType()); + } catch (CloudException e) { + if (e.getCause() instanceof OSSException) { + OSSException cause = (OSSException) e.getCause(); + switch (cause.getErrorCode()) { + case OSSErrorCode.ACCESS_DENIED: + return new ConnectionTestResult(TestResult.akAccessDenied(storageConfig.getAccessKeyId()), + config.getType()); + case OSSErrorCode.INVALID_ACCESS_KEY_ID: + return new ConnectionTestResult(TestResult.invalidAccessKeyId(storageConfig.getAccessKeyId()), + config.getType()); + case OSSErrorCode.SIGNATURE_DOES_NOT_MATCH: + return new ConnectionTestResult( + TestResult.signatureDoesNotMatch(storageConfig.getAccessKeyId()), config.getType()); + default: + return new ConnectionTestResult(TestResult.unknownError(e), config.getType()); + } + } + // TODO:process s3 error message + return new ConnectionTestResult(TestResult.unknownError(e), config.getType()); + } + } + + private String getRegion(ConnectionConfig config) { + Pattern pattern = Pattern.compile(getEndPointRegex(config.getDialectType())); + Matcher matcher = pattern.matcher(config.getHost()); + if (matcher.find()) { + return matcher.group(1); + } else { + throw new UnExpectedException("Illegal endpoint"); + } + } + + private CloudProvider getCloudProvider(DialectType type) { + switch (type) { + case COS: + return CloudProvider.TENCENT_CLOUD; + case OBS: + return CloudProvider.HUAWEI_CLOUD; + case S3A: + return CloudProvider.AWS; + case OSS: + return CloudProvider.ALIBABA_CLOUD; + default: + throw new UnExpectedException(); + } + } + + private String getEndPointRegex(DialectType type) { + switch (type) { + case COS: + return cosRegex; + case OBS: + return obsRegex; + case S3A: + return s3Regex; + case OSS: + return ossRegex; + default: + throw new UnExpectedException(); + } + } + + private String generateTempFileName() { + return tempFileNamePrefix + UUID.randomUUID(); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index a056cfab8b..bc5ed359f4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -511,6 +511,9 @@ public Boolean internalSyncDataSourceSchemas(@NonNull Long dataSourceId) throws } try { ConnectionConfig connection = connectionService.getForConnectionSkipPermissionCheck(dataSourceId); + if (connection.getDialectType().isFileSystem()) { + return true; + } horizontalDataPermissionValidator.checkCurrentOrganization(connection); organizationService.get(connection.getOrganizationId()).ifPresent(organization -> { if (organization.getType() == OrganizationType.INDIVIDUAL) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/CloudObjectStorageClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/CloudObjectStorageClient.java index 19417b462b..b6d646f36e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/CloudObjectStorageClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/CloudObjectStorageClient.java @@ -291,13 +291,13 @@ public long calculatePartSize(long fileLength) { * 也就是杭州的client只允许操作杭州的bucket，不允许跨域操作 */ private void validateBucket() { - if (objectStorageConfiguration.getCloudProvider() != CloudProvider.ALIBABA_CLOUD) { - return; - } String bucketName = getBucketName(); boolean isExist = publicEndpointCloudObjectStorage.doesBucketExist(bucketName); Verify.verify(isExist, String.format("object storage bucket '%s' not exists", bucketName)); + if (objectStorageConfiguration.getCloudProvider() != CloudProvider.ALIBABA_CLOUD) { + return; + } String region = objectStorageConfiguration.getRegion(); if (StringUtils.isNotEmpty(region)) { String location = publicEndpointCloudObjectStorage.getBucketLocation(bucketName); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorage.java index 3b4606b011..fd6f84179a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorage.java @@ -16,6 +16,7 @@ package com.oceanbase.odc.service.objectstorage.cloud; import java.io.File; +import java.io.InputStream; import java.net.URL; import java.util.Date; import java.util.List; @@ -60,6 +61,9 @@ public interface CloudObjectStorage { PutObjectResult putObject(String bucketName, String key, File file, ObjectMetadata metadata) throws CloudException; + PutObjectResult putObject(String bucketName, String key, InputStream in, ObjectMetadata metadata) + throws CloudException; + default PutObjectResult putObject(String bucketName, String key, File file) { return putObject(bucketName, key, file, null); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AlibabaCloudClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AlibabaCloudClient.java index 3a67bcb876..ccae6d094a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AlibabaCloudClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AlibabaCloudClient.java @@ -168,6 +168,23 @@ public PutObjectResult putObject(String bucketName, String key, File file, Objec return putObject; } + @Override + public PutObjectResult putObject(String bucketName, String key, InputStream in, ObjectMetadata metadata) + throws CloudException { + PutObjectResult putObject = callOssMethod("Put object", () -> { + com.aliyun.oss.model.ObjectMetadata objectMetadata = toOss(metadata); + com.aliyun.oss.model.PutObjectResult ossResult = oss.putObject(bucketName, key, in, objectMetadata); + PutObjectResult result = new PutObjectResult(); + result.setVersionId(ossResult.getVersionId()); + result.setETag(ossResult.getETag()); + result.setRequestId(ossResult.getRequestId()); + result.setClientCRC(ossResult.getClientCRC()); + result.setServerCRC(ossResult.getServerCRC()); + return result; + }); + return putObject; + } + @Override public CopyObjectResult copyObject(String bucketName, String from, String to) throws CloudException { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AmazonCloudClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AmazonCloudClient.java index b786892028..28013e3368 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AmazonCloudClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AmazonCloudClient.java @@ -182,6 +182,23 @@ public PutObjectResult putObject(String bucketName, String key, File file, Objec }); } + @Override + public PutObjectResult putObject(String bucketName, String key, InputStream in, ObjectMetadata metadata) + throws CloudException { + return callAmazonMethod("Put object", () -> { + com.amazonaws.services.s3.model.ObjectMetadata objectMetadata = toS3(metadata); + PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, in, objectMetadata); + if (metadata.hasTag()) { + putRequest.withTagging(toS3(metadata.getTagging())); + } + com.amazonaws.services.s3.model.PutObjectResult s3Result = s3.putObject(putRequest); + PutObjectResult result = new PutObjectResult(); + result.setVersionId(s3Result.getVersionId()); + result.setETag(s3Result.getETag()); + return result; + }); + } + @Override public CopyObjectResult copyObject(String bucketName, String from, String to) throws CloudException { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/NullCloudClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/NullCloudClient.java index e52c38038c..6620c343aa 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/NullCloudClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/NullCloudClient.java @@ -16,6 +16,7 @@ package com.oceanbase.odc.service.objectstorage.cloud.client; import java.io.File; +import java.io.InputStream; import java.net.URL; import java.util.Date; import java.util.List; @@ -76,6 +77,12 @@ public PutObjectResult putObject(String bucketName, String key, File file, Objec throw new UnsupportedException(); } + @Override + public PutObjectResult putObject(String bucketName, String key, InputStream in, ObjectMetadata metadata) + throws CloudException { + throw new UnsupportedException(); + } + @Override public CopyObjectResult copyObject(String bucketName, String from, String to) throws CloudException { diff --git a/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/TestResult.java b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/TestResult.java index aa20507701..e2230ee08f 100644 --- a/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/TestResult.java +++ b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/TestResult.java @@ -84,6 +84,22 @@ public static TestResult initScriptFailed(Throwable throwable) { return fail(ErrorCodes.ConnectionInitScriptFailed, new String[] {message}); } + public static TestResult bucketNotExist(String bucketName) { + return fail(ErrorCodes.BucketNotExist, new String[] {bucketName}); + } + + public static TestResult invalidAccessKeyId(String accessKeyId) { + return fail(ErrorCodes.InvalidAccessKeyId, new String[] {accessKeyId}); + } + + public static TestResult akAccessDenied(String accessKeyId) { + return fail(ErrorCodes.AccessDenied, new String[] {accessKeyId}); + } + + public static TestResult signatureDoesNotMatch(String accessKeyId) { + return fail(ErrorCodes.SignatureDoesNotMatch, new String[] {accessKeyId}); + } + public static TestResult success() { TestResult result = new TestResult(); result.setActive(true); From 6faed2db41052d5930375915f343c699badaf639 Mon Sep 17 00:00:00 2001 From: tinker Date: Wed, 20 Nov 2024 11:44:42 +0800 Subject: [PATCH 02/50] add i18 --- .../resources/i18n/ErrorMessages.properties | 5 ++- .../i18n/ErrorMessages_zh_CN.properties | 5 ++- .../FileSystemConnectionTesting.java | 31 +++++++++++-------- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages.properties index 85f4974381..bdde667b08 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages.properties @@ -200,4 +200,7 @@ com.oceanbase.odc.ErrorCodes.TimeDataTypePrecisionMismatch=The time type precisi com.oceanbase.odc.ErrorCodes.DatabaseAccessDenied=Database access is denied because the current user does not have {0} permissions for the database. com.oceanbase.odc.ErrorCodes.ObQueryProfileNotSupported=Query profile is only available for OceanBase Database with versions equal to or higher than {0}. com.oceanbase.odc.ErrorCodes.WorksheetEditVersionConflict=Someone has just modified this file. Please refresh the page to get the latest version and continue editing. -com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=The database user type used in the workspace must be a super account. \ No newline at end of file +com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=The database user type used in the workspace must be a super account. +com.oceanbase.odc.ErrorCodes.BucketNotExist=Bucket does not exist +com.oceanbase.odc.ErrorCodes.InvalidAccessKeyId=Invalid access key id +com.oceanbase.odc.ErrorCodes.SignatureDoesNotMatch=Invalid access key secret \ No newline at end of file diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties index c4ea9bdadd..ff7717ce84 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties @@ -199,4 +199,7 @@ com.oceanbase.odc.ErrorCodes.TimeDataTypePrecisionMismatch=时间类型精度不 com.oceanbase.odc.ErrorCodes.DatabaseAccessDenied=数据库访问被拒绝，因为当前用户没有数据库的{0}权限 com.oceanbase.odc.ErrorCodes.ObQueryProfileNotSupported=执行详情仅在高于或等于 {0} 的 OceanBase 版本可用 com.oceanbase.odc.ErrorCodes.WorksheetEditVersionConflict=有人刚刚修改了这个工作簿，请刷新页面以获取最新版本并重新编辑 -com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=工作空间中使用的数据库用户类型必须为超级账号 \ No newline at end of file +com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=工作空间中使用的数据库用户类型必须为超级账号 +com.oceanbase.odc.ErrorCodes.BucketNotExist=桶不存在 +com.oceanbase.odc.ErrorCodes.InvalidAccessKeyId=无效的 AccessKeyId +com.oceanbase.odc.ErrorCodes.SignatureDoesNotMatch=无效的 AccessKeySecret \ No newline at end of file diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java index 5f969639ee..d1361cc248 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java @@ -25,6 +25,7 @@ import com.aliyun.oss.OSSErrorCode; import com.aliyun.oss.OSSException; +import com.oceanbase.odc.core.shared.PreConditions; import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.plugin.connect.api.TestResult; import com.oceanbase.odc.service.cloud.model.CloudProvider; @@ -49,15 +50,16 @@ @Component public class FileSystemConnectionTesting { - private static final String cosRegex = "cos\\.(\\w+-\\w+)\\.myqcloud\\.com"; - private static final String ossRegex = "oss-([a-zA-Z0-9-]+)\\.aliyuncs\\.com"; - private static final String obsRegex = "obs\\.([a-zA-Z0-9-]+)\\.myhuaweicloud\\.com"; - private static final String s3Regex = "s3\\.([a-zA-Z0-9-]+)\\.amazonaws\\.com(\\.cn)?"; - private static final String tempFileNamePrefix = "odc-test-object"; - private static final String tempFileData = "This is a test object to check read and write permissions."; - private static final String OSS_ERROR_CODE_INVALID_ACCESS_KEY_ID = "InvalidAccessKeyId"; + private static final String COS_ENDPOINT_REGEX = "cos\\.(\\w+-\\w+)\\.myqcloud\\.com"; + private static final String OSS_ENDPOINT_REGEX = "oss-([a-zA-Z0-9-]+)\\.aliyuncs\\.com"; + private static final String OBS_ENDPOINT_REGEX = "obs\\.([a-zA-Z0-9-]+)\\.myhuaweicloud\\.com"; + private static final String S3_ENDPOINT_REGEX = "s3\\.([a-zA-Z0-9-]+)\\.amazonaws\\.com(\\.cn)?"; + private static final String TMP_FILE_NAME_PREFIX = "odc-test-object-"; + private static final String TMP_TEST_DATA = "This is a test object to check read and write permissions."; public ConnectionTestResult test(@NonNull ConnectionConfig config) { + PreConditions.notBlank(config.getPassword(), "AccessKeySecret"); + PreConditions.notBlank(config.getDefaultSchema(), "Bucket"); ObjectStorageConfiguration storageConfig = new ObjectStorageConfiguration(); storageConfig.setAccessKeyId(config.getUsername()); storageConfig.setAccessKeySecret(config.getPassword()); @@ -71,7 +73,7 @@ public ConnectionTestResult test(@NonNull ConnectionConfig config) { String tempFileName = generateTempFileName(); String objectKey = config.getDefaultSchema() + tempFileName; cloudClient.putObject(storageConfig.getBucketName(), objectKey, - new ByteArrayInputStream(tempFileData.getBytes(StandardCharsets.UTF_8)), new ObjectMetadata()); + new ByteArrayInputStream(TMP_TEST_DATA.getBytes(StandardCharsets.UTF_8)), new ObjectMetadata()); DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(); deleteObjectsRequest.setBucketName(storageConfig.getBucketName()); deleteObjectsRequest.setKey(objectKey); @@ -90,6 +92,9 @@ public ConnectionTestResult test(@NonNull ConnectionConfig config) { case OSSErrorCode.SIGNATURE_DOES_NOT_MATCH: return new ConnectionTestResult( TestResult.signatureDoesNotMatch(storageConfig.getAccessKeyId()), config.getType()); + case OSSErrorCode.NO_SUCH_BUCKET: + return new ConnectionTestResult(TestResult.bucketNotExist(storageConfig.getBucketName()), + config.getType()); default: return new ConnectionTestResult(TestResult.unknownError(e), config.getType()); } @@ -127,19 +132,19 @@ private CloudProvider getCloudProvider(DialectType type) { private String getEndPointRegex(DialectType type) { switch (type) { case COS: - return cosRegex; + return COS_ENDPOINT_REGEX; case OBS: - return obsRegex; + return OBS_ENDPOINT_REGEX; case S3A: - return s3Regex; + return S3_ENDPOINT_REGEX; case OSS: - return ossRegex; + return OSS_ENDPOINT_REGEX; default: throw new UnExpectedException(); } } private String generateTempFileName() { - return tempFileNamePrefix + UUID.randomUUID(); + return TMP_FILE_NAME_PREFIX + UUID.randomUUID(); } } From 87bc44504718af339209750ebee646bbf77138db Mon Sep 17 00:00:00 2001 From: tinker Date: Wed, 20 Nov 2024 16:38:36 +0800 Subject: [PATCH 03/50] fix access denied --- .../odc/core/shared/constant/ConnectType.java | 8 ++++---- .../odc/core/shared/constant/DialectType.java | 9 +-------- .../service/connection/ConnectionTesting.java | 3 ++- .../connection/ConnectionValidator.java | 3 ++- .../FileSystemConnectionTesting.java | 18 ++++++++++-------- .../connection/database/DatabaseService.java | 3 --- 6 files changed, 19 insertions(+), 25 deletions(-) diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java index 6674d8922c..6d2572833b 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java @@ -35,10 +35,10 @@ public enum ConnectType { // reserved for future version ODP_SHARDING_OB_ORACLE(DialectType.OB_ORACLE), ORACLE(DialectType.ORACLE), - OSS(DialectType.OSS), - OBS(DialectType.OBS), - COS(DialectType.COS), - S3A(DialectType.S3A), + OSS(DialectType.FILE_SYSTEM), + OBS(DialectType.FILE_SYSTEM), + COS(DialectType.FILE_SYSTEM), + S3A(DialectType.FILE_SYSTEM), UNKNOWN(DialectType.UNKNOWN), ; diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/DialectType.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/DialectType.java index f529ccd0c3..b83c369e90 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/DialectType.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/DialectType.java @@ -29,10 +29,7 @@ public enum DialectType { ODP_SHARDING_OB_MYSQL, DORIS, POSTGRESQL, - OSS, - OBS, - COS, - S3A, + FILE_SYSTEM, UNKNOWN, ; @@ -71,8 +68,4 @@ public boolean isPostgreSql() { return POSTGRESQL == this; } - public boolean isFileSystem() { - return OSS == this || OBS == this || COS == this || S3A == this; - } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java index 767018e86b..06c3f762ff 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java @@ -99,7 +99,8 @@ public ConnectionTestResult test(@NotNull @Valid TestConnectionReq req) { if (req.getAccountType() == ConnectionAccountType.SYS_READ) { connectionConfig.setDefaultSchema(null); } - return connectionConfig.getDialectType().isFileSystem() ? fileSystemConnectionTesting.test(connectionConfig) + return connectionConfig.getDialectType() == DialectType.FILE_SYSTEM + ? fileSystemConnectionTesting.test(connectionConfig) : test(connectionConfig); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionValidator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionValidator.java index f25f057c09..fcab8f4504 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionValidator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionValidator.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.exception.AccessDeniedException; import com.oceanbase.odc.service.collaboration.environment.EnvironmentService; @@ -43,7 +44,7 @@ public class ConnectionValidator { void validateForUpsert(ConnectionConfig connection) { PreConditions.notNull(connection, "connection"); PreConditions.notBlank(connection.getHost(), "connection.host"); - if (!connection.getDialectType().isFileSystem()) { + if (connection.getDialectType() != DialectType.FILE_SYSTEM) { PreConditions.notNull(connection.getPort(), "connection.port"); } PreConditions.validNotSqlInjection(connection.getUsername(), "username"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java index d1361cc248..e306aff10b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java @@ -26,7 +26,7 @@ import com.aliyun.oss.OSSErrorCode; import com.aliyun.oss.OSSException; import com.oceanbase.odc.core.shared.PreConditions; -import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.core.shared.constant.ConnectType; import com.oceanbase.odc.plugin.connect.api.TestResult; import com.oceanbase.odc.service.cloud.model.CloudProvider; import com.oceanbase.odc.service.connection.model.ConnectionConfig; @@ -60,18 +60,20 @@ public class FileSystemConnectionTesting { public ConnectionTestResult test(@NonNull ConnectionConfig config) { PreConditions.notBlank(config.getPassword(), "AccessKeySecret"); PreConditions.notBlank(config.getDefaultSchema(), "Bucket"); + String[] splitPath = config.getDefaultSchema().split("/"); + String bucketName = splitPath[0]; + String path = splitPath.length > 1 ? "/" + splitPath[1] : "/"; ObjectStorageConfiguration storageConfig = new ObjectStorageConfiguration(); storageConfig.setAccessKeyId(config.getUsername()); storageConfig.setAccessKeySecret(config.getPassword()); - storageConfig.setBucketName(config.getDefaultSchema().split("/")[0]); + storageConfig.setBucketName(bucketName); storageConfig.setRegion(getRegion(config)); - storageConfig.setCloudProvider(getCloudProvider(config.getDialectType())); + storageConfig.setCloudProvider(getCloudProvider(config.getType())); storageConfig.setPublicEndpoint(config.getHost()); try { CloudClient cloudClient = new CloudResourceConfigurations.CloudClientBuilder().generateCloudClient(storageConfig); - String tempFileName = generateTempFileName(); - String objectKey = config.getDefaultSchema() + tempFileName; + String objectKey = path + generateTempFileName(); cloudClient.putObject(storageConfig.getBucketName(), objectKey, new ByteArrayInputStream(TMP_TEST_DATA.getBytes(StandardCharsets.UTF_8)), new ObjectMetadata()); DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(); @@ -105,7 +107,7 @@ public ConnectionTestResult test(@NonNull ConnectionConfig config) { } private String getRegion(ConnectionConfig config) { - Pattern pattern = Pattern.compile(getEndPointRegex(config.getDialectType())); + Pattern pattern = Pattern.compile(getEndPointRegex(config.getType())); Matcher matcher = pattern.matcher(config.getHost()); if (matcher.find()) { return matcher.group(1); @@ -114,7 +116,7 @@ private String getRegion(ConnectionConfig config) { } } - private CloudProvider getCloudProvider(DialectType type) { + private CloudProvider getCloudProvider(ConnectType type) { switch (type) { case COS: return CloudProvider.TENCENT_CLOUD; @@ -129,7 +131,7 @@ private CloudProvider getCloudProvider(DialectType type) { } } - private String getEndPointRegex(DialectType type) { + private String getEndPointRegex(ConnectType type) { switch (type) { case COS: return COS_ENDPOINT_REGEX; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index bc5ed359f4..a056cfab8b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -511,9 +511,6 @@ public Boolean internalSyncDataSourceSchemas(@NonNull Long dataSourceId) throws } try { ConnectionConfig connection = connectionService.getForConnectionSkipPermissionCheck(dataSourceId); - if (connection.getDialectType().isFileSystem()) { - return true; - } horizontalDataPermissionValidator.checkCurrentOrganization(connection); organizationService.get(connection.getOrganizationId()).ifPresent(organization -> { if (organization.getType() == OrganizationType.INDIVIDUAL) { From 5913e084b42649a39064174b2b252a75dfeef877 Mon Sep 17 00:00:00 2001 From: tinker Date: Wed, 20 Nov 2024 16:59:11 +0800 Subject: [PATCH 04/50] fix access denied --- .../odc/service/connection/FileSystemConnectionTesting.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java index e306aff10b..2c4db35d5e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java @@ -60,7 +60,7 @@ public class FileSystemConnectionTesting { public ConnectionTestResult test(@NonNull ConnectionConfig config) { PreConditions.notBlank(config.getPassword(), "AccessKeySecret"); PreConditions.notBlank(config.getDefaultSchema(), "Bucket"); - String[] splitPath = config.getDefaultSchema().split("/"); + String[] splitPath = config.getDefaultSchema().split("/", 2); String bucketName = splitPath[0]; String path = splitPath.length > 1 ? "/" + splitPath[1] : "/"; ObjectStorageConfiguration storageConfig = new ObjectStorageConfiguration(); From a3d34fa8e7fef7e9fa0aff1c1c695a95d083f44c Mon Sep 17 00:00:00 2001 From: tinker Date: Wed, 20 Nov 2024 17:08:35 +0800 Subject: [PATCH 05/50] fix access denied --- .../odc/service/connection/FileSystemConnectionTesting.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java index 2c4db35d5e..d01456004b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java @@ -62,7 +62,7 @@ public ConnectionTestResult test(@NonNull ConnectionConfig config) { PreConditions.notBlank(config.getDefaultSchema(), "Bucket"); String[] splitPath = config.getDefaultSchema().split("/", 2); String bucketName = splitPath[0]; - String path = splitPath.length > 1 ? "/" + splitPath[1] : "/"; + String path = splitPath.length > 1 ? splitPath[1] : ""; ObjectStorageConfiguration storageConfig = new ObjectStorageConfiguration(); storageConfig.setAccessKeyId(config.getUsername()); storageConfig.setAccessKeySecret(config.getPassword()); From b4fc647ee212ab60a9750313dd10f3b863d261e3 Mon Sep 17 00:00:00 2001 From: tinker Date: Wed, 20 Nov 2024 17:13:52 +0800 Subject: [PATCH 06/50] fix delete obj failed --- .../odc/service/connection/FileSystemConnectionTesting.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java index d01456004b..aa3428f37d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java @@ -17,6 +17,7 @@ import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -78,7 +79,7 @@ public ConnectionTestResult test(@NonNull ConnectionConfig config) { new ByteArrayInputStream(TMP_TEST_DATA.getBytes(StandardCharsets.UTF_8)), new ObjectMetadata()); DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(); deleteObjectsRequest.setBucketName(storageConfig.getBucketName()); - deleteObjectsRequest.setKey(objectKey); + deleteObjectsRequest.setKeys(Collections.singletonList(objectKey)); cloudClient.deleteObjects(deleteObjectsRequest); return ConnectionTestResult.success(config.getType()); } catch (CloudException e) { From ee4b526afba46e170616353fba3e056b183e41c4 Mon Sep 17 00:00:00 2001 From: tinker Date: Thu, 28 Nov 2024 18:32:01 +0800 Subject: [PATCH 07/50] support sync database --- .../connection/database/DatabaseService.java | 42 +++++++++++++++++++ .../database/DatabaseSyncManager.java | 4 +- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index a056cfab8b..f44d8420ed 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -528,6 +528,48 @@ public Boolean internalSyncDataSourceSchemas(@NonNull Long dataSourceId) throws } } + @SkipAuthorize("internal usage") + public Boolean internalSyncFileSystemSchemas(@NonNull Long dataSourceId) throws InterruptedException { + Lock lock = jdbcLockRegistry.obtain(connectionService.getUpdateDsSchemaLockKey(dataSourceId)); + if (!lock.tryLock(3, TimeUnit.SECONDS)) { + throw new ConflictException(ErrorCodes.ResourceSynchronizing, + new Object[] {ResourceType.ODC_DATABASE.getLocalizedMessage()}, "Can not acquire jdbc lock"); + } + try { + ConnectionConfig connection = connectionService.getForConnectionSkipPermissionCheck(dataSourceId); + horizontalDataPermissionValidator.checkCurrentOrganization(connection); + List existDatabases = databaseRepository.findByConnectionId(dataSourceId).stream() + .filter(DatabaseEntity::getExisted).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(existDatabases) + || !existDatabases.stream().map(DatabaseEntity::getName).collect( + Collectors.toSet()).contains(connection.getDefaultSchema())) { + DatabaseEntity entity = new DatabaseEntity(); + entity.setDatabaseId(com.oceanbase.odc.common.util.StringUtils.uuid()); + entity.setOrganizationId(connection.getOrganizationId()); + entity.setName(connection.getDefaultSchema()); + entity.setProjectId(connection.getProjectId()); + entity.setConnectionId(connection.getId()); + entity.setEnvironmentId(connection.getEnvironmentId()); + entity.setSyncStatus(DatabaseSyncStatus.SUCCEEDED); + entity.setExisted(true); + entity.setObjectSyncStatus(DBObjectSyncStatus.SYNCED); + entity.setConnectType(connection.getType()); + databaseRepository.save(entity); + } else { + DeleteDatabasesReq deleteDatabasesReq = new DeleteDatabasesReq(); + deleteDatabasesReq + .setDatabaseIds(existDatabases.stream().map(DatabaseEntity::getId).collect(Collectors.toSet())); + deleteDatabases(deleteDatabasesReq); + } + return true; + } catch (Exception ex) { + log.warn("Sync database failed, dataSourceId={}, errorMessage={}", dataSourceId, ex.getLocalizedMessage()); + return false; + } finally { + lock.unlock(); + } + } + public int updateEnvironmentIdByConnectionId(@NotNull Long environmentId, @NotNull Long connectionId) { return databaseRepository.setEnvironmentIdByConnectionId(environmentId, connectionId); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseSyncManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseSyncManager.java index 332c3fd493..2d299552c7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseSyncManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseSyncManager.java @@ -29,6 +29,7 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.oceanbase.odc.core.authority.util.SkipAuthorize; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.exception.BadRequestException; import com.oceanbase.odc.metadb.iam.UserEntity; import com.oceanbase.odc.service.connection.model.ConnectionConfig; @@ -91,7 +92,8 @@ public Future submitSyncDataSourceAndDBSchemaTask(@NonNull ConnectionCo private Boolean syncDBForDataSource(@NonNull ConnectionConfig dataSource) throws InterruptedException { Long creatorId = dataSource.getCreatorId(); SecurityContextUtils.setCurrentUser(creatorId, dataSource.getOrganizationId(), getAccountName(creatorId)); - return databaseService.internalSyncDataSourceSchemas(dataSource.getId()); + return dataSource.getDialectType() == DialectType.FILE_SYSTEM ? databaseService.internalSyncFileSystemSchemas( + dataSource.getId()) : databaseService.internalSyncDataSourceSchemas(dataSource.getId()); } private Future doExecute(Supplier> supplier) { From 5139b643b3039c3c0aaa7f83ce4427967df1c422 Mon Sep 17 00:00:00 2001 From: tinker Date: Thu, 5 Dec 2024 14:36:29 +0800 Subject: [PATCH 08/50] set default schema --- .../com/oceanbase/odc/core/shared/constant/ConnectType.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java index 6d2572833b..db92a1cff0 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java @@ -58,7 +58,7 @@ public boolean isODPSharding() { } public boolean isDefaultSchemaRequired() { - return isODPSharding(); + return isODPSharding() || this.dialectType == DialectType.FILE_SYSTEM; } public static ConnectType from(DialectType dialectType) { From 088dc8ec917d36384bde8a3960f56e7154dc4e36 Mon Sep 17 00:00:00 2001 From: tinker Date: Fri, 6 Dec 2024 15:25:14 +0800 Subject: [PATCH 09/50] fix --- .../odc/service/connection/database/DatabaseService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index f44d8420ed..0bac63ebd8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -554,6 +554,7 @@ public Boolean internalSyncFileSystemSchemas(@NonNull Long dataSourceId) throws entity.setExisted(true); entity.setObjectSyncStatus(DBObjectSyncStatus.SYNCED); entity.setConnectType(connection.getType()); + entity.setType(DatabaseType.PHYSICAL); databaseRepository.save(entity); } else { DeleteDatabasesReq deleteDatabasesReq = new DeleteDatabasesReq(); @@ -563,7 +564,7 @@ public Boolean internalSyncFileSystemSchemas(@NonNull Long dataSourceId) throws } return true; } catch (Exception ex) { - log.warn("Sync database failed, dataSourceId={}, errorMessage={}", dataSourceId, ex.getLocalizedMessage()); + log.warn("Sync database failed, dataSourceId={}, errorMessage={}", dataSourceId, ex); return false; } finally { lock.unlock(); From e3862df809e4b1dd41646bc5e0697de085648d35 Mon Sep 17 00:00:00 2001 From: tinker Date: Mon, 16 Dec 2024 17:08:18 +0800 Subject: [PATCH 10/50] support archive to file system --- pom.xml | 2 +- .../odc/core/shared/constant/ErrorCodes.java | 3 +- .../resources/i18n/ErrorMessages.properties | 3 +- .../i18n/ErrorMessages_zh_CN.properties | 3 +- .../i18n/ErrorMessages_zh_TW.properties | 3 +- .../odc/service/dlm/DLMConfiguration.java | 10 - .../odc/service/dlm/DLMJobFactory.java | 3 +- .../odc/service/dlm/DLMJobStore.java | 87 +------ .../odc/service/dlm/DataArchiveJobStore.java | 214 ------------------ .../odc/service/dlm/DataSourceInfoMapper.java | 62 +++-- .../processor/AbstractDlmPreprocessor.java | 2 +- .../processor/DataArchivePreprocessor.java | 41 +--- .../base/dataarchive/DataArchiveTask.java | 37 ++- 13 files changed, 83 insertions(+), 387 deletions(-) delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataArchiveJobStore.java diff --git a/pom.xml b/pom.xml index cc36555862..56dd63bd11 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,7 @@ 2.1.6 - 1.1.6.bp + 1.1.7 2.11.0 diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java index caaa1dc632..248e08ab3d 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java @@ -317,7 +317,8 @@ public enum ErrorCodes implements ErrorCode { */ BucketNotExist, InvalidAccessKeyId, - SignatureDoesNotMatch; + SignatureDoesNotMatch, + UnsupportedSyncTableStructure; @Override public String code() { diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages.properties index bdde667b08..3570b3d911 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages.properties @@ -203,4 +203,5 @@ com.oceanbase.odc.ErrorCodes.WorksheetEditVersionConflict=Someone has just modif com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=The database user type used in the workspace must be a super account. com.oceanbase.odc.ErrorCodes.BucketNotExist=Bucket does not exist com.oceanbase.odc.ErrorCodes.InvalidAccessKeyId=Invalid access key id -com.oceanbase.odc.ErrorCodes.SignatureDoesNotMatch=Invalid access key secret \ No newline at end of file +com.oceanbase.odc.ErrorCodes.SignatureDoesNotMatch=Invalid access key secret +com.oceanbase.odc.ErrorCodes.UnsupportedSyncTableStructure=Sync table structure is not supported for {0} to {1} \ No newline at end of file diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties index ff7717ce84..765f11caab 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties @@ -202,4 +202,5 @@ com.oceanbase.odc.ErrorCodes.WorksheetEditVersionConflict=有人刚刚修改了 com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=工作空间中使用的数据库用户类型必须为超级账号 com.oceanbase.odc.ErrorCodes.BucketNotExist=桶不存在 com.oceanbase.odc.ErrorCodes.InvalidAccessKeyId=无效的 AccessKeyId -com.oceanbase.odc.ErrorCodes.SignatureDoesNotMatch=无效的 AccessKeySecret \ No newline at end of file +com.oceanbase.odc.ErrorCodes.SignatureDoesNotMatch=无效的 AccessKeySecret +com.oceanbase.odc.ErrorCodes.UnsupportedSyncTableStructure=结构同步暂不支持 {0} 到 {1} \ No newline at end of file diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties index 5e8ed7b042..dfd1e9db4c 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties @@ -199,4 +199,5 @@ com.oceanbase.odc.ErrorCodes.TimeDataTypePrecisionMismatch=時間類型精度不 com.oceanbase.odc.ErrorCodes.DatabaseAccessDenied=資料庫存取被拒絕，因為目前使用者沒有資料庫的{0}權限 com.oceanbase.odc.ErrorCodes.ObQueryProfileNotSupported=執行詳情僅在高於或等於 {0} 的 OceanBase 版本可用 com.oceanbase.odc.ErrorCodes.WorksheetEditVersionConflict=有人剛剛修改了這個工作簿，請重新整理頁面以取得最新版本並繼續編輯 -com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=工作空間中使用的資料庫用戶類型必須為超級帳號 \ No newline at end of file +com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=工作空間中使用的資料庫用戶類型必須為超級帳號 +com.oceanbase.odc.ErrorCodes.UnsupportedSyncTableStructure=結構同步暫不支持 {0} 到 {1} \ No newline at end of file diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMConfiguration.java index 475793c5e3..4d188abd6f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMConfiguration.java @@ -16,11 +16,9 @@ package com.oceanbase.odc.service.dlm; import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import com.oceanbase.tools.migrator.common.enums.ShardingStrategy; -import com.oceanbase.tools.migrator.core.IJobStore; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -36,9 +34,6 @@ @Configuration public class DLMConfiguration { - @Value("${odc.task.dlm.thread-pool-size:15}") - private int dlmThreadPoolSize; - @Value("${odc.task.dlm.single-task-read-write-ratio:0.5}") private double readWriteRatio; @@ -54,9 +49,4 @@ public class DLMConfiguration { @Value("${odc.task.dlm.default-scan-batch-size:10000}") private int defaultScanBatchSize; - @Bean - public DLMJobFactory dlmJobFactory(IJobStore jobStore) { - return new DLMJobFactory(jobStore); - } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobFactory.java index e11e83d500..c8e234bf0c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobFactory.java @@ -18,9 +18,9 @@ import com.oceanbase.odc.service.dlm.model.DlmTableUnit; import com.oceanbase.odc.service.dlm.model.DlmTableUnitParameters; import com.oceanbase.tools.migrator.common.dto.HistoryJob; -import com.oceanbase.tools.migrator.core.IJobStore; import com.oceanbase.tools.migrator.core.JobFactory; import com.oceanbase.tools.migrator.core.JobReq; +import com.oceanbase.tools.migrator.core.store.IJobStore; import com.oceanbase.tools.migrator.job.Job; import lombok.extern.slf4j.Slf4j; @@ -41,7 +41,6 @@ public Job createJob(DlmTableUnit parameters) { HistoryJob historyJob = new HistoryJob(); historyJob.setId(parameters.getDlmTableUnitId()); historyJob.setJobType(parameters.getType()); - historyJob.setTableId(-1L); historyJob.setPrintSqlTrace(false); historyJob.setSourceTable(parameters.getTableName()); historyJob.setTargetTable(parameters.getTargetTableName()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java index 94414bf025..08c2885bc3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java @@ -25,27 +25,19 @@ import java.util.Map; import com.alibaba.druid.pool.DruidDataSource; -import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.dlm.model.DlmTableUnit; -import com.oceanbase.odc.service.dlm.model.RateLimitConfiguration; -import com.oceanbase.odc.service.schedule.job.DLMJobReq; -import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.tools.migrator.common.dto.JobStatistic; -import com.oceanbase.tools.migrator.common.dto.TableSizeInfo; import com.oceanbase.tools.migrator.common.dto.TaskGenerator; import com.oceanbase.tools.migrator.common.element.PrimaryKey; import com.oceanbase.tools.migrator.common.exception.JobException; import com.oceanbase.tools.migrator.common.exception.JobSqlException; -import com.oceanbase.tools.migrator.common.meta.TableMeta; -import com.oceanbase.tools.migrator.core.IJobStore; import com.oceanbase.tools.migrator.core.handler.genarator.GeneratorStatus; import com.oceanbase.tools.migrator.core.handler.genarator.GeneratorType; -import com.oceanbase.tools.migrator.core.meta.ClusterMeta; import com.oceanbase.tools.migrator.core.meta.JobMeta; import com.oceanbase.tools.migrator.core.meta.TaskMeta; -import com.oceanbase.tools.migrator.core.meta.TenantMeta; +import com.oceanbase.tools.migrator.core.store.IJobStore; import lombok.extern.slf4j.Slf4j; @@ -60,7 +52,6 @@ public class DLMJobStore implements IJobStore { private DruidDataSource dataSource; private boolean enableBreakpointRecovery = false; private Map dlmTableUnits; - private Map jobParameters; public DLMJobStore(ConnectionConfig metaDBConfig) { @@ -141,11 +132,6 @@ public void storeTaskGenerator(TaskGenerator taskGenerator) throws SQLException } } - @Override - public void bindGeneratorToJob(String s, TaskGenerator taskGenerator) throws SQLException { - - } - @Override public JobStatistic getJobStatistic(String s) throws JobException { return new JobStatistic(); @@ -225,7 +211,8 @@ public void storeTaskMeta(TaskMeta taskMeta) throws SQLException { } @Override - public Long getAbnormalTaskIndex(String jobId) { + public long getAbnormalTaskCount(String jobId) { + long count = 9999; if (enableBreakpointRecovery) { try (Connection conn = dataSource.getConnection(); PreparedStatement ps = conn.prepareStatement( @@ -233,78 +220,14 @@ public Long getAbnormalTaskIndex(String jobId) { ps.setString(1, jobId); ResultSet resultSet = ps.executeQuery(); if (resultSet.next()) { - long count = resultSet.getLong(1); - return count > 0 ? count : null; + count = resultSet.getLong(1); } } catch (Exception ignored) { log.warn("Get abnormal task failed.jobId={}", jobId); } } - return null; - } - - @Override - public void updateTableSizeInfo(TableSizeInfo tableSizeInfo, long l) { - - } - - @Override - public void updateLimiter(JobMeta jobMeta) { - try { - RateLimitConfiguration params; - if (jobParameters.containsKey(JobParametersKeyConstants.DLM_RATE_LIMIT_CONFIG)) { - params = JsonUtils.fromJson( - jobParameters.get(JobParametersKeyConstants.DLM_RATE_LIMIT_CONFIG), - RateLimitConfiguration.class); - } else { - DLMJobReq dlmJobReq = JsonUtils.fromJson( - jobParameters.get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), - DLMJobReq.class); - params = dlmJobReq.getRateLimit(); - } - if (params.getDataSizeLimit() != null) { - setClusterLimitConfig(jobMeta.getSourceCluster(), params.getDataSizeLimit()); - setClusterLimitConfig(jobMeta.getTargetCluster(), params.getDataSizeLimit()); - setTenantLimitConfig(jobMeta.getSourceTenant(), params.getDataSizeLimit()); - setTenantLimitConfig(jobMeta.getTargetTenant(), params.getDataSizeLimit()); - log.info("Update rate limit success,dataSizeLimit={}", params.getDataSizeLimit()); - } - if (params.getRowLimit() != null) { - setTableLimitConfig(jobMeta.getTargetTableMeta(), params.getRowLimit()); - setTableLimitConfig(jobMeta.getSourceTableMeta(), params.getRowLimit()); - log.info("Update rate limit success,rowLimit={}", params.getRowLimit()); - } - } catch (Exception e) { - log.warn("Update rate limit failed,errorMsg={}", e.getMessage()); - setClusterLimitConfig(jobMeta.getSourceCluster(), 1024); - setClusterLimitConfig(jobMeta.getTargetCluster(), 1024); - setTenantLimitConfig(jobMeta.getSourceTenant(), 1024); - setTenantLimitConfig(jobMeta.getTargetTenant(), 1024); - setTableLimitConfig(jobMeta.getTargetTableMeta(), 1000); - setTableLimitConfig(jobMeta.getSourceTableMeta(), 1000); - } + return count; } - public void setJobParameters(Map jobParameters) { - this.jobParameters = jobParameters; - } - - private void setClusterLimitConfig(ClusterMeta clusterMeta, long dataSizeLimit) { - clusterMeta.setReadSizeLimit(dataSizeLimit); - clusterMeta.setWriteSizeLimit(dataSizeLimit); - clusterMeta.setWriteUsedQuota(1); - clusterMeta.setReadUsedQuota(1); - } - private void setTenantLimitConfig(TenantMeta tenantMeta, long dataSizeLimit) { - tenantMeta.setReadSizeLimit(dataSizeLimit); - tenantMeta.setWriteSizeLimit(dataSizeLimit); - tenantMeta.setWriteUsedQuota(1); - tenantMeta.setReadUsedQuota(1); - } - - private void setTableLimitConfig(TableMeta tableMeta, int rowLimit) { - tableMeta.setReadRowCountLimit(rowLimit); - tableMeta.setWriteRowCountLimit(rowLimit); - } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataArchiveJobStore.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataArchiveJobStore.java deleted file mode 100644 index f373059cb0..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataArchiveJobStore.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.dlm; - -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -import com.oceanbase.odc.common.json.JsonUtils; -import com.oceanbase.odc.metadb.dlm.DlmTableUnitRepository; -import com.oceanbase.odc.metadb.dlm.TaskGeneratorEntity; -import com.oceanbase.odc.metadb.dlm.TaskGeneratorRepository; -import com.oceanbase.odc.metadb.dlm.TaskUnitEntity; -import com.oceanbase.odc.metadb.dlm.TaskUnitRepository; -import com.oceanbase.odc.service.dlm.model.RateLimitConfiguration; -import com.oceanbase.odc.service.dlm.utils.DlmJobIdUtil; -import com.oceanbase.odc.service.dlm.utils.TaskGeneratorMapper; -import com.oceanbase.odc.service.dlm.utils.TaskUnitMapper; -import com.oceanbase.odc.service.schedule.model.DlmTableUnitStatistic; -import com.oceanbase.tools.migrator.common.dto.JobStatistic; -import com.oceanbase.tools.migrator.common.dto.TableSizeInfo; -import com.oceanbase.tools.migrator.common.dto.TaskGenerator; -import com.oceanbase.tools.migrator.common.meta.TableMeta; -import com.oceanbase.tools.migrator.core.IJobStore; -import com.oceanbase.tools.migrator.core.meta.ClusterMeta; -import com.oceanbase.tools.migrator.core.meta.JobMeta; -import com.oceanbase.tools.migrator.core.meta.TaskMeta; -import com.oceanbase.tools.migrator.core.meta.TenantMeta; - -import lombok.extern.slf4j.Slf4j; - -/** - * @Author：tinker - * @Date: 2023/5/8 19:27 - * @Descripition: TODO Store runtime data and use it to resume execution from a breakpoint. - */ -@Component -@Slf4j -public class DataArchiveJobStore implements IJobStore { - - @Value("${odc.task.dlm.support-breakpoint-recovery:false}") - private boolean supportBreakpointRecovery; - @Autowired - private DlmLimiterService limiterService; - @Autowired - private TaskGeneratorRepository taskGeneratorRepository; - @Autowired - private TaskUnitRepository taskUnitRepository; - @Autowired - private DlmTableUnitRepository dlmTableUnitRepository; - - private final TaskGeneratorMapper taskGeneratorMapper = TaskGeneratorMapper.INSTANCE; - private final TaskUnitMapper taskUnitMapper = TaskUnitMapper.INSTANCE; - - @Override - public TaskGenerator getTaskGenerator(String generatorId, String jobId) { - if (supportBreakpointRecovery) { - return taskGeneratorRepository.findByJobId(jobId).map(taskGeneratorMapper::entityToModel) - .orElse(null); - } - return null; - } - - @Override - public void storeTaskGenerator(TaskGenerator taskGenerator) { - if (supportBreakpointRecovery) { - Optional optional = taskGeneratorRepository.findByGeneratorId(taskGenerator.getId()); - TaskGeneratorEntity entity; - if (optional.isPresent()) { - entity = optional.get(); - entity.setStatus(taskGenerator.getGeneratorStatus().name()); - entity.setTaskCount(taskGenerator.getTaskCount()); - entity.setPartitionSavePoint(taskGenerator.getGeneratorPartitionSavepoint()); - entity.setProcessedRowCount(taskGenerator.getProcessedRowCount()); - entity.setProcessedDataSize(taskGenerator.getProcessedDataSize()); - if (taskGenerator.getGeneratorSavePoint() != null) { - entity.setPrimaryKeySavePoint(taskGenerator.getGeneratorSavePoint().toSqlString()); - } - } else { - entity = taskGeneratorMapper.modelToEntity(taskGenerator); - } - taskGeneratorRepository.save(entity); - } - } - - @Override - public void bindGeneratorToJob(String jobId, TaskGenerator taskGenerator) {} - - @Override - public JobStatistic getJobStatistic(String s) { - return new JobStatistic(); - } - - @Override - public void storeJobStatistic(JobMeta jobMeta) { - DlmTableUnitStatistic dlmExecutionDetail = new DlmTableUnitStatistic(); - dlmExecutionDetail.setProcessedRowCount(jobMeta.getJobStat().getRowCount()); - dlmExecutionDetail.setProcessedRowsPerSecond(jobMeta.getJobStat().getAvgRowCount()); - dlmExecutionDetail.setReadRowCount(jobMeta.getJobStat().getReadRowCount()); - dlmExecutionDetail.setReadRowsPerSecond(jobMeta.getJobStat().getAvgReadRowCount()); - dlmTableUnitRepository.updateStatisticByDlmTableUnitId(jobMeta.getJobId(), - JsonUtils.toJson(dlmExecutionDetail)); - } - - @Override - public List getTaskMeta(JobMeta jobMeta) { - if (supportBreakpointRecovery) { - List tasks = taskUnitRepository.findByGeneratorId(jobMeta.getGenerator().getId()).stream().map( - taskUnitMapper::entityToModel).collect( - Collectors.toList()); - tasks.forEach(o -> o.setJobMeta(jobMeta)); - return tasks; - } - return null; - } - - @Override - public void storeTaskMeta(TaskMeta taskMeta) { - if (supportBreakpointRecovery) { - Optional optional = taskUnitRepository.findByJobIdAndGeneratorIdAndTaskIndex( - taskMeta.getJobMeta().getJobId(), taskMeta.getGeneratorId(), taskMeta.getTaskIndex()); - TaskUnitEntity entity; - if (optional.isPresent()) { - entity = optional.get(); - entity.setStatus(taskMeta.getTaskStatus().name()); - entity.setPartitionName(taskMeta.getPartitionName()); - if (taskMeta.getMinPrimaryKey() != null) { - entity.setLowerBoundPrimaryKey(taskMeta.getMinPrimaryKey().toSqlString()); - } - if (taskMeta.getMaxPrimaryKey() != null) { - entity.setUpperBoundPrimaryKey(taskMeta.getMaxPrimaryKey().toSqlString()); - } - if (taskMeta.getCursorPrimaryKey() != null) { - entity.setPrimaryKeyCursor(taskMeta.getCursorPrimaryKey().toSqlString()); - } - } else { - entity = taskUnitMapper.modelToEntity(taskMeta); - } - taskUnitRepository.save(entity); - } - } - - @Override - public Long getAbnormalTaskIndex(String jobId) { - if (supportBreakpointRecovery) { - Long abnormalTaskCount = taskUnitRepository.findAbnormalTaskByJobId(jobId); - if (abnormalTaskCount != 0) { - return abnormalTaskCount; - } - } - return null; - } - - @Override - public void updateTableSizeInfo(TableSizeInfo tableSizeInfo, long l) { - - } - - @Override - public void updateLimiter(JobMeta jobMeta) { - RateLimitConfiguration rateLimit; - try { - rateLimit = limiterService - .getByOrderIdOrElseDefaultConfig(Long.parseLong(DlmJobIdUtil.getJobName(jobMeta.getJobId()))); - } catch (Exception e) { - log.warn("Update limiter failed,jobId={},error={}", - jobMeta.getJobId(), e); - return; - } - setClusterLimitConfig(jobMeta.getSourceCluster(), rateLimit.getDataSizeLimit()); - setClusterLimitConfig(jobMeta.getTargetCluster(), rateLimit.getDataSizeLimit()); - setTenantLimitConfig(jobMeta.getSourceTenant(), rateLimit.getDataSizeLimit()); - setTenantLimitConfig(jobMeta.getTargetTenant(), rateLimit.getDataSizeLimit()); - setTableLimitConfig(jobMeta.getSourceTableMeta(), rateLimit.getRowLimit()); - setTableLimitConfig(jobMeta.getTargetTableMeta(), rateLimit.getRowLimit()); - } - - private void setClusterLimitConfig(ClusterMeta clusterMeta, long dataSizeLimit) { - clusterMeta.setReadSizeLimit(dataSizeLimit); - clusterMeta.setWriteSizeLimit(dataSizeLimit); - clusterMeta.setWriteUsedQuota(1); - clusterMeta.setReadUsedQuota(1); - } - - private void setTenantLimitConfig(TenantMeta tenantMeta, long dataSizeLimit) { - tenantMeta.setReadSizeLimit(dataSizeLimit); - tenantMeta.setWriteSizeLimit(dataSizeLimit); - tenantMeta.setWriteUsedQuota(1); - tenantMeta.setReadUsedQuota(1); - } - - private void setTableLimitConfig(TableMeta tableMeta, int rowLimit) { - tableMeta.setReadRowCountLimit(rowLimit); - tableMeta.setWriteRowCountLimit(rowLimit); - } - -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java index f817395291..16decc6e91 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java @@ -27,8 +27,8 @@ import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; import com.oceanbase.odc.service.session.factory.OBConsoleDataSourceFactory; import com.oceanbase.tools.migrator.common.configure.DataSourceInfo; -import com.oceanbase.tools.migrator.common.enums.DataBaseType; -import com.oceanbase.tools.migrator.common.util.EncryptUtils; +import com.oceanbase.tools.migrator.common.enums.DatasourceType; +import com.oceanbase.tools.migrator.datasource.fs.FileFormat; import lombok.extern.slf4j.Slf4j; @@ -44,12 +44,12 @@ public static ConnectionConfig toConnectionConfig(DataSourceInfo dataSourceInfo) ConnectionConfig connectionConfig = new ConnectionConfig(); connectionConfig.setDefaultSchema(dataSourceInfo.getDatabaseName()); connectionConfig.setPassword(dataSourceInfo.getPassword()); - connectionConfig.setHost(dataSourceInfo.getIp()); + connectionConfig.setHost(dataSourceInfo.getHost()); connectionConfig.setPort(dataSourceInfo.getPort()); - connectionConfig.setUsername(dataSourceInfo.getFullUserName()); - connectionConfig.setType(ConnectType.valueOf(dataSourceInfo.getDatabaseType().name())); + connectionConfig.setUsername(dataSourceInfo.getUsername()); + connectionConfig.setType(ConnectType.valueOf(dataSourceInfo.getType().name())); // convert full username to native user name - if (dataSourceInfo.getDatabaseType() == DataBaseType.OB_ORACLE) { + if (dataSourceInfo.getType() == DatasourceType.OB_ORACLE) { String userName = connectionConfig.getUsername(); if (userName.contains("#")) { userName = userName.split("#")[0]; @@ -61,12 +61,11 @@ public static ConnectionConfig toConnectionConfig(DataSourceInfo dataSourceInfo) userName = userName.replace("\"", ""); } connectionConfig.setUsername(userName); - connectionConfig.setTenantName(dataSourceInfo.getTenantName()); - connectionConfig.setClusterName(dataSourceInfo.getClusterName()); } return connectionConfig; } + public static DataSourceInfo toDataSourceInfo(ConnectionConfig connectionConfig, String schemaName) { DataSourceInfo dataSourceInfo = new DataSourceInfo(); dataSourceInfo.setDatabaseName(connectionConfig.getDefaultSchema()); @@ -74,48 +73,45 @@ public static DataSourceInfo toDataSourceInfo(ConnectionConfig connectionConfig, if (StringUtils.isNotEmpty(connectionConfig.getPassword())) { dataSourceInfo.setPassword(connectionConfig.getPassword()); } - dataSourceInfo.setIp(connectionConfig.getHost()); + dataSourceInfo.setHost(connectionConfig.getHost()); dataSourceInfo.setPort(connectionConfig.getPort()); switch (connectionConfig.getDialectType()) { case DORIS: case MYSQL: { - dataSourceInfo.setFullUserName(connectionConfig.getUsername()); - dataSourceInfo.setDatabaseType(DataBaseType.MYSQL); + dataSourceInfo.setUsername(connectionConfig.getUsername()); + dataSourceInfo.setType(DatasourceType.MYSQL); break; } case OB_MYSQL: { dataSourceInfo - .setFullUserName(OBConsoleDataSourceFactory.getUsername(connectionConfig)); - dataSourceInfo.setDatabaseType(DataBaseType.OB_MYSQL); - dataSourceInfo.setSysUser(connectionConfig.getSysTenantUsername()); - dataSourceInfo.setClusterName(connectionConfig.getClusterName()); - if (StringUtils.isNotEmpty(connectionConfig.getSysTenantPassword())) { - try { - dataSourceInfo.setSysPassword(EncryptUtils.encode(connectionConfig.getSysTenantPassword())); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - dataSourceInfo.setSysDatabaseName("oceanbase"); + .setUsername(OBConsoleDataSourceFactory.getUsername(connectionConfig)); + dataSourceInfo.setType(DatasourceType.OB_MYSQL); break; } case OB_ORACLE: - dataSourceInfo.setFullUserName(OBConsoleDataSourceFactory.getUsername(connectionConfig)); - dataSourceInfo.setClusterName(connectionConfig.getClusterName()); - dataSourceInfo.setTenantName(connectionConfig.getTenantName()); - dataSourceInfo.setDatabaseType(DataBaseType.OB_ORACLE); + dataSourceInfo.setUsername(OBConsoleDataSourceFactory.getUsername(connectionConfig)); + dataSourceInfo.setType(DatasourceType.OB_ORACLE); break; case POSTGRESQL: - dataSourceInfo.setFullUserName(connectionConfig.getUsername()); + dataSourceInfo.setUsername(connectionConfig.getUsername()); connectionConfig.setDefaultSchema(schemaName); String jdbcUrl = getJdbcUrl(connectionConfig) + "&stringtype=unspecified"; - dataSourceInfo.setJdbcUrl(jdbcUrl); - dataSourceInfo.setDatabaseType(DataBaseType.POSTGRESQL); + dataSourceInfo.setUrl(jdbcUrl); + dataSourceInfo.setType(DatasourceType.POSTGRESQL); break; case ORACLE: - dataSourceInfo.setJdbcUrl(getJdbcUrl(connectionConfig)); - dataSourceInfo.setDatabaseType(DataBaseType.ORACLE); - dataSourceInfo.setFullUserName(getOracleUsername(connectionConfig)); + dataSourceInfo.setUrl(getJdbcUrl(connectionConfig)); + dataSourceInfo.setType(DatasourceType.ORACLE); + dataSourceInfo.setUsername(getOracleUsername(connectionConfig)); + break; + case FILE_SYSTEM: + dataSourceInfo.setHost(connectionConfig.getHost()); + dataSourceInfo.setType(DatasourceType.valueOf(connectionConfig.getType().name())); + dataSourceInfo.setUsername(connectionConfig.getUsername()); + dataSourceInfo.setPassword(connectionConfig.getPassword()); + dataSourceInfo.setFileFormat(FileFormat.CSV); + dataSourceInfo.setUrl(connectionConfig.getDefaultSchema()); + dataSourceInfo.setDefaultCharset("UTF-8"); break; default: log.warn(String.format("Unsupported datasource type:%s", connectionConfig.getDialectType())); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/AbstractDlmPreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/AbstractDlmPreprocessor.java index b7bd32427e..352f267210 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/AbstractDlmPreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/AbstractDlmPreprocessor.java @@ -181,7 +181,7 @@ public void supportDataArchivingLink(ConnectionConfig sourceDs, ConnectionConfig targetDs.getRegion())); } if (sourceDs.getDialectType().isMysql()) { - if (!targetDs.getDialectType().isMysql()) { + if (!targetDs.getDialectType().isMysql() && targetDs.getDialectType() != DialectType.FILE_SYSTEM) { throw new UnsupportedException( String.format("Unsupported data link from %s to %s.", sourceDs.getDialectType(), targetDs.getDialectType())); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java index 63c3b30f67..08d2f6604c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java @@ -18,18 +18,14 @@ import org.springframework.beans.factory.annotation.Autowired; import com.oceanbase.odc.core.session.ConnectionSession; -import com.oceanbase.odc.core.session.ConnectionSessionConstants; import com.oceanbase.odc.core.session.ConnectionSessionFactory; -import com.oceanbase.odc.core.shared.constant.DialectType; -import com.oceanbase.odc.core.shared.exception.UnsupportedException; -import com.oceanbase.odc.plugin.connect.api.InformationExtensionPoint; +import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.dlm.DLMConfiguration; -import com.oceanbase.odc.service.dlm.DLMTableStructureSynchronizer; import com.oceanbase.odc.service.dlm.model.DataArchiveParameters; -import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; import com.oceanbase.odc.service.schedule.model.OperationType; import com.oceanbase.odc.service.schedule.model.ScheduleChangeParams; import com.oceanbase.odc.service.schedule.model.ScheduleType; @@ -65,12 +61,14 @@ public void process(ScheduleChangeParams req) { Database sourceDb = databaseService.detail(parameters.getSourceDatabaseId()); Database targetDb = databaseService.detail(parameters.getTargetDataBaseId()); supportDataArchivingLink(sourceDb.getDataSource(), targetDb.getDataSource()); + if (!parameters.getSyncTableStructure().isEmpty()) { + PreConditions.validArgumentState(sourceDb.getDialectType() != targetDb.getDialectType(), + ErrorCodes.UnsupportedSyncTableStructure, null, null); + } ConnectionConfig sourceDs = sourceDb.getDataSource(); sourceDs.setDefaultSchema(sourceDb.getName()); ConnectionSessionFactory sourceSessionFactory = new DefaultConnectSessionFactory(sourceDs); - ConnectionSessionFactory targetSessionFactory = new DefaultConnectSessionFactory(targetDb.getDataSource()); ConnectionSession sourceSession = sourceSessionFactory.generateSession(); - ConnectionSession targetSession = targetSessionFactory.generateSession(); try { if (parameters.isFullDatabase()) { parameters.setTables(getAllTables(sourceSession, sourceDb.getName())); @@ -78,36 +76,9 @@ public void process(ScheduleChangeParams req) { if (parameters.getTables().isEmpty()) { throw new IllegalArgumentException("The table list is empty."); } - DialectType sourceDbType = sourceSession.getDialectType(); - DialectType targetDbType = targetSession.getDialectType(); - InformationExtensionPoint sourceInformation = - ConnectionPluginUtil.getInformationExtension(sourceDbType); - InformationExtensionPoint targetInformation = - ConnectionPluginUtil.getInformationExtension(targetDbType); - String sourceDbVersion = - sourceSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY).execute( - sourceInformation::getDBVersion); - String targetDbVersion = - targetSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY).execute( - targetInformation::getDBVersion); - if (!parameters.getSyncTableStructure().isEmpty()) { - boolean supportedSyncTableStructure = DLMTableStructureSynchronizer.isSupportedSyncTableStructure( - sourceDbType, sourceDbVersion, targetDbType, targetDbVersion); - if (!supportedSyncTableStructure) { - log.warn( - "Synchronization of table structure is unsupported,sourceDbType={},sourceDbVersion={},targetDbType={},targetDbVersion={}", - sourceDbType, - sourceDbVersion, targetDbType, targetDbVersion); - throw new UnsupportedException(String.format( - "Synchronization of table structure is unsupported,sourceDbType=%s,sourceDbVersion=%s,targetDbType=%s,targetDbVersion=%s", - sourceDbType, - sourceDbVersion, targetDbType, targetDbVersion)); - } - } checkTableAndCondition(sourceSession, sourceDb, parameters.getTables(), parameters.getVariables()); } finally { sourceSession.expire(); - targetSession.expire(); } log.info("Data archive preprocessing has been completed."); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java index 9e8a1b0760..43f0ae7feb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java @@ -32,6 +32,7 @@ import com.oceanbase.odc.service.dlm.DataSourceInfoMapper; import com.oceanbase.odc.service.dlm.model.DlmTableUnit; import com.oceanbase.odc.service.dlm.model.DlmTableUnitParameters; +import com.oceanbase.odc.service.dlm.model.RateLimitConfiguration; import com.oceanbase.odc.service.dlm.utils.DlmJobIdUtil; import com.oceanbase.odc.service.schedule.job.DLMJobReq; import com.oceanbase.odc.service.schedule.model.DlmTableUnitStatistic; @@ -40,6 +41,7 @@ import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.util.JobUtils; import com.oceanbase.tools.migrator.common.enums.JobType; +import com.oceanbase.tools.migrator.core.meta.JobMeta; import com.oceanbase.tools.migrator.job.Job; import com.oceanbase.tools.migrator.task.CheckMode; @@ -73,7 +75,6 @@ protected void doInit(JobContext context) { @Override public boolean start() throws Exception { - jobStore.setJobParameters(jobContext.getJobParameters()); DLMJobReq parameters = JsonUtils.fromJson( jobContext.getJobParameters().get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), @@ -213,13 +214,39 @@ public boolean modify(Map jobParameters) { if (!super.modify(jobParameters)) { return false; } - afterModifiedJobParameters(); + updateLimiter(jobParameters); return true; } - protected void afterModifiedJobParameters() { - if (jobStore != null) { - jobStore.setJobParameters(jobContext.getJobParameters()); + public void updateLimiter(Map jobParameters) { + if (job == null || job.getJobMeta() == null) { + return; + } + JobMeta jobMeta = job.getJobMeta(); + try { + RateLimitConfiguration params; + if (jobParameters.containsKey(JobParametersKeyConstants.DLM_RATE_LIMIT_CONFIG)) { + params = JsonUtils.fromJson( + jobParameters.get(JobParametersKeyConstants.DLM_RATE_LIMIT_CONFIG), + RateLimitConfiguration.class); + } else { + DLMJobReq dlmJobReq = JsonUtils.fromJson( + jobParameters.get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), + DLMJobReq.class); + params = dlmJobReq.getRateLimit(); + } + if (params.getDataSizeLimit() != null) { + jobMeta.getSourceLimiterConfig().setDataSizeLimit(params.getDataSizeLimit()); + jobMeta.getTargetLimiterConfig().setDataSizeLimit(params.getDataSizeLimit()); + log.info("Update rate limit success,dataSizeLimit={}", params.getDataSizeLimit()); + } + if (params.getRowLimit() != null) { + jobMeta.getSourceLimiterConfig().setRowLimit(params.getRowLimit()); + jobMeta.getTargetLimiterConfig().setRowLimit(params.getRowLimit()); + log.info("Update rate limit success,rowLimit={}", params.getRowLimit()); + } + } catch (Exception e) { + log.warn("Update rate limit failed,errorMsg={}", e.getMessage()); } } From ada7ab12c0e9edce4b4f7494b8e2bea32999d5bb Mon Sep 17 00:00:00 2001 From: tinker Date: Tue, 17 Dec 2024 12:12:10 +0800 Subject: [PATCH 11/50] upgrade hadoop-common from 3.3.4 to 3.3.6 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 56dd63bd11..2c7a98ca7d 100644 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,7 @@ 1.2.19 1.5.4 - 3.3.4 + 3.3.6 4.1.94.Final 1.2.1 2.1.6 From 2824cf54eb58d17ac0d1198f9a19245770419fb4 Mon Sep 17 00:00:00 2001 From: tinker Date: Tue, 17 Dec 2024 12:21:41 +0800 Subject: [PATCH 12/50] bugfix --- .../main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java index 08c2885bc3..db2db7e6a8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java @@ -212,7 +212,7 @@ public void storeTaskMeta(TaskMeta taskMeta) throws SQLException { @Override public long getAbnormalTaskCount(String jobId) { - long count = 9999; + long count = 0; if (enableBreakpointRecovery) { try (Connection conn = dataSource.getConnection(); PreparedStatement ps = conn.prepareStatement( From df72a73253c3c65a0e20ce748a7355e00f75b893 Mon Sep 17 00:00:00 2001 From: tinker Date: Tue, 17 Dec 2024 12:39:30 +0800 Subject: [PATCH 13/50] bugfix --- .../com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java index 16decc6e91..34bd0f4fcd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java @@ -110,7 +110,7 @@ public static DataSourceInfo toDataSourceInfo(ConnectionConfig connectionConfig, dataSourceInfo.setUsername(connectionConfig.getUsername()); dataSourceInfo.setPassword(connectionConfig.getPassword()); dataSourceInfo.setFileFormat(FileFormat.CSV); - dataSourceInfo.setUrl(connectionConfig.getDefaultSchema()); + dataSourceInfo.setDatabaseName(connectionConfig.getDefaultSchema()); dataSourceInfo.setDefaultCharset("UTF-8"); break; default: From a9f02c2426f7b2375c60ec66f56111d69f45eeee Mon Sep 17 00:00:00 2001 From: tinker Date: Wed, 18 Dec 2024 17:24:05 +0800 Subject: [PATCH 14/50] bugfix --- .../oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java index 9c61909e5a..d599ac0734 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java @@ -36,9 +36,8 @@ public interface TaskGeneratorMapper { @Mapping(source = "generatorId", target = "id") @Mapping(source = "status", target = "generatorStatus") - @Mapping(target = "generatorType", constant = "AUTO") - @Mapping(source = "partitionSavePoint", target = "generatorPartitionSavepoint") - @Mapping(target = "generatorSavePoint", + @Mapping(source = "partitionSavePoint", target = "partitionSavePoint") + @Mapping(target = "primaryKeySavePoint", expression = "java(com.oceanbase.tools.migrator.common.element.PrimaryKey.valuesOf(entity.getPrimaryKeySavePoint()))") TaskGenerator entityToModel(TaskGeneratorEntity entity); From badc00277076ec6c35a0da62f776faff207b1fa5 Mon Sep 17 00:00:00 2001 From: tinker Date: Wed, 18 Dec 2024 17:45:53 +0800 Subject: [PATCH 15/50] bugfix --- .../odc/service/dlm/DLMJobStore.java | 43 ++++++++----------- .../dlm/utils/TaskGeneratorMapper.java | 2 +- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java index db2db7e6a8..435f751095 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java @@ -31,11 +31,7 @@ import com.oceanbase.tools.migrator.common.dto.JobStatistic; import com.oceanbase.tools.migrator.common.dto.TaskGenerator; import com.oceanbase.tools.migrator.common.element.PrimaryKey; -import com.oceanbase.tools.migrator.common.exception.JobException; -import com.oceanbase.tools.migrator.common.exception.JobSqlException; import com.oceanbase.tools.migrator.core.handler.genarator.GeneratorStatus; -import com.oceanbase.tools.migrator.core.handler.genarator.GeneratorType; -import com.oceanbase.tools.migrator.core.meta.JobMeta; import com.oceanbase.tools.migrator.core.meta.TaskMeta; import com.oceanbase.tools.migrator.core.store.IJobStore; @@ -70,7 +66,7 @@ public void destroy() { } @Override - public TaskGenerator getTaskGenerator(String generatorId, String jobId) throws SQLException { + public TaskGenerator getTaskGenerator(String jobId) throws SQLException { if (enableBreakpointRecovery) { try (Connection conn = dataSource.getConnection(); PreparedStatement ps = conn.prepareStatement( @@ -80,15 +76,14 @@ public TaskGenerator getTaskGenerator(String generatorId, String jobId) throws S if (resultSet.next()) { TaskGenerator taskGenerator = new TaskGenerator(); taskGenerator.setId(resultSet.getString("generator_id")); - taskGenerator.setGeneratorType(GeneratorType.valueOf(resultSet.getString("type"))); taskGenerator.setGeneratorStatus(GeneratorStatus.valueOf(resultSet.getString("status"))); taskGenerator.setJobId(jobId); taskGenerator.setTaskCount(resultSet.getInt("task_count")); taskGenerator - .setGeneratorSavePoint(PrimaryKey.valuesOf(resultSet.getString("primary_key_save_point"))); + .setPrimaryKeySavePoint(PrimaryKey.valuesOf(resultSet.getString("primary_key_save_point"))); taskGenerator.setProcessedDataSize(resultSet.getLong("processed_row_count")); taskGenerator.setProcessedDataSize(resultSet.getLong("processed_data_size")); - taskGenerator.setGeneratorPartitionSavepoint(resultSet.getString("partition_save_point")); + taskGenerator.setPartitionSavePoint(resultSet.getString("partition_save_point")); log.info("Load task generator success.jobId={}", jobId); return taskGenerator; } @@ -118,11 +113,11 @@ public void storeTaskGenerator(TaskGenerator taskGenerator) throws SQLException ps.setLong(3, taskGenerator.getProcessedDataSize()); ps.setLong(4, taskGenerator.getProcessedRowCount()); ps.setString(5, taskGenerator.getGeneratorStatus().name()); - ps.setString(6, GeneratorType.AUTO.name()); + ps.setString(6, ""); ps.setLong(7, taskGenerator.getTaskCount()); - ps.setString(8, taskGenerator.getGeneratorSavePoint() == null ? "" - : taskGenerator.getGeneratorSavePoint().toSqlString()); - ps.setString(9, taskGenerator.getGeneratorPartitionSavepoint()); + ps.setString(8, taskGenerator.getPrimaryKeySavePoint() == null ? "" + : taskGenerator.getPrimaryKeySavePoint().toSqlString()); + ps.setString(9, taskGenerator.getPartitionSavePoint()); if (ps.executeUpdate() == 1) { log.info("Update task generator success.jobId={}", taskGenerator.getJobId()); } else { @@ -133,34 +128,34 @@ public void storeTaskGenerator(TaskGenerator taskGenerator) throws SQLException } @Override - public JobStatistic getJobStatistic(String s) throws JobException { + public JobStatistic getJobStatistic(String s) throws SQLException { return new JobStatistic(); } @Override - public void storeJobStatistic(JobMeta jobMeta) throws JobSqlException { - dlmTableUnits.get(jobMeta.getJobId()).getStatistic().setProcessedRowCount(jobMeta.getJobStat().getRowCount()); - dlmTableUnits.get(jobMeta.getJobId()).getStatistic() - .setProcessedRowsPerSecond(jobMeta.getJobStat().getAvgRowCount()); - - dlmTableUnits.get(jobMeta.getJobId()).getStatistic().setReadRowCount(jobMeta.getJobStat().getReadRowCount()); - dlmTableUnits.get(jobMeta.getJobId()).getStatistic() - .setReadRowsPerSecond(jobMeta.getJobStat().getAvgReadRowCount()); + public void storeJobStatistic(JobStatistic jobStatistic) throws SQLException { + dlmTableUnits.get(jobStatistic.getJobId()).getStatistic() + .setProcessedRowCount(jobStatistic.getRowCount().get()); + dlmTableUnits.get(jobStatistic.getJobId()).getStatistic() + .setProcessedRowsPerSecond(jobStatistic.getRowCountPerSeconds()); + + dlmTableUnits.get(jobStatistic.getJobId()).getStatistic().setReadRowCount(jobStatistic.getReadRowCount().get()); + dlmTableUnits.get(jobStatistic.getJobId()).getStatistic() + .setReadRowsPerSecond(jobStatistic.getReadRowCountPerSeconds()); } @Override - public List getTaskMeta(JobMeta jobMeta) throws SQLException { + public List loadUnfinishedTask(String generatorId) throws SQLException { if (enableBreakpointRecovery) { try (Connection conn = dataSource.getConnection(); PreparedStatement ps = conn.prepareStatement( "select * from dlm_task_unit where generator_id = ? AND status !='SUCCESS'")) { - ps.setString(1, jobMeta.getGenerator().getId()); + ps.setString(1, generatorId); ResultSet resultSet = ps.executeQuery(); List taskMetas = new LinkedList<>(); while (resultSet.next()) { TaskMeta taskMeta = new TaskMeta(); taskMeta.setTaskIndex(resultSet.getLong("task_index")); - taskMeta.setJobMeta(jobMeta); taskMeta.setGeneratorId(resultSet.getString("generator_id")); taskMeta.setTaskStatus(com.oceanbase.tools.migrator.common.enums.TaskStatus .valueOf(resultSet.getString("status"))); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java index d599ac0734..8e124d27e7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java @@ -44,7 +44,7 @@ public interface TaskGeneratorMapper { @InheritInverseConfiguration @Mapping(target = "type", constant = "AUTO") @Mapping(target = "primaryKeySavePoint", - expression = "java(model.getGeneratorSavePoint() != null ?model.getGeneratorSavePoint().toSqlString():null)") + expression = "java(model.getPrimaryKeySavePoint() != null ?model.getPrimaryKeySavePoint().toSqlString():null)") @Mapping(target = "id", ignore = true) TaskGeneratorEntity modelToEntity(TaskGenerator model); From 5a1efb6e32318d46591e6be11dae945e7cdb70f2 Mon Sep 17 00:00:00 2001 From: tinker Date: Mon, 23 Dec 2024 17:11:23 +0800 Subject: [PATCH 16/50] pull mode --- .../com/oceanbase/odc/service/task/util/JobPropertiesUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobPropertiesUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobPropertiesUtils.java index a0418ea523..342c5b9e29 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobPropertiesUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobPropertiesUtils.java @@ -63,7 +63,7 @@ public static void setMonitorMode(@NonNull Map jobProperties, public static TaskMonitorMode getMonitorMode(@NonNull Map jobProperties) { String monitorMode = jobProperties.get("monitorMode"); - return StringUtils.isBlank(monitorMode) ? TaskMonitorMode.PUSH : TaskMonitorMode.valueOf(monitorMode); + return StringUtils.isBlank(monitorMode) ? TaskMonitorMode.PULL : TaskMonitorMode.valueOf(monitorMode); } public static void setExecutorListenPort(@NonNull Map jobProperties, @NonNull Integer listenPort) { From 1014e846b6775211ed82a298eb92c0a033951efe Mon Sep 17 00:00:00 2001 From: tinker Date: Mon, 23 Dec 2024 17:42:21 +0800 Subject: [PATCH 17/50] revert --- server/odc-server/src/main/resources/log4j2-task.xml | 10 ++++++++++ .../odc/service/task/util/JobPropertiesUtils.java | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/server/odc-server/src/main/resources/log4j2-task.xml b/server/odc-server/src/main/resources/log4j2-task.xml index b79d3d78bf..8b084ad5e7 100644 --- a/server/odc-server/src/main/resources/log4j2-task.xml +++ b/server/odc-server/src/main/resources/log4j2-task.xml @@ -59,6 +59,11 @@ + + + + + @@ -77,6 +82,11 @@ + + + + + diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobPropertiesUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobPropertiesUtils.java index 342c5b9e29..a0418ea523 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobPropertiesUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobPropertiesUtils.java @@ -63,7 +63,7 @@ public static void setMonitorMode(@NonNull Map jobProperties, public static TaskMonitorMode getMonitorMode(@NonNull Map jobProperties) { String monitorMode = jobProperties.get("monitorMode"); - return StringUtils.isBlank(monitorMode) ? TaskMonitorMode.PULL : TaskMonitorMode.valueOf(monitorMode); + return StringUtils.isBlank(monitorMode) ? TaskMonitorMode.PUSH : TaskMonitorMode.valueOf(monitorMode); } public static void setExecutorListenPort(@NonNull Map jobProperties, @NonNull Integer listenPort) { From 1ae8572648c452d0f701fa0582e638d39fbf6e32 Mon Sep 17 00:00:00 2001 From: tinker Date: Mon, 30 Dec 2024 13:52:41 +0800 Subject: [PATCH 18/50] cannot sync oss datasource --- .../connection/ConnectionEventPublisher.java | 52 +++++++++++ .../service/connection/ConnectionService.java | 6 ++ .../connection/database/DatabaseService.java | 47 +--------- .../database/DatabaseSyncManager.java | 4 +- .../event/UpdateDatasourceEvent.java | 37 ++++++++ .../listener/UpdateDatasourceListener.java | 88 +++++++++++++++++++ .../db/schema/DBSchemaSyncTaskManager.java | 4 + 7 files changed, 192 insertions(+), 46 deletions(-) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionEventPublisher.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpdateDatasourceEvent.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionEventPublisher.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionEventPublisher.java new file mode 100644 index 0000000000..4920c18554 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionEventPublisher.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.oceanbase.odc.common.event.AbstractEvent; +import com.oceanbase.odc.common.event.LocalEventPublisher; +import com.oceanbase.odc.service.connection.listener.UpdateDatasourceListener; + +import lombok.NonNull; + +/** + * @Author：tinker + * @Date: 2024/12/30 10:57 + * @Descripition: + */ + +@Component +public class ConnectionEventPublisher { + @Autowired + private LocalEventPublisher localEventPublisher; + + @Autowired + private UpdateDatasourceListener updateDatasourceListener; + + @PostConstruct + public void init() { + localEventPublisher.addEventListener(updateDatasourceListener); + } + + public void publishEvent(@NonNull T event) { + localEventPublisher.publishEvent(event); + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java index 60dcb7b5d1..4f09ecc38c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java @@ -105,6 +105,7 @@ import com.oceanbase.odc.service.connection.ConnectionStatusManager.CheckState; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.DatabaseSyncManager; +import com.oceanbase.odc.service.connection.event.UpdateDatasourceEvent; import com.oceanbase.odc.service.connection.model.ConnectProperties; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.connection.model.OBTenantEndpoint; @@ -221,6 +222,9 @@ public class ConnectionService { @Autowired private TransactionTemplate txTemplate; + @Autowired + private ConnectionEventPublisher connectionEventPublisher; + private final ConnectionMapper mapper = ConnectionMapper.INSTANCE; public static final String DEFAULT_MIN_PRIVILEGE = "read"; @@ -249,6 +253,7 @@ public ConnectionConfig create(@NotNull @Valid ConnectionConfig connection, @Not } }); databaseSyncManager.submitSyncDataSourceAndDBSchemaTask(saved); + connectionEventPublisher.publishEvent(new UpdateDatasourceEvent(saved)); return saved; } @@ -684,6 +689,7 @@ private ConnectionConfig updateConnectionConfig(Long id, ConnectionConfig connec } }); databaseSyncManager.submitSyncDataSourceAndDBSchemaTask(config); + connectionEventPublisher.publishEvent(new UpdateDatasourceEvent(config)); return config; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index 83c36e3e49..f34fe41ffe 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -64,6 +64,7 @@ import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.constant.OrganizationType; import com.oceanbase.odc.core.shared.constant.ResourceRoleName; @@ -518,6 +519,9 @@ public Boolean internalSyncDataSourceSchemas(@NonNull Long dataSourceId) throws Optional organizationOpt = Optional.empty(); try { connection = connectionService.getForConnectionSkipPermissionCheck(dataSourceId); + if (connection.getDialectType() == DialectType.FILE_SYSTEM) { + return true; + } horizontalDataPermissionValidator.checkCurrentOrganization(connection); organizationOpt = organizationService.get(connection.getOrganizationId()); Organization organization = @@ -538,49 +542,6 @@ public Boolean internalSyncDataSourceSchemas(@NonNull Long dataSourceId) throws } } - @SkipAuthorize("internal usage") - public Boolean internalSyncFileSystemSchemas(@NonNull Long dataSourceId) throws InterruptedException { - Lock lock = jdbcLockRegistry.obtain(connectionService.getUpdateDsSchemaLockKey(dataSourceId)); - if (!lock.tryLock(3, TimeUnit.SECONDS)) { - throw new ConflictException(ErrorCodes.ResourceSynchronizing, - new Object[] {ResourceType.ODC_DATABASE.getLocalizedMessage()}, "Can not acquire jdbc lock"); - } - try { - ConnectionConfig connection = connectionService.getForConnectionSkipPermissionCheck(dataSourceId); - horizontalDataPermissionValidator.checkCurrentOrganization(connection); - List existDatabases = databaseRepository.findByConnectionId(dataSourceId).stream() - .filter(DatabaseEntity::getExisted).collect(Collectors.toList()); - if (CollectionUtils.isEmpty(existDatabases) - || !existDatabases.stream().map(DatabaseEntity::getName).collect( - Collectors.toSet()).contains(connection.getDefaultSchema())) { - DatabaseEntity entity = new DatabaseEntity(); - entity.setDatabaseId(com.oceanbase.odc.common.util.StringUtils.uuid()); - entity.setOrganizationId(connection.getOrganizationId()); - entity.setName(connection.getDefaultSchema()); - entity.setProjectId(connection.getProjectId()); - entity.setConnectionId(connection.getId()); - entity.setEnvironmentId(connection.getEnvironmentId()); - entity.setSyncStatus(DatabaseSyncStatus.SUCCEEDED); - entity.setExisted(true); - entity.setObjectSyncStatus(DBObjectSyncStatus.SYNCED); - entity.setConnectType(connection.getType()); - entity.setType(DatabaseType.PHYSICAL); - databaseRepository.save(entity); - } else { - DeleteDatabasesReq deleteDatabasesReq = new DeleteDatabasesReq(); - deleteDatabasesReq - .setDatabaseIds(existDatabases.stream().map(DatabaseEntity::getId).collect(Collectors.toSet())); - deleteDatabases(deleteDatabasesReq); - } - return true; - } catch (Exception ex) { - log.warn("Sync database failed, dataSourceId={}, errorMessage={}", dataSourceId, ex); - return false; - } finally { - lock.unlock(); - } - } - public int updateEnvironmentIdByConnectionId(@NotNull Long environmentId, @NotNull Long connectionId) { return databaseRepository.setEnvironmentIdByConnectionId(environmentId, connectionId); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseSyncManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseSyncManager.java index e971828b0f..17e11aebd8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseSyncManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseSyncManager.java @@ -29,7 +29,6 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.oceanbase.odc.core.authority.util.SkipAuthorize; -import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.exception.BadRequestException; import com.oceanbase.odc.metadb.iam.UserEntity; import com.oceanbase.odc.service.connection.ConnectionSyncHistoryService; @@ -95,8 +94,7 @@ public Future submitSyncDataSourceAndDBSchemaTask(@NonNull ConnectionCo private Boolean syncDBForDataSource(@NonNull ConnectionConfig dataSource) throws InterruptedException { Long creatorId = dataSource.getCreatorId(); SecurityContextUtils.setCurrentUser(creatorId, dataSource.getOrganizationId(), getAccountName(creatorId)); - return dataSource.getDialectType() == DialectType.FILE_SYSTEM ? databaseService.internalSyncFileSystemSchemas( - dataSource.getId()) : databaseService.internalSyncDataSourceSchemas(dataSource.getId()); + return databaseService.internalSyncDataSourceSchemas(dataSource.getId()); } private Future doExecute(Supplier> supplier) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpdateDatasourceEvent.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpdateDatasourceEvent.java new file mode 100644 index 0000000000..733e8a83fe --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpdateDatasourceEvent.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.event; + +import com.oceanbase.odc.common.event.AbstractEvent; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; + +import lombok.Getter; + +/** + * @Author：tianke + * @Date: 2024/12/30 10:41 + * @Descripition: + */ +public class UpdateDatasourceEvent extends AbstractEvent { + + @Getter + private ConnectionConfig connectionConfig; + + public UpdateDatasourceEvent(ConnectionConfig connectionCOnfig) { + super(connectionCOnfig, "UpdateDatasourceEvent"); + + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java new file mode 100644 index 0000000000..4a98c1b625 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.listener; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import com.oceanbase.odc.common.event.AbstractEventListener; +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.metadb.connection.DatabaseEntity; +import com.oceanbase.odc.metadb.connection.DatabaseRepository; +import com.oceanbase.odc.service.connection.database.model.DatabaseSyncStatus; +import com.oceanbase.odc.service.connection.database.model.DatabaseType; +import com.oceanbase.odc.service.connection.event.UpdateDatasourceEvent; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.db.schema.model.DBObjectSyncStatus; + +import lombok.extern.slf4j.Slf4j; + +/** + * @Author：tinker + * @Date: 2024/12/30 10:47 + * @Descripition: + */ +@Slf4j +@Component +public class UpdateDatasourceListener extends AbstractEventListener { + + @Autowired + private DatabaseRepository databaseRepository; + + @Override + public void onEvent(UpdateDatasourceEvent event) { + + ConnectionConfig connectionConfig = event.getConnectionConfig(); + if (connectionConfig.getDialectType() != DialectType.FILE_SYSTEM) { + return; + } + List byConnectionId = databaseRepository.findByConnectionId(connectionConfig.getId()); + DatabaseEntity entity = null; + if (!CollectionUtils.isEmpty(byConnectionId)) { + List toBeDelete = byConnectionId.stream().filter( + o -> !connectionConfig.getDefaultSchema().equals(o.getName())).map(DatabaseEntity::getId).collect( + Collectors.toList()); + if (!toBeDelete.isEmpty()) { + databaseRepository.deleteAllById(toBeDelete); + } + Optional existed = byConnectionId.stream().filter( + o -> connectionConfig.getDefaultSchema().equals(o.getName())).findFirst(); + if (existed.isPresent()) { + entity = existed.get(); + } + } + // create or update + entity = entity == null ? new DatabaseEntity() : entity; + entity.setDatabaseId(com.oceanbase.odc.common.util.StringUtils.uuid()); + entity.setOrganizationId(connectionConfig.getOrganizationId()); + entity.setName(connectionConfig.getDefaultSchema()); + entity.setProjectId(connectionConfig.getProjectId()); + entity.setConnectionId(connectionConfig.getId()); + entity.setEnvironmentId(connectionConfig.getEnvironmentId()); + entity.setSyncStatus(DatabaseSyncStatus.SUCCEEDED); + entity.setExisted(true); + entity.setObjectSyncStatus(DBObjectSyncStatus.SYNCED); + entity.setConnectType(connectionConfig.getType()); + entity.setType(DatabaseType.PHYSICAL); + databaseRepository.save(entity); + + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java index a7440a5cb0..428c5b2495 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java @@ -34,6 +34,7 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.exception.ConflictException; import com.oceanbase.odc.core.shared.exception.NotFoundException; @@ -113,6 +114,9 @@ public void submitTaskByDatabases(@NonNull Collection databases) { } public void submitTaskByDataSource(@NonNull ConnectionConfig dataSource) { + if (dataSource.getDialectType() == DialectType.FILE_SYSTEM) { + return; + } List databases = databaseService.listExistDatabasesByConnectionId(dataSource.getId()); databases.removeIf(e -> (syncProperties.isBlockExclusionsWhenSyncDbSchemas() && syncProperties.getExcludeSchemas(dataSource.getDialectType()).contains(e.getName())) From 461393aeecec2580be5a9d87f1e5746cc9d0445d Mon Sep 17 00:00:00 2001 From: tinker Date: Mon, 30 Dec 2024 14:13:29 +0800 Subject: [PATCH 19/50] code format --- .../odc/service/connection/ConnectionService.java | 6 +++--- ...dateDatasourceEvent.java => UpsertDatasourceEvent.java} | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/{UpdateDatasourceEvent.java => UpsertDatasourceEvent.java} (80%) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java index 4f09ecc38c..abeb4fb57d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java @@ -105,7 +105,7 @@ import com.oceanbase.odc.service.connection.ConnectionStatusManager.CheckState; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.DatabaseSyncManager; -import com.oceanbase.odc.service.connection.event.UpdateDatasourceEvent; +import com.oceanbase.odc.service.connection.event.UpsertDatasourceEvent; import com.oceanbase.odc.service.connection.model.ConnectProperties; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.connection.model.OBTenantEndpoint; @@ -253,7 +253,7 @@ public ConnectionConfig create(@NotNull @Valid ConnectionConfig connection, @Not } }); databaseSyncManager.submitSyncDataSourceAndDBSchemaTask(saved); - connectionEventPublisher.publishEvent(new UpdateDatasourceEvent(saved)); + connectionEventPublisher.publishEvent(new UpsertDatasourceEvent(saved)); return saved; } @@ -689,7 +689,7 @@ private ConnectionConfig updateConnectionConfig(Long id, ConnectionConfig connec } }); databaseSyncManager.submitSyncDataSourceAndDBSchemaTask(config); - connectionEventPublisher.publishEvent(new UpdateDatasourceEvent(config)); + connectionEventPublisher.publishEvent(new UpsertDatasourceEvent(config)); return config; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpdateDatasourceEvent.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpsertDatasourceEvent.java similarity index 80% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpdateDatasourceEvent.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpsertDatasourceEvent.java index 733e8a83fe..cb4f9bcd28 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpdateDatasourceEvent.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpsertDatasourceEvent.java @@ -25,13 +25,14 @@ * @Date: 2024/12/30 10:41 * @Descripition: */ -public class UpdateDatasourceEvent extends AbstractEvent { +public class UpsertDatasourceEvent extends AbstractEvent { @Getter private ConnectionConfig connectionConfig; - public UpdateDatasourceEvent(ConnectionConfig connectionCOnfig) { - super(connectionCOnfig, "UpdateDatasourceEvent"); + public UpsertDatasourceEvent(ConnectionConfig connectionConfig) { + super(connectionConfig, "UpsertDatasourceEvent"); + this.connectionConfig = connectionConfig; } } From 2b5f98855b091812f562a16f66f9fc28a73f181f Mon Sep 17 00:00:00 2001 From: tinker Date: Mon, 30 Dec 2024 14:19:59 +0800 Subject: [PATCH 20/50] code format --- .../connection/listener/UpdateDatasourceListener.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java index 4a98c1b625..456312fea5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java @@ -29,7 +29,7 @@ import com.oceanbase.odc.metadb.connection.DatabaseRepository; import com.oceanbase.odc.service.connection.database.model.DatabaseSyncStatus; import com.oceanbase.odc.service.connection.database.model.DatabaseType; -import com.oceanbase.odc.service.connection.event.UpdateDatasourceEvent; +import com.oceanbase.odc.service.connection.event.UpsertDatasourceEvent; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.db.schema.model.DBObjectSyncStatus; @@ -42,13 +42,13 @@ */ @Slf4j @Component -public class UpdateDatasourceListener extends AbstractEventListener { +public class UpdateDatasourceListener extends AbstractEventListener { @Autowired private DatabaseRepository databaseRepository; @Override - public void onEvent(UpdateDatasourceEvent event) { + public void onEvent(UpsertDatasourceEvent event) { ConnectionConfig connectionConfig = event.getConnectionConfig(); if (connectionConfig.getDialectType() != DialectType.FILE_SYSTEM) { From 40c0b8526767c308aa760e3c9e2695a840982704 Mon Sep 17 00:00:00 2001 From: tinker Date: Mon, 30 Dec 2024 16:46:36 +0800 Subject: [PATCH 21/50] opt dlm task implements --- .../odc/service/dlm/DLMJobStore.java | 19 +-- .../odc/service/dlm/model/DlmTableUnit.java | 4 + .../schedule/job/DataArchiveDeleteJob.java | 1 + .../service/schedule/job/DataArchiveJob.java | 1 + .../schedule/job/DataArchiveRollbackJob.java | 1 + .../service/schedule/job/DataDeleteJob.java | 1 + .../base/dataarchive/DataArchiveTask.java | 158 +++++++++--------- 7 files changed, 93 insertions(+), 92 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java index 73179105f5..ef541e95fb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java @@ -16,16 +16,13 @@ package com.oceanbase.odc.service.dlm; import java.sql.Connection; -import java.sql.Date; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.LinkedList; import java.util.List; -import java.util.Map; import com.alibaba.druid.pool.DruidDataSource; -import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.dlm.model.DlmTableUnit; import com.oceanbase.tools.migrator.common.dto.JobStatistic; @@ -35,6 +32,7 @@ import com.oceanbase.tools.migrator.core.meta.TaskMeta; import com.oceanbase.tools.migrator.core.store.IJobStore; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; /** @@ -47,16 +45,13 @@ public class DLMJobStore implements IJobStore { private DruidDataSource dataSource; private boolean enableBreakpointRecovery = false; - private Map dlmTableUnits; + @Setter + private DlmTableUnit dlmTableUnit; public DLMJobStore(ConnectionConfig metaDBConfig) { } - public void setDlmTableUnits(Map dlmTableUnits) { - this.dlmTableUnits = dlmTableUnits; - } - public void destroy() { if (dataSource == null) { return; @@ -137,13 +132,13 @@ public JobStatistic getJobStatistic(String s) throws SQLException { @Override public void storeJobStatistic(JobStatistic jobStatistic) throws SQLException { - dlmTableUnits.get(jobStatistic.getJobId()).getStatistic() + dlmTableUnit.getStatistic() .setProcessedRowCount(jobStatistic.getRowCount().get()); - dlmTableUnits.get(jobStatistic.getJobId()).getStatistic() + dlmTableUnit.getStatistic() .setProcessedRowsPerSecond(jobStatistic.getRowCountPerSeconds()); - dlmTableUnits.get(jobStatistic.getJobId()).getStatistic().setReadRowCount(jobStatistic.getReadRowCount().get()); - dlmTableUnits.get(jobStatistic.getJobId()).getStatistic() + dlmTableUnit.getStatistic().setReadRowCount(jobStatistic.getReadRowCount().get()); + dlmTableUnit.getStatistic() .setReadRowsPerSecond(jobStatistic.getReadRowCountPerSeconds()); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DlmTableUnit.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DlmTableUnit.java index 2370ed086d..929d90ecbc 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DlmTableUnit.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DlmTableUnit.java @@ -16,9 +16,11 @@ package com.oceanbase.odc.service.dlm.model; import java.util.Date; +import java.util.Set; import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.schedule.model.DlmTableUnitStatistic; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.migrator.common.configure.DataSourceInfo; import com.oceanbase.tools.migrator.common.enums.JobType; @@ -60,4 +62,6 @@ public class DlmTableUnit { private Date endTime; + private Set syncTableStructure; + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveDeleteJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveDeleteJob.java index be4bd69f4f..c0be02423a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveDeleteJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveDeleteJob.java @@ -65,6 +65,7 @@ public void executeJob(JobExecutionContext context) { DLMJobReq parameters = getDLMJobReq(dataArchiveTask.getJobId()); parameters.setJobType(JobType.DELETE); + parameters.setFireTime(context.getFireTime()); parameters.setScheduleTaskId(taskEntity.getId()); parameters .setRateLimit(limiterService.getByOrderIdOrElseDefaultConfig(Long.parseLong(taskEntity.getJobName()))); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveJob.java index ef6b448532..6534f436aa 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveJob.java @@ -48,6 +48,7 @@ private void executeInTaskFramework(JobExecutionContext context) { parameters.setScheduleTaskId(taskEntity.getId()); parameters.setJobType(JobType.MIGRATE); parameters.setTables(dataArchiveParameters.getTables()); + parameters.setFireTime(context.getFireTime()); for (DataArchiveTableConfig tableConfig : parameters.getTables()) { tableConfig.setConditionExpression(StringUtils.isNotEmpty(tableConfig.getConditionExpression()) ? DataArchiveConditionUtil.parseCondition(tableConfig.getConditionExpression(), diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveRollbackJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveRollbackJob.java index 67ef30c5d8..dc139884da 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveRollbackJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveRollbackJob.java @@ -63,6 +63,7 @@ public void executeJob(JobExecutionContext context) { DataSourceInfo tempDataSource = parameters.getSourceDs(); parameters.setSourceDs(parameters.getTargetDs()); parameters.setTargetDs(tempDataSource); + parameters.setFireTime(context.getFireTime()); parameters .setRateLimit(limiterService.getByOrderIdOrElseDefaultConfig(Long.parseLong(taskEntity.getJobName()))); parameters.getTables().forEach(o -> { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataDeleteJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataDeleteJob.java index 392abd3f72..00e2a60fc6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataDeleteJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataDeleteJob.java @@ -72,6 +72,7 @@ private void executeInTaskFramework(JobExecutionContext context) { parameters.getSourceDs().setQueryTimeout(dataDeleteParameters.getQueryTimeout()); parameters.getTargetDs().setQueryTimeout(dataDeleteParameters.getQueryTimeout()); parameters.setShardingStrategy(dataDeleteParameters.getShardingStrategy()); + parameters.setFireTime(context.getFireTime()); Long jobId = publishJob(parameters, dataDeleteParameters.getTimeoutMillis(), dataDeleteParameters.getDatabaseId()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java index a4b75585e6..d8a0d59510 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java @@ -15,15 +15,15 @@ */ package com.oceanbase.odc.service.task.base.dataarchive; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Date; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; +import org.springframework.util.CollectionUtils; + import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.dlm.DLMJobFactory; @@ -60,7 +60,8 @@ public class DataArchiveTask extends TaskBase> { private DLMJobStore jobStore; private double progress = 0.0; private Job job; - private Map result; + private List toDoList; + private int currentIndex = 0; private boolean isToStop = false; public DataArchiveTask() {} @@ -69,96 +70,70 @@ public DataArchiveTask() {} protected void doInit(JobContext context) { jobStore = new DLMJobStore(JobUtils.getMetaDBConnectionConfig()); jobFactory = new DLMJobFactory(jobStore); - log.info("Init data-archive job env succeed,jobIdentity={}", context.getJobIdentity()); - } - - @Override - public boolean start() throws Exception { - - DLMJobReq parameters = - JsonUtils.fromJson( - jobContext.getJobParameters().get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), - DLMJobReq.class); - if (parameters.getFireTime() == null) { - parameters.setFireTime(new Date()); - } try { - result = getDlmTableUnits(parameters).stream() - .collect(Collectors.toMap(DlmTableUnit::getDlmTableUnitId, o -> o)); - jobStore.setDlmTableUnits(result); + DLMJobReq parameters = + JsonUtils.fromJson( + jobContext.getJobParameters().get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), + DLMJobReq.class); + initTableUnit(parameters); } catch (Exception e) { - log.warn("Get dlm job failed!", e); - context.getExceptionListener().onException(e); - return false; + log.warn("Initialization of the DLM job was failed,jobIdentity={}", context.getJobIdentity(), e); } - Set dlmTableUnitIds = result.keySet(); + log.info("Initialization of the DLM job was successful. Number of tables to be processed = {},jobIdentity={}", + toDoList.size(), context.getJobIdentity()); + } - for (String dlmTableUnitId : dlmTableUnitIds) { - DlmTableUnit dlmTableUnit = result.get(dlmTableUnitId); - if (isToStop) { - log.info("Job is terminated,jobIdentity={}", jobContext.getJobIdentity()); - break; - } + @Override + public boolean start() throws Exception { + while (!isToStop && currentIndex < toDoList.size()) { + DlmTableUnit dlmTableUnit = toDoList.get(currentIndex); if (dlmTableUnit.getStatus() == TaskStatus.DONE) { log.info("The table had been completed,tableName={}", dlmTableUnit.getTableName()); + currentIndex++; continue; } - startTableUnit(dlmTableUnitId); - if (parameters.getJobType() == JobType.MIGRATE) { - try { - DLMTableStructureSynchronizer.sync( - DataSourceInfoMapper.toConnectionConfig(parameters.getSourceDs()), - DataSourceInfoMapper.toConnectionConfig(parameters.getTargetDs()), - dlmTableUnit.getTableName(), dlmTableUnit.getTargetTableName(), - parameters.getSyncTableStructure()); - } catch (Exception e) { - log.warn("Failed to sync target table structure,table will be ignored,tableName={}", - dlmTableUnit.getTableName(), e); - if (!parameters.getSyncTableStructure().isEmpty()) { - finishTableUnit(dlmTableUnitId, TaskStatus.FAILED); - continue; - } - } - } + syncTableStructure(dlmTableUnit); try { job = jobFactory.createJob(dlmTableUnit); - log.info("Init {} job succeed,DLMJobId={}", job.getJobMeta().getJobType(), job.getJobMeta().getJobId()); - log.info("{} job start,DLMJobId={}", job.getJobMeta().getJobType(), job.getJobMeta().getJobId()); - if (isToStop) { - finishTableUnit(dlmTableUnitId, TaskStatus.CANCELED); - job.stop(); - log.info("The task has stopped."); - break; - } else { - job.run(); - } - log.info("{} job finished,DLMJobId={}", dlmTableUnit.getType(), dlmTableUnitId); - finishTableUnit(dlmTableUnitId, TaskStatus.DONE); } catch (Throwable e) { - log.error("{} job failed,DLMJobId={},errorMsg={}", dlmTableUnit.getType(), dlmTableUnitId, e); - // set task status to failed if any job failed. - if (job != null && job.getJobMeta().isToStop()) { - finishTableUnit(dlmTableUnitId, TaskStatus.CANCELED); - } else { - finishTableUnit(dlmTableUnitId, TaskStatus.FAILED); - context.getExceptionListener().onException(e); - } + log.error("Failed to create job,dlmTableUnitId={}", dlmTableUnit.getDlmTableUnitId(), e); + dlmTableUnit.setStatus(TaskStatus.FAILED); + currentIndex++; + continue; } + log.info("Init {} job succeed,dlmTableUnitId={}", dlmTableUnit.getType(), dlmTableUnit.getDlmTableUnitId()); + try { + job.run(); + log.info("{} job finished,dlmTableUnitId={}", dlmTableUnit.getType(), dlmTableUnit.getDlmTableUnitId()); + dlmTableUnit.setStatus(TaskStatus.DONE); + } catch (Throwable e) { + dlmTableUnit.setStatus(isToStop ? TaskStatus.CANCELED : TaskStatus.FAILED); + context.getExceptionListener().onException(e); + } + dlmTableUnit.setEndTime(new Date()); + currentIndex++; } + log.info("All tables have been processed,jobIdentity={}.\n{}", jobContext.getJobIdentity(), buildReport()); return true; } - private void startTableUnit(String dlmTableUnitId) { - result.get(dlmTableUnitId).setStatus(TaskStatus.RUNNING); - result.get(dlmTableUnitId).setStartTime(new Date()); - } - - private void finishTableUnit(String dlmTableUnitId, TaskStatus status) { - result.get(dlmTableUnitId).setStatus(status); - result.get(dlmTableUnitId).setEndTime(new Date()); + private void syncTableStructure(DlmTableUnit tableUnit) { + if (tableUnit.getType() != JobType.MIGRATE || CollectionUtils.isEmpty(tableUnit.getSyncTableStructure())) { + return; + } + try { + DLMTableStructureSynchronizer.sync( + DataSourceInfoMapper.toConnectionConfig(tableUnit.getSourceDatasourceInfo()), + DataSourceInfoMapper.toConnectionConfig(tableUnit.getTargetDatasourceInfo()), + tableUnit.getTableName(), tableUnit.getTargetTableName(), + tableUnit.getSyncTableStructure()); + } catch (Exception e) { + log.warn("Failed to sync target table structure,tableName={}", + tableUnit.getTableName(), e); + } } - private List getDlmTableUnits(DLMJobReq req) throws SQLException { + private void initTableUnit(DLMJobReq req) { List dlmTableUnits = new LinkedList<>(); req.getTables().forEach(table -> { DlmTableUnit dlmTableUnit = new DlmTableUnit(); @@ -183,9 +158,32 @@ private List getDlmTableUnits(DLMJobReq req) throws SQLException { dlmTableUnit.setStatus(TaskStatus.PREPARING); dlmTableUnit.setType(req.getJobType()); dlmTableUnit.setStatistic(new DlmTableUnitStatistic()); + dlmTableUnit.setSyncTableStructure(req.getSyncTableStructure()); dlmTableUnits.add(dlmTableUnit); }); - return dlmTableUnits; + toDoList = new LinkedList<>(dlmTableUnits); + } + + private String buildReport() { + StringBuilder sb = new StringBuilder(); + sb.append("Job report:\n"); + sb.append("Total tables: ").append(toDoList.size()).append("\n"); + sb.append("Success tables: ") + .append(toDoList.stream().filter(t -> t.getStatus() == TaskStatus.DONE).map(DlmTableUnit::getTableName) + .collect( + Collectors.joining(","))) + .append("\n"); + sb.append("Failed tables: ") + .append(toDoList.stream().filter(t -> t.getStatus() == TaskStatus.FAILED) + .map(DlmTableUnit::getTableName).collect( + Collectors.joining(","))) + .append("\n"); + sb.append("Canceled tables: ") + .append(toDoList.stream().filter(t -> t.getStatus() == TaskStatus.CANCELED) + .map(DlmTableUnit::getTableName).collect( + Collectors.joining(","))) + .append("\n"); + return sb.toString(); } @Override @@ -194,9 +192,9 @@ public void stop() throws Exception { if (job != null) { try { job.stop(); - result.forEach((k, v) -> { - if (!v.getStatus().isTerminated()) { - v.setStatus(TaskStatus.CANCELED); + toDoList.forEach(t -> { + if (!t.getStatus().isTerminated()) { + t.setStatus(TaskStatus.CANCELED); } }); } catch (Exception e) { @@ -258,6 +256,6 @@ public double getProgress() { @Override public List getTaskResult() { - return new ArrayList<>(result.values()); + return new ArrayList<>(toDoList); } } From 1ee57f06bfad4ad3da4341a855c84dfa99033936 Mon Sep 17 00:00:00 2001 From: tinker Date: Tue, 31 Dec 2024 12:04:30 +0800 Subject: [PATCH 22/50] fix npe --- .../odc/service/task/base/dataarchive/DataArchiveTask.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java index d8a0d59510..b8e4f87792 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java @@ -94,6 +94,7 @@ public boolean start() throws Exception { } syncTableStructure(dlmTableUnit); try { + jobStore.setDlmTableUnit(dlmTableUnit); job = jobFactory.createJob(dlmTableUnit); } catch (Throwable e) { log.error("Failed to create job,dlmTableUnitId={}", dlmTableUnit.getDlmTableUnitId(), e); From e644922197de126c71503d6c8ebc3005ad3707b4 Mon Sep 17 00:00:00 2001 From: tinker Date: Thu, 2 Jan 2025 20:57:25 +0800 Subject: [PATCH 23/50] store partition range in disk --- .../odc/service/dlm/DLMJobStore.java | 2 ++ .../dlm/model/DataArchiveTableConfig.java | 6 ++++ .../service/schedule/job/AbstractDlmJob.java | 29 +++++++++++++------ .../schedule/model/DlmTableUnitStatistic.java | 7 +++++ 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java index ef541e95fb..b52bbdcf1f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java @@ -93,6 +93,8 @@ public TaskGenerator getTaskGenerator(String jobId) throws SQLException { @Override public void storeTaskGenerator(TaskGenerator taskGenerator) throws SQLException { + dlmTableUnit.getStatistic().setPartName2MaxKey(taskGenerator.getPartName2MaxKey()); + dlmTableUnit.getStatistic().setPartName2MinKey(taskGenerator.getPartName2MinKey()); if (enableBreakpointRecovery) { StringBuilder sb = new StringBuilder(); sb.append("INSERT INTO dlm_task_generator "); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DataArchiveTableConfig.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DataArchiveTableConfig.java index b852f42cdb..6a88248b79 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DataArchiveTableConfig.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DataArchiveTableConfig.java @@ -15,8 +15,10 @@ */ package com.oceanbase.odc.service.dlm.model; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import com.oceanbase.odc.common.util.StringUtils; @@ -40,6 +42,10 @@ public class DataArchiveTableConfig { // the sql condition such as "gmt_create < '2023-01-01'" private String conditionExpression; + private Map partName2MinKey = new HashMap<>(); + + private Map partName2MaxKey = new HashMap<>(); + public String getTargetTableName() { return StringUtils.isEmpty(targetTableName) ? tableName : targetTableName; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java index e3216a388e..697ac8ed57 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java @@ -17,6 +17,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; import org.quartz.JobExecutionContext; @@ -29,12 +30,13 @@ import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.dlm.DLMService; import com.oceanbase.odc.service.dlm.DataSourceInfoMapper; import com.oceanbase.odc.service.dlm.DlmLimiterService; +import com.oceanbase.odc.service.dlm.model.DlmTableUnit; import com.oceanbase.odc.service.quartz.util.ScheduleTaskUtils; import com.oceanbase.odc.service.schedule.ScheduleService; import com.oceanbase.odc.service.task.base.dataarchive.DataArchiveTask; -import com.oceanbase.odc.service.task.config.TaskFrameworkEnabledProperties; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.executor.task.TaskDescription; import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; @@ -58,11 +60,9 @@ public abstract class AbstractDlmJob implements OdcJob { public final DatabaseService databaseService; public final ScheduleService scheduleService; public final DlmLimiterService limiterService; + public final DLMService dlmService; public JobScheduler jobScheduler = null; - - public final TaskFrameworkEnabledProperties taskFrameworkProperties; - public final TaskFrameworkService taskFrameworkService; @@ -71,11 +71,9 @@ public AbstractDlmJob() { databaseService = SpringContextUtil.getBean(DatabaseService.class); scheduleService = SpringContextUtil.getBean(ScheduleService.class); limiterService = SpringContextUtil.getBean(DlmLimiterService.class); - taskFrameworkProperties = SpringContextUtil.getBean(TaskFrameworkEnabledProperties.class); taskFrameworkService = SpringContextUtil.getBean(TaskFrameworkService.class); - if (taskFrameworkProperties.isEnabled()) { - jobScheduler = SpringContextUtil.getBean(JobScheduler.class); - } + jobScheduler = SpringContextUtil.getBean(JobScheduler.class); + dlmService = SpringContextUtil.getBean(DLMService.class); } public DataSourceInfo getDataSourceInfo(Long databaseId) { @@ -129,10 +127,23 @@ public Long publishJob(DLMJobReq parameters, Long timeoutMillis, CloudProvider p } public DLMJobReq getDLMJobReq(Long jobId) { - return JsonUtils.fromJson(JsonUtils.fromJson( + DLMJobReq dlmJobReq = JsonUtils.fromJson(JsonUtils.fromJson( taskFrameworkService.find(jobId).getJobParametersJson(), new TypeReference>() {}).get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), DLMJobReq.class); + Map tableName2Unit = + dlmService.findByScheduleTaskId(dlmJobReq.getScheduleTaskId()).stream() + .collect( + Collectors.toMap(DlmTableUnit::getTableName, o -> o)); + dlmJobReq.getTables().forEach(o -> { + if (tableName2Unit.containsKey(o.getTableName()) + && tableName2Unit.get(o.getTableName()).getStatistic() != null) { + o.setPartName2MinKey(tableName2Unit.get(o.getTableName()).getStatistic().getPartName2MinKey()); + o.setPartName2MaxKey(tableName2Unit.get(o.getTableName()).getStatistic().getPartName2MaxKey()); + } + }); + + return dlmJobReq; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/DlmTableUnitStatistic.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/DlmTableUnitStatistic.java index 063accdf22..5faf400afc 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/DlmTableUnitStatistic.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/DlmTableUnitStatistic.java @@ -15,6 +15,9 @@ */ package com.oceanbase.odc.service.schedule.model; +import java.util.HashMap; +import java.util.Map; + import lombok.Data; /** @@ -33,4 +36,8 @@ public class DlmTableUnitStatistic { private Long readRowsPerSecond = 0L; + private Map partName2MinKey = new HashMap<>(); + + private Map partName2MaxKey = new HashMap<>(); + } From 175c06c37cf9282b1dedd2b5590588bd1731bcb8 Mon Sep 17 00:00:00 2001 From: tinker Date: Thu, 2 Jan 2025 21:07:14 +0800 Subject: [PATCH 24/50] bugfix --- .../odc/service/schedule/processor/DataArchivePreprocessor.java | 2 +- .../odc/service/task/base/dataarchive/DataArchiveTask.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java index 08d2f6604c..51e8d72bbd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java @@ -63,7 +63,7 @@ public void process(ScheduleChangeParams req) { supportDataArchivingLink(sourceDb.getDataSource(), targetDb.getDataSource()); if (!parameters.getSyncTableStructure().isEmpty()) { PreConditions.validArgumentState(sourceDb.getDialectType() != targetDb.getDialectType(), - ErrorCodes.UnsupportedSyncTableStructure, null, null); + ErrorCodes.UnsupportedSyncTableStructure, new Object[]{sourceDb.getDialectType(),targetDb.getDialectType()}, null); } ConnectionConfig sourceDs = sourceDb.getDataSource(); sourceDs.setDefaultSchema(sourceDb.getName()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java index b8e4f87792..54ebd269da 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java @@ -148,6 +148,8 @@ private void initTableUnit(DLMJobReq req) { jobParameter.setMigratePartitions(table.getPartitions()); jobParameter.setSyncDBObjectType(req.getSyncTableStructure()); jobParameter.setShardingStrategy(req.getShardingStrategy()); + jobParameter.setPartName2MinKey(table.getPartName2MinKey()); + jobParameter.setPartName2MaxKey(table.getPartName2MaxKey()); dlmTableUnit.setParameters(jobParameter); dlmTableUnit.setDlmTableUnitId(DlmJobIdUtil.generateHistoryJobId(req.getJobName(), req.getJobType().name(), req.getScheduleTaskId(), dlmTableUnits.size())); From 0e0f18670d189d4967d23569043cc804cfc8768b Mon Sep 17 00:00:00 2001 From: tinker Date: Thu, 2 Jan 2025 21:09:58 +0800 Subject: [PATCH 25/50] code format --- .../service/schedule/processor/DataArchivePreprocessor.java | 3 ++- .../odc/service/task/base/dataarchive/DataArchiveTask.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java index 51e8d72bbd..dad948983c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java @@ -63,7 +63,8 @@ public void process(ScheduleChangeParams req) { supportDataArchivingLink(sourceDb.getDataSource(), targetDb.getDataSource()); if (!parameters.getSyncTableStructure().isEmpty()) { PreConditions.validArgumentState(sourceDb.getDialectType() != targetDb.getDialectType(), - ErrorCodes.UnsupportedSyncTableStructure, new Object[]{sourceDb.getDialectType(),targetDb.getDialectType()}, null); + ErrorCodes.UnsupportedSyncTableStructure, + new Object[] {sourceDb.getDialectType(), targetDb.getDialectType()}, null); } ConnectionConfig sourceDs = sourceDb.getDataSource(); sourceDs.setDefaultSchema(sourceDb.getName()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java index 54ebd269da..ee40d2dc8c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java @@ -104,6 +104,7 @@ public boolean start() throws Exception { } log.info("Init {} job succeed,dlmTableUnitId={}", dlmTableUnit.getType(), dlmTableUnit.getDlmTableUnitId()); try { + dlmTableUnit.setStatus(TaskStatus.RUNNING); job.run(); log.info("{} job finished,dlmTableUnitId={}", dlmTableUnit.getType(), dlmTableUnit.getDlmTableUnitId()); dlmTableUnit.setStatus(TaskStatus.DONE); From a50bfc2f2519386d9bb98a5453edcb5d7506403f Mon Sep 17 00:00:00 2001 From: tinker Date: Mon, 6 Jan 2025 10:11:46 +0800 Subject: [PATCH 26/50] opt error message --- .../oceanbase/odc/core/shared/constant/ErrorCodes.java | 1 + .../src/main/resources/i18n/ErrorMessages.properties | 3 ++- .../main/resources/i18n/ErrorMessages_zh_CN.properties | 3 ++- .../main/resources/i18n/ErrorMessages_zh_TW.properties | 3 ++- server/odc-server/src/main/resources/data.sql | 3 ++- .../odc/service/schedule/ScheduleService.java | 10 ++++++---- 6 files changed, 15 insertions(+), 8 deletions(-) diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java index 248e08ab3d..e72965dc22 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java @@ -170,6 +170,7 @@ public enum ErrorCodes implements ErrorCode { // Schedule AlterScheduleExists, InvalidCronExpression, + ScheduleIntervalTooShort, // Partition plan PartitionPlanNoDropPreviewSqlGenerated, diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages.properties index 3570b3d911..a1aec59753 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages.properties @@ -204,4 +204,5 @@ com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=The database u com.oceanbase.odc.ErrorCodes.BucketNotExist=Bucket does not exist com.oceanbase.odc.ErrorCodes.InvalidAccessKeyId=Invalid access key id com.oceanbase.odc.ErrorCodes.SignatureDoesNotMatch=Invalid access key secret -com.oceanbase.odc.ErrorCodes.UnsupportedSyncTableStructure=Sync table structure is not supported for {0} to {1} \ No newline at end of file +com.oceanbase.odc.ErrorCodes.UnsupportedSyncTableStructure=Sync table structure is not supported for {0} to {1} +com.oceanbase.odc.ErrorCodes.ScheduleIntervalTooShort=The execution interval is configured too short, please reconfigure. The minimum interval is: {0} seconds. \ No newline at end of file diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties index 765f11caab..0a070012b2 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties @@ -203,4 +203,5 @@ com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=工作空间 com.oceanbase.odc.ErrorCodes.BucketNotExist=桶不存在 com.oceanbase.odc.ErrorCodes.InvalidAccessKeyId=无效的 AccessKeyId com.oceanbase.odc.ErrorCodes.SignatureDoesNotMatch=无效的 AccessKeySecret -com.oceanbase.odc.ErrorCodes.UnsupportedSyncTableStructure=结构同步暂不支持 {0} 到 {1} \ No newline at end of file +com.oceanbase.odc.ErrorCodes.UnsupportedSyncTableStructure=结构同步暂不支持 {0} 到 {1} +com.oceanbase.odc.ErrorCodes.ScheduleIntervalTooShort=执行间隔配置过短，请重新配置。最小间隔为：{0} 秒 \ No newline at end of file diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties index dfd1e9db4c..d7cb846d75 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties @@ -200,4 +200,5 @@ com.oceanbase.odc.ErrorCodes.DatabaseAccessDenied=資料庫存取被拒絕，因 com.oceanbase.odc.ErrorCodes.ObQueryProfileNotSupported=執行詳情僅在高於或等於 {0} 的 OceanBase 版本可用 com.oceanbase.odc.ErrorCodes.WorksheetEditVersionConflict=有人剛剛修改了這個工作簿，請重新整理頁面以取得最新版本並繼續編輯 com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=工作空間中使用的資料庫用戶類型必須為超級帳號 -com.oceanbase.odc.ErrorCodes.UnsupportedSyncTableStructure=結構同步暫不支持 {0} 到 {1} \ No newline at end of file +com.oceanbase.odc.ErrorCodes.UnsupportedSyncTableStructure=結構同步暫不支持 {0} 到 {1} +com.oceanbase.odc.ErrorCodes.ScheduleIntervalTooShort=執行間隔設定過短，請重新設定。最小間隔應為：{0} 秒 \ No newline at end of file diff --git a/server/odc-server/src/main/resources/data.sql b/server/odc-server/src/main/resources/data.sql index 3e9ce4e440..aa137b7500 100644 --- a/server/odc-server/src/main/resources/data.sql +++ b/server/odc-server/src/main/resources/data.sql @@ -859,4 +859,5 @@ INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('o INSERT INTO `config_system_configuration` (`key`, `value`, `application`, `profile`, `label`, `description`) VALUES ('odc.session.kill-query-or-session.max-supported-ob-version', '4.2.5', 'odc', 'default', 'master', 'Max OBVersion kill session or kill query supported, only take effect when value greater than 0') -ON DUPLICATE KEY UPDATE `id`=`id`; \ No newline at end of file +ON DUPLICATE KEY UPDATE `id`=`id`; +INSERT INTO config_system_configuration ( `key`, `value`, `description` ) VALUES( 'odc.schedule.min-interval', '600', 'DLM 单个任务默认每秒行限制' ) ON DUPLICATE KEY UPDATE `id` = `id`; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 6e8baa762f..f1b821e51d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -42,6 +42,7 @@ import org.quartz.Trigger; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.cglib.beans.BeanMap; import org.springframework.context.annotation.Lazy; import org.springframework.core.io.InputStreamResource; @@ -151,6 +152,9 @@ @Service @SkipAuthorize public class ScheduleService { + + @Value("${odc.schedule.min-interval:600}") + private int minInterval; @Autowired private ScheduleRepository scheduleRepository; @@ -415,10 +419,8 @@ private void validateTriggerConfig(TriggerConfig triggerConfig) { throw new IllegalArgumentException("Invalid cron expression"); } long intervalMills = nextFiveFireTimes.get(1).getTime() - nextFiveFireTimes.get(0).getTime(); - if (intervalMills / 1000 < 10 * 60) { - throw new IllegalArgumentException( - "The interval between weeks is too short. The minimum interval is 10 minutes."); - } + PreConditions.validArgumentState(intervalMills / 1000 > minInterval, ErrorCodes.ScheduleIntervalTooShort, + new Object[] {minInterval}, null); } } From 65b2917cbdad80eee111bf5b3ca817f0f98c00f5 Mon Sep 17 00:00:00 2001 From: tinker Date: Mon, 6 Jan 2025 11:19:05 +0800 Subject: [PATCH 27/50] opt task log --- server/odc-server/src/main/resources/log4j2-task.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/odc-server/src/main/resources/log4j2-task.xml b/server/odc-server/src/main/resources/log4j2-task.xml index 8b084ad5e7..06d93a7be3 100644 --- a/server/odc-server/src/main/resources/log4j2-task.xml +++ b/server/odc-server/src/main/resources/log4j2-task.xml @@ -117,6 +117,9 @@ + + + From 442b36498155258f7b645d8c91092b47c7fd09fe Mon Sep 17 00:00:00 2001 From: tinker Date: Mon, 6 Jan 2025 15:11:23 +0800 Subject: [PATCH 28/50] create table for all type --- .../odc/service/dlm/DLMTableStructureSynchronizer.java | 3 ++- .../odc/service/task/base/dataarchive/DataArchiveTask.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java index eee02f6a3d..07a974b368 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java @@ -94,11 +94,12 @@ public static void sync(ConnectionConfig srcConfig, ConnectionConfig tgtConfig, DBTableStructureComparator comparator = new DBTableStructureComparator(tgtTableEditor, tgtConfig.getType().getDialectType(), srcConfig.getDefaultSchema(), tgtConfig.getDefaultSchema()); List changeSqlScript = new LinkedList<>(); + targetType.remove(DBObjectType.TABLE); if (tgtTable == null) { srcTable.setSchemaName(tgtConfig.getDefaultSchema()); srcTable.setName(tgtTableName); changeSqlScript.add(tgtTableEditor.generateCreateObjectDDL(srcTable)); - } else { + } else if (!targetType.isEmpty()) { DBObjectComparisonResult result = comparator.compare(srcTable, tgtTable); if (result.getComparisonResult() == ComparisonResult.INCONSISTENT) { changeSqlScript = result.getSubDBObjectComparisonResult().stream() diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java index ee40d2dc8c..7dcb621538 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java @@ -120,7 +120,7 @@ public boolean start() throws Exception { } private void syncTableStructure(DlmTableUnit tableUnit) { - if (tableUnit.getType() != JobType.MIGRATE || CollectionUtils.isEmpty(tableUnit.getSyncTableStructure())) { + if (tableUnit.getType() != JobType.MIGRATE) { return; } try { From f6e5f2c8fb442aab75f2eaf71c1e557d0278ea87 Mon Sep 17 00:00:00 2001 From: tinker Date: Mon, 6 Jan 2025 17:54:44 +0800 Subject: [PATCH 29/50] enable save point --- .../com/oceanbase/odc/service/dlm/DLMJobStore.java | 10 +++++++++- .../service/task/base/dataarchive/DataArchiveTask.java | 2 -- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java index b52bbdcf1f..9b64145573 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java @@ -25,6 +25,7 @@ import com.alibaba.druid.pool.DruidDataSource; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.dlm.model.DlmTableUnit; +import com.oceanbase.odc.service.session.factory.DruidDataSourceFactory; import com.oceanbase.tools.migrator.common.dto.JobStatistic; import com.oceanbase.tools.migrator.common.dto.TaskGenerator; import com.oceanbase.tools.migrator.common.element.PrimaryKey; @@ -44,11 +45,18 @@ public class DLMJobStore implements IJobStore { private DruidDataSource dataSource; - private boolean enableBreakpointRecovery = false; + private boolean enableBreakpointRecovery = true; @Setter private DlmTableUnit dlmTableUnit; public DLMJobStore(ConnectionConfig metaDBConfig) { + try { + DruidDataSourceFactory druidDataSourceFactory = new DruidDataSourceFactory(metaDBConfig); + dataSource = (DruidDataSource) druidDataSourceFactory.getDataSource(); + } catch (Exception e) { + log.warn("Failed to connect to the meta database; closing save point."); + enableBreakpointRecovery = false; + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java index 7dcb621538..58c6645b0c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java @@ -22,8 +22,6 @@ import java.util.Map; import java.util.stream.Collectors; -import org.springframework.util.CollectionUtils; - import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.dlm.DLMJobFactory; From e2848b59b8812af1c226458b5e980030ebe687ec Mon Sep 17 00:00:00 2001 From: tinker Date: Mon, 6 Jan 2025 19:17:15 +0800 Subject: [PATCH 30/50] opt log info --- .../src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java index 9b64145573..b293ffeb0b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java @@ -183,7 +183,6 @@ public List loadUnfinishedTask(String generatorId) throws SQLException @Override public void storeTaskMeta(TaskMeta taskMeta) throws SQLException { if (enableBreakpointRecovery) { - log.info("start to store taskMeta:{}", taskMeta); StringBuilder sb = new StringBuilder(); sb.append("INSERT INTO dlm_task_unit "); sb.append( From e10cb0a1e1e1ae21760c50e03e8b7ace5a1ce77f Mon Sep 17 00:00:00 2001 From: tinker Date: Tue, 7 Jan 2025 11:04:50 +0800 Subject: [PATCH 31/50] bugfix --- server/odc-server/src/main/resources/data.sql | 3 +-- .../oceanbase/odc/service/dlm/DLMJobFactory.java | 2 ++ .../odc/service/dlm/model/DlmTableUnit.java | 5 +++++ .../processor/CreateFlowInstanceProcessAspect.java | 13 ------------- .../odc/service/schedule/ScheduleService.java | 4 ++-- .../task/base/dataarchive/DataArchiveTask.java | 6 ++++++ 6 files changed, 16 insertions(+), 17 deletions(-) diff --git a/server/odc-server/src/main/resources/data.sql b/server/odc-server/src/main/resources/data.sql index aa137b7500..3e9ce4e440 100644 --- a/server/odc-server/src/main/resources/data.sql +++ b/server/odc-server/src/main/resources/data.sql @@ -859,5 +859,4 @@ INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('o INSERT INTO `config_system_configuration` (`key`, `value`, `application`, `profile`, `label`, `description`) VALUES ('odc.session.kill-query-or-session.max-supported-ob-version', '4.2.5', 'odc', 'default', 'master', 'Max OBVersion kill session or kill query supported, only take effect when value greater than 0') -ON DUPLICATE KEY UPDATE `id`=`id`; -INSERT INTO config_system_configuration ( `key`, `value`, `description` ) VALUES( 'odc.schedule.min-interval', '600', 'DLM 单个任务默认每秒行限制' ) ON DUPLICATE KEY UPDATE `id` = `id`; +ON DUPLICATE KEY UPDATE `id`=`id`; \ No newline at end of file diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobFactory.java index c8e234bf0c..2829ddb2ab 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobFactory.java @@ -55,6 +55,8 @@ public Job createJob(DlmTableUnit parameters) { req.setHistoryJob(historyJob); req.setSourceDs(parameters.getSourceDatasourceInfo()); req.setTargetDs(parameters.getTargetDatasourceInfo()); + req.setSourceLimitConfig(parameters.getSourceLimitConfig()); + req.setTargetLimitConfig(parameters.getTargetLimitConfig()); return super.createJob(req); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DlmTableUnit.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DlmTableUnit.java index 929d90ecbc..03a0b03183 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DlmTableUnit.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DlmTableUnit.java @@ -23,6 +23,7 @@ import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.migrator.common.configure.DataSourceInfo; import com.oceanbase.tools.migrator.common.enums.JobType; +import com.oceanbase.tools.migrator.limiter.LimiterConfig; import lombok.Data; @@ -50,6 +51,10 @@ public class DlmTableUnit { private DataSourceInfo targetDatasourceInfo; + private LimiterConfig sourceLimitConfig; + + private LimiterConfig targetLimitConfig; + private DlmTableUnitStatistic statistic; private DlmTableUnitParameters parameters; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java index 0fa3896d45..256fef7270 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java @@ -44,9 +44,6 @@ import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.partitionplan.model.PartitionPlanConfig; import com.oceanbase.odc.service.quartz.util.QuartzCronExpressionUtils; -import com.oceanbase.odc.service.schedule.flowtask.AlterScheduleParameters; -import com.oceanbase.odc.service.schedule.model.JobType; -import com.oceanbase.odc.service.schedule.model.OperationType; import com.oceanbase.odc.service.schedule.model.TriggerConfig; import com.oceanbase.odc.service.schedule.model.TriggerStrategy; @@ -74,8 +71,6 @@ public class CreateFlowInstanceProcessAspect implements InitializingBean { @Value("${odc.task.trigger.minimum-interval:600}") private Long triggerMinimumIntervalSeconds; - private final Map scheduleTaskPreprocessors = new HashMap<>(); - private final Map flowTaskPreprocessors = new HashMap<>(); @Pointcut("@annotation(com.oceanbase.odc.service.flow.processor.EnablePreprocess) && args(com.oceanbase.odc.service.flow.model.CreateFlowInstanceReq)") @@ -148,14 +143,6 @@ private void validateTriggerConfig(CreateFlowInstanceReq req) { if (parameters.getDroppingTrigger() != null) { validateTriggerConfig(parameters.getDroppingTrigger()); } - return; - } - if (req.getParameters() instanceof AlterScheduleParameters) { - AlterScheduleParameters parameters = (AlterScheduleParameters) req.getParameters(); - if (parameters.getOperationType() == OperationType.CREATE - || parameters.getOperationType() == OperationType.UPDATE) { - validateTriggerConfig(parameters.getTriggerConfig()); - } } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index f1b821e51d..3e98bc4b49 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -153,7 +153,7 @@ @SkipAuthorize public class ScheduleService { - @Value("${odc.schedule.min-interval:600}") + @Value("${odc.task.trigger.minimum-interval:600}") private int minInterval; @Autowired private ScheduleRepository scheduleRepository; @@ -241,6 +241,7 @@ public List dispatchCreateSchedule(CreateFlowInstanceReq ScheduleChangeParams scheduleChangeParams; switch (parameters.getOperationType()) { case CREATE: { + validateTriggerConfig(parameters.getTriggerConfig()); CreateScheduleReq createScheduleReq = new CreateScheduleReq(); createScheduleReq.setParameters(parameters.getScheduleTaskParameters()); createScheduleReq.setTriggerConfig(parameters.getTriggerConfig()); @@ -278,7 +279,6 @@ public ChangeScheduleResp changeSchedule(ScheduleChangeParams req) { // create or load target schedule if (req.getOperationType() == OperationType.CREATE) { PreConditions.notNull(req.getCreateScheduleReq(), "req.createScheduleReq"); - validateTriggerConfig(req.getCreateScheduleReq().getTriggerConfig()); ScheduleEntity entity = new ScheduleEntity(); entity.setName(req.getCreateScheduleReq().getName()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java index 58c6645b0c..babbf55579 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java @@ -41,6 +41,7 @@ import com.oceanbase.tools.migrator.common.enums.JobType; import com.oceanbase.tools.migrator.core.meta.JobMeta; import com.oceanbase.tools.migrator.job.Job; +import com.oceanbase.tools.migrator.limiter.LimiterConfig; import com.oceanbase.tools.migrator.task.CheckMode; import lombok.extern.slf4j.Slf4j; @@ -161,6 +162,11 @@ private void initTableUnit(DLMJobReq req) { dlmTableUnit.setType(req.getJobType()); dlmTableUnit.setStatistic(new DlmTableUnitStatistic()); dlmTableUnit.setSyncTableStructure(req.getSyncTableStructure()); + LimiterConfig limiterConfig = new LimiterConfig(); + limiterConfig.setDataSizeLimit(req.getRateLimit().getDataSizeLimit()); + limiterConfig.setRowLimit(req.getRateLimit().getRowLimit()); + dlmTableUnit.setSourceLimitConfig(limiterConfig); + dlmTableUnit.setTargetLimitConfig(limiterConfig); dlmTableUnits.add(dlmTableUnit); }); toDoList = new LinkedList<>(dlmTableUnits); From 244b52c0b4acb64a2092755fcb26a22bbbb86213 Mon Sep 17 00:00:00 2001 From: tinker Date: Tue, 7 Jan 2025 11:52:48 +0800 Subject: [PATCH 32/50] bugfix --- .../odc/service/task/base/dataarchive/DataArchiveTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java index babbf55579..409b0f8ccc 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java @@ -97,7 +97,7 @@ public boolean start() throws Exception { job = jobFactory.createJob(dlmTableUnit); } catch (Throwable e) { log.error("Failed to create job,dlmTableUnitId={}", dlmTableUnit.getDlmTableUnitId(), e); - dlmTableUnit.setStatus(TaskStatus.FAILED); + dlmTableUnit.setStatus(isToStop ? TaskStatus.CANCELED : TaskStatus.FAILED); currentIndex++; continue; } From 27707b9323b856e2906a015861d1058cec588610 Mon Sep 17 00:00:00 2001 From: tinker Date: Wed, 8 Jan 2025 10:13:49 +0800 Subject: [PATCH 33/50] modify oss configuration --- .../FileSystemConnectionTesting.java | 56 +++++++++---------- .../connection/model/ConnectionConfig.java | 7 +++ .../odc/service/dlm/DataSourceInfoMapper.java | 2 +- 3 files changed, 33 insertions(+), 32 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java index aa3428f37d..04e8c88ee3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java @@ -16,11 +16,11 @@ package com.oceanbase.odc.service.connection; import java.io.ByteArrayInputStream; +import java.net.URI; import java.nio.charset.StandardCharsets; +import java.text.MessageFormat; import java.util.Collections; import java.util.UUID; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.springframework.stereotype.Component; @@ -51,30 +51,30 @@ @Component public class FileSystemConnectionTesting { - private static final String COS_ENDPOINT_REGEX = "cos\\.(\\w+-\\w+)\\.myqcloud\\.com"; - private static final String OSS_ENDPOINT_REGEX = "oss-([a-zA-Z0-9-]+)\\.aliyuncs\\.com"; - private static final String OBS_ENDPOINT_REGEX = "obs\\.([a-zA-Z0-9-]+)\\.myhuaweicloud\\.com"; - private static final String S3_ENDPOINT_REGEX = "s3\\.([a-zA-Z0-9-]+)\\.amazonaws\\.com(\\.cn)?"; + private static final String COS_ENDPOINT_PATTERN = "cos.{0}.myqcloud.com"; + private static final String OBS_ENDPOINT_PATTERN = "obs.{0}.myhuaweicloud.com"; + private static final String OSS_ENDPOINT_PATTERN = "oss-{0}.aliyuncs.com"; + private static final String S3_ENDPOINT_GLOBAL_PATTERN = "s3.{0}.amazonaws.com"; + private static final String S3_ENDPOINT_CN_PATTERN = "s3.{0}.amazonaws.com.cn"; + private static final String TMP_FILE_NAME_PREFIX = "odc-test-object-"; private static final String TMP_TEST_DATA = "This is a test object to check read and write permissions."; public ConnectionTestResult test(@NonNull ConnectionConfig config) { PreConditions.notBlank(config.getPassword(), "AccessKeySecret"); - PreConditions.notBlank(config.getDefaultSchema(), "Bucket"); - String[] splitPath = config.getDefaultSchema().split("/", 2); - String bucketName = splitPath[0]; - String path = splitPath.length > 1 ? splitPath[1] : ""; + URI uri = URI.create(config.getHost()); ObjectStorageConfiguration storageConfig = new ObjectStorageConfiguration(); storageConfig.setAccessKeyId(config.getUsername()); storageConfig.setAccessKeySecret(config.getPassword()); - storageConfig.setBucketName(bucketName); - storageConfig.setRegion(getRegion(config)); + storageConfig.setBucketName(uri.getAuthority()); + storageConfig.setRegion(config.getRegion()); storageConfig.setCloudProvider(getCloudProvider(config.getType())); - storageConfig.setPublicEndpoint(config.getHost()); + storageConfig.setPublicEndpoint(getEndPointByRegion(config.getType(), config.getRegion())); try { CloudClient cloudClient = new CloudResourceConfigurations.CloudClientBuilder().generateCloudClient(storageConfig); - String objectKey = path + generateTempFileName(); + String objectKey = uri.getPath().endsWith("/") ? uri.getPath() + generateTempFileName() + : uri.getPath() + "/" + generateTempFileName(); cloudClient.putObject(storageConfig.getBucketName(), objectKey, new ByteArrayInputStream(TMP_TEST_DATA.getBytes(StandardCharsets.UTF_8)), new ObjectMetadata()); DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(); @@ -107,16 +107,6 @@ public ConnectionTestResult test(@NonNull ConnectionConfig config) { } } - private String getRegion(ConnectionConfig config) { - Pattern pattern = Pattern.compile(getEndPointRegex(config.getType())); - Matcher matcher = pattern.matcher(config.getHost()); - if (matcher.find()) { - return matcher.group(1); - } else { - throw new UnExpectedException("Illegal endpoint"); - } - } - private CloudProvider getCloudProvider(ConnectType type) { switch (type) { case COS: @@ -132,18 +122,22 @@ private CloudProvider getCloudProvider(ConnectType type) { } } - private String getEndPointRegex(ConnectType type) { + private static String getEndPointByRegion(ConnectType type, String region) { switch (type) { case COS: - return COS_ENDPOINT_REGEX; + return MessageFormat.format(COS_ENDPOINT_PATTERN, region); + case OSS: + return MessageFormat.format(OSS_ENDPOINT_PATTERN, region); case OBS: - return OBS_ENDPOINT_REGEX; + return MessageFormat.format(OBS_ENDPOINT_PATTERN, region); case S3A: - return S3_ENDPOINT_REGEX; - case OSS: - return OSS_ENDPOINT_REGEX; + // Note there is a difference of Top-Level Domain between cn and global regions. + if (region.startsWith("cn-")) { + return MessageFormat.format(S3_ENDPOINT_CN_PATTERN, region); + } + return MessageFormat.format(S3_ENDPOINT_GLOBAL_PATTERN, region); default: - throw new UnExpectedException(); + throw new IllegalArgumentException("regionToEndpoint is not applicable for storageType " + type); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectionConfig.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectionConfig.java index b9bd8d1778..679ba0fdad 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectionConfig.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectionConfig.java @@ -546,6 +546,13 @@ public String getCloudProvider() { return o == null ? null : o.toString(); } + public void setRegion(@NotNull String region) { + if (this.attributes == null) { + attributes = new HashMap<>(); + } + attributes.put(REGION, region); + } + public String getRegion() { if (this.attributes == null) { return null; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java index 34bd0f4fcd..383e64ff44 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java @@ -110,7 +110,7 @@ public static DataSourceInfo toDataSourceInfo(ConnectionConfig connectionConfig, dataSourceInfo.setUsername(connectionConfig.getUsername()); dataSourceInfo.setPassword(connectionConfig.getPassword()); dataSourceInfo.setFileFormat(FileFormat.CSV); - dataSourceInfo.setDatabaseName(connectionConfig.getDefaultSchema()); + dataSourceInfo.setRegion(connectionConfig.getRegion()); dataSourceInfo.setDefaultCharset("UTF-8"); break; default: From 2d3b5aa59898df051acdf9bef04d7ac50fe15a66 Mon Sep 17 00:00:00 2001 From: tinker Date: Wed, 8 Jan 2025 10:52:52 +0800 Subject: [PATCH 34/50] modify oss configuration --- .../odc/core/shared/constant/ConnectType.java | 2 +- .../service/connection/ConnectionTesting.java | 1 + .../FileSystemConnectionTesting.java | 1 + .../connection/model/TestConnectionReq.java | 19 +++++++++++++++++++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java index db92a1cff0..6d2572833b 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java @@ -58,7 +58,7 @@ public boolean isODPSharding() { } public boolean isDefaultSchemaRequired() { - return isODPSharding() || this.dialectType == DialectType.FILE_SYSTEM; + return isODPSharding(); } public static ConnectType from(DialectType dialectType) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java index 06c3f762ff..f92bb5987b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java @@ -236,6 +236,7 @@ private ConnectionConfig reqToConnectionConfig(TestConnectionReq req) { config.setServiceName(req.getServiceName()); config.setUserRole(req.getUserRole()); config.setCatalogName(req.getCatalogName()); + config.setRegion(req.getRegion()); OBTenantEndpoint endpoint = req.getEndpoint(); if (Objects.nonNull(endpoint) && OceanBaseAccessMode.IC_PROXY == endpoint.getAccessMode()) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java index 04e8c88ee3..e7e0f3a9c6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java @@ -62,6 +62,7 @@ public class FileSystemConnectionTesting { public ConnectionTestResult test(@NonNull ConnectionConfig config) { PreConditions.notBlank(config.getPassword(), "AccessKeySecret"); + PreConditions.notBlank(config.getRegion(), "Region"); URI uri = URI.create(config.getHost()); ObjectStorageConfiguration storageConfig = new ObjectStorageConfiguration(); storageConfig.setAccessKeyId(config.getUsername()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/TestConnectionReq.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/TestConnectionReq.java index 8ca19b8fc9..1074d37549 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/TestConnectionReq.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/TestConnectionReq.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.connection.model; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -46,6 +47,9 @@ @Data @ToString(exclude = {"password"}) public class TestConnectionReq implements CloudConnectionConfig, SSLConnectionConfig { + + private static final String REGION = "region"; + /** * Connection ID，用于编辑连接页面未传密码参数时从已保存的连接信息中获取对应密码字段 */ @@ -157,6 +161,21 @@ public DialectType getDialectType() { return this.dialectType; } + public void setRegion(String region) { + if (this.attributes == null) { + attributes = new HashMap<>(); + } + attributes.put(REGION, region); + } + + public String getRegion() { + if (this.attributes == null) { + return null; + } + Object o = attributes.get(REGION); + return o == null ? null : o.toString(); + } + public static TestConnectionReq fromConnection(ConnectionConfig connection, ConnectionAccountType accountType) { PreConditions.notNull(accountType, "AccountType"); From 032a8164dac042a2cafaecb95f379d00e2cda6f5 Mon Sep 17 00:00:00 2001 From: tinker Date: Wed, 8 Jan 2025 11:11:47 +0800 Subject: [PATCH 35/50] fix object key --- .../odc/service/connection/FileSystemConnectionTesting.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java index e7e0f3a9c6..c041ed9f2b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java @@ -74,8 +74,8 @@ public ConnectionTestResult test(@NonNull ConnectionConfig config) { try { CloudClient cloudClient = new CloudResourceConfigurations.CloudClientBuilder().generateCloudClient(storageConfig); - String objectKey = uri.getPath().endsWith("/") ? uri.getPath() + generateTempFileName() - : uri.getPath() + "/" + generateTempFileName(); + String objectKey = uri.getPath().endsWith("/") ? uri.getAuthority() + uri.getPath() + generateTempFileName() + : uri.getAuthority() + uri.getPath() + "/" + generateTempFileName(); cloudClient.putObject(storageConfig.getBucketName(), objectKey, new ByteArrayInputStream(TMP_TEST_DATA.getBytes(StandardCharsets.UTF_8)), new ObjectMetadata()); DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(); From b855ec0d3ea6a2680f67a8cbd50789fea79e88e4 Mon Sep 17 00:00:00 2001 From: tinker Date: Wed, 8 Jan 2025 11:22:51 +0800 Subject: [PATCH 36/50] use file url as schema name --- .../connection/listener/UpdateDatasourceListener.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java index 456312fea5..f52b9832bb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java @@ -58,13 +58,13 @@ public void onEvent(UpsertDatasourceEvent event) { DatabaseEntity entity = null; if (!CollectionUtils.isEmpty(byConnectionId)) { List toBeDelete = byConnectionId.stream().filter( - o -> !connectionConfig.getDefaultSchema().equals(o.getName())).map(DatabaseEntity::getId).collect( + o -> !connectionConfig.getHost().equals(o.getName())).map(DatabaseEntity::getId).collect( Collectors.toList()); if (!toBeDelete.isEmpty()) { databaseRepository.deleteAllById(toBeDelete); } Optional existed = byConnectionId.stream().filter( - o -> connectionConfig.getDefaultSchema().equals(o.getName())).findFirst(); + o -> connectionConfig.getHost().equals(o.getName())).findFirst(); if (existed.isPresent()) { entity = existed.get(); } @@ -73,7 +73,7 @@ public void onEvent(UpsertDatasourceEvent event) { entity = entity == null ? new DatabaseEntity() : entity; entity.setDatabaseId(com.oceanbase.odc.common.util.StringUtils.uuid()); entity.setOrganizationId(connectionConfig.getOrganizationId()); - entity.setName(connectionConfig.getDefaultSchema()); + entity.setName(connectionConfig.getHost()); entity.setProjectId(connectionConfig.getProjectId()); entity.setConnectionId(connectionConfig.getId()); entity.setEnvironmentId(connectionConfig.getEnvironmentId()); From 218f9c5c45048b6831b28b0369d60a4ac662c040 Mon Sep 17 00:00:00 2001 From: tinker Date: Thu, 9 Jan 2025 13:11:14 +0800 Subject: [PATCH 37/50] opt oracle username --- .../odc/service/dlm/DataSourceInfoMapper.java | 27 +++++++++---------- .../base/dataarchive/DataArchiveTask.java | 5 +--- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java index 383e64ff44..6ea1d3b4d3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java @@ -48,20 +48,6 @@ public static ConnectionConfig toConnectionConfig(DataSourceInfo dataSourceInfo) connectionConfig.setPort(dataSourceInfo.getPort()); connectionConfig.setUsername(dataSourceInfo.getUsername()); connectionConfig.setType(ConnectType.valueOf(dataSourceInfo.getType().name())); - // convert full username to native user name - if (dataSourceInfo.getType() == DatasourceType.OB_ORACLE) { - String userName = connectionConfig.getUsername(); - if (userName.contains("#")) { - userName = userName.split("#")[0]; - } - if (userName.contains("@")) { - userName = userName.split("@")[0]; - } - if (userName.contains("\"")) { - userName = userName.replace("\"", ""); - } - connectionConfig.setUsername(userName); - } return connectionConfig; } @@ -89,8 +75,19 @@ public static DataSourceInfo toDataSourceInfo(ConnectionConfig connectionConfig, break; } case OB_ORACLE: - dataSourceInfo.setUsername(OBConsoleDataSourceFactory.getUsername(connectionConfig)); dataSourceInfo.setType(DatasourceType.OB_ORACLE); + // convert full username to native user name + String userName = connectionConfig.getUsername(); + if (userName.contains("#")) { + userName = userName.split("#")[0]; + } + if (userName.contains("@")) { + userName = userName.split("@")[0]; + } + if (userName.contains("\"")) { + userName = userName.replaceAll("\"", ""); + } + dataSourceInfo.setUsername(userName); break; case POSTGRESQL: dataSourceInfo.setUsername(connectionConfig.getUsername()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java index 409b0f8ccc..07c5c3289f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java @@ -27,7 +27,6 @@ import com.oceanbase.odc.service.dlm.DLMJobFactory; import com.oceanbase.odc.service.dlm.DLMJobStore; import com.oceanbase.odc.service.dlm.DLMTableStructureSynchronizer; -import com.oceanbase.odc.service.dlm.DataSourceInfoMapper; import com.oceanbase.odc.service.dlm.model.DlmTableUnit; import com.oceanbase.odc.service.dlm.model.DlmTableUnitParameters; import com.oceanbase.odc.service.dlm.model.RateLimitConfiguration; @@ -123,9 +122,7 @@ private void syncTableStructure(DlmTableUnit tableUnit) { return; } try { - DLMTableStructureSynchronizer.sync( - DataSourceInfoMapper.toConnectionConfig(tableUnit.getSourceDatasourceInfo()), - DataSourceInfoMapper.toConnectionConfig(tableUnit.getTargetDatasourceInfo()), + DLMTableStructureSynchronizer.sync(tableUnit.getSourceDatasourceInfo(), tableUnit.getTargetDatasourceInfo(), tableUnit.getTableName(), tableUnit.getTargetTableName(), tableUnit.getSyncTableStructure()); } catch (Exception e) { From 0b297604a31adcf68406bfcdd8cdad92a3e21cbc Mon Sep 17 00:00:00 2001 From: tinker Date: Thu, 9 Jan 2025 14:08:53 +0800 Subject: [PATCH 38/50] rollback commit --- .../odc/service/dlm/DataSourceInfoMapper.java | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java index 6ea1d3b4d3..e33817c7f0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java @@ -75,19 +75,8 @@ public static DataSourceInfo toDataSourceInfo(ConnectionConfig connectionConfig, break; } case OB_ORACLE: + dataSourceInfo.setUsername(OBConsoleDataSourceFactory.getUsername(connectionConfig)); dataSourceInfo.setType(DatasourceType.OB_ORACLE); - // convert full username to native user name - String userName = connectionConfig.getUsername(); - if (userName.contains("#")) { - userName = userName.split("#")[0]; - } - if (userName.contains("@")) { - userName = userName.split("@")[0]; - } - if (userName.contains("\"")) { - userName = userName.replaceAll("\"", ""); - } - dataSourceInfo.setUsername(userName); break; case POSTGRESQL: dataSourceInfo.setUsername(connectionConfig.getUsername()); From bd6eda1eab72522af989e2c3b9043929e6cc20a1 Mon Sep 17 00:00:00 2001 From: tinker Date: Thu, 9 Jan 2025 14:18:46 +0800 Subject: [PATCH 39/50] enable ob sync table structure --- .../odc/service/dlm/DLMTableStructureSynchronizer.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java index 07a974b368..c772f46ac9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java @@ -129,11 +129,10 @@ public static void sync(ConnectionConfig srcConfig, ConnectionConfig tgtConfig, public static boolean isSupportedSyncTableStructure(DialectType srcType, String srcVersion, DialectType tgtType, String tgtVersion) { - // only supports MySQL or OBMySQL - if (!srcType.isMysql() || !tgtType.isMysql()) { + if (srcType != tgtType) { return false; } - if (srcType != tgtType) { + if (!srcType.isOceanbase() && !srcType.isMysql()) { return false; } // unsupported MySQL versions below 5.7.0 From 47da9d0f31a2616d9fc4dafd358aef304a61a8c2 Mon Sep 17 00:00:00 2001 From: tinker Date: Thu, 9 Jan 2025 18:13:55 +0800 Subject: [PATCH 40/50] adapt check point --- .../java/com/oceanbase/odc/service/dlm/DLMJobStore.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java index b293ffeb0b..c911936b03 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java @@ -101,8 +101,10 @@ public TaskGenerator getTaskGenerator(String jobId) throws SQLException { @Override public void storeTaskGenerator(TaskGenerator taskGenerator) throws SQLException { - dlmTableUnit.getStatistic().setPartName2MaxKey(taskGenerator.getPartName2MaxKey()); - dlmTableUnit.getStatistic().setPartName2MinKey(taskGenerator.getPartName2MinKey()); + taskGenerator.getPartName2MaxKey() + .forEach((k, v) -> dlmTableUnit.getStatistic().getPartName2MaxKey().put(k, v.getSqlString())); + taskGenerator.getPartName2MinKey() + .forEach((k, v) -> dlmTableUnit.getStatistic().getPartName2MinKey().put(k, v.getSqlString())); if (enableBreakpointRecovery) { StringBuilder sb = new StringBuilder(); sb.append("INSERT INTO dlm_task_generator "); From 21b6d62db276cf0fe9fa9064b2916312173976be Mon Sep 17 00:00:00 2001 From: tinker Date: Thu, 9 Jan 2025 18:39:34 +0800 Subject: [PATCH 41/50] fix region is null --- .../oceanbase/odc/service/connection/ConnectionService.java | 5 ++++- .../service/task/processor/result/DLMResultProcessor.java | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java index abeb4fb57d..ee391b9f10 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java @@ -398,8 +398,11 @@ public ConnectionConfig getWithoutPermissionCheck(@NotNull Long id) { @SkipAuthorize("odc internal usage") public List listByOrganizationId(@NonNull Long organizationId) { - return entitiesToModels(repository.findByOrganizationIdOrderByNameAsc(organizationId), organizationId, true, + List connectionConfigs = entitiesToModels( + repository.findByOrganizationIdOrderByNameAsc(organizationId), organizationId, true, true); + fullFillAttributes(connectionConfigs); + return connectionConfigs; } @SkipAuthorize("odc internal usage") diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/DLMResultProcessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/DLMResultProcessor.java index e61bd64b3f..50c428e1bb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/DLMResultProcessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/DLMResultProcessor.java @@ -48,7 +48,7 @@ public class DLMResultProcessor extends DLMProcessorMatcher implements ResultPro @Override public void process(TaskResult result) { - log.info("Start refresh result,result={}", result.getResultJson()); + log.info("Start refresh result,jobIdentity={}", result.getJobIdentity()); try { List dlmTableUnits = JsonUtils.fromJson(result.getResultJson(), new TypeReference>() {}); From 69896eddcae39182d2bdd9e9530e936d10e2d95f Mon Sep 17 00:00:00 2001 From: tinker Date: Thu, 9 Jan 2025 19:10:59 +0800 Subject: [PATCH 42/50] update status when task failed --- .../service/quartz/executor/AbstractQuartzJob.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/executor/AbstractQuartzJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/executor/AbstractQuartzJob.java index 661574d65c..d37fb4ecef 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/executor/AbstractQuartzJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/executor/AbstractQuartzJob.java @@ -27,7 +27,10 @@ import org.quartz.JobKey; import org.quartz.UnableToInterruptJobException; +import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.core.shared.exception.UnsupportedException; +import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; +import com.oceanbase.odc.metadb.schedule.ScheduleTaskRepository; import com.oceanbase.odc.service.common.util.SpringContextUtil; import com.oceanbase.odc.service.monitor.DefaultMeterName; import com.oceanbase.odc.service.monitor.MeterKey; @@ -75,6 +78,16 @@ public void execute(JobExecutionContext context) throws JobExecutionException { sendEndMetric(); } catch (Exception e) { sendFailedMetric(); + try { + log.info("Start to update schedule task status to failed,jobKey={}", jobKey); + ScheduleTaskRepository taskRepository = SpringContextUtil.getBean(ScheduleTaskRepository.class); + ScheduleTaskEntity taskEntity = (ScheduleTaskEntity) context.getResult(); + if (taskEntity != null && taskEntity.getId() != null) { + taskRepository.updateStatusById(taskEntity.getId(), TaskStatus.FAILED); + } + } catch (Exception innerException) { + log.warn("Update schedule task status failed.", innerException); + } log.warn("Job execute failed,job key={},fire time={}.", context.getJobDetail().getKey(), context.getFireTime(), e); } finally { From 000107231dddb1644f49da1075a344059f47773a Mon Sep 17 00:00:00 2001 From: tinker Date: Fri, 10 Jan 2025 15:33:53 +0800 Subject: [PATCH 43/50] bugfix --- .../main/java/com/oceanbase/odc/service/dlm/DLMService.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMService.java index fe02913217..9862737d42 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMService.java @@ -97,6 +97,9 @@ public void createOrUpdateDlmTableUnits(List dlmTableUnits) { DlmTableUnitEntity entity; if (entityOptional.isPresent()) { entity = entityOptional.get(); + if (entity.getStatus() == TaskStatus.DONE) { + return; + } entity.setStatistic(JsonUtils.toJson(o.getStatistic())); entity.setStatus(o.getStatus()); entity.setStartTime(o.getStartTime()); From b01986d9780c7914e8e6b3d665f0a72aa170ca8e Mon Sep 17 00:00:00 2001 From: tinker Date: Fri, 10 Jan 2025 16:23:09 +0800 Subject: [PATCH 44/50] merge 4.3.x --- client | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client b/client index 0021be9fee..97074b7ddf 160000 --- a/client +++ b/client @@ -1 +1 @@ -Subproject commit 0021be9feed6d121c190f1aa9565c0ab9fa34dc5 +Subproject commit 97074b7ddffb212b0a43687bb88665d94450eb7b From 1c6d191976a55b9337b4301ae606f97c369b3da5 Mon Sep 17 00:00:00 2001 From: tinker Date: Fri, 10 Jan 2025 18:18:53 +0800 Subject: [PATCH 45/50] record global range in statistic --- .../java/com/oceanbase/odc/service/dlm/DLMJobStore.java | 6 ++++++ .../odc/service/schedule/model/DlmTableUnitStatistic.java | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java index c911936b03..3d31cf1c77 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java @@ -105,6 +105,12 @@ public void storeTaskGenerator(TaskGenerator taskGenerator) throws SQLException .forEach((k, v) -> dlmTableUnit.getStatistic().getPartName2MaxKey().put(k, v.getSqlString())); taskGenerator.getPartName2MinKey() .forEach((k, v) -> dlmTableUnit.getStatistic().getPartName2MinKey().put(k, v.getSqlString())); + if (taskGenerator.getGlobalMaxKey() != null) { + dlmTableUnit.getStatistic().setGlobalMaxKey(taskGenerator.getGlobalMaxKey().getSqlString()); + } + if (taskGenerator.getGlobalMinKey() != null) { + dlmTableUnit.getStatistic().setGlobalMinKey(taskGenerator.getGlobalMinKey().getSqlString()); + } if (enableBreakpointRecovery) { StringBuilder sb = new StringBuilder(); sb.append("INSERT INTO dlm_task_generator "); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/DlmTableUnitStatistic.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/DlmTableUnitStatistic.java index 5faf400afc..f6aa86192e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/DlmTableUnitStatistic.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/DlmTableUnitStatistic.java @@ -36,6 +36,10 @@ public class DlmTableUnitStatistic { private Long readRowsPerSecond = 0L; + private String globalMinKey; + + private String globalMaxKey; + private Map partName2MinKey = new HashMap<>(); private Map partName2MaxKey = new HashMap<>(); From 45423916a8f6e02a6d4e35a6cfd349c639c02b9d Mon Sep 17 00:00:00 2001 From: tinker Date: Fri, 10 Jan 2025 18:25:14 +0800 Subject: [PATCH 46/50] record global range in statistic --- .../odc/service/dlm/model/DataArchiveTableConfig.java | 4 ++++ .../oceanbase/odc/service/schedule/job/AbstractDlmJob.java | 2 ++ 2 files changed, 6 insertions(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DataArchiveTableConfig.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DataArchiveTableConfig.java index 6a88248b79..06103cadc9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DataArchiveTableConfig.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DataArchiveTableConfig.java @@ -42,6 +42,10 @@ public class DataArchiveTableConfig { // the sql condition such as "gmt_create < '2023-01-01'" private String conditionExpression; + private String minKey; + + private String maxKey; + private Map partName2MinKey = new HashMap<>(); private Map partName2MaxKey = new HashMap<>(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java index 697ac8ed57..f523cf6c1a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java @@ -140,6 +140,8 @@ public DLMJobReq getDLMJobReq(Long jobId) { && tableName2Unit.get(o.getTableName()).getStatistic() != null) { o.setPartName2MinKey(tableName2Unit.get(o.getTableName()).getStatistic().getPartName2MinKey()); o.setPartName2MaxKey(tableName2Unit.get(o.getTableName()).getStatistic().getPartName2MaxKey()); + o.setMinKey(tableName2Unit.get(o.getTableName()).getStatistic().getGlobalMinKey()); + o.setMaxKey(tableName2Unit.get(o.getTableName()).getStatistic().getGlobalMaxKey()); } }); From 4525c772ef4ecb4c48d9fb24220dac50fce89d14 Mon Sep 17 00:00:00 2001 From: tinker Date: Mon, 13 Jan 2025 15:06:16 +0800 Subject: [PATCH 47/50] upgrade sdk version to 1.2.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dc7a3fb991..00e4729510 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,7 @@ 2.1.6 - 1.1.7 + 1.2.0 2.11.0 From 7e8bd0e438cb2e7fa37860d080f750678de00ffa Mon Sep 17 00:00:00 2001 From: tinker Date: Mon, 13 Jan 2025 17:53:43 +0800 Subject: [PATCH 48/50] rsp comment --- .../oceanbase/odc/service/connection/ConnectionTesting.java | 2 +- ...ConnectionTesting.java => FileSystemConnectionTester.java} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename server/odc-service/src/main/java/com/oceanbase/odc/service/connection/{FileSystemConnectionTesting.java => FileSystemConnectionTester.java} (98%) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java index f92bb5987b..bc046a116c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java @@ -75,7 +75,7 @@ public class ConnectionTesting { @Autowired private CloudMetadataClient cloudMetadataClient; @Autowired - private FileSystemConnectionTesting fileSystemConnectionTesting; + private FileSystemConnectionTester fileSystemConnectionTesting; @Value("${odc.sdk.test-connect.query-timeout-seconds:2}") private int queryTimeoutSeconds = 2; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTester.java similarity index 98% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTester.java index c041ed9f2b..43d5c37be3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTesting.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTester.java @@ -49,7 +49,7 @@ */ @Component -public class FileSystemConnectionTesting { +public class FileSystemConnectionTester { private static final String COS_ENDPOINT_PATTERN = "cos.{0}.myqcloud.com"; private static final String OBS_ENDPOINT_PATTERN = "obs.{0}.myhuaweicloud.com"; @@ -84,7 +84,7 @@ public ConnectionTestResult test(@NonNull ConnectionConfig config) { cloudClient.deleteObjects(deleteObjectsRequest); return ConnectionTestResult.success(config.getType()); } catch (CloudException e) { - if (e.getCause() instanceof OSSException) { + if (e.getCause() != null && e.getCause() instanceof OSSException) { OSSException cause = (OSSException) e.getCause(); switch (cause.getErrorCode()) { case OSSErrorCode.ACCESS_DENIED: From 6bb7aa8505130002d4b9adc723d9c90b026fcb95 Mon Sep 17 00:00:00 2001 From: tinker Date: Mon, 13 Jan 2025 17:57:50 +0800 Subject: [PATCH 49/50] rsp comment --- .../odc/service/connection/ConnectionTesting.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java index bc046a116c..ea7a4b31d5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java @@ -99,13 +99,14 @@ public ConnectionTestResult test(@NotNull @Valid TestConnectionReq req) { if (req.getAccountType() == ConnectionAccountType.SYS_READ) { connectionConfig.setDefaultSchema(null); } - return connectionConfig.getDialectType() == DialectType.FILE_SYSTEM - ? fileSystemConnectionTesting.test(connectionConfig) - : test(connectionConfig); + return test(connectionConfig); } public ConnectionTestResult test(@NonNull ConnectionConfig config) { ConnectType type = config.getType(); + if (type.getDialectType() == DialectType.FILE_SYSTEM) { + return fileSystemConnectionTesting.test(config); + } try { /** * 进行连接测试时需要关注的值有一个 {@link ConnectType}， 容易产生问题信息主要是两个：{@code username}, {@code defaultSchema} 首先分析 From 0bf893244008a1a5bb3a625fb8144026336d318e Mon Sep 17 00:00:00 2001 From: tinker Date: Mon, 13 Jan 2025 19:45:46 +0800 Subject: [PATCH 50/50] rsp comments --- server/odc-server/src/main/resources/data.sql | 8 +++++++- .../com/oceanbase/odc/service/dlm/DLMConfiguration.java | 6 ++++++ .../odc/service/schedule/job/AbstractDlmJob.java | 6 ++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/server/odc-server/src/main/resources/data.sql b/server/odc-server/src/main/resources/data.sql index 3e9ce4e440..b64d204ef5 100644 --- a/server/odc-server/src/main/resources/data.sql +++ b/server/odc-server/src/main/resources/data.sql @@ -859,4 +859,10 @@ INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('o INSERT INTO `config_system_configuration` (`key`, `value`, `application`, `profile`, `label`, `description`) VALUES ('odc.session.kill-query-or-session.max-supported-ob-version', '4.2.5', 'odc', 'default', 'master', 'Max OBVersion kill session or kill query supported, only take effect when value greater than 0') -ON DUPLICATE KEY UPDATE `id`=`id`; \ No newline at end of file +ON DUPLICATE KEY UPDATE `id`=`id`; +INSERT INTO config_system_configuration ( `key`, `value`, `description` ) VALUES('odc.task.dlm.session-limiting.enabled', 'true', +'Explosion-proof current limiting switch of mysql/oracle' ) +ON DUPLICATE KEY UPDATE `id` = `id`; +INSERT INTO config_system_configuration ( `key`, `value`, `description` ) VALUES('odc.task.dlm.session-limiting-ratio', '25', +'The ratio of oracle/mysql active sessions to the maximum number of connections allowed' ) +ON DUPLICATE KEY UPDATE `id` = `id`; \ No newline at end of file diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMConfiguration.java index 4d188abd6f..c79c3dd3fe 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMConfiguration.java @@ -49,4 +49,10 @@ public class DLMConfiguration { @Value("${odc.task.dlm.default-scan-batch-size:10000}") private int defaultScanBatchSize; + @Value("${odc.task.dlm.session-limiting.enabled:true}") + private boolean sessionLimitingEnabled; + + @Value("${odc.task.dlm.session-limiting-ratio:25}") + private int sessionLimitingRatio; + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java index f523cf6c1a..98c202020e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java @@ -30,6 +30,7 @@ import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.dlm.DLMConfiguration; import com.oceanbase.odc.service.dlm.DLMService; import com.oceanbase.odc.service.dlm.DataSourceInfoMapper; import com.oceanbase.odc.service.dlm.DlmLimiterService; @@ -65,6 +66,8 @@ public abstract class AbstractDlmJob implements OdcJob { public JobScheduler jobScheduler = null; public final TaskFrameworkService taskFrameworkService; + public final DLMConfiguration dlmConfiguration; + public AbstractDlmJob() { scheduleTaskRepository = SpringContextUtil.getBean(ScheduleTaskRepository.class); @@ -74,6 +77,7 @@ public AbstractDlmJob() { taskFrameworkService = SpringContextUtil.getBean(TaskFrameworkService.class); jobScheduler = SpringContextUtil.getBean(JobScheduler.class); dlmService = SpringContextUtil.getBean(DLMService.class); + dlmConfiguration = SpringContextUtil.getBean(DLMConfiguration.class); } public DataSourceInfo getDataSourceInfo(Long databaseId) { @@ -81,6 +85,8 @@ public DataSourceInfo getDataSourceInfo(Long databaseId) { ConnectionConfig config = databaseService.findDataSourceForTaskById(databaseId); DataSourceInfo dataSourceInfo = DataSourceInfoMapper.toDataSourceInfo(config, db.getName()); dataSourceInfo.setDatabaseName(db.getName()); + dataSourceInfo.setSessionLimitRatio(dlmConfiguration.getSessionLimitingRatio()); + dataSourceInfo.setEnabledLimit(dlmConfiguration.isSessionLimitingEnabled()); return dataSourceInfo; }