Skip to content

Commit

Permalink
Add cluster state checksum in manifest
Browse files Browse the repository at this point in the history 
Signed-off-by: Himshikha Gupta <[email protected]>
  • Loading branch information
Himshikha Gupta committed Aug 13, 2024
1 parent c6189a9 commit 379fe90
Show file tree
Hide file tree
Showing 6 changed files with 541 additions and 9 deletions.
1 change: 1 addition & 0 deletions server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -735,6 +735,7 @@ public void apply(Settings value, Settings current, Settings previous) {
IndicesService.CLUSTER_INDEX_RESTRICT_REPLICATION_TYPE_SETTING,
RemoteRoutingTableBlobStore.REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING,
RemoteRoutingTableBlobStore.REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING,
RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING,

// Admission Control Settings
AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE,
Expand Down
74 changes: 69 additions & 5 deletions server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment {
// also we introduce index routing-metadata, diff and other attributes as part of manifest
// required for state publication
public static final int CODEC_V3 = 3; // In Codec V3, we have introduced new diff field in diff-manifest's routing_table_diff
public static final int CODEC_V4 = 4; // In Codec V4, we are adding checksum of cluster state.

private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term");
private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version");
Expand Down Expand Up @@ -73,6 +74,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment {
);
private static final ParseField UPLOADED_CLUSTER_STATE_CUSTOM_METADATA = new ParseField("uploaded_cluster_state_custom_metadata");
private static final ParseField DIFF_MANIFEST = new ParseField("diff_manifest");
private static final ParseField CHECKSUM = new ParseField("checksum");

private static ClusterMetadataManifest.Builder manifestV0Builder(Object[] fields) {
return ClusterMetadataManifest.builder()
Expand Down Expand Up @@ -114,6 +116,10 @@ private static ClusterMetadataManifest.Builder manifestV3Builder(Object[] fields
return manifestV2Builder(fields);
}

private static ClusterMetadataManifest.Builder manifestV4Builder(Object[] fields) {
return manifestV3Builder(fields).checksum(checksum(fields));
}

private static long term(Object[] fields) {
return (long) fields[0];
}
Expand Down Expand Up @@ -216,6 +222,10 @@ private static ClusterStateDiffManifest diffManifest(Object[] fields) {
return (ClusterStateDiffManifest) fields[23];
}

private static ClusterStateChecksum checksum(Object[] fields) {
return (ClusterStateChecksum) fields[24];
}

private static final ConstructingObjectParser<ClusterMetadataManifest, Void> PARSER_V0 = new ConstructingObjectParser<>(
"cluster_metadata_manifest",
fields -> manifestV0Builder(fields).build()
Expand All @@ -236,13 +246,20 @@ private static ClusterStateDiffManifest diffManifest(Object[] fields) {
fields -> manifestV3Builder(fields).build()
);

private static final ConstructingObjectParser<ClusterMetadataManifest, Void> CURRENT_PARSER = PARSER_V3;
private static final ConstructingObjectParser<ClusterMetadataManifest, Void> PARSER_V4 = new ConstructingObjectParser<>(
"cluster_metadata_manifest",
fields -> manifestV4Builder(fields).build()
);

private static final ConstructingObjectParser<ClusterMetadataManifest, Void> CURRENT_PARSER = PARSER_V4;

static {
declareParser(PARSER_V0, CODEC_V0);
declareParser(PARSER_V1, CODEC_V1);
declareParser(PARSER_V2, CODEC_V2);
declareParser(PARSER_V3, CODEC_V3);
declareParser(PARSER_V4, CODEC_V4);

}

private static void declareParser(ConstructingObjectParser<ClusterMetadataManifest, Void> parser, long codec_version) {
Expand Down Expand Up @@ -324,6 +341,13 @@ private static void declareParser(ConstructingObjectParser<ClusterMetadataManife
DIFF_MANIFEST
);
}
if (codec_version >= CODEC_V4) {
parser.declareObject(
ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> ClusterStateChecksum.fromXContent(p),
CHECKSUM
);
}
}

private final int codecVersion;
Expand Down Expand Up @@ -351,6 +375,7 @@ private static void declareParser(ConstructingObjectParser<ClusterMetadataManife
private final UploadedMetadataAttribute uploadedHashesOfConsistentSettings;
private final Map<String, UploadedMetadataAttribute> uploadedClusterStateCustomMap;
private final ClusterStateDiffManifest diffManifest;
private ClusterStateChecksum clusterStateChecksum;

public List<UploadedIndexMetadata> getIndices() {
return indices;
Expand Down Expand Up @@ -459,6 +484,10 @@ public List<UploadedIndexMetadata> getIndicesRouting() {
return indicesRouting;
}

public ClusterStateChecksum getClusterStateChecksum() {
return clusterStateChecksum;
}

public ClusterMetadataManifest(
long clusterTerm,
long version,
Expand All @@ -484,7 +513,8 @@ public ClusterMetadataManifest(
UploadedMetadataAttribute uploadedTransientSettingsMetadata,
UploadedMetadataAttribute uploadedHashesOfConsistentSettings,
Map<String, UploadedMetadataAttribute> uploadedClusterStateCustomMap,
ClusterStateDiffManifest diffManifest
ClusterStateDiffManifest diffManifest,
ClusterStateChecksum clusterStateChecksum
) {
this.clusterTerm = clusterTerm;
this.stateVersion = version;
Expand Down Expand Up @@ -515,6 +545,7 @@ public ClusterMetadataManifest(
this.uploadedClusterStateCustomMap = Collections.unmodifiableMap(
uploadedClusterStateCustomMap != null ? uploadedClusterStateCustomMap : new HashMap<>()
);
this.clusterStateChecksum = clusterStateChecksum;
}

public ClusterMetadataManifest(StreamInput in) throws IOException {
Expand All @@ -528,6 +559,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException {
this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new));
this.previousClusterUUID = in.readString();
this.clusterUUIDCommitted = in.readBoolean();
clusterStateChecksum = null;
if (in.getVersion().onOrAfter(Version.V_2_15_0)) {
this.codecVersion = in.readInt();
this.uploadedCoordinationMetadata = new UploadedMetadataAttribute(in);
Expand Down Expand Up @@ -590,6 +622,9 @@ public ClusterMetadataManifest(StreamInput in) throws IOException {
this.uploadedHashesOfConsistentSettings = null;
this.uploadedClusterStateCustomMap = null;
}
if (in.getVersion().onOrAfter(Version.V_2_17_0) && in.readBoolean()) {
clusterStateChecksum = new ClusterStateChecksum(in);
}
}

public static Builder builder() {
Expand Down Expand Up @@ -687,6 +722,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion());
builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName());
}
if (onOrAfterCodecVersion(CODEC_V4)) {
if (getClusterStateChecksum() != null) {
builder.startObject(CHECKSUM.getPreferredName());
getClusterStateChecksum().toXContent(builder, params);
builder.endObject();
}
}
return builder;
}

Expand Down Expand Up @@ -746,6 +788,14 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeInt(codecVersion);
out.writeString(globalMetadataFileName);
}
if (out.getVersion().onOrAfter(Version.V_2_17_0)) {
if (clusterStateChecksum != null) {
out.writeBoolean(true);
clusterStateChecksum.writeTo(out);
} else {
out.writeBoolean(false);
}
}
}

@Override
Expand Down Expand Up @@ -781,7 +831,8 @@ public boolean equals(Object o) {
&& Objects.equals(uploadedTransientSettingsMetadata, that.uploadedTransientSettingsMetadata)
&& Objects.equals(uploadedHashesOfConsistentSettings, that.uploadedHashesOfConsistentSettings)
&& Objects.equals(uploadedClusterStateCustomMap, that.uploadedClusterStateCustomMap)
&& Objects.equals(diffManifest, that.diffManifest);
&& Objects.equals(diffManifest, that.diffManifest)
&& Objects.equals(clusterStateChecksum, that.clusterStateChecksum);
}

@Override
Expand Down Expand Up @@ -811,7 +862,8 @@ public int hashCode() {
uploadedTransientSettingsMetadata,
uploadedHashesOfConsistentSettings,
uploadedClusterStateCustomMap,
diffManifest
diffManifest,
clusterStateChecksum
);
}

Expand All @@ -836,6 +888,10 @@ public static ClusterMetadataManifest fromXContentV2(XContentParser parser) thro
return PARSER_V2.parse(parser, null);
}

public static ClusterMetadataManifest fromXContentV3(XContentParser parser) throws IOException {
return PARSER_V3.parse(parser, null);
}

public static ClusterMetadataManifest fromXContent(XContentParser parser) throws IOException {
return CURRENT_PARSER.parse(parser, null);
}
Expand Down Expand Up @@ -872,6 +928,7 @@ public static class Builder {
private UploadedMetadataAttribute hashesOfConsistentSettings;
private Map<String, UploadedMetadataAttribute> clusterStateCustomMetadataMap;
private ClusterStateDiffManifest diffManifest;
private ClusterStateChecksum checksum;

public Builder indices(List<UploadedIndexMetadata> indices) {
this.indices = indices;
Expand Down Expand Up @@ -1011,6 +1068,11 @@ public Builder diffManifest(ClusterStateDiffManifest diffManifest) {
return this;
}

public Builder checksum(ClusterStateChecksum checksum) {
this.checksum = checksum;
return this;
}

public Builder() {
indices = new ArrayList<>();
customMetadataMap = new HashMap<>();
Expand Down Expand Up @@ -1043,6 +1105,7 @@ public Builder(ClusterMetadataManifest manifest) {
this.diffManifest = manifest.diffManifest;
this.hashesOfConsistentSettings = manifest.uploadedHashesOfConsistentSettings;
this.clusterStateCustomMetadataMap = manifest.uploadedClusterStateCustomMap;
this.checksum = manifest.clusterStateChecksum;
}

public ClusterMetadataManifest build() {
Expand Down Expand Up @@ -1071,7 +1134,8 @@ public ClusterMetadataManifest build() {
transientSettingsMetadata,
hashesOfConsistentSettings,
clusterStateCustomMetadataMap,
diffManifest
diffManifest,
checksum
);
}

Expand Down
Loading

0 comments on commit 379fe90

Please sign in to comment.