Skip to content

Commit

Permalink
fix: 11997: Backport the fix for 11996 to release 0.47 (#11999)
Browse files Browse the repository at this point in the history
Signed-off-by: Artem Ananev <[email protected]>
  • Loading branch information
artemananiev authored Mar 11, 2024
1 parent 1dbe451 commit e77c6fd
Show file tree
Hide file tree
Showing 4 changed files with 272 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -1217,7 +1217,11 @@ private void writeLeavesToPathToKeyValue(
longKeyToPath.put(key, INVALID_PATH);
}
} else {
objectKeyToPath.deleteIfEqual(leafRecord.getKey(), path);
if (isReconnect) {
objectKeyToPath.deleteIfEqual(leafRecord.getKey(), path);
} else {
objectKeyToPath.delete(leafRecord.getKey());
}
}
statisticsUpdater.countFlushLeavesDeleted();

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
/*
* Copyright (C) 2016-2024 Hedera Hashgraph, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.swirlds.virtual.merkle;

import com.hedera.pbj.runtime.io.ReadableSequentialData;
import com.hedera.pbj.runtime.io.WritableSequentialData;
import com.swirlds.common.io.streams.SerializableDataInputStream;
import com.swirlds.common.io.streams.SerializableDataOutputStream;
import com.swirlds.virtualmap.VirtualKey;
import java.io.IOException;
import java.nio.ByteBuffer;

public final class TestObjectKey implements VirtualKey {

public static final int BYTES = Long.BYTES * 2;

private long k;

public TestObjectKey() {}

public TestObjectKey(long value) {
this.k = value;
}

public TestObjectKey copy() {
return new TestObjectKey(k);
}

@Override
public int getVersion() {
return 1;
}

long getValue() {
return k;
}

@Override
public void serialize(SerializableDataOutputStream out) throws IOException {
out.writeLong(k);
out.writeLong(k);
}

void serialize(final WritableSequentialData out) {
out.writeLong(k);
out.writeLong(k);
}

void serialize(final ByteBuffer buffer) {
buffer.putLong(k);
buffer.putLong(k);
}

@Override
public void deserialize(SerializableDataInputStream in, int version) throws IOException {
k = in.readLong();
long kk = in.readLong();
assert k == kk : "Malformed TestObjectKey";
}

void deserialize(final ReadableSequentialData in) {
k = in.readLong();
long kk = in.readLong();
assert k == kk : "Malformed TestObjectKey";
}

void deserialize(final ByteBuffer buffer) {
k = buffer.getLong();
long kk = buffer.getLong();
assert k == kk : "Malformed TestObjectKey";
}

@Override
public int hashCode() {
return Long.hashCode(k);
}

@Override
public String toString() {
if (Character.isAlphabetic((char) k)) {
return "TestObjectKey{ " + ((char) k) + " }";
} else {
return "TestObjectKey{ " + k + " }";
}
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
TestObjectKey other = (TestObjectKey) o;
return k == other.k;
}

@Override
public long getClassId() {
return 0x255bb9565ebfad4bL;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/*
* Copyright (C) 2016-2024 Hedera Hashgraph, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.swirlds.virtual.merkle;

import com.hedera.pbj.runtime.io.ReadableSequentialData;
import com.hedera.pbj.runtime.io.WritableSequentialData;
import com.hedera.pbj.runtime.io.buffer.BufferedData;
import com.swirlds.merkledb.serialize.KeySerializer;
import java.nio.ByteBuffer;

public class TestObjectKeySerializer implements KeySerializer<TestObjectKey> {

public TestObjectKeySerializer() {
// required for deserialization
}

@Override
public long getClassId() {
return 8838922;
}

@Override
public int getVersion() {
return 1;
}

@Override
public int getSerializedSize() {
return TestObjectKey.BYTES;
}

@Override
public long getCurrentDataVersion() {
return 1;
}

@Override
public void serialize(final TestObjectKey data, final WritableSequentialData out) {
data.serialize(out);
}

@Override
public void serialize(TestObjectKey data, ByteBuffer buffer) {
data.serialize(buffer);
}

@Override
public TestObjectKey deserialize(final ReadableSequentialData in) {
final TestObjectKey key = new TestObjectKey();
key.deserialize(in);
return key;
}

@Override
public TestObjectKey deserialize(final ByteBuffer buffer, final long dataVersion) {
final TestObjectKey key = new TestObjectKey();
key.deserialize(buffer);
return key;
}

@Override
public boolean equals(final BufferedData buffer, final TestObjectKey keyToCompare) {
return (buffer.readLong() == keyToCompare.getValue()) && (buffer.readLong() == keyToCompare.getValue());
}

@Override
public boolean equals(final ByteBuffer buffer, final int dataVersion, final TestObjectKey keyToCompare) {
return (buffer.getLong() == keyToCompare.getValue()) && (buffer.getLong() == keyToCompare.getValue());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

import static com.swirlds.common.test.fixtures.junit.tags.TestQualifierTags.TIME_CONSUMING;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;

Expand All @@ -26,27 +28,44 @@
import com.swirlds.merkledb.MerkleDbTableConfig;
import com.swirlds.virtual.merkle.TestKey;
import com.swirlds.virtual.merkle.TestKeySerializer;
import com.swirlds.virtual.merkle.TestObjectKey;
import com.swirlds.virtual.merkle.TestObjectKeySerializer;
import com.swirlds.virtual.merkle.TestValue;
import com.swirlds.virtual.merkle.TestValueSerializer;
import com.swirlds.virtualmap.VirtualMap;
import com.swirlds.virtualmap.datasource.VirtualDataSourceBuilder;
import com.swirlds.virtualmap.datasource.VirtualLeafRecord;
import com.swirlds.virtualmap.internal.RecordAccessor;
import com.swirlds.virtualmap.internal.merkle.VirtualRootNode;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Tags;
import org.junit.jupiter.api.Test;

final class MapTest {

VirtualDataSourceBuilder<TestKey, TestValue> createBuilder() {
VirtualDataSourceBuilder<TestKey, TestValue> createLongBuilder() {
final MerkleDbTableConfig<TestKey, TestValue> tableConfig = new MerkleDbTableConfig<>(
(short) 1, DigestType.SHA_384,
(short) 1, new TestKeySerializer(),
(short) 1, new TestValueSerializer());
return new MerkleDbDataSourceBuilder<>(tableConfig);
}

VirtualMap<TestKey, TestValue> createMap(String label) {
return new VirtualMap<>(label, createBuilder());
VirtualDataSourceBuilder<TestObjectKey, TestValue> createGenericBuilder() {
final MerkleDbTableConfig<TestObjectKey, TestValue> tableConfig = new MerkleDbTableConfig<>(
(short) 1, DigestType.SHA_384,
(short) 1, new TestObjectKeySerializer(),
(short) 1, new TestValueSerializer());
return new MerkleDbDataSourceBuilder<>(tableConfig);
}

VirtualMap<TestKey, TestValue> createLongMap(String label) {
return new VirtualMap<>(label, createLongBuilder());
}

VirtualMap<TestObjectKey, TestValue> createObjectMap(String label) {
return new VirtualMap<>(label, createGenericBuilder());
}

@Test
Expand All @@ -56,7 +75,7 @@ VirtualMap<TestKey, TestValue> createMap(String label) {
void insertRemoveAndModifyOneMillion() throws InterruptedException {
final int changesPerBatch = 15_432; // Some unexpected size just to be crazy
final int max = 1_000_000;
VirtualMap<TestKey, TestValue> map = createMap("insertRemoveAndModifyOneMillion");
VirtualMap<TestKey, TestValue> map = createLongMap("insertRemoveAndModifyOneMillion");
try {
for (int i = 0; i < max; i++) {
if (i > 0 && i % changesPerBatch == 0) {
Expand Down Expand Up @@ -102,4 +121,51 @@ void insertRemoveAndModifyOneMillion() throws InterruptedException {
map.release();
}
}

@Test
@Tags({@Tag("VirtualMerkle")})
@DisplayName("Delete a value that was moved to a different virtual path")
void deletedObjectLeavesOnFlush() throws InterruptedException {
VirtualMap<TestObjectKey, TestValue> map = createObjectMap("deletedObjectLeavesOnFlush");
for (int i = 0; i < 8; i++) {
map.put(new TestObjectKey(i), new TestValue(i));
}

VirtualRootNode<TestObjectKey, TestValue> rootNode = map.getRight();
rootNode.enableFlush();

RecordAccessor<TestObjectKey, TestValue> records = rootNode.getRecords();
// Check that key/value 0 is at path 7
VirtualLeafRecord<TestObjectKey, TestValue> leaf = records.findLeafRecord(7, false);
assertNotNull(leaf);
assertEquals(new TestObjectKey(0), leaf.getKey());
assertEquals(new TestValue(0), leaf.getValue());

VirtualMap<TestObjectKey, TestValue> copy = map.copy();
map.release();
map = copy;
rootNode.waitUntilFlushed();

// Move key/value to a different path, then delete
map.remove(new TestObjectKey(0));
map.remove(new TestObjectKey(2));
map.put(new TestObjectKey(8), new TestValue(8));
map.put(new TestObjectKey(0), new TestValue(0));
map.remove(new TestObjectKey(0));

rootNode = map.getRight();
rootNode.enableFlush();

copy = map.copy();
map.release();
map = copy;
rootNode.waitUntilFlushed();

// During this second flush, key/value 0 must be deleted from the map despite its
// path in the virtual tree doesn't match the path in the data source
assertFalse(map.containsKey(new TestObjectKey(0)));
assertNull(map.get(new TestObjectKey(0)));

map.release();
}
}

0 comments on commit e77c6fd

Please sign in to comment.