Skip to content

Commit

Permalink
fixes, bump version
Browse files Browse the repository at this point in the history
  • Loading branch information
RuedigerMoeller committed Jul 31, 2017
1 parent b019cbf commit 4d07930
Show file tree
Hide file tree
Showing 9 changed files with 201 additions and 27 deletions.
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>de.ruedigermoeller</groupId>
<artifactId>fst</artifactId>
<version>2.51</version>
<version>2.52</version>
<packaging>bundle</packaging>

<description>A fast java serialization drop in-replacement and some serialization based utils such as Structs and OffHeap Memory.</description>
Expand Down
23 changes: 15 additions & 8 deletions src/main/java/org/nustaq/serialization/FSTConfiguration.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.fasterxml.jackson.core.ObjectCodec;
import com.fasterxml.jackson.core.SerializableString;
import com.fasterxml.jackson.core.io.IOContext;
import com.fasterxml.jackson.core.json.JsonWriteContext;
import com.fasterxml.jackson.core.json.UTF8JsonGenerator;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
import org.nustaq.offheap.bytez.onheap.HeapBytez;
Expand All @@ -36,12 +37,14 @@
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;


/**
* Created with IntelliJ IDEA.
* User: ruedi
Expand Down Expand Up @@ -257,7 +260,10 @@ public static FSTConfiguration createJsonNoRefConfiguration() {
/**
* create a json conf with given attributes. Note that shared refs = true for jason might be not as stable as for binary encodings
* as fst relies on stream positions to identify objects within a given input, so any inbetween formatting will break proper reference
* resolution
* resolution.
*
* WARNING: use of sharedrefs = true is Deprecated as its flakey
*
* @param prettyPrint
* @param shareReferences
* @return
Expand All @@ -269,6 +275,10 @@ public static FSTConfiguration createJsonConfiguration(boolean prettyPrint, bool
return createJsonConfiguration(prettyPrint,shareReferences,null);
}

public static FSTConfiguration createJsonConfiguration() {
return createJsonConfiguration(false,false);
}

protected static FSTConfiguration createJsonConfiguration(boolean prettyPrint, boolean shareReferences, ConcurrentHashMap<FieldKey,FSTClazzInfo.FSTFieldInfo> shared ) {
if ( prettyPrint && shareReferences ) {
throw new RuntimeException("cannot use prettyPrint with shared refs to 'true'. Set shareRefs to false.");
Expand Down Expand Up @@ -467,7 +477,7 @@ protected static FSTConfiguration initDefaultFstConfigurationInternal(FSTConfigu
// for most cases don't register for subclasses as in many cases we'd like to fallback to JDK implementation
// (e.g. TreeMap) in order to guarantee complete serialization
reg.putSerializer(ArrayList.class, new FSTArrayListSerializer(), false);
reg.putSerializer(Vector.class, new FSTCollectionSerializer(), false);
reg.putSerializer(Vector.class, new FSTCollectionSerializer(), true);
reg.putSerializer(LinkedList.class, new FSTCollectionSerializer(), false); // subclass should register manually
reg.putSerializer(HashSet.class, new FSTCollectionSerializer(), false); // subclass should register manually
reg.putSerializer(HashMap.class, new FSTMapSerializer(), false); // subclass should register manually
Expand Down Expand Up @@ -923,6 +933,8 @@ public FSTObjectInput getObjectInputCopyFrom( byte arr[],int off, int len ) {

protected FSTObjectInput getIn() {
FSTObjectInput fstObjectInput = (FSTObjectInput) streamCoderFactory.getInput().get();
if ( fstObjectInput != null && fstObjectInput.isClosed() )
fstObjectInput = null;
if ( fstObjectInput == null ) {
streamCoderFactory.getInput().set(new FSTObjectInput(this));
return getIn();
Expand Down Expand Up @@ -1191,13 +1203,8 @@ public String asJsonString(Object o) {
if ( getCoderSpecific() instanceof JsonFactory == false ) {
return "can be called on JsonConfiguration only";
} else {
try {
return new String(asByteArray(o),"UTF-8");
} catch (UnsupportedEncodingException e) {
FSTUtil.<RuntimeException>rethrow(e);
}
return new String(asByteArray(o), StandardCharsets.UTF_8);
}
return null;
}

/**
Expand Down
4 changes: 4 additions & 0 deletions src/main/java/org/nustaq/serialization/FSTObjectInput.java
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,10 @@ protected void setCodec(FSTDecoder codec) {
this.codec = codec;
}

public boolean isClosed() {
return closed;
}

protected static class CallbackEntry {
ObjectInputValidation cb;
int prio;
Expand Down
15 changes: 13 additions & 2 deletions src/main/java/org/nustaq/serialization/FSTObjectOutput.java
Original file line number Diff line number Diff line change
Expand Up @@ -451,10 +451,21 @@ protected FSTClazzInfo writeObjectWithContext(FSTClazzInfo.FSTFieldInfo referenc
return originalInfo;
}
}

if (! writeObjectHeader(serializationInfo, referencee, toWrite) ) { // skip in case codec can write object as primitive
defaultWriteObject(toWrite, serializationInfo);
if ( serializationInfo.isExternalizable() )
ser = serializationInfo.getSer();
if ( ser == null ) {
defaultWriteObject(toWrite, serializationInfo);
if ( serializationInfo.isExternalizable() )
getCodec().externalEnd(serializationInfo);
} else {
// handle edge case: there is a serializer registered for replaced class
// copied from below :(
int pos = getCodec().getWritten();
// write object depending on type (custom, externalizable, serializable/java, default)
ser.writeObject(this, toWrite, serializationInfo, referencee, pos);
getCodec().externalEnd(serializationInfo);
}
}
return originalInfo;
} else { // object has custom serializer
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonStreamContext;
import com.fasterxml.jackson.core.ObjectCodec;
import com.fasterxml.jackson.core.io.SerializedString;
import com.fasterxml.jackson.core.json.JsonWriteContext;
import org.nustaq.serialization.*;
import org.nustaq.serialization.util.FSTOutputStream;
import org.nustaq.serialization.util.FSTUtil;
Expand Down Expand Up @@ -168,6 +170,13 @@ public void close() throws IOException {

@Override
public void reset(byte[] outbytes) {
if ( gen != null ) {
try {
createGenerator();
} catch (Exception e) {
FSTUtil.<RuntimeException>rethrow(e);
}
}
if (outbytes==null) {
out.reset();
} else {
Expand Down
16 changes: 16 additions & 0 deletions src/test/json/StringEnc.java
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,20 @@ public void testStringenc() throws UnsupportedEncodingException {
Assert.assertTrue(x.equals("ÄÖasdß"));
}

static class PJ implements Serializable {
int a = 10;
}

@Test
public void testLeadingSpaceBug() {
FSTConfiguration fst = FSTConfiguration.createJsonConfiguration();
PJ pojo = new PJ();
String x = fst.asJsonString(pojo);
System.out.println(x);
String x1 = fst.asJsonString(pojo);
System.out.println(x1);
Assert.assertTrue(x.equals(x1));

}

}
2 changes: 1 addition & 1 deletion src/test/offheap/StringOffHeapTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ public void hardcore() throws Exception {
FSTCoder coder = createCoder();
coder.getConf().registerClass(TestRec.class);

int count = 5;
int count = 2;
// while (true)
while (count-- > 0)
{
Expand Down
15 changes: 0 additions & 15 deletions src/test/ser/RawMemTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -49,18 +49,12 @@ public void test() {
System.out.println("binary");
deser = smallBench(conf, original, deser);
deser = smallBench(conf, original, deser);
deser = smallBench(conf, original, deser);
deser = smallBench(conf, original, deser);
deser = smallBench(conf, original, deser);
assertTrue(DeepEquals.deepEquals(original, deser));

System.out.println("default");
conf = FSTConfiguration.createDefaultConfiguration();
deser = smallBench(conf, original, deser);
deser = smallBench(conf, original, deser);
deser = smallBench(conf, original, deser);
deser = smallBench(conf, original, deser);
deser = smallBench(conf, original, deser);
assertTrue(DeepEquals.deepEquals(original, deser));

System.out.println("Default LEN:"+FSTConfiguration.createDefaultConfiguration().asByteArray(original).length);
Expand Down Expand Up @@ -99,8 +93,6 @@ public void testOffHeapCoder0( boolean shared ) throws Exception {
MallocBytezAllocator alloc = new MallocBytezAllocator();
MallocBytez bytez = (MallocBytez) alloc.alloc(1000 * 1000);

ohbench(original, coder, bytez);
ohbench(original, coder, bytez);
ohbench(original, coder, bytez);
ohbench(original, coder, bytez);
Object deser = ohbench(original, coder, bytez);
Expand All @@ -109,8 +101,6 @@ public void testOffHeapCoder0( boolean shared ) throws Exception {
System.out.println("-----");
ohbench(smallClazz, coder, bytez);
ohbench(smallClazz, coder, bytez);
ohbench(smallClazz, coder, bytez);
ohbench(smallClazz, coder, bytez);
deser = ohbench(smallClazz, coder, bytez);
assertTrue(DeepEquals.deepEquals(smallClazz, deser));

Expand Down Expand Up @@ -158,8 +148,6 @@ public void testOnHeapCoder0(boolean shared) throws Exception {
Object deser = coder.toObject(arr, 0, (int) arr.length);
assertTrue(DeepEquals.deepEquals(deser,original));

onhbench(original, coder, arr, 0);
onhbench(original, coder, arr, 0);
onhbench(original, coder, arr, 0);
onhbench(original, coder, arr, 0);
deser = onhbench(original, coder, arr, 0);
Expand All @@ -168,9 +156,6 @@ public void testOnHeapCoder0(boolean shared) throws Exception {
System.out.println("-----");
deser = onhbench(smallClazz, coder, arr, 0);
deser = onhbench(smallClazz, coder, arr, 0);
deser = onhbench(smallClazz, coder, arr, 0);
deser = onhbench(smallClazz, coder, arr, 0);
deser = onhbench(smallClazz, coder, arr, 0);
assertTrue(DeepEquals.deepEquals(smallClazz, deser));

boolean lenEx = false;
Expand Down
142 changes: 142 additions & 0 deletions src/test/ser/serializers/Github204.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
package ser.serializers;

import org.junit.Test;
import org.nustaq.serialization.FSTConfiguration;
import org.nustaq.serialization.FSTObjectInput;
import org.nustaq.serialization.FSTObjectOutput;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.ObjectStreamException;
import java.io.Serializable;
import java.util.*;

import static org.junit.Assert.assertEquals;

/**
* Created by ruedi on 31.07.17.
*/
public class Github204 {

@Test
public void testMapWriteReplace() throws Exception {
Map<String, Object> map = new MyMap<>();
map.put("value", 1234.5);

Map<String, Object> map1 = new MyMap<>();
map1.put("othervalue", 5678.9);

ByteArrayOutputStream buf = new ByteArrayOutputStream();
FSTObjectOutput out = new FSTObjectOutput(buf, getTestConfiguration());
out.writeObject(map);
out.writeObject(map1);
out.writeObject("Ensure Stream is not corrupted");
out.close();

FSTObjectInput in = getTestConfiguration().getObjectInput(new ByteArrayInputStream(buf.toByteArray()));
Map<?, ?> res = (Map<?, ?>) in.readObject();
Map<?, ?> res1 = (Map<?, ?>) in.readObject();
in.close();

assertEquals(LinkedHashMap.class, res.getClass());
assertEquals(1, res.size());
assertEquals(1234.5, res.get("value"));
assertEquals(LinkedHashMap.class, res1.getClass());
assertEquals(1, res1.size());
assertEquals(5678.9, res1.get("othervalue"));
}

private FSTConfiguration getTestConfiguration() {
return FSTConfiguration.createDefaultConfiguration();
}

public static class MyMap<K, V> implements Map<K, V>, Serializable {
private final Map<K, V> data;

public MyMap() {
this(new HashMap());
}


public MyMap(Map<K, V> data) {
this.data = data;
}

@Override
public int size() {
return data.size();
}

@Override
public boolean isEmpty() {
return data.isEmpty();
}

@Override
public boolean containsKey(Object key) {
return data.containsKey(key);
}

@Override
public boolean containsValue(Object value) {
return data.containsValue(value);
}

@Override
public V get(Object key) {
return data.get(key);
}

@Override
public V put(K key, V value) {
return data.put(key, value);
}

@Override
public V remove(Object key) {
return data.remove(key);
}

@Override
public void putAll(Map<? extends K, ? extends V> m) {
data.putAll(m);
}

@Override
public void clear() {
data.clear();
}

@Override
public Set<K> keySet() {
return data.keySet();
}

@Override
public Collection<V> values() {
return data.values();
}

@Override
public Set<Entry<K, V>> entrySet() {
return data.entrySet();
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
MyMap<?, ?> myMap = (MyMap<?, ?>) o;
return Objects.equals(data, myMap.data);
}

@Override
public int hashCode() {
return Objects.hash(data);
}

private Object writeReplace() throws ObjectStreamException {
return new LinkedHashMap<K, V>(this);
}
}
}

0 comments on commit 4d07930

Please sign in to comment.