From c0aef67589a098c01fbae8d1efd8a9f8d7c19c6e Mon Sep 17 00:00:00 2001 From: Shawn Yang Date: Fri, 19 Apr 2024 12:37:38 +0800 Subject: [PATCH] delete guide files (#110) This PR delete guide files since we havev renamed those files in https://github.com/apache/incubator-fury/pull/1538 --- docs/guide/java_object_graph_guide.md | 416 ----------------- docs/guide/xlang_object_graph_guide.md | 596 ------------------------- 2 files changed, 1012 deletions(-) delete mode 100644 docs/guide/java_object_graph_guide.md delete mode 100644 docs/guide/xlang_object_graph_guide.md diff --git a/docs/guide/java_object_graph_guide.md b/docs/guide/java_object_graph_guide.md deleted file mode 100644 index 997b7a121..000000000 --- a/docs/guide/java_object_graph_guide.md +++ /dev/null @@ -1,416 +0,0 @@ ---- -title: Java Object Graph Guide -sidebar_position: 0 -id: java_object_graph_guide ---- - -# Java object graph serialization - -When only java object serialization needed, this mode will have better performance compared to cross-language object -graph serialization. - -## Quick Start - -Note that fury creation is not cheap, the **fury instances should be reused between serializations** instead of creating -it everytime. -You should keep fury to a static global variable, or instance variable of some singleton object or limited objects. - -Fury for single-thread usage: - -```java -import java.util.List; -import java.util.Arrays; - -import org.apache.fury.*; -import org.apache.fury.config.*; - -public class Example { - public static void main(String[] args) { - SomeClass object = new SomeClass(); - // Note that Fury instances should be reused between - // multiple serializations of different objects. - Fury fury = Fury.builder().withLanguage(Language.JAVA) - // Allow to deserialize objects unknown types, more flexible - // but may be insecure if the classes contains malicious code. - .requireClassRegistration(true) - .build(); - // Registering types can reduce class name serialization overhead, but not mandatory. - // If class registration enabled, all custom types must be registered. - fury.register(SomeClass.class); - byte[] bytes = fury.serialize(object); - System.out.println(fury.deserialize(bytes)); - } -} -``` - -Fury for multiple-thread usage: - -```java -import java.util.List; -import java.util.Arrays; - -import org.apache.fury.*; -import org.apache.fury.config.*; - -public class Example { - public static void main(String[] args) { - SomeClass object = new SomeClass(); - // Note that Fury instances should be reused between - // multiple serializations of different objects. - ThreadSafeFury fury = new ThreadLocalFury(classLoader -> { - Fury f = Fury.builder().withLanguage(Language.JAVA) - .withClassLoader(classLoader).build(); - f.register(SomeClass.class); - return f; - }); - byte[] bytes = fury.serialize(object); - System.out.println(fury.deserialize(bytes)); - } -} -``` - -Fury instances reuse example: - -```java -import java.util.List; -import java.util.Arrays; - -import org.apache.fury.*; -import org.apache.fury.config.*; - -public class Example { - // reuse fury. - private static final ThreadSafeFury fury = Fury.builder() - // Allow to deserialize objects unknown types, more flexible - // but may be insecure if the classes contains malicious code. - .requireClassRegistration(true) - .buildThreadSafeFury(); - - public static void main(String[] args) { - SomeClass object = new SomeClass(); - byte[] bytes = fury.serialize(object); - System.out.println(fury.deserialize(bytes)); - } -} -``` - -## FuryBuilder options - -| Option Name | Description | Default Value | -|-------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------| -| `timeRefIgnored` | Whether to ignore reference tracking of all time types registered in `TimeSerializers` and subclasses of those types when ref tracking is enabled. If ignored, ref tracking of every time type can be enabled by invoking `Fury#registerSerializer(Class, Serializer)`. For example, `fury.registerSerializer(Date.class, new DateSerializer(fury, true))`. Note that enabling ref tracking should happen before serializer codegen of any types which contain time fields. Otherwise, those fields will still skip ref tracking. | `true` | -| `compressInt` | Enables or disables int compression for smaller size. | `true` | -| `compressLong` | Enables or disables long compression for smaller size. | `true` | -| `compressString` | Enables or disables string compression for smaller size. | `true` | -| `classLoader` | The classloader should not be updated; Fury caches class metadata. Use `LoaderBinding` or `ThreadSafeFury` for classloader updates. | `Thread.currentThread().getContextClassLoader()` | -| `compatibleMode` | Type forward/backward compatibility config. Also Related to `checkClassVersion` config. `SCHEMA_CONSISTENT`: Class schema must be consistent between serialization peer and deserialization peer. `COMPATIBLE`: Class schema can be different between serialization peer and deserialization peer. They can add/delete fields independently. | `CompatibleMode.SCHEMA_CONSISTENT` | -| `checkClassVersion` | Determines whether to check the consistency of the class schema. If enabled, Fury checks, writes, and checks consistency using the `classVersionHash`. It will be automatically disabled when `CompatibleMode#COMPATIBLE` is enabled. Disabling is not recommended unless you can ensure the class won't evolve. | `false` | -| `checkJdkClassSerializable` | Enables or disables checking of `Serializable` interface for classes under `java.*`. If a class under `java.*` is not `Serializable`, Fury will throw an `UnsupportedOperationException`. | `true` | -| `registerGuavaTypes` | Whether to pre-register Guava types such as `RegularImmutableMap`/`RegularImmutableList`. These types are not public API, but seem pretty stable. | `true` | -| `requireClassRegistration` | Disabling may allow unknown classes to be deserialized, potentially causing security risks. | `true` | -| `suppressClassRegistrationWarnings` | Whether to suppress class registration warnings. The warnings can be used for security audit, but may be annoying, this suppression will be enabled by default. | `true` | -| `shareMetaContext` | Enables or disables meta share mode. | `false` | -| `deserializeUnexistedClass` | Enables or disables deserialization/skipping of data for non-existent classes. | `false` | -| `codeGenEnabled` | Disabling may result in faster initial serialization but slower subsequent serializations. | `true` | -| `asyncCompilationEnabled` | If enabled, serialization uses interpreter mode first and switches to JIT serialization after async serializer JIT for a class is finished. | `false` | -| `scalaOptimizationEnabled` | Enables or disables Scala-specific serialization optimization. | `false` | - -## Advanced Usage - -### Fury creation - -Single thread fury: - -```java -Fury fury = Fury.builder() - .withLanguage(Language.JAVA) - // enable reference tracking for shared/circular reference. - // Disable it will have better performance if no duplicate reference. - .withRefTracking(false) - .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) - // enable type forward/backward compatibility - // disable it for small size and better performance. - // .withCompatibleMode(CompatibleMode.COMPATIBLE) - // enable async multi-threaded compilation. - .withAsyncCompilation(true) - .build(); - byte[] bytes = fury.serialize(object); - System.out.println(fury.deserialize(bytes)); -``` - -Thread-safe fury: - -```java -ThreadSafeFury fury = Fury.builder() - .withLanguage(Language.JAVA) - // enable reference tracking for shared/circular reference. - // Disable it will have better performance if no duplicate reference. - .withRefTracking(false) - // compress int for smaller size - // .withIntCompressed(true) - // compress long for smaller size - // .withLongCompressed(true) - .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) - // enable type forward/backward compatibility - // disable it for small size and better performance. - // .withCompatibleMode(CompatibleMode.COMPATIBLE) - // enable async multi-threaded compilation. - .withAsyncCompilation(true) - .buildThreadSafeFury(); - byte[] bytes = fury.serialize(object); - System.out.println(fury.deserialize(bytes)); -``` - -### Smaller size -`FuryBuilder#withIntCompressed`/`FuryBuilder#withLongCompressed` can be used to compress int/long for smaller size. -Normally compress int is enough. - -Both compression are enabled by default, if the serialized is not important, for example, you use flatbuffers for -serialization before, which doesn't compress anything, then you should disable compression. If your data are all numbers, -the compression may bring 80% performance regression. - -For int compression, fury use 1~5 bytes for encoding. First bit in every byte indicate whether has next byte. if first bit is set, then next byte will be read util first bit of next byte is unset. - -For long compression, fury support two encoding: -- Fury SLI(Small long as int) Encoding (**used by default**): - - If long is in [-1073741824, 1073741823], encode as 4 bytes int: `| little-endian: ((int) value) << 1 |` - - Otherwise write as 9 bytes: `| 0b1 | little-endian 8bytes long |` -- Fury PVL(Progressive Variable-length Long) Encoding: - - First bit in every byte indicate whether has next byte. if first bit is set, then next byte will be read util first bit of next byte is unset. - - Negative number will be converted to positive number by ` (v << 1) ^ (v >> 63)` to reduce cost of small negative numbers. - -If a number are `long` type, it can't be represented by smaller bytes mostly, the compression won't get good enough result, -not worthy compared to performance cost. Maybe you should try to disable long compression if you find it didn't bring much -space savings. - -### Implement a customized serializer -In some cases, you may want to implement a serializer for your type, especially some class customize serialization by JDK -writeObject/writeReplace/readObject/readResolve, which is very inefficient. For example, you don't want following `Foo#writeObject` -got invoked, you can take following `FooSerializer` as an example: -```java -class Foo { - public long f1; - private void writeObject(ObjectOutputStream s) throws IOException { - System.out.println(f1); - s.defaultWriteObject(); - } -} - -class FooSerializer extends Serializer { - public FooSerializer(Fury fury) { - super(fury, Foo.class); - } - - @Override - public void write(MemoryBuffer buffer, Foo value) { - buffer.writeInt64(value.f1); - } - - @Override - public Foo read(MemoryBuffer buffer) { - Foo foo = new Foo(); - foo.f1 = buffer.readInt64(); - return foo; - } -} -``` - -Register serializer: -```java -Fury fury = getFury(); -fury.registerSerializer(Foo.class, new FooSerializer(fury)); -``` - -### Security & Class Registration - -`FuryBuilder#requireClassRegistration` can be used to disable class registration, this will allow to deserialize objects -unknown types, -more flexible but **may be insecure if the classes contains malicious code**. - -**Do not disable class registration unless you can ensure your environment is secure**. -Malicious code in `init/equals/hashCode` can be executed when deserializing unknown/untrusted types when this option -disabled. - -Class registration can not only reduce security risks, but also avoid classname serialization cost. - -You can register class with API `Fury#register`. - -Note that class registration order is important, serialization and deserialization peer -should have same registration order. - -```java -Fury fury = xxx; -fury.register(SomeClass.class); -fury.register(SomeClass1.class, 200); -``` - -If you invoke `FuryBuilder#requireClassRegistration(false)` to disable class registration check, -you can set `org.apache.fury.resolver.ClassChecker` by `ClassResolver#setClassChecker` to control which classes are allowed -for serialization. For example, you can allow classes started with `org.example.*` by: -```java -Fury fury = xxx; -fury.getClassResolver().setClassChecker((classResolver, className) -> className.startsWith("org.example.")); -``` -```java -AllowListChecker checker = new AllowListChecker(AllowListChecker.CheckLevel.STRICT); -ThreadSafeFury fury = new ThreadLocalFury(classLoader -> { - Fury f = Fury.builder().requireClassRegistration(true).withClassLoader(classLoader).build(); - f.getClassResolver().setClassChecker(checker); - checker.addListener(f.getClassResolver()); - return f; -}); -checker.allowClass("org.example.*"); -``` - -Fury also provided a `org.apache.fury.resolver.AllowListChecker` which is allowed/disallowed list based checker to simplify -the customization of class check mechanism. You can use this checker or implement more sophisticated checker by yourself. - -### Serializer Registration - -You can also register a custom serializer for a class by `Fury#registerSerializer` API. - -Or implement `java.io.Externalizable` for a class. - -### Zero-Copy Serialization - -```java -import org.apache.fury.*; -import org.apache.fury.config.*; -import org.apache.fury.serializers.BufferObject; -import org.apache.fury.memory.MemoryBuffer; - -import java.util.*; -import java.util.stream.Collectors; - -public class ZeroCopyExample { - // Note that fury instance should be reused instead of creation every time. - static Fury fury = Fury.builder() - .withLanguage(Language.JAVA) - .build(); - - // mvn exec:java -Dexec.mainClass="io.ray.fury.examples.ZeroCopyExample" - public static void main(String[] args) { - List list = Arrays.asList("str", new byte[1000], new int[100], new double[100]); - Collection bufferObjects = new ArrayList<>(); - byte[] bytes = fury.serialize(list, e -> !bufferObjects.add(e)); - List buffers = bufferObjects.stream() - .map(BufferObject::toBuffer).collect(Collectors.toList()); - System.out.println(fury.deserialize(bytes, buffers)); - } -} -``` - -### Meta Sharing - -Fury supports share type metadata (class name, field name, final field type information, etc.) between multiple -serializations in a context (ex. TCP connection), and this information will be sent to the peer during the first -serialization in the context. Based on this metadata, the peer can rebuild the same deserializer, which avoids -transmitting metadata for subsequent serializations and reduces network traffic pressure and supports type -forward/backward compatibility automatically. - -```java -// Fury.builder() -// .withLanguage(Language.JAVA) -// .withRefTracking(false) -// // share meta across serialization. -// .withMetaContextShare(true) -// Not thread-safe fury. -MetaContext context = xxx; -fury.getSerializationContext().setMetaContext(context); -byte[] bytes = fury.serialize(o); -// Not thread-safe fury. -MetaContext context = xxx; -fury.getSerializationContext().setMetaContext(context); -fury.deserialize(bytes) - -// Thread-safe fury -fury.setClassLoader(beanA.getClass().getClassLoader()); -byte[] serialized = fury.execute( - f -> { - f.getSerializationContext().setMetaContext(context); - return f.serialize(beanA); - } -); -// thread-safe fury -fury.setClassLoader(beanA.getClass().getClassLoader()); -Object newObj = fury.execute( - f -> { - f.getSerializationContext().setMetaContext(context); - return f.deserialize(serialized); - } -); -``` - -### Deserialize un-exited classes. - -Fury support deserializing unexisted classes, this feature can be enabled -by `FuryBuilder#deserializeUnexistedClass(true)`. When enabled, and metadata sharing enabled, Fury will store -the deserialized data of this type in a lazy subclass of Map. By using the lazy map implemented by Fury, the rebalance -cost of filling map during deserialization can be avoided, which further improves performance. If this data is sent to -another process and the class exists in this process, the data will be deserialized into the object of this type without -losing any information. - -If metadata sharing is not enabled, the new class data will be skipped and an `UnexistedSkipClass` stub object will be -returned. - -## Migration - -### JDK migration - -If you use jdk serialization before, and you can't upgrade your client and server at the same time, which is common for -online application. Fury provided an util method `org.apache.fury.serializer.JavaSerializer.serializedByJDK` to check whether -the binary are generated by jdk serialization, you use following pattern to make exiting serialization protocol-aware, -then upgrade serialization to fury in an async rolling-up way: - -```java -if (JavaSerializer.serializedByJDK(bytes)) { - ObjectInputStream objectInputStream = xxx; - return objectInputStream.readObject(); -} else { - return fury.deserialize(bytes); -} -``` - -### Upgrade fury -Currently binary compatibility is ensured for minor versions only. For example, if you are using fury`v0.2.0`, binary compatibility will -be provided if you upgrade to fury `v0.2.1`. But if upgrade to fury `v0.4.1`, no binary compatibility are ensured. -Most of the time there is no need to upgrade fury to newer major version, the current version is fast and compact enough, -and we provide some minor fix for recent older versions. - -But if you do want to upgrade fury for better performance and smaller size, you need to write fury version as header to serialized data -using code like following to keep binary compatibility: -```java -MemoryBuffer buffer = xxx; -buffer.writeVarInt32(2); -fury.serialize(buffer, obj); -``` -Then for deserialization, you need: -```java -MemoryBuffer buffer = xxx; -int furyVersion = buffer.readVarInt32() -Fury fury = getFury(furyVersion); -fury.deserialize(buffer); -``` -`getFury` is a method to load corresponding fury, you can shade and relocate different version of fury to different -package, and load fury by version. - -If you upgrade fury by minor version, or you won't have data serialized by older fury, you can upgrade fury directly, -no need to `versioning` the data. - -## Trouble shooting -### Class inconsistency and class version check -If you create fury without setting `CompatibleMode` to `org.apache.fury.config.CompatibleMode.COMPATIBLE`, and you got a strange -serialization error, it may be caused by class inconsistency between serialization peer and deserialization peer. - -In such cases, you can invoke `FuryBuilder#withClassVersionCheck` to create fury to validate it, if deserialization throws `org.apache.fury.exception.ClassNotCompatibleException`, it shows class are inconsistent, and you should create fury with -`FuryBuilder#withCompaibleMode(CompatibleMode.COMPATIBLE)`. - -`CompatibleMode.COMPATIBLE` has more performance and space cost, do not set it by default if your classes are always consistent between serialization and deserialization. - -### Use wrong API for deserialization -If you serialize an object by invoking `Fury#serialize`, you should invoke `Fury#deserialize` for deserialization instead of -`Fury#deserializeJavaObject`. - -If you serialize an object by invoking `Fury#serializeJavaObject`, you should invoke `Fury#deserializeJavaObject` for deserialization instead of `Fury#deserializeJavaObjectAndClass`/`Fury#deserialize`. - -If you serialize an object by invoking `Fury#serializeJavaObjectAndClass`, you should invoke `Fury#deserializeJavaObjectAndClass` for deserialization instead of `Fury#deserializeJavaObject`/`Fury#deserialize`. diff --git a/docs/guide/xlang_object_graph_guide.md b/docs/guide/xlang_object_graph_guide.md deleted file mode 100644 index 62ae18809..000000000 --- a/docs/guide/xlang_object_graph_guide.md +++ /dev/null @@ -1,596 +0,0 @@ ---- -title: Xlang Object Graph Guide -sidebar_position: 2 -id: xlang_object_graph_guide ---- - -## Cross-language object graph serialization - -### Serialize built-in types -Common types can be serialized automatically: primitive numeric types, string, binary, array, list, map and so on. - - -**Java** - -```java -import org.apache.fury.*; -import org.apache.fury.config.*; - -import java.util.*; - -public class Example1 { - public static void main(String[] args) { - Fury fury = Fury.builder().withLanguage(Language.XLANG).build(); - List list = ofArrayList(true, false, "str", -1.1, 1, new int[100], new double[20]); - byte[] bytes = fury.serialize(list); - // bytes can be data serialized by other languages. - fury.deserialize(bytes); - Map map = new HashMap<>(); - map.put("k1", "v1"); - map.put("k2", list); - map.put("k3", -1); - bytes = fury.serialize(map); - // bytes can be data serialized by other languages. - fury.deserialize(bytes); - } -} -``` - -**Python** - -```python -import pyfury -import numpy as np - -fury = pyfury.Fury() -object_list = [True, False, "str", -1.1, 1, - np.full(100, 0, dtype=np.int32), np.full(20, 0.0, dtype=np.double)] -data = fury.serialize(object_list) -# bytes can be data serialized by other languages. -new_list = fury.deserialize(data) -object_map = {"k1": "v1", "k2": object_list, "k3": -1} -data = fury.serialize(object_map) -# bytes can be data serialized by other languages. -new_map = fury.deserialize(data) -print(new_map) -``` - -**Golang** - -```go -package main - -import furygo "github.com/apache/incubator-fury/fury/go/fury" -import "fmt" - -func main() { - list := []interface{}{true, false, "str", -1.1, 1, make([]int32, 10), make([]float64, 20)} - fury := furygo.NewFury() - bytes, err := fury.Marshal(list) - if err != nil { - panic(err) - } - var newValue interface{} - // bytes can be data serialized by other languages. - if err := fury.Unmarshal(bytes, &newValue); err != nil { - panic(err) - } - fmt.Println(newValue) - dict := map[string]interface{}{ - "k1": "v1", - "k2": list, - "k3": -1, - } - bytes, err = fury.Marshal(dict) - if err != nil { - panic(err) - } - // bytes can be data serialized by other languages. - if err := fury.Unmarshal(bytes, &newValue); err != nil { - panic(err) - } - fmt.Println(newValue) -} -``` - -**JavaScript** - -```javascript -import Fury from '@furyjs/fury'; - -/** - * @furyjs/hps use v8's fast-calls-api that can be called directly by jit, ensure that the version of Node is 20 or above. - * Experimental feature, installation success cannot be guaranteed at this moment - * If you are unable to install the module, replace it with `const hps = null;` - **/ -import hps from '@furyjs/hps'; - -const fury = new Fury({ hps }); -const input = fury.serialize('hello fury'); -const result = fury.deserialize(input); -console.log(result); -``` - -**Rust** - -```rust -use chrono::{NaiveDate, NaiveDateTime}; -use fury::{from_buffer, to_buffer, Fury}; -use std::collections::HashMap; - -fn run() { - let bin: Vec = to_buffer(&"hello".to_string()); - let obj: String = from_buffer(&bin).expect("should success"); - assert_eq!("hello".to_string(), obj); -} -``` - -### Serialize custom types -Serializing user-defined types needs registering the custom type using the register API to establish the mapping relationship between the type in different languages. - -**Java** - -```java -import org.apache.fury.*; -import org.apache.fury.config.*; -import java.util.*; - -public class Example2 { - public static class SomeClass1 { - Object f1; - Map f2; - } - - public static class SomeClass2 { - Object f1; - String f2; - List f3; - Map f4; - Byte f5; - Short f6; - Integer f7; - Long f8; - Float f9; - Double f10; - short[] f11; - List f12; - } - - public static Object createObject() { - SomeClass1 obj1 = new SomeClass1(); - obj1.f1 = true; - obj1.f2 = ofHashMap((byte) -1, 2); - SomeClass2 obj = new SomeClass2(); - obj.f1 = obj1; - obj.f2 = "abc"; - obj.f3 = ofArrayList("abc", "abc"); - obj.f4 = ofHashMap((byte) 1, 2); - obj.f5 = Byte.MAX_VALUE; - obj.f6 = Short.MAX_VALUE; - obj.f7 = Integer.MAX_VALUE; - obj.f8 = Long.MAX_VALUE; - obj.f9 = 1.0f / 2; - obj.f10 = 1 / 3.0; - obj.f11 = new short[]{(short) 1, (short) 2}; - obj.f12 = ofArrayList((short) -1, (short) 4); - return obj; - } - - // mvn exec:java -Dexec.mainClass="org.apache.fury.examples.Example2" - public static void main(String[] args) { - Fury fury = Fury.builder().withLanguage(Language.XLANG).build(); - fury.register(SomeClass1.class, "example.SomeClass1"); - fury.register(SomeClass2.class, "example.SomeClass2"); - byte[] bytes = fury.serialize(createObject()); - // bytes can be data serialized by other languages. - System.out.println(fury.deserialize(bytes)); - } -} -``` - -**Python** - -```python -from dataclasses import dataclass -from typing import List, Dict, Any -import pyfury, array - - -@dataclass -class SomeClass1: - f1: Any - f2: Dict[pyfury.Int8Type, pyfury.Int32Type] - - -@dataclass -class SomeClass2: - f1: Any = None - f2: str = None - f3: List[str] = None - f4: Dict[pyfury.Int8Type, pyfury.Int32Type] = None - f5: pyfury.Int8Type = None - f6: pyfury.Int16Type = None - f7: pyfury.Int32Type = None - # int type will be taken as `pyfury.Int64Type`. - # use `pyfury.Int32Type` for type hint if peer - # are more narrow type. - f8: int = None - f9: pyfury.Float32Type = None - # float type will be taken as `pyfury.Float64Type` - f10: float = None - f11: pyfury.Int16ArrayType = None - f12: List[pyfury.Int16Type] = None - - -if __name__ == "__main__": - f = pyfury.Fury() - f.register_class(SomeClass1, type_tag="example.SomeClass1") - f.register_class(SomeClass2, type_tag="example.SomeClass2") - obj1 = SomeClass1(f1=True, f2={-1: 2}) - obj = SomeClass2( - f1=obj1, - f2="abc", - f3=["abc", "abc"], - f4={1: 2}, - f5=2 ** 7 - 1, - f6=2 ** 15 - 1, - f7=2 ** 31 - 1, - f8=2 ** 63 - 1, - f9=1.0 / 2, - f10=1 / 3.0, - f11=array.array("h", [1, 2]), - f12=[-1, 4], - ) - data = f.serialize(obj) - # bytes can be data serialized by other languages. - print(f.deserialize(data)) -``` - -**Golang** - -```go -package main - -import furygo "github.com/apache/incubator-fury/fury/go/fury" -import "fmt" - -func main() { - type SomeClass1 struct { - F1 interface{} - F2 string - F3 []interface{} - F4 map[int8]int32 - F5 int8 - F6 int16 - F7 int32 - F8 int64 - F9 float32 - F10 float64 - F11 []int16 - F12 fury.Int16Slice - } - - type SomeClas2 struct { - F1 interface{} - F2 map[int8]int32 - } - fury := furygo.NewFury() - if err := fury.RegisterTagType("example.SomeClass1", SomeClass1{}); err != nil { - panic(err) - } - if err := fury.RegisterTagType("example.SomeClass2", SomeClass2{}); err != nil { - panic(err) - } - obj1 := &SomeClass1{} - obj1.F1 = true - obj1.F2 = map[int8]int32{-1: 2} - obj := &SomeClass1{} - obj.F1 = obj1 - obj.F2 = "abc" - obj.F3 = []interface{}{"abc", "abc"} - f4 := map[int8]int32{1: 2} - obj.F4 = f4 - obj.F5 = fury.MaxInt8 - obj.F6 = fury.MaxInt16 - obj.F7 = fury.MaxInt32 - obj.F8 = fury.MaxInt64 - obj.F9 = 1.0 / 2 - obj.F10 = 1 / 3.0 - obj.F11 = []int16{1, 2} - obj.F12 = []int16{-1, 4} - bytes, err := fury.Marshal(obj); - if err != nil { - panic(err) - } - var newValue interface{} - // bytes can be data serialized by other languages. - if err := fury.Unmarshal(bytes, &newValue); err != nil { - panic(err) - } - fmt.Println(newValue) -} -``` - -**JavaScript** - -```javascript -import Fury, { Type, InternalSerializerType } from '@furyjs/fury'; - -/** - * @furyjs/hps use v8's fast-calls-api that can be called directly by jit, ensure that the version of Node is 20 or above. - * Experimental feature, installation success cannot be guaranteed at this moment - * If you are unable to install the module, replace it with `const hps = null;` - **/ -import hps from '@furyjs/hps'; - -// Now we describe data structures using JSON, but in the future, we will use more ways. -const description = Type.object('example.foo', { - foo: Type.string(), -}); -const fury = new Fury({ hps }); -const { serialize, deserialize } = fury.registerSerializer(description); -const input = serialize({ foo: 'hello fury' }); -const result = deserialize(input); -console.log(result); -``` - -**Rust** - -```rust -use chrono::{NaiveDate, NaiveDateTime}; -use fury::{from_buffer, to_buffer, Fury}; -use std::collections::HashMap; - -#[test] -fn complex_struct() { - #[derive(Fury, Debug, PartialEq)] - #[tag("example.foo2")] - struct Animal { - category: String, - } - - #[derive(Fury, Debug, PartialEq)] - #[tag("example.foo")] - struct Person { - c1: Vec, // binary - c2: Vec, // primitive array - animal: Vec, - c3: Vec>, - name: String, - c4: HashMap, - age: u16, - op: Option, - op2: Option, - date: NaiveDate, - time: NaiveDateTime, - c5: f32, - c6: f64, - } - let person: Person = Person { - c1: vec![1, 2, 3], - c2: vec![5, 6, 7], - c3: vec![vec![1, 2], vec![1, 3]], - animal: vec![Animal { - category: "Dog".to_string(), - }], - c4: HashMap::from([ - ("hello1".to_string(), "hello2".to_string()), - ("hello2".to_string(), "hello3".to_string()), - ]), - age: 12, - name: "helo".to_string(), - op: Some("option".to_string()), - op2: None, - date: NaiveDate::from_ymd_opt(2025, 12, 12).unwrap(), - time: NaiveDateTime::from_timestamp_opt(1689912359, 0).unwrap(), - c5: 2.0, - c6: 4.0, - }; - - let bin: Vec = to_buffer(&person); - let obj: Person = from_buffer(&bin).expect("should success"); - assert_eq!(person, obj); -} -``` - -### Serialize Shared Reference and Circular Reference -Shared reference and circular reference can be serialized automatically, no duplicate data or recursion error. - -**Java** - -```java -import org.apache.fury.*; -import org.apache.fury.config.*; -import java.util.*; - -public class ReferenceExample { - public static class SomeClass { - SomeClass f1; - Map f2; - Map f3; - } - - public static Object createObject() { - SomeClass obj = new SomeClass(); - obj.f1 = obj; - obj.f2 = ofHashMap("k1", "v1", "k2", "v2"); - obj.f3 = obj.f2; - return obj; - } - - // mvn exec:java -Dexec.mainClass="org.apache.fury.examples.ReferenceExample" - public static void main(String[] args) { - Fury fury = Fury.builder().withLanguage(Language.XLANG) - .withRefTracking(true).build(); - fury.register(SomeClass.class, "example.SomeClass"); - byte[] bytes = fury.serialize(createObject()); - // bytes can be data serialized by other languages. - System.out.println(fury.deserialize(bytes)); - } -} -``` - -**Python** - -```python -from typing import Dict -import pyfury - -class SomeClass: - f1: "SomeClass" - f2: Dict[str, str] - f3: Dict[str, str] - -fury = pyfury.Fury(ref_tracking=True) -fury.register_class(SomeClass, type_tag="example.SomeClass") -obj = SomeClass() -obj.f2 = {"k1": "v1", "k2": "v2"} -obj.f1, obj.f3 = obj, obj.f2 -data = fury.serialize(obj) -# bytes can be data serialized by other languages. -print(fury.deserialize(data)) -``` - -**Golang** - -```go -package main - -import furygo "github.com/apache/incubator-fury/fury/go/fury" -import "fmt" - -func main() { - type SomeClass struct { - F1 *SomeClass - F2 map[string]string - F3 map[string]string - } - fury := furygo.NewFury(true) - if err := fury.RegisterTagType("example.SomeClass", SomeClass{}); err != nil { - panic(err) - } - value := &SomeClass{F2: map[string]string{"k1": "v1", "k2": "v2"}} - value.F3 = value.F2 - value.F1 = value - bytes, err := fury.Marshal(value) - if err != nil { - } - var newValue interface{} - // bytes can be data serialized by other languages. - if err := fury.Unmarshal(bytes, &newValue); err != nil { - panic(err) - } - fmt.Println(newValue) -} -``` - -**JavaScript** - -```javascript -import Fury, { Type } from '@furyjs/fury'; -/** - * @furyjs/hps use v8's fast-calls-api that can be called directly by jit, ensure that the version of Node is 20 or above. - * Experimental feature, installation success cannot be guaranteed at this moment - * If you are unable to install the module, replace it with `const hps = null;` - **/ -import hps from '@furyjs/hps'; - -const description = Type.object('example.foo', { - foo: Type.string(), - bar: Type.object('example.foo'), -}); - -const fury = new Fury({ hps }); -const { serialize, deserialize } = fury.registerSerializer(description); -const data: any = { - foo: 'hello fury', -}; -data.bar = data; -const input = serialize(data); -const result = deserialize(input); -console.log(result.bar.foo === result.foo); -``` - -**JavaScript** -Reference cannot be implemented because of rust ownership restrictions - - -### Zero-Copy Serialization - -**Java** - -```java -import org.apache.fury.*; -import org.apache.fury.config.*; -import org.apache.fury.serializers.BufferObject; -import org.apache.fury.memory.MemoryBuffer; - -import java.util.*; -import java.util.stream.Collectors; - -public class ZeroCopyExample { - // mvn exec:java -Dexec.mainClass="io.ray.fury.examples.ZeroCopyExample" - public static void main(String[] args) { - Fury fury = Fury.builder().withLanguage(Language.XLANG).build(); - List list = ofArrayList("str", new byte[1000], new int[100], new double[100]); - Collection bufferObjects = new ArrayList<>(); - byte[] bytes = fury.serialize(list, e -> !bufferObjects.add(e)); - // bytes can be data serialized by other languages. - List buffers = bufferObjects.stream() - .map(BufferObject::toBuffer).collect(Collectors.toList()); - System.out.println(fury.deserialize(bytes, buffers)); - } -} -``` - -**Python** - -```python -import array -import pyfury -import numpy as np - -fury = pyfury.Fury() -list_ = ["str", bytes(bytearray(1000)), - array.array("i", range(100)), np.full(100, 0.0, dtype=np.double)] -serialized_objects = [] -data = fury.serialize(list_, buffer_callback=serialized_objects.append) -buffers = [o.to_buffer() for o in serialized_objects] -# bytes can be data serialized by other languages. -print(fury.deserialize(data, buffers=buffers)) -``` - -**Golang** - -```go -package main - -import furygo "github.com/apache/incubator-fury/fury/go/fury" -import "fmt" - -func main() { - fury := furygo.NewFury() - list := []interface{}{"str", make([]byte, 1000)} - buf := fury.NewByteBuffer(nil) - var bufferObjects []fury.BufferObject - fury.Serialize(buf, list, func(o fury.BufferObject) bool { - bufferObjects = append(bufferObjects, o) - return false - }) - var newList []interface{} - var buffers []*fury.ByteBuffer - for _, o := range bufferObjects { - buffers = append(buffers, o.ToBuffer()) - } - if err := fury.Deserialize(buf, &newList, buffers); err != nil { - panic(err) - } - fmt.Println(newList) -} -``` - -**JavaScript** - -```javascript -// Coming soon -```