From 78357c285051659f287eccb6f156e4666d9bc5cc Mon Sep 17 00:00:00 2001 From: Albert Skalt Date: Mon, 31 Jul 2023 23:37:09 +0300 Subject: [PATCH] go-tupleconv: add documentation --- README.md | 177 ++++++++++++++++++++++++++- example_test.go | 283 ++++++++++++++++++++++++++++++++++++++++++++ testdata/config.lua | 38 ++++++ 3 files changed, 497 insertions(+), 1 deletion(-) create mode 100644 example_test.go create mode 100644 testdata/config.lua diff --git a/README.md b/README.md index 12e4b7b..7e131c6 100644 --- a/README.md +++ b/README.md @@ -1 +1,176 @@ -# Parser in Go for Tarantool types +# Tarantool tuples converter in Go + +[![Go Reference][godoc-badge]][godoc-url] +[![Actions Status][actions-badge]][actions-url] +[![Code Coverage][coverage-badge]][coverage-url] + +## Table of contents +* [Documentation](#documentation) + * [Converter](#converter) + * [Mapper](#mapper) + * [Mappers to tarantool types](#mappers-to-tarantool-types) + * [Example](#example) + * [String to nullable](#string-to-nullable) + * [String to any/scalar](#string-to-anyscalar) + * [Customization](#customization) +## Documentation + +### Converter +`Converter[S,T]` converts objects of type `S` into objects of type `T`. Converters +are basic entities on which mappers are based. +Implementations of some converters are available, for example, converters +from strings to golang types. +Usage example: +```golang +// Basic converter. +strToBoolConv := tupleconv.MakeStringToBoolConverter() +result, err := strToBoolConv.Convert("true") // true + +// Function based converter. +funcConv := tupleconv.MakeFuncConverter(func(s string) (string, error) { + return s + " world!", nil +}) +result, err = funcConv.Convert("hello") // hello world! +``` +**Note 1**: You can use the provided converters. + +**Note 2**: You can create your own converters based on the functions +with `tupleconv.MakeFuncConverter`. + +**Note 3**: You can create your own converters, implementing +`Converter[S,T]` interface. + +### Mapper +`Mapper` is an object that converts tuples. It is built using a list of +converters. +Usage example: +```golang +// Mapper example. +mapper := tupleconv.MakeMapper[string, any]([]tupleconv.Converter[string, any]{ + tupleconv.MakeFuncConverter(func(s string) (any, error) { + return s + "1", nil + }), + tupleconv.MakeFuncConverter(func(s string) (any, error) { + iVal, err := strconv.Atoi(s) + if err != nil { + return nil, errors.New("can't convert") + } + return iVal + 1, nil + }), +}) +result, err := mapper.Map([]string{"a", "4"}) // []any{"a1", 5} +result, err = mapper.Map([]string{"0"}) // []any{"01"} +``` +```golang +// Single mapper example. +toStringMapper := tupleconv.MakeMapper([]tupleconv.Converter[any, string]{}). + WithDefaultConverter(tupleconv.MakeFuncConverter( + func(s any) (string, error) { + return fmt.Sprintln(s), nil + }), +) +res, err := toStringMapper.Map([]any{1, 2.5, nil}) // ["1\n", "2.5\n", "\n"] +``` +**Note 1**: To create a mapper, an array of converters is needed, each +of which transforms a certain type S into type T. + +**Note 2**: To perform tuple mapping, you can use the function +`Map`, which will return control to the calling code upon the first error. + +**Note 3**: You can set a default converter that will be applied if the tuple length exceeds +the size of the primary converters list. +For example, if you only set a default converter, `Map` will work like the `map` function in +functional programming languages. + +**Note 4**: If tuple length is less than converters list length, then only corresponding converters +will be applied. + +### Mappers to tarantool types + +#### Example +For building an array of converters, especially when it comes to conversions to +tarantool types, there is a built-in solution. +Let's consider an example: +```golang +factory := tupleconv.MakeStringToTTConvFactory(). + WithDecimalSeparators(",.") + +spaceFmt := []tupleconv.SpaceField{ + {Type: tupleconv.TypeUnsigned}, + {Type: tupleconv.TypeDouble, IsNullable: true}, + {Type: tupleconv.TypeString}, +} + +converters, _ := tupleconv.MakeTypeToTTConverters[string](factory, spaceFmt) +mapper := tupleconv.MakeMapper(converters) +result, err := mapper.Map([]string{"1", "-2,2", "some_string"}) // [1, -2.2, "some_string"] +``` +**Note 1**: To build an array of converters, the space format and a +certain object implementing `TTConvFactory` are used. Function +`MakeTypeToTTConverters` takes these entities and gives the converters list. + +**Note 2**: `TTConvFactory[Type]` is capable of building a +converter from `Type` to each tarantool type. + +**Note 3**: There is a basic factory available called +`StringToTTConvFactory`, which is used for conversions from strings to +tarantool types. + +**Note 4**: `StringToTTConvFactory` can be configured with options like +`WithDecimalSeparators`. + +#### String to nullable +When converting nullable types with `StringToTTConvFactory`, first, an attempt +is made to convert to null. + +For example, empty string is interpreted like `null` with default options. +If a field has a `string` type and is `nullable`, then an empty string will be +converted to null during the conversion process, rather than being +converted to empty string. + + +#### String to any/scalar +When converting to `any`/`scalar` with `StringToTTConvFactory`, by default, +an attempt will be made to convert them to the following types, +in the following order: +- `number` +- `decimal` +- `boolean` +- `datetime` +- `uuid` +- `interval` +- `string` + +#### Customization +`TTConvFactory[Type]` is an interface that can build a mapper from +`Type` to each tarantool type. +To customize the behavior for specific types, one can +inherit from the existing factory and override the necessary methods. +For example, let's make the standard factory for conversion from strings to +tarantool types always convert `any` type to a string: +```golang +type customFactory struct { + tupleconv.StringToTTConvFactory +} + +func (f *customFactory) MakeTypeToAnyMapper() tupleconv.Converter[string, any] { + return tupleconv.MakeFuncConverter(func(s string) (any, error) { + return s, nil + }) +} + +func example() { + factory := &customFactory{} + spaceFmt := []tupleconv.SpaceField{{Type: "any"}} + converters, _ := tupleconv.MakeTypeToTTConverters[string](factory, spaceFmt) + + res, err := converters[0].Convert("12") // "12" +} +``` + +[godoc-badge]: https://pkg.go.dev/badge/github.com/tarantool/go-tupleconv.svg +[godoc-url]: https://pkg.go.dev/github.com/tarantool/go-tupleconv +[actions-badge]: https://github.com/tarantool/go-tupleconv/actions/workflows/test.yml/badge.svg +[actions-url]: https://github.com/tarantool/go-tupleconv/actions/workflows/test.yml +[coverage-badge]: https://coveralls.io/repos/github/tarantool/go-tupleconv/badge.svg?branch=master +[coverage-url]: https://coveralls.io/github/tarantool/go-tupleconv?branch=master \ No newline at end of file diff --git a/example_test.go b/example_test.go new file mode 100644 index 0000000..347fd01 --- /dev/null +++ b/example_test.go @@ -0,0 +1,283 @@ +package tupleconv_test + +import ( + "errors" + "fmt" + "github.com/tarantool/go-tarantool" + "github.com/tarantool/go-tarantool/datetime" + "github.com/tarantool/go-tarantool/test_helpers" + "github.com/tarantool/go-tupleconv" + "strconv" + "strings" + "time" + + _ "github.com/tarantool/go-tarantool/uuid" +) + +type filterIntConverter struct { + toFilter string +} + +func (c filterIntConverter) Convert(src string) (int64, error) { + src = strings.ReplaceAll(src, c.toFilter, "") + return strconv.ParseInt(src, 10, 64) +} + +func ExampleConverter() { + // Basic converter. + strToBoolConv := tupleconv.MakeStringToBoolConverter() + result, err := strToBoolConv.Convert("true") + fmt.Println(result, err) + + // Function based converter. + funcConv := tupleconv.MakeFuncConverter(func(s string) (string, error) { + return s + " world!", nil + }) + result, err = funcConv.Convert("hello") + fmt.Println(result, err) + + var filterConv tupleconv.Converter[string, int64] = filterIntConverter{toFilter: "th"} + result, err = filterConv.Convert("100th") + fmt.Println(result, err) + + // Output: + // true + // hello world! + // 100 +} + +func ExampleMapper_basicMapper() { + // Mapper example. + mapper := tupleconv.MakeMapper[string, any]([]tupleconv.Converter[string, any]{ + tupleconv.MakeFuncConverter(func(s string) (any, error) { + return s + "1", nil + }), + tupleconv.MakeFuncConverter(func(s string) (any, error) { + iVal, err := strconv.Atoi(s) + if err != nil { + return nil, errors.New("can't convert") + } + return iVal + 1, nil + }), + }) + result, err := mapper.Map([]string{"a", "4"}) + fmt.Println(result, err) + + result, err = mapper.Map([]string{"0"}) + fmt.Println(result, err) + + // Output: + // [a1 5] + // [01] +} + +func ExampleMapper_singleMapper() { + // Single mapper example. + toStringMapper := tupleconv.MakeMapper([]tupleconv.Converter[any, string]{}). + WithDefaultConverter(tupleconv.MakeFuncConverter( + func(s any) (string, error) { + return fmt.Sprintln(s), nil + }), + ) + res, err := toStringMapper.Map([]any{1, 2.5, nil}) + fmt.Println(res, err) + + // Output: + // [1 + // 2.5 + // + //] +} + +func ExampleStringToTTConvFactory() { + factory := tupleconv.MakeStringToTTConvFactory(). + WithDecimalSeparators(",.") + + spaceFmt := []tupleconv.SpaceField{ + {Type: tupleconv.TypeUnsigned}, + {Type: tupleconv.TypeDouble, IsNullable: true}, + {Type: tupleconv.TypeString}, + } + + converters, _ := tupleconv.MakeTypeToTTConverters[string](factory, spaceFmt) + mapper := tupleconv.MakeMapper(converters) + result, err := mapper.Map([]string{"1", "-2,2", "some_string"}) + fmt.Println(result, err) + + // Output: + // [1 -2.2 some_string] +} + +func ExampleStringToTTConvFactory_manualConverters() { + factory := tupleconv.MakeStringToTTConvFactory(). + WithDecimalSeparators(",.") + + fieldTypes := []tupleconv.TypeName{ + tupleconv.TypeUnsigned, + tupleconv.TypeDouble, + tupleconv.TypeString, + } + + converters := make([]tupleconv.Converter[string, any], 0) + for _, typ := range fieldTypes { + conv, _ := tupleconv.GetConverterByType[string](factory, typ) + converters = append(converters, conv) + } + + mapper := tupleconv.MakeMapper(converters) + result, err := mapper.Map([]string{"1", "-2,2", "some_string"}) + fmt.Println(result, err) + + // Output: + // [1 -2.2 some_string] +} + +func ExampleStringToTTConvFactory_convertNullable() { + factory := tupleconv.MakeStringToTTConvFactory(). + WithNullValue("2.5") + + converters, _ := tupleconv.MakeTypeToTTConverters[string](factory, []tupleconv.SpaceField{ + {Type: tupleconv.TypeDouble, IsNullable: true}, + }) + fmt.Println(converters[0].Convert("2.5")) + + // Output: + // +} + +type customFactory struct { + tupleconv.StringToTTConvFactory +} + +func (f *customFactory) MakeTypeToAnyMapper() tupleconv.Converter[string, any] { + return tupleconv.MakeFuncConverter(func(s string) (any, error) { + return s, nil + }) +} + +func ExampleTTConvFactory_custom() { + facture := &customFactory{} + spaceFmt := []tupleconv.SpaceField{{Type: tupleconv.TypeAny}} + converters, _ := tupleconv.MakeTypeToTTConverters[string](facture, spaceFmt) + + res, err := converters[0].Convert("12") + fmt.Println(res, err) + + // Output: + // 12 +} + +const workDir = "work_dir" +const server = "127.0.0.1:3014" + +func upTarantool() (func(), error) { + inst, err := test_helpers.StartTarantool(test_helpers.StartOpts{ + InitScript: "testdata/config.lua", + Listen: server, + WorkDir: workDir, + User: "test", + Pass: "password", + WaitStart: 100 * time.Millisecond, + ConnectRetry: 3, + RetryTimeout: 500 * time.Millisecond, + }) + if err != nil { + test_helpers.StopTarantoolWithCleanup(inst) + return nil, nil + } + + cleanup := func() { + test_helpers.StopTarantoolWithCleanup(inst) + } + return cleanup, nil +} + +func ExampleMap_insertMappedTuples() { + cleanupTarantool, err := upTarantool() + if err != nil { + fmt.Println(err) + return + } + defer cleanupTarantool() + + conn, _ := tarantool.Connect(server, tarantool.Opts{ + User: "test", + Pass: "password", + }) + var spaceFmtResp [][]tupleconv.SpaceField + _ = conn.CallTyped("get_test_space_fmt", []any{}, &spaceFmtResp) + spaceFmt := spaceFmtResp[0] + fmt.Println(spaceFmt[0:3]) + + fac := tupleconv.MakeStringToTTConvFactory() + converters, _ := tupleconv.MakeTypeToTTConverters[string](fac, spaceFmt) + decoder := tupleconv.MakeMapper(converters). + WithDefaultConverter(fac.GetStringConverter()) + + dt1 := "2020-08-22T11:27:43.123456789-02:00" + dt2 := "1880-01-01T00:00:00Z" + uuid := "00000000-0000-0000-0000-000000000001" + interval := "1,2,3,4,5,6,7,8,1" + + tuples := [][]string{ + {"1", "true", "12", "143.5", dt1, "", "str", "", "[1,2,3]", "190", ""}, + {"2", "f", "0", "-42", dt2, interval, "abacaba", "", "[]", uuid, "150"}, + + // Extra fields. + {"4", "1", "12", "143.5", dt1, "", "str", uuid, "[1,2,3]", "190", "", "extra", "blabla"}, + } + + for _, tuple := range tuples { + mapped, err := decoder.Map(tuple) + if err != nil { + fmt.Println(err) + return + } + insertReq := tarantool.NewInsertRequest("test_space").Tuple(mapped) + resp, err := conn.Do(insertReq).Get() + if err != nil { + fmt.Println(err) + return + } + fmt.Println("insert response code =", resp.Code) + } + + selectReq := tarantool.NewSelectRequest("test_space") + resp, err := conn.Do(selectReq).Get() + if err != nil { + fmt.Println(err) + return + } + + tuple0, _ := resp.Data[0].([]any) + encoder := tupleconv.MakeMapper[any, string]([]tupleconv.Converter[any, string]{}). + WithDefaultConverter(tupleconv.MakeFuncConverter(func(s any) (string, error) { + asDatetime, isDatetime := s.(datetime.Datetime) + if isDatetime { + return fmt.Sprintln(asDatetime.ToTime()), nil + } else { + return fmt.Sprintln(s), nil + } + })) + + encodedTuple0, _ := encoder.Map(tuple0) + fmt.Println(encodedTuple0) + + // Output: + // [{0 id unsigned false} {0 boolean boolean false} {0 number number false}] + // insert response code = 0 + // insert response code = 0 + // insert response code = 0 + // [1 + // true + // 12 + // 143.5 + // 2020-08-22 13:27:43.123456789 +0000 UTC + // + // str + // + // [1 2 3] + // 190 + // + //] +} diff --git a/testdata/config.lua b/testdata/config.lua new file mode 100644 index 0000000..19d393a --- /dev/null +++ b/testdata/config.lua @@ -0,0 +1,38 @@ +-- Do not set listen for now so connector won't be +-- able to send requests until everything is configured. + +box.cfg { + work_dir = os.getenv("TEST_TNT_WORK_DIR"), +} + +box.once('init', function() + box.schema.user.create('test', { password = 'password' }) + box.schema.user.grant('test', 'execute,read,write', 'universe') + + box.schema.create_space('test_space', { + format = { + { name = 'id', type = 'unsigned' }, + { name = 'boolean', type = 'boolean' }, + { name = 'number', type = 'number' }, + { name = 'decimal', type = 'decimal' }, + { name = 'datetime', type = 'datetime' }, + { name = 'interval', type = 'interval', is_nullable = true }, + { name = 'string', type = 'string', is_nullable = true }, + { name = 'uuid', type = 'uuid', is_nullable = true }, + { name = 'array', type = 'array' }, + { name = 'any', type = 'any' }, + { name = 'scalar', type = 'scalar', is_nullable = true }, + } + }) + + box.space.test_space:create_index('primary') +end) + +function get_test_space_fmt() + return box.space.test_space:format() +end + +-- Set listen only when every other thing is configured. +box.cfg { + listen = os.getenv("TEST_TNT_LISTEN"), +}