Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history 
Signed-off-by: ThreadDao <[email protected]>
  • Loading branch information
@ThreadDao
ThreadDao committed Oct 31, 2023
1 parent 1802e08 commit 22ff7be
Show file tree
Hide file tree
Showing 7 changed files with 251 additions and 78 deletions.
2 changes: 1 addition & 1 deletion client/collection.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ func (c *GrpcClient) CreateCollection(ctx context.Context, collSchema *entity.Sc
for _, o := range opts {
o(opt)
}

collSchema.EnableDynamicField = opt.EnableDynamicSchema
return c.requestCreateCollection(ctx, collSchema, opt, shardNum)
}

Expand Down
138 changes: 74 additions & 64 deletions test/common/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,17 @@ const (
DefaultBinaryVecFieldName = "binaryVec"
DefaultDynamicNumberField = "dynamicNumber"
DefaultDynamicStringField = "dynamicString"
DefaultDynamicBoolField = "dynamicBool"
DefaultDynamicListField = "dynamicList"
RowCount = "row_count"
DefaultTimeout = 120
DefaultDim = int64(128)
DefaultShards = int32(2)
DefaultNb = 3000
DefaultNq = 5
DefaultTopK = 10
DefaultDynamicBoolField = "dynamicBool"
DefaultDynamicListField = "dynamicList"
RowCount = "row_count"
DefaultTimeout = 120
DefaultDim = int64(128)
DefaultShards = int32(2)
DefaultNb = 3000
DefaultNq = 5
DefaultTopK = 10
TestCapacity = 100 // default array field capacity
TestMaxLen = 100 // default varchar field max length
)

// const default value from milvus
Expand All @@ -47,7 +49,7 @@ const (
DefaultDb = "default"
DefaultConsistencyLevel = entity.ClBounded
MaxDim = 32768
DefaultMaxLength = int64(65535)
MaxLength = int64(65535)
MaxCollectionNameLen = 255
DefaultRgCapacity = 1000000
RetentionDuration = 40 // common.retentionDuration
Expand All @@ -63,6 +65,17 @@ var IndexStateValue = map[string]int32{
"Retry": 5,
}

var ArrayFieldType = []entity.FieldType{
entity.FieldTypeBool,
entity.FieldTypeInt8,
entity.FieldTypeInt16,
entity.FieldTypeInt32,
entity.FieldTypeInt64,
entity.FieldTypeFloat,
entity.FieldTypeDouble,
entity.FieldTypeVarChar,
}

var r *rand.Rand

func init() {
Expand Down Expand Up @@ -120,37 +133,44 @@ func GenDefaultBinaryFields(autoID bool, dim int64) []*entity.Field {

// GenDefaultVarcharFields gen default fields with varchar, floatVector field
func GenDefaultVarcharFields(autoID bool) []*entity.Field {
varcharField := GenField(DefaultVarcharFieldName, entity.FieldTypeVarChar, WithIsPrimaryKey(true), WithAutoID(autoID), WithMaxLength(DefaultMaxLength))
varcharField := GenField(DefaultVarcharFieldName, entity.FieldTypeVarChar, WithIsPrimaryKey(true), WithAutoID(autoID), WithMaxLength(MaxLength))
binaryVecField := GenField(DefaultBinaryVecFieldName, entity.FieldTypeBinaryVector, WithDim(DefaultDim))
fields := []*entity.Field{
varcharField, binaryVecField,
}
return fields
}

// GenAllArrayFields gen all array fields
func GenAllArrayFields() []*entity.Field {
fields := []*entity.Field{
GenField("boolArray", entity.FieldTypeArray, WithElementType(entity.FieldTypeBool), WithMaxCapacity(TestCapacity)),
GenField("int8Array", entity.FieldTypeArray, WithElementType(entity.FieldTypeInt8), WithMaxCapacity(TestCapacity)),
GenField("int16Array", entity.FieldTypeArray, WithElementType(entity.FieldTypeInt16), WithMaxCapacity(TestCapacity)),
GenField("int32Array", entity.FieldTypeArray, WithElementType(entity.FieldTypeInt32), WithMaxCapacity(TestCapacity)),
GenField("int64Array", entity.FieldTypeArray, WithElementType(entity.FieldTypeInt64), WithMaxCapacity(TestCapacity)),
GenField("floatArray", entity.FieldTypeArray, WithElementType(entity.FieldTypeFloat), WithMaxCapacity(TestCapacity)),
GenField("doubleArray", entity.FieldTypeArray, WithElementType(entity.FieldTypeDouble), WithMaxCapacity(TestCapacity)),
GenField("varcharArray", entity.FieldTypeArray, WithElementType(entity.FieldTypeVarChar), WithMaxLength(TestMaxLen), WithMaxCapacity(TestCapacity)),
}
return fields
}

// GenAllFields gen fields with all scala field types
func GenAllFields() []*entity.Field {
allFields := []*entity.Field{
GenField("int64", entity.FieldTypeInt64, WithIsPrimaryKey(true)), // int64
GenField("bool", entity.FieldTypeBool), // bool
GenField("int8", entity.FieldTypeInt8), // int8
GenField("int16", entity.FieldTypeInt16), // int16
GenField("int32", entity.FieldTypeInt32), // int32
GenField("float", entity.FieldTypeFloat), // float
GenField("double", entity.FieldTypeDouble), // double
GenField("varchar", entity.FieldTypeVarChar, WithMaxLength(DefaultMaxLength)), // varchar
GenField("json", entity.FieldTypeJSON), // json
GenField("boolArray", entity.FieldTypeArray, WithElementType(entity.FieldTypeBool), WithMaxCapacity(100)),
GenField("int8Array", entity.FieldTypeArray, WithElementType(entity.FieldTypeInt8), WithMaxCapacity(100)),
GenField("int16Array", entity.FieldTypeArray, WithElementType(entity.FieldTypeInt16), WithMaxCapacity(100)),
GenField("int32Array", entity.FieldTypeArray, WithElementType(entity.FieldTypeInt32), WithMaxCapacity(100)),
GenField("int64Array", entity.FieldTypeArray, WithElementType(entity.FieldTypeInt64), WithMaxCapacity(100)),
GenField("floatArray", entity.FieldTypeArray, WithElementType(entity.FieldTypeFloat), WithMaxCapacity(100)),
GenField("doubleArray", entity.FieldTypeArray, WithElementType(entity.FieldTypeDouble), WithMaxCapacity(100)),
GenField("varcharArray", entity.FieldTypeArray, WithElementType(entity.FieldTypeVarChar), WithMaxLength(100), WithMaxCapacity(100)),
GenField("int64", entity.FieldTypeInt64, WithIsPrimaryKey(true)), // int64
GenField("bool", entity.FieldTypeBool), // bool
GenField("int8", entity.FieldTypeInt8), // int8
GenField("int16", entity.FieldTypeInt16), // int16
GenField("int32", entity.FieldTypeInt32), // int32
GenField("float", entity.FieldTypeFloat), // float
GenField("double", entity.FieldTypeDouble), // double
GenField("varchar", entity.FieldTypeVarChar, WithMaxLength(MaxLength)), // varchar
GenField("json", entity.FieldTypeJSON), // json
GenField("floatVec", entity.FieldTypeFloatVector, WithDim(DefaultDim)), // float vector
}

allFields = append(allFields, GenAllArrayFields()...)
return allFields
}

Expand All @@ -165,32 +185,6 @@ func GenDefaultColumnData(start int, nb int, dim int64) (entity.Column, entity.C
GenColumnData(start, nb, entity.FieldTypeFloatVector, DefaultFloatVecFieldName, WithVectorDim(dim))
}

type GenColumnDataOption func(opt *genDataOpt)

type genDataOpt struct {
dim int64
ElementType entity.FieldType
capacity int64
}

func WithVectorDim(dim int64) GenColumnDataOption {
return func(opt *genDataOpt) {
opt.dim = dim
}
}

func WithArrayElementType(eleType entity.FieldType) GenColumnDataOption {
return func(opt *genDataOpt) {
opt.ElementType = eleType
}
}

func WithArrayCapacity(capacity int64) GenColumnDataOption {
return func(opt *genDataOpt) {
opt.capacity = capacity
}
}

// GenColumnData GenColumnDataOption
func GenColumnData(start int, nb int, fieldType entity.FieldType, fieldName string, opts ...GenColumnDataOption) entity.Column {
opt := &genDataOpt{}
Expand Down Expand Up @@ -422,7 +416,30 @@ func GenDefaultVarcharData(start int, nb int, dim int64) (entity.Column, entity.
return varColumn, binaryColumn
}

func GenAllFieldsData(start int, nb int, dim int64) []entity.Column {
func GenAllArrayData(start int, nb int, opts ...GenColumnDataOption) []entity.Column {
// how to pass different capacity for different column
opt := &genDataOpt{}
for _, o := range opts {
o(opt)
}
data := []entity.Column {
GenColumnData(start, nb, entity.FieldTypeArray, "boolArray", WithArrayElementType(entity.FieldTypeBool), WithArrayCapacity(opt.capacity)),
GenColumnData(start, nb, entity.FieldTypeArray, "int8Array", WithArrayElementType(entity.FieldTypeInt8), WithArrayCapacity(20)),
GenColumnData(start, nb, entity.FieldTypeArray, "int16Array", WithArrayElementType(entity.FieldTypeInt16), WithArrayCapacity(30)),
GenColumnData(start, nb, entity.FieldTypeArray, "int32Array", WithArrayElementType(entity.FieldTypeInt32), WithArrayCapacity(30)),
GenColumnData(start, nb, entity.FieldTypeArray, "int64Array", WithArrayElementType(entity.FieldTypeInt64), WithArrayCapacity(30)),
GenColumnData(start, nb, entity.FieldTypeArray, "floatArray", WithArrayElementType(entity.FieldTypeFloat), WithArrayCapacity(100)),
GenColumnData(start, nb, entity.FieldTypeArray, "doubleArray", WithArrayElementType(entity.FieldTypeDouble), WithArrayCapacity(30)),
//GenColumnData(start, nb, entity.FieldTypeArray, "varcharArray", WithArrayElementType(entity.FieldTypeVarChar), WithArrayCapacity(100)),
}
return data
}

func GenAllFieldsData(start int, nb int, dim int64, opts ...GenColumnDataOption) []entity.Column {
opt := &genDataOpt{}
for _, o := range opts {
o(opt)
}
// prepare data
data := []entity.Column{
GenColumnData(start, nb, entity.FieldTypeInt64, "int64"),
Expand All @@ -434,16 +451,9 @@ func GenAllFieldsData(start int, nb int, dim int64) []entity.Column {
GenColumnData(start, nb, entity.FieldTypeDouble, "double"),
GenColumnData(start, nb, entity.FieldTypeVarChar, "varchar"),
GenDefaultJSONData("json", start, nb),
GenColumnData(start, nb, entity.FieldTypeArray, "boolArray", WithArrayElementType(entity.FieldTypeBool), WithArrayCapacity(10)),
GenColumnData(start, nb, entity.FieldTypeArray, "int8Array", WithArrayElementType(entity.FieldTypeInt8), WithArrayCapacity(20)),
GenColumnData(start, nb, entity.FieldTypeArray, "int16Array", WithArrayElementType(entity.FieldTypeInt16), WithArrayCapacity(30)),
GenColumnData(start, nb, entity.FieldTypeArray, "int32Array", WithArrayElementType(entity.FieldTypeInt32), WithArrayCapacity(30)),
GenColumnData(start, nb, entity.FieldTypeArray, "int64Array", WithArrayElementType(entity.FieldTypeInt64), WithArrayCapacity(30)),
GenColumnData(start, nb, entity.FieldTypeArray, "floatArray", WithArrayElementType(entity.FieldTypeFloat), WithArrayCapacity(100)),
GenColumnData(start, nb, entity.FieldTypeArray, "doubleArray", WithArrayElementType(entity.FieldTypeDouble), WithArrayCapacity(30)),
GenColumnData(start, nb, entity.FieldTypeArray, "varcharArray", WithArrayElementType(entity.FieldTypeVarChar), WithArrayCapacity(100)),
GenColumnData(start, nb, entity.FieldTypeFloatVector, "floatVec", WithVectorDim(dim)),
}
data = append(data, GenAllArrayData(start, nb)...)
return data
}

Expand Down
29 changes: 28 additions & 1 deletion test/common/utils_client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func WithEnableDynamicField(enableDF bool) CreateSchemaOption {
}
}

// gen schema
// GenSchema gen schema
func GenSchema(name string, autoID bool, fields []*entity.Field, opts ...CreateSchemaOption) *entity.Schema {
schema := &entity.Schema{
CollectionName: name,
Expand All @@ -110,3 +110,30 @@ func GenSchema(name string, autoID bool, fields []*entity.Field, opts ...CreateS
}

// -- create schema --

// GenColumnDataOption -- create column data --
type GenColumnDataOption func(opt *genDataOpt)
type genDataOpt struct {
dim int64
ElementType entity.FieldType
capacity int64
}

func WithVectorDim(dim int64) GenColumnDataOption {
return func(opt *genDataOpt) {
opt.dim = dim
}
}

func WithArrayElementType(eleType entity.FieldType) GenColumnDataOption {
return func(opt *genDataOpt) {
opt.ElementType = eleType
}
}

func WithArrayCapacity(capacity int64) GenColumnDataOption {
return func(opt *genDataOpt) {
opt.capacity = capacity
}
}
// -- create column data --
44 changes: 41 additions & 3 deletions test/testcases/collection_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ package testcases

import (
"fmt"
"github.com/milvus-io/milvus-sdk-go/v2/client"
"log"
"testing"
"time"
Expand Down Expand Up @@ -446,6 +447,30 @@ func TestCreateCollectionDynamicSchema(t *testing.T) {
// check describe collection
collection, _ := mc.DescribeCollection(ctx, collName)
common.CheckCollection(t, collection, collName, common.DefaultShards, schema, common.DefaultConsistencyLevel)
require.Truef(t, collection.Schema.EnableDynamicField, "Expected collection.Schema.EnableDynamicField is True")

// check collName in ListCollections
collections, errListCollection := mc.ListCollections(ctx)
common.CheckErr(t, errListCollection, true)
common.CheckContainsCollection(t, collections, collName)
}

// test create collection enable dynamic field by collection opt
func TestCreateCollectionDynamic(t *testing.T) {
t.Skip("Waiting for congqi to update schema.EnableDynamicField according to the CreateCollectionOption.EnableDynamicSchema")
ctx := createContext(t, time.Second*common.DefaultTimeout)
mc := createMilvusClient(ctx, t)

collName := common.GenRandomString(6)
schema := common.GenSchema(collName, false, common.GenDefaultFields(false))

err := mc.CreateCollection(ctx, schema, common.DefaultShards, client.WithEnableDynamicSchema(true))
common.CheckErr(t, err, true)

// check describe collection
collection, _ := mc.DescribeCollection(ctx, collName)
common.CheckCollection(t, collection, collName, common.DefaultShards, schema, common.DefaultConsistencyLevel)
require.Truef(t, collection.Schema.EnableDynamicField, "Expected collection.Schema.EnableDynamicField is True")

// check collName in ListCollections
collections, errListCollection := mc.ListCollections(ctx)
Expand Down Expand Up @@ -487,6 +512,7 @@ func TestCreateArrayFieldInvalidCapacity(t *testing.T) {
}
}

// test create collection varchar array with invalid max length
func TestCreateVarcharArrayInvalidLength(t *testing.T) {
ctx := createContext(t, time.Second*common.DefaultTimeout)
mc := createMilvusClient(ctx, t)
Expand All @@ -498,12 +524,12 @@ func TestCreateVarcharArrayInvalidLength(t *testing.T) {
common.CheckErr(t, err, false, "type param(max_length) should be specified for varChar field")

// invalid max length
for _, invalidLength := range []int64{-1, 0, common.DefaultMaxLength + 1} {
arrayField := common.GenField(common.DefaultArrayFieldName, entity.FieldTypeArray, common.WithElementType(entity.FieldTypeFloat),
for _, invalidLength := range []int64{-1, 0, common.MaxLength + 1} {
arrayField := common.GenField(common.DefaultArrayFieldName, entity.FieldTypeArray, common.WithElementType(entity.FieldTypeVarChar),
common.WithMaxCapacity(100), common.WithMaxLength(invalidLength))
schema := common.GenSchema(common.GenRandomString(6), false, append(fields, arrayField), common.WithEnableDynamicField(true))
err := mc.CreateCollection(ctx, schema, common.DefaultShards)
common.CheckErr(t, err, false, "the maximum capacity specified for a Array should be in (0, 4096]")
common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 65535]: invalid parameter")
}
}

Expand Down Expand Up @@ -536,6 +562,18 @@ func TestCreateArrayNotSupportedFieldType(t *testing.T) {
}
}

func TestCreateCollectionAllFields(t *testing.T) {
ctx := createContext(t, time.Second*common.DefaultTimeout)
mc := createMilvusClient(ctx, t)
allFields := common.GenAllFields()
collName := common.GenRandomString(6)
schema := common.GenSchema(collName, false, allFields)

// create collection
errCreateCollection := mc.CreateCollection(ctx, schema, common.DefaultShards)
common.CheckErr(t, errCreateCollection, true)
}

// -- Get Collection Statistics --

func TestGetStaticsCollectionNotExisted(t *testing.T) {
Expand Down
Loading

0 comments on commit 22ff7be

Please sign in to comment.