From 7af8cccb7cb02ad1aeec9b31b2531121b7c71747 Mon Sep 17 00:00:00 2001 From: jasonkarel Date: Wed, 8 Jan 2025 15:22:13 +0700 Subject: [PATCH 1/4] chore(server): move export csv items and GeoJSON to Item Usecase Layer --- server/internal/adapter/integration/item.go | 59 +-- server/internal/usecase/interactor/item.go | 82 +++++ .../usecase/interactor/item_export.go | 62 ++++ .../usecase/interactor/item_export_test.go | 97 +++++ .../internal/usecase/interactor/item_test.go | 338 ++++++++++++++++++ server/internal/usecase/interfaces/item.go | 15 + 6 files changed, 610 insertions(+), 43 deletions(-) create mode 100644 server/internal/usecase/interactor/item_export.go create mode 100644 server/internal/usecase/interactor/item_export_test.go diff --git a/server/internal/adapter/integration/item.go b/server/internal/adapter/integration/item.go index f79f876905..237273f943 100644 --- a/server/internal/adapter/integration/item.go +++ b/server/internal/adapter/integration/item.go @@ -3,7 +3,6 @@ package integration import ( "context" "errors" - "io" "github.com/reearth/reearth-cms/server/internal/usecase" "github.com/reearth/reearth-cms/server/pkg/model" @@ -74,13 +73,12 @@ func (s *Server) ItemsAsGeoJSON(ctx context.Context, request ItemsAsGeoJSONReque op := adapter.Operator(ctx) uc := adapter.Usecases(ctx) - sp, err := uc.Schema.FindByModel(ctx, request.ModelId, op) + schemaPackage, err := uc.Schema.FindByModel(ctx, request.ModelId, op) if err != nil { - return ItemsAsGeoJSON400Response{}, err + return ItemsAsGeoJSON404Response{}, err } - p := fromPagination(request.Params.Page, request.Params.PerPage) - items, _, err := uc.Item.FindBySchema(ctx, sp.Schema().ID(), nil, p, op) + featureCollections, err := uc.Item.ItemsAsGeoJSON(ctx, schemaPackage, request.Params.Page, request.Params.PerPage, op) if err != nil { if errors.Is(err, rerror.ErrNotFound) { return ItemsAsGeoJSON404Response{}, err @@ -88,14 +86,9 @@ func (s *Server) ItemsAsGeoJSON(ctx context.Context, request ItemsAsGeoJSONReque return ItemsAsGeoJSON400Response{}, err } - fc, err := featureCollectionFromItems(items, sp.Schema()) - if err != nil { - return ItemsAsGeoJSON400Response{}, err - } - return ItemsAsGeoJSON200JSONResponse{ - Features: fc.Features, - Type: fc.Type, + Features: featureCollections.FeatureCollections.Features, + Type: featureCollections.FeatureCollections.Type, }, nil } @@ -103,13 +96,12 @@ func (s *Server) ItemsAsCSV(ctx context.Context, request ItemsAsCSVRequestObject op := adapter.Operator(ctx) uc := adapter.Usecases(ctx) - sp, err := uc.Schema.FindByModel(ctx, request.ModelId, op) + schemaPackage, err := uc.Schema.FindByModel(ctx, request.ModelId, op) if err != nil { - return ItemsAsCSV400Response{}, err + return ItemsAsCSV404Response{}, err } - p := fromPagination(request.Params.Page, request.Params.PerPage) - items, _, err := uc.Item.FindBySchema(ctx, sp.Schema().ID(), nil, p, op) + pr, err := uc.Item.ItemsAsCSV(ctx, schemaPackage, request.Params.Page, request.Params.PerPage, op) if err != nil { if errors.Is(err, rerror.ErrNotFound) { return ItemsAsCSV404Response{}, err @@ -117,14 +109,8 @@ func (s *Server) ItemsAsCSV(ctx context.Context, request ItemsAsCSVRequestObject return ItemsAsCSV400Response{}, err } - pr, pw := io.Pipe() - err = csvFromItems(pw, items, sp.Schema()) - if err != nil { - return ItemsAsCSV400Response{}, err - } - return ItemsAsCSV200TextcsvResponse{ - Body: pr, + Body: pr.PipeReader, }, nil } @@ -217,13 +203,12 @@ func (s *Server) ItemsWithProjectAsGeoJSON(ctx context.Context, request ItemsWit return ItemsWithProjectAsGeoJSON400Response{}, err } - sp, err := uc.Schema.FindByModel(ctx, m.ID(), op) + schemaPackage, err := uc.Schema.FindByModel(ctx, m.ID(), op) if err != nil { return ItemsWithProjectAsGeoJSON400Response{}, err } - p := fromPagination(request.Params.Page, request.Params.PerPage) - items, _, err := uc.Item.FindBySchema(ctx, sp.Schema().ID(), nil, p, op) + featureCollections, err := uc.Item.ItemsAsGeoJSON(ctx, schemaPackage, request.Params.Page, request.Params.PerPage, op) if err != nil { if errors.Is(err, rerror.ErrNotFound) { return ItemsWithProjectAsGeoJSON404Response{}, err @@ -231,14 +216,9 @@ func (s *Server) ItemsWithProjectAsGeoJSON(ctx context.Context, request ItemsWit return ItemsWithProjectAsGeoJSON400Response{}, err } - fc, err := featureCollectionFromItems(items, sp.Schema()) - if err != nil { - return ItemsWithProjectAsGeoJSON400Response{}, err - } - return ItemsWithProjectAsGeoJSON200JSONResponse{ - Features: fc.Features, - Type: fc.Type, + Features: featureCollections.FeatureCollections.Features, + Type: featureCollections.FeatureCollections.Type, }, nil } @@ -262,13 +242,12 @@ func (s *Server) ItemsWithProjectAsCSV(ctx context.Context, request ItemsWithPro return ItemsWithProjectAsCSV400Response{}, err } - sp, err := uc.Schema.FindByModel(ctx, m.ID(), op) + schemaPackage, err := uc.Schema.FindByModel(ctx, m.ID(), op) if err != nil { return ItemsWithProjectAsCSV400Response{}, err } - p := fromPagination(request.Params.Page, request.Params.PerPage) - items, _, err := uc.Item.FindBySchema(ctx, sp.Schema().ID(), nil, p, op) + pr, err := uc.Item.ItemsAsCSV(ctx, schemaPackage, request.Params.Page, request.Params.PerPage, op) if err != nil { if errors.Is(err, rerror.ErrNotFound) { return ItemsWithProjectAsCSV404Response{}, err @@ -276,14 +255,8 @@ func (s *Server) ItemsWithProjectAsCSV(ctx context.Context, request ItemsWithPro return ItemsWithProjectAsCSV400Response{}, err } - pr, pw := io.Pipe() - err = csvFromItems(pw, items, sp.Schema()) - if err != nil { - return ItemsWithProjectAsCSV400Response{}, err - } - return ItemsWithProjectAsCSV200TextcsvResponse{ - Body: pr, + Body: pr.PipeReader, }, nil } diff --git a/server/internal/usecase/interactor/item.go b/server/internal/usecase/interactor/item.go index e66a157a49..c4c06bfd1e 100644 --- a/server/internal/usecase/interactor/item.go +++ b/server/internal/usecase/interactor/item.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "time" "github.com/reearth/reearth-cms/server/internal/usecase" @@ -25,6 +26,9 @@ import ( "golang.org/x/exp/slices" ) +const maxPerPage = 100 +const defaultPerPage int64 = 50 + type Item struct { repos *repo.Container gateways *gateway.Container @@ -1179,3 +1183,81 @@ func (i Item) getReferencedItems(ctx context.Context, fields []*item.Field) ([]i } return i.repos.Item.FindByIDs(ctx, ids, nil) } + +// ItemsAsCSV exports items data in content to csv file by modelID. +func (i Item) ItemsAsCSV(ctx context.Context, schemaPackage *schema.Package, page *int, perPage *int, operator *usecase.Operator) (interfaces.ExportItemsToCSVResponse, error) { + if operator.AcOperator.User == nil && operator.Integration == nil { + return interfaces.ExportItemsToCSVResponse{}, interfaces.ErrInvalidOperator + } + return Run1(ctx, operator, i.repos, Usecase().Transaction(), func(ctx context.Context) (interfaces.ExportItemsToCSVResponse, error) { + + // fromPagination + paginationOffset := fromPagination(page, perPage) + + items, _, err := i.repos.Item.FindBySchema(ctx, schemaPackage.Schema().ID(), nil, nil, paginationOffset) + if err != nil { + return interfaces.ExportItemsToCSVResponse{}, err + } + + pr, pw := io.Pipe() + err = csvFromItems(pw, items, schemaPackage.Schema()) + if err != nil { + return interfaces.ExportItemsToCSVResponse{}, err + } + + return interfaces.ExportItemsToCSVResponse{ + PipeReader: pr, + }, nil + }) +} + +// ItemsAsGeoJSON converts items to Geo JSON type given a model ID +func (i Item) ItemsAsGeoJSON(ctx context.Context, schemaPackage *schema.Package, page *int, perPage *int, operator *usecase.Operator) (interfaces.ExportItemsToGeoJSONResponse, error) { + + if operator.AcOperator.User == nil && operator.Integration == nil { + return interfaces.ExportItemsToGeoJSONResponse{}, interfaces.ErrInvalidOperator + } + + return Run1(ctx, operator, i.repos, Usecase().Transaction(), func(ctx context.Context) (interfaces.ExportItemsToGeoJSONResponse, error) { + + // fromPagination + paginationOffset := fromPagination(page, perPage) + + items, _, err := i.repos.Item.FindBySchema(ctx, schemaPackage.Schema().ID(), nil, nil, paginationOffset) + if err != nil { + return interfaces.ExportItemsToGeoJSONResponse{}, err + } + + featureCollections, err := featureCollectionFromItems(items, schemaPackage.Schema()) + if err != nil { + return interfaces.ExportItemsToGeoJSONResponse{}, err + } + + return interfaces.ExportItemsToGeoJSONResponse{ + FeatureCollections: featureCollections, + }, nil + }) +} + +func fromPagination(page, perPage *int) *usecasex.Pagination { + p := int64(1) + if page != nil && *page > 0 { + p = int64(*page) + } + + pp := defaultPerPage + if perPage != nil { + if ppr := *perPage; 1 <= ppr { + if ppr > maxPerPage { + pp = int64(maxPerPage) + } else { + pp = int64(ppr) + } + } + } + + return usecasex.OffsetPagination{ + Offset: (p - 1) * pp, + Limit: pp, + }.Wrap() +} diff --git a/server/internal/usecase/interactor/item_export.go b/server/internal/usecase/interactor/item_export.go new file mode 100644 index 0000000000..cae4952807 --- /dev/null +++ b/server/internal/usecase/interactor/item_export.go @@ -0,0 +1,62 @@ +package interactor + +import ( + "encoding/csv" + "io" + + "github.com/labstack/gommon/log" + "github.com/reearth/reearth-cms/server/pkg/integrationapi" + "github.com/reearth/reearth-cms/server/pkg/item" + "github.com/reearth/reearth-cms/server/pkg/schema" + "github.com/reearth/reearthx/i18n" + "github.com/reearth/reearthx/rerror" + "github.com/samber/lo" +) + +var ( + pointFieldIsNotSupportedError = rerror.NewE(i18n.T("point type is not supported in any geometry field in this model")) +) + +// GeoJSON +func featureCollectionFromItems(ver item.VersionedList, s *schema.Schema) (*integrationapi.FeatureCollection, error) { + return integrationapi.FeatureCollectionFromItems(ver, s) +} + +// CSV +func csvFromItems(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) error { + if !s.IsPointFieldSupported() { + return pointFieldIsNotSupportedError + } + go handleCSVGeneration(pw, l, s) + return nil +} +func handleCSVGeneration(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) { + err := generateCSV(pw, l, s) + if err != nil { + log.Errorf("failed to generate CSV: %v", err) + _ = pw.CloseWithError(err) + } else { + _ = pw.Close() + } +} +func generateCSV(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) error { + w := csv.NewWriter(pw) + defer w.Flush() + headers := integrationapi.BuildCSVHeaders(s) + if err := w.Write(headers); err != nil { + return err + } + nonGeoFields := lo.Filter(s.Fields(), func(f *schema.Field, _ int) bool { + return !f.IsGeometryField() + }) + for _, ver := range l { + row, ok := integrationapi.RowFromItem(ver.Value(), nonGeoFields) + if ok { + if err := w.Write(row); err != nil { + return err + } + } + } + + return w.Error() +} diff --git a/server/internal/usecase/interactor/item_export_test.go b/server/internal/usecase/interactor/item_export_test.go new file mode 100644 index 0000000000..67e3af31b7 --- /dev/null +++ b/server/internal/usecase/interactor/item_export_test.go @@ -0,0 +1,97 @@ +package interactor + +import ( + "io" + "testing" + + "github.com/reearth/reearth-cms/server/pkg/id" + "github.com/reearth/reearth-cms/server/pkg/item" + "github.com/reearth/reearth-cms/server/pkg/schema" + "github.com/reearth/reearth-cms/server/pkg/value" + "github.com/reearth/reearth-cms/server/pkg/version" + "github.com/reearth/reearthx/account/accountdomain" + "github.com/reearth/reearthx/util" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" +) + +func TestCSVFromItems(t *testing.T) { + iid := id.NewItemID() + sid := id.NewSchemaID() + mid := id.NewModelID() + tid := id.NewThreadID() + pid := id.NewProjectID() + gst := schema.GeometryObjectSupportedTypeList{schema.GeometryObjectSupportedTypePoint, schema.GeometryObjectSupportedTypeLineString} + gest := schema.GeometryEditorSupportedTypeList{schema.GeometryEditorSupportedTypePoint, schema.GeometryEditorSupportedTypeLineString} + sf1 := schema.NewField(schema.NewGeometryObject(gst).TypeProperty()).NewID().Name("geo1").Key(id.RandomKey()).MustBuild() + sf3 := schema.NewField(schema.NewGeometryEditor(gest).TypeProperty()).NewID().Name("geo2").Key(id.RandomKey()).MustBuild() + in4, _ := schema.NewInteger(lo.ToPtr(int64(1)), lo.ToPtr(int64(100))) + tp4 := in4.TypeProperty() + sf4 := schema.NewField(tp4).NewID().Name("age").Key(id.RandomKey()).MustBuild() + sf5 := schema.NewField(schema.NewBool().TypeProperty()).NewID().Name("isMarried").Key(id.RandomKey()).MustBuild() + s1 := schema.New().ID(sid).Fields([]*schema.Field{sf1, sf3, sf4, sf5}).Workspace(accountdomain.NewWorkspaceID()).Project(pid).MustBuild() + fi1 := item.NewField(sf1.ID(), value.TypeGeometryObject.Value("{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}").AsMultiple(), nil) + fi2 := item.NewField(sf3.ID(), value.TypeGeometryEditor.Value("{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}").AsMultiple(), nil) + fi3 := item.NewField(sf4.ID(), value.TypeInteger.Value(30).AsMultiple(), nil) + fi4 := item.NewField(sf5.ID(), value.TypeBool.Value(true).AsMultiple(), nil) + i1 := item.New(). + ID(iid). + Schema(sid). + Project(pid). + Fields([]*item.Field{fi1, fi2, fi3, fi4}). + Model(mid). + Thread(tid). + MustBuild() + v1 := version.New() + vi1 := version.MustBeValue(v1, nil, version.NewRefs(version.Latest), util.Now(), i1) + // with geometry fields + ver1 := item.VersionedList{vi1} + _, pw := io.Pipe() + err := csvFromItems(pw, ver1, s1) + assert.Nil(t, err) + // no geometry fields + iid2 := id.NewItemID() + sid2 := id.NewSchemaID() + mid2 := id.NewModelID() + tid2 := id.NewThreadID() + sf2 := schema.NewField(schema.NewText(lo.ToPtr(10)).TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() + s2 := schema.New().ID(sid).Fields([]*schema.Field{sf2}).Workspace(accountdomain.NewWorkspaceID()).Project(pid).MustBuild() + i2 := item.New(). + ID(iid2). + Schema(sid2). + Project(pid). + Fields([]*item.Field{item.NewField(sf2.ID(), value.TypeText.Value("test").AsMultiple(), nil)}). + Model(mid2). + Thread(tid2). + MustBuild() + v2 := version.New() + vi2 := version.MustBeValue(v2, nil, version.NewRefs(version.Latest), util.Now(), i2) + ver2 := item.VersionedList{vi2} + expectErr2 := pointFieldIsNotSupportedError + _, pw1 := io.Pipe() + err = csvFromItems(pw1, ver2, s2) + assert.Equal(t, expectErr2, err) + // point field is not supported + iid3 := id.NewItemID() + sid3 := id.NewSchemaID() + mid3 := id.NewModelID() + tid3 := id.NewThreadID() + gst2 := schema.GeometryObjectSupportedTypeList{schema.GeometryObjectSupportedTypeLineString, schema.GeometryObjectSupportedTypePolygon} + sf6 := schema.NewField(schema.NewGeometryObject(gst2).TypeProperty()).NewID().Name("geo3").Key(id.RandomKey()).MustBuild() + s3 := schema.New().ID(sid).Fields([]*schema.Field{sf6}).Workspace(accountdomain.NewWorkspaceID()).Project(pid).MustBuild() + i3 := item.New(). + ID(iid3). + Schema(sid3). + Project(pid). + Fields([]*item.Field{item.NewField(sf6.ID(), value.TypeText.Value("{\n \"coordinates\": [\n [\n 139.65439725962517,\n 36.34793305387103\n ],\n [\n 139.61688622815393,\n 35.910803456352724\n ]\n ],\n \"type\": \"LineString\"\n}").AsMultiple(), nil)}). + Model(mid3). + Thread(tid3). + MustBuild() + v3 := version.New() + vi3 := version.MustBeValue(v3, nil, version.NewRefs(version.Latest), util.Now(), i3) + ver3 := item.VersionedList{vi3} + expectErr3 := pointFieldIsNotSupportedError + _, pw2 := io.Pipe() + err = csvFromItems(pw2, ver3, s3) + assert.Equal(t, expectErr3, err) +} diff --git a/server/internal/usecase/interactor/item_test.go b/server/internal/usecase/interactor/item_test.go index a8ce0df478..081cee6b4f 100644 --- a/server/internal/usecase/interactor/item_test.go +++ b/server/internal/usecase/interactor/item_test.go @@ -3,6 +3,7 @@ package interactor import ( "context" "errors" + "io" "testing" "time" @@ -11,6 +12,7 @@ import ( "github.com/reearth/reearth-cms/server/internal/usecase/interfaces" "github.com/reearth/reearth-cms/server/internal/usecase/repo" "github.com/reearth/reearth-cms/server/pkg/id" + "github.com/reearth/reearth-cms/server/pkg/integrationapi" "github.com/reearth/reearth-cms/server/pkg/item" "github.com/reearth/reearth-cms/server/pkg/model" "github.com/reearth/reearth-cms/server/pkg/project" @@ -22,6 +24,7 @@ import ( "github.com/reearth/reearthx/account/accountdomain/user" "github.com/reearth/reearthx/account/accountdomain/workspace" "github.com/reearth/reearthx/account/accountusecase" + "github.com/reearth/reearthx/i18n" "github.com/reearth/reearthx/rerror" "github.com/reearth/reearthx/usecasex" "github.com/reearth/reearthx/util" @@ -1082,3 +1085,338 @@ func TestWorkFlow(t *testing.T) { assert.NoError(t, err) assert.Equal(t, map[id.ItemID]item.Status{i.ID(): item.StatusPublic}, status) } + +func TestItem_ItemsAsCSV(t *testing.T) { + r := []workspace.Role{workspace.RoleReader, workspace.RoleWriter} + w := accountdomain.NewWorkspaceID() + prj := project.New().NewID().Workspace(w).RequestRoles(r).MustBuild() + + gst := schema.GeometryObjectSupportedTypeList{schema.GeometryObjectSupportedTypePoint, schema.GeometryObjectSupportedTypeLineString} + gest := schema.GeometryEditorSupportedTypeList{schema.GeometryEditorSupportedTypePoint, schema.GeometryEditorSupportedTypeLineString} + + // Geometry Object type + sid1 := id.NewSchemaID() + fid1 := id.NewFieldID() + sf1 := schema.NewField(schema.NewGeometryObject(gst).TypeProperty()).NewID().Name("geo1").Key(id.RandomKey()).ID(fid1).MustBuild() + s1 := schema.New().ID(sid1).Workspace(w).Project(prj.ID()).Fields(schema.FieldList{sf1}).MustBuild() + sp1 := schema.NewPackage(s1, nil, nil, nil) + m1 := model.New().NewID().Schema(s1.ID()).Key(id.RandomKey()).Project(s1.Project()).MustBuild() + fi1 := item.NewField(sf1.ID(), value.TypeGeometryObject.Value("{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}").AsMultiple(), nil) + fs1 := []*item.Field{fi1} + i1 := item.New().ID(id.NewItemID()).Schema(s1.ID()).Model(m1.ID()).Project(s1.Project()).Thread(id.NewThreadID()).Fields(fs1).MustBuild() + i1IDStr := i1.ID().String() + + // GeometryEditor type item + sid2 := id.NewSchemaID() + fid2 := id.NewFieldID() + sf2 := schema.NewField(schema.NewGeometryEditor(gest).TypeProperty()).NewID().Name("geo2").Key(id.RandomKey()).ID(fid2).MustBuild() + s2 := schema.New().ID(sid2).Workspace(accountdomain.NewWorkspaceID()).Project(prj.ID()).Fields(schema.FieldList{sf2}).MustBuild() + m2 := model.New().NewID().Schema(s2.ID()).Key(id.RandomKey()).Project(s2.Project()).MustBuild() + fi2 := item.NewField(sf2.ID(), value.TypeGeometryEditor.Value("{\"coordinates\": [[[ ],[138.90306434425662,36.33622175736386],[138.67187898370287,36.33622175736386],[138.67187898370287,36.11737907906834],[138.90306434425662,36.11737907906834]]],\"type\": \"Polygon\"}").AsMultiple(), nil) + fs2 := []*item.Field{fi2} + i2 := item.New().NewID().Schema(s2.ID()).Model(m2.ID()).Project(s2.Project()).Thread(id.NewThreadID()).Fields(fs2).MustBuild() + sp2 := schema.NewPackage(s2, nil, nil, nil) + + // integer type item + fid3 := id.NewFieldID() + in4, _ := schema.NewInteger(lo.ToPtr(int64(1)), lo.ToPtr(int64(100))) + tp4 := in4.TypeProperty() + sf3 := schema.NewField(tp4).NewID().Name("age").Key(id.RandomKey()).ID(fid3).MustBuild() + s3 := schema.New().ID(sid2).Workspace(accountdomain.NewWorkspaceID()).Project(prj.ID()).Fields(schema.FieldList{sf3}).MustBuild() + m3 := model.New().NewID().Schema(s3.ID()).Key(id.RandomKey()).Project(s3.Project()).MustBuild() + fs3 := []*item.Field{item.NewField(sf3.ID(), value.TypeReference.Value(nil).AsMultiple(), nil)} + i3 := item.New().NewID().Schema(s3.ID()).Model(m3.ID()).Project(s3.Project()).Thread(id.NewThreadID()).Fields(fs3).MustBuild() + sp3 := schema.NewPackage(s3, nil, nil, nil) + + page1 := 1 + perPage1 := 10 + + wid := accountdomain.NewWorkspaceID() + u := user.New().NewID().Email("aaa@bbb.com").Workspace(wid).Name("foo").MustBuild() + op := &usecase.Operator{ + AcOperator: &accountusecase.Operator{ + User: lo.ToPtr(u.ID()), + }, + } + + opUserNil := &usecase.Operator{ + AcOperator: &accountusecase.Operator{}, + } + ctx := context.Background() + + type args struct { + ctx context.Context + schemaPackage *schema.Package + page *int + perPage *int + op *usecase.Operator + } + tests := []struct { + name string + args args + seedsItems item.List + seedSchemas *schema.Schema + seedModels *model.Model + want []byte + wantError error + }{ + { + name: "success", + args: args{ + ctx: ctx, + schemaPackage: sp1, + page: &page1, + perPage: &perPage1, + op: op, + }, + seedsItems: item.List{i1}, + seedSchemas: s1, + seedModels: m1, + want: []byte("id,location_lat,location_lng\n" + i1IDStr + ",139.28179282584915,36.58570985749664\n"), + wantError: nil, + }, + { + name: "success geometry editor type", + args: args{ + ctx: ctx, + schemaPackage: sp2, + page: &page1, + perPage: &perPage1, + op: op, + }, + seedsItems: item.List{i2}, + seedSchemas: s2, + seedModels: m2, + want: []byte("id,location_lat,location_lng\n"), + wantError: nil, + }, + { + name: "error point type is not supported in any geometry field non geometry field", + args: args{ + ctx: ctx, + schemaPackage: sp3, + page: &page1, + perPage: &perPage1, + op: op, + }, + seedsItems: item.List{i3}, + seedSchemas: s3, + seedModels: m3, + want: []byte(nil), + wantError: pointFieldIsNotSupportedError, + }, + { + name: "error operator user is nil", + args: args{ + ctx: ctx, + schemaPackage: sp3, + page: &page1, + perPage: &perPage1, + op: opUserNil, + }, + want: []byte(nil), + wantError: interfaces.ErrInvalidOperator, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + db := memory.New() + for _, seed := range tt.seedsItems { + err := db.Item.Save(ctx, seed) + assert.NoError(t, err) + } + + if tt.seedSchemas != nil { + err := db.Schema.Save(ctx, tt.seedSchemas) + assert.NoError(t, err) + } + if tt.seedModels != nil { + err := db.Model.Save(ctx, tt.seedModels) + assert.NoError(t, err) + } + itemUC := NewItem(db, nil) + itemUC.ignoreEvent = true + + pr, err := itemUC.ItemsAsCSV(ctx, tt.args.schemaPackage, tt.args.page, tt.args.perPage, tt.args.op) + + var result []byte + if pr.PipeReader != nil { + result, _ = io.ReadAll(pr.PipeReader) + } + + assert.Equal(t, tt.want, result) + assert.Equal(t, tt.wantError, err) + }) + } +} + +func TestItem_ItemsAsGeoJSON(t *testing.T) { + r := []workspace.Role{workspace.RoleReader, workspace.RoleWriter} + w := accountdomain.NewWorkspaceID() + prj := project.New().NewID().Workspace(w).RequestRoles(r).MustBuild() + + gst := schema.GeometryObjectSupportedTypeList{schema.GeometryObjectSupportedTypePoint, schema.GeometryObjectSupportedTypeLineString} + gest := schema.GeometryEditorSupportedTypeList{schema.GeometryEditorSupportedTypePoint, schema.GeometryEditorSupportedTypeLineString} + + sid1 := id.NewSchemaID() + fid1 := id.NewFieldID() + sf1 := schema.NewField(schema.NewGeometryObject(gst).TypeProperty()).NewID().Name("geo1").Key(id.RandomKey()).ID(fid1).MustBuild() + s1 := schema.New().ID(sid1).Workspace(w).Project(prj.ID()).Fields(schema.FieldList{sf1}).MustBuild() + sp1 := schema.NewPackage(s1, nil, nil, nil) + m1 := model.New().NewID().Schema(s1.ID()).Key(id.RandomKey()).Project(s1.Project()).MustBuild() + fi1 := item.NewField(sf1.ID(), value.TypeGeometryObject.Value("{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}").AsMultiple(), nil) + fs1 := []*item.Field{fi1} + i1 := item.New().ID(id.NewItemID()).Schema(s1.ID()).Model(m1.ID()).Project(s1.Project()).Thread(id.NewThreadID()).Fields(fs1).MustBuild() + + v1 := version.New() + vi1 := version.MustBeValue(v1, nil, version.NewRefs(version.Latest), util.Now(), i1) + // with geometry fields + ver1 := item.VersionedList{vi1} + + fc1, _ := featureCollectionFromItems(ver1, s1) + + sid2 := id.NewSchemaID() + fid2 := id.NewFieldID() + sf2 := schema.NewField(schema.NewGeometryEditor(gest).TypeProperty()).NewID().Name("geo2").Key(id.RandomKey()).ID(fid2).MustBuild() + s2 := schema.New().ID(sid2).Workspace(accountdomain.NewWorkspaceID()).Project(prj.ID()).Fields(schema.FieldList{sf2}).MustBuild() + sp2 := schema.NewPackage(s2, nil, nil, nil) + m2 := model.New().NewID().Schema(s2.ID()).Key(id.RandomKey()).Project(s2.Project()).MustBuild() + fi2 := item.NewField(sf2.ID(), value.TypeGeometryEditor.Value("{\"coordinates\": [[[138.90306434425662,36.11737907906834],[138.90306434425662,36.33622175736386],[138.67187898370287,36.33622175736386],[138.67187898370287,36.11737907906834],[138.90306434425662,36.11737907906834]]],\"type\": \"Polygon\"}").AsMultiple(), nil) + fs2 := []*item.Field{fi2} + i2 := item.New().NewID().Schema(s2.ID()).Model(m2.ID()).Project(s2.Project()).Thread(id.NewThreadID()).Fields(fs2).MustBuild() + v2 := version.New() + vi2 := version.MustBeValue(v2, nil, version.NewRefs(version.Latest), util.Now(), i2) + + ver2 := item.VersionedList{vi2} + fc2, _ := featureCollectionFromItems(ver2, s2) + + fid3 := id.NewFieldID() + in4, _ := schema.NewInteger(lo.ToPtr(int64(1)), lo.ToPtr(int64(100))) + tp4 := in4.TypeProperty() + sf3 := schema.NewField(tp4).NewID().Name("age").Key(id.RandomKey()).ID(fid3).MustBuild() + s3 := schema.New().ID(sid2).Workspace(accountdomain.NewWorkspaceID()).Project(prj.ID()).Fields(schema.FieldList{sf3}).MustBuild() + sp3 := schema.NewPackage(s3, nil, nil, nil) + m3 := model.New().NewID().Schema(s3.ID()).Key(id.RandomKey()).Project(s3.Project()).MustBuild() + fs3 := []*item.Field{item.NewField(sf3.ID(), value.TypeReference.Value(nil).AsMultiple(), nil)} + i3 := item.New().NewID().Schema(s3.ID()).Model(m3.ID()).Project(s3.Project()).Thread(id.NewThreadID()).Fields(fs3).MustBuild() + + page1 := 1 + perPage1 := 10 + + wid := accountdomain.NewWorkspaceID() + u := user.New().NewID().Email("aaa@bbb.com").Workspace(wid).Name("foo").MustBuild() + op := &usecase.Operator{ + AcOperator: &accountusecase.Operator{ + User: lo.ToPtr(u.ID()), + }, + } + + opUserNil := &usecase.Operator{ + AcOperator: &accountusecase.Operator{}, + } + + type args struct { + ctx context.Context + schemaPackage *schema.Package + page *int + perPage *int + op *usecase.Operator + } + tests := []struct { + name string + args args + seedsItems item.List + seedSchemas *schema.Schema + seedModels *model.Model + want *integrationapi.FeatureCollection + wantError error + }{ + { + name: "success", + args: args{ + ctx: context.Background(), + schemaPackage: sp1, + page: &page1, + perPage: &perPage1, + op: op, + }, + seedsItems: item.List{i1}, + seedSchemas: s1, + seedModels: m1, + want: fc1, + wantError: nil, + }, + { + name: "success geometry editor type", + args: args{ + ctx: context.Background(), + schemaPackage: sp2, + page: &page1, + perPage: &perPage1, + op: op, + }, + seedsItems: item.List{i2}, + seedSchemas: s2, + seedModels: m2, + want: fc2, + wantError: nil, + }, + { + name: "error no geometry field in this model / integer", + args: args{ + ctx: context.Background(), + schemaPackage: sp3, + page: &page1, + perPage: &perPage1, + op: op, + }, + seedsItems: item.List{i3}, + seedSchemas: s3, + seedModels: m3, + want: nil, + wantError: rerror.NewE(i18n.T("no geometry field in this model")), + }, + { + name: "error operator user is nil", + args: args{ + ctx: context.Background(), + schemaPackage: sp3, + page: &page1, + perPage: &perPage1, + op: opUserNil, + }, + want: nil, + wantError: interfaces.ErrInvalidOperator, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + db := memory.New() + + for _, seed := range tt.seedsItems { + err := db.Item.Save(ctx, seed) + assert.NoError(t, err) + } + + if tt.seedSchemas != nil { + err := db.Schema.Save(ctx, tt.seedSchemas) + assert.NoError(t, err) + } + if tt.seedModels != nil { + err := db.Model.Save(ctx, tt.seedModels) + assert.NoError(t, err) + } + itemUC := NewItem(db, nil) + itemUC.ignoreEvent = true + result, err := itemUC.ItemsAsGeoJSON(ctx, tt.args.schemaPackage, tt.args.page, tt.args.perPage, tt.args.op) + + assert.Equal(t, tt.want, result.FeatureCollections) + assert.Equal(t, tt.wantError, err) + }) + } +} diff --git a/server/internal/usecase/interfaces/item.go b/server/internal/usecase/interfaces/item.go index 425706c1d9..3935c820b3 100644 --- a/server/internal/usecase/interfaces/item.go +++ b/server/internal/usecase/interfaces/item.go @@ -2,10 +2,12 @@ package interfaces import ( "context" + "io" "time" "github.com/reearth/reearth-cms/server/internal/usecase" "github.com/reearth/reearth-cms/server/pkg/id" + "github.com/reearth/reearth-cms/server/pkg/integrationapi" "github.com/reearth/reearth-cms/server/pkg/item" "github.com/reearth/reearth-cms/server/pkg/model" "github.com/reearth/reearth-cms/server/pkg/schema" @@ -80,6 +82,15 @@ type ImportItemsResponse struct { NewFields schema.FieldList } +// ExportItemsToCSVResponse contains exported csv data from items +type ExportItemsToCSVResponse struct { + PipeReader *io.PipeReader +} + +type ExportItemsToGeoJSONResponse struct { + FeatureCollections *integrationapi.FeatureCollection +} + type Item interface { FindByID(context.Context, id.ItemID, *usecase.Operator) (item.Versioned, error) FindPublicByID(context.Context, id.ItemID, *usecase.Operator) (item.Versioned, error) @@ -99,4 +110,8 @@ type Item interface { Publish(context.Context, id.ItemIDList, *usecase.Operator) (item.VersionedList, error) Unpublish(context.Context, id.ItemIDList, *usecase.Operator) (item.VersionedList, error) Import(context.Context, ImportItemsParam, *usecase.Operator) (ImportItemsResponse, error) + // ItemsAsCSV exports items data in content to csv file by schema package. + ItemsAsCSV(context.Context, *schema.Package, *int, *int, *usecase.Operator) (ExportItemsToCSVResponse, error) + // ItemsAsGeoJSON converts items to Geo JSON type given thge schema package. + ItemsAsGeoJSON(context.Context, *schema.Package, *int, *int, *usecase.Operator) (ExportItemsToGeoJSONResponse, error) } From 4c1a6890e5aa038f220dd3445b2644306a8b9661 Mon Sep 17 00:00:00 2001 From: jasonkarel Date: Wed, 8 Jan 2025 15:26:22 +0700 Subject: [PATCH 2/4] fix comments of the function --- server/internal/usecase/interactor/item.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/internal/usecase/interactor/item.go b/server/internal/usecase/interactor/item.go index c4c06bfd1e..8955c3dc25 100644 --- a/server/internal/usecase/interactor/item.go +++ b/server/internal/usecase/interactor/item.go @@ -1184,7 +1184,7 @@ func (i Item) getReferencedItems(ctx context.Context, fields []*item.Field) ([]i return i.repos.Item.FindByIDs(ctx, ids, nil) } -// ItemsAsCSV exports items data in content to csv file by modelID. +// ItemsAsCSV exports items data in content to csv file by schema package. func (i Item) ItemsAsCSV(ctx context.Context, schemaPackage *schema.Package, page *int, perPage *int, operator *usecase.Operator) (interfaces.ExportItemsToCSVResponse, error) { if operator.AcOperator.User == nil && operator.Integration == nil { return interfaces.ExportItemsToCSVResponse{}, interfaces.ErrInvalidOperator @@ -1211,7 +1211,7 @@ func (i Item) ItemsAsCSV(ctx context.Context, schemaPackage *schema.Package, pag }) } -// ItemsAsGeoJSON converts items to Geo JSON type given a model ID +// ItemsAsGeoJSON converts items to Geo JSON type given the schema package func (i Item) ItemsAsGeoJSON(ctx context.Context, schemaPackage *schema.Package, page *int, perPage *int, operator *usecase.Operator) (interfaces.ExportItemsToGeoJSONResponse, error) { if operator.AcOperator.User == nil && operator.Integration == nil { From 2585e32a5fe2029534c3c993395fbd5b5fc29708 Mon Sep 17 00:00:00 2001 From: jasonkarel Date: Wed, 8 Jan 2025 16:13:12 +0700 Subject: [PATCH 3/4] delete old item export file --- .../adapter/integration/item_export.go | 70 ------------ .../adapter/integration/item_export_test.go | 100 ------------------ 2 files changed, 170 deletions(-) delete mode 100644 server/internal/adapter/integration/item_export.go delete mode 100644 server/internal/adapter/integration/item_export_test.go diff --git a/server/internal/adapter/integration/item_export.go b/server/internal/adapter/integration/item_export.go deleted file mode 100644 index fcc4ee77ee..0000000000 --- a/server/internal/adapter/integration/item_export.go +++ /dev/null @@ -1,70 +0,0 @@ -package integration - -import ( - "encoding/csv" - "io" - - "github.com/reearth/reearth-cms/server/pkg/integrationapi" - "github.com/reearth/reearth-cms/server/pkg/item" - "github.com/reearth/reearth-cms/server/pkg/schema" - "github.com/reearth/reearthx/i18n" - "github.com/reearth/reearthx/log" - "github.com/reearth/reearthx/rerror" - "github.com/samber/lo" -) - -var ( - pointFieldIsNotSupportedError = rerror.NewE(i18n.T("point type is not supported in any geometry field in this model")) -) - -// GeoJSON -func featureCollectionFromItems(ver item.VersionedList, s *schema.Schema) (*integrationapi.FeatureCollection, error) { - return integrationapi.FeatureCollectionFromItems(ver, s) -} - -// CSV -func csvFromItems(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) error { - if !s.IsPointFieldSupported() { - return pointFieldIsNotSupportedError - } - - go handleCSVGeneration(pw, l, s) - - return nil -} - -func handleCSVGeneration(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) { - err := generateCSV(pw, l, s) - if err != nil { - log.Errorf("failed to generate CSV: %v", err) - _ = pw.CloseWithError(err) - } else { - _ = pw.Close() - } -} - -func generateCSV(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) error { - w := csv.NewWriter(pw) - defer w.Flush() - - headers := integrationapi.BuildCSVHeaders(s) - if err := w.Write(headers); err != nil { - return err - } - - nonGeoFields := lo.Filter(s.Fields(), func(f *schema.Field, _ int) bool { - return !f.IsGeometryField() - }) - - for _, ver := range l { - row, ok := integrationapi.RowFromItem(ver.Value(), nonGeoFields) - if ok { - if err := w.Write(row); err != nil { - return err - } - } - } - - return w.Error() -} - diff --git a/server/internal/adapter/integration/item_export_test.go b/server/internal/adapter/integration/item_export_test.go deleted file mode 100644 index 774f31abf7..0000000000 --- a/server/internal/adapter/integration/item_export_test.go +++ /dev/null @@ -1,100 +0,0 @@ -package integration - -import ( - "io" - "testing" - - "github.com/reearth/reearth-cms/server/pkg/id" - "github.com/reearth/reearth-cms/server/pkg/item" - "github.com/reearth/reearth-cms/server/pkg/schema" - "github.com/reearth/reearth-cms/server/pkg/value" - "github.com/reearth/reearth-cms/server/pkg/version" - "github.com/reearth/reearthx/account/accountdomain" - "github.com/reearth/reearthx/util" - "github.com/samber/lo" - "github.com/stretchr/testify/assert" -) - -func TestCSVFromItems(t *testing.T) { - iid := id.NewItemID() - sid := id.NewSchemaID() - mid := id.NewModelID() - tid := id.NewThreadID() - pid := id.NewProjectID() - gst := schema.GeometryObjectSupportedTypeList{schema.GeometryObjectSupportedTypePoint, schema.GeometryObjectSupportedTypeLineString} - gest := schema.GeometryEditorSupportedTypeList{schema.GeometryEditorSupportedTypePoint, schema.GeometryEditorSupportedTypeLineString} - sf1 := schema.NewField(schema.NewGeometryObject(gst).TypeProperty()).NewID().Name("geo1").Key(id.RandomKey()).MustBuild() - sf3 := schema.NewField(schema.NewGeometryEditor(gest).TypeProperty()).NewID().Name("geo2").Key(id.RandomKey()).MustBuild() - in4, _ := schema.NewInteger(lo.ToPtr(int64(1)), lo.ToPtr(int64(100))) - tp4 := in4.TypeProperty() - sf4 := schema.NewField(tp4).NewID().Name("age").Key(id.RandomKey()).MustBuild() - sf5 := schema.NewField(schema.NewBool().TypeProperty()).NewID().Name("isMarried").Key(id.RandomKey()).MustBuild() - s1 := schema.New().ID(sid).Fields([]*schema.Field{sf1, sf3, sf4, sf5}).Workspace(accountdomain.NewWorkspaceID()).Project(pid).MustBuild() - fi1 := item.NewField(sf1.ID(), value.TypeGeometryObject.Value("{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}").AsMultiple(), nil) - fi2 := item.NewField(sf3.ID(), value.TypeGeometryEditor.Value("{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}").AsMultiple(), nil) - fi3 := item.NewField(sf4.ID(), value.TypeInteger.Value(30).AsMultiple(), nil) - fi4 := item.NewField(sf5.ID(), value.TypeBool.Value(true).AsMultiple(), nil) - i1 := item.New(). - ID(iid). - Schema(sid). - Project(pid). - Fields([]*item.Field{fi1, fi2, fi3, fi4}). - Model(mid). - Thread(tid). - MustBuild() - v1 := version.New() - vi1 := version.MustBeValue(v1, nil, version.NewRefs(version.Latest), util.Now(), i1) - - // with geometry fields - ver1 := item.VersionedList{vi1} - _, pw := io.Pipe() - err := csvFromItems(pw, ver1, s1) - assert.Nil(t, err) - - // no geometry fields - iid2 := id.NewItemID() - sid2 := id.NewSchemaID() - mid2 := id.NewModelID() - tid2 := id.NewThreadID() - sf2 := schema.NewField(schema.NewText(lo.ToPtr(10)).TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() - s2 := schema.New().ID(sid).Fields([]*schema.Field{sf2}).Workspace(accountdomain.NewWorkspaceID()).Project(pid).MustBuild() - i2 := item.New(). - ID(iid2). - Schema(sid2). - Project(pid). - Fields([]*item.Field{item.NewField(sf2.ID(), value.TypeText.Value("test").AsMultiple(), nil)}). - Model(mid2). - Thread(tid2). - MustBuild() - v2 := version.New() - vi2 := version.MustBeValue(v2, nil, version.NewRefs(version.Latest), util.Now(), i2) - ver2 := item.VersionedList{vi2} - expectErr2 := pointFieldIsNotSupportedError - _, pw1 := io.Pipe() - err = csvFromItems(pw1, ver2, s2) - assert.Equal(t, expectErr2, err) - - // point field is not supported - iid3 := id.NewItemID() - sid3 := id.NewSchemaID() - mid3 := id.NewModelID() - tid3 := id.NewThreadID() - gst2 := schema.GeometryObjectSupportedTypeList{schema.GeometryObjectSupportedTypeLineString, schema.GeometryObjectSupportedTypePolygon} - sf6 := schema.NewField(schema.NewGeometryObject(gst2).TypeProperty()).NewID().Name("geo3").Key(id.RandomKey()).MustBuild() - s3 := schema.New().ID(sid).Fields([]*schema.Field{sf6}).Workspace(accountdomain.NewWorkspaceID()).Project(pid).MustBuild() - i3 := item.New(). - ID(iid3). - Schema(sid3). - Project(pid). - Fields([]*item.Field{item.NewField(sf6.ID(), value.TypeText.Value("{\n \"coordinates\": [\n [\n 139.65439725962517,\n 36.34793305387103\n ],\n [\n 139.61688622815393,\n 35.910803456352724\n ]\n ],\n \"type\": \"LineString\"\n}").AsMultiple(), nil)}). - Model(mid3). - Thread(tid3). - MustBuild() - v3 := version.New() - vi3 := version.MustBeValue(v3, nil, version.NewRefs(version.Latest), util.Now(), i3) - ver3 := item.VersionedList{vi3} - expectErr3 := pointFieldIsNotSupportedError - _, pw2 := io.Pipe() - err = csvFromItems(pw2, ver3, s3) - assert.Equal(t, expectErr3, err) -} \ No newline at end of file From 8a9b4cff5eb3701daddbd62be05c617fbd07d463 Mon Sep 17 00:00:00 2001 From: jasonkarel Date: Wed, 8 Jan 2025 16:22:32 +0700 Subject: [PATCH 4/4] add error return consistency --- server/internal/adapter/integration/item.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/server/internal/adapter/integration/item.go b/server/internal/adapter/integration/item.go index 237273f943..479db985ec 100644 --- a/server/internal/adapter/integration/item.go +++ b/server/internal/adapter/integration/item.go @@ -75,7 +75,10 @@ func (s *Server) ItemsAsGeoJSON(ctx context.Context, request ItemsAsGeoJSONReque schemaPackage, err := uc.Schema.FindByModel(ctx, request.ModelId, op) if err != nil { - return ItemsAsGeoJSON404Response{}, err + if errors.Is(err, rerror.ErrNotFound) { + return ItemsAsGeoJSON404Response{}, err + } + return ItemsAsGeoJSON400Response{}, err } featureCollections, err := uc.Item.ItemsAsGeoJSON(ctx, schemaPackage, request.Params.Page, request.Params.PerPage, op) @@ -98,7 +101,10 @@ func (s *Server) ItemsAsCSV(ctx context.Context, request ItemsAsCSVRequestObject schemaPackage, err := uc.Schema.FindByModel(ctx, request.ModelId, op) if err != nil { - return ItemsAsCSV404Response{}, err + if errors.Is(err, rerror.ErrNotFound) { + return ItemsAsCSV404Response{}, err + } + return ItemsAsCSV400Response{}, err } pr, err := uc.Item.ItemsAsCSV(ctx, schemaPackage, request.Params.Page, request.Params.PerPage, op) @@ -205,6 +211,9 @@ func (s *Server) ItemsWithProjectAsGeoJSON(ctx context.Context, request ItemsWit schemaPackage, err := uc.Schema.FindByModel(ctx, m.ID(), op) if err != nil { + if errors.Is(err, rerror.ErrNotFound) { + return ItemsWithProjectAsGeoJSON404Response{}, err + } return ItemsWithProjectAsGeoJSON400Response{}, err } @@ -244,6 +253,9 @@ func (s *Server) ItemsWithProjectAsCSV(ctx context.Context, request ItemsWithPro schemaPackage, err := uc.Schema.FindByModel(ctx, m.ID(), op) if err != nil { + if errors.Is(err, rerror.ErrNotFound) { + return ItemsWithProjectAsCSV404Response{}, err + } return ItemsWithProjectAsCSV400Response{}, err }