Skip to content

Commit

Permalink
feat(ci,server,worker): implement schemata and items copy (#1331)
Browse files Browse the repository at this point in the history 
* add copy endpoint

* wip: server implementation

* update copy model function name

* wip: copier ci

* remove copy of the internal server files in worker docker image

* wip: worker

* wip: repo

* apply new changes from worker to server

* refactor changes type

* refactor copier in mongo

* feat: refactor copier main file

* add e2e test

* wip: server test cases

* wip: worker tests

* update TestSchema_CopyFrom

* requested changes copier

* add key to params

* add a log after data successfully copied

* improve triggerCopyEvent logging

* fix db URI

* add more unit tests
  • Loading branch information
@nourbalaha
nourbalaha authored Jan 15, 2025
1 parent c0a9ef2 commit 61a5e68
Show file tree
Hide file tree
Showing 32 changed files with 1,294 additions and 99 deletions.
106 changes: 106 additions & 0 deletions .github/workflows/build_copier.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
name: copier-build
on:
workflow_run:
workflows: [ci-worker]
types: [completed]
branches: [main, release]
concurrency:
group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch }}
cancel-in-progress: true

jobs:
info:
name: Collect information
runs-on: ubuntu-latest
if: github.event.workflow_run.conclusion != 'failure' && github.event.repository.full_name == 'reearth/reearth-cms' && (github.event.workflow_run.head_branch == 'release' || !startsWith(github.event.workflow_run.head_commit.message, 'v'))
outputs:
sha_short: ${{ steps.info.outputs.sha_short }}
new_tag: ${{ steps.info.outputs.new_tag }}
new_tag_short: ${{ steps.info.outputs.new_tag_short }}
name: ${{ steps.info.outputs.name }}
steps:
- name: checkout
uses: actions/checkout@v4
- name: Fetch tags
run: git fetch --prune --unshallow --tags
- name: Get info
id: info
# The tag name should be retrieved lazily, as tagging may be delayed.
env:
BRANCH: ${{ github.event.workflow_run.head_branch }}
run: |
echo "sha_short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
if [[ "$BRANCH" = "release" ]]; then
TAG=$(git tag --points-at HEAD)
if [[ ! -z "$TAG" ]]; then
echo "new_tag=$TAG" >> "$GITHUB_OUTPUT"
echo "new_tag_short=${TAG#v}" >> "$GITHUB_OUTPUT"
else
echo "name=rc" >> "$GITHUB_OUTPUT"
fi
else
echo "name=nightly" >> "$GITHUB_OUTPUT"
fi
- name: Show info
env:
SHA_SHORT: ${{ steps.info.outputs.sha_short }}
NEW_TAG: ${{ steps.info.outputs.new_tag }}
NEW_TAG_SHORT: ${{ steps.info.outputs.new_tag_short }}
NAME: ${{ steps.info.outputs.name }}
run: echo "sha_short=$SHA_SHORT, new_tag=$NEW_TAG, new_tag_short=$NEW_TAG_SHORT, name=$NAME"

docker:
name: Build and push Docker image
runs-on: ubuntu-latest
needs:
- info
if: needs.info.outputs.name || needs.info.outputs.new_tag
env:
IMAGE_NAME: reearth/reearth-cms-copier
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Get options
id: options
env:
TAG: ${{ needs.info.outputs.new_tag_short }}
NAME: ${{ needs.info.outputs.name }}
SHA: ${{ needs.info.outputs.sha_short }}
run: |
if [[ -n $TAG ]]; then
PLATFORMS=linux/amd64,linux/arm64
VERSION=$TAG
TAGS=$IMAGE_NAME:$TAG
if [[ ! $TAG =~ '-' ]]; then
TAGS+=,${IMAGE_NAME}:${TAG%.*}
TAGS+=,${IMAGE_NAME}:${TAG%%.*}
TAGS+=,${IMAGE_NAME}:latest
fi
else
PLATFORMS=linux/amd64
VERSION=$SHA
TAGS=$IMAGE_NAME:$NAME
fi
echo "platforms=$PLATFORMS" >> "$GITHUB_OUTPUT"
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
- name: Build and push docker image
uses: docker/build-push-action@v6
with:
context: ./worker
file: ./worker/copier.Dockerfile
platforms: ${{ steps.options.outputs.platforms }}
push: true
build-args: VERSION=${{ steps.options.outputs.version }}
tags: ${{ steps.options.outputs.tags }}
cache-from: type=gha
cache-to: type=gha,mode=max
53 changes: 53 additions & 0 deletions .github/workflows/build_worker.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,3 +123,56 @@ jobs:
tags: ${{ steps.options.outputs.tags }}
cache-from: type=gha
cache-to: type=gha,mode=max

docker_copier:
runs-on: ubuntu-latest
if: inputs.name || inputs.new_tag
env:
IMAGE_NAME: reearth/reearth-cms-copier
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Get options
id: options
env:
TAG: ${{ inputs.new_tag_short }}
NAME: ${{ inputs.name }}
SHA: ${{ inputs.sha_short }}
run: |
if [[ -n $TAG ]]; then
PLATFORMS=linux/amd64,linux/arm64
VERSION=$TAG
TAGS=$IMAGE_NAME:$TAG
if [[ ! $TAG =~ '-' ]]; then
TAGS+=,${IMAGE_NAME}:${TAG%.*}
TAGS+=,${IMAGE_NAME}:${TAG%%.*}
TAGS+=,${IMAGE_NAME}:latest
fi
else
PLATFORMS=linux/amd64
VERSION=$SHA
TAGS=$IMAGE_NAME:$NAME
fi
echo "platforms=$PLATFORMS" >> "$GITHUB_OUTPUT"
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
- name: Build and push docker image
uses: docker/build-push-action@v6
with:
context: ./worker
file: ./worker/copier.Dockerfile
platforms: ${{ steps.options.outputs.platforms }}
push: true
build-args: VERSION=${{ steps.options.outputs.version }}
tags: ${{ steps.options.outputs.tags }}
cache-from: type=gha
cache-to: type=gha,mode=max
85 changes: 85 additions & 0 deletions server/e2e/integration_model_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,91 @@ func TestIntegrationModelGetAPI(t *testing.T) {
obj.Value("lastModified").NotNull()
}

// POST /models/{modelId}/copy
func TestIntegrationModelCopy(t *testing.T) {
endpoint := "/api/models/{modelId}/copy"
e := StartServer(t, &app.Config{}, true, baseSeeder)

e.POST(endpoint, id.NewModelID()).
Expect().
Status(http.StatusUnauthorized)

e.POST(endpoint, id.NewModelID()).
WithHeader("authorization", "secret_abc").
Expect().
Status(http.StatusUnauthorized)

e.POST(endpoint, id.NewModelID()).
WithHeader("authorization", "Bearer secret_abc").
Expect().
Status(http.StatusUnauthorized)

oldModelId := mId1.String()
oldModel := e.GET("/api/models/{modelId}", oldModelId).
WithHeader("authorization", "Bearer "+secret).
Expect().
Status(http.StatusOK).
JSON().
Object()

newName := "new name"
newKey := id.RandomKey().Ref().StringRef()
newModel := e.POST(endpoint, oldModelId).
WithHeader("authorization", "Bearer "+secret).
WithJSON(map[string]interface{}{
"name": newName,
"key": newKey,
}).
Expect().
Status(http.StatusOK).
JSON().
Object()

newModel.
ContainsKey("id").
ContainsKey("projectId").
ContainsKey("schemaId").
ContainsKey("public").
ContainsKey("createdAt").
ContainsKey("updatedAt").
ContainsKey("key")

newModelID := newModel.Value("id").String()
newModelID.NotEqual(oldModelId)
copiedModel := e.GET("/api/models/{modelId}", newModelID.Raw()).
WithHeader("authorization", "Bearer "+secret).
Expect().
Status(http.StatusOK).
JSON().
Object()
copiedModel.
HasValue("id", newModelID.Raw()).
HasValue("projectId", oldModel.Value("projectId").String().Raw()).
HasValue("public", oldModel.Value("public").Boolean().Raw()).
HasValue("name", newName).
HasValue("key", newKey).
HasValue("description", oldModel.Value("description").String().Raw())

copiedModel.Value("schemaId").NotNull()
oldSchemaId := oldModel.Value("schemaId").String()
copiedSchemaId := copiedModel.Value("schemaId").String()
copiedSchemaId.NotEqual(oldSchemaId.Raw())

oldSchema := oldModel.Value("schema").Object()
copiedSchema := copiedModel.Value("schema").Object()
copiedSchema.Value("fields").Array().Length().IsEqual(oldSchema.Value("fields").Array().Length().Raw())
copiedSchema.Value("titleField").String().IsEqual(oldSchema.Value("titleField").String().Raw())

copiedModel.Value("metadataSchemaId").NotNull()
oldMetadataSchemaId := oldModel.Value("metadataSchemaId").String()
copiedMetadataSchemaId := copiedModel.Value("metadataSchemaId").String()
copiedMetadataSchemaId.NotEqual(oldMetadataSchemaId.Raw())

oldMetadataSchema := oldModel.Value("metadataSchema").Object()
copiedMetadataSchema := copiedModel.Value("metadataSchema").Object()
copiedMetadataSchema.Value("fields").Array().Length().IsEqual(oldMetadataSchema.Value("fields").Array().Length().Raw())
}

// PATCH /models/{modelId}
func TestIntegrationModelUpdateAPI(t *testing.T) {
endpoint := "/api/models/{modelId}"
Expand Down
32 changes: 32 additions & 0 deletions server/internal/adapter/integration/model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,38 @@ func (s *Server) ModelGet(ctx context.Context, request ModelGetRequestObject) (M
return ModelGet200JSONResponse(integrationapi.NewModel(m, sp, lastModified)), nil
}

func (s *Server) CopyModel(ctx context.Context, request CopyModelRequestObject) (CopyModelResponseObject, error) {
uc := adapter.Usecases(ctx)
op := adapter.Operator(ctx)

m, err := uc.Model.Copy(ctx, interfaces.CopyModelParam{
ModelId: request.ModelId,
Name: request.Body.Name,
Key: request.Body.Key,
}, op)
if err != nil {
if errors.Is(err, rerror.ErrNotFound) {
return CopyModel404Response{}, err
}
return CopyModel500Response{}, err
}

sp, err := uc.Schema.FindByModel(ctx, m.ID(), op)
if err != nil {
if errors.Is(err, rerror.ErrNotFound) {
return CopyModel404Response{}, err
}
return CopyModel500Response{}, err
}

lastModified, err := uc.Item.LastModifiedByModel(ctx, m.ID(), op)
if err != nil && !errors.Is(err, rerror.ErrNotFound) {
return CopyModel500Response{}, err
}

return CopyModel200JSONResponse(integrationapi.NewModel(m, sp, lastModified)), nil
}

func (s *Server) ModelGetWithProject(ctx context.Context, request ModelGetWithProjectRequestObject) (ModelGetWithProjectResponseObject, error) {
uc := adapter.Usecases(ctx)
op := adapter.Operator(ctx)
Expand Down
Loading

0 comments on commit 61a5e68

Please sign in to comment.