diff --git a/.github/workflows/base-docker-publish.yml b/.github/workflows/base-docker-publish.yml index 5c3ee7b6f..635ccb536 100644 --- a/.github/workflows/base-docker-publish.yml +++ b/.github/workflows/base-docker-publish.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Go 1.19.2 uses: actions/setup-go@v4 diff --git a/.github/workflows/chart-release.yml b/.github/workflows/chart-release.yml index 473f1766e..09fdf03a9 100644 --- a/.github/workflows/chart-release.yml +++ b/.github/workflows/chart-release.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1969c1297..7d908eefb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: github_token: ${{ github.token }} - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get tools dependencies run: make tools diff --git a/api/converter/from_pb.go b/api/converter/from_pb.go index 1aef86d37..b05098ba2 100644 --- a/api/converter/from_pb.go +++ b/api/converter/from_pb.go @@ -442,6 +442,12 @@ func fromStyle(pbStyle *api.Operation_Style) (*operations.Style, error) { if err != nil { return nil, err } + createdAtMapByActor, err := fromCreatedAtMapByActor( + pbStyle.CreatedAtMapByActor, + ) + if err != nil { + return nil, err + } executedAt, err := fromTimeTicket(pbStyle.ExecutedAt) if err != nil { return nil, err @@ -450,6 +456,7 @@ func fromStyle(pbStyle *api.Operation_Style) (*operations.Style, error) { parentCreatedAt, from, to, + createdAtMapByActor, pbStyle.Attributes, executedAt, ), nil diff --git a/api/converter/to_pb.go b/api/converter/to_pb.go index beec819a5..4772936f6 100644 --- a/api/converter/to_pb.go +++ b/api/converter/to_pb.go @@ -356,11 +356,12 @@ func toEdit(e *operations.Edit) (*api.Operation_Edit_, error) { func toStyle(style *operations.Style) (*api.Operation_Style_, error) { return &api.Operation_Style_{ Style: &api.Operation_Style{ - ParentCreatedAt: ToTimeTicket(style.ParentCreatedAt()), - From: toTextNodePos(style.From()), - To: toTextNodePos(style.To()), - Attributes: style.Attributes(), - ExecutedAt: ToTimeTicket(style.ExecutedAt()), + ParentCreatedAt: ToTimeTicket(style.ParentCreatedAt()), + From: toTextNodePos(style.From()), + To: toTextNodePos(style.To()), + CreatedAtMapByActor: toCreatedAtMapByActor(style.CreatedAtMapByActor()), + Attributes: style.Attributes(), + ExecutedAt: ToTimeTicket(style.ExecutedAt()), }, }, nil } diff --git a/design/retention.md b/design/retention.md index 54ce78c25..06262385f 100644 --- a/design/retention.md +++ b/design/retention.md @@ -56,7 +56,7 @@ In conclusion, when a snapshot is created, it should not simply delete the chang This can be expressed as a picture above(when SnapshotInterval=10, SnapshotThreshold=5). Assuming that there are a series of C (changes) in chronological order (ServerSeq), S (snapshots) are being created at intervals of 10 and the synchronized ServerSeq is being recorded in SyncedSeq. At this time, there may be a situation where Client A's synchronization is delayed for some reason. -In this situation, if all previous Cs are deleted when S2 is created, Client A must pull C19 and C20 for synchronization, but it is already deleted and does not exist. This is the reason why the previous changes are deleted based on the minimum synced ServerSeq in the actual implementation. +In this situation, if all previous Cs are deleted when S3 is created, Client A must pull C19 and C20 for synchronization, but it is already deleted and does not exist. This is the reason why the previous changes are deleted based on the minimum synced ServerSeq in the actual implementation. ### How it was implemented as code diff --git a/pkg/document/crdt/rga_tree_split.go b/pkg/document/crdt/rga_tree_split.go index cabb68b0d..54ab7306d 100644 --- a/pkg/document/crdt/rga_tree_split.go +++ b/pkg/document/crdt/rga_tree_split.go @@ -262,6 +262,12 @@ func (s *RGATreeSplitNode[V]) Remove(removedAt *time.Ticket, latestCreatedAt *ti return false } +// canStyle checks if node is able to set style. +func (s *RGATreeSplitNode[V]) canStyle(editedAt *time.Ticket, latestCreatedAt *time.Ticket) bool { + return !s.createdAt().After(latestCreatedAt) && + (s.removedAt == nil || editedAt.After(s.removedAt)) +} + // Value returns the value of this node. func (s *RGATreeSplitNode[V]) Value() V { return s.value diff --git a/pkg/document/crdt/text.go b/pkg/document/crdt/text.go index 148f3d959..be33ce9be 100644 --- a/pkg/document/crdt/text.go +++ b/pkg/document/crdt/text.go @@ -250,28 +250,58 @@ func (t *Text) Edit( func (t *Text) Style( from, to *RGATreeSplitNodePos, + latestCreatedAtMapByActor map[string]*time.Ticket, attributes map[string]string, executedAt *time.Ticket, -) error { +) (map[string]*time.Ticket, error) { // 01. Split nodes with from and to _, toRight, err := t.rgaTreeSplit.findNodeWithSplit(to, executedAt) if err != nil { - return err + return nil, err } _, fromRight, err := t.rgaTreeSplit.findNodeWithSplit(from, executedAt) if err != nil { - return err + return nil, err } // 02. style nodes between from and to nodes := t.rgaTreeSplit.findBetween(fromRight, toRight) + createdAtMapByActor := make(map[string]*time.Ticket) + var toBeStyled []*RGATreeSplitNode[*TextValue] + for _, node := range nodes { + actorIDHex := node.id.createdAt.ActorIDHex() + + var latestCreatedAt *time.Ticket + if len(latestCreatedAtMapByActor) == 0 { + latestCreatedAt = time.MaxTicket + } else { + createdAt, ok := latestCreatedAtMapByActor[actorIDHex] + if ok { + latestCreatedAt = createdAt + } else { + latestCreatedAt = time.InitialTicket + } + } + + if node.canStyle(executedAt, latestCreatedAt) { + latestCreatedAt = createdAtMapByActor[actorIDHex] + createdAt := node.id.createdAt + if latestCreatedAt == nil || createdAt.After(latestCreatedAt) { + createdAtMapByActor[actorIDHex] = createdAt + } + toBeStyled = append(toBeStyled, node) + } + } + + for _, node := range toBeStyled { val := node.value for key, value := range attributes { val.attrs.Set(key, value, executedAt) } } - return nil + + return createdAtMapByActor, nil } // Nodes returns the internal nodes of this Text. diff --git a/pkg/document/crdt/text_test.go b/pkg/document/crdt/text_test.go index e69ae829f..0b2ee5ed1 100644 --- a/pkg/document/crdt/text_test.go +++ b/pkg/document/crdt/text_test.go @@ -80,7 +80,7 @@ func TestText(t *testing.T) { assert.Equal(t, `[{"val":"Hello "},{"val":"Yorkie"}]`, text.Marshal()) fromPos, toPos, _ = text.CreateRange(0, 1) - err = text.Style(fromPos, toPos, map[string]string{"b": "1"}, ctx.IssueTimeTicket()) + _, err = text.Style(fromPos, toPos, nil, map[string]string{"b": "1"}, ctx.IssueTimeTicket()) assert.NoError(t, err) assert.Equal( t, diff --git a/pkg/document/json/text.go b/pkg/document/json/text.go index abac1becc..fc053e6f7 100644 --- a/pkg/document/json/text.go +++ b/pkg/document/json/text.go @@ -103,12 +103,14 @@ func (p *Text) Style(from, to int, attributes map[string]string) *Text { } ticket := p.context.IssueTimeTicket() - if err := p.Text.Style( + maxCreationMapByActor, err := p.Text.Style( fromPos, toPos, + nil, attributes, ticket, - ); err != nil { + ) + if err != nil { panic(err) } @@ -116,6 +118,7 @@ func (p *Text) Style(from, to int, attributes map[string]string) *Text { p.CreatedAt(), fromPos, toPos, + maxCreationMapByActor, attributes, ticket, )) diff --git a/pkg/document/operations/style.go b/pkg/document/operations/style.go index 069f8a4d1..5394bc9de 100644 --- a/pkg/document/operations/style.go +++ b/pkg/document/operations/style.go @@ -32,6 +32,10 @@ type Style struct { // to is the end point of the range to apply the style to. to *crdt.RGATreeSplitNodePos + // latestCreatedAtMapByActor is a map that stores the latest creation time + // by actor for the nodes included in the range to apply the style to. + latestCreatedAtMapByActor map[string]*time.Ticket + // attributes represents the text style. attributes map[string]string @@ -44,15 +48,17 @@ func NewStyle( parentCreatedAt *time.Ticket, from *crdt.RGATreeSplitNodePos, to *crdt.RGATreeSplitNodePos, + latestCreatedAtMapByActor map[string]*time.Ticket, attributes map[string]string, executedAt *time.Ticket, ) *Style { return &Style{ - parentCreatedAt: parentCreatedAt, - from: from, - to: to, - attributes: attributes, - executedAt: executedAt, + parentCreatedAt: parentCreatedAt, + from: from, + to: to, + latestCreatedAtMapByActor: latestCreatedAtMapByActor, + attributes: attributes, + executedAt: executedAt, } } @@ -64,7 +70,8 @@ func (e *Style) Execute(root *crdt.Root) error { return ErrNotApplicableDataType } - return obj.Style(e.from, e.to, e.attributes, e.executedAt) + _, err := obj.Style(e.from, e.to, e.latestCreatedAtMapByActor, e.attributes, e.executedAt) + return err } // From returns the start point of the editing range. @@ -96,3 +103,9 @@ func (e *Style) ParentCreatedAt() *time.Ticket { func (e *Style) Attributes() map[string]string { return e.attributes } + +// CreatedAtMapByActor returns the map that stores the latest creation time +// by actor for the nodes included in the range to apply the style to. +func (e *Style) CreatedAtMapByActor() map[string]*time.Ticket { + return e.latestCreatedAtMapByActor +} diff --git a/test/bench/document_bench_test.go b/test/bench/document_bench_test.go index 2f43393a0..2cf0dc666 100644 --- a/test/bench/document_bench_test.go +++ b/test/bench/document_bench_test.go @@ -470,6 +470,155 @@ func BenchmarkDocument(b *testing.B) { b.Run("object 10000", func(b *testing.B) { benchmarkObject(10000, b) }) + + b.Run("tree 100", func(b *testing.B) { + benchmarkTree(100, b) + }) + + b.Run("tree 1000", func(b *testing.B) { + benchmarkTree(1000, b) + }) + + b.Run("tree 10000", func(b *testing.B) { + benchmarkTree(10000, b) + }) + + b.Run("tree delete all 1000", func(b *testing.B) { + benchmarkTreeDeleteAll(1000, b) + }) + + b.Run("tree edit gc 100", func(b *testing.B) { + benchmarkTreeEditGC(100, b) + }) + + b.Run("tree edit gc 1000", func(b *testing.B) { + benchmarkTreeEditGC(1000, b) + }) + + b.Run("tree split gc 100", func(b *testing.B) { + benchmarkTreeSplitGC(100, b) + }) + + b.Run("tree split gc 1000", func(b *testing.B) { + benchmarkTreeSplitGC(1000, b) + }) + +} + +func benchmarkTree(cnt int, b *testing.B) { + for i := 0; i < b.N; i++ { + doc := document.New("d1") + + err := doc.Update(func(root *json.Object, p *presence.Presence) error { + tree := root.SetNewTree("t", &json.TreeNode{ + Type: "root", + Children: []json.TreeNode{{ + Type: "p", + Children: []json.TreeNode{}, + }}, + }) + for c := 1; c <= cnt; c++ { + tree.Edit(c, c, &json.TreeNode{Type: "text", Value: "a"}) + } + return nil + }) + assert.NoError(b, err) + } +} + +func benchmarkTreeDeleteAll(cnt int, b *testing.B) { + for i := 0; i < b.N; i++ { + doc := document.New("d1") + + err := doc.Update(func(root *json.Object, p *presence.Presence) error { + tree := root.SetNewTree("t", &json.TreeNode{ + Type: "root", + Children: []json.TreeNode{{ + Type: "p", + Children: []json.TreeNode{}, + }}, + }) + for c := 1; c <= cnt; c++ { + tree.Edit(c, c, &json.TreeNode{Type: "text", Value: "a"}) + } + return nil + }) + + err = doc.Update(func(root *json.Object, p *presence.Presence) error { + tree := root.GetTree("t") + tree.Edit(1, cnt+1) + + return nil + }) + assert.NoError(b, err) + } +} + +func benchmarkTreeEditGC(cnt int, b *testing.B) { + for i := 0; i < b.N; i++ { + doc := document.New("d1") + + err := doc.Update(func(root *json.Object, p *presence.Presence) error { + tree := root.SetNewTree("t", &json.TreeNode{ + Type: "root", + Children: []json.TreeNode{{ + Type: "p", + Children: []json.TreeNode{}, + }}, + }) + for c := 1; c <= cnt; c++ { + tree.Edit(c, c, &json.TreeNode{Type: "text", Value: "a"}) + } + return nil + }) + + err = doc.Update(func(root *json.Object, p *presence.Presence) error { + tree := root.GetTree("t") + for c := 1; c <= cnt; c++ { + tree.Edit(c, c+1, &json.TreeNode{Type: "text", Value: "b"}) + } + + return nil + }) + assert.NoError(b, err) + assert.Equal(b, cnt, doc.GarbageLen()) + assert.Equal(b, cnt, doc.GarbageCollect(time.MaxTicket)) + } +} + +func benchmarkTreeSplitGC(cnt int, b *testing.B) { + for i := 0; i < b.N; i++ { + doc := document.New("d1") + + var builder strings.Builder + for i := 0; i < cnt; i++ { + builder.WriteString("a") + } + err := doc.Update(func(root *json.Object, p *presence.Presence) error { + tree := root.SetNewTree("t", &json.TreeNode{ + Type: "root", + Children: []json.TreeNode{{ + Type: "p", + Children: []json.TreeNode{}, + }}, + }) + tree.Edit(1, 1, &json.TreeNode{Type: "text", Value: builder.String()}) + + return nil + }) + + err = doc.Update(func(root *json.Object, p *presence.Presence) error { + tree := root.GetTree("t") + for c := 1; c <= cnt; c++ { + tree.Edit(c, c+1, &json.TreeNode{Type: "text", Value: "b"}) + } + + return nil + }) + assert.NoError(b, err) + assert.Equal(b, cnt, doc.GarbageLen()) + assert.Equal(b, cnt, doc.GarbageCollect(time.MaxTicket)) + } } func benchmarkText(cnt int, b *testing.B) { diff --git a/test/integration/text_test.go b/test/integration/text_test.go index 3d1e0287f..b85be27d4 100644 --- a/test/integration/text_test.go +++ b/test/integration/text_test.go @@ -96,6 +96,43 @@ func TestText(t *testing.T) { syncClientsThenAssertEqual(t, []clientAndDocPair{{c1, d1}, {c2, d2}}) }) + t.Run("concurrent insertion and deletion test", func(t *testing.T) { + ctx := context.Background() + d1 := document.New(helper.TestDocKey(t)) + err := c1.Attach(ctx, d1) + assert.NoError(t, err) + + err = d1.Update(func(root *json.Object, p *presence.Presence) error { + root.SetNewText("k1").Edit(0, 0, "AB") + return nil + }, "set a new text by c1") + assert.NoError(t, err) + err = c1.Sync(ctx) + assert.NoError(t, err) + + d2 := document.New(helper.TestDocKey(t)) + err = c2.Attach(ctx, d2) + assert.NoError(t, err) + assert.Equal(t, `{"k1":[{"val":"AB"}]}`, d2.Marshal()) + + err = d1.Update(func(root *json.Object, p *presence.Presence) error { + root.GetText("k1").Edit(0, 2, "") + return nil + }) + assert.NoError(t, err) + assert.Equal(t, `{"k1":[]}`, d1.Marshal()) + + err = d2.Update(func(root *json.Object, p *presence.Presence) error { + root.GetText("k1").Edit(1, 1, "C") + return nil + }) + assert.NoError(t, err) + assert.Equal(t, `{"k1":[{"val":"A"},{"val":"C"},{"val":"B"}]}`, d2.Marshal()) + + syncClientsThenAssertEqual(t, []clientAndDocPair{{c1, d1}, {c2, d2}}) + assert.Equal(t, `{"k1":[{"val":"C"}]}`, d1.Marshal()) + }) + t.Run("rich text test", func(t *testing.T) { ctx := context.Background() d1 := document.New(helper.TestDocKey(t)) @@ -226,4 +263,45 @@ func TestText(t *testing.T) { assert.True(t, d1.Root().GetText("k1").CheckWeight()) assert.True(t, d2.Root().GetText("k1").CheckWeight()) }) + + // Peritext test + t.Run("ex2. concurrent formatting and insertion test", func(t *testing.T) { + ctx := context.Background() + d1 := document.New(helper.TestDocKey(t)) + err := c1.Attach(ctx, d1) + assert.NoError(t, err) + + err = d1.Update(func(root *json.Object, p *presence.Presence) error { + root.SetNewText("k1").Edit(0, 0, "The fox jumped.", nil) + return nil + }) + assert.NoError(t, err) + err = c1.Sync(ctx) + assert.NoError(t, err) + + d2 := document.New(helper.TestDocKey(t)) + err = c2.Attach(ctx, d2) + assert.NoError(t, err) + assert.Equal(t, `{"k1":[{"val":"The fox jumped."}]}`, d2.Marshal()) + + err = d1.Update(func(root *json.Object, p *presence.Presence) error { + root.GetText("k1").Style(0, 15, map[string]string{"b": "1"}) + return nil + }) + assert.NoError(t, err) + assert.Equal(t, `{"k1":[{"attrs":{"b":"1"},"val":"The fox jumped."}]}`, d1.Marshal()) + + err = d2.Update(func(root *json.Object, p *presence.Presence) error { + root.GetText("k1").Edit(4, 4, "brown ") + return nil + }) + assert.NoError(t, err) + assert.Equal(t, `{"k1":[{"val":"The "},{"val":"brown "},{"val":"fox jumped."}]}`, d2.Marshal()) + + syncClientsThenAssertEqual(t, []clientAndDocPair{{c1, d1}, {c2, d2}}) + assert.Equal(t, `{"k1":[{"attrs":{"b":"1"},"val":"The "},{"val":"brown "},{"attrs":{"b":"1"},"val":"fox jumped."}]}`, d1.Marshal()) + + // TODO(MoonGyu1): d1 and d2 should have the result below after applying mark operation + // assert.Equal(t, `{"k1":[{"attrs":{"b":"1"},"val":"The "},{"attrs":{"b":"1"},"val":"brown "},{"attrs":{"b":"1"},"val":"fox jumped."}]}`, d1.Marshal()) + }) }