From 56fa208733c2de19541e552d90fa26f4aa06ef02 Mon Sep 17 00:00:00 2001 From: kirillston Date: Tue, 2 Jul 2024 12:09:57 +0200 Subject: [PATCH 01/71] GO-3192 Add test --- core/block/editor/basic/basic_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/core/block/editor/basic/basic_test.go b/core/block/editor/basic/basic_test.go index 62a0e91c8..d738ecdae 100644 --- a/core/block/editor/basic/basic_test.go +++ b/core/block/editor/basic/basic_test.go @@ -146,6 +146,24 @@ func TestBasic_Move(t *testing.T) { assert.Len(t, sb.NewState().Pick("4").Model().ChildrenIds, 1) }) + t.Run("incorrect table cell move", func(t *testing.T) { + // given + sb := smarttest.New("test") + sb.AddBlock(simple.New(&model.Block{Id: "test", ChildrenIds: []string{"table"}})). + AddBlock(simple.New(&model.Block{Id: "table", ChildrenIds: []string{"rows", "columns"}})). + AddBlock(simple.New(&model.Block{Id: "columns", ChildrenIds: []string{"column"}, Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableColumns}}})). + AddBlock(simple.New(&model.Block{Id: "column", ChildrenIds: []string{}, Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}})). + AddBlock(simple.New(&model.Block{Id: "rows", ChildrenIds: []string{"row"}, Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableRows}}})). + AddBlock(simple.New(&model.Block{Id: "row", ChildrenIds: []string{"cell"}, Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{IsHeader: false}}})). + AddBlock(simple.New(&model.Block{Id: "cell", ChildrenIds: []string{}})) + + b := NewBasic(sb, nil, converter.NewLayoutConverter()) + st := sb.NewState() + + err := b.Move(st, st, "rows", model.Block_Bottom, []string{"cell"}) + require.NoError(t, err) + require.NoError(t, sb.Apply(st)) + }) t.Run("header", func(t *testing.T) { sb := smarttest.New("test") sb.AddBlock(simple.New(&model.Block{Id: "test"})) From 44a3c95a909e49252a0e00120238b8821190f5e5 Mon Sep 17 00:00:00 2001 From: kirillston Date: Tue, 2 Jul 2024 16:35:01 +0200 Subject: [PATCH 02/71] GO-3192 Add table blocks move check --- core/block/editor/basic/basic.go | 26 ++++++ core/block/editor/basic/basic_test.go | 112 +++++++++++++++++++++----- core/block/editor/table/table.go | 22 +++-- 3 files changed, 134 insertions(+), 26 deletions(-) diff --git a/core/block/editor/basic/basic.go b/core/block/editor/basic/basic.go index 86a8d6d41..34ef14878 100644 --- a/core/block/editor/basic/basic.go +++ b/core/block/editor/basic/basic.go @@ -10,6 +10,7 @@ import ( "github.com/anyproto/anytype-heart/core/block/editor/converter" "github.com/anyproto/anytype-heart/core/block/editor/smartblock" "github.com/anyproto/anytype-heart/core/block/editor/state" + "github.com/anyproto/anytype-heart/core/block/editor/table" "github.com/anyproto/anytype-heart/core/block/editor/template" "github.com/anyproto/anytype-heart/core/block/restriction" "github.com/anyproto/anytype-heart/core/block/simple" @@ -98,6 +99,8 @@ type Updatable interface { Update(ctx session.Context, apply func(b simple.Block) error, blockIds ...string) (err error) } +var ErrCannotMoveTableBlocks = fmt.Errorf("can not move table blocks") + func NewBasic( sb smartblock.SmartBlock, objectStore objectstore.ObjectStore, @@ -236,6 +239,11 @@ func (bs *basic) Move(srcState, destState *state.State, targetBlockId string, po } } + targetBlockId, position, err = checkTableBlocksMove(srcState, targetBlockId, position, blockIds) + if err != nil { + return err + } + var replacementCandidate simple.Block for _, id := range blockIds { if b := srcState.Pick(id); b != nil { @@ -278,6 +286,24 @@ func (bs *basic) Move(srcState, destState *state.State, targetBlockId string, po return srcState.InsertTo(targetBlockId, position, blockIds...) } +func checkTableBlocksMove(st *state.State, target string, pos model.BlockPosition, blockIds []string) (string, model.BlockPosition, error) { + for _, id := range blockIds { + t := table.GetTableRootBlock(st, id) + if t != nil && t.Model().Id != id { + // we should not move table blocks except table root block + return "", 0, ErrCannotMoveTableBlocks + } + } + + t := table.GetTableRootBlock(st, target) + if t != nil { + // if the target is one of table blocks, we should insert blocks under the table + return t.Model().Id, model.Block_Bottom, nil + } + + return target, pos, nil +} + func (bs *basic) Replace(ctx session.Context, id string, block *model.Block) (newId string, err error) { s := bs.NewStateCtx(ctx) if block.GetContent() == nil { diff --git a/core/block/editor/basic/basic_test.go b/core/block/editor/basic/basic_test.go index d738ecdae..0957bdcca 100644 --- a/core/block/editor/basic/basic_test.go +++ b/core/block/editor/basic/basic_test.go @@ -1,6 +1,8 @@ package basic import ( + "errors" + "math/rand" "testing" "github.com/gogo/protobuf/types" @@ -146,24 +148,6 @@ func TestBasic_Move(t *testing.T) { assert.Len(t, sb.NewState().Pick("4").Model().ChildrenIds, 1) }) - t.Run("incorrect table cell move", func(t *testing.T) { - // given - sb := smarttest.New("test") - sb.AddBlock(simple.New(&model.Block{Id: "test", ChildrenIds: []string{"table"}})). - AddBlock(simple.New(&model.Block{Id: "table", ChildrenIds: []string{"rows", "columns"}})). - AddBlock(simple.New(&model.Block{Id: "columns", ChildrenIds: []string{"column"}, Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableColumns}}})). - AddBlock(simple.New(&model.Block{Id: "column", ChildrenIds: []string{}, Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}})). - AddBlock(simple.New(&model.Block{Id: "rows", ChildrenIds: []string{"row"}, Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableRows}}})). - AddBlock(simple.New(&model.Block{Id: "row", ChildrenIds: []string{"cell"}, Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{IsHeader: false}}})). - AddBlock(simple.New(&model.Block{Id: "cell", ChildrenIds: []string{}})) - - b := NewBasic(sb, nil, converter.NewLayoutConverter()) - st := sb.NewState() - - err := b.Move(st, st, "rows", model.Block_Bottom, []string{"cell"}) - require.NoError(t, err) - require.NoError(t, sb.Apply(st)) - }) t.Run("header", func(t *testing.T) { sb := smarttest.New("test") sb.AddBlock(simple.New(&model.Block{Id: "test"})) @@ -276,6 +260,98 @@ func TestBasic_Move(t *testing.T) { }) } +func TestBasic_MoveTableBlocks(t *testing.T) { + getSB := func() *smarttest.SmartTest { + sb := smarttest.New("test") + sb.AddBlock(simple.New(&model.Block{Id: "test", ChildrenIds: []string{"upper", "table", "block"}})). + AddBlock(simple.New(&model.Block{Id: "table", ChildrenIds: []string{"columns", "rows"}, Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}})). + AddBlock(simple.New(&model.Block{Id: "columns", ChildrenIds: []string{"column"}, Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableColumns}}})). + AddBlock(simple.New(&model.Block{Id: "column", ChildrenIds: []string{}, Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}})). + AddBlock(simple.New(&model.Block{Id: "rows", ChildrenIds: []string{"row"}, Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableRows}}})). + AddBlock(simple.New(&model.Block{Id: "row", ChildrenIds: []string{"cell"}, Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{IsHeader: false}}})). + AddBlock(simple.New(&model.Block{Id: "cell", ChildrenIds: []string{}})). + AddBlock(simple.New(&model.Block{Id: "block", ChildrenIds: []string{}})). + AddBlock(simple.New(&model.Block{Id: "upper", ChildrenIds: []string{}})) + return sb + } + + for _, block := range []string{"columns", "rows", "column", "row", "cell"} { + t.Run("moving non-root table block '"+block+"' leads to error", func(t *testing.T) { + // given + sb := getSB() + b := NewBasic(sb, nil, converter.NewLayoutConverter()) + st := sb.NewState() + + // when + err := b.Move(st, st, "block", model.Block_Bottom, []string{block}) + + // then + assert.Error(t, err) + assert.True(t, errors.Is(err, ErrCannotMoveTableBlocks)) + }) + } + + t.Run("no error on moving root table block", func(t *testing.T) { + // given + sb := getSB() + b := NewBasic(sb, nil, converter.NewLayoutConverter()) + st := sb.NewState() + + // when + err := b.Move(st, st, "block", model.Block_Bottom, []string{"table"}) + + // then + assert.NoError(t, err) + assert.Equal(t, []string{"upper", "block", "table"}, st.Pick("test").Model().ChildrenIds) + }) + + t.Run("moving table block from invalid table leads to error", func(t *testing.T) { + // given + sb := getSB() + b := NewBasic(sb, nil, converter.NewLayoutConverter()) + st := sb.NewState() + st.Unlink("columns") + + // when + err := b.Move(st, st, "block", model.Block_Bottom, []string{"cell"}) + + // then + assert.Error(t, err) + assert.True(t, errors.Is(err, ErrCannotMoveTableBlocks)) + }) + + for _, block := range []string{"table", "columns", "rows", "column", "row", "cell"} { + t.Run("moving a block to '"+block+"' block leads to moving it under the table", func(t *testing.T) { + // given + sb := getSB() + b := NewBasic(sb, nil, converter.NewLayoutConverter()) + st := sb.NewState() + + // when + err := b.Move(st, st, block, model.BlockPosition(rand.Intn(6)), []string{"upper"}) + + // then + assert.NoError(t, err) + assert.Equal(t, []string{"table", "upper", "block"}, st.Pick("test").Model().ChildrenIds) + }) + } + + t.Run("moving a block to the invalid table leads to moving it under the table", func(t *testing.T) { + // given + sb := getSB() + b := NewBasic(sb, nil, converter.NewLayoutConverter()) + st := sb.NewState() + st.Unlink("columns") + + // when + err := b.Move(st, st, "cell", model.BlockPosition(rand.Intn(6)), []string{"upper"}) + + // then + assert.NoError(t, err) + assert.Equal(t, []string{"table", "upper", "block"}, st.Pick("test").Model().ChildrenIds) + }) +} + func TestBasic_MoveToAnotherObject(t *testing.T) { t.Run("basic", func(t *testing.T) { sb1 := smarttest.New("test1") diff --git a/core/block/editor/table/table.go b/core/block/editor/table/table.go index ce4191be7..b33f473b6 100644 --- a/core/block/editor/table/table.go +++ b/core/block/editor/table/table.go @@ -868,14 +868,7 @@ func NewTable(s *state.State, id string) (*Table, error) { s: s, } - next := s.Pick(id) - for next != nil { - if next.Model().GetTable() != nil { - tb.block = next - break - } - next = s.PickParentOf(next.Model().Id) - } + tb.block = GetTableRootBlock(s, id) if tb.block == nil { return nil, fmt.Errorf("root table block is not found") } @@ -901,6 +894,19 @@ func NewTable(s *state.State, id string) (*Table, error) { return &tb, nil } +// GetTableRootBlock iterates over parents of block. Returns nil in case root table block is not found +func GetTableRootBlock(s *state.State, id string) (block simple.Block) { + next := s.Pick(id) + for next != nil { + if next.Model().GetTable() != nil { + block = next + break + } + next = s.PickParentOf(next.Model().Id) + } + return block +} + // destructureDivs removes child dividers from block func destructureDivs(s *state.State, blockID string) { parent := s.Pick(blockID) From aadbaf10ba436e1a81cb09a0caa539c91c43a8a4 Mon Sep 17 00:00:00 2001 From: kirillston Date: Tue, 2 Jul 2024 17:05:50 +0200 Subject: [PATCH 03/71] GO-3192 Allow moving relatively to table block --- core/block/editor/basic/basic.go | 2 +- core/block/editor/basic/basic_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/block/editor/basic/basic.go b/core/block/editor/basic/basic.go index 34ef14878..8db10caf9 100644 --- a/core/block/editor/basic/basic.go +++ b/core/block/editor/basic/basic.go @@ -296,7 +296,7 @@ func checkTableBlocksMove(st *state.State, target string, pos model.BlockPositio } t := table.GetTableRootBlock(st, target) - if t != nil { + if t != nil && t.Model().Id != target { // if the target is one of table blocks, we should insert blocks under the table return t.Model().Id, model.Block_Bottom, nil } diff --git a/core/block/editor/basic/basic_test.go b/core/block/editor/basic/basic_test.go index 0957bdcca..a6c5b57fb 100644 --- a/core/block/editor/basic/basic_test.go +++ b/core/block/editor/basic/basic_test.go @@ -320,7 +320,7 @@ func TestBasic_MoveTableBlocks(t *testing.T) { assert.True(t, errors.Is(err, ErrCannotMoveTableBlocks)) }) - for _, block := range []string{"table", "columns", "rows", "column", "row", "cell"} { + for _, block := range []string{"columns", "rows", "column", "row", "cell"} { t.Run("moving a block to '"+block+"' block leads to moving it under the table", func(t *testing.T) { // given sb := getSB() From 230f5d87e1ffa672243c35593a9dadab21b866c3 Mon Sep 17 00:00:00 2001 From: kirillston Date: Tue, 2 Jul 2024 18:05:53 +0200 Subject: [PATCH 04/71] GO-3192 Allow moving rows and inserting into cell --- core/block/editor/basic/basic.go | 31 ++++++++++++-- core/block/editor/basic/basic_test.go | 59 +++++++++++++++++++++++---- 2 files changed, 79 insertions(+), 11 deletions(-) diff --git a/core/block/editor/basic/basic.go b/core/block/editor/basic/basic.go index 8db10caf9..d3e2dcd4b 100644 --- a/core/block/editor/basic/basic.go +++ b/core/block/editor/basic/basic.go @@ -2,6 +2,7 @@ package basic import ( "fmt" + "slices" "github.com/globalsign/mgo/bson" "github.com/gogo/protobuf/types" @@ -288,8 +289,22 @@ func (bs *basic) Move(srcState, destState *state.State, targetBlockId string, po func checkTableBlocksMove(st *state.State, target string, pos model.BlockPosition, blockIds []string) (string, model.BlockPosition, error) { for _, id := range blockIds { - t := table.GetTableRootBlock(st, id) - if t != nil && t.Model().Id != id { + root := table.GetTableRootBlock(st, id) + if root != nil && root.Model().Id != id { + t, err := table.NewTable(st, root.Model().Id) + if err != nil { + return "", 0, ErrCannotMoveTableBlocks + } + + // we allow moving rows between each other + rows := t.RowIDs() + if slices.Contains(rows, id) && slices.Contains(rows, target) { + if pos == model.Block_Bottom || pos == model.Block_Top { + return target, pos, nil + } + return "", 0, fmt.Errorf("failed to move rows: position should be Top or Bottom, got %s", model.BlockPosition_name[int32(pos)]) + } + // we should not move table blocks except table root block return "", 0, ErrCannotMoveTableBlocks } @@ -297,13 +312,23 @@ func checkTableBlocksMove(st *state.State, target string, pos model.BlockPositio t := table.GetTableRootBlock(st, target) if t != nil && t.Model().Id != target { - // if the target is one of table blocks, we should insert blocks under the table + // we allow inserting blocks into table cell + if isTableCell(target) && slices.Contains([]model.BlockPosition{model.Block_Inner, model.Block_Replace, model.Block_InnerFirst}, pos) { + return target, pos, nil + } + + // if the target is one of table blocks, but not cell or table root, we should insert blocks under the table return t.Model().Id, model.Block_Bottom, nil } return target, pos, nil } +func isTableCell(id string) bool { + _, _, err := table.ParseCellID(id) + return err == nil +} + func (bs *basic) Replace(ctx session.Context, id string, block *model.Block) (newId string, err error) { s := bs.NewStateCtx(ctx) if block.GetContent() == nil { diff --git a/core/block/editor/basic/basic_test.go b/core/block/editor/basic/basic_test.go index a6c5b57fb..00ff79ec4 100644 --- a/core/block/editor/basic/basic_test.go +++ b/core/block/editor/basic/basic_test.go @@ -267,15 +267,16 @@ func TestBasic_MoveTableBlocks(t *testing.T) { AddBlock(simple.New(&model.Block{Id: "table", ChildrenIds: []string{"columns", "rows"}, Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}})). AddBlock(simple.New(&model.Block{Id: "columns", ChildrenIds: []string{"column"}, Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableColumns}}})). AddBlock(simple.New(&model.Block{Id: "column", ChildrenIds: []string{}, Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}})). - AddBlock(simple.New(&model.Block{Id: "rows", ChildrenIds: []string{"row"}, Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableRows}}})). - AddBlock(simple.New(&model.Block{Id: "row", ChildrenIds: []string{"cell"}, Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{IsHeader: false}}})). - AddBlock(simple.New(&model.Block{Id: "cell", ChildrenIds: []string{}})). + AddBlock(simple.New(&model.Block{Id: "rows", ChildrenIds: []string{"row", "row2"}, Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableRows}}})). + AddBlock(simple.New(&model.Block{Id: "row", ChildrenIds: []string{"column-row"}, Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{IsHeader: false}}})). + AddBlock(simple.New(&model.Block{Id: "row2", ChildrenIds: []string{}, Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{IsHeader: false}}})). + AddBlock(simple.New(&model.Block{Id: "column-row", ChildrenIds: []string{}})). AddBlock(simple.New(&model.Block{Id: "block", ChildrenIds: []string{}})). AddBlock(simple.New(&model.Block{Id: "upper", ChildrenIds: []string{}})) return sb } - for _, block := range []string{"columns", "rows", "column", "row", "cell"} { + for _, block := range []string{"columns", "rows", "column", "row", "column-row"} { t.Run("moving non-root table block '"+block+"' leads to error", func(t *testing.T) { // given sb := getSB() @@ -305,6 +306,33 @@ func TestBasic_MoveTableBlocks(t *testing.T) { assert.Equal(t, []string{"upper", "block", "table"}, st.Pick("test").Model().ChildrenIds) }) + t.Run("no error on moving one row between another", func(t *testing.T) { + // given + sb := getSB() + b := NewBasic(sb, nil, converter.NewLayoutConverter()) + st := sb.NewState() + + // when + err := b.Move(st, st, "row2", model.Block_Bottom, []string{"row"}) + + // then + assert.NoError(t, err) + assert.Equal(t, []string{"row2", "row"}, st.Pick("rows").Model().ChildrenIds) + }) + + t.Run("moving rows with incorrect position leads to error", func(t *testing.T) { + // given + sb := getSB() + b := NewBasic(sb, nil, converter.NewLayoutConverter()) + st := sb.NewState() + + // when + err := b.Move(st, st, "row2", model.Block_Left, []string{"row"}) + + // then + assert.Error(t, err) + }) + t.Run("moving table block from invalid table leads to error", func(t *testing.T) { // given sb := getSB() @@ -313,14 +341,14 @@ func TestBasic_MoveTableBlocks(t *testing.T) { st.Unlink("columns") // when - err := b.Move(st, st, "block", model.Block_Bottom, []string{"cell"}) + err := b.Move(st, st, "block", model.Block_Bottom, []string{"column-row"}) // then assert.Error(t, err) assert.True(t, errors.Is(err, ErrCannotMoveTableBlocks)) }) - for _, block := range []string{"columns", "rows", "column", "row", "cell"} { + for _, block := range []string{"columns", "rows", "column", "row"} { t.Run("moving a block to '"+block+"' block leads to moving it under the table", func(t *testing.T) { // given sb := getSB() @@ -328,7 +356,7 @@ func TestBasic_MoveTableBlocks(t *testing.T) { st := sb.NewState() // when - err := b.Move(st, st, block, model.BlockPosition(rand.Intn(6)), []string{"upper"}) + err := b.Move(st, st, block, model.BlockPosition(rand.Intn(len(model.BlockPosition_name))), []string{"upper"}) // then assert.NoError(t, err) @@ -344,12 +372,27 @@ func TestBasic_MoveTableBlocks(t *testing.T) { st.Unlink("columns") // when - err := b.Move(st, st, "cell", model.BlockPosition(rand.Intn(6)), []string{"upper"}) + err := b.Move(st, st, "rows", model.BlockPosition(rand.Intn(6)), []string{"upper"}) // then assert.NoError(t, err) assert.Equal(t, []string{"table", "upper", "block"}, st.Pick("test").Model().ChildrenIds) }) + + t.Run("moving a block to table cell is allowed", func(t *testing.T) { + // given + sb := getSB() + b := NewBasic(sb, nil, converter.NewLayoutConverter()) + st := sb.NewState() + + // when + err := b.Move(st, st, "column-row", model.Block_Inner, []string{"upper"}) + + // then + assert.NoError(t, err) + assert.Equal(t, []string{"table", "block"}, st.Pick("test").Model().ChildrenIds) + assert.Equal(t, []string{"upper"}, st.Pick("column-row").Model().ChildrenIds) + }) } func TestBasic_MoveToAnotherObject(t *testing.T) { From 1fdd07d775f850527bef9c9a6faa2019d6198ca4 Mon Sep 17 00:00:00 2001 From: kirillston Date: Tue, 9 Jul 2024 19:45:33 +0300 Subject: [PATCH 05/71] GO-3192 Check all rows --- core/block/editor/basic/basic.go | 28 ++++++++++++--------------- core/block/editor/basic/basic_test.go | 26 +++++++++++++++++++++++++ util/slice/slice.go | 12 ++++++++++++ util/slice/slice_test.go | 8 ++++++++ 4 files changed, 58 insertions(+), 16 deletions(-) diff --git a/core/block/editor/basic/basic.go b/core/block/editor/basic/basic.go index d3e2dcd4b..077fc8eed 100644 --- a/core/block/editor/basic/basic.go +++ b/core/block/editor/basic/basic.go @@ -288,23 +288,19 @@ func (bs *basic) Move(srcState, destState *state.State, targetBlockId string, po } func checkTableBlocksMove(st *state.State, target string, pos model.BlockPosition, blockIds []string) (string, model.BlockPosition, error) { + if t, _ := table.NewTable(st, target); t != nil { + // we allow moving rows between each other + if slice.IsSubSlice(append(blockIds, target), t.RowIDs()) { + if pos == model.Block_Bottom || pos == model.Block_Top { + return target, pos, nil + } + return "", 0, fmt.Errorf("failed to move rows: position should be Top or Bottom, got %s", model.BlockPosition_name[int32(pos)]) + } + } + for _, id := range blockIds { - root := table.GetTableRootBlock(st, id) - if root != nil && root.Model().Id != id { - t, err := table.NewTable(st, root.Model().Id) - if err != nil { - return "", 0, ErrCannotMoveTableBlocks - } - - // we allow moving rows between each other - rows := t.RowIDs() - if slices.Contains(rows, id) && slices.Contains(rows, target) { - if pos == model.Block_Bottom || pos == model.Block_Top { - return target, pos, nil - } - return "", 0, fmt.Errorf("failed to move rows: position should be Top or Bottom, got %s", model.BlockPosition_name[int32(pos)]) - } - + t := table.GetTableRootBlock(st, id) + if t != nil && t.Model().Id != id { // we should not move table blocks except table root block return "", 0, ErrCannotMoveTableBlocks } diff --git a/core/block/editor/basic/basic_test.go b/core/block/editor/basic/basic_test.go index 00ff79ec4..156b46dca 100644 --- a/core/block/editor/basic/basic_test.go +++ b/core/block/editor/basic/basic_test.go @@ -333,6 +333,32 @@ func TestBasic_MoveTableBlocks(t *testing.T) { assert.Error(t, err) }) + t.Run("moving rows and some other blocks between another leads to error", func(t *testing.T) { + // given + sb := getSB() + b := NewBasic(sb, nil, converter.NewLayoutConverter()) + st := sb.NewState() + + // when + err := b.Move(st, st, "row2", model.Block_Top, []string{"row", "rows"}) + + // then + assert.Error(t, err) + }) + + t.Run("moving the row between itself leads to error", func(t *testing.T) { + // given + sb := getSB() + b := NewBasic(sb, nil, converter.NewLayoutConverter()) + st := sb.NewState() + + // when + err := b.Move(st, st, "row2", model.Block_Bottom, []string{"row2"}) + + // then + assert.Error(t, err) + }) + t.Run("moving table block from invalid table leads to error", func(t *testing.T) { // given sb := getSB() diff --git a/util/slice/slice.go b/util/slice/slice.go index 4c87d0084..7ea2064df 100644 --- a/util/slice/slice.go +++ b/util/slice/slice.go @@ -237,3 +237,15 @@ func FilterCID(cids []string) []string { return err == nil }) } + +func IsSubSlice[T comparable](subs, s []T) bool { + if len(subs) > len(s) { + return false + } + for _, e := range subs { + if !slices.Contains(s, e) { + return false + } + } + return true +} diff --git a/util/slice/slice_test.go b/util/slice/slice_test.go index 3d642bd33..5bf818ca6 100644 --- a/util/slice/slice_test.go +++ b/util/slice/slice_test.go @@ -100,3 +100,11 @@ func TestUnsortedEquals(t *testing.T) { assert.False(t, UnsortedEqual([]string{"a", "b", "c"}, []string{"a", "b"})) assert.False(t, UnsortedEqual([]string{"a", "b", "c"}, []string{"a", "b", "c", "d"})) } + +func TestIsSubSlice(t *testing.T) { + assert.True(t, IsSubSlice([]byte("abc"), []byte("csabd"))) + assert.True(t, IsSubSlice([]string{}, []string{})) + assert.True(t, IsSubSlice([]string{}, []string{"hello"})) + assert.True(t, IsSubSlice([]string{"a", "c"}, []string{"a", "b", "c", "d"})) + assert.False(t, IsSubSlice([]string{"a", "c", "b", "a"}, []string{"a", "b", "c"})) +} From f49cbcf063acdf195392e6a5833aee74a5270aae Mon Sep 17 00:00:00 2001 From: kirillston Date: Wed, 10 Jul 2024 15:36:11 +0300 Subject: [PATCH 06/71] GO-3192 Rename slice method --- core/block/editor/basic/basic.go | 2 +- util/slice/slice.go | 7 ++----- util/slice/slice_test.go | 15 +++++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/block/editor/basic/basic.go b/core/block/editor/basic/basic.go index 077fc8eed..515336d1c 100644 --- a/core/block/editor/basic/basic.go +++ b/core/block/editor/basic/basic.go @@ -290,7 +290,7 @@ func (bs *basic) Move(srcState, destState *state.State, targetBlockId string, po func checkTableBlocksMove(st *state.State, target string, pos model.BlockPosition, blockIds []string) (string, model.BlockPosition, error) { if t, _ := table.NewTable(st, target); t != nil { // we allow moving rows between each other - if slice.IsSubSlice(append(blockIds, target), t.RowIDs()) { + if slice.ContainsAll(t.RowIDs(), append(blockIds, target)...) { if pos == model.Block_Bottom || pos == model.Block_Top { return target, pos, nil } diff --git a/util/slice/slice.go b/util/slice/slice.go index 7ea2064df..4e7e10cb6 100644 --- a/util/slice/slice.go +++ b/util/slice/slice.go @@ -238,11 +238,8 @@ func FilterCID(cids []string) []string { }) } -func IsSubSlice[T comparable](subs, s []T) bool { - if len(subs) > len(s) { - return false - } - for _, e := range subs { +func ContainsAll[T comparable](s []T, items ...T) bool { + for _, e := range items { if !slices.Contains(s, e) { return false } diff --git a/util/slice/slice_test.go b/util/slice/slice_test.go index 5bf818ca6..0cb15704a 100644 --- a/util/slice/slice_test.go +++ b/util/slice/slice_test.go @@ -101,10 +101,13 @@ func TestUnsortedEquals(t *testing.T) { assert.False(t, UnsortedEqual([]string{"a", "b", "c"}, []string{"a", "b", "c", "d"})) } -func TestIsSubSlice(t *testing.T) { - assert.True(t, IsSubSlice([]byte("abc"), []byte("csabd"))) - assert.True(t, IsSubSlice([]string{}, []string{})) - assert.True(t, IsSubSlice([]string{}, []string{"hello"})) - assert.True(t, IsSubSlice([]string{"a", "c"}, []string{"a", "b", "c", "d"})) - assert.False(t, IsSubSlice([]string{"a", "c", "b", "a"}, []string{"a", "b", "c"})) +func TestContainsAll(t *testing.T) { + assert.True(t, ContainsAll([]byte("csabd"), []byte("abc")...)) + assert.True(t, ContainsAll([]string{})) + assert.True(t, ContainsAll([]string{"hello"})) + assert.True(t, ContainsAll([]string{"a", "b", "c", "d"}, "a", "c")) + assert.True(t, ContainsAll([]string{"a", "b", "c", "d"}, "a", "a")) + assert.True(t, ContainsAll([]string{"a", "b", "c"}, "a", "c", "b", "a")) + assert.False(t, ContainsAll([]string{}, "z")) + assert.False(t, ContainsAll([]string{"y", "x"}, "z")) } From 8a265221d324a66e025c9ccac0853420e6bbc697 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 16 Jul 2024 22:34:31 +0200 Subject: [PATCH 07/71] GO-3769 WIP update logic --- core/block/object/treesyncer/treesyncer.go | 21 +++++-- core/syncstatus/detailsupdater/updater.go | 21 ++++--- .../syncstatus/objectsyncstatus/syncstatus.go | 59 +++++++++++++++---- go.mod | 12 ++-- go.sum | 58 ++++++++++++++---- 5 files changed, 130 insertions(+), 41 deletions(-) diff --git a/core/block/object/treesyncer/treesyncer.go b/core/block/object/treesyncer/treesyncer.go index 92551db95..dc28027c2 100644 --- a/core/block/object/treesyncer/treesyncer.go +++ b/core/block/object/treesyncer/treesyncer.go @@ -17,6 +17,7 @@ import ( "go.uber.org/zap" "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/util/slice" ) var log = logger.NewNamed(treemanager.CName) @@ -72,6 +73,10 @@ type SyncDetailsUpdater interface { UpdateDetails(objectId []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) } +type peerIdData struct { + existing, missing []string +} + type treeSyncer struct { sync.Mutex mainCtx context.Context @@ -88,6 +93,7 @@ type treeSyncer struct { nodeConf nodeconf.NodeConf syncedTreeRemover SyncedTreeRemover syncDetailsUpdater SyncDetailsUpdater + peerData map[string]peerIdData } func NewTreeSyncer(spaceId string) treesyncer.TreeSyncer { @@ -105,6 +111,7 @@ func NewTreeSyncer(spaceId string) treesyncer.TreeSyncer { func (t *treeSyncer) Init(a *app.App) (err error) { t.isSyncing = true + t.peerData = map[string]peerIdData{} t.treeManager = app.MustComponent[treemanager.TreeManager](a) t.peerManager = app.MustComponent[PeerStatusChecker](a) t.nodeConf = app.MustComponent[nodeconf.NodeConf](a) @@ -164,10 +171,16 @@ func (t *treeSyncer) ShouldSync(peerId string) bool { func (t *treeSyncer) SyncAll(ctx context.Context, peerId string, existing, missing []string) error { t.Lock() defer t.Unlock() - var err error + var ( + err error + peerData = t.peerData[peerId] + ) + existingRemoved, existingAdded := slice.DifferenceRemovedAdded(peerData.existing, existing) + peerData.existing = existing + peerData.missing = missing isResponsible := slices.Contains(t.nodeConf.NodeIds(t.spaceId), peerId) - defer t.sendResultEvent(err, isResponsible, peerId, existing) - t.sendSyncingEvent(peerId, existing, missing, isResponsible) + t.sendResultEvent(peerId, existingRemoved, err, isResponsible) + t.sendSyncingEvent(peerId, existingAdded, missing, isResponsible) reqExec, exists := t.requestPools[peerId] if !exists { reqExec = newExecutor(t.requests, 0) @@ -219,7 +232,7 @@ func (t *treeSyncer) sendSyncingEvent(peerId string, existing []string, missing } } -func (t *treeSyncer) sendResultEvent(err error, nodePeer bool, peerId string, existing []string) { +func (t *treeSyncer) sendResultEvent(peerId string, existing []string, err error, nodePeer bool) { if nodePeer && !t.peerManager.IsPeerOffline(peerId) { if err != nil { t.sendDetailsUpdates(existing, domain.ObjectError, domain.NetworkError) diff --git a/core/syncstatus/detailsupdater/updater.go b/core/syncstatus/detailsupdater/updater.go index eed81303b..15e6a10f6 100644 --- a/core/syncstatus/detailsupdater/updater.go +++ b/core/syncstatus/detailsupdater/updater.go @@ -161,11 +161,14 @@ func (u *syncStatusUpdater) updateObjectDetails(syncStatusDetails *syncStatusDet func (u *syncStatusUpdater) setObjectDetails(syncStatusDetails *syncStatusDetails, record *types.Struct, objectId string) error { status := syncStatusDetails.status syncError := syncStatusDetails.syncError + isFileStatus := false if fileStatus, ok := record.GetFields()[bundle.RelationKeyFileBackupStatus.String()]; ok { + isFileStatus = true status, syncError = mapFileStatus(filesyncstatus.Status(int(fileStatus.GetNumberValue()))) } + // we want to update sync date for other stuff changed := u.hasRelationsChange(record, status, syncError) - if !changed { + if !changed && isFileStatus { return nil } if !u.isLayoutSuitableForSyncRelations(record) { @@ -260,15 +263,15 @@ func (u *syncStatusUpdater) setSyncDetails(sb smartblock.SmartBlock, status doma Key: bundle.RelationKeySyncStatus.String(), Value: pbtypes.Int64(int64(status)), }, + { + Key: bundle.RelationKeySyncError.String(), + Value: pbtypes.Int64(int64(syncError)), + }, + { + Key: bundle.RelationKeySyncDate.String(), + Value: pbtypes.Int64(time.Now().Unix()), + }, } - syncStatusDetails = append(syncStatusDetails, &model.Detail{ - Key: bundle.RelationKeySyncError.String(), - Value: pbtypes.Int64(int64(syncError)), - }) - syncStatusDetails = append(syncStatusDetails, &model.Detail{ - Key: bundle.RelationKeySyncDate.String(), - Value: pbtypes.Int64(time.Now().Unix()), - }) return d.SetDetails(nil, syncStatusDetails, false) } return nil diff --git a/core/syncstatus/objectsyncstatus/syncstatus.go b/core/syncstatus/objectsyncstatus/syncstatus.go index 8f993067e..d6bc72708 100644 --- a/core/syncstatus/objectsyncstatus/syncstatus.go +++ b/core/syncstatus/objectsyncstatus/syncstatus.go @@ -46,6 +46,7 @@ const ( type StatusUpdater interface { HeadsChange(treeId string, heads []string) HeadsReceive(senderId, treeId string, heads []string) + HeadsApply(senderId, treeId string, heads []string, allAdded bool) RemoveAllExcept(senderId string, differentRemoteIds []string) } @@ -64,6 +65,7 @@ type StatusService interface { type treeHeadsEntry struct { heads []string stateCounter uint64 + isUpdated bool syncStatus SyncStatus } @@ -87,10 +89,10 @@ type syncStatusService struct { spaceId string treeHeads map[string]treeHeadsEntry watchers map[string]struct{} + synced []string + tempSynced map[string]struct{} stateCounter uint64 - treeStatusBuf []treeStatus - updateIntervalSecs int updateTimeout time.Duration @@ -102,8 +104,9 @@ type syncStatusService struct { func NewSyncStatusService() StatusService { return &syncStatusService{ - treeHeads: map[string]treeHeadsEntry{}, - watchers: map[string]struct{}{}, + treeHeads: map[string]treeHeadsEntry{}, + watchers: map[string]struct{}{}, + tempSynced: map[string]struct{}{}, } } @@ -158,9 +161,26 @@ func (s *syncStatusService) HeadsChange(treeId string, heads []string) { s.updateDetails(treeId, domain.ObjectSyncing) } -func (s *syncStatusService) update(ctx context.Context) (err error) { - s.treeStatusBuf = s.treeStatusBuf[:0] +func (s *syncStatusService) HeadsApply(senderId, treeId string, heads []string, allAdded bool) { + s.Lock() + defer s.Unlock() + if len(heads) == 0 || !s.isSenderResponsible(senderId) { + if allAdded { + s.tempSynced[treeId] = struct{}{} + } + return + } + _, ok := s.treeHeads[treeId] + if ok { + return + } + if allAdded { + s.synced = append(s.synced, treeId) + } +} +func (s *syncStatusService) update(ctx context.Context) (err error) { + var treeStatusBuf []treeStatus s.Lock() if s.updateReceiver == nil { s.Unlock() @@ -174,11 +194,19 @@ func (s *syncStatusService) update(ctx context.Context) (err error) { s.Unlock() return } - s.treeStatusBuf = append(s.treeStatusBuf, treeStatus{treeId, treeHeads.syncStatus}) + if !treeHeads.isUpdated { + treeHeads.isUpdated = true + s.treeHeads[treeId] = treeHeads + treeStatusBuf = append(treeStatusBuf, treeStatus{treeId, treeHeads.syncStatus}) + } } + for _, treeId := range s.synced { + treeStatusBuf = append(treeStatusBuf, treeStatus{treeId, StatusSynced}) + } + s.synced = s.synced[:0] s.Unlock() s.updateReceiver.UpdateNodeStatus() - for _, entry := range s.treeStatusBuf { + for _, entry := range treeStatusBuf { err = s.updateReceiver.UpdateTree(ctx, entry.treeId, entry.status) if err != nil { return @@ -220,6 +248,7 @@ func (s *syncStatusService) HeadsReceive(senderId, treeId string, heads []string }) if len(curTreeHeads.heads) == 0 { curTreeHeads.syncStatus = StatusSynced + curTreeHeads.isUpdated = false } s.treeHeads[treeId] = curTreeHeads } @@ -277,8 +306,18 @@ func (s *syncStatusService) RemoveAllExcept(senderId string, differentRemoteIds } // if we didn't find our treeId in heads ids which are different from us and node if _, found := slices.BinarySearch(differentRemoteIds, treeId); !found { - entry.syncStatus = StatusSynced - s.treeHeads[treeId] = entry + if entry.syncStatus != StatusSynced { + entry.syncStatus = StatusSynced + entry.isUpdated = false + s.treeHeads[treeId] = entry + } + } + } + // responsible node has those ids + for treeId := range s.tempSynced { + delete(s.tempSynced, treeId) + if _, found := slices.BinarySearch(differentRemoteIds, treeId); !found { + s.synced = append(s.synced, treeId) } } } diff --git a/go.mod b/go.mod index 8800d3bfc..5b6326417 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/PuerkitoBio/goquery v1.9.2 github.com/VividCortex/ewma v1.2.0 github.com/adrium/goheif v0.0.0-20230113233934-ca402e77a786 - github.com/anyproto/any-sync v0.4.21 + github.com/anyproto/any-sync v0.4.22-alpha.1 github.com/anyproto/go-naturaldate/v2 v2.0.2-0.20230524105841-9829cfd13438 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de github.com/avast/retry-go/v4 v4.6.0 @@ -56,8 +56,8 @@ require ( github.com/joho/godotenv v1.5.1 github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e github.com/kelseyhightower/envconfig v1.4.0 - github.com/klauspost/compress v1.17.7 - github.com/libp2p/go-libp2p v0.33.2 + github.com/klauspost/compress v1.17.8 + github.com/libp2p/go-libp2p v0.35.1 github.com/libp2p/zeroconf/v2 v2.2.0 github.com/logrusorgru/aurora v2.0.3+incompatible github.com/magiconair/properties v1.8.7 @@ -144,7 +144,7 @@ require ( github.com/chigopher/pathlib v0.19.1 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dsoprea/go-iptc v0.0.0-20200609062250-162ae6b44feb // indirect @@ -210,7 +210,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multiaddr v0.12.3 // indirect + github.com/multiformats/go-multiaddr v0.12.4 // indirect github.com/multiformats/go-multicodec v0.9.0 // indirect github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect @@ -221,7 +221,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/prometheus/client_model v0.6.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/pseudomuto/protokit v0.2.1 // indirect diff --git a/go.sum b/go.sum index 9005efdb5..bf4afbb6a 100644 --- a/go.sum +++ b/go.sum @@ -83,8 +83,10 @@ github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxB github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= -github.com/anyproto/any-sync v0.4.21 h1:L0/IUUrliZWm74RQgvrf9YKzfuvn9Mya+iuAbDKvU+Q= -github.com/anyproto/any-sync v0.4.21/go.mod h1:sO/zUrmnCZKnH/3KaRH3JQSZMuINS3X7ZJa+d4YgfkA= +github.com/anyproto/any-sync v0.4.22-0.20240716173953-cc827ebbc8b2 h1:y2+207FGo5gMqcWXPbTfZGRSRkf2cZt+M5AtikInFzE= +github.com/anyproto/any-sync v0.4.22-0.20240716173953-cc827ebbc8b2/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= +github.com/anyproto/any-sync v0.4.22-alpha.1 h1:8hgiiW2fAPLwY5kcwKBfYvat3jpk8Hfw5h+QwdZCHvg= +github.com/anyproto/any-sync v0.4.22-alpha.1/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580 h1:Ba80IlCCxkZ9H1GF+7vFu/TSpPvbpDCxXJ5ogc4euYc= github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580/go.mod h1:T/uWAYxrXdaXw64ihI++9RMbKTCpKd/yE9+saARew7k= github.com/anyproto/go-chash v0.1.0 h1:I9meTPjXFRfXZHRJzjOHC/XF7Q5vzysKkiT/grsogXY= @@ -249,8 +251,8 @@ github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= @@ -752,8 +754,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= -github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= @@ -796,8 +798,8 @@ github.com/libp2p/go-libp2p v0.7.0/go.mod h1:hZJf8txWeCduQRDC/WSqBGMxaTHCOYHt2xS github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniVO7zIHGMw= github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.13.0/go.mod h1:pM0beYdACRfHO1WcJlp65WXyG2A6NqYM+t2DTVAJxMo= -github.com/libp2p/go-libp2p v0.33.2 h1:vCdwnFxoGOXMKmaGHlDSnL4bM3fQeW8pgIa9DECnb40= -github.com/libp2p/go-libp2p v0.33.2/go.mod h1:zTeppLuCvUIkT118pFVzA8xzP/p2dJYOMApCkFh0Yww= +github.com/libp2p/go-libp2p v0.35.1 h1:Hm7Ub2BF+GCb14ojcsEK6WAy5it5smPDK02iXSZLl50= +github.com/libp2p/go-libp2p v0.35.1/go.mod h1:Dnkgba5hsfSv5dvvXC8nfqk44hH0gIKKno+HOMU0fdc= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= @@ -1154,6 +1156,38 @@ github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pion/datachannel v1.5.6 h1:1IxKJntfSlYkpUj8LlYRSWpYiTTC02nUrOE8T3DqGeg= +github.com/pion/datachannel v1.5.6/go.mod h1:1eKT6Q85pRnr2mHiWHxJwO50SfZRtWHTsNIVb/NfGW4= +github.com/pion/dtls/v2 v2.2.11 h1:9U/dpCYl1ySttROPWJgqWKEylUdT0fXp/xst6JwY5Ks= +github.com/pion/dtls/v2 v2.2.11/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= +github.com/pion/ice/v2 v2.3.25 h1:M5rJA07dqhi3nobJIg+uPtcVjFECTrhcR3n0ns8kDZs= +github.com/pion/ice/v2 v2.3.25/go.mod h1:KXJJcZK7E8WzrBEYnV4UtqEZsGeWfHxsNqhVcVvgjxw= +github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M= +github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4= +github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= +github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= +github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8= +github.com/pion/mdns v0.0.12/go.mod h1:VExJjv8to/6Wqm1FXK+Ii/Z9tsVk/F5sD/N70cnYFbk= +github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= +github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= +github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE= +github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= +github.com/pion/rtp v1.8.6 h1:MTmn/b0aWWsAzux2AmP8WGllusBVw4NPYPVFFd7jUPw= +github.com/pion/rtp v1.8.6/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/sctp v1.8.16 h1:PKrMs+o9EMLRvFfXq59WFsC+V8mN1wnKzqrv+3D/gYY= +github.com/pion/sctp v1.8.16/go.mod h1:P6PbDVA++OJMrVNg2AL3XtYHV4uD6dvfyOovCgMs0PE= +github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY= +github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= +github.com/pion/srtp/v2 v2.0.18 h1:vKpAXfawO9RtTRKZJbG4y0v1b11NZxQnxRl85kGuUlo= +github.com/pion/srtp/v2 v2.0.18/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA= +github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= +github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= +github.com/pion/transport/v2 v2.2.5 h1:iyi25i/21gQck4hfRhomF6SktmUQjRsRW4WJdhfc3Kc= +github.com/pion/transport/v2 v2.2.5/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= +github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= +github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= +github.com/pion/webrtc/v3 v3.2.40 h1:Wtfi6AZMQg+624cvCXUuSmrKWepSB7zfgYDOYqsSOVU= +github.com/pion/webrtc/v3 v3.2.40/go.mod h1:M1RAe3TNTD1tzyvqHrbVODfwdPGSXOUo/OgpoGGJqFY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -1184,8 +1218,8 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= -github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -1213,8 +1247,8 @@ github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/quic-go v0.44.0 h1:So5wOr7jyO4vzL2sd8/pD9Kesciv91zSk8BoFngItQ0= github.com/quic-go/quic-go v0.44.0/go.mod h1:z4cx/9Ny9UtGITIPzmPTXh1ULfOyWh4qGQlpnPcWmek= -github.com/quic-go/webtransport-go v0.6.0 h1:CvNsKqc4W2HljHJnoT+rMmbRJybShZ0YPFDD3NxaZLY= -github.com/quic-go/webtransport-go v0.6.0/go.mod h1:9KjU4AEBqEQidGHNDkZrb8CAa1abRaosM2yGOyiikEc= +github.com/quic-go/webtransport-go v0.8.0 h1:HxSrwun11U+LlmwpgM1kEqIqH90IT4N8auv/cD7QFJg= +github.com/quic-go/webtransport-go v0.8.0/go.mod h1:N99tjprW432Ut5ONql/aUhSLT0YVSlwHohQsuac9WaM= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= From 574e691d9b5351503076d6fda18062e669216f10 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 17 Jul 2024 09:39:10 +0200 Subject: [PATCH 08/71] GO-3769 Add missing to space --- core/block/object/treesyncer/treesyncer.go | 37 ++++++++++------------ core/syncstatus/detailsupdater/updater.go | 25 +++++++++++++++ 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/core/block/object/treesyncer/treesyncer.go b/core/block/object/treesyncer/treesyncer.go index dc28027c2..2a614a9fc 100644 --- a/core/block/object/treesyncer/treesyncer.go +++ b/core/block/object/treesyncer/treesyncer.go @@ -70,11 +70,7 @@ type PeerStatusChecker interface { type SyncDetailsUpdater interface { app.Component - UpdateDetails(objectId []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) -} - -type peerIdData struct { - existing, missing []string + UpdateSpaceDetails(existing []string, hasMissing bool, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) } type treeSyncer struct { @@ -93,7 +89,7 @@ type treeSyncer struct { nodeConf nodeconf.NodeConf syncedTreeRemover SyncedTreeRemover syncDetailsUpdater SyncDetailsUpdater - peerData map[string]peerIdData + peerData map[string][]string } func NewTreeSyncer(spaceId string) treesyncer.TreeSyncer { @@ -111,7 +107,7 @@ func NewTreeSyncer(spaceId string) treesyncer.TreeSyncer { func (t *treeSyncer) Init(a *app.App) (err error) { t.isSyncing = true - t.peerData = map[string]peerIdData{} + t.peerData = map[string][]string{} t.treeManager = app.MustComponent[treemanager.TreeManager](a) t.peerManager = app.MustComponent[PeerStatusChecker](a) t.nodeConf = app.MustComponent[nodeconf.NodeConf](a) @@ -175,12 +171,11 @@ func (t *treeSyncer) SyncAll(ctx context.Context, peerId string, existing, missi err error peerData = t.peerData[peerId] ) - existingRemoved, existingAdded := slice.DifferenceRemovedAdded(peerData.existing, existing) - peerData.existing = existing - peerData.missing = missing + existingRemoved, existingAdded := slice.DifferenceRemovedAdded(peerData, existing) + t.peerData[peerId] = existing isResponsible := slices.Contains(t.nodeConf.NodeIds(t.spaceId), peerId) - t.sendResultEvent(peerId, existingRemoved, err, isResponsible) - t.sendSyncingEvent(peerId, existingAdded, missing, isResponsible) + t.sendResultEvent(peerId, len(missing) != 0, existingRemoved, err, isResponsible) + t.sendSyncingEvent(peerId, len(missing) != 0, existingAdded, isResponsible) reqExec, exists := t.requestPools[peerId] if !exists { reqExec = newExecutor(t.requests, 0) @@ -219,31 +214,31 @@ func (t *treeSyncer) SyncAll(ctx context.Context, peerId string, existing, missi return nil } -func (t *treeSyncer) sendSyncingEvent(peerId string, existing []string, missing []string, nodePeer bool) { +func (t *treeSyncer) sendSyncingEvent(peerId string, hasMissing bool, existing []string, nodePeer bool) { if !nodePeer { return } if t.peerManager.IsPeerOffline(peerId) { - t.sendDetailsUpdates(existing, domain.ObjectError, domain.NetworkError) + t.sendDetailsUpdates(existing, hasMissing, domain.ObjectError, domain.NetworkError) return } - if len(existing) != 0 || len(missing) != 0 { - t.sendDetailsUpdates(existing, domain.ObjectSyncing, domain.Null) + if len(existing) != 0 || hasMissing { + t.sendDetailsUpdates(existing, hasMissing, domain.ObjectSyncing, domain.Null) } } -func (t *treeSyncer) sendResultEvent(peerId string, existing []string, err error, nodePeer bool) { +func (t *treeSyncer) sendResultEvent(peerId string, hasMissing bool, existing []string, err error, nodePeer bool) { if nodePeer && !t.peerManager.IsPeerOffline(peerId) { if err != nil { - t.sendDetailsUpdates(existing, domain.ObjectError, domain.NetworkError) + t.sendDetailsUpdates(existing, hasMissing, domain.ObjectError, domain.NetworkError) } else { - t.sendDetailsUpdates(existing, domain.ObjectSynced, domain.Null) + t.sendDetailsUpdates(existing, hasMissing, domain.ObjectSynced, domain.Null) } } } -func (t *treeSyncer) sendDetailsUpdates(existing []string, status domain.ObjectSyncStatus, syncError domain.SyncError) { - t.syncDetailsUpdater.UpdateDetails(existing, status, syncError, t.spaceId) +func (t *treeSyncer) sendDetailsUpdates(existing []string, hasMissing bool, status domain.ObjectSyncStatus, syncError domain.SyncError) { + t.syncDetailsUpdater.UpdateSpaceDetails(existing, hasMissing, status, syncError, t.spaceId) } func (t *treeSyncer) requestTree(peerId, id string) { diff --git a/core/syncstatus/detailsupdater/updater.go b/core/syncstatus/detailsupdater/updater.go index 15e6a10f6..53e741f8f 100644 --- a/core/syncstatus/detailsupdater/updater.go +++ b/core/syncstatus/detailsupdater/updater.go @@ -39,6 +39,7 @@ type syncStatusDetails struct { type Updater interface { app.ComponentRunnable + UpdateSpaceDetails(existing []string, hasMissing bool, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) UpdateDetails(objectId []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) } @@ -54,6 +55,7 @@ type syncStatusUpdater struct { batcher *mb.MB[*syncStatusDetails] spaceService space.Service spaceSyncStatus SpaceStatusUpdater + missing map[string]struct{} entries map[string]*syncStatusDetails mx sync.Mutex @@ -65,6 +67,7 @@ func NewUpdater() Updater { return &syncStatusUpdater{ batcher: mb.New[*syncStatusDetails](0), finish: make(chan struct{}), + missing: make(map[string]struct{}), entries: make(map[string]*syncStatusDetails, 0), } } @@ -118,6 +121,20 @@ func (u *syncStatusUpdater) UpdateDetails(objectId []string, status domain.Objec } } +func (u *syncStatusUpdater) UpdateSpaceDetails(existing []string, hasMissing bool, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) { + if spaceId == u.spaceService.TechSpaceId() { + return + } + u.mx.Lock() + if hasMissing { + u.missing[spaceId] = struct{}{} + } else { + delete(u.missing, spaceId) + } + u.mx.Unlock() + u.UpdateDetails(existing, status, syncError, spaceId) +} + func (u *syncStatusUpdater) updateDetails(syncStatusDetails *syncStatusDetails) { details := u.extractObjectDetails(syncStatusDetails) for _, detail := range details { @@ -231,6 +248,14 @@ func mapObjectSyncToSpaceSyncStatus(status domain.ObjectSyncStatus, syncError do func (u *syncStatusUpdater) sendSpaceStatusUpdate(err error, syncStatusDetails *syncStatusDetails, status domain.SpaceSyncStatus, syncError domain.SyncError) { if err == nil { + if status == domain.Synced { + u.mx.Lock() + _, missing := u.missing[syncStatusDetails.spaceId] + u.mx.Unlock() + if missing { + status = domain.Syncing + } + } u.spaceSyncStatus.SendUpdate(domain.MakeSyncStatus(syncStatusDetails.spaceId, status, syncError, domain.Objects)) } } From 011570054de5045b6e26bdbf3e2a043c3d6e9fa1 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 17 Jul 2024 14:55:30 +0200 Subject: [PATCH 09/71] GO-3769 Update any-sync --- core/block/object/treesyncer/treesyncer.go | 24 +++++++-------- core/domain/syncstatus.go | 9 +++--- core/syncstatus/detailsupdater/updater.go | 35 ++++++++-------------- go.mod | 2 +- go.sum | 2 ++ 5 files changed, 32 insertions(+), 40 deletions(-) diff --git a/core/block/object/treesyncer/treesyncer.go b/core/block/object/treesyncer/treesyncer.go index 2a614a9fc..e3ad7bcd1 100644 --- a/core/block/object/treesyncer/treesyncer.go +++ b/core/block/object/treesyncer/treesyncer.go @@ -70,7 +70,7 @@ type PeerStatusChecker interface { type SyncDetailsUpdater interface { app.Component - UpdateSpaceDetails(existing []string, hasMissing bool, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) + UpdateSpaceDetails(existing []string, missingCount int, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) } type treeSyncer struct { @@ -174,8 +174,8 @@ func (t *treeSyncer) SyncAll(ctx context.Context, peerId string, existing, missi existingRemoved, existingAdded := slice.DifferenceRemovedAdded(peerData, existing) t.peerData[peerId] = existing isResponsible := slices.Contains(t.nodeConf.NodeIds(t.spaceId), peerId) - t.sendResultEvent(peerId, len(missing) != 0, existingRemoved, err, isResponsible) - t.sendSyncingEvent(peerId, len(missing) != 0, existingAdded, isResponsible) + t.sendResultEvent(peerId, len(missing), existingRemoved, err, isResponsible) + t.sendSyncingEvent(peerId, len(missing), existingAdded, isResponsible) reqExec, exists := t.requestPools[peerId] if !exists { reqExec = newExecutor(t.requests, 0) @@ -214,31 +214,31 @@ func (t *treeSyncer) SyncAll(ctx context.Context, peerId string, existing, missi return nil } -func (t *treeSyncer) sendSyncingEvent(peerId string, hasMissing bool, existing []string, nodePeer bool) { +func (t *treeSyncer) sendSyncingEvent(peerId string, missingCount int, existing []string, nodePeer bool) { if !nodePeer { return } if t.peerManager.IsPeerOffline(peerId) { - t.sendDetailsUpdates(existing, hasMissing, domain.ObjectError, domain.NetworkError) + t.sendDetailsUpdates(existing, missingCount, domain.ObjectError, domain.NetworkError) return } - if len(existing) != 0 || hasMissing { - t.sendDetailsUpdates(existing, hasMissing, domain.ObjectSyncing, domain.Null) + if len(existing) != 0 || missingCount != 0 { + t.sendDetailsUpdates(existing, missingCount, domain.ObjectSyncing, domain.Null) } } -func (t *treeSyncer) sendResultEvent(peerId string, hasMissing bool, existing []string, err error, nodePeer bool) { +func (t *treeSyncer) sendResultEvent(peerId string, missingCount int, existing []string, err error, nodePeer bool) { if nodePeer && !t.peerManager.IsPeerOffline(peerId) { if err != nil { - t.sendDetailsUpdates(existing, hasMissing, domain.ObjectError, domain.NetworkError) + t.sendDetailsUpdates(existing, missingCount, domain.ObjectError, domain.NetworkError) } else { - t.sendDetailsUpdates(existing, hasMissing, domain.ObjectSynced, domain.Null) + t.sendDetailsUpdates(existing, missingCount, domain.ObjectSynced, domain.Null) } } } -func (t *treeSyncer) sendDetailsUpdates(existing []string, hasMissing bool, status domain.ObjectSyncStatus, syncError domain.SyncError) { - t.syncDetailsUpdater.UpdateSpaceDetails(existing, hasMissing, status, syncError, t.spaceId) +func (t *treeSyncer) sendDetailsUpdates(existing []string, missingCount int, status domain.ObjectSyncStatus, syncError domain.SyncError) { + t.syncDetailsUpdater.UpdateSpaceDetails(existing, missingCount, status, syncError, t.spaceId) } func (t *treeSyncer) requestTree(peerId, id string) { diff --git a/core/domain/syncstatus.go b/core/domain/syncstatus.go index 178cece1a..cf3c50a2d 100644 --- a/core/domain/syncstatus.go +++ b/core/domain/syncstatus.go @@ -37,10 +37,11 @@ const ( ) type SpaceSync struct { - SpaceId string - Status SpaceSyncStatus - SyncError SyncError - SyncType SyncType + SpaceId string + Status SpaceSyncStatus + SyncError SyncError + SyncType SyncType + MissingObjects int } func MakeSyncStatus(spaceId string, status SpaceSyncStatus, syncError SyncError, syncType SyncType) *SpaceSync { diff --git a/core/syncstatus/detailsupdater/updater.go b/core/syncstatus/detailsupdater/updater.go index 53e741f8f..c4ed776fa 100644 --- a/core/syncstatus/detailsupdater/updater.go +++ b/core/syncstatus/detailsupdater/updater.go @@ -39,7 +39,7 @@ type syncStatusDetails struct { type Updater interface { app.ComponentRunnable - UpdateSpaceDetails(existing []string, hasMissing bool, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) + UpdateSpaceDetails(existing []string, missing int, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) UpdateDetails(objectId []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) } @@ -55,7 +55,7 @@ type syncStatusUpdater struct { batcher *mb.MB[*syncStatusDetails] spaceService space.Service spaceSyncStatus SpaceStatusUpdater - missing map[string]struct{} + missing map[string]int entries map[string]*syncStatusDetails mx sync.Mutex @@ -67,7 +67,7 @@ func NewUpdater() Updater { return &syncStatusUpdater{ batcher: mb.New[*syncStatusDetails](0), finish: make(chan struct{}), - missing: make(map[string]struct{}), + missing: make(map[string]int), entries: make(map[string]*syncStatusDetails, 0), } } @@ -121,13 +121,13 @@ func (u *syncStatusUpdater) UpdateDetails(objectId []string, status domain.Objec } } -func (u *syncStatusUpdater) UpdateSpaceDetails(existing []string, hasMissing bool, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) { +func (u *syncStatusUpdater) UpdateSpaceDetails(existing []string, missing int, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) { if spaceId == u.spaceService.TechSpaceId() { return } u.mx.Lock() - if hasMissing { - u.missing[spaceId] = struct{}{} + if missing != 0 { + u.missing[spaceId] = missing } else { delete(u.missing, spaceId) } @@ -135,17 +135,6 @@ func (u *syncStatusUpdater) UpdateSpaceDetails(existing []string, hasMissing boo u.UpdateDetails(existing, status, syncError, spaceId) } -func (u *syncStatusUpdater) updateDetails(syncStatusDetails *syncStatusDetails) { - details := u.extractObjectDetails(syncStatusDetails) - for _, detail := range details { - id := pbtypes.GetString(detail.Details, bundle.RelationKeyId.String()) - err := u.setObjectDetails(syncStatusDetails, detail.Details, id) - if err != nil { - log.Errorf("failed to update object details %s", err) - } - } -} - func (u *syncStatusUpdater) extractObjectDetails(syncStatusDetails *syncStatusDetails) []database.Record { details, err := u.objectStore.Query(database.Query{ Filters: []*model.BlockContentDataviewFilter{ @@ -248,15 +237,18 @@ func mapObjectSyncToSpaceSyncStatus(status domain.ObjectSyncStatus, syncError do func (u *syncStatusUpdater) sendSpaceStatusUpdate(err error, syncStatusDetails *syncStatusDetails, status domain.SpaceSyncStatus, syncError domain.SyncError) { if err == nil { + domainStatus := domain.MakeSyncStatus(syncStatusDetails.spaceId, status, syncError, domain.Objects) if status == domain.Synced { u.mx.Lock() - _, missing := u.missing[syncStatusDetails.spaceId] + cnt := u.missing[syncStatusDetails.spaceId] u.mx.Unlock() - if missing { + if cnt != 0 { status = domain.Syncing } + domainStatus.MissingObjects = cnt } - u.spaceSyncStatus.SendUpdate(domain.MakeSyncStatus(syncStatusDetails.spaceId, status, syncError, domain.Objects)) + + u.spaceSyncStatus.SendUpdate(domainStatus) } } @@ -339,8 +331,5 @@ func (u *syncStatusUpdater) processEvents() { } } } - if len(status.objectIds) == 0 { - u.updateDetails(status) - } } } diff --git a/go.mod b/go.mod index 5b6326417..f4aa11d16 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/PuerkitoBio/goquery v1.9.2 github.com/VividCortex/ewma v1.2.0 github.com/adrium/goheif v0.0.0-20230113233934-ca402e77a786 - github.com/anyproto/any-sync v0.4.22-alpha.1 + github.com/anyproto/any-sync v0.4.22-alpha.1.0.20240717122315-3a22302fda91 github.com/anyproto/go-naturaldate/v2 v2.0.2-0.20230524105841-9829cfd13438 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de github.com/avast/retry-go/v4 v4.6.0 diff --git a/go.sum b/go.sum index bf4afbb6a..9ff02fcc9 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,8 @@ github.com/anyproto/any-sync v0.4.22-0.20240716173953-cc827ebbc8b2 h1:y2+207FGo5 github.com/anyproto/any-sync v0.4.22-0.20240716173953-cc827ebbc8b2/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= github.com/anyproto/any-sync v0.4.22-alpha.1 h1:8hgiiW2fAPLwY5kcwKBfYvat3jpk8Hfw5h+QwdZCHvg= github.com/anyproto/any-sync v0.4.22-alpha.1/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= +github.com/anyproto/any-sync v0.4.22-alpha.1.0.20240717122315-3a22302fda91 h1:WkyVWywVGi7P72yo5ykpNhZxoNeYCLaNdEJUyGCVr7c= +github.com/anyproto/any-sync v0.4.22-alpha.1.0.20240717122315-3a22302fda91/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580 h1:Ba80IlCCxkZ9H1GF+7vFu/TSpPvbpDCxXJ5ogc4euYc= github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580/go.mod h1:T/uWAYxrXdaXw64ihI++9RMbKTCpKd/yE9+saARew7k= github.com/anyproto/go-chash v0.1.0 h1:I9meTPjXFRfXZHRJzjOHC/XF7Q5vzysKkiT/grsogXY= From b91b210d615b0acdfc92ed07da24bd983f61a9c5 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 17 Jul 2024 17:54:22 +0200 Subject: [PATCH 10/71] GO-3769 Update sync --- core/block/object/treesyncer/treesyncer.go | 24 +++--- core/domain/syncstatus.go | 11 +-- core/syncstatus/detailsupdater/updater.go | 28 ++----- .../syncstatus/objectsyncstatus/syncstatus.go | 21 +++++- .../syncstatus/spacesyncstatus/objectstate.go | 10 ++- .../syncstatus/spacesyncstatus/spacestatus.go | 73 ++++++++++++------- go.mod | 2 +- go.sum | 2 + 8 files changed, 98 insertions(+), 73 deletions(-) diff --git a/core/block/object/treesyncer/treesyncer.go b/core/block/object/treesyncer/treesyncer.go index e3ad7bcd1..5fb3b6946 100644 --- a/core/block/object/treesyncer/treesyncer.go +++ b/core/block/object/treesyncer/treesyncer.go @@ -70,7 +70,7 @@ type PeerStatusChecker interface { type SyncDetailsUpdater interface { app.Component - UpdateSpaceDetails(existing []string, missingCount int, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) + UpdateSpaceDetails(existing, missing []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) } type treeSyncer struct { @@ -174,8 +174,8 @@ func (t *treeSyncer) SyncAll(ctx context.Context, peerId string, existing, missi existingRemoved, existingAdded := slice.DifferenceRemovedAdded(peerData, existing) t.peerData[peerId] = existing isResponsible := slices.Contains(t.nodeConf.NodeIds(t.spaceId), peerId) - t.sendResultEvent(peerId, len(missing), existingRemoved, err, isResponsible) - t.sendSyncingEvent(peerId, len(missing), existingAdded, isResponsible) + t.sendResultEvent(peerId, existingRemoved, missing, err, isResponsible) + t.sendSyncingEvent(peerId, existingAdded, missing, isResponsible) reqExec, exists := t.requestPools[peerId] if !exists { reqExec = newExecutor(t.requests, 0) @@ -214,31 +214,31 @@ func (t *treeSyncer) SyncAll(ctx context.Context, peerId string, existing, missi return nil } -func (t *treeSyncer) sendSyncingEvent(peerId string, missingCount int, existing []string, nodePeer bool) { +func (t *treeSyncer) sendSyncingEvent(peerId string, existing, missing []string, nodePeer bool) { if !nodePeer { return } if t.peerManager.IsPeerOffline(peerId) { - t.sendDetailsUpdates(existing, missingCount, domain.ObjectError, domain.NetworkError) + t.sendDetailsUpdates(existing, missing, domain.ObjectError, domain.NetworkError) return } - if len(existing) != 0 || missingCount != 0 { - t.sendDetailsUpdates(existing, missingCount, domain.ObjectSyncing, domain.Null) + if len(existing) != 0 || len(missing) != 0 { + t.sendDetailsUpdates(existing, missing, domain.ObjectSyncing, domain.Null) } } -func (t *treeSyncer) sendResultEvent(peerId string, missingCount int, existing []string, err error, nodePeer bool) { +func (t *treeSyncer) sendResultEvent(peerId string, existing, missing []string, err error, nodePeer bool) { if nodePeer && !t.peerManager.IsPeerOffline(peerId) { if err != nil { - t.sendDetailsUpdates(existing, missingCount, domain.ObjectError, domain.NetworkError) + t.sendDetailsUpdates(existing, missing, domain.ObjectError, domain.NetworkError) } else { - t.sendDetailsUpdates(existing, missingCount, domain.ObjectSynced, domain.Null) + t.sendDetailsUpdates(existing, missing, domain.ObjectSynced, domain.Null) } } } -func (t *treeSyncer) sendDetailsUpdates(existing []string, missingCount int, status domain.ObjectSyncStatus, syncError domain.SyncError) { - t.syncDetailsUpdater.UpdateSpaceDetails(existing, missingCount, status, syncError, t.spaceId) +func (t *treeSyncer) sendDetailsUpdates(existing, missing []string, status domain.ObjectSyncStatus, syncError domain.SyncError) { + t.syncDetailsUpdater.UpdateSpaceDetails(existing, missing, status, syncError, t.spaceId) } func (t *treeSyncer) requestTree(peerId, id string) { diff --git a/core/domain/syncstatus.go b/core/domain/syncstatus.go index cf3c50a2d..b5917ca96 100644 --- a/core/domain/syncstatus.go +++ b/core/domain/syncstatus.go @@ -37,11 +37,12 @@ const ( ) type SpaceSync struct { - SpaceId string - Status SpaceSyncStatus - SyncError SyncError - SyncType SyncType - MissingObjects int + SpaceId string + Status SpaceSyncStatus + SyncError SyncError + SyncType SyncType + // MissingObjects is a list of object IDs that are missing, it is not set every time + MissingObjects []string } func MakeSyncStatus(spaceId string, status SpaceSyncStatus, syncError SyncError, syncType SyncType) *SpaceSync { diff --git a/core/syncstatus/detailsupdater/updater.go b/core/syncstatus/detailsupdater/updater.go index c4ed776fa..14e6f139f 100644 --- a/core/syncstatus/detailsupdater/updater.go +++ b/core/syncstatus/detailsupdater/updater.go @@ -39,13 +39,14 @@ type syncStatusDetails struct { type Updater interface { app.ComponentRunnable - UpdateSpaceDetails(existing []string, missing int, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) + UpdateSpaceDetails(existing, missing []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) UpdateDetails(objectId []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) } type SpaceStatusUpdater interface { app.Component SendUpdate(status *domain.SpaceSync) + UpdateMissingIds(spaceId string, ids []string) } type syncStatusUpdater struct { @@ -55,7 +56,6 @@ type syncStatusUpdater struct { batcher *mb.MB[*syncStatusDetails] spaceService space.Service spaceSyncStatus SpaceStatusUpdater - missing map[string]int entries map[string]*syncStatusDetails mx sync.Mutex @@ -67,7 +67,6 @@ func NewUpdater() Updater { return &syncStatusUpdater{ batcher: mb.New[*syncStatusDetails](0), finish: make(chan struct{}), - missing: make(map[string]int), entries: make(map[string]*syncStatusDetails, 0), } } @@ -121,17 +120,11 @@ func (u *syncStatusUpdater) UpdateDetails(objectId []string, status domain.Objec } } -func (u *syncStatusUpdater) UpdateSpaceDetails(existing []string, missing int, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) { +func (u *syncStatusUpdater) UpdateSpaceDetails(existing, missing []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) { if spaceId == u.spaceService.TechSpaceId() { return } - u.mx.Lock() - if missing != 0 { - u.missing[spaceId] = missing - } else { - delete(u.missing, spaceId) - } - u.mx.Unlock() + u.spaceSyncStatus.UpdateMissingIds(spaceId, missing) u.UpdateDetails(existing, status, syncError, spaceId) } @@ -237,18 +230,7 @@ func mapObjectSyncToSpaceSyncStatus(status domain.ObjectSyncStatus, syncError do func (u *syncStatusUpdater) sendSpaceStatusUpdate(err error, syncStatusDetails *syncStatusDetails, status domain.SpaceSyncStatus, syncError domain.SyncError) { if err == nil { - domainStatus := domain.MakeSyncStatus(syncStatusDetails.spaceId, status, syncError, domain.Objects) - if status == domain.Synced { - u.mx.Lock() - cnt := u.missing[syncStatusDetails.spaceId] - u.mx.Unlock() - if cnt != 0 { - status = domain.Syncing - } - domainStatus.MissingObjects = cnt - } - - u.spaceSyncStatus.SendUpdate(domainStatus) + u.spaceSyncStatus.SendUpdate(domain.MakeSyncStatus(syncStatusDetails.spaceId, status, syncError, domain.Objects)) } } diff --git a/core/syncstatus/objectsyncstatus/syncstatus.go b/core/syncstatus/objectsyncstatus/syncstatus.go index d6bc72708..56000d567 100644 --- a/core/syncstatus/objectsyncstatus/syncstatus.go +++ b/core/syncstatus/objectsyncstatus/syncstatus.go @@ -15,12 +15,12 @@ import ( "github.com/anyproto/any-sync/commonspace/spacestorage" "github.com/anyproto/any-sync/nodeconf" "github.com/anyproto/any-sync/util/periodicsync" - "github.com/anyproto/any-sync/util/slice" "golang.org/x/exp/slices" "github.com/anyproto/anytype-heart/core/anytype/config" "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" + "github.com/anyproto/anytype-heart/util/slice" ) const ( @@ -47,6 +47,7 @@ type StatusUpdater interface { HeadsChange(treeId string, heads []string) HeadsReceive(senderId, treeId string, heads []string) HeadsApply(senderId, treeId string, heads []string, allAdded bool) + ObjectReceive(senderId, treeId string, heads []string) RemoveAllExcept(senderId string, differentRemoteIds []string) } @@ -161,6 +162,20 @@ func (s *syncStatusService) HeadsChange(treeId string, heads []string) { s.updateDetails(treeId, domain.ObjectSyncing) } +func (s *syncStatusService) ObjectReceive(senderId, treeId string, heads []string) { + s.Lock() + defer s.Unlock() + if len(heads) == 0 || !s.isSenderResponsible(senderId) { + s.tempSynced[treeId] = struct{}{} + return + } + _, ok := s.treeHeads[treeId] + if ok { + return + } + s.synced = append(s.synced, treeId) +} + func (s *syncStatusService) HeadsApply(senderId, treeId string, heads []string, allAdded bool) { s.Lock() defer s.Unlock() @@ -243,9 +258,7 @@ func (s *syncStatusService) HeadsReceive(senderId, treeId string, heads []string curTreeHeads.heads[idx] = "" } } - curTreeHeads.heads = slice.DiscardFromSlice(curTreeHeads.heads, func(h string) bool { - return h == "" - }) + curTreeHeads.heads = slice.RemoveMut(curTreeHeads.heads, "") if len(curTreeHeads.heads) == 0 { curTreeHeads.syncStatus = StatusSynced curTreeHeads.isUpdated = false diff --git a/core/syncstatus/spacesyncstatus/objectstate.go b/core/syncstatus/spacesyncstatus/objectstate.go index 0266f926e..c5a2a0564 100644 --- a/core/syncstatus/spacesyncstatus/objectstate.go +++ b/core/syncstatus/spacesyncstatus/objectstate.go @@ -1,14 +1,18 @@ package spacesyncstatus import ( + "fmt" "sync" + "github.com/samber/lo" + "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/pkg/lib/bundle" "github.com/anyproto/anytype-heart/pkg/lib/database" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" "github.com/anyproto/anytype-heart/util/pbtypes" + "github.com/anyproto/anytype-heart/util/slice" ) type ObjectState struct { @@ -37,7 +41,11 @@ func (o *ObjectState) SetObjectsNumber(status *domain.SpaceSync) { o.objectSyncCountBySpace[status.SpaceId] = 0 default: records := o.getSyncingObjects(status) - o.objectSyncCountBySpace[status.SpaceId] = len(records) + ids := lo.Map(records, func(r database.Record, idx int) string { + return pbtypes.GetString(r.Details, bundle.RelationKeyId.String()) + }) + _, added := slice.DifferenceRemovedAdded(ids, status.MissingObjects) + o.objectSyncCountBySpace[status.SpaceId] = len(records) + len(added) } } diff --git a/core/syncstatus/spacesyncstatus/spacestatus.go b/core/syncstatus/spacesyncstatus/spacestatus.go index 278178987..ab82e4128 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus.go +++ b/core/syncstatus/spacesyncstatus/spacestatus.go @@ -2,9 +2,14 @@ package spacesyncstatus import ( "context" + "sync" + "time" "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/app/logger" + "github.com/anyproto/any-sync/util/periodicsync" "github.com/cheggaaa/mb/v3" + "github.com/samber/lo" "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/event" @@ -12,6 +17,7 @@ import ( "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/logging" + "github.com/anyproto/anytype-heart/util/slice" ) const service = "core.syncstatus.spacesyncstatus" @@ -21,6 +27,7 @@ var log = logging.Logger("anytype-mw-space-status") type Updater interface { app.ComponentRunnable SendUpdate(spaceSync *domain.SpaceSync) + UpdateMissingIds(spaceId string, ids []string) } type SpaceIdGetter interface { @@ -53,8 +60,11 @@ type spaceSyncStatus struct { ctx context.Context ctxCancel context.CancelFunc spaceIdGetter SpaceIdGetter - - finish chan struct{} + curStatuses map[string]*domain.SpaceSync + missingIds map[string][]string + mx sync.Mutex + periodicCall periodicsync.PeriodicSync + finish chan struct{} } func NewSpaceSyncStatus() Updater { @@ -65,11 +75,14 @@ func (s *spaceSyncStatus) Init(a *app.App) (err error) { s.eventSender = app.MustComponent[event.Sender](a) s.networkConfig = app.MustComponent[NetworkConfig](a) store := app.MustComponent[objectstore.ObjectStore](a) + s.curStatuses = make(map[string]*domain.SpaceSync) + s.missingIds = make(map[string][]string) s.filesState = NewFileState(store) s.objectsState = NewObjectState(store) s.spaceIdGetter = app.MustComponent[SpaceIdGetter](a) sessionHookRunner := app.MustComponent[session.HookRunner](a) sessionHookRunner.RegisterHook(s.sendSyncEventForNewSession) + s.periodicCall = periodicsync.NewPeriodicSync(10, time.Second*5, s.update, logger.CtxLogger{Logger: log.Desugar()}) return } @@ -85,6 +98,12 @@ func (s *spaceSyncStatus) sendSyncEventForNewSession(ctx session.Context) error return nil } +func (s *spaceSyncStatus) UpdateMissingIds(spaceId string, ids []string) { + s.mx.Lock() + defer s.mx.Unlock() + s.missingIds[spaceId] = ids +} + func (s *spaceSyncStatus) Run(ctx context.Context) (err error) { if s.networkConfig.GetNetworkMode() == pb.RpcAccount_LocalOnly { s.sendLocalOnlyEvent() @@ -94,10 +113,29 @@ func (s *spaceSyncStatus) Run(ctx context.Context) (err error) { s.sendStartEvent(s.spaceIdGetter.AllSpaceIds()) } s.ctx, s.ctxCancel = context.WithCancel(context.Background()) - go s.processEvents() + s.periodicCall.Run() return } +func (s *spaceSyncStatus) update(ctx context.Context) error { + s.mx.Lock() + missingIds := lo.MapEntries(s.missingIds, func(key string, value []string) (string, []string) { + return key, slice.Copy(value) + }) + statuses := lo.MapToSlice(s.curStatuses, func(key string, value *domain.SpaceSync) *domain.SpaceSync { + return value + }) + s.mx.Unlock() + for _, st := range statuses { + if st.SpaceId == s.spaceIdGetter.TechSpaceId() { + continue + } + st.MissingObjects = missingIds[st.SpaceId] + s.updateSpaceSyncStatus(st) + } + return nil +} + func (s *spaceSyncStatus) sendEventToSession(spaceId, token string) { s.eventSender.SendToSession(token, &pb.Event{ Messages: []*pb.EventMessage{{ @@ -134,25 +172,9 @@ func (s *spaceSyncStatus) sendLocalOnlyEvent() { } func (s *spaceSyncStatus) SendUpdate(status *domain.SpaceSync) { - e := s.batcher.Add(context.Background(), status) - if e != nil { - log.Errorf("failed to add space sync event to queue %s", e) - } -} - -func (s *spaceSyncStatus) processEvents() { - defer close(s.finish) - for { - status, err := s.batcher.WaitOne(s.ctx) - if err != nil { - log.Errorf("failed to get event from batcher: %s", err) - return - } - if status.SpaceId == s.spaceIdGetter.TechSpaceId() { - continue - } - s.updateSpaceSyncStatus(status) - } + s.mx.Lock() + defer s.mx.Unlock() + s.curStatuses[status.SpaceId] = status } func (s *spaceSyncStatus) updateSpaceSyncStatus(receivedStatus *domain.SpaceSync) { @@ -204,11 +226,8 @@ func (s *spaceSyncStatus) needToSendEvent(status domain.SpaceSyncStatus, currSyn } func (s *spaceSyncStatus) Close(ctx context.Context) (err error) { - if s.ctxCancel != nil { - s.ctxCancel() - } - <-s.finish - return s.batcher.Close() + s.periodicCall.Close() + return } func (s *spaceSyncStatus) makeSpaceSyncEvent(spaceId string) *pb.EventSpaceSyncStatusUpdate { diff --git a/go.mod b/go.mod index f4aa11d16..468c27f5d 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/PuerkitoBio/goquery v1.9.2 github.com/VividCortex/ewma v1.2.0 github.com/adrium/goheif v0.0.0-20230113233934-ca402e77a786 - github.com/anyproto/any-sync v0.4.22-alpha.1.0.20240717122315-3a22302fda91 + github.com/anyproto/any-sync v0.4.22-alpha.2 github.com/anyproto/go-naturaldate/v2 v2.0.2-0.20230524105841-9829cfd13438 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de github.com/avast/retry-go/v4 v4.6.0 diff --git a/go.sum b/go.sum index 9ff02fcc9..58cefa808 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ github.com/anyproto/any-sync v0.4.22-alpha.1 h1:8hgiiW2fAPLwY5kcwKBfYvat3jpk8Hfw github.com/anyproto/any-sync v0.4.22-alpha.1/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= github.com/anyproto/any-sync v0.4.22-alpha.1.0.20240717122315-3a22302fda91 h1:WkyVWywVGi7P72yo5ykpNhZxoNeYCLaNdEJUyGCVr7c= github.com/anyproto/any-sync v0.4.22-alpha.1.0.20240717122315-3a22302fda91/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= +github.com/anyproto/any-sync v0.4.22-alpha.2 h1:q9eAASQU3VEYYQH8VQAPZwzWHwcAku1mXZlWWs96VMk= +github.com/anyproto/any-sync v0.4.22-alpha.2/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580 h1:Ba80IlCCxkZ9H1GF+7vFu/TSpPvbpDCxXJ5ogc4euYc= github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580/go.mod h1:T/uWAYxrXdaXw64ihI++9RMbKTCpKd/yE9+saARew7k= github.com/anyproto/go-chash v0.1.0 h1:I9meTPjXFRfXZHRJzjOHC/XF7Q5vzysKkiT/grsogXY= From ef51336dad468b6353a244ad4b1652261e5b4089 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 17 Jul 2024 18:06:04 +0200 Subject: [PATCH 11/71] GO-3769 Decrease iteration cycle --- core/syncstatus/spacesyncstatus/spacestatus.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/syncstatus/spacesyncstatus/spacestatus.go b/core/syncstatus/spacesyncstatus/spacestatus.go index ab82e4128..e50d23141 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus.go +++ b/core/syncstatus/spacesyncstatus/spacestatus.go @@ -82,7 +82,7 @@ func (s *spaceSyncStatus) Init(a *app.App) (err error) { s.spaceIdGetter = app.MustComponent[SpaceIdGetter](a) sessionHookRunner := app.MustComponent[session.HookRunner](a) sessionHookRunner.RegisterHook(s.sendSyncEventForNewSession) - s.periodicCall = periodicsync.NewPeriodicSync(10, time.Second*5, s.update, logger.CtxLogger{Logger: log.Desugar()}) + s.periodicCall = periodicsync.NewPeriodicSync(1, time.Second*5, s.update, logger.CtxLogger{Logger: log.Desugar()}) return } @@ -123,6 +123,7 @@ func (s *spaceSyncStatus) update(ctx context.Context) error { return key, slice.Copy(value) }) statuses := lo.MapToSlice(s.curStatuses, func(key string, value *domain.SpaceSync) *domain.SpaceSync { + delete(s.curStatuses, key) return value }) s.mx.Unlock() @@ -131,6 +132,8 @@ func (s *spaceSyncStatus) update(ctx context.Context) error { continue } st.MissingObjects = missingIds[st.SpaceId] + // if the there are too many updates and this hurts performance, + // we may skip some iterations and not do the updates for example s.updateSpaceSyncStatus(st) } return nil From e93867647c1c64a378d01c1cf5391dfbb01a8211 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 17 Jul 2024 21:17:04 +0200 Subject: [PATCH 12/71] GO-3769 Change logic --- core/block/object/treesyncer/treesyncer.go | 30 +----- core/syncstatus/detailsupdater/updater.go | 99 ++++++++++++------- .../syncstatus/objectsyncstatus/syncstatus.go | 4 +- 3 files changed, 72 insertions(+), 61 deletions(-) diff --git a/core/block/object/treesyncer/treesyncer.go b/core/block/object/treesyncer/treesyncer.go index 5fb3b6946..87ceb36ec 100644 --- a/core/block/object/treesyncer/treesyncer.go +++ b/core/block/object/treesyncer/treesyncer.go @@ -17,7 +17,6 @@ import ( "go.uber.org/zap" "github.com/anyproto/anytype-heart/core/domain" - "github.com/anyproto/anytype-heart/util/slice" ) var log = logger.NewNamed(treemanager.CName) @@ -89,7 +88,6 @@ type treeSyncer struct { nodeConf nodeconf.NodeConf syncedTreeRemover SyncedTreeRemover syncDetailsUpdater SyncDetailsUpdater - peerData map[string][]string } func NewTreeSyncer(spaceId string) treesyncer.TreeSyncer { @@ -107,7 +105,6 @@ func NewTreeSyncer(spaceId string) treesyncer.TreeSyncer { func (t *treeSyncer) Init(a *app.App) (err error) { t.isSyncing = true - t.peerData = map[string][]string{} t.treeManager = app.MustComponent[treemanager.TreeManager](a) t.peerManager = app.MustComponent[PeerStatusChecker](a) t.nodeConf = app.MustComponent[nodeconf.NodeConf](a) @@ -164,18 +161,11 @@ func (t *treeSyncer) ShouldSync(peerId string) bool { return t.isSyncing } -func (t *treeSyncer) SyncAll(ctx context.Context, peerId string, existing, missing []string) error { +func (t *treeSyncer) SyncAll(ctx context.Context, peerId string, existing, missing []string) (err error) { t.Lock() defer t.Unlock() - var ( - err error - peerData = t.peerData[peerId] - ) - existingRemoved, existingAdded := slice.DifferenceRemovedAdded(peerData, existing) - t.peerData[peerId] = existing isResponsible := slices.Contains(t.nodeConf.NodeIds(t.spaceId), peerId) - t.sendResultEvent(peerId, existingRemoved, missing, err, isResponsible) - t.sendSyncingEvent(peerId, existingAdded, missing, isResponsible) + t.sendSyncEvents(peerId, existing, missing, isResponsible) reqExec, exists := t.requestPools[peerId] if !exists { reqExec = newExecutor(t.requests, 0) @@ -214,29 +204,17 @@ func (t *treeSyncer) SyncAll(ctx context.Context, peerId string, existing, missi return nil } -func (t *treeSyncer) sendSyncingEvent(peerId string, existing, missing []string, nodePeer bool) { +func (t *treeSyncer) sendSyncEvents(peerId string, existing, missing []string, nodePeer bool) { if !nodePeer { return } if t.peerManager.IsPeerOffline(peerId) { t.sendDetailsUpdates(existing, missing, domain.ObjectError, domain.NetworkError) - return - } - if len(existing) != 0 || len(missing) != 0 { + } else { t.sendDetailsUpdates(existing, missing, domain.ObjectSyncing, domain.Null) } } -func (t *treeSyncer) sendResultEvent(peerId string, existing, missing []string, err error, nodePeer bool) { - if nodePeer && !t.peerManager.IsPeerOffline(peerId) { - if err != nil { - t.sendDetailsUpdates(existing, missing, domain.ObjectError, domain.NetworkError) - } else { - t.sendDetailsUpdates(existing, missing, domain.ObjectSynced, domain.Null) - } - } -} - func (t *treeSyncer) sendDetailsUpdates(existing, missing []string, status domain.ObjectSyncStatus, syncError domain.SyncError) { t.syncDetailsUpdater.UpdateSpaceDetails(existing, missing, status, syncError, t.spaceId) } diff --git a/core/syncstatus/detailsupdater/updater.go b/core/syncstatus/detailsupdater/updater.go index 14e6f139f..eff148bc6 100644 --- a/core/syncstatus/detailsupdater/updater.go +++ b/core/syncstatus/detailsupdater/updater.go @@ -3,6 +3,7 @@ package detailsupdater import ( "context" "errors" + "fmt" "slices" "sync" "time" @@ -24,6 +25,7 @@ import ( "github.com/anyproto/anytype-heart/pkg/lib/pb/model" "github.com/anyproto/anytype-heart/space" "github.com/anyproto/anytype-heart/util/pbtypes" + "github.com/anyproto/anytype-heart/util/slice" ) var log = logging.Logger(CName) @@ -31,16 +33,17 @@ var log = logging.Logger(CName) const CName = "core.syncstatus.objectsyncstatus.updater" type syncStatusDetails struct { - objectIds []string - status domain.ObjectSyncStatus - syncError domain.SyncError - spaceId string + objectId string + markAllSyncedExcept []string + status domain.ObjectSyncStatus + syncError domain.SyncError + spaceId string } type Updater interface { app.ComponentRunnable UpdateSpaceDetails(existing, missing []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) - UpdateDetails(objectId []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) + UpdateDetails(objectId string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) } type SpaceStatusUpdater interface { @@ -96,21 +99,20 @@ func (u *syncStatusUpdater) Name() (name string) { return CName } -func (u *syncStatusUpdater) UpdateDetails(objectId []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) { +func (u *syncStatusUpdater) UpdateDetails(objectId string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) { if spaceId == u.spaceService.TechSpaceId() { return } - for _, id := range objectId { - u.mx.Lock() - u.entries[id] = &syncStatusDetails{ - status: status, - syncError: syncError, - spaceId: spaceId, - } - u.mx.Unlock() + u.mx.Lock() + u.entries[objectId] = &syncStatusDetails{ + objectId: objectId, + status: status, + syncError: syncError, + spaceId: spaceId, } + u.mx.Unlock() err := u.batcher.TryAdd(&syncStatusDetails{ - objectIds: objectId, + objectId: objectId, status: status, syncError: syncError, spaceId: spaceId, @@ -125,28 +127,36 @@ func (u *syncStatusUpdater) UpdateSpaceDetails(existing, missing []string, statu return } u.spaceSyncStatus.UpdateMissingIds(spaceId, missing) - u.UpdateDetails(existing, status, syncError, spaceId) + err := u.batcher.TryAdd(&syncStatusDetails{ + markAllSyncedExcept: existing, + status: status, + syncError: syncError, + spaceId: spaceId, + }) + if err != nil { + log.Errorf("failed to add sync details update to queue: %s", err) + } } -func (u *syncStatusUpdater) extractObjectDetails(syncStatusDetails *syncStatusDetails) []database.Record { - details, err := u.objectStore.Query(database.Query{ +func (u *syncStatusUpdater) getSyncingObjects(spaceId string) []string { + ids, _, err := u.objectStore.QueryObjectIDs(database.Query{ Filters: []*model.BlockContentDataviewFilter{ { RelationKey: bundle.RelationKeySyncStatus.String(), - Condition: model.BlockContentDataviewFilter_NotEqual, - Value: pbtypes.Int64(int64(syncStatusDetails.status)), + Condition: model.BlockContentDataviewFilter_Equal, + Value: pbtypes.Int64(int64(domain.ObjectSyncing)), }, { RelationKey: bundle.RelationKeySpaceId.String(), Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.String(syncStatusDetails.spaceId), + Value: pbtypes.String(spaceId), }, }, }) if err != nil { log.Errorf("failed to update object details %s", err) } - return details + return ids } func (u *syncStatusUpdater) updateObjectDetails(syncStatusDetails *syncStatusDetails, objectId string) error { @@ -296,22 +306,45 @@ func (u *syncStatusUpdater) hasRelationsChange(record *types.Struct, status doma func (u *syncStatusUpdater) processEvents() { defer close(u.finish) + updateSpecificObject := func(details *syncStatusDetails) { + u.mx.Lock() + objectStatus := u.entries[details.objectId] + delete(u.entries, details.objectId) + u.mx.Unlock() + if objectStatus != nil { + err := u.updateObjectDetails(objectStatus, details.objectId) + if err != nil { + log.Errorf("failed to update details %s", err) + } + } + } + syncAllObjectsExcept := func(details *syncStatusDetails) { + ids := u.getSyncingObjects(details.spaceId) + removed, added := slice.DifferenceRemovedAdded(details.markAllSyncedExcept, ids) + details.status = domain.ObjectSynced + for _, id := range added { + err := u.updateObjectDetails(details, id) + if err != nil { + log.Errorf("failed to update details %s", err) + } + } + details.status = domain.ObjectSyncing + for _, id := range removed { + err := u.updateObjectDetails(details, id) + if err != nil { + log.Errorf("failed to update details %s", err) + } + } + } for { status, err := u.batcher.WaitOne(u.ctx) if err != nil { return } - for _, id := range status.objectIds { - u.mx.Lock() - objectStatus := u.entries[id] - delete(u.entries, id) - u.mx.Unlock() - if objectStatus != nil { - err := u.updateObjectDetails(objectStatus, id) - if err != nil { - log.Errorf("failed to update details %s", err) - } - } + if status.objectId == "" { + syncAllObjectsExcept(status) + } else { + updateSpecificObject(status) } } } diff --git a/core/syncstatus/objectsyncstatus/syncstatus.go b/core/syncstatus/objectsyncstatus/syncstatus.go index 56000d567..f81c782d2 100644 --- a/core/syncstatus/objectsyncstatus/syncstatus.go +++ b/core/syncstatus/objectsyncstatus/syncstatus.go @@ -77,7 +77,7 @@ type treeStatus struct { type Updater interface { app.Component - UpdateDetails(objectId []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) + UpdateDetails(objectId string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) } type syncStatusService struct { @@ -354,5 +354,5 @@ func (s *syncStatusService) updateDetails(treeId string, status domain.ObjectSyn syncErr = domain.IncompatibleVersion status = domain.ObjectError } - s.syncDetailsUpdater.UpdateDetails([]string{treeId}, status, syncErr, s.spaceId) + s.syncDetailsUpdater.UpdateDetails(treeId, status, syncErr, s.spaceId) } From 49723190f5191b87853ea032ec0bd4c2165085f5 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 17 Jul 2024 21:26:34 +0200 Subject: [PATCH 13/71] GO-3769 Update interval --- core/syncstatus/objectsyncstatus/syncstatus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/syncstatus/objectsyncstatus/syncstatus.go b/core/syncstatus/objectsyncstatus/syncstatus.go index f81c782d2..1bc1ded09 100644 --- a/core/syncstatus/objectsyncstatus/syncstatus.go +++ b/core/syncstatus/objectsyncstatus/syncstatus.go @@ -24,7 +24,7 @@ import ( ) const ( - syncUpdateInterval = 5 + syncUpdateInterval = 2 syncTimeout = time.Second ) From 2161fd71085b004320b00a56c0ba9bcc21828bdf Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 18 Jul 2024 16:20:51 +0200 Subject: [PATCH 14/71] GO-3769 WIP refactor sync logic --- core/block/object/treesyncer/treesyncer.go | 18 +- core/syncstatus/detailsupdater/updater.go | 60 ++-- core/syncstatus/filestatus.go | 10 +- .../syncstatus/objectsyncstatus/syncstatus.go | 132 +------- .../syncstatus/spacesyncstatus/objectstate.go | 1 + .../syncstatus/spacesyncstatus/spacestatus.go | 307 +++++++++--------- space/spacecore/peermanager/manager.go | 6 +- 7 files changed, 182 insertions(+), 352 deletions(-) diff --git a/core/block/object/treesyncer/treesyncer.go b/core/block/object/treesyncer/treesyncer.go index 87ceb36ec..03d1d39b2 100644 --- a/core/block/object/treesyncer/treesyncer.go +++ b/core/block/object/treesyncer/treesyncer.go @@ -15,8 +15,6 @@ import ( "github.com/anyproto/any-sync/net/streampool" "github.com/anyproto/any-sync/nodeconf" "go.uber.org/zap" - - "github.com/anyproto/anytype-heart/core/domain" ) var log = logger.NewNamed(treemanager.CName) @@ -69,7 +67,7 @@ type PeerStatusChecker interface { type SyncDetailsUpdater interface { app.Component - UpdateSpaceDetails(existing, missing []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) + UpdateSpaceDetails(existing, missing []string, spaceId string) } type treeSyncer struct { @@ -165,7 +163,7 @@ func (t *treeSyncer) SyncAll(ctx context.Context, peerId string, existing, missi t.Lock() defer t.Unlock() isResponsible := slices.Contains(t.nodeConf.NodeIds(t.spaceId), peerId) - t.sendSyncEvents(peerId, existing, missing, isResponsible) + t.sendSyncEvents(existing, missing, isResponsible) reqExec, exists := t.requestPools[peerId] if !exists { reqExec = newExecutor(t.requests, 0) @@ -204,19 +202,15 @@ func (t *treeSyncer) SyncAll(ctx context.Context, peerId string, existing, missi return nil } -func (t *treeSyncer) sendSyncEvents(peerId string, existing, missing []string, nodePeer bool) { +func (t *treeSyncer) sendSyncEvents(existing, missing []string, nodePeer bool) { if !nodePeer { return } - if t.peerManager.IsPeerOffline(peerId) { - t.sendDetailsUpdates(existing, missing, domain.ObjectError, domain.NetworkError) - } else { - t.sendDetailsUpdates(existing, missing, domain.ObjectSyncing, domain.Null) - } + t.sendDetailsUpdates(existing, missing) } -func (t *treeSyncer) sendDetailsUpdates(existing, missing []string, status domain.ObjectSyncStatus, syncError domain.SyncError) { - t.syncDetailsUpdater.UpdateSpaceDetails(existing, missing, status, syncError, t.spaceId) +func (t *treeSyncer) sendDetailsUpdates(existing, missing []string) { + t.syncDetailsUpdater.UpdateSpaceDetails(existing, missing, t.spaceId) } func (t *treeSyncer) requestTree(peerId, id string) { diff --git a/core/syncstatus/detailsupdater/updater.go b/core/syncstatus/detailsupdater/updater.go index eff148bc6..16eae9492 100644 --- a/core/syncstatus/detailsupdater/updater.go +++ b/core/syncstatus/detailsupdater/updater.go @@ -36,19 +36,18 @@ type syncStatusDetails struct { objectId string markAllSyncedExcept []string status domain.ObjectSyncStatus - syncError domain.SyncError spaceId string } type Updater interface { app.ComponentRunnable - UpdateSpaceDetails(existing, missing []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) - UpdateDetails(objectId string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) + UpdateSpaceDetails(existing, missing []string, spaceId string) + UpdateDetails(objectId string, status domain.ObjectSyncStatus, spaceId string) } type SpaceStatusUpdater interface { app.Component - SendUpdate(status *domain.SpaceSync) + Refresh(spaceId string) UpdateMissingIds(spaceId string, ids []string) } @@ -99,40 +98,38 @@ func (u *syncStatusUpdater) Name() (name string) { return CName } -func (u *syncStatusUpdater) UpdateDetails(objectId string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) { +func (u *syncStatusUpdater) UpdateDetails(objectId string, status domain.ObjectSyncStatus, spaceId string) { if spaceId == u.spaceService.TechSpaceId() { return } u.mx.Lock() u.entries[objectId] = &syncStatusDetails{ - objectId: objectId, - status: status, - syncError: syncError, - spaceId: spaceId, + objectId: objectId, + status: status, + spaceId: spaceId, } u.mx.Unlock() err := u.batcher.TryAdd(&syncStatusDetails{ - objectId: objectId, - status: status, - syncError: syncError, - spaceId: spaceId, + objectId: objectId, + status: status, + spaceId: spaceId, }) if err != nil { log.Errorf("failed to add sync details update to queue: %s", err) } } -func (u *syncStatusUpdater) UpdateSpaceDetails(existing, missing []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) { +func (u *syncStatusUpdater) UpdateSpaceDetails(existing, missing []string, spaceId string) { if spaceId == u.spaceService.TechSpaceId() { return } u.spaceSyncStatus.UpdateMissingIds(spaceId, missing) err := u.batcher.TryAdd(&syncStatusDetails{ markAllSyncedExcept: existing, - status: status, - syncError: syncError, + status: domain.ObjectSyncing, spaceId: spaceId, }) + fmt.Println("[x]: sending update to batcher, len(existing)", len(existing), "len(missing)", len(missing), "spaceId", spaceId) if err != nil { log.Errorf("failed to add sync details update to queue: %s", err) } @@ -169,7 +166,7 @@ func (u *syncStatusUpdater) updateObjectDetails(syncStatusDetails *syncStatusDet func (u *syncStatusUpdater) setObjectDetails(syncStatusDetails *syncStatusDetails, record *types.Struct, objectId string) error { status := syncStatusDetails.status - syncError := syncStatusDetails.syncError + syncError := domain.Null isFileStatus := false if fileStatus, ok := record.GetFields()[bundle.RelationKeyFileBackupStatus.String()]; ok { isFileStatus = true @@ -187,8 +184,7 @@ func (u *syncStatusUpdater) setObjectDetails(syncStatusDetails *syncStatusDetail if err != nil { return err } - spaceStatus := mapObjectSyncToSpaceSyncStatus(status, syncError) - defer u.sendSpaceStatusUpdate(err, syncStatusDetails, spaceStatus, syncError) + defer u.spaceSyncStatus.Refresh(syncStatusDetails.spaceId) err = spc.DoLockedIfNotExists(objectId, func() error { return u.objectStore.ModifyObjectDetails(objectId, func(details *types.Struct) (*types.Struct, error) { if details == nil || details.Fields == nil { @@ -223,27 +219,6 @@ func (u *syncStatusUpdater) isLayoutSuitableForSyncRelations(details *types.Stru return !slices.Contains(layoutsWithoutSyncRelations, layout) } -func mapObjectSyncToSpaceSyncStatus(status domain.ObjectSyncStatus, syncError domain.SyncError) domain.SpaceSyncStatus { - switch status { - case domain.ObjectSynced: - return domain.Synced - case domain.ObjectSyncing, domain.ObjectQueued: - return domain.Syncing - case domain.ObjectError: - // don't send error to space if file were oversized - if syncError != domain.Oversized { - return domain.Error - } - } - return domain.Synced -} - -func (u *syncStatusUpdater) sendSpaceStatusUpdate(err error, syncStatusDetails *syncStatusDetails, status domain.SpaceSyncStatus, syncError domain.SyncError) { - if err == nil { - u.spaceSyncStatus.SendUpdate(domain.MakeSyncStatus(syncStatusDetails.spaceId, status, syncError, domain.Objects)) - } -} - func mapFileStatus(status filesyncstatus.Status) (domain.ObjectSyncStatus, domain.SyncError) { var syncError domain.SyncError switch status { @@ -321,6 +296,11 @@ func (u *syncStatusUpdater) processEvents() { syncAllObjectsExcept := func(details *syncStatusDetails) { ids := u.getSyncingObjects(details.spaceId) removed, added := slice.DifferenceRemovedAdded(details.markAllSyncedExcept, ids) + if len(removed)+len(added) == 0 { + u.spaceSyncStatus.Refresh(details.spaceId) + return + } + fmt.Println("[x]: marking synced, len(synced)", len(added), "len(syncing)", len(removed), "spaceId", details.spaceId) details.status = domain.ObjectSynced for _, id := range added { err := u.updateObjectDetails(details, id) diff --git a/core/syncstatus/filestatus.go b/core/syncstatus/filestatus.go index fd75d6781..f484e49cb 100644 --- a/core/syncstatus/filestatus.go +++ b/core/syncstatus/filestatus.go @@ -30,7 +30,7 @@ func (s *service) onFileLimited(objectId string, _ domain.FullFileId, bytesLeftP } func (s *service) OnFileDelete(fileId domain.FullFileId) { - s.sendSpaceStatusUpdate(filesyncstatus.Synced, fileId.SpaceId, 0) + s.spaceSyncStatus.Refresh(fileId.SpaceId) } func (s *service) indexFileSyncStatus(fileObjectId string, status filesyncstatus.Status, bytesLeftPercentage float64) error { @@ -56,7 +56,7 @@ func (s *service) indexFileSyncStatus(fileObjectId string, status filesyncstatus if err != nil { return fmt.Errorf("update tree: %w", err) } - s.sendSpaceStatusUpdate(status, spaceId, bytesLeftPercentage) + s.spaceSyncStatus.Refresh(spaceId) return nil } @@ -82,12 +82,6 @@ func provideFileStatusDetails(status filesyncstatus.Status, newStatus int64) []* return details } -func (s *service) sendSpaceStatusUpdate(status filesyncstatus.Status, spaceId string, bytesLeftPercentage float64) { - spaceStatus, spaceError := getSyncStatus(status, bytesLeftPercentage) - syncStatus := domain.MakeSyncStatus(spaceId, spaceStatus, spaceError, domain.Files) - s.spaceSyncStatus.SendUpdate(syncStatus) -} - func getFileObjectStatus(status filesyncstatus.Status) (domain.ObjectSyncStatus, domain.SyncError) { var ( objectSyncStatus domain.ObjectSyncStatus diff --git a/core/syncstatus/objectsyncstatus/syncstatus.go b/core/syncstatus/objectsyncstatus/syncstatus.go index 1bc1ded09..5653c4013 100644 --- a/core/syncstatus/objectsyncstatus/syncstatus.go +++ b/core/syncstatus/objectsyncstatus/syncstatus.go @@ -2,7 +2,6 @@ package objectsyncstatus import ( "context" - "fmt" "sync" "time" @@ -11,7 +10,6 @@ import ( "github.com/anyproto/any-sync/commonspace/spacestate" "github.com/anyproto/any-sync/commonspace/syncstatus" - "github.com/anyproto/any-sync/commonspace/object/tree/treestorage" "github.com/anyproto/any-sync/commonspace/spacestorage" "github.com/anyproto/any-sync/nodeconf" "github.com/anyproto/any-sync/util/periodicsync" @@ -20,7 +18,6 @@ import ( "github.com/anyproto/anytype-heart/core/anytype/config" "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" - "github.com/anyproto/anytype-heart/util/slice" ) const ( @@ -63,13 +60,6 @@ type StatusService interface { StatusWatcher } -type treeHeadsEntry struct { - heads []string - stateCounter uint64 - isUpdated bool - syncStatus SyncStatus -} - type treeStatus struct { treeId string status SyncStatus @@ -77,7 +67,7 @@ type treeStatus struct { type Updater interface { app.Component - UpdateDetails(objectId string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) + UpdateDetails(objectId string, status domain.ObjectSyncStatus, spaceId string) } type syncStatusService struct { @@ -88,8 +78,6 @@ type syncStatusService struct { storage spacestorage.SpaceStorage spaceId string - treeHeads map[string]treeHeadsEntry - watchers map[string]struct{} synced []string tempSynced map[string]struct{} stateCounter uint64 @@ -105,8 +93,6 @@ type syncStatusService struct { func NewSyncStatusService() StatusService { return &syncStatusService{ - treeHeads: map[string]treeHeadsEntry{}, - watchers: map[string]struct{}{}, tempSynced: map[string]struct{}{}, } } @@ -147,18 +133,6 @@ func (s *syncStatusService) Run(ctx context.Context) error { } func (s *syncStatusService) HeadsChange(treeId string, heads []string) { - s.Lock() - defer s.Unlock() - - var headsCopy []string - headsCopy = append(headsCopy, heads...) - - s.treeHeads[treeId] = treeHeadsEntry{ - heads: headsCopy, - stateCounter: s.stateCounter, - syncStatus: StatusNotSynced, - } - s.stateCounter++ s.updateDetails(treeId, domain.ObjectSyncing) } @@ -169,10 +143,6 @@ func (s *syncStatusService) ObjectReceive(senderId, treeId string, heads []strin s.tempSynced[treeId] = struct{}{} return } - _, ok := s.treeHeads[treeId] - if ok { - return - } s.synced = append(s.synced, treeId) } @@ -185,10 +155,6 @@ func (s *syncStatusService) HeadsApply(senderId, treeId string, heads []string, } return } - _, ok := s.treeHeads[treeId] - if ok { - return - } if allAdded { s.synced = append(s.synced, treeId) } @@ -201,20 +167,6 @@ func (s *syncStatusService) update(ctx context.Context) (err error) { s.Unlock() return } - for treeId := range s.watchers { - // that means that we haven't yet got the status update - treeHeads, exists := s.treeHeads[treeId] - if !exists { - err = fmt.Errorf("treeHeads should always exist for watchers") - s.Unlock() - return - } - if !treeHeads.isUpdated { - treeHeads.isUpdated = true - s.treeHeads[treeId] = treeHeads - treeStatusBuf = append(treeStatusBuf, treeStatus{treeId, treeHeads.syncStatus}) - } - } for _, treeId := range s.synced { treeStatusBuf = append(treeStatusBuf, treeStatus{treeId, StatusSynced}) } @@ -239,67 +191,13 @@ func mapStatus(status SyncStatus) domain.ObjectSyncStatus { } func (s *syncStatusService) HeadsReceive(senderId, treeId string, heads []string) { - s.Lock() - defer s.Unlock() - - curTreeHeads, ok := s.treeHeads[treeId] - if !ok || curTreeHeads.syncStatus == StatusSynced { - return - } - - // checking if other node is responsible - if len(heads) == 0 || !s.isSenderResponsible(senderId) { - return - } - - // checking if we received the head that we are interested in - for _, head := range heads { - if idx, found := slices.BinarySearch(curTreeHeads.heads, head); found { - curTreeHeads.heads[idx] = "" - } - } - curTreeHeads.heads = slice.RemoveMut(curTreeHeads.heads, "") - if len(curTreeHeads.heads) == 0 { - curTreeHeads.syncStatus = StatusSynced - curTreeHeads.isUpdated = false - } - s.treeHeads[treeId] = curTreeHeads } func (s *syncStatusService) Watch(treeId string) (err error) { - s.Lock() - defer s.Unlock() - _, ok := s.treeHeads[treeId] - if !ok { - var ( - st treestorage.TreeStorage - heads []string - ) - st, err = s.storage.TreeStorage(treeId) - if err != nil { - return - } - heads, err = st.Heads() - if err != nil { - return - } - slices.Sort(heads) - s.stateCounter++ - s.treeHeads[treeId] = treeHeadsEntry{ - heads: heads, - stateCounter: s.stateCounter, - syncStatus: StatusUnknown, - } - } - - s.watchers[treeId] = struct{}{} - return + return nil } func (s *syncStatusService) Unwatch(treeId string) { - s.Lock() - defer s.Unlock() - delete(s.watchers, treeId) } func (s *syncStatusService) RemoveAllExcept(senderId string, differentRemoteIds []string) { @@ -312,21 +210,6 @@ func (s *syncStatusService) RemoveAllExcept(senderId string, differentRemoteIds defer s.Unlock() slices.Sort(differentRemoteIds) - for treeId, entry := range s.treeHeads { - // if the current update is outdated - if entry.stateCounter > s.stateCounter { - continue - } - // if we didn't find our treeId in heads ids which are different from us and node - if _, found := slices.BinarySearch(differentRemoteIds, treeId); !found { - if entry.syncStatus != StatusSynced { - entry.syncStatus = StatusSynced - entry.isUpdated = false - s.treeHeads[treeId] = entry - } - } - } - // responsible node has those ids for treeId := range s.tempSynced { delete(s.tempSynced, treeId) if _, found := slices.BinarySearch(differentRemoteIds, treeId); !found { @@ -345,14 +228,5 @@ func (s *syncStatusService) isSenderResponsible(senderId string) bool { } func (s *syncStatusService) updateDetails(treeId string, status domain.ObjectSyncStatus) { - var syncErr domain.SyncError - if s.nodeStatus.GetNodeStatus(s.spaceId) != nodestatus.Online || s.config.IsLocalOnlyMode() { - syncErr = domain.NetworkError - status = domain.ObjectError - } - if s.nodeConfService.NetworkCompatibilityStatus() == nodeconf.NetworkCompatibilityStatusIncompatible { - syncErr = domain.IncompatibleVersion - status = domain.ObjectError - } - s.syncDetailsUpdater.UpdateDetails(treeId, status, syncErr, s.spaceId) + s.syncDetailsUpdater.UpdateDetails(treeId, status, s.spaceId) } diff --git a/core/syncstatus/spacesyncstatus/objectstate.go b/core/syncstatus/spacesyncstatus/objectstate.go index c5a2a0564..9d9fd3907 100644 --- a/core/syncstatus/spacesyncstatus/objectstate.go +++ b/core/syncstatus/spacesyncstatus/objectstate.go @@ -45,6 +45,7 @@ func (o *ObjectState) SetObjectsNumber(status *domain.SpaceSync) { return pbtypes.GetString(r.Details, bundle.RelationKeyId.String()) }) _, added := slice.DifferenceRemovedAdded(ids, status.MissingObjects) + fmt.Println("[x]: added", len(added), "records", len(records)) o.objectSyncCountBySpace[status.SpaceId] = len(records) + len(added) } } diff --git a/core/syncstatus/spacesyncstatus/spacestatus.go b/core/syncstatus/spacesyncstatus/spacestatus.go index e50d23141..d9fe3af89 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus.go +++ b/core/syncstatus/spacesyncstatus/spacestatus.go @@ -7,16 +7,24 @@ import ( "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/app/logger" + "github.com/anyproto/any-sync/nodeconf" "github.com/anyproto/any-sync/util/periodicsync" "github.com/cheggaaa/mb/v3" "github.com/samber/lo" "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/event" + "github.com/anyproto/anytype-heart/core/files" "github.com/anyproto/anytype-heart/core/session" + "github.com/anyproto/anytype-heart/core/syncstatus/filesyncstatus" + "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/database" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/logging" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/pbtypes" "github.com/anyproto/anytype-heart/util/slice" ) @@ -24,12 +32,20 @@ const service = "core.syncstatus.spacesyncstatus" var log = logging.Logger("anytype-mw-space-status") +// nodeconfservice +// nodestatus +// GetNodeUsage(ctx context.Context) (*NodeUsageResponse, error) + type Updater interface { app.ComponentRunnable - SendUpdate(spaceSync *domain.SpaceSync) + Refresh(spaceId string) UpdateMissingIds(spaceId string, ids []string) } +type NodeUsage interface { + GetNodeUsage(ctx context.Context) (*files.NodeUsageResponse, error) +} + type SpaceIdGetter interface { app.Component TechSpaceId() string @@ -52,6 +68,10 @@ type NetworkConfig interface { type spaceSyncStatus struct { eventSender event.Sender networkConfig NetworkConfig + nodeStatus nodestatus.NodeStatus + nodeConf nodeconf.Service + nodeUsage NodeUsage + store objectstore.ObjectStore batcher *mb.MB[*domain.SpaceSync] filesState State @@ -60,7 +80,7 @@ type spaceSyncStatus struct { ctx context.Context ctxCancel context.CancelFunc spaceIdGetter SpaceIdGetter - curStatuses map[string]*domain.SpaceSync + curStatuses map[string]struct{} missingIds map[string][]string mx sync.Mutex periodicCall periodicsync.PeriodicSync @@ -74,11 +94,12 @@ func NewSpaceSyncStatus() Updater { func (s *spaceSyncStatus) Init(a *app.App) (err error) { s.eventSender = app.MustComponent[event.Sender](a) s.networkConfig = app.MustComponent[NetworkConfig](a) - store := app.MustComponent[objectstore.ObjectStore](a) - s.curStatuses = make(map[string]*domain.SpaceSync) + s.nodeStatus = app.MustComponent[nodestatus.NodeStatus](a) + s.nodeConf = app.MustComponent[nodeconf.Service](a) + s.nodeUsage = app.MustComponent[NodeUsage](a) + s.store = app.MustComponent[objectstore.ObjectStore](a) + s.curStatuses = make(map[string]struct{}) s.missingIds = make(map[string][]string) - s.filesState = NewFileState(store) - s.objectsState = NewObjectState(store) s.spaceIdGetter = app.MustComponent[SpaceIdGetter](a) sessionHookRunner := app.MustComponent[session.HookRunner](a) sessionHookRunner.RegisterHook(s.sendSyncEventForNewSession) @@ -122,28 +143,37 @@ func (s *spaceSyncStatus) update(ctx context.Context) error { missingIds := lo.MapEntries(s.missingIds, func(key string, value []string) (string, []string) { return key, slice.Copy(value) }) - statuses := lo.MapToSlice(s.curStatuses, func(key string, value *domain.SpaceSync) *domain.SpaceSync { + statuses := lo.MapToSlice(s.curStatuses, func(key string, value struct{}) string { delete(s.curStatuses, key) - return value + return key }) s.mx.Unlock() - for _, st := range statuses { - if st.SpaceId == s.spaceIdGetter.TechSpaceId() { + for _, spaceId := range statuses { + if spaceId == s.spaceIdGetter.TechSpaceId() { continue } - st.MissingObjects = missingIds[st.SpaceId] // if the there are too many updates and this hurts performance, // we may skip some iterations and not do the updates for example - s.updateSpaceSyncStatus(st) + s.updateSpaceSyncStatus(spaceId, missingIds[spaceId]) } return nil } func (s *spaceSyncStatus) sendEventToSession(spaceId, token string) { + s.mx.Lock() + missingObjects := s.missingIds[spaceId] + s.mx.Unlock() + params := syncParams{ + bytesLeftPercentage: s.getBytesLeftPercentage(spaceId), + connectionStatus: s.nodeStatus.GetNodeStatus(spaceId), + compatibility: s.nodeConf.NetworkCompatibilityStatus(), + filesSyncingCount: s.getFileSyncingObjectsCount(spaceId), + objectsSyncingCount: s.getObjectSyncingObjectsCount(spaceId, missingObjects), + } s.eventSender.SendToSession(token, &pb.Event{ Messages: []*pb.EventMessage{{ Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ - SpaceSyncStatusUpdate: s.makeSpaceSyncEvent(spaceId), + SpaceSyncStatusUpdate: s.makeSyncEvent(spaceId, params), }, }}, }) @@ -151,13 +181,10 @@ func (s *spaceSyncStatus) sendEventToSession(spaceId, token string) { func (s *spaceSyncStatus) sendStartEvent(spaceIds []string) { for _, id := range spaceIds { - s.eventSender.Broadcast(&pb.Event{ - Messages: []*pb.EventMessage{{ - Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ - SpaceSyncStatusUpdate: s.makeSpaceSyncEvent(id), - }, - }}, - }) + s.mx.Lock() + missingObjects := s.missingIds[id] + s.mx.Unlock() + s.updateSpaceSyncStatus(id, missingObjects) } } @@ -174,58 +201,90 @@ func (s *spaceSyncStatus) sendLocalOnlyEvent() { }) } -func (s *spaceSyncStatus) SendUpdate(status *domain.SpaceSync) { +func (s *spaceSyncStatus) Refresh(spaceId string) { s.mx.Lock() defer s.mx.Unlock() - s.curStatuses[status.SpaceId] = status + s.curStatuses[spaceId] = struct{}{} } -func (s *spaceSyncStatus) updateSpaceSyncStatus(receivedStatus *domain.SpaceSync) { - currSyncStatus := s.getSpaceSyncStatus(receivedStatus.SpaceId) - if s.isStatusNotChanged(receivedStatus, currSyncStatus) { - return +func (s *spaceSyncStatus) getObjectSyncingObjectsCount(spaceId string, missingObjects []string) int { + ids, _, err := s.store.QueryObjectIDs(database.Query{ + Filters: []*model.BlockContentDataviewFilter{ + { + RelationKey: bundle.RelationKeySyncStatus.String(), + Condition: model.BlockContentDataviewFilter_Equal, + Value: pbtypes.Int64(int64(domain.Syncing)), + }, + { + RelationKey: bundle.RelationKeyLayout.String(), + Condition: model.BlockContentDataviewFilter_NotIn, + Value: pbtypes.IntList( + int(model.ObjectType_file), + int(model.ObjectType_image), + int(model.ObjectType_video), + int(model.ObjectType_audio), + ), + }, + { + RelationKey: bundle.RelationKeySpaceId.String(), + Condition: model.BlockContentDataviewFilter_Equal, + Value: pbtypes.String(spaceId), + }, + }, + }) + if err != nil { + log.Errorf("failed to query file status: %s", err) } - state := s.getCurrentState(receivedStatus) - prevObjectNumber := s.getObjectNumber(receivedStatus.SpaceId) - state.SetObjectsNumber(receivedStatus) - newObjectNumber := s.getObjectNumber(receivedStatus.SpaceId) - state.SetSyncStatusAndErr(receivedStatus.Status, receivedStatus.SyncError, receivedStatus.SpaceId) + _, added := slice.DifferenceRemovedAdded(ids, missingObjects) + return len(ids) + len(added) +} - spaceStatus := s.getSpaceSyncStatus(receivedStatus.SpaceId) - - // send synced event only if files and objects are all synced - if !s.needToSendEvent(spaceStatus, currSyncStatus, prevObjectNumber, newObjectNumber) { - return +func (s *spaceSyncStatus) getFileSyncingObjectsCount(spaceId string) int { + _, num, err := s.store.QueryObjectIDs(database.Query{ + Filters: []*model.BlockContentDataviewFilter{ + { + RelationKey: bundle.RelationKeyFileBackupStatus.String(), + Condition: model.BlockContentDataviewFilter_In, + Value: pbtypes.IntList(int(filesyncstatus.Syncing), int(filesyncstatus.Queued)), + }, + { + RelationKey: bundle.RelationKeySpaceId.String(), + Condition: model.BlockContentDataviewFilter_Equal, + Value: pbtypes.String(spaceId), + }, + }, + }) + if err != nil { + log.Errorf("failed to query file status: %s", err) } + return num +} + +func (s *spaceSyncStatus) getBytesLeftPercentage(spaceId string) float64 { + nodeUsage, err := s.nodeUsage.GetNodeUsage(context.Background()) + if err != nil { + log.Errorf("failed to get node usage: %s", err) + return 0 + } + return float64(nodeUsage.Usage.BytesLeft) / float64(nodeUsage.Usage.AccountBytesLimit) +} + +func (s *spaceSyncStatus) updateSpaceSyncStatus(spaceId string, missingObjects []string) { + params := syncParams{ + bytesLeftPercentage: s.getBytesLeftPercentage(spaceId), + connectionStatus: s.nodeStatus.GetNodeStatus(spaceId), + compatibility: s.nodeConf.NetworkCompatibilityStatus(), + filesSyncingCount: s.getFileSyncingObjectsCount(spaceId), + objectsSyncingCount: s.getObjectSyncingObjectsCount(spaceId, missingObjects), + } + // fmt.Println("[x]: space status", event.Status, "space id", receivedStatus.SpaceId, "network", event.Network, "error", event.Error, "object number", event.SyncingObjectsCounter, "isFile", isFileState) s.eventSender.Broadcast(&pb.Event{ Messages: []*pb.EventMessage{{ Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ - SpaceSyncStatusUpdate: s.makeSpaceSyncEvent(receivedStatus.SpaceId), + SpaceSyncStatusUpdate: s.makeSyncEvent(spaceId, params), }, }}, }) - state.ResetSpaceErrorStatus(receivedStatus.SpaceId, receivedStatus.SyncError) -} - -func (s *spaceSyncStatus) isStatusNotChanged(status *domain.SpaceSync, syncStatus domain.SpaceSyncStatus) bool { - if status.Status == domain.Syncing { - // we need to check if number of syncing object is changed first - return false - } - syncErrNotChanged := s.getError(status.SpaceId) == mapError(status.SyncError) - if syncStatus == domain.Unknown { - return false - } - statusNotChanged := syncStatus == status.Status - if syncErrNotChanged && statusNotChanged { - return true - } - return false -} - -func (s *spaceSyncStatus) needToSendEvent(status domain.SpaceSyncStatus, currSyncStatus domain.SpaceSyncStatus, prevObjectNumber int64, newObjectNumber int64) bool { - // that because we get update on syncing objects count, so we need to send updated object counter to client - return (status == domain.Syncing && prevObjectNumber != newObjectNumber) || currSyncStatus != status } func (s *spaceSyncStatus) Close(ctx context.Context) (err error) { @@ -233,86 +292,42 @@ func (s *spaceSyncStatus) Close(ctx context.Context) (err error) { return } -func (s *spaceSyncStatus) makeSpaceSyncEvent(spaceId string) *pb.EventSpaceSyncStatusUpdate { +type syncParams struct { + bytesLeftPercentage float64 + connectionStatus nodestatus.ConnectionStatus + compatibility nodeconf.NetworkCompatibilityStatus + filesSyncingCount int + objectsSyncingCount int +} + +func (s *spaceSyncStatus) makeSyncEvent(spaceId string, params syncParams) *pb.EventSpaceSyncStatusUpdate { + status := pb.EventSpace_Synced + err := pb.EventSpace_Null + syncingObjectsCount := int64(params.objectsSyncingCount + params.filesSyncingCount) + if syncingObjectsCount > 0 { + status = pb.EventSpace_Syncing + } + if params.bytesLeftPercentage < 0.1 { + err = pb.EventSpace_StorageLimitExceed + } + if params.connectionStatus == nodestatus.ConnectionError { + status = pb.EventSpace_Offline + err = pb.EventSpace_NetworkError + } + if params.compatibility == nodeconf.NetworkCompatibilityStatusIncompatible { + status = pb.EventSpace_Error + err = pb.EventSpace_IncompatibleVersion + } + return &pb.EventSpaceSyncStatusUpdate{ Id: spaceId, - Status: mapStatus(s.getSpaceSyncStatus(spaceId)), + Status: status, Network: mapNetworkMode(s.networkConfig.GetNetworkMode()), - Error: s.getError(spaceId), - SyncingObjectsCounter: s.getObjectNumber(spaceId), + Error: err, + SyncingObjectsCounter: syncingObjectsCount, } } -func (s *spaceSyncStatus) getObjectNumber(spaceId string) int64 { - return int64(s.filesState.GetSyncObjectCount(spaceId) + s.objectsState.GetSyncObjectCount(spaceId)) -} - -func (s *spaceSyncStatus) getSpaceSyncStatus(spaceId string) domain.SpaceSyncStatus { - filesStatus := s.filesState.GetSyncStatus(spaceId) - objectsStatus := s.objectsState.GetSyncStatus(spaceId) - - if s.isUnknown(filesStatus, objectsStatus) { - return domain.Unknown - } - if s.isOfflineStatus(filesStatus, objectsStatus) { - return domain.Offline - } - - if s.isSyncedStatus(filesStatus, objectsStatus) { - return domain.Synced - } - - if s.isErrorStatus(filesStatus, objectsStatus) { - return domain.Error - } - - if s.isSyncingStatus(filesStatus, objectsStatus) { - return domain.Syncing - } - return domain.Synced -} - -func (s *spaceSyncStatus) isSyncingStatus(filesStatus domain.SpaceSyncStatus, objectsStatus domain.SpaceSyncStatus) bool { - return filesStatus == domain.Syncing || objectsStatus == domain.Syncing -} - -func (s *spaceSyncStatus) isErrorStatus(filesStatus domain.SpaceSyncStatus, objectsStatus domain.SpaceSyncStatus) bool { - return filesStatus == domain.Error || objectsStatus == domain.Error -} - -func (s *spaceSyncStatus) isSyncedStatus(filesStatus domain.SpaceSyncStatus, objectsStatus domain.SpaceSyncStatus) bool { - return filesStatus == domain.Synced && objectsStatus == domain.Synced -} - -func (s *spaceSyncStatus) isOfflineStatus(filesStatus domain.SpaceSyncStatus, objectsStatus domain.SpaceSyncStatus) bool { - return filesStatus == domain.Offline || objectsStatus == domain.Offline -} - -func (s *spaceSyncStatus) getCurrentState(status *domain.SpaceSync) State { - if status.SyncType == domain.Files { - return s.filesState - } - return s.objectsState -} - -func (s *spaceSyncStatus) getError(spaceId string) pb.EventSpaceSyncError { - syncErr := s.filesState.GetSyncErr(spaceId) - if syncErr != domain.Null { - return mapError(syncErr) - } - - syncErr = s.objectsState.GetSyncErr(spaceId) - if syncErr != domain.Null { - return mapError(syncErr) - } - - return pb.EventSpace_Null -} - -func (s *spaceSyncStatus) isUnknown(filesStatus domain.SpaceSyncStatus, objectsStatus domain.SpaceSyncStatus) bool { - return filesStatus == domain.Unknown && objectsStatus == domain.Unknown -} - func mapNetworkMode(mode pb.RpcAccountNetworkMode) pb.EventSpaceNetwork { switch mode { case pb.RpcAccount_LocalOnly: @@ -323,29 +338,3 @@ func mapNetworkMode(mode pb.RpcAccountNetworkMode) pb.EventSpaceNetwork { return pb.EventSpace_Anytype } } - -func mapStatus(status domain.SpaceSyncStatus) pb.EventSpaceStatus { - switch status { - case domain.Syncing: - return pb.EventSpace_Syncing - case domain.Offline: - return pb.EventSpace_Offline - case domain.Error: - return pb.EventSpace_Error - default: - return pb.EventSpace_Synced - } -} - -func mapError(err domain.SyncError) pb.EventSpaceSyncError { - switch err { - case domain.NetworkError: - return pb.EventSpace_NetworkError - case domain.IncompatibleVersion: - return pb.EventSpace_IncompatibleVersion - case domain.StorageLimitExceed: - return pb.EventSpace_StorageLimitExceed - default: - return pb.EventSpace_Null - } -} diff --git a/space/spacecore/peermanager/manager.go b/space/spacecore/peermanager/manager.go index d2065898f..50258d707 100644 --- a/space/spacecore/peermanager/manager.go +++ b/space/spacecore/peermanager/manager.go @@ -15,7 +15,6 @@ import ( "github.com/anyproto/any-sync/net/peer" "go.uber.org/zap" - "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" "github.com/anyproto/anytype-heart/space/spacecore/peerstore" ) @@ -35,7 +34,7 @@ type NodeStatus interface { type Updater interface { app.ComponentRunnable - SendUpdate(spaceSync *domain.SpaceSync) + Refresh(spaceId string) } type PeerToPeerStatus interface { @@ -214,9 +213,8 @@ func (n *clientPeerManager) fetchResponsiblePeers() { for _, p := range n.responsiblePeers { n.nodeStatus.SetNodesStatus(n.spaceId, p.Id(), nodestatus.ConnectionError) } - n.spaceSyncService.SendUpdate(domain.MakeSyncStatus(n.spaceId, domain.Offline, domain.Null, domain.Objects)) } - + n.spaceSyncService.Refresh(n.spaceId) peerIds := n.peerStore.LocalPeerIds(n.spaceId) var needUpdate bool for _, peerId := range peerIds { From 73e36787901c4a7341d0ad9ab0b370b4eea0fa23 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 18 Jul 2024 16:54:17 +0200 Subject: [PATCH 15/71] GO-3769 Fix file sync --- core/syncstatus/spacesyncstatus/spacestatus.go | 8 +++++--- pkg/lib/localstore/objectstore/queries.go | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/core/syncstatus/spacesyncstatus/spacestatus.go b/core/syncstatus/spacesyncstatus/spacestatus.go index d9fe3af89..6b2b7ead4 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus.go +++ b/core/syncstatus/spacesyncstatus/spacestatus.go @@ -2,6 +2,7 @@ package spacesyncstatus import ( "context" + "fmt" "sync" "time" @@ -139,6 +140,7 @@ func (s *spaceSyncStatus) Run(ctx context.Context) (err error) { } func (s *spaceSyncStatus) update(ctx context.Context) error { + // TODO: use subscriptions inside middleware instead of this s.mx.Lock() missingIds := lo.MapEntries(s.missingIds, func(key string, value []string) (string, []string) { return key, slice.Copy(value) @@ -240,7 +242,7 @@ func (s *spaceSyncStatus) getObjectSyncingObjectsCount(spaceId string, missingOb } func (s *spaceSyncStatus) getFileSyncingObjectsCount(spaceId string) int { - _, num, err := s.store.QueryObjectIDs(database.Query{ + recs, _, err := s.store.QueryObjectIDs(database.Query{ Filters: []*model.BlockContentDataviewFilter{ { RelationKey: bundle.RelationKeyFileBackupStatus.String(), @@ -257,7 +259,7 @@ func (s *spaceSyncStatus) getFileSyncingObjectsCount(spaceId string) int { if err != nil { log.Errorf("failed to query file status: %s", err) } - return num + return len(recs) } func (s *spaceSyncStatus) getBytesLeftPercentage(spaceId string) float64 { @@ -318,7 +320,7 @@ func (s *spaceSyncStatus) makeSyncEvent(spaceId string, params syncParams) *pb.E status = pb.EventSpace_Error err = pb.EventSpace_IncompatibleVersion } - + fmt.Println("[x]: status: connection", params.connectionStatus, ", space id", spaceId, ", compatibility", params.compatibility, ", object number", syncingObjectsCount, ", bytes left", params.bytesLeftPercentage) return &pb.EventSpaceSyncStatusUpdate{ Id: spaceId, Status: status, diff --git a/pkg/lib/localstore/objectstore/queries.go b/pkg/lib/localstore/objectstore/queries.go index 4fc9e39b5..a2905b156 100644 --- a/pkg/lib/localstore/objectstore/queries.go +++ b/pkg/lib/localstore/objectstore/queries.go @@ -463,7 +463,7 @@ func (s *dsObjectStore) QueryObjectIDs(q database.Query) (ids []string, total in for _, rec := range recs { ids = append(ids, pbtypes.GetString(rec.Details, bundle.RelationKeyId.String())) } - return ids, 0, nil + return ids, len(recs), nil } func (s *dsObjectStore) QueryByID(ids []string) (records []database.Record, err error) { From aa7280173d994d5ef6d9b61e3c786e23e4aaf743 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 18 Jul 2024 23:34:55 +0200 Subject: [PATCH 16/71] GO-3769 Add subscription --- .../spacesyncstatus/objectsubscription.go | 198 ++++++++++++++++++ .../syncstatus/spacesyncstatus/spacestatus.go | 105 +++++----- .../spacesyncstatus/syncingobjects.go | 103 +++++++++ 3 files changed, 349 insertions(+), 57 deletions(-) create mode 100644 core/syncstatus/spacesyncstatus/objectsubscription.go create mode 100644 core/syncstatus/spacesyncstatus/syncingobjects.go diff --git a/core/syncstatus/spacesyncstatus/objectsubscription.go b/core/syncstatus/spacesyncstatus/objectsubscription.go new file mode 100644 index 000000000..46d33f6db --- /dev/null +++ b/core/syncstatus/spacesyncstatus/objectsubscription.go @@ -0,0 +1,198 @@ +package spacesyncstatus + +import ( + "context" + "sync" + + "github.com/cheggaaa/mb/v3" + "github.com/gogo/protobuf/types" + "github.com/huandu/skiplist" + + "github.com/anyproto/anytype-heart/core/subscription" + "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/util/pbtypes" +) + +type entry[T any] struct { + id string + data T +} + +func newEmptyEntry[T any](id string) *entry[T] { + return &entry[T]{id: id} +} + +func newEntry[T any](id string, data T) *entry[T] { + return &entry[T]{id: id, data: data} +} + +type ( + extract[T any] func(*types.Struct) (string, T) + compare[T any] func(T, T) int + update[T any] func(string, *types.Value, T) T + unset[T any] func([]string, T) T +) + +type SubscriptionParams[T any] struct { + Request subscription.SubscribeRequest + Extract extract[T] + Order compare[T] + Update update[T] + Unset unset[T] +} + +func NewIdSubscription(service subscription.Service, request subscription.SubscribeRequest) *ObjectSubscription[struct{}] { + return &ObjectSubscription[struct{}]{ + request: request, + service: service, + ch: make(chan struct{}), + order: nil, + extract: func(t *types.Struct) (string, struct{}) { + return pbtypes.GetString(t, bundle.RelationKeyId.String()), struct{}{} + }, + update: func(s string, value *types.Value, s2 struct{}) struct{} { + return struct{}{} + }, + unset: func(strings []string, s struct{}) struct{} { + return struct{}{} + }, + } +} + +func NewObjectSubscription[T any](service subscription.Service, params SubscriptionParams[T]) *ObjectSubscription[T] { + return &ObjectSubscription[T]{ + request: params.Request, + service: service, + ch: make(chan struct{}), + order: params.Order, + extract: params.Extract, + update: params.Update, + unset: params.Unset, + } +} + +type ObjectSubscription[T any] struct { + request subscription.SubscribeRequest + service subscription.Service + ch chan struct{} + events *mb.MB[*pb.EventMessage] + ctx context.Context + cancel context.CancelFunc + skl *skiplist.SkipList + order compare[T] + extract extract[T] + update update[T] + unset unset[T] + mx sync.Mutex +} + +func (o *ObjectSubscription[T]) Run() error { + resp, err := o.service.Search(o.request) + if err != nil { + return err + } + o.ctx, o.cancel = context.WithCancel(context.Background()) + o.events = resp.Output + o.skl = skiplist.New(o) + for _, rec := range resp.Records { + id, data := o.extract(rec) + e := &entry[T]{id: id, data: data} + o.skl.Set(e, nil) + } + go o.read() + return nil +} + +func (o *ObjectSubscription[T]) Close() { + o.cancel() + <-o.ch + return +} + +func (o *ObjectSubscription[T]) Len() int { + o.mx.Lock() + defer o.mx.Unlock() + return o.skl.Len() +} + +func (o *ObjectSubscription[T]) Iterate(iter func(id string, data T) bool) { + o.mx.Lock() + defer o.mx.Unlock() + cur := o.skl.Front() + for cur != nil { + el := cur.Key().(*entry[T]) + if !iter(el.id, el.data) { + return + } + cur = cur.Next() + } + return +} + +func (o *ObjectSubscription[T]) Compare(lhs, rhs interface{}) (comp int) { + le := lhs.(*entry[T]) + re := rhs.(*entry[T]) + if le.id == re.id { + return 0 + } + if o.order != nil { + comp = o.order(le.data, re.data) + } + if comp == 0 { + if le.id > re.id { + return 1 + } else { + return -1 + } + } + return comp +} + +func (o *ObjectSubscription[T]) CalcScore(key interface{}) float64 { + return 0 +} + +func (o *ObjectSubscription[T]) read() { + defer close(o.ch) + readEvent := func(event *pb.EventMessage) { + o.mx.Lock() + defer o.mx.Unlock() + switch v := event.Value.(type) { + case *pb.EventMessageValueOfSubscriptionAdd: + o.skl.Set(newEmptyEntry[T](v.SubscriptionAdd.Id), nil) + case *pb.EventMessageValueOfSubscriptionRemove: + o.skl.Remove(newEmptyEntry[T](v.SubscriptionRemove.Id)) + case *pb.EventMessageValueOfObjectDetailsAmend: + curEntry := o.skl.Get(newEmptyEntry[T](v.ObjectDetailsAmend.Id)) + if curEntry == nil { + return + } + e := curEntry.Key().(*entry[T]) + for _, value := range v.ObjectDetailsAmend.Details { + e.data = o.update(value.Key, value.Value, e.data) + } + case *pb.EventMessageValueOfObjectDetailsUnset: + curEntry := o.skl.Get(newEmptyEntry[T](v.ObjectDetailsUnset.Id)) + if curEntry == nil { + return + } + e := curEntry.Key().(*entry[T]) + e.data = o.unset(v.ObjectDetailsUnset.Keys, e.data) + case *pb.EventMessageValueOfObjectDetailsSet: + curEntry := o.skl.Get(newEmptyEntry[T](v.ObjectDetailsSet.Id)) + if curEntry == nil { + return + } + e := curEntry.Key().(*entry[T]) + _, e.data = o.extract(v.ObjectDetailsSet.Details) + } + } + for { + event, err := o.events.WaitOne(o.ctx) + if err != nil { + return + } + readEvent(event) + } +} diff --git a/core/syncstatus/spacesyncstatus/spacestatus.go b/core/syncstatus/spacesyncstatus/spacestatus.go index 6b2b7ead4..f641f35a7 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus.go +++ b/core/syncstatus/spacesyncstatus/spacestatus.go @@ -17,15 +17,11 @@ import ( "github.com/anyproto/anytype-heart/core/event" "github.com/anyproto/anytype-heart/core/files" "github.com/anyproto/anytype-heart/core/session" - "github.com/anyproto/anytype-heart/core/syncstatus/filesyncstatus" + "github.com/anyproto/anytype-heart/core/subscription" "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" "github.com/anyproto/anytype-heart/pb" - "github.com/anyproto/anytype-heart/pkg/lib/bundle" - "github.com/anyproto/anytype-heart/pkg/lib/database" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/logging" - "github.com/anyproto/anytype-heart/pkg/lib/pb/model" - "github.com/anyproto/anytype-heart/util/pbtypes" "github.com/anyproto/anytype-heart/util/slice" ) @@ -67,13 +63,15 @@ type NetworkConfig interface { } type spaceSyncStatus struct { - eventSender event.Sender - networkConfig NetworkConfig - nodeStatus nodestatus.NodeStatus - nodeConf nodeconf.Service - nodeUsage NodeUsage - store objectstore.ObjectStore - batcher *mb.MB[*domain.SpaceSync] + eventSender event.Sender + networkConfig NetworkConfig + nodeStatus nodestatus.NodeStatus + nodeConf nodeconf.Service + nodeUsage NodeUsage + store objectstore.ObjectStore + batcher *mb.MB[*domain.SpaceSync] + subscriptionService subscription.Service + subs map[string]*syncingObjects filesState State objectsState State @@ -98,8 +96,10 @@ func (s *spaceSyncStatus) Init(a *app.App) (err error) { s.nodeStatus = app.MustComponent[nodestatus.NodeStatus](a) s.nodeConf = app.MustComponent[nodeconf.Service](a) s.nodeUsage = app.MustComponent[NodeUsage](a) + s.subscriptionService = app.MustComponent[subscription.Service](a) s.store = app.MustComponent[objectstore.ObjectStore](a) s.curStatuses = make(map[string]struct{}) + s.subs = make(map[string]*syncingObjects) s.missingIds = make(map[string][]string) s.spaceIdGetter = app.MustComponent[SpaceIdGetter](a) sessionHookRunner := app.MustComponent[session.HookRunner](a) @@ -210,56 +210,39 @@ func (s *spaceSyncStatus) Refresh(spaceId string) { } func (s *spaceSyncStatus) getObjectSyncingObjectsCount(spaceId string, missingObjects []string) int { - ids, _, err := s.store.QueryObjectIDs(database.Query{ - Filters: []*model.BlockContentDataviewFilter{ - { - RelationKey: bundle.RelationKeySyncStatus.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.Int64(int64(domain.Syncing)), - }, - { - RelationKey: bundle.RelationKeyLayout.String(), - Condition: model.BlockContentDataviewFilter_NotIn, - Value: pbtypes.IntList( - int(model.ObjectType_file), - int(model.ObjectType_image), - int(model.ObjectType_video), - int(model.ObjectType_audio), - ), - }, - { - RelationKey: bundle.RelationKeySpaceId.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.String(spaceId), - }, - }, - }) - if err != nil { - log.Errorf("failed to query file status: %s", err) + s.mx.Lock() + curSub := s.subs[spaceId] + s.mx.Unlock() + if curSub == nil { + curSub = newSyncingObjects(spaceId, s.subscriptionService) + err := curSub.run() + if err != nil { + log.Errorf("failed to run subscription: %s", err) + return 0 + } + s.mx.Lock() + s.subs[spaceId] = curSub + s.mx.Unlock() } - _, added := slice.DifferenceRemovedAdded(ids, missingObjects) - return len(ids) + len(added) + return curSub.SyncingObjectsCount(missingObjects) } func (s *spaceSyncStatus) getFileSyncingObjectsCount(spaceId string) int { - recs, _, err := s.store.QueryObjectIDs(database.Query{ - Filters: []*model.BlockContentDataviewFilter{ - { - RelationKey: bundle.RelationKeyFileBackupStatus.String(), - Condition: model.BlockContentDataviewFilter_In, - Value: pbtypes.IntList(int(filesyncstatus.Syncing), int(filesyncstatus.Queued)), - }, - { - RelationKey: bundle.RelationKeySpaceId.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.String(spaceId), - }, - }, - }) - if err != nil { - log.Errorf("failed to query file status: %s", err) + s.mx.Lock() + curSub := s.subs[spaceId] + s.mx.Unlock() + if curSub == nil { + curSub = newSyncingObjects(spaceId, s.subscriptionService) + err := curSub.run() + if err != nil { + log.Errorf("failed to run subscription: %s", err) + return 0 + } + s.mx.Lock() + s.subs[spaceId] = curSub + s.mx.Unlock() } - return len(recs) + return curSub.FileSyncingObjectsCount() } func (s *spaceSyncStatus) getBytesLeftPercentage(spaceId string) float64 { @@ -291,6 +274,14 @@ func (s *spaceSyncStatus) updateSpaceSyncStatus(spaceId string, missingObjects [ func (s *spaceSyncStatus) Close(ctx context.Context) (err error) { s.periodicCall.Close() + s.mx.Lock() + subs := lo.MapToSlice(s.subs, func(key string, value *syncingObjects) *syncingObjects { + return value + }) + s.mx.Unlock() + for _, sub := range subs { + sub.Close() + } return } diff --git a/core/syncstatus/spacesyncstatus/syncingobjects.go b/core/syncstatus/spacesyncstatus/syncingobjects.go new file mode 100644 index 000000000..582d5ba31 --- /dev/null +++ b/core/syncstatus/spacesyncstatus/syncingobjects.go @@ -0,0 +1,103 @@ +package spacesyncstatus + +import ( + "fmt" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/core/subscription" + "github.com/anyproto/anytype-heart/core/syncstatus/filesyncstatus" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/pbtypes" + "github.com/anyproto/anytype-heart/util/slice" +) + +type syncingObjects struct { + fileSubscription *ObjectSubscription[struct{}] + objectSubscription *ObjectSubscription[struct{}] + service subscription.Service + spaceId string +} + +func newSyncingObjects(spaceId string, service subscription.Service) *syncingObjects { + return &syncingObjects{ + service: service, + spaceId: spaceId, + } +} + +func (s *syncingObjects) run() error { + objectReq := subscription.SubscribeRequest{ + SubId: fmt.Sprintf("spacestatus.objects.%s", s.spaceId), + Internal: true, + NoDepSubscription: true, + Keys: []string{bundle.RelationKeyId.String()}, + Filters: []*model.BlockContentDataviewFilter{ + { + RelationKey: bundle.RelationKeySyncStatus.String(), + Condition: model.BlockContentDataviewFilter_Equal, + Value: pbtypes.Int64(int64(domain.Syncing)), + }, + { + RelationKey: bundle.RelationKeyLayout.String(), + Condition: model.BlockContentDataviewFilter_NotIn, + Value: pbtypes.IntList( + int(model.ObjectType_file), + int(model.ObjectType_image), + int(model.ObjectType_video), + int(model.ObjectType_audio), + ), + }, + { + RelationKey: bundle.RelationKeySpaceId.String(), + Condition: model.BlockContentDataviewFilter_Equal, + Value: pbtypes.String(s.spaceId), + }, + }, + } + fileReq := subscription.SubscribeRequest{ + SubId: fmt.Sprintf("spacestatus.files.%s", s.spaceId), + Internal: true, + NoDepSubscription: true, + Keys: []string{bundle.RelationKeyId.String()}, + Filters: []*model.BlockContentDataviewFilter{ + { + RelationKey: bundle.RelationKeyFileBackupStatus.String(), + Condition: model.BlockContentDataviewFilter_In, + Value: pbtypes.IntList(int(filesyncstatus.Syncing), int(filesyncstatus.Queued)), + }, + { + RelationKey: bundle.RelationKeySpaceId.String(), + Condition: model.BlockContentDataviewFilter_Equal, + Value: pbtypes.String(s.spaceId), + }, + }, + } + s.fileSubscription = NewIdSubscription(s.service, fileReq) + s.objectSubscription = NewIdSubscription(s.service, objectReq) + errFiles := s.fileSubscription.Run() + errObjects := s.objectSubscription.Run() + if errFiles != nil || errObjects != nil { + return fmt.Errorf("error running syncing objects: %v %v", errFiles, errObjects) + } + return nil +} + +func (s *syncingObjects) Close() { + s.fileSubscription.Close() + s.objectSubscription.Close() +} + +func (s *syncingObjects) SyncingObjectsCount(missing []string) int { + ids := make([]string, 0, s.objectSubscription.Len()) + s.objectSubscription.Iterate(func(id string, _ struct{}) bool { + ids = append(ids, id) + return true + }) + _, added := slice.DifferenceRemovedAdded(ids, missing) + return len(ids) + len(added) +} + +func (s *syncingObjects) FileSyncingObjectsCount() int { + return s.fileSubscription.Len() +} From a003316544dee015d02fea29a415737c27f4eaf0 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 19 Jul 2024 00:21:29 +0200 Subject: [PATCH 17/71] GO-3769 Use sync subscriptions everywhere --- core/anytype/bootstrap.go | 2 + core/syncstatus/detailsupdater/updater.go | 90 +++++++------------ .../syncstatus/spacesyncstatus/spacestatus.go | 48 +++------- .../objectsubscription.go | 2 +- .../syncingobjects.go | 12 ++- .../syncsubscritions/syncsubscriptions.go | 82 +++++++++++++++++ 6 files changed, 137 insertions(+), 99 deletions(-) rename core/syncstatus/{spacesyncstatus => syncsubscritions}/objectsubscription.go (99%) rename core/syncstatus/{spacesyncstatus => syncsubscritions}/syncingobjects.go (91%) create mode 100644 core/syncstatus/syncsubscritions/syncsubscriptions.go diff --git a/core/anytype/bootstrap.go b/core/anytype/bootstrap.go index fa8700668..3e7583f5b 100644 --- a/core/anytype/bootstrap.go +++ b/core/anytype/bootstrap.go @@ -81,6 +81,7 @@ import ( "github.com/anyproto/anytype-heart/core/syncstatus/detailsupdater" "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" "github.com/anyproto/anytype-heart/core/syncstatus/spacesyncstatus" + "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscritions" "github.com/anyproto/anytype-heart/core/wallet" "github.com/anyproto/anytype-heart/metrics" "github.com/anyproto/anytype-heart/pkg/lib/core" @@ -277,6 +278,7 @@ func Bootstrap(a *app.App, components ...app.Component) { Register(debug.New()). Register(collection.New()). Register(subscription.New()). + Register(syncsubscritions.New()). Register(builtinobjects.New()). Register(bookmark.New()). Register(importer.New()). diff --git a/core/syncstatus/detailsupdater/updater.go b/core/syncstatus/detailsupdater/updater.go index 16eae9492..70e4996e2 100644 --- a/core/syncstatus/detailsupdater/updater.go +++ b/core/syncstatus/detailsupdater/updater.go @@ -18,8 +18,8 @@ import ( "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/syncstatus/detailsupdater/helper" "github.com/anyproto/anytype-heart/core/syncstatus/filesyncstatus" + "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscritions" "github.com/anyproto/anytype-heart/pkg/lib/bundle" - "github.com/anyproto/anytype-heart/pkg/lib/database" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/logging" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" @@ -52,12 +52,13 @@ type SpaceStatusUpdater interface { } type syncStatusUpdater struct { - objectStore objectstore.ObjectStore - ctx context.Context - ctxCancel context.CancelFunc - batcher *mb.MB[*syncStatusDetails] - spaceService space.Service - spaceSyncStatus SpaceStatusUpdater + objectStore objectstore.ObjectStore + ctx context.Context + ctxCancel context.CancelFunc + batcher *mb.MB[*syncStatusDetails] + spaceService space.Service + spaceSyncStatus SpaceStatusUpdater + syncSubscriptions syncsubscritions.SyncSubscriptions entries map[string]*syncStatusDetails mx sync.Mutex @@ -91,6 +92,7 @@ func (u *syncStatusUpdater) Init(a *app.App) (err error) { u.objectStore = app.MustComponent[objectstore.ObjectStore](a) u.spaceService = app.MustComponent[space.Service](a) u.spaceSyncStatus = app.MustComponent[SpaceStatusUpdater](a) + u.syncSubscriptions = app.MustComponent[syncsubscritions.SyncSubscriptions](a) return nil } @@ -136,50 +138,25 @@ func (u *syncStatusUpdater) UpdateSpaceDetails(existing, missing []string, space } func (u *syncStatusUpdater) getSyncingObjects(spaceId string) []string { - ids, _, err := u.objectStore.QueryObjectIDs(database.Query{ - Filters: []*model.BlockContentDataviewFilter{ - { - RelationKey: bundle.RelationKeySyncStatus.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.Int64(int64(domain.ObjectSyncing)), - }, - { - RelationKey: bundle.RelationKeySpaceId.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.String(spaceId), - }, - }, - }) + sub, err := u.syncSubscriptions.GetSubscription(spaceId) if err != nil { - log.Errorf("failed to update object details %s", err) + return nil } + ids := make([]string, 0, sub.GetObjectSubscription().Len()) + sub.GetObjectSubscription().Iterate(func(id string, _ struct{}) bool { + ids = append(ids, id) + return true + }) return ids } func (u *syncStatusUpdater) updateObjectDetails(syncStatusDetails *syncStatusDetails, objectId string) error { - record, err := u.objectStore.GetDetails(objectId) - if err != nil { - return err - } - return u.setObjectDetails(syncStatusDetails, record.Details, objectId) + return u.setObjectDetails(syncStatusDetails, objectId) } -func (u *syncStatusUpdater) setObjectDetails(syncStatusDetails *syncStatusDetails, record *types.Struct, objectId string) error { +func (u *syncStatusUpdater) setObjectDetails(syncStatusDetails *syncStatusDetails, objectId string) error { status := syncStatusDetails.status syncError := domain.Null - isFileStatus := false - if fileStatus, ok := record.GetFields()[bundle.RelationKeyFileBackupStatus.String()]; ok { - isFileStatus = true - status, syncError = mapFileStatus(filesyncstatus.Status(int(fileStatus.GetNumberValue()))) - } - // we want to update sync date for other stuff - changed := u.hasRelationsChange(record, status, syncError) - if !changed && isFileStatus { - return nil - } - if !u.isLayoutSuitableForSyncRelations(record) { - return nil - } spc, err := u.spaceService.Get(u.ctx, syncStatusDetails.spaceId) if err != nil { return err @@ -190,6 +167,12 @@ func (u *syncStatusUpdater) setObjectDetails(syncStatusDetails *syncStatusDetail if details == nil || details.Fields == nil { details = &types.Struct{Fields: map[string]*types.Value{}} } + if !u.isLayoutSuitableForSyncRelations(details) { + return details, nil + } + if fileStatus, ok := details.GetFields()[bundle.RelationKeyFileBackupStatus.String()]; ok { + status, syncError = mapFileStatus(filesyncstatus.Status(int(fileStatus.GetNumberValue()))) + } details.Fields[bundle.RelationKeySyncStatus.String()] = pbtypes.Int64(int64(status)) details.Fields[bundle.RelationKeySyncError.String()] = pbtypes.Int64(int64(syncError)) details.Fields[bundle.RelationKeySyncDate.String()] = pbtypes.Int64(time.Now().Unix()) @@ -241,6 +224,13 @@ func (u *syncStatusUpdater) setSyncDetails(sb smartblock.SmartBlock, status doma if !slices.Contains(helper.SyncRelationsSmartblockTypes(), sb.Type()) { return nil } + details := sb.CombinedDetails() + if !u.isLayoutSuitableForSyncRelations(details) { + return nil + } + if fileStatus, ok := details.GetFields()[bundle.RelationKeyFileBackupStatus.String()]; ok { + status, syncError = mapFileStatus(filesyncstatus.Status(int(fileStatus.GetNumberValue()))) + } if d, ok := sb.(basic.DetailsSettable); ok { syncStatusDetails := []*model.Detail{ { @@ -261,24 +251,6 @@ func (u *syncStatusUpdater) setSyncDetails(sb smartblock.SmartBlock, status doma return nil } -func (u *syncStatusUpdater) hasRelationsChange(record *types.Struct, status domain.ObjectSyncStatus, syncError domain.SyncError) bool { - var changed bool - if record == nil || len(record.GetFields()) == 0 { - changed = true - } - if pbtypes.Get(record, bundle.RelationKeySyncStatus.String()) == nil || - pbtypes.Get(record, bundle.RelationKeySyncError.String()) == nil { - changed = true - } - if pbtypes.GetInt64(record, bundle.RelationKeySyncStatus.String()) != int64(status) { - changed = true - } - if pbtypes.GetInt64(record, bundle.RelationKeySyncError.String()) != int64(syncError) { - changed = true - } - return changed -} - func (u *syncStatusUpdater) processEvents() { defer close(u.finish) updateSpecificObject := func(details *syncStatusDetails) { diff --git a/core/syncstatus/spacesyncstatus/spacestatus.go b/core/syncstatus/spacesyncstatus/spacestatus.go index f641f35a7..1f95e01dc 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus.go +++ b/core/syncstatus/spacesyncstatus/spacestatus.go @@ -19,6 +19,7 @@ import ( "github.com/anyproto/anytype-heart/core/session" "github.com/anyproto/anytype-heart/core/subscription" "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" + "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscritions" "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/logging" @@ -71,7 +72,7 @@ type spaceSyncStatus struct { store objectstore.ObjectStore batcher *mb.MB[*domain.SpaceSync] subscriptionService subscription.Service - subs map[string]*syncingObjects + subs syncsubscritions.SyncSubscriptions filesState State objectsState State @@ -99,7 +100,7 @@ func (s *spaceSyncStatus) Init(a *app.App) (err error) { s.subscriptionService = app.MustComponent[subscription.Service](a) s.store = app.MustComponent[objectstore.ObjectStore](a) s.curStatuses = make(map[string]struct{}) - s.subs = make(map[string]*syncingObjects) + s.subs = app.MustComponent[syncsubscritions.SyncSubscriptions](a) s.missingIds = make(map[string][]string) s.spaceIdGetter = app.MustComponent[SpaceIdGetter](a) sessionHookRunner := app.MustComponent[session.HookRunner](a) @@ -210,37 +211,19 @@ func (s *spaceSyncStatus) Refresh(spaceId string) { } func (s *spaceSyncStatus) getObjectSyncingObjectsCount(spaceId string, missingObjects []string) int { - s.mx.Lock() - curSub := s.subs[spaceId] - s.mx.Unlock() - if curSub == nil { - curSub = newSyncingObjects(spaceId, s.subscriptionService) - err := curSub.run() - if err != nil { - log.Errorf("failed to run subscription: %s", err) - return 0 - } - s.mx.Lock() - s.subs[spaceId] = curSub - s.mx.Unlock() + curSub, err := s.subs.GetSubscription(spaceId) + if err != nil { + log.Errorf("failed to get subscription: %s", err) + return 0 } return curSub.SyncingObjectsCount(missingObjects) } func (s *spaceSyncStatus) getFileSyncingObjectsCount(spaceId string) int { - s.mx.Lock() - curSub := s.subs[spaceId] - s.mx.Unlock() - if curSub == nil { - curSub = newSyncingObjects(spaceId, s.subscriptionService) - err := curSub.run() - if err != nil { - log.Errorf("failed to run subscription: %s", err) - return 0 - } - s.mx.Lock() - s.subs[spaceId] = curSub - s.mx.Unlock() + curSub, err := s.subs.GetSubscription(spaceId) + if err != nil { + log.Errorf("failed to get subscription: %s", err) + return 0 } return curSub.FileSyncingObjectsCount() } @@ -262,7 +245,6 @@ func (s *spaceSyncStatus) updateSpaceSyncStatus(spaceId string, missingObjects [ filesSyncingCount: s.getFileSyncingObjectsCount(spaceId), objectsSyncingCount: s.getObjectSyncingObjectsCount(spaceId, missingObjects), } - // fmt.Println("[x]: space status", event.Status, "space id", receivedStatus.SpaceId, "network", event.Network, "error", event.Error, "object number", event.SyncingObjectsCounter, "isFile", isFileState) s.eventSender.Broadcast(&pb.Event{ Messages: []*pb.EventMessage{{ Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ @@ -274,14 +256,6 @@ func (s *spaceSyncStatus) updateSpaceSyncStatus(spaceId string, missingObjects [ func (s *spaceSyncStatus) Close(ctx context.Context) (err error) { s.periodicCall.Close() - s.mx.Lock() - subs := lo.MapToSlice(s.subs, func(key string, value *syncingObjects) *syncingObjects { - return value - }) - s.mx.Unlock() - for _, sub := range subs { - sub.Close() - } return } diff --git a/core/syncstatus/spacesyncstatus/objectsubscription.go b/core/syncstatus/syncsubscritions/objectsubscription.go similarity index 99% rename from core/syncstatus/spacesyncstatus/objectsubscription.go rename to core/syncstatus/syncsubscritions/objectsubscription.go index 46d33f6db..ab5d334b2 100644 --- a/core/syncstatus/spacesyncstatus/objectsubscription.go +++ b/core/syncstatus/syncsubscritions/objectsubscription.go @@ -1,4 +1,4 @@ -package spacesyncstatus +package syncsubscritions import ( "context" diff --git a/core/syncstatus/spacesyncstatus/syncingobjects.go b/core/syncstatus/syncsubscritions/syncingobjects.go similarity index 91% rename from core/syncstatus/spacesyncstatus/syncingobjects.go rename to core/syncstatus/syncsubscritions/syncingobjects.go index 582d5ba31..9dbee1b6e 100644 --- a/core/syncstatus/spacesyncstatus/syncingobjects.go +++ b/core/syncstatus/syncsubscritions/syncingobjects.go @@ -1,4 +1,4 @@ -package spacesyncstatus +package syncsubscritions import ( "fmt" @@ -26,7 +26,7 @@ func newSyncingObjects(spaceId string, service subscription.Service) *syncingObj } } -func (s *syncingObjects) run() error { +func (s *syncingObjects) Run() error { objectReq := subscription.SubscribeRequest{ SubId: fmt.Sprintf("spacestatus.objects.%s", s.spaceId), Internal: true, @@ -88,6 +88,14 @@ func (s *syncingObjects) Close() { s.objectSubscription.Close() } +func (s *syncingObjects) GetFileSubscription() *ObjectSubscription[struct{}] { + return s.fileSubscription +} + +func (s *syncingObjects) GetObjectSubscription() *ObjectSubscription[struct{}] { + return s.objectSubscription +} + func (s *syncingObjects) SyncingObjectsCount(missing []string) int { ids := make([]string, 0, s.objectSubscription.Len()) s.objectSubscription.Iterate(func(id string, _ struct{}) bool { diff --git a/core/syncstatus/syncsubscritions/syncsubscriptions.go b/core/syncstatus/syncsubscritions/syncsubscriptions.go new file mode 100644 index 000000000..65fec81ea --- /dev/null +++ b/core/syncstatus/syncsubscritions/syncsubscriptions.go @@ -0,0 +1,82 @@ +package syncsubscritions + +import ( + "context" + "sync" + + "github.com/anyproto/any-sync/app" + "github.com/samber/lo" + + "github.com/anyproto/anytype-heart/core/subscription" +) + +const CName = "client.syncstatus.syncsubscritions" + +type SyncSubscription interface { + Run() error + Close() + GetFileSubscription() *ObjectSubscription[struct{}] + GetObjectSubscription() *ObjectSubscription[struct{}] + SyncingObjectsCount(missing []string) int + FileSyncingObjectsCount() int +} + +type SyncSubscriptions interface { + app.ComponentRunnable + GetSubscription(id string) (SyncSubscription, error) +} + +func New() SyncSubscriptions { + return &syncSubscriptions{ + subs: make(map[string]SyncSubscription), + } +} + +type syncSubscriptions struct { + sync.Mutex + service subscription.Service + subs map[string]SyncSubscription +} + +func (s *syncSubscriptions) Init(a *app.App) (err error) { + s.service = app.MustComponent[subscription.Service](a) + return +} + +func (s *syncSubscriptions) Name() (name string) { + return CName +} + +func (s *syncSubscriptions) Run(ctx context.Context) (err error) { + return nil +} + +func (s *syncSubscriptions) GetSubscription(id string) (SyncSubscription, error) { + s.Lock() + curSub := s.subs[id] + s.Unlock() + if curSub != nil { + return curSub, nil + } + sub := newSyncingObjects(id, s.service) + err := sub.Run() + if err != nil { + return nil, err + } + s.Lock() + s.subs[id] = sub + s.Unlock() + return sub, nil +} + +func (s *syncSubscriptions) Close(ctx context.Context) (err error) { + s.Lock() + subs := lo.MapToSlice(s.subs, func(key string, value SyncSubscription) SyncSubscription { + return value + }) + s.Unlock() + for _, sub := range subs { + sub.Close() + } + return nil +} From 7c7bf025edda821fd313ed13d2994e8b4d6fe629 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 19 Jul 2024 10:55:53 +0200 Subject: [PATCH 18/71] GO-3769 Remove sorting from sub --- .../syncsubscritions/objectsubscription.go | 64 ++++--------------- 1 file changed, 14 insertions(+), 50 deletions(-) diff --git a/core/syncstatus/syncsubscritions/objectsubscription.go b/core/syncstatus/syncsubscritions/objectsubscription.go index ab5d334b2..ab823ad89 100644 --- a/core/syncstatus/syncsubscritions/objectsubscription.go +++ b/core/syncstatus/syncsubscritions/objectsubscription.go @@ -6,7 +6,6 @@ import ( "github.com/cheggaaa/mb/v3" "github.com/gogo/protobuf/types" - "github.com/huandu/skiplist" "github.com/anyproto/anytype-heart/core/subscription" "github.com/anyproto/anytype-heart/pb" @@ -29,7 +28,6 @@ func newEntry[T any](id string, data T) *entry[T] { type ( extract[T any] func(*types.Struct) (string, T) - compare[T any] func(T, T) int update[T any] func(string, *types.Value, T) T unset[T any] func([]string, T) T ) @@ -37,7 +35,6 @@ type ( type SubscriptionParams[T any] struct { Request subscription.SubscribeRequest Extract extract[T] - Order compare[T] Update update[T] Unset unset[T] } @@ -47,7 +44,6 @@ func NewIdSubscription(service subscription.Service, request subscription.Subscr request: request, service: service, ch: make(chan struct{}), - order: nil, extract: func(t *types.Struct) (string, struct{}) { return pbtypes.GetString(t, bundle.RelationKeyId.String()), struct{}{} }, @@ -65,7 +61,6 @@ func NewObjectSubscription[T any](service subscription.Service, params Subscript request: params.Request, service: service, ch: make(chan struct{}), - order: params.Order, extract: params.Extract, update: params.Update, unset: params.Unset, @@ -79,8 +74,7 @@ type ObjectSubscription[T any] struct { events *mb.MB[*pb.EventMessage] ctx context.Context cancel context.CancelFunc - skl *skiplist.SkipList - order compare[T] + sub map[string]*entry[T] extract extract[T] update update[T] unset unset[T] @@ -94,11 +88,10 @@ func (o *ObjectSubscription[T]) Run() error { } o.ctx, o.cancel = context.WithCancel(context.Background()) o.events = resp.Output - o.skl = skiplist.New(o) + o.sub = map[string]*entry[T]{} for _, rec := range resp.Records { id, data := o.extract(rec) - e := &entry[T]{id: id, data: data} - o.skl.Set(e, nil) + o.sub[id] = newEntry(id, data) } go o.read() return nil @@ -113,46 +106,20 @@ func (o *ObjectSubscription[T]) Close() { func (o *ObjectSubscription[T]) Len() int { o.mx.Lock() defer o.mx.Unlock() - return o.skl.Len() + return len(o.sub) } func (o *ObjectSubscription[T]) Iterate(iter func(id string, data T) bool) { o.mx.Lock() defer o.mx.Unlock() - cur := o.skl.Front() - for cur != nil { - el := cur.Key().(*entry[T]) - if !iter(el.id, el.data) { + for id, ent := range o.sub { + if !iter(id, ent.data) { return } - cur = cur.Next() } return } -func (o *ObjectSubscription[T]) Compare(lhs, rhs interface{}) (comp int) { - le := lhs.(*entry[T]) - re := rhs.(*entry[T]) - if le.id == re.id { - return 0 - } - if o.order != nil { - comp = o.order(le.data, re.data) - } - if comp == 0 { - if le.id > re.id { - return 1 - } else { - return -1 - } - } - return comp -} - -func (o *ObjectSubscription[T]) CalcScore(key interface{}) float64 { - return 0 -} - func (o *ObjectSubscription[T]) read() { defer close(o.ch) readEvent := func(event *pb.EventMessage) { @@ -160,32 +127,29 @@ func (o *ObjectSubscription[T]) read() { defer o.mx.Unlock() switch v := event.Value.(type) { case *pb.EventMessageValueOfSubscriptionAdd: - o.skl.Set(newEmptyEntry[T](v.SubscriptionAdd.Id), nil) + o.sub[v.SubscriptionAdd.Id] = newEmptyEntry[T](v.SubscriptionAdd.Id) case *pb.EventMessageValueOfSubscriptionRemove: - o.skl.Remove(newEmptyEntry[T](v.SubscriptionRemove.Id)) + delete(o.sub, v.SubscriptionRemove.Id) case *pb.EventMessageValueOfObjectDetailsAmend: - curEntry := o.skl.Get(newEmptyEntry[T](v.ObjectDetailsAmend.Id)) + curEntry := o.sub[v.ObjectDetailsAmend.Id] if curEntry == nil { return } - e := curEntry.Key().(*entry[T]) for _, value := range v.ObjectDetailsAmend.Details { - e.data = o.update(value.Key, value.Value, e.data) + curEntry.data = o.update(value.Key, value.Value, curEntry.data) } case *pb.EventMessageValueOfObjectDetailsUnset: - curEntry := o.skl.Get(newEmptyEntry[T](v.ObjectDetailsUnset.Id)) + curEntry := o.sub[v.ObjectDetailsUnset.Id] if curEntry == nil { return } - e := curEntry.Key().(*entry[T]) - e.data = o.unset(v.ObjectDetailsUnset.Keys, e.data) + curEntry.data = o.unset(v.ObjectDetailsUnset.Keys, curEntry.data) case *pb.EventMessageValueOfObjectDetailsSet: - curEntry := o.skl.Get(newEmptyEntry[T](v.ObjectDetailsSet.Id)) + curEntry := o.sub[v.ObjectDetailsSet.Id] if curEntry == nil { return } - e := curEntry.Key().(*entry[T]) - _, e.data = o.extract(v.ObjectDetailsSet.Details) + _, curEntry.data = o.extract(v.ObjectDetailsSet.Details) } } for { From 19d5a1062a7215b99ce916a5a56edeea3894295a Mon Sep 17 00:00:00 2001 From: kirillston Date: Fri, 19 Jul 2024 17:00:35 +0300 Subject: [PATCH 19/71] GO-3192 Move check to table pkg --- core/block/editor/basic/basic.go | 43 +- core/block/editor/basic/basic_test.go | 5 +- core/block/editor/table/editor.go | 765 ++++++++++++++++++++++++ core/block/editor/table/table.go | 798 ++------------------------ core/block/editor/table/table_test.go | 31 +- 5 files changed, 828 insertions(+), 814 deletions(-) create mode 100644 core/block/editor/table/editor.go diff --git a/core/block/editor/basic/basic.go b/core/block/editor/basic/basic.go index 515336d1c..54ed8f4af 100644 --- a/core/block/editor/basic/basic.go +++ b/core/block/editor/basic/basic.go @@ -2,7 +2,6 @@ package basic import ( "fmt" - "slices" "github.com/globalsign/mgo/bson" "github.com/gogo/protobuf/types" @@ -100,8 +99,6 @@ type Updatable interface { Update(ctx session.Context, apply func(b simple.Block) error, blockIds ...string) (err error) } -var ErrCannotMoveTableBlocks = fmt.Errorf("can not move table blocks") - func NewBasic( sb smartblock.SmartBlock, objectStore objectstore.ObjectStore, @@ -240,7 +237,7 @@ func (bs *basic) Move(srcState, destState *state.State, targetBlockId string, po } } - targetBlockId, position, err = checkTableBlocksMove(srcState, targetBlockId, position, blockIds) + targetBlockId, position, err = table.CheckTableBlocksMove(srcState, targetBlockId, position, blockIds) if err != nil { return err } @@ -287,44 +284,6 @@ func (bs *basic) Move(srcState, destState *state.State, targetBlockId string, po return srcState.InsertTo(targetBlockId, position, blockIds...) } -func checkTableBlocksMove(st *state.State, target string, pos model.BlockPosition, blockIds []string) (string, model.BlockPosition, error) { - if t, _ := table.NewTable(st, target); t != nil { - // we allow moving rows between each other - if slice.ContainsAll(t.RowIDs(), append(blockIds, target)...) { - if pos == model.Block_Bottom || pos == model.Block_Top { - return target, pos, nil - } - return "", 0, fmt.Errorf("failed to move rows: position should be Top or Bottom, got %s", model.BlockPosition_name[int32(pos)]) - } - } - - for _, id := range blockIds { - t := table.GetTableRootBlock(st, id) - if t != nil && t.Model().Id != id { - // we should not move table blocks except table root block - return "", 0, ErrCannotMoveTableBlocks - } - } - - t := table.GetTableRootBlock(st, target) - if t != nil && t.Model().Id != target { - // we allow inserting blocks into table cell - if isTableCell(target) && slices.Contains([]model.BlockPosition{model.Block_Inner, model.Block_Replace, model.Block_InnerFirst}, pos) { - return target, pos, nil - } - - // if the target is one of table blocks, but not cell or table root, we should insert blocks under the table - return t.Model().Id, model.Block_Bottom, nil - } - - return target, pos, nil -} - -func isTableCell(id string) bool { - _, _, err := table.ParseCellID(id) - return err == nil -} - func (bs *basic) Replace(ctx session.Context, id string, block *model.Block) (newId string, err error) { s := bs.NewStateCtx(ctx) if block.GetContent() == nil { diff --git a/core/block/editor/basic/basic_test.go b/core/block/editor/basic/basic_test.go index 156b46dca..bf7e47df7 100644 --- a/core/block/editor/basic/basic_test.go +++ b/core/block/editor/basic/basic_test.go @@ -12,6 +12,7 @@ import ( "github.com/anyproto/anytype-heart/core/block/editor/converter" "github.com/anyproto/anytype-heart/core/block/editor/smartblock" "github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest" + "github.com/anyproto/anytype-heart/core/block/editor/table" "github.com/anyproto/anytype-heart/core/block/editor/template" "github.com/anyproto/anytype-heart/core/block/restriction" "github.com/anyproto/anytype-heart/core/block/simple" @@ -288,7 +289,7 @@ func TestBasic_MoveTableBlocks(t *testing.T) { // then assert.Error(t, err) - assert.True(t, errors.Is(err, ErrCannotMoveTableBlocks)) + assert.True(t, errors.Is(err, table.ErrCannotMoveTableBlocks)) }) } @@ -371,7 +372,7 @@ func TestBasic_MoveTableBlocks(t *testing.T) { // then assert.Error(t, err) - assert.True(t, errors.Is(err, ErrCannotMoveTableBlocks)) + assert.True(t, errors.Is(err, table.ErrCannotMoveTableBlocks)) }) for _, block := range []string{"columns", "rows", "column", "row"} { diff --git a/core/block/editor/table/editor.go b/core/block/editor/table/editor.go new file mode 100644 index 000000000..07146b870 --- /dev/null +++ b/core/block/editor/table/editor.go @@ -0,0 +1,765 @@ +package table + +import ( + "fmt" + "sort" + + "github.com/globalsign/mgo/bson" + + "github.com/anyproto/anytype-heart/core/block/editor/smartblock" + "github.com/anyproto/anytype-heart/core/block/editor/state" + "github.com/anyproto/anytype-heart/core/block/simple" + "github.com/anyproto/anytype-heart/core/block/simple/text" + "github.com/anyproto/anytype-heart/core/block/source" + "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +// nolint:revive,interfacebloat +type TableEditor interface { + TableCreate(s *state.State, req pb.RpcBlockTableCreateRequest) (string, error) + RowCreate(s *state.State, req pb.RpcBlockTableRowCreateRequest) (string, error) + RowDelete(s *state.State, req pb.RpcBlockTableRowDeleteRequest) error + ColumnDelete(s *state.State, req pb.RpcBlockTableColumnDeleteRequest) error + ColumnMove(s *state.State, req pb.RpcBlockTableColumnMoveRequest) error + RowDuplicate(s *state.State, req pb.RpcBlockTableRowDuplicateRequest) (newRowID string, err error) + RowListFill(s *state.State, req pb.RpcBlockTableRowListFillRequest) error + RowListClean(s *state.State, req pb.RpcBlockTableRowListCleanRequest) error + RowSetHeader(s *state.State, req pb.RpcBlockTableRowSetHeaderRequest) error + ColumnListFill(s *state.State, req pb.RpcBlockTableColumnListFillRequest) error + cleanupTables(_ smartblock.ApplyInfo) error + ColumnCreate(s *state.State, req pb.RpcBlockTableColumnCreateRequest) (string, error) + cloneColumnStyles(s *state.State, srcColID string, targetColID string) error + ColumnDuplicate(s *state.State, req pb.RpcBlockTableColumnDuplicateRequest) (id string, err error) + Expand(s *state.State, req pb.RpcBlockTableExpandRequest) error + Sort(s *state.State, req pb.RpcBlockTableSortRequest) error + CellCreate(s *state.State, rowID string, colID string, b *model.Block) (string, error) +} + +type editor struct { + sb smartblock.SmartBlock + + generateRowID func() string + generateColID func() string +} + +var _ TableEditor = &editor{} + +func NewEditor(sb smartblock.SmartBlock) TableEditor { + genID := func() string { + return bson.NewObjectId().Hex() + } + + t := editor{ + sb: sb, + generateRowID: genID, + generateColID: genID, + } + if sb != nil { + sb.AddHook(t.cleanupTables, smartblock.HookOnBlockClose) + } + return &t +} + +func (t *editor) TableCreate(s *state.State, req pb.RpcBlockTableCreateRequest) (string, error) { + if t.sb != nil { + if err := t.sb.Restrictions().Object.Check(model.Restrictions_Blocks); err != nil { + return "", err + } + } + + tableBlock := simple.New(&model.Block{ + Content: &model.BlockContentOfTable{ + Table: &model.BlockContentTable{}, + }, + }) + if !s.Add(tableBlock) { + return "", fmt.Errorf("add table block") + } + + if err := s.InsertTo(req.TargetId, req.Position, tableBlock.Model().Id); err != nil { + return "", fmt.Errorf("insert block: %w", err) + } + + columnIds := make([]string, 0, req.Columns) + for i := uint32(0); i < req.Columns; i++ { + id, err := t.addColumnHeader(s) + if err != nil { + return "", err + } + columnIds = append(columnIds, id) + } + columnsLayout := simple.New(&model.Block{ + ChildrenIds: columnIds, + Content: &model.BlockContentOfLayout{ + Layout: &model.BlockContentLayout{ + Style: model.BlockContentLayout_TableColumns, + }, + }, + }) + if !s.Add(columnsLayout) { + return "", fmt.Errorf("add columns block") + } + + rowIDs := make([]string, 0, req.Rows) + for i := uint32(0); i < req.Rows; i++ { + id, err := t.addRow(s) + if err != nil { + return "", err + } + rowIDs = append(rowIDs, id) + } + + rowsLayout := simple.New(&model.Block{ + ChildrenIds: rowIDs, + Content: &model.BlockContentOfLayout{ + Layout: &model.BlockContentLayout{ + Style: model.BlockContentLayout_TableRows, + }, + }, + }) + if !s.Add(rowsLayout) { + return "", fmt.Errorf("add rows block") + } + + tableBlock.Model().ChildrenIds = []string{columnsLayout.Model().Id, rowsLayout.Model().Id} + + if req.WithHeaderRow { + headerID := rowIDs[0] + + if err := t.RowSetHeader(s, pb.RpcBlockTableRowSetHeaderRequest{ + TargetId: headerID, + IsHeader: true, + }); err != nil { + return "", fmt.Errorf("row set header: %w", err) + } + + if err := t.RowListFill(s, pb.RpcBlockTableRowListFillRequest{ + BlockIds: []string{headerID}, + }); err != nil { + return "", fmt.Errorf("fill header row: %w", err) + } + + row, err := getRow(s, headerID) + if err != nil { + return "", fmt.Errorf("get header row: %w", err) + } + + for _, cellID := range row.Model().ChildrenIds { + cell := s.Get(cellID) + if cell == nil { + return "", fmt.Errorf("get header cell id %s", cellID) + } + + cell.Model().BackgroundColor = "grey" + } + } + + return tableBlock.Model().Id, nil +} + +func (t *editor) RowCreate(s *state.State, req pb.RpcBlockTableRowCreateRequest) (string, error) { + switch req.Position { + case model.Block_Top, model.Block_Bottom: + case model.Block_Inner: + tb, err := NewTable(s, req.TargetId) + if err != nil { + return "", fmt.Errorf("initialize table state: %w", err) + } + req.TargetId = tb.Rows().Id + default: + return "", fmt.Errorf("position is not supported") + } + + rowID, err := t.addRow(s) + if err != nil { + return "", err + } + if err := s.InsertTo(req.TargetId, req.Position, rowID); err != nil { + return "", fmt.Errorf("insert row: %w", err) + } + return rowID, nil +} + +func (t *editor) RowDelete(s *state.State, req pb.RpcBlockTableRowDeleteRequest) error { + _, err := pickRow(s, req.TargetId) + if err != nil { + return fmt.Errorf("pick target row: %w", err) + } + + if !s.Unlink(req.TargetId) { + return fmt.Errorf("unlink row block") + } + return nil +} + +func (t *editor) ColumnDelete(s *state.State, req pb.RpcBlockTableColumnDeleteRequest) error { + _, err := pickColumn(s, req.TargetId) + if err != nil { + return fmt.Errorf("pick target column: %w", err) + } + + tb, err := NewTable(s, req.TargetId) + if err != nil { + return fmt.Errorf("initialize table state: %w", err) + } + + for _, rowID := range tb.RowIDs() { + row, err := pickRow(s, rowID) + if err != nil { + return fmt.Errorf("pick row %s: %w", rowID, err) + } + + for _, cellID := range row.Model().ChildrenIds { + _, colID, err := ParseCellID(cellID) + if err != nil { + return fmt.Errorf("parse cell id %s: %w", cellID, err) + } + + if colID == req.TargetId { + if !s.Unlink(cellID) { + return fmt.Errorf("unlink cell %s", cellID) + } + break + } + } + } + if !s.Unlink(req.TargetId) { + return fmt.Errorf("unlink column header") + } + + return nil +} + +func (t *editor) ColumnMove(s *state.State, req pb.RpcBlockTableColumnMoveRequest) error { + switch req.Position { + case model.Block_Left: + req.Position = model.Block_Top + case model.Block_Right: + req.Position = model.Block_Bottom + default: + return fmt.Errorf("position is not supported") + } + _, err := pickColumn(s, req.TargetId) + if err != nil { + return fmt.Errorf("get target column: %w", err) + } + _, err = pickColumn(s, req.DropTargetId) + if err != nil { + return fmt.Errorf("get drop target column: %w", err) + } + + tb, err := NewTable(s, req.TargetId) + if err != nil { + return fmt.Errorf("init table block: %w", err) + } + + if !s.Unlink(req.TargetId) { + return fmt.Errorf("unlink target column") + } + if err = s.InsertTo(req.DropTargetId, req.Position, req.TargetId); err != nil { + return fmt.Errorf("insert column: %w", err) + } + + colIdx := tb.MakeColumnIndex() + + for _, id := range tb.RowIDs() { + row, err := getRow(s, id) + if err != nil { + return fmt.Errorf("get row %s: %w", id, err) + } + normalizeRow(colIdx, row) + } + + return nil +} + +func (t *editor) RowDuplicate(s *state.State, req pb.RpcBlockTableRowDuplicateRequest) (newRowID string, err error) { + srcRow, err := pickRow(s, req.BlockId) + if err != nil { + return "", fmt.Errorf("pick source row: %w", err) + } + + newRow := srcRow.Copy() + newRow.Model().Id = t.generateRowID() + if !s.Add(newRow) { + return "", fmt.Errorf("add new row %s", newRow.Model().Id) + } + if err = s.InsertTo(req.TargetId, req.Position, newRow.Model().Id); err != nil { + return "", fmt.Errorf("insert column: %w", err) + } + + for i, srcID := range newRow.Model().ChildrenIds { + cell := s.Pick(srcID) + if cell == nil { + return "", fmt.Errorf("cell %s is not found", srcID) + } + _, colID, err := ParseCellID(srcID) + if err != nil { + return "", fmt.Errorf("parse cell id %s: %w", srcID, err) + } + + newCell := cell.Copy() + newCell.Model().Id = MakeCellID(newRow.Model().Id, colID) + if !s.Add(newCell) { + return "", fmt.Errorf("add new cell %s", newCell.Model().Id) + } + newRow.Model().ChildrenIds[i] = newCell.Model().Id + } + + return newRow.Model().Id, nil +} + +func (t *editor) RowListFill(s *state.State, req pb.RpcBlockTableRowListFillRequest) error { + if len(req.BlockIds) == 0 { + return fmt.Errorf("empty row list") + } + + tb, err := NewTable(s, req.BlockIds[0]) + if err != nil { + return fmt.Errorf("init table: %w", err) + } + + columns := tb.ColumnIDs() + + for _, rowID := range req.BlockIds { + row, err := getRow(s, rowID) + if err != nil { + return fmt.Errorf("get row %s: %w", rowID, err) + } + + newIds := make([]string, 0, len(columns)) + for _, colID := range columns { + id := MakeCellID(rowID, colID) + newIds = append(newIds, id) + + if !s.Exists(id) { + _, err := addCell(s, rowID, colID) + if err != nil { + return fmt.Errorf("add cell %s: %w", id, err) + } + } + } + row.Model().ChildrenIds = newIds + } + return nil +} + +func (t *editor) RowListClean(s *state.State, req pb.RpcBlockTableRowListCleanRequest) error { + if len(req.BlockIds) == 0 { + return fmt.Errorf("empty row list") + } + + for _, rowID := range req.BlockIds { + row, err := pickRow(s, rowID) + if err != nil { + return fmt.Errorf("pick row: %w", err) + } + + for _, cellID := range row.Model().ChildrenIds { + cell := s.Pick(cellID) + if v, ok := cell.(text.Block); ok && v.IsEmpty() { + s.Unlink(cellID) + } + } + } + return nil +} + +func (t *editor) RowSetHeader(s *state.State, req pb.RpcBlockTableRowSetHeaderRequest) error { + tb, err := NewTable(s, req.TargetId) + if err != nil { + return fmt.Errorf("init table: %w", err) + } + + row, err := getRow(s, req.TargetId) + if err != nil { + return fmt.Errorf("get target row: %w", err) + } + + if row.Model().GetTableRow().IsHeader != req.IsHeader { + row.Model().GetTableRow().IsHeader = req.IsHeader + + err = normalizeRows(s, tb) + if err != nil { + return fmt.Errorf("normalize rows: %w", err) + } + } + + return nil +} + +func (t *editor) ColumnListFill(s *state.State, req pb.RpcBlockTableColumnListFillRequest) error { + if len(req.BlockIds) == 0 { + return fmt.Errorf("empty row list") + } + + tb, err := NewTable(s, req.BlockIds[0]) + if err != nil { + return fmt.Errorf("init table: %w", err) + } + + rows := tb.RowIDs() + + for _, colID := range req.BlockIds { + for _, rowID := range rows { + id := MakeCellID(rowID, colID) + if s.Exists(id) { + continue + } + _, err := addCell(s, rowID, colID) + if err != nil { + return fmt.Errorf("add cell %s: %w", id, err) + } + + row, err := getRow(s, rowID) + if err != nil { + return fmt.Errorf("get row %s: %w", rowID, err) + } + + row.Model().ChildrenIds = append(row.Model().ChildrenIds, id) + } + } + + colIdx := tb.MakeColumnIndex() + for _, rowID := range rows { + row, err := getRow(s, rowID) + if err != nil { + return fmt.Errorf("get row %s: %w", rowID, err) + } + normalizeRow(colIdx, row) + } + + return nil +} + +func (t *editor) cleanupTables(_ smartblock.ApplyInfo) error { + if t.sb == nil { + return fmt.Errorf("nil smartblock") + } + s := t.sb.NewState() + + err := s.Iterate(func(b simple.Block) bool { + if b.Model().GetTable() == nil { + return true + } + + tb, err := NewTable(s, b.Model().Id) + if err != nil { + log.Errorf("cleanup: init table %s: %s", b.Model().Id, err) + return true + } + err = t.RowListClean(s, pb.RpcBlockTableRowListCleanRequest{ + BlockIds: tb.RowIDs(), + }) + if err != nil { + log.Errorf("cleanup table %s: %s", b.Model().Id, err) + return true + } + return true + }) + if err != nil { + log.Errorf("cleanup iterate: %s", err) + } + + if err = t.sb.Apply(s, smartblock.KeepInternalFlags); err != nil { + if err == source.ErrReadOnly { + return nil + } + log.Errorf("cleanup apply: %s", err) + } + return nil +} + +func (t *editor) ColumnCreate(s *state.State, req pb.RpcBlockTableColumnCreateRequest) (string, error) { + switch req.Position { + case model.Block_Left: + req.Position = model.Block_Top + if _, err := pickColumn(s, req.TargetId); err != nil { + return "", fmt.Errorf("pick column: %w", err) + } + case model.Block_Right: + req.Position = model.Block_Bottom + if _, err := pickColumn(s, req.TargetId); err != nil { + return "", fmt.Errorf("pick column: %w", err) + } + case model.Block_Inner: + tb, err := NewTable(s, req.TargetId) + if err != nil { + return "", fmt.Errorf("initialize table state: %w", err) + } + req.TargetId = tb.Columns().Id + default: + return "", fmt.Errorf("position is not supported") + } + + colID, err := t.addColumnHeader(s) + if err != nil { + return "", err + } + if err = s.InsertTo(req.TargetId, req.Position, colID); err != nil { + return "", fmt.Errorf("insert column header: %w", err) + } + + return colID, t.cloneColumnStyles(s, req.TargetId, colID) +} + +func (t *editor) cloneColumnStyles(s *state.State, srcColID, targetColID string) error { + tb, err := NewTable(s, srcColID) + if err != nil { + return fmt.Errorf("init table block: %w", err) + } + colIdx := tb.MakeColumnIndex() + + for _, rowID := range tb.RowIDs() { + row, err := pickRow(s, rowID) + if err != nil { + return fmt.Errorf("pick row: %w", err) + } + + var protoBlock simple.Block + for _, cellID := range row.Model().ChildrenIds { + _, colID, err := ParseCellID(cellID) + if err != nil { + return fmt.Errorf("parse cell id: %w", err) + } + + if colID == srcColID { + protoBlock = s.Pick(cellID) + } + } + + if protoBlock != nil && protoBlock.Model().BackgroundColor != "" { + targetCellID := MakeCellID(rowID, targetColID) + + if !s.Exists(targetCellID) { + _, err := addCell(s, rowID, targetColID) + if err != nil { + return fmt.Errorf("add cell: %w", err) + } + } + cell := s.Get(targetCellID) + cell.Model().BackgroundColor = protoBlock.Model().BackgroundColor + + row = s.Get(row.Model().Id) + row.Model().ChildrenIds = append(row.Model().ChildrenIds, targetCellID) + normalizeRow(colIdx, row) + } + } + + return nil +} + +func (t *editor) ColumnDuplicate(s *state.State, req pb.RpcBlockTableColumnDuplicateRequest) (id string, err error) { + switch req.Position { + case model.Block_Left: + req.Position = model.Block_Top + case model.Block_Right: + req.Position = model.Block_Bottom + default: + return "", fmt.Errorf("position is not supported") + } + + srcCol, err := pickColumn(s, req.BlockId) + if err != nil { + return "", fmt.Errorf("pick source column: %w", err) + } + + _, err = pickColumn(s, req.TargetId) + if err != nil { + return "", fmt.Errorf("pick target column: %w", err) + } + + tb, err := NewTable(s, req.TargetId) + if err != nil { + return "", fmt.Errorf("init table block: %w", err) + } + + newCol := srcCol.Copy() + newCol.Model().Id = t.generateColID() + if !s.Add(newCol) { + return "", fmt.Errorf("add column block") + } + if err = s.InsertTo(req.TargetId, req.Position, newCol.Model().Id); err != nil { + return "", fmt.Errorf("insert column: %w", err) + } + + colIdx := tb.MakeColumnIndex() + + for _, rowID := range tb.RowIDs() { + row, err := getRow(s, rowID) + if err != nil { + return "", fmt.Errorf("get row %s: %w", rowID, err) + } + + var cellID string + for _, id := range row.Model().ChildrenIds { + _, colID, err := ParseCellID(id) + if err != nil { + return "", fmt.Errorf("parse cell %s in row %s: %w", cellID, rowID, err) + } + if colID == req.BlockId { + cellID = id + break + } + } + if cellID == "" { + continue + } + + cell := s.Pick(cellID) + if cell == nil { + return "", fmt.Errorf("cell %s is not found", cellID) + } + cell = cell.Copy() + cell.Model().Id = MakeCellID(rowID, newCol.Model().Id) + + if !s.Add(cell) { + return "", fmt.Errorf("add cell block") + } + + row.Model().ChildrenIds = append(row.Model().ChildrenIds, cell.Model().Id) + normalizeRow(colIdx, row) + } + + return newCol.Model().Id, nil +} + +func (t *editor) Expand(s *state.State, req pb.RpcBlockTableExpandRequest) error { + tb, err := NewTable(s, req.TargetId) + if err != nil { + return fmt.Errorf("init table block: %w", err) + } + + for i := uint32(0); i < req.Columns; i++ { + _, err := t.ColumnCreate(s, pb.RpcBlockTableColumnCreateRequest{ + TargetId: req.TargetId, + Position: model.Block_Inner, + }) + if err != nil { + return fmt.Errorf("create column: %w", err) + } + } + + for i := uint32(0); i < req.Rows; i++ { + rows := tb.Rows() + _, err := t.RowCreate(s, pb.RpcBlockTableRowCreateRequest{ + TargetId: rows.ChildrenIds[len(rows.ChildrenIds)-1], + Position: model.Block_Bottom, + }) + if err != nil { + return fmt.Errorf("create row: %w", err) + } + } + return nil +} + +func (t *editor) Sort(s *state.State, req pb.RpcBlockTableSortRequest) error { + _, err := pickColumn(s, req.ColumnId) + if err != nil { + return fmt.Errorf("pick column: %w", err) + } + + tb, err := NewTable(s, req.ColumnId) + if err != nil { + return fmt.Errorf("init table block: %w", err) + } + + rows := s.Get(tb.Rows().Id) + sorter := tableSorter{ + rowIDs: make([]string, 0, len(rows.Model().ChildrenIds)), + values: make([]string, len(rows.Model().ChildrenIds)), + } + + var headers []string + + var i int + for _, rowID := range rows.Model().ChildrenIds { + row, err := pickRow(s, rowID) + if err != nil { + return fmt.Errorf("pick row %s: %w", rowID, err) + } + if row.Model().GetTableRow().GetIsHeader() { + headers = append(headers, rowID) + continue + } + + sorter.rowIDs = append(sorter.rowIDs, rowID) + for _, cellID := range row.Model().ChildrenIds { + _, colID, err := ParseCellID(cellID) + if err != nil { + return fmt.Errorf("parse cell id %s: %w", cellID, err) + } + if colID == req.ColumnId { + cell := s.Pick(cellID) + if cell == nil { + return fmt.Errorf("cell %s is not found", cellID) + } + sorter.values[i] = cell.Model().GetText().GetText() + } + } + i++ + } + + if req.Type == model.BlockContentDataviewSort_Asc { + sort.Stable(sorter) + } else { + sort.Stable(sort.Reverse(sorter)) + } + + // nolint:gocritic + rows.Model().ChildrenIds = append(headers, sorter.rowIDs...) + + return nil +} + +func (t *editor) CellCreate(s *state.State, rowID string, colID string, b *model.Block) (string, error) { + tb, err := NewTable(s, rowID) + if err != nil { + return "", fmt.Errorf("initialize table state: %w", err) + } + + row, err := getRow(s, rowID) + if err != nil { + return "", fmt.Errorf("get row: %w", err) + } + if _, err = pickColumn(s, colID); err != nil { + return "", fmt.Errorf("pick column: %w", err) + } + + cellID, err := addCell(s, rowID, colID) + if err != nil { + return "", fmt.Errorf("add cell: %w", err) + } + cell := s.Get(cellID) + cell.Model().Content = b.Content + if err := s.InsertTo(rowID, model.Block_Inner, cellID); err != nil { + return "", fmt.Errorf("insert to: %w", err) + } + + colIdx := tb.MakeColumnIndex() + normalizeRow(colIdx, row) + + return cellID, nil +} + +func (t *editor) addColumnHeader(s *state.State) (string, error) { + b := simple.New(&model.Block{ + Id: t.generateColID(), + Content: &model.BlockContentOfTableColumn{ + TableColumn: &model.BlockContentTableColumn{}, + }, + }) + if !s.Add(b) { + return "", fmt.Errorf("add column block") + } + return b.Model().Id, nil +} + +func (t *editor) addRow(s *state.State) (string, error) { + row := makeRow(t.generateRowID()) + if !s.Add(row) { + return "", fmt.Errorf("add row block") + } + return row.Model().Id, nil +} diff --git a/core/block/editor/table/table.go b/core/block/editor/table/table.go index b33f473b6..5491b3427 100644 --- a/core/block/editor/table/table.go +++ b/core/block/editor/table/table.go @@ -2,751 +2,20 @@ package table import ( "fmt" - "sort" + "slices" "strings" - "github.com/globalsign/mgo/bson" - - "github.com/anyproto/anytype-heart/core/block/editor/smartblock" "github.com/anyproto/anytype-heart/core/block/editor/state" "github.com/anyproto/anytype-heart/core/block/simple" "github.com/anyproto/anytype-heart/core/block/simple/table" - "github.com/anyproto/anytype-heart/core/block/simple/text" - "github.com/anyproto/anytype-heart/core/block/source" - "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/logging" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/slice" ) var log = logging.Logger("anytype-simple-tables") -// nolint:revive,interfacebloat -type TableEditor interface { - TableCreate(s *state.State, req pb.RpcBlockTableCreateRequest) (string, error) - RowCreate(s *state.State, req pb.RpcBlockTableRowCreateRequest) (string, error) - RowDelete(s *state.State, req pb.RpcBlockTableRowDeleteRequest) error - ColumnDelete(s *state.State, req pb.RpcBlockTableColumnDeleteRequest) error - ColumnMove(s *state.State, req pb.RpcBlockTableColumnMoveRequest) error - RowDuplicate(s *state.State, req pb.RpcBlockTableRowDuplicateRequest) (newRowID string, err error) - RowListFill(s *state.State, req pb.RpcBlockTableRowListFillRequest) error - RowListClean(s *state.State, req pb.RpcBlockTableRowListCleanRequest) error - RowSetHeader(s *state.State, req pb.RpcBlockTableRowSetHeaderRequest) error - ColumnListFill(s *state.State, req pb.RpcBlockTableColumnListFillRequest) error - cleanupTables(_ smartblock.ApplyInfo) error - ColumnCreate(s *state.State, req pb.RpcBlockTableColumnCreateRequest) (string, error) - cloneColumnStyles(s *state.State, srcColID string, targetColID string) error - ColumnDuplicate(s *state.State, req pb.RpcBlockTableColumnDuplicateRequest) (id string, err error) - Expand(s *state.State, req pb.RpcBlockTableExpandRequest) error - Sort(s *state.State, req pb.RpcBlockTableSortRequest) error - CellCreate(s *state.State, rowID string, colID string, b *model.Block) (string, error) -} - -type Editor struct { - sb smartblock.SmartBlock - - generateRowID func() string - generateColID func() string -} - -var _ TableEditor = &Editor{} - -func NewEditor(sb smartblock.SmartBlock) *Editor { - genID := func() string { - return bson.NewObjectId().Hex() - } - - t := Editor{ - sb: sb, - generateRowID: genID, - generateColID: genID, - } - if sb != nil { - sb.AddHook(t.cleanupTables, smartblock.HookOnBlockClose) - } - return &t -} - -func (t *Editor) TableCreate(s *state.State, req pb.RpcBlockTableCreateRequest) (string, error) { - if t.sb != nil { - if err := t.sb.Restrictions().Object.Check(model.Restrictions_Blocks); err != nil { - return "", err - } - } - - tableBlock := simple.New(&model.Block{ - Content: &model.BlockContentOfTable{ - Table: &model.BlockContentTable{}, - }, - }) - if !s.Add(tableBlock) { - return "", fmt.Errorf("add table block") - } - - if err := s.InsertTo(req.TargetId, req.Position, tableBlock.Model().Id); err != nil { - return "", fmt.Errorf("insert block: %w", err) - } - - columnIds := make([]string, 0, req.Columns) - for i := uint32(0); i < req.Columns; i++ { - id, err := t.addColumnHeader(s) - if err != nil { - return "", err - } - columnIds = append(columnIds, id) - } - columnsLayout := simple.New(&model.Block{ - ChildrenIds: columnIds, - Content: &model.BlockContentOfLayout{ - Layout: &model.BlockContentLayout{ - Style: model.BlockContentLayout_TableColumns, - }, - }, - }) - if !s.Add(columnsLayout) { - return "", fmt.Errorf("add columns block") - } - - rowIDs := make([]string, 0, req.Rows) - for i := uint32(0); i < req.Rows; i++ { - id, err := t.addRow(s) - if err != nil { - return "", err - } - rowIDs = append(rowIDs, id) - } - - rowsLayout := simple.New(&model.Block{ - ChildrenIds: rowIDs, - Content: &model.BlockContentOfLayout{ - Layout: &model.BlockContentLayout{ - Style: model.BlockContentLayout_TableRows, - }, - }, - }) - if !s.Add(rowsLayout) { - return "", fmt.Errorf("add rows block") - } - - tableBlock.Model().ChildrenIds = []string{columnsLayout.Model().Id, rowsLayout.Model().Id} - - if req.WithHeaderRow { - headerID := rowIDs[0] - - if err := t.RowSetHeader(s, pb.RpcBlockTableRowSetHeaderRequest{ - TargetId: headerID, - IsHeader: true, - }); err != nil { - return "", fmt.Errorf("row set header: %w", err) - } - - if err := t.RowListFill(s, pb.RpcBlockTableRowListFillRequest{ - BlockIds: []string{headerID}, - }); err != nil { - return "", fmt.Errorf("fill header row: %w", err) - } - - row, err := getRow(s, headerID) - if err != nil { - return "", fmt.Errorf("get header row: %w", err) - } - - for _, cellID := range row.Model().ChildrenIds { - cell := s.Get(cellID) - if cell == nil { - return "", fmt.Errorf("get header cell id %s", cellID) - } - - cell.Model().BackgroundColor = "grey" - } - } - - return tableBlock.Model().Id, nil -} - -func (t *Editor) RowCreate(s *state.State, req pb.RpcBlockTableRowCreateRequest) (string, error) { - switch req.Position { - case model.Block_Top, model.Block_Bottom: - case model.Block_Inner: - tb, err := NewTable(s, req.TargetId) - if err != nil { - return "", fmt.Errorf("initialize table state: %w", err) - } - req.TargetId = tb.Rows().Id - default: - return "", fmt.Errorf("position is not supported") - } - - rowID, err := t.addRow(s) - if err != nil { - return "", err - } - if err := s.InsertTo(req.TargetId, req.Position, rowID); err != nil { - return "", fmt.Errorf("insert row: %w", err) - } - return rowID, nil -} - -func (t *Editor) RowDelete(s *state.State, req pb.RpcBlockTableRowDeleteRequest) error { - _, err := pickRow(s, req.TargetId) - if err != nil { - return fmt.Errorf("pick target row: %w", err) - } - - if !s.Unlink(req.TargetId) { - return fmt.Errorf("unlink row block") - } - return nil -} - -func (t *Editor) ColumnDelete(s *state.State, req pb.RpcBlockTableColumnDeleteRequest) error { - _, err := pickColumn(s, req.TargetId) - if err != nil { - return fmt.Errorf("pick target column: %w", err) - } - - tb, err := NewTable(s, req.TargetId) - if err != nil { - return fmt.Errorf("initialize table state: %w", err) - } - - for _, rowID := range tb.RowIDs() { - row, err := pickRow(s, rowID) - if err != nil { - return fmt.Errorf("pick row %s: %w", rowID, err) - } - - for _, cellID := range row.Model().ChildrenIds { - _, colID, err := ParseCellID(cellID) - if err != nil { - return fmt.Errorf("parse cell id %s: %w", cellID, err) - } - - if colID == req.TargetId { - if !s.Unlink(cellID) { - return fmt.Errorf("unlink cell %s", cellID) - } - break - } - } - } - if !s.Unlink(req.TargetId) { - return fmt.Errorf("unlink column header") - } - - return nil -} - -func (t *Editor) ColumnMove(s *state.State, req pb.RpcBlockTableColumnMoveRequest) error { - switch req.Position { - case model.Block_Left: - req.Position = model.Block_Top - case model.Block_Right: - req.Position = model.Block_Bottom - default: - return fmt.Errorf("position is not supported") - } - _, err := pickColumn(s, req.TargetId) - if err != nil { - return fmt.Errorf("get target column: %w", err) - } - _, err = pickColumn(s, req.DropTargetId) - if err != nil { - return fmt.Errorf("get drop target column: %w", err) - } - - tb, err := NewTable(s, req.TargetId) - if err != nil { - return fmt.Errorf("init table block: %w", err) - } - - if !s.Unlink(req.TargetId) { - return fmt.Errorf("unlink target column") - } - if err = s.InsertTo(req.DropTargetId, req.Position, req.TargetId); err != nil { - return fmt.Errorf("insert column: %w", err) - } - - colIdx := tb.MakeColumnIndex() - - for _, id := range tb.RowIDs() { - row, err := getRow(s, id) - if err != nil { - return fmt.Errorf("get row %s: %w", id, err) - } - normalizeRow(colIdx, row) - } - - return nil -} - -func (t *Editor) RowDuplicate(s *state.State, req pb.RpcBlockTableRowDuplicateRequest) (newRowID string, err error) { - srcRow, err := pickRow(s, req.BlockId) - if err != nil { - return "", fmt.Errorf("pick source row: %w", err) - } - - newRow := srcRow.Copy() - newRow.Model().Id = t.generateRowID() - if !s.Add(newRow) { - return "", fmt.Errorf("add new row %s", newRow.Model().Id) - } - if err = s.InsertTo(req.TargetId, req.Position, newRow.Model().Id); err != nil { - return "", fmt.Errorf("insert column: %w", err) - } - - for i, srcID := range newRow.Model().ChildrenIds { - cell := s.Pick(srcID) - if cell == nil { - return "", fmt.Errorf("cell %s is not found", srcID) - } - _, colID, err := ParseCellID(srcID) - if err != nil { - return "", fmt.Errorf("parse cell id %s: %w", srcID, err) - } - - newCell := cell.Copy() - newCell.Model().Id = MakeCellID(newRow.Model().Id, colID) - if !s.Add(newCell) { - return "", fmt.Errorf("add new cell %s", newCell.Model().Id) - } - newRow.Model().ChildrenIds[i] = newCell.Model().Id - } - - return newRow.Model().Id, nil -} - -func (t *Editor) RowListFill(s *state.State, req pb.RpcBlockTableRowListFillRequest) error { - if len(req.BlockIds) == 0 { - return fmt.Errorf("empty row list") - } - - tb, err := NewTable(s, req.BlockIds[0]) - if err != nil { - return fmt.Errorf("init table: %w", err) - } - - columns := tb.ColumnIDs() - - for _, rowID := range req.BlockIds { - row, err := getRow(s, rowID) - if err != nil { - return fmt.Errorf("get row %s: %w", rowID, err) - } - - newIds := make([]string, 0, len(columns)) - for _, colID := range columns { - id := MakeCellID(rowID, colID) - newIds = append(newIds, id) - - if !s.Exists(id) { - _, err := addCell(s, rowID, colID) - if err != nil { - return fmt.Errorf("add cell %s: %w", id, err) - } - } - } - row.Model().ChildrenIds = newIds - } - return nil -} - -func (t *Editor) RowListClean(s *state.State, req pb.RpcBlockTableRowListCleanRequest) error { - if len(req.BlockIds) == 0 { - return fmt.Errorf("empty row list") - } - - for _, rowID := range req.BlockIds { - row, err := pickRow(s, rowID) - if err != nil { - return fmt.Errorf("pick row: %w", err) - } - - for _, cellID := range row.Model().ChildrenIds { - cell := s.Pick(cellID) - if v, ok := cell.(text.Block); ok && v.IsEmpty() { - s.Unlink(cellID) - } - } - } - return nil -} - -func (t *Editor) RowSetHeader(s *state.State, req pb.RpcBlockTableRowSetHeaderRequest) error { - tb, err := NewTable(s, req.TargetId) - if err != nil { - return fmt.Errorf("init table: %w", err) - } - - row, err := getRow(s, req.TargetId) - if err != nil { - return fmt.Errorf("get target row: %w", err) - } - - if row.Model().GetTableRow().IsHeader != req.IsHeader { - row.Model().GetTableRow().IsHeader = req.IsHeader - - err = normalizeRows(s, tb) - if err != nil { - return fmt.Errorf("normalize rows: %w", err) - } - } - - return nil -} - -func (t *Editor) ColumnListFill(s *state.State, req pb.RpcBlockTableColumnListFillRequest) error { - if len(req.BlockIds) == 0 { - return fmt.Errorf("empty row list") - } - - tb, err := NewTable(s, req.BlockIds[0]) - if err != nil { - return fmt.Errorf("init table: %w", err) - } - - rows := tb.RowIDs() - - for _, colID := range req.BlockIds { - for _, rowID := range rows { - id := MakeCellID(rowID, colID) - if s.Exists(id) { - continue - } - _, err := addCell(s, rowID, colID) - if err != nil { - return fmt.Errorf("add cell %s: %w", id, err) - } - - row, err := getRow(s, rowID) - if err != nil { - return fmt.Errorf("get row %s: %w", rowID, err) - } - - row.Model().ChildrenIds = append(row.Model().ChildrenIds, id) - } - } - - colIdx := tb.MakeColumnIndex() - for _, rowID := range rows { - row, err := getRow(s, rowID) - if err != nil { - return fmt.Errorf("get row %s: %w", rowID, err) - } - normalizeRow(colIdx, row) - } - - return nil -} - -func (t *Editor) cleanupTables(_ smartblock.ApplyInfo) error { - if t.sb == nil { - return fmt.Errorf("nil smartblock") - } - s := t.sb.NewState() - - err := s.Iterate(func(b simple.Block) bool { - if b.Model().GetTable() == nil { - return true - } - - tb, err := NewTable(s, b.Model().Id) - if err != nil { - log.Errorf("cleanup: init table %s: %s", b.Model().Id, err) - return true - } - err = t.RowListClean(s, pb.RpcBlockTableRowListCleanRequest{ - BlockIds: tb.RowIDs(), - }) - if err != nil { - log.Errorf("cleanup table %s: %s", b.Model().Id, err) - return true - } - return true - }) - if err != nil { - log.Errorf("cleanup iterate: %s", err) - } - - if err = t.sb.Apply(s, smartblock.KeepInternalFlags); err != nil { - if err == source.ErrReadOnly { - return nil - } - log.Errorf("cleanup apply: %s", err) - } - return nil -} - -func (t *Editor) ColumnCreate(s *state.State, req pb.RpcBlockTableColumnCreateRequest) (string, error) { - switch req.Position { - case model.Block_Left: - req.Position = model.Block_Top - if _, err := pickColumn(s, req.TargetId); err != nil { - return "", fmt.Errorf("pick column: %w", err) - } - case model.Block_Right: - req.Position = model.Block_Bottom - if _, err := pickColumn(s, req.TargetId); err != nil { - return "", fmt.Errorf("pick column: %w", err) - } - case model.Block_Inner: - tb, err := NewTable(s, req.TargetId) - if err != nil { - return "", fmt.Errorf("initialize table state: %w", err) - } - req.TargetId = tb.Columns().Id - default: - return "", fmt.Errorf("position is not supported") - } - - colID, err := t.addColumnHeader(s) - if err != nil { - return "", err - } - if err = s.InsertTo(req.TargetId, req.Position, colID); err != nil { - return "", fmt.Errorf("insert column header: %w", err) - } - - return colID, t.cloneColumnStyles(s, req.TargetId, colID) -} - -func (t *Editor) cloneColumnStyles(s *state.State, srcColID, targetColID string) error { - tb, err := NewTable(s, srcColID) - if err != nil { - return fmt.Errorf("init table block: %w", err) - } - colIdx := tb.MakeColumnIndex() - - for _, rowID := range tb.RowIDs() { - row, err := pickRow(s, rowID) - if err != nil { - return fmt.Errorf("pick row: %w", err) - } - - var protoBlock simple.Block - for _, cellID := range row.Model().ChildrenIds { - _, colID, err := ParseCellID(cellID) - if err != nil { - return fmt.Errorf("parse cell id: %w", err) - } - - if colID == srcColID { - protoBlock = s.Pick(cellID) - } - } - - if protoBlock != nil && protoBlock.Model().BackgroundColor != "" { - targetCellID := MakeCellID(rowID, targetColID) - - if !s.Exists(targetCellID) { - _, err := addCell(s, rowID, targetColID) - if err != nil { - return fmt.Errorf("add cell: %w", err) - } - } - cell := s.Get(targetCellID) - cell.Model().BackgroundColor = protoBlock.Model().BackgroundColor - - row = s.Get(row.Model().Id) - row.Model().ChildrenIds = append(row.Model().ChildrenIds, targetCellID) - normalizeRow(colIdx, row) - } - } - - return nil -} - -func (t *Editor) ColumnDuplicate(s *state.State, req pb.RpcBlockTableColumnDuplicateRequest) (id string, err error) { - switch req.Position { - case model.Block_Left: - req.Position = model.Block_Top - case model.Block_Right: - req.Position = model.Block_Bottom - default: - return "", fmt.Errorf("position is not supported") - } - - srcCol, err := pickColumn(s, req.BlockId) - if err != nil { - return "", fmt.Errorf("pick source column: %w", err) - } - - _, err = pickColumn(s, req.TargetId) - if err != nil { - return "", fmt.Errorf("pick target column: %w", err) - } - - tb, err := NewTable(s, req.TargetId) - if err != nil { - return "", fmt.Errorf("init table block: %w", err) - } - - newCol := srcCol.Copy() - newCol.Model().Id = t.generateColID() - if !s.Add(newCol) { - return "", fmt.Errorf("add column block") - } - if err = s.InsertTo(req.TargetId, req.Position, newCol.Model().Id); err != nil { - return "", fmt.Errorf("insert column: %w", err) - } - - colIdx := tb.MakeColumnIndex() - - for _, rowID := range tb.RowIDs() { - row, err := getRow(s, rowID) - if err != nil { - return "", fmt.Errorf("get row %s: %w", rowID, err) - } - - var cellID string - for _, id := range row.Model().ChildrenIds { - _, colID, err := ParseCellID(id) - if err != nil { - return "", fmt.Errorf("parse cell %s in row %s: %w", cellID, rowID, err) - } - if colID == req.BlockId { - cellID = id - break - } - } - if cellID == "" { - continue - } - - cell := s.Pick(cellID) - if cell == nil { - return "", fmt.Errorf("cell %s is not found", cellID) - } - cell = cell.Copy() - cell.Model().Id = MakeCellID(rowID, newCol.Model().Id) - - if !s.Add(cell) { - return "", fmt.Errorf("add cell block") - } - - row.Model().ChildrenIds = append(row.Model().ChildrenIds, cell.Model().Id) - normalizeRow(colIdx, row) - } - - return newCol.Model().Id, nil -} - -func (t *Editor) Expand(s *state.State, req pb.RpcBlockTableExpandRequest) error { - tb, err := NewTable(s, req.TargetId) - if err != nil { - return fmt.Errorf("init table block: %w", err) - } - - for i := uint32(0); i < req.Columns; i++ { - _, err := t.ColumnCreate(s, pb.RpcBlockTableColumnCreateRequest{ - TargetId: req.TargetId, - Position: model.Block_Inner, - }) - if err != nil { - return fmt.Errorf("create column: %w", err) - } - } - - for i := uint32(0); i < req.Rows; i++ { - rows := tb.Rows() - _, err := t.RowCreate(s, pb.RpcBlockTableRowCreateRequest{ - TargetId: rows.ChildrenIds[len(rows.ChildrenIds)-1], - Position: model.Block_Bottom, - }) - if err != nil { - return fmt.Errorf("create row: %w", err) - } - } - return nil -} - -func (t *Editor) Sort(s *state.State, req pb.RpcBlockTableSortRequest) error { - _, err := pickColumn(s, req.ColumnId) - if err != nil { - return fmt.Errorf("pick column: %w", err) - } - - tb, err := NewTable(s, req.ColumnId) - if err != nil { - return fmt.Errorf("init table block: %w", err) - } - - rows := s.Get(tb.Rows().Id) - sorter := tableSorter{ - rowIDs: make([]string, 0, len(rows.Model().ChildrenIds)), - values: make([]string, len(rows.Model().ChildrenIds)), - } - - var headers []string - - var i int - for _, rowID := range rows.Model().ChildrenIds { - row, err := pickRow(s, rowID) - if err != nil { - return fmt.Errorf("pick row %s: %w", rowID, err) - } - if row.Model().GetTableRow().GetIsHeader() { - headers = append(headers, rowID) - continue - } - - sorter.rowIDs = append(sorter.rowIDs, rowID) - for _, cellID := range row.Model().ChildrenIds { - _, colID, err := ParseCellID(cellID) - if err != nil { - return fmt.Errorf("parse cell id %s: %w", cellID, err) - } - if colID == req.ColumnId { - cell := s.Pick(cellID) - if cell == nil { - return fmt.Errorf("cell %s is not found", cellID) - } - sorter.values[i] = cell.Model().GetText().GetText() - } - } - i++ - } - - if req.Type == model.BlockContentDataviewSort_Asc { - sort.Stable(sorter) - } else { - sort.Stable(sort.Reverse(sorter)) - } - - // nolint:gocritic - rows.Model().ChildrenIds = append(headers, sorter.rowIDs...) - - return nil -} - -func (t *Editor) CellCreate(s *state.State, rowID string, colID string, b *model.Block) (string, error) { - tb, err := NewTable(s, rowID) - if err != nil { - return "", fmt.Errorf("initialize table state: %w", err) - } - - row, err := getRow(s, rowID) - if err != nil { - return "", fmt.Errorf("get row: %w", err) - } - if _, err = pickColumn(s, colID); err != nil { - return "", fmt.Errorf("pick column: %w", err) - } - - cellID, err := addCell(s, rowID, colID) - if err != nil { - return "", fmt.Errorf("add cell: %w", err) - } - cell := s.Get(cellID) - cell.Model().Content = b.Content - if err := s.InsertTo(rowID, model.Block_Inner, cellID); err != nil { - return "", fmt.Errorf("insert to: %w", err) - } - - colIdx := tb.MakeColumnIndex() - normalizeRow(colIdx, row) - - return cellID, nil -} +var ErrCannotMoveTableBlocks = fmt.Errorf("can not move table blocks") type tableSorter struct { rowIDs []string @@ -766,27 +35,6 @@ func (t tableSorter) Swap(i, j int) { t.rowIDs[i], t.rowIDs[j] = t.rowIDs[j], t.rowIDs[i] } -func (t *Editor) addColumnHeader(s *state.State) (string, error) { - b := simple.New(&model.Block{ - Id: t.generateColID(), - Content: &model.BlockContentOfTableColumn{ - TableColumn: &model.BlockContentTableColumn{}, - }, - }) - if !s.Add(b) { - return "", fmt.Errorf("add column block") - } - return b.Model().Id, nil -} - -func (t *Editor) addRow(s *state.State) (string, error) { - row := makeRow(t.generateRowID()) - if !s.Add(row) { - return "", fmt.Errorf("add row block") - } - return row.Model().Id, nil -} - func makeRow(id string) simple.Block { return simple.New(&model.Block{ Id: id, @@ -842,6 +90,11 @@ func ParseCellID(id string) (rowID string, colID string, err error) { return toks[0], toks[1], nil } +func isTableCell(id string) bool { + _, _, err := ParseCellID(id) + return err == nil +} + func addCell(s *state.State, rowID, colID string) (string, error) { c := simple.New(&model.Block{ Id: MakeCellID(rowID, colID), @@ -1012,3 +265,38 @@ func (tb Table) Iterate(f func(b simple.Block, pos CellPosition) bool) error { } return nil } + +// CheckTableBlocksMove checks if Insert operation is allowed in case table blocks are affected +func CheckTableBlocksMove(st *state.State, target string, pos model.BlockPosition, blockIds []string) (string, model.BlockPosition, error) { + // nolint:errcheck + if t, _ := NewTable(st, target); t != nil { + // we allow moving rows between each other + if slice.ContainsAll(t.RowIDs(), append(blockIds, target)...) { + if pos == model.Block_Bottom || pos == model.Block_Top { + return target, pos, nil + } + return "", 0, fmt.Errorf("failed to move rows: position should be Top or Bottom, got %s", model.BlockPosition_name[int32(pos)]) + } + } + + for _, id := range blockIds { + t := GetTableRootBlock(st, id) + if t != nil && t.Model().Id != id { + // we should not move table blocks except table root block + return "", 0, ErrCannotMoveTableBlocks + } + } + + t := GetTableRootBlock(st, target) + if t != nil && t.Model().Id != target { + // we allow inserting blocks into table cell + if isTableCell(target) && slices.Contains([]model.BlockPosition{model.Block_Inner, model.Block_Replace, model.Block_InnerFirst}, pos) { + return target, pos, nil + } + + // if the target is one of table blocks, but not cell or table root, we should insert blocks under the table + return t.Model().Id, model.Block_Bottom, nil + } + + return target, pos, nil +} diff --git a/core/block/editor/table/table_test.go b/core/block/editor/table/table_test.go index 3dbd1faca..9fb54c52c 100644 --- a/core/block/editor/table/table_test.go +++ b/core/block/editor/table/table_test.go @@ -104,7 +104,7 @@ func TestRowCreate(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - tb := Editor{ + tb := editor{ generateRowID: idFromSlice([]string{tc.newRowId}), } id, err := tb.RowCreate(tc.source, tc.req) @@ -171,7 +171,7 @@ func TestRowListClean(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - tb := Editor{} + tb := editor{} err := tb.RowListClean(tc.source, tc.req) require.NoError(t, err) assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) @@ -222,7 +222,7 @@ func TestExpand(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - tb := Editor{ + tb := editor{ generateColID: idFromSlice(tc.newColIds), generateRowID: idFromSlice(tc.newRowIds), } @@ -292,7 +292,7 @@ func TestRowListFill(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - tb := Editor{} + tb := editor{} err := tb.RowListFill(tc.source, tc.req) require.NoError(t, err) assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) @@ -353,7 +353,7 @@ func TestColumnListFill(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - tb := Editor{} + tb := editor{} err := tb.ColumnListFill(tc.source, tc.req) require.NoError(t, err) assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) @@ -411,7 +411,7 @@ func TestColumnCreate(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - tb := Editor{ + tb := editor{ generateColID: idFromSlice([]string{tc.newColId}), } id, err := tb.ColumnCreate(tc.source, tc.req) @@ -506,7 +506,7 @@ func TestColumnDuplicate(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - tb := Editor{ + tb := editor{ generateColID: idFromSlice([]string{tc.newColId}), } id, err := tb.ColumnDuplicate(tc.source, tc.req) @@ -604,7 +604,7 @@ func TestRowDuplicate(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - tb := Editor{ + tb := editor{ generateRowID: idFromSlice([]string{tc.newRowId}), } id, err := tb.RowDuplicate(tc.source, tc.req) @@ -660,7 +660,7 @@ func TestColumnMove(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - tb := Editor{} + tb := editor{} err := tb.ColumnMove(tc.source, tc.req) require.NoError(t, err) assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) @@ -709,7 +709,7 @@ func TestColumnDelete(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - tb := Editor{} + tb := editor{} err := tb.ColumnDelete(tc.source, tc.req) require.NoError(t, err) assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) @@ -866,7 +866,7 @@ func TestSort(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - tb := Editor{} + tb := editor{} err := tb.Sort(tc.source, tc.req) require.NoError(t, err) assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) @@ -911,7 +911,7 @@ func TestRowSetHeader(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - tb := Editor{} + tb := editor{} err := tb.RowSetHeader(tc.source, tc.req) require.NoError(t, err) assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) @@ -933,9 +933,10 @@ func TestEditorAPI(t *testing.T) { }), }).(*state.State) - ed := NewEditor(nil) - ed.generateColID = idFromSlice([]string{"col1", "col2", "col3"}) - ed.generateRowID = idFromSlice([]string{"row1", "row2"}) + ed := editor{ + generateColID: idFromSlice([]string{"col1", "col2", "col3"}), + generateRowID: idFromSlice([]string{"row1", "row2"}), + } tableID, err := ed.TableCreate(s, pb.RpcBlockTableCreateRequest{ TargetId: "root", From 8d399fed60ae28413fc0e6212c14b1437c4a1552 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 19 Jul 2024 18:41:37 +0200 Subject: [PATCH 20/71] GO-3769 Add error on limit reached --- core/syncstatus/spacesyncstatus/spacestatus.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/syncstatus/spacesyncstatus/spacestatus.go b/core/syncstatus/spacesyncstatus/spacestatus.go index 1f95e01dc..4b6db2547 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus.go +++ b/core/syncstatus/spacesyncstatus/spacestatus.go @@ -275,6 +275,7 @@ func (s *spaceSyncStatus) makeSyncEvent(spaceId string, params syncParams) *pb.E status = pb.EventSpace_Syncing } if params.bytesLeftPercentage < 0.1 { + status = pb.EventSpace_Error err = pb.EventSpace_StorageLimitExceed } if params.connectionStatus == nodestatus.ConnectionError { From cb961e7abba40fae8d62dda2704a6ad262dc05e5 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 22 Jul 2024 13:30:19 +0200 Subject: [PATCH 21/71] GO-3769 Fix local mode --- .../syncstatus/spacesyncstatus/spacestatus.go | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/core/syncstatus/spacesyncstatus/spacestatus.go b/core/syncstatus/spacesyncstatus/spacestatus.go index 4b6db2547..b113b08e0 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus.go +++ b/core/syncstatus/spacesyncstatus/spacestatus.go @@ -84,6 +84,7 @@ type spaceSyncStatus struct { missingIds map[string][]string mx sync.Mutex periodicCall periodicsync.PeriodicSync + isLocal bool finish chan struct{} } @@ -103,6 +104,7 @@ func (s *spaceSyncStatus) Init(a *app.App) (err error) { s.subs = app.MustComponent[syncsubscritions.SyncSubscriptions](a) s.missingIds = make(map[string][]string) s.spaceIdGetter = app.MustComponent[SpaceIdGetter](a) + s.isLocal = s.networkConfig.GetNetworkMode() == pb.RpcAccount_LocalOnly sessionHookRunner := app.MustComponent[session.HookRunner](a) sessionHookRunner.RegisterHook(s.sendSyncEventForNewSession) s.periodicCall = periodicsync.NewPeriodicSync(1, time.Second*5, s.update, logger.CtxLogger{Logger: log.Desugar()}) @@ -128,13 +130,7 @@ func (s *spaceSyncStatus) UpdateMissingIds(spaceId string, ids []string) { } func (s *spaceSyncStatus) Run(ctx context.Context) (err error) { - if s.networkConfig.GetNetworkMode() == pb.RpcAccount_LocalOnly { - s.sendLocalOnlyEvent() - close(s.finish) - return - } else { - s.sendStartEvent(s.spaceIdGetter.AllSpaceIds()) - } + s.sendStartEvent(s.spaceIdGetter.AllSpaceIds()) s.ctx, s.ctxCancel = context.WithCancel(context.Background()) s.periodicCall.Run() return @@ -166,6 +162,10 @@ func (s *spaceSyncStatus) sendEventToSession(spaceId, token string) { s.mx.Lock() missingObjects := s.missingIds[spaceId] s.mx.Unlock() + if s.isLocal { + s.sendLocalOnlyEventToSession(spaceId, token) + return + } params := syncParams{ bytesLeftPercentage: s.getBytesLeftPercentage(spaceId), connectionStatus: s.nodeStatus.GetNodeStatus(spaceId), @@ -191,11 +191,26 @@ func (s *spaceSyncStatus) sendStartEvent(spaceIds []string) { } } -func (s *spaceSyncStatus) sendLocalOnlyEvent() { +func (s *spaceSyncStatus) sendLocalOnlyEvent(spaceId string) { s.eventSender.Broadcast(&pb.Event{ Messages: []*pb.EventMessage{{ Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ + Id: spaceId, + Status: pb.EventSpace_Offline, + Network: pb.EventSpace_LocalOnly, + }, + }, + }}, + }) +} + +func (s *spaceSyncStatus) sendLocalOnlyEventToSession(spaceId, token string) { + s.eventSender.SendToSession(token, &pb.Event{ + Messages: []*pb.EventMessage{{ + Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ + SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ + Id: spaceId, Status: pb.EventSpace_Offline, Network: pb.EventSpace_LocalOnly, }, @@ -238,6 +253,10 @@ func (s *spaceSyncStatus) getBytesLeftPercentage(spaceId string) float64 { } func (s *spaceSyncStatus) updateSpaceSyncStatus(spaceId string, missingObjects []string) { + if s.isLocal { + s.sendLocalOnlyEvent(spaceId) + return + } params := syncParams{ bytesLeftPercentage: s.getBytesLeftPercentage(spaceId), connectionStatus: s.nodeStatus.GetNodeStatus(spaceId), From 6cb6b0677f9e5ec55de3a72b635756a20ec5f8e7 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 23 Jul 2024 00:23:42 +0200 Subject: [PATCH 22/71] GO-3769 Add old status events --- .../syncstatus/objectsyncstatus/syncstatus.go | 95 ++++++++++++++++--- 1 file changed, 83 insertions(+), 12 deletions(-) diff --git a/core/syncstatus/objectsyncstatus/syncstatus.go b/core/syncstatus/objectsyncstatus/syncstatus.go index 5653c4013..d10c25cd2 100644 --- a/core/syncstatus/objectsyncstatus/syncstatus.go +++ b/core/syncstatus/objectsyncstatus/syncstatus.go @@ -7,6 +7,7 @@ import ( "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/app/logger" + "github.com/anyproto/any-sync/commonspace/object/tree/treestorage" "github.com/anyproto/any-sync/commonspace/spacestate" "github.com/anyproto/any-sync/commonspace/syncstatus" @@ -18,10 +19,11 @@ import ( "github.com/anyproto/anytype-heart/core/anytype/config" "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" + "github.com/anyproto/anytype-heart/util/slice" ) const ( - syncUpdateInterval = 2 + syncUpdateInterval = 3 syncTimeout = time.Second ) @@ -40,6 +42,11 @@ const ( StatusNotSynced ) +type treeHeadsEntry struct { + heads []string + syncStatus SyncStatus +} + type StatusUpdater interface { HeadsChange(treeId string, heads []string) HeadsReceive(senderId, treeId string, heads []string) @@ -77,10 +84,11 @@ type syncStatusService struct { updateReceiver UpdateReceiver storage spacestorage.SpaceStorage - spaceId string - synced []string - tempSynced map[string]struct{} - stateCounter uint64 + spaceId string + synced []string + tempSynced map[string]struct{} + treeHeads map[string]treeHeadsEntry + watchers map[string]struct{} updateIntervalSecs int updateTimeout time.Duration @@ -94,6 +102,8 @@ type syncStatusService struct { func NewSyncStatusService() StatusService { return &syncStatusService{ tempSynced: map[string]struct{}{}, + treeHeads: map[string]treeHeadsEntry{}, + watchers: map[string]struct{}{}, } } @@ -155,30 +165,56 @@ func (s *syncStatusService) HeadsApply(senderId, treeId string, heads []string, } return } - if allAdded { - s.synced = append(s.synced, treeId) + if !allAdded { + return + } + s.synced = append(s.synced, treeId) + if curTreeHeads, ok := s.treeHeads[treeId]; ok { + // checking if we received the head that we are interested in + for _, head := range heads { + if idx, found := slices.BinarySearch(curTreeHeads.heads, head); found { + curTreeHeads.heads[idx] = "" + } + } + curTreeHeads.heads = slice.RemoveMut(curTreeHeads.heads, "") + if len(curTreeHeads.heads) == 0 { + curTreeHeads.syncStatus = StatusSynced + } + s.treeHeads[treeId] = curTreeHeads } } func (s *syncStatusService) update(ctx context.Context) (err error) { - var treeStatusBuf []treeStatus + var ( + updateDetailsStatuses []treeStatus + updateThreadStatuses []treeStatus + ) s.Lock() if s.updateReceiver == nil { s.Unlock() return } for _, treeId := range s.synced { - treeStatusBuf = append(treeStatusBuf, treeStatus{treeId, StatusSynced}) + updateDetailsStatuses = append(updateDetailsStatuses, treeStatus{treeId, StatusSynced}) + } + for treeId := range s.watchers { + treeHeads, exists := s.treeHeads[treeId] + if !exists { + continue + } + updateThreadStatuses = append(updateThreadStatuses, treeStatus{treeId, treeHeads.syncStatus}) } s.synced = s.synced[:0] s.Unlock() s.updateReceiver.UpdateNodeStatus() - for _, entry := range treeStatusBuf { + for _, entry := range updateDetailsStatuses { + s.updateDetails(entry.treeId, mapStatus(entry.status)) + } + for _, entry := range updateThreadStatuses { err = s.updateReceiver.UpdateTree(ctx, entry.treeId, entry.status) if err != nil { return } - s.updateDetails(entry.treeId, mapStatus(entry.status)) } return } @@ -194,10 +230,37 @@ func (s *syncStatusService) HeadsReceive(senderId, treeId string, heads []string } func (s *syncStatusService) Watch(treeId string) (err error) { - return nil + s.Lock() + defer s.Unlock() + _, ok := s.treeHeads[treeId] + if !ok { + var ( + st treestorage.TreeStorage + heads []string + ) + st, err = s.storage.TreeStorage(treeId) + if err != nil { + return + } + heads, err = st.Heads() + if err != nil { + return + } + slices.Sort(heads) + s.treeHeads[treeId] = treeHeadsEntry{ + heads: heads, + syncStatus: StatusUnknown, + } + } + + s.watchers[treeId] = struct{}{} + return } func (s *syncStatusService) Unwatch(treeId string) { + s.Lock() + defer s.Unlock() + delete(s.watchers, treeId) } func (s *syncStatusService) RemoveAllExcept(senderId string, differentRemoteIds []string) { @@ -210,6 +273,14 @@ func (s *syncStatusService) RemoveAllExcept(senderId string, differentRemoteIds defer s.Unlock() slices.Sort(differentRemoteIds) + for treeId, entry := range s.treeHeads { + if _, found := slices.BinarySearch(differentRemoteIds, treeId); !found { + if entry.syncStatus != StatusSynced { + entry.syncStatus = StatusSynced + s.treeHeads[treeId] = entry + } + } + } for treeId := range s.tempSynced { delete(s.tempSynced, treeId) if _, found := slices.BinarySearch(differentRemoteIds, treeId); !found { From 794cb94c6c8c1441432013f63eab95be94fd0d32 Mon Sep 17 00:00:00 2001 From: kirillston Date: Tue, 23 Jul 2024 12:20:59 +0300 Subject: [PATCH 23/71] GO-3192 Fix lint --- core/block/editor/table/editor.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/block/editor/table/editor.go b/core/block/editor/table/editor.go index 5ee585a85..83ac42363 100644 --- a/core/block/editor/table/editor.go +++ b/core/block/editor/table/editor.go @@ -1,6 +1,7 @@ package table import ( + "errors" "fmt" "sort" @@ -463,7 +464,7 @@ func (t *editor) cleanupTables(_ smartblock.ApplyInfo) error { } if err = t.sb.Apply(s, smartblock.KeepInternalFlags); err != nil { - if err == source.ErrReadOnly { + if errors.Is(err, source.ErrReadOnly) { return nil } log.Errorf("cleanup apply: %s", err) From 826a24e92cbba9c908b742e1626707a1fb17140a Mon Sep 17 00:00:00 2001 From: kirillston Date: Tue, 23 Jul 2024 13:35:56 +0300 Subject: [PATCH 24/71] GO-3192 We cannot move blocks to cell --- core/block/editor/basic/basic_test.go | 17 +---------------- core/block/editor/table/table.go | 26 +++++++------------------- 2 files changed, 8 insertions(+), 35 deletions(-) diff --git a/core/block/editor/basic/basic_test.go b/core/block/editor/basic/basic_test.go index 5a78d1d48..ba6d3a22f 100644 --- a/core/block/editor/basic/basic_test.go +++ b/core/block/editor/basic/basic_test.go @@ -375,7 +375,7 @@ func TestBasic_MoveTableBlocks(t *testing.T) { assert.True(t, errors.Is(err, table.ErrCannotMoveTableBlocks)) }) - for _, block := range []string{"columns", "rows", "column", "row"} { + for _, block := range []string{"columns", "rows", "column", "row", "column-row"} { t.Run("moving a block to '"+block+"' block leads to moving it under the table", func(t *testing.T) { // given sb := getSB() @@ -405,21 +405,6 @@ func TestBasic_MoveTableBlocks(t *testing.T) { assert.NoError(t, err) assert.Equal(t, []string{"table", "upper", "block"}, st.Pick("test").Model().ChildrenIds) }) - - t.Run("moving a block to table cell is allowed", func(t *testing.T) { - // given - sb := getSB() - b := NewBasic(sb, nil, converter.NewLayoutConverter()) - st := sb.NewState() - - // when - err := b.Move(st, st, "column-row", model.Block_Inner, []string{"upper"}) - - // then - assert.NoError(t, err) - assert.Equal(t, []string{"table", "block"}, st.Pick("test").Model().ChildrenIds) - assert.Equal(t, []string{"upper"}, st.Pick("column-row").Model().ChildrenIds) - }) } func TestBasic_MoveToAnotherObject(t *testing.T) { diff --git a/core/block/editor/table/table.go b/core/block/editor/table/table.go index 5491b3427..311d6c7d5 100644 --- a/core/block/editor/table/table.go +++ b/core/block/editor/table/table.go @@ -2,7 +2,6 @@ package table import ( "fmt" - "slices" "strings" "github.com/anyproto/anytype-heart/core/block/editor/state" @@ -90,11 +89,6 @@ func ParseCellID(id string) (rowID string, colID string, err error) { return toks[0], toks[1], nil } -func isTableCell(id string) bool { - _, _, err := ParseCellID(id) - return err == nil -} - func addCell(s *state.State, rowID, colID string) (string, error) { c := simple.New(&model.Block{ Id: MakeCellID(rowID, colID), @@ -121,7 +115,7 @@ func NewTable(s *state.State, id string) (*Table, error) { s: s, } - tb.block = GetTableRootBlock(s, id) + tb.block = PickTableRootBlock(s, id) if tb.block == nil { return nil, fmt.Errorf("root table block is not found") } @@ -147,8 +141,8 @@ func NewTable(s *state.State, id string) (*Table, error) { return &tb, nil } -// GetTableRootBlock iterates over parents of block. Returns nil in case root table block is not found -func GetTableRootBlock(s *state.State, id string) (block simple.Block) { +// PickTableRootBlock iterates over parents of block. Returns nil in case root table block is not found +func PickTableRootBlock(s *state.State, id string) (block simple.Block) { next := s.Pick(id) for next != nil { if next.Model().GetTable() != nil { @@ -268,8 +262,7 @@ func (tb Table) Iterate(f func(b simple.Block, pos CellPosition) bool) error { // CheckTableBlocksMove checks if Insert operation is allowed in case table blocks are affected func CheckTableBlocksMove(st *state.State, target string, pos model.BlockPosition, blockIds []string) (string, model.BlockPosition, error) { - // nolint:errcheck - if t, _ := NewTable(st, target); t != nil { + if t, err := NewTable(st, target); err == nil && t != nil { // we allow moving rows between each other if slice.ContainsAll(t.RowIDs(), append(blockIds, target)...) { if pos == model.Block_Bottom || pos == model.Block_Top { @@ -280,21 +273,16 @@ func CheckTableBlocksMove(st *state.State, target string, pos model.BlockPositio } for _, id := range blockIds { - t := GetTableRootBlock(st, id) + t := PickTableRootBlock(st, id) if t != nil && t.Model().Id != id { // we should not move table blocks except table root block return "", 0, ErrCannotMoveTableBlocks } } - t := GetTableRootBlock(st, target) + t := PickTableRootBlock(st, target) if t != nil && t.Model().Id != target { - // we allow inserting blocks into table cell - if isTableCell(target) && slices.Contains([]model.BlockPosition{model.Block_Inner, model.Block_Replace, model.Block_InnerFirst}, pos) { - return target, pos, nil - } - - // if the target is one of table blocks, but not cell or table root, we should insert blocks under the table + // if the target is one of table blocks, but not table root, we should insert blocks under the table return t.Model().Id, model.Block_Bottom, nil } From 24444a1d4b97a7d0cf8b9522e7523cb9fae81f67 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 23 Jul 2024 19:41:14 +0200 Subject: [PATCH 25/71] GO-3769 Simplify updater --- core/syncstatus/detailsupdater/updater.go | 100 +++++------ core/syncstatus/spacesyncstatus/filestate.go | 105 ------------ .../spacesyncstatus/filestate_test.go | 161 ------------------ .../syncstatus/spacesyncstatus/objectstate.go | 121 ------------- .../spacesyncstatus/objectstate_test.go | 157 ----------------- 5 files changed, 44 insertions(+), 600 deletions(-) delete mode 100644 core/syncstatus/spacesyncstatus/filestate.go delete mode 100644 core/syncstatus/spacesyncstatus/filestate_test.go delete mode 100644 core/syncstatus/spacesyncstatus/objectstate.go delete mode 100644 core/syncstatus/spacesyncstatus/objectstate_test.go diff --git a/core/syncstatus/detailsupdater/updater.go b/core/syncstatus/detailsupdater/updater.go index 70e4996e2..48d414934 100644 --- a/core/syncstatus/detailsupdater/updater.go +++ b/core/syncstatus/detailsupdater/updater.go @@ -3,7 +3,6 @@ package detailsupdater import ( "context" "errors" - "fmt" "slices" "sync" "time" @@ -33,10 +32,9 @@ var log = logging.Logger(CName) const CName = "core.syncstatus.objectsyncstatus.updater" type syncStatusDetails struct { - objectId string - markAllSyncedExcept []string - status domain.ObjectSyncStatus - spaceId string + objectId string + status domain.ObjectSyncStatus + spaceId string } type Updater interface { @@ -55,7 +53,7 @@ type syncStatusUpdater struct { objectStore objectstore.ObjectStore ctx context.Context ctxCancel context.CancelFunc - batcher *mb.MB[*syncStatusDetails] + batcher *mb.MB[string] spaceService space.Service spaceSyncStatus SpaceStatusUpdater syncSubscriptions syncsubscritions.SyncSubscriptions @@ -68,7 +66,7 @@ type syncStatusUpdater struct { func NewUpdater() Updater { return &syncStatusUpdater{ - batcher: mb.New[*syncStatusDetails](0), + batcher: mb.New[string](0), finish: make(chan struct{}), entries: make(map[string]*syncStatusDetails, 0), } @@ -104,14 +102,7 @@ func (u *syncStatusUpdater) UpdateDetails(objectId string, status domain.ObjectS if spaceId == u.spaceService.TechSpaceId() { return } - u.mx.Lock() - u.entries[objectId] = &syncStatusDetails{ - objectId: objectId, - status: status, - spaceId: spaceId, - } - u.mx.Unlock() - err := u.batcher.TryAdd(&syncStatusDetails{ + err := u.addToQueue(&syncStatusDetails{ objectId: objectId, status: status, spaceId: spaceId, @@ -121,19 +112,43 @@ func (u *syncStatusUpdater) UpdateDetails(objectId string, status domain.ObjectS } } +func (u *syncStatusUpdater) addToQueue(details *syncStatusDetails) error { + u.mx.Lock() + u.entries[details.objectId] = details + u.mx.Unlock() + return u.batcher.TryAdd(details.objectId) +} + func (u *syncStatusUpdater) UpdateSpaceDetails(existing, missing []string, spaceId string) { if spaceId == u.spaceService.TechSpaceId() { return } u.spaceSyncStatus.UpdateMissingIds(spaceId, missing) - err := u.batcher.TryAdd(&syncStatusDetails{ - markAllSyncedExcept: existing, - status: domain.ObjectSyncing, - spaceId: spaceId, - }) - fmt.Println("[x]: sending update to batcher, len(existing)", len(existing), "len(missing)", len(missing), "spaceId", spaceId) - if err != nil { - log.Errorf("failed to add sync details update to queue: %s", err) + ids := u.getSyncingObjects(spaceId) + removed, added := slice.DifferenceRemovedAdded(existing, ids) + if len(removed)+len(added) == 0 { + u.spaceSyncStatus.Refresh(spaceId) + return + } + for _, id := range added { + err := u.addToQueue(&syncStatusDetails{ + objectId: id, + status: domain.ObjectSynced, + spaceId: spaceId, + }) + if err != nil { + log.Errorf("failed to add sync details update to queue: %s", err) + } + } + for _, id := range removed { + err := u.addToQueue(&syncStatusDetails{ + objectId: id, + status: domain.ObjectSyncing, + spaceId: spaceId, + }) + if err != nil { + log.Errorf("failed to add sync details update to queue: %s", err) + } } } @@ -253,50 +268,23 @@ func (u *syncStatusUpdater) setSyncDetails(sb smartblock.SmartBlock, status doma func (u *syncStatusUpdater) processEvents() { defer close(u.finish) - updateSpecificObject := func(details *syncStatusDetails) { + updateSpecificObject := func(objectId string) { u.mx.Lock() - objectStatus := u.entries[details.objectId] - delete(u.entries, details.objectId) + objectStatus := u.entries[objectId] + delete(u.entries, objectId) u.mx.Unlock() if objectStatus != nil { - err := u.updateObjectDetails(objectStatus, details.objectId) - if err != nil { - log.Errorf("failed to update details %s", err) - } - } - } - syncAllObjectsExcept := func(details *syncStatusDetails) { - ids := u.getSyncingObjects(details.spaceId) - removed, added := slice.DifferenceRemovedAdded(details.markAllSyncedExcept, ids) - if len(removed)+len(added) == 0 { - u.spaceSyncStatus.Refresh(details.spaceId) - return - } - fmt.Println("[x]: marking synced, len(synced)", len(added), "len(syncing)", len(removed), "spaceId", details.spaceId) - details.status = domain.ObjectSynced - for _, id := range added { - err := u.updateObjectDetails(details, id) - if err != nil { - log.Errorf("failed to update details %s", err) - } - } - details.status = domain.ObjectSyncing - for _, id := range removed { - err := u.updateObjectDetails(details, id) + err := u.updateObjectDetails(objectStatus, objectId) if err != nil { log.Errorf("failed to update details %s", err) } } } for { - status, err := u.batcher.WaitOne(u.ctx) + objectId, err := u.batcher.WaitOne(u.ctx) if err != nil { return } - if status.objectId == "" { - syncAllObjectsExcept(status) - } else { - updateSpecificObject(status) - } + updateSpecificObject(objectId) } } diff --git a/core/syncstatus/spacesyncstatus/filestate.go b/core/syncstatus/spacesyncstatus/filestate.go deleted file mode 100644 index d972e0502..000000000 --- a/core/syncstatus/spacesyncstatus/filestate.go +++ /dev/null @@ -1,105 +0,0 @@ -package spacesyncstatus - -import ( - "sync" - - "github.com/anyproto/anytype-heart/core/domain" - "github.com/anyproto/anytype-heart/core/syncstatus/filesyncstatus" - "github.com/anyproto/anytype-heart/pkg/lib/bundle" - "github.com/anyproto/anytype-heart/pkg/lib/database" - "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" - "github.com/anyproto/anytype-heart/pkg/lib/pb/model" - "github.com/anyproto/anytype-heart/util/pbtypes" -) - -type FileState struct { - fileSyncCountBySpace map[string]int - fileSyncStatusBySpace map[string]domain.SpaceSyncStatus - filesErrorBySpace map[string]domain.SyncError - sync.Mutex - - store objectstore.ObjectStore -} - -func NewFileState(store objectstore.ObjectStore) *FileState { - return &FileState{ - fileSyncCountBySpace: make(map[string]int, 0), - fileSyncStatusBySpace: make(map[string]domain.SpaceSyncStatus, 0), - filesErrorBySpace: make(map[string]domain.SyncError, 0), - - store: store, - } -} - -func (f *FileState) SetObjectsNumber(status *domain.SpaceSync) { - f.Lock() - defer f.Unlock() - switch status.Status { - case domain.Error, domain.Offline: - f.fileSyncCountBySpace[status.SpaceId] = 0 - default: - records, err := f.store.Query(database.Query{ - Filters: []*model.BlockContentDataviewFilter{ - { - RelationKey: bundle.RelationKeyFileBackupStatus.String(), - Condition: model.BlockContentDataviewFilter_In, - Value: pbtypes.IntList(int(filesyncstatus.Syncing), int(filesyncstatus.Queued)), - }, - { - RelationKey: bundle.RelationKeySpaceId.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.String(status.SpaceId), - }, - }, - }) - if err != nil { - log.Errorf("failed to query file status: %s", err) - } - f.fileSyncCountBySpace[status.SpaceId] = len(records) - } -} - -func (f *FileState) SetSyncStatusAndErr(status domain.SpaceSyncStatus, syncErr domain.SyncError, spaceId string) { - f.Lock() - defer f.Unlock() - switch status { - case domain.Synced: - f.fileSyncStatusBySpace[spaceId] = domain.Synced - f.filesErrorBySpace[spaceId] = syncErr - if number := f.fileSyncCountBySpace[spaceId]; number > 0 { - f.fileSyncStatusBySpace[spaceId] = domain.Syncing - return - } - case domain.Error, domain.Syncing, domain.Offline: - f.fileSyncStatusBySpace[spaceId] = status - f.filesErrorBySpace[spaceId] = syncErr - } -} - -func (f *FileState) GetSyncStatus(spaceId string) domain.SpaceSyncStatus { - f.Lock() - defer f.Unlock() - if status, ok := f.fileSyncStatusBySpace[spaceId]; ok { - return status - } - return domain.Unknown -} - -func (f *FileState) GetSyncObjectCount(spaceId string) int { - f.Lock() - defer f.Unlock() - return f.fileSyncCountBySpace[spaceId] -} - -func (f *FileState) ResetSpaceErrorStatus(spaceId string, syncError domain.SyncError) { - // show StorageLimitExceed only once - if syncError == domain.StorageLimitExceed { - f.SetSyncStatusAndErr(domain.Synced, domain.Null, spaceId) - } -} - -func (f *FileState) GetSyncErr(spaceId string) domain.SyncError { - f.Lock() - defer f.Unlock() - return f.filesErrorBySpace[spaceId] -} diff --git a/core/syncstatus/spacesyncstatus/filestate_test.go b/core/syncstatus/spacesyncstatus/filestate_test.go deleted file mode 100644 index 249d38afb..000000000 --- a/core/syncstatus/spacesyncstatus/filestate_test.go +++ /dev/null @@ -1,161 +0,0 @@ -package spacesyncstatus - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/anyproto/anytype-heart/core/domain" - "github.com/anyproto/anytype-heart/core/syncstatus/filesyncstatus" - "github.com/anyproto/anytype-heart/pkg/lib/bundle" - "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" - "github.com/anyproto/anytype-heart/util/pbtypes" -) - -func TestFileState_GetSyncObjectCount(t *testing.T) { - t.Run("GetSyncObjectCount", func(t *testing.T) { - // given - fileState := NewFileState(nil) - - // when - fileState.fileSyncCountBySpace["spaceId"] = 1 - objectCount := fileState.GetSyncObjectCount("spaceId") - - // then - assert.Equal(t, 1, objectCount) - }) - t.Run("GetSyncObjectCount: zero value", func(t *testing.T) { - // given - fileState := NewFileState(nil) - - // when - objectCount := fileState.GetSyncObjectCount("spaceId") - - // then - assert.Equal(t, 0, objectCount) - }) -} - -func TestFileState_GetSyncStatus(t *testing.T) { - t.Run("GetSyncStatus", func(t *testing.T) { - // given - fileState := NewFileState(nil) - - // when - fileState.fileSyncStatusBySpace["spaceId"] = domain.Syncing - syncStatus := fileState.GetSyncStatus("spaceId") - - // then - assert.Equal(t, domain.Syncing, syncStatus) - }) - t.Run("GetSyncStatus: zero value", func(t *testing.T) { - // given - fileState := NewFileState(nil) - - // when - syncStatus := fileState.GetSyncStatus("spaceId") - - // then - assert.Equal(t, domain.Unknown, syncStatus) - }) -} - -func TestFileState_SetObjectsNumber(t *testing.T) { - t.Run("SetObjectsNumber", func(t *testing.T) { - // given - storeFixture := objectstore.NewStoreFixture(t) - storeFixture.AddObjects(t, []objectstore.TestObject{ - { - bundle.RelationKeyId: pbtypes.String("id1"), - bundle.RelationKeyFileBackupStatus: pbtypes.Int64(int64(filesyncstatus.Syncing)), - bundle.RelationKeySpaceId: pbtypes.String("spaceId"), - }, - { - bundle.RelationKeyId: pbtypes.String("id2"), - bundle.RelationKeyFileBackupStatus: pbtypes.Int64(int64(filesyncstatus.Synced)), - bundle.RelationKeySpaceId: pbtypes.String("spaceId"), - }, - { - bundle.RelationKeyId: pbtypes.String("id3"), - bundle.RelationKeyFileBackupStatus: pbtypes.Int64(int64(filesyncstatus.Syncing)), - bundle.RelationKeySpaceId: pbtypes.String("spaceId"), - }, - }) - fileState := NewFileState(storeFixture) - syncStatus := domain.MakeSyncStatus("spaceId", domain.Syncing, domain.Null, domain.Files) - - // when - fileState.SetObjectsNumber(syncStatus) - - // then - assert.Equal(t, 2, fileState.GetSyncObjectCount("spaceId")) - }) - t.Run("SetObjectsNumber: no file object", func(t *testing.T) { - // given - storeFixture := objectstore.NewStoreFixture(t) - fileState := NewFileState(storeFixture) - syncStatus := domain.MakeSyncStatus("spaceId", domain.Synced, domain.Null, domain.Files) - - // when - fileState.SetObjectsNumber(syncStatus) - - // then - assert.Equal(t, 0, fileState.GetSyncObjectCount("spaceId")) - }) -} - -func TestFileState_SetSyncStatus(t *testing.T) { - t.Run("SetSyncStatusAndErr, status synced", func(t *testing.T) { - // given - fileState := NewFileState(objectstore.NewStoreFixture(t)) - - // when - fileState.SetSyncStatusAndErr(domain.Synced, domain.Null, "spaceId") - - // then - assert.Equal(t, domain.Synced, fileState.GetSyncStatus("spaceId")) - }) - t.Run("SetSyncStatusAndErr, sync in progress", func(t *testing.T) { - // given - fileState := NewFileState(objectstore.NewStoreFixture(t)) - - // when - syncStatus := domain.MakeSyncStatus("spaceId", domain.Syncing, domain.Null, domain.Files) - fileState.SetSyncStatusAndErr(syncStatus.Status, domain.Null, syncStatus.SpaceId) - - // then - assert.Equal(t, domain.Syncing, fileState.GetSyncStatus("spaceId")) - }) - t.Run("SetSyncStatusAndErr, sync is finished with error", func(t *testing.T) { - // given - fileState := NewFileState(objectstore.NewStoreFixture(t)) - - // when - syncStatus := domain.MakeSyncStatus("spaceId", domain.Error, domain.Null, domain.Files) - fileState.SetSyncStatusAndErr(syncStatus.Status, domain.Null, syncStatus.SpaceId) - - // then - assert.Equal(t, domain.Error, fileState.GetSyncStatus("spaceId")) - }) - t.Run("SetSyncStatusAndErr, offline", func(t *testing.T) { - // given - fileState := NewFileState(objectstore.NewStoreFixture(t)) - - // when - fileState.SetSyncStatusAndErr(domain.Offline, domain.Null, "spaceId") - - // then - assert.Equal(t, domain.Offline, fileState.GetSyncStatus("spaceId")) - }) - t.Run("SetSyncStatusAndErr, syncing status", func(t *testing.T) { - // given - fileState := NewFileState(objectstore.NewStoreFixture(t)) - - // when - fileState.fileSyncCountBySpace["spaceId"] = 1 - fileState.SetSyncStatusAndErr(domain.Synced, domain.Null, "spaceId") - - // then - assert.Equal(t, domain.Syncing, fileState.GetSyncStatus("spaceId")) - }) -} diff --git a/core/syncstatus/spacesyncstatus/objectstate.go b/core/syncstatus/spacesyncstatus/objectstate.go deleted file mode 100644 index 9d9fd3907..000000000 --- a/core/syncstatus/spacesyncstatus/objectstate.go +++ /dev/null @@ -1,121 +0,0 @@ -package spacesyncstatus - -import ( - "fmt" - "sync" - - "github.com/samber/lo" - - "github.com/anyproto/anytype-heart/core/domain" - "github.com/anyproto/anytype-heart/pkg/lib/bundle" - "github.com/anyproto/anytype-heart/pkg/lib/database" - "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" - "github.com/anyproto/anytype-heart/pkg/lib/pb/model" - "github.com/anyproto/anytype-heart/util/pbtypes" - "github.com/anyproto/anytype-heart/util/slice" -) - -type ObjectState struct { - objectSyncStatusBySpace map[string]domain.SpaceSyncStatus - objectSyncCountBySpace map[string]int - objectSyncErrBySpace map[string]domain.SyncError - sync.Mutex - - store objectstore.ObjectStore -} - -func NewObjectState(store objectstore.ObjectStore) *ObjectState { - return &ObjectState{ - objectSyncCountBySpace: make(map[string]int, 0), - objectSyncStatusBySpace: make(map[string]domain.SpaceSyncStatus, 0), - objectSyncErrBySpace: make(map[string]domain.SyncError, 0), - store: store, - } -} - -func (o *ObjectState) SetObjectsNumber(status *domain.SpaceSync) { - o.Lock() - defer o.Unlock() - switch status.Status { - case domain.Error, domain.Offline: - o.objectSyncCountBySpace[status.SpaceId] = 0 - default: - records := o.getSyncingObjects(status) - ids := lo.Map(records, func(r database.Record, idx int) string { - return pbtypes.GetString(r.Details, bundle.RelationKeyId.String()) - }) - _, added := slice.DifferenceRemovedAdded(ids, status.MissingObjects) - fmt.Println("[x]: added", len(added), "records", len(records)) - o.objectSyncCountBySpace[status.SpaceId] = len(records) + len(added) - } -} - -func (o *ObjectState) getSyncingObjects(status *domain.SpaceSync) []database.Record { - records, err := o.store.Query(database.Query{ - Filters: []*model.BlockContentDataviewFilter{ - { - RelationKey: bundle.RelationKeySyncStatus.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.Int64(int64(domain.Syncing)), - }, - { - RelationKey: bundle.RelationKeyLayout.String(), - Condition: model.BlockContentDataviewFilter_NotIn, - Value: pbtypes.IntList( - int(model.ObjectType_file), - int(model.ObjectType_image), - int(model.ObjectType_video), - int(model.ObjectType_audio), - ), - }, - { - RelationKey: bundle.RelationKeySpaceId.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.String(status.SpaceId), - }, - }, - }) - if err != nil { - log.Errorf("failed to query file status: %s", err) - } - return records -} - -func (o *ObjectState) SetSyncStatusAndErr(status domain.SpaceSyncStatus, syncErr domain.SyncError, spaceId string) { - o.Lock() - defer o.Unlock() - if objectNumber, ok := o.objectSyncCountBySpace[spaceId]; ok && objectNumber > 0 { - o.objectSyncStatusBySpace[spaceId] = domain.Syncing - o.objectSyncErrBySpace[spaceId] = domain.Null - return - } else if ok && objectNumber == 0 && status == domain.Syncing { - o.objectSyncStatusBySpace[spaceId] = domain.Synced - o.objectSyncErrBySpace[spaceId] = domain.Null - return - } - o.objectSyncStatusBySpace[spaceId] = status - o.objectSyncErrBySpace[spaceId] = syncErr -} - -func (o *ObjectState) GetSyncStatus(spaceId string) domain.SpaceSyncStatus { - o.Lock() - defer o.Unlock() - if status, ok := o.objectSyncStatusBySpace[spaceId]; ok { - return status - } - return domain.Unknown -} - -func (o *ObjectState) GetSyncObjectCount(spaceId string) int { - o.Lock() - defer o.Unlock() - return o.objectSyncCountBySpace[spaceId] -} - -func (o *ObjectState) GetSyncErr(spaceId string) domain.SyncError { - o.Lock() - defer o.Unlock() - return o.objectSyncErrBySpace[spaceId] -} - -func (o *ObjectState) ResetSpaceErrorStatus(string, domain.SyncError) {} diff --git a/core/syncstatus/spacesyncstatus/objectstate_test.go b/core/syncstatus/spacesyncstatus/objectstate_test.go deleted file mode 100644 index 23e416d98..000000000 --- a/core/syncstatus/spacesyncstatus/objectstate_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package spacesyncstatus - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/anyproto/anytype-heart/core/domain" - "github.com/anyproto/anytype-heart/pkg/lib/bundle" - "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" - "github.com/anyproto/anytype-heart/util/pbtypes" -) - -func TestObjectState_GetSyncObjectCount(t *testing.T) { - t.Run("GetSyncObjectCount", func(t *testing.T) { - // given - objectState := NewObjectState(objectstore.NewStoreFixture(t)) - - // when - objectState.objectSyncCountBySpace["spaceId"] = 1 - objectCount := objectState.GetSyncObjectCount("spaceId") - - // then - assert.Equal(t, 1, objectCount) - }) - t.Run("GetSyncObjectCount: zero value", func(t *testing.T) { - // given - objectState := NewObjectState(objectstore.NewStoreFixture(t)) - - // when - objectCount := objectState.GetSyncObjectCount("spaceId") - - // then - assert.Equal(t, 0, objectCount) - }) -} - -func TestObjectState_GetSyncStatus(t *testing.T) { - t.Run("GetSyncStatus", func(t *testing.T) { - // given - objectState := NewObjectState(objectstore.NewStoreFixture(t)) - - // when - objectState.objectSyncStatusBySpace["spaceId"] = domain.Syncing - syncStatus := objectState.GetSyncStatus("spaceId") - - // then - assert.Equal(t, domain.Syncing, syncStatus) - }) - t.Run("GetSyncStatus: zero value", func(t *testing.T) { - // given - objectState := NewObjectState(objectstore.NewStoreFixture(t)) - - // when - syncStatus := objectState.GetSyncStatus("spaceId") - - // then - assert.Equal(t, domain.Unknown, syncStatus) - }) -} - -func TestObjectState_SetObjectsNumber(t *testing.T) { - t.Run("SetObjectsNumber", func(t *testing.T) { - // given - storeFixture := objectstore.NewStoreFixture(t) - objectState := NewObjectState(storeFixture) - syncStatus := domain.MakeSyncStatus("spaceId", domain.Syncing, domain.Null, domain.Objects) - storeFixture.AddObjects(t, []objectstore.TestObject{ - { - bundle.RelationKeyId: pbtypes.String("id1"), - bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.Syncing)), - bundle.RelationKeySpaceId: pbtypes.String("spaceId"), - }, - { - bundle.RelationKeyId: pbtypes.String("id2"), - bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.Syncing)), - bundle.RelationKeySpaceId: pbtypes.String("spaceId"), - }, - }) - - // when - objectState.SetObjectsNumber(syncStatus) - - // then - assert.Equal(t, 2, objectState.GetSyncObjectCount("spaceId")) - }) - t.Run("SetObjectsNumber: no object", func(t *testing.T) { - // given - objectState := NewObjectState(objectstore.NewStoreFixture(t)) - syncStatus := domain.MakeSyncStatus("spaceId", domain.Synced, domain.Null, domain.Objects) - - // when - objectState.SetObjectsNumber(syncStatus) - - // then - assert.Equal(t, 0, objectState.GetSyncObjectCount("spaceId")) - }) -} - -func TestObjectState_SetSyncStatus(t *testing.T) { - t.Run("SetSyncStatusAndErr, status synced", func(t *testing.T) { - // given - objectState := NewObjectState(objectstore.NewStoreFixture(t)) - - // when - syncStatus := domain.MakeSyncStatus("spaceId", domain.Synced, domain.Null, domain.Objects) - objectState.SetSyncStatusAndErr(syncStatus.Status, domain.Null, syncStatus.SpaceId) - - // then - assert.Equal(t, domain.Synced, objectState.GetSyncStatus("spaceId")) - }) - t.Run("SetSyncStatusAndErr, sync in progress", func(t *testing.T) { - // given - objectState := NewObjectState(objectstore.NewStoreFixture(t)) - - // when - syncStatus := domain.MakeSyncStatus("spaceId", domain.Syncing, domain.Null, domain.Objects) - objectState.SetSyncStatusAndErr(syncStatus.Status, domain.Null, syncStatus.SpaceId) - - // then - assert.Equal(t, domain.Syncing, objectState.GetSyncStatus("spaceId")) - }) - t.Run("SetSyncStatusAndErr, sync is finished with error", func(t *testing.T) { - // given - objectState := NewObjectState(objectstore.NewStoreFixture(t)) - - // when - syncStatus := domain.MakeSyncStatus("spaceId", domain.Error, domain.Null, domain.Objects) - objectState.SetSyncStatusAndErr(syncStatus.Status, domain.Null, syncStatus.SpaceId) - - // then - assert.Equal(t, domain.Error, objectState.GetSyncStatus("spaceId")) - }) - t.Run("SetSyncStatusAndErr, offline", func(t *testing.T) { - // given - objectState := NewObjectState(objectstore.NewStoreFixture(t)) - - // when - syncStatus := domain.MakeSyncStatus("spaceId", domain.Offline, domain.Null, domain.Objects) - objectState.SetSyncStatusAndErr(syncStatus.Status, domain.Null, syncStatus.SpaceId) - - // then - assert.Equal(t, domain.Offline, objectState.GetSyncStatus("spaceId")) - }) - t.Run("SetSyncStatusAndErr, syncing", func(t *testing.T) { - // given - objectState := NewObjectState(objectstore.NewStoreFixture(t)) - - // when - syncStatus := domain.MakeSyncStatus("spaceId", domain.Syncing, domain.Null, domain.Objects) - objectState.SetObjectsNumber(syncStatus) - objectState.SetSyncStatusAndErr(syncStatus.Status, domain.Null, syncStatus.SpaceId) - - // then - assert.Equal(t, domain.Synced, objectState.GetSyncStatus("spaceId")) - }) -} From 40a24efe0eacbdff5c6a81cd14873848b46059bc Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Tue, 23 Jul 2024 22:10:55 +0200 Subject: [PATCH 26/71] GO-3769 Test object sync status --- .../mock_SyncDetailsUpdater.go | 30 +- .../mock_SpaceStatusUpdater.go | 62 +++- .../mock_objectsyncstatus/mock_Updater.go | 19 +- .../syncstatus/objectsyncstatus/syncstatus.go | 7 +- .../objectsyncstatus/syncstatus_test.go | 318 +++++------------- .../mock_spacesyncstatus/mock_Updater.go | 62 +++- .../mock_peermanager/mock_Updater.go | 68 ++-- 7 files changed, 237 insertions(+), 329 deletions(-) diff --git a/core/block/object/treesyncer/mock_treesyncer/mock_SyncDetailsUpdater.go b/core/block/object/treesyncer/mock_treesyncer/mock_SyncDetailsUpdater.go index e17ba1fcf..3e9b0805f 100644 --- a/core/block/object/treesyncer/mock_treesyncer/mock_SyncDetailsUpdater.go +++ b/core/block/object/treesyncer/mock_treesyncer/mock_SyncDetailsUpdater.go @@ -4,7 +4,6 @@ package mock_treesyncer import ( app "github.com/anyproto/any-sync/app" - domain "github.com/anyproto/anytype-heart/core/domain" mock "github.com/stretchr/testify/mock" ) @@ -112,38 +111,37 @@ func (_c *MockSyncDetailsUpdater_Name_Call) RunAndReturn(run func() string) *Moc return _c } -// UpdateDetails provides a mock function with given fields: objectId, status, syncError, spaceId -func (_m *MockSyncDetailsUpdater) UpdateDetails(objectId []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) { - _m.Called(objectId, status, syncError, spaceId) +// UpdateSpaceDetails provides a mock function with given fields: existing, missing, spaceId +func (_m *MockSyncDetailsUpdater) UpdateSpaceDetails(existing []string, missing []string, spaceId string) { + _m.Called(existing, missing, spaceId) } -// MockSyncDetailsUpdater_UpdateDetails_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateDetails' -type MockSyncDetailsUpdater_UpdateDetails_Call struct { +// MockSyncDetailsUpdater_UpdateSpaceDetails_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateSpaceDetails' +type MockSyncDetailsUpdater_UpdateSpaceDetails_Call struct { *mock.Call } -// UpdateDetails is a helper method to define mock.On call -// - objectId []string -// - status domain.ObjectSyncStatus -// - syncError domain.SyncError +// UpdateSpaceDetails is a helper method to define mock.On call +// - existing []string +// - missing []string // - spaceId string -func (_e *MockSyncDetailsUpdater_Expecter) UpdateDetails(objectId interface{}, status interface{}, syncError interface{}, spaceId interface{}) *MockSyncDetailsUpdater_UpdateDetails_Call { - return &MockSyncDetailsUpdater_UpdateDetails_Call{Call: _e.mock.On("UpdateDetails", objectId, status, syncError, spaceId)} +func (_e *MockSyncDetailsUpdater_Expecter) UpdateSpaceDetails(existing interface{}, missing interface{}, spaceId interface{}) *MockSyncDetailsUpdater_UpdateSpaceDetails_Call { + return &MockSyncDetailsUpdater_UpdateSpaceDetails_Call{Call: _e.mock.On("UpdateSpaceDetails", existing, missing, spaceId)} } -func (_c *MockSyncDetailsUpdater_UpdateDetails_Call) Run(run func(objectId []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string)) *MockSyncDetailsUpdater_UpdateDetails_Call { +func (_c *MockSyncDetailsUpdater_UpdateSpaceDetails_Call) Run(run func(existing []string, missing []string, spaceId string)) *MockSyncDetailsUpdater_UpdateSpaceDetails_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].([]string), args[1].(domain.ObjectSyncStatus), args[2].(domain.SyncError), args[3].(string)) + run(args[0].([]string), args[1].([]string), args[2].(string)) }) return _c } -func (_c *MockSyncDetailsUpdater_UpdateDetails_Call) Return() *MockSyncDetailsUpdater_UpdateDetails_Call { +func (_c *MockSyncDetailsUpdater_UpdateSpaceDetails_Call) Return() *MockSyncDetailsUpdater_UpdateSpaceDetails_Call { _c.Call.Return() return _c } -func (_c *MockSyncDetailsUpdater_UpdateDetails_Call) RunAndReturn(run func([]string, domain.ObjectSyncStatus, domain.SyncError, string)) *MockSyncDetailsUpdater_UpdateDetails_Call { +func (_c *MockSyncDetailsUpdater_UpdateSpaceDetails_Call) RunAndReturn(run func([]string, []string, string)) *MockSyncDetailsUpdater_UpdateSpaceDetails_Call { _c.Call.Return(run) return _c } diff --git a/core/syncstatus/detailsupdater/mock_detailsupdater/mock_SpaceStatusUpdater.go b/core/syncstatus/detailsupdater/mock_detailsupdater/mock_SpaceStatusUpdater.go index 2db307c1b..9bdcc3a84 100644 --- a/core/syncstatus/detailsupdater/mock_detailsupdater/mock_SpaceStatusUpdater.go +++ b/core/syncstatus/detailsupdater/mock_detailsupdater/mock_SpaceStatusUpdater.go @@ -5,8 +5,6 @@ package mock_detailsupdater import ( app "github.com/anyproto/any-sync/app" - domain "github.com/anyproto/anytype-heart/core/domain" - mock "github.com/stretchr/testify/mock" ) @@ -114,35 +112,69 @@ func (_c *MockSpaceStatusUpdater_Name_Call) RunAndReturn(run func() string) *Moc return _c } -// SendUpdate provides a mock function with given fields: status -func (_m *MockSpaceStatusUpdater) SendUpdate(status *domain.SpaceSync) { - _m.Called(status) +// Refresh provides a mock function with given fields: spaceId +func (_m *MockSpaceStatusUpdater) Refresh(spaceId string) { + _m.Called(spaceId) } -// MockSpaceStatusUpdater_SendUpdate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SendUpdate' -type MockSpaceStatusUpdater_SendUpdate_Call struct { +// MockSpaceStatusUpdater_Refresh_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Refresh' +type MockSpaceStatusUpdater_Refresh_Call struct { *mock.Call } -// SendUpdate is a helper method to define mock.On call -// - status *domain.SpaceSync -func (_e *MockSpaceStatusUpdater_Expecter) SendUpdate(status interface{}) *MockSpaceStatusUpdater_SendUpdate_Call { - return &MockSpaceStatusUpdater_SendUpdate_Call{Call: _e.mock.On("SendUpdate", status)} +// Refresh is a helper method to define mock.On call +// - spaceId string +func (_e *MockSpaceStatusUpdater_Expecter) Refresh(spaceId interface{}) *MockSpaceStatusUpdater_Refresh_Call { + return &MockSpaceStatusUpdater_Refresh_Call{Call: _e.mock.On("Refresh", spaceId)} } -func (_c *MockSpaceStatusUpdater_SendUpdate_Call) Run(run func(status *domain.SpaceSync)) *MockSpaceStatusUpdater_SendUpdate_Call { +func (_c *MockSpaceStatusUpdater_Refresh_Call) Run(run func(spaceId string)) *MockSpaceStatusUpdater_Refresh_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(*domain.SpaceSync)) + run(args[0].(string)) }) return _c } -func (_c *MockSpaceStatusUpdater_SendUpdate_Call) Return() *MockSpaceStatusUpdater_SendUpdate_Call { +func (_c *MockSpaceStatusUpdater_Refresh_Call) Return() *MockSpaceStatusUpdater_Refresh_Call { _c.Call.Return() return _c } -func (_c *MockSpaceStatusUpdater_SendUpdate_Call) RunAndReturn(run func(*domain.SpaceSync)) *MockSpaceStatusUpdater_SendUpdate_Call { +func (_c *MockSpaceStatusUpdater_Refresh_Call) RunAndReturn(run func(string)) *MockSpaceStatusUpdater_Refresh_Call { + _c.Call.Return(run) + return _c +} + +// UpdateMissingIds provides a mock function with given fields: spaceId, ids +func (_m *MockSpaceStatusUpdater) UpdateMissingIds(spaceId string, ids []string) { + _m.Called(spaceId, ids) +} + +// MockSpaceStatusUpdater_UpdateMissingIds_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateMissingIds' +type MockSpaceStatusUpdater_UpdateMissingIds_Call struct { + *mock.Call +} + +// UpdateMissingIds is a helper method to define mock.On call +// - spaceId string +// - ids []string +func (_e *MockSpaceStatusUpdater_Expecter) UpdateMissingIds(spaceId interface{}, ids interface{}) *MockSpaceStatusUpdater_UpdateMissingIds_Call { + return &MockSpaceStatusUpdater_UpdateMissingIds_Call{Call: _e.mock.On("UpdateMissingIds", spaceId, ids)} +} + +func (_c *MockSpaceStatusUpdater_UpdateMissingIds_Call) Run(run func(spaceId string, ids []string)) *MockSpaceStatusUpdater_UpdateMissingIds_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].([]string)) + }) + return _c +} + +func (_c *MockSpaceStatusUpdater_UpdateMissingIds_Call) Return() *MockSpaceStatusUpdater_UpdateMissingIds_Call { + _c.Call.Return() + return _c +} + +func (_c *MockSpaceStatusUpdater_UpdateMissingIds_Call) RunAndReturn(run func(string, []string)) *MockSpaceStatusUpdater_UpdateMissingIds_Call { _c.Call.Return(run) return _c } diff --git a/core/syncstatus/objectsyncstatus/mock_objectsyncstatus/mock_Updater.go b/core/syncstatus/objectsyncstatus/mock_objectsyncstatus/mock_Updater.go index 23b422e97..86a0111cb 100644 --- a/core/syncstatus/objectsyncstatus/mock_objectsyncstatus/mock_Updater.go +++ b/core/syncstatus/objectsyncstatus/mock_objectsyncstatus/mock_Updater.go @@ -112,9 +112,9 @@ func (_c *MockUpdater_Name_Call) RunAndReturn(run func() string) *MockUpdater_Na return _c } -// UpdateDetails provides a mock function with given fields: objectId, status, syncError, spaceId -func (_m *MockUpdater) UpdateDetails(objectId []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string) { - _m.Called(objectId, status, syncError, spaceId) +// UpdateDetails provides a mock function with given fields: objectId, status, spaceId +func (_m *MockUpdater) UpdateDetails(objectId string, status domain.ObjectSyncStatus, spaceId string) { + _m.Called(objectId, status, spaceId) } // MockUpdater_UpdateDetails_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateDetails' @@ -123,17 +123,16 @@ type MockUpdater_UpdateDetails_Call struct { } // UpdateDetails is a helper method to define mock.On call -// - objectId []string +// - objectId string // - status domain.ObjectSyncStatus -// - syncError domain.SyncError // - spaceId string -func (_e *MockUpdater_Expecter) UpdateDetails(objectId interface{}, status interface{}, syncError interface{}, spaceId interface{}) *MockUpdater_UpdateDetails_Call { - return &MockUpdater_UpdateDetails_Call{Call: _e.mock.On("UpdateDetails", objectId, status, syncError, spaceId)} +func (_e *MockUpdater_Expecter) UpdateDetails(objectId interface{}, status interface{}, spaceId interface{}) *MockUpdater_UpdateDetails_Call { + return &MockUpdater_UpdateDetails_Call{Call: _e.mock.On("UpdateDetails", objectId, status, spaceId)} } -func (_c *MockUpdater_UpdateDetails_Call) Run(run func(objectId []string, status domain.ObjectSyncStatus, syncError domain.SyncError, spaceId string)) *MockUpdater_UpdateDetails_Call { +func (_c *MockUpdater_UpdateDetails_Call) Run(run func(objectId string, status domain.ObjectSyncStatus, spaceId string)) *MockUpdater_UpdateDetails_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].([]string), args[1].(domain.ObjectSyncStatus), args[2].(domain.SyncError), args[3].(string)) + run(args[0].(string), args[1].(domain.ObjectSyncStatus), args[2].(string)) }) return _c } @@ -143,7 +142,7 @@ func (_c *MockUpdater_UpdateDetails_Call) Return() *MockUpdater_UpdateDetails_Ca return _c } -func (_c *MockUpdater_UpdateDetails_Call) RunAndReturn(run func([]string, domain.ObjectSyncStatus, domain.SyncError, string)) *MockUpdater_UpdateDetails_Call { +func (_c *MockUpdater_UpdateDetails_Call) RunAndReturn(run func(string, domain.ObjectSyncStatus, string)) *MockUpdater_UpdateDetails_Call { _c.Call.Return(run) return _c } diff --git a/core/syncstatus/objectsyncstatus/syncstatus.go b/core/syncstatus/objectsyncstatus/syncstatus.go index d10c25cd2..125a8fe77 100644 --- a/core/syncstatus/objectsyncstatus/syncstatus.go +++ b/core/syncstatus/objectsyncstatus/syncstatus.go @@ -79,7 +79,6 @@ type Updater interface { type syncStatusService struct { sync.Mutex - configuration nodeconf.NodeConf periodicSync periodicsync.PeriodicSync updateReceiver UpdateReceiver storage spacestorage.SpaceStorage @@ -112,7 +111,6 @@ func (s *syncStatusService) Init(a *app.App) (err error) { s.updateIntervalSecs = syncUpdateInterval s.updateTimeout = syncTimeout s.spaceId = sharedState.SpaceId - s.configuration = app.MustComponent[nodeconf.NodeConf](a) s.storage = app.MustComponent[spacestorage.SpaceStorage](a) s.periodicSync = periodicsync.NewPeriodicSync( s.updateIntervalSecs, @@ -143,6 +141,9 @@ func (s *syncStatusService) Run(ctx context.Context) error { } func (s *syncStatusService) HeadsChange(treeId string, heads []string) { + s.Lock() + s.treeHeads[treeId] = treeHeadsEntry{heads: heads, syncStatus: StatusNotSynced} + s.Unlock() s.updateDetails(treeId, domain.ObjectSyncing) } @@ -295,7 +296,7 @@ func (s *syncStatusService) Close(ctx context.Context) error { } func (s *syncStatusService) isSenderResponsible(senderId string) bool { - return slices.Contains(s.configuration.NodeIds(s.spaceId), senderId) + return slices.Contains(s.nodeConfService.NodeIds(s.spaceId), senderId) } func (s *syncStatusService) updateDetails(treeId string, status domain.ObjectSyncStatus) { diff --git a/core/syncstatus/objectsyncstatus/syncstatus_test.go b/core/syncstatus/objectsyncstatus/syncstatus_test.go index f6d6869b1..49ddbcca7 100644 --- a/core/syncstatus/objectsyncstatus/syncstatus_test.go +++ b/core/syncstatus/objectsyncstatus/syncstatus_test.go @@ -5,257 +5,141 @@ import ( "testing" "github.com/anyproto/any-sync/app" - "github.com/anyproto/any-sync/commonspace/object/accountdata" - "github.com/anyproto/any-sync/commonspace/object/acl/list" - "github.com/anyproto/any-sync/commonspace/object/tree/objecttree" + "github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto" "github.com/anyproto/any-sync/commonspace/object/tree/treestorage" "github.com/anyproto/any-sync/commonspace/spacestate" "github.com/anyproto/any-sync/commonspace/spacestorage/mock_spacestorage" - "github.com/anyproto/any-sync/nodeconf" "github.com/anyproto/any-sync/nodeconf/mock_nodeconf" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" "github.com/anyproto/anytype-heart/core/anytype/config" "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" "github.com/anyproto/anytype-heart/core/syncstatus/objectsyncstatus/mock_objectsyncstatus" - "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/tests/testutil" ) -func Test_HeadsChange(t *testing.T) { +func Test_UseCases(t *testing.T) { t.Run("HeadsChange: new object", func(t *testing.T) { - // given - s := newFixture(t) - s.service.EXPECT().NetworkCompatibilityStatus().Return(nodeconf.NetworkCompatibilityStatusOk) - s.detailsUpdater.EXPECT().UpdateDetails([]string{"id"}, domain.ObjectSyncing, domain.Null, "spaceId") + s := newFixture(t, "spaceId") + s.detailsUpdater.EXPECT().UpdateDetails("id", domain.ObjectSyncing, "spaceId") - // when s.HeadsChange("id", []string{"head1", "head2"}) - // then assert.NotNil(t, s.treeHeads["id"]) assert.Equal(t, []string{"head1", "head2"}, s.treeHeads["id"].heads) }) - t.Run("local only", func(t *testing.T) { - // given - s := newFixture(t) - s.config.NetworkMode = pb.RpcAccount_LocalOnly - s.service.EXPECT().NetworkCompatibilityStatus().Return(nodeconf.NetworkCompatibilityStatusOk) - s.detailsUpdater.EXPECT().UpdateDetails([]string{"id"}, domain.ObjectError, domain.NetworkError, "spaceId") + t.Run("HeadsChange then HeadsApply: responsible", func(t *testing.T) { + s := newFixture(t, "spaceId") + s.detailsUpdater.EXPECT().UpdateDetails("id", domain.ObjectSyncing, "spaceId") - // when s.HeadsChange("id", []string{"head1", "head2"}) - // then assert.NotNil(t, s.treeHeads["id"]) assert.Equal(t, []string{"head1", "head2"}, s.treeHeads["id"].heads) - }) - t.Run("HeadsChange: update existing object", func(t *testing.T) { - // given - s := newFixture(t) - s.config.NetworkMode = pb.RpcAccount_DefaultConfig - s.detailsUpdater.EXPECT().UpdateDetails([]string{"id"}, domain.ObjectSyncing, domain.Null, "spaceId") - s.service.EXPECT().NetworkCompatibilityStatus().Return(nodeconf.NetworkCompatibilityStatusOk).Times(2) - // when - s.HeadsChange("id", []string{"head1", "head2"}) - s.HeadsChange("id", []string{"head3"}) - - // then - assert.NotNil(t, s.treeHeads["id"]) - assert.Equal(t, []string{"head3"}, s.treeHeads["id"].heads) - }) - t.Run("HeadsChange: node offline", func(t *testing.T) { - // given - s := newFixture(t) s.service.EXPECT().NodeIds("spaceId").Return([]string{"peerId"}) - s.nodeStatus.SetNodesStatus("spaceId", "peerId", nodestatus.ConnectionError) - s.detailsUpdater.EXPECT().UpdateDetails([]string{"id"}, domain.ObjectError, domain.NetworkError, "spaceId") - s.service.EXPECT().NetworkCompatibilityStatus().Return(nodeconf.NetworkCompatibilityStatusOk).Times(2) - // when - s.HeadsChange("id", []string{"head1", "head2"}) - s.HeadsChange("id", []string{"head3"}) + s.HeadsApply("peerId", "id", []string{"head1", "head2"}, true) - // then - assert.NotNil(t, s.treeHeads["id"]) - assert.Equal(t, []string{"head3"}, s.treeHeads["id"].heads) - }) - t.Run("HeadsChange: network incompatible", func(t *testing.T) { - // given - s := newFixture(t) - s.detailsUpdater.EXPECT().UpdateDetails([]string{"id"}, domain.ObjectError, domain.IncompatibleVersion, "spaceId") - s.service.EXPECT().NetworkCompatibilityStatus().Return(nodeconf.NetworkCompatibilityStatusIncompatible).Times(1) - - // when - s.HeadsChange("id", []string{"head3"}) - - // then - assert.NotNil(t, s.treeHeads["id"]) - assert.Equal(t, []string{"head3"}, s.treeHeads["id"].heads) - }) -} - -func TestSyncStatusService_HeadsReceive(t *testing.T) { - t.Run("HeadsReceive: heads not changed ", func(t *testing.T) { - // given - s := newFixture(t) - - // when - s.HeadsReceive("peerId", "id", []string{"head1", "head2"}) - - // then - _, ok := s.treeHeads["id"] - assert.False(t, ok) - }) - t.Run("HeadsReceive: object synced", func(t *testing.T) { - // given - s := newFixture(t) - - // when - s.treeHeads["id"] = treeHeadsEntry{ - syncStatus: StatusSynced, - } - s.HeadsReceive("peerId", "id", []string{"head1", "head2"}) - - // then assert.NotNil(t, s.treeHeads["id"]) assert.Equal(t, StatusSynced, s.treeHeads["id"].syncStatus) + assert.Equal(t, s.synced, []string{"id"}) }) - t.Run("HeadsReceive: sender in not responsible", func(t *testing.T) { - // given - s := newFixture(t) - s.service.EXPECT().NodeIds(s.spaceId).Return([]string{"peerId2"}) - s.detailsUpdater.EXPECT().UpdateDetails([]string{"id"}, domain.ObjectSyncing, domain.Null, "spaceId") - s.service.EXPECT().NetworkCompatibilityStatus().Return(nodeconf.NetworkCompatibilityStatusOk) + t.Run("HeadsChange then HeadsApply: not responsible", func(t *testing.T) { + s := newFixture(t, "spaceId") + s.detailsUpdater.EXPECT().UpdateDetails("id", domain.ObjectSyncing, "spaceId") - // when - s.HeadsChange("id", []string{"head1"}) - s.HeadsReceive("peerId", "id", []string{"head2"}) + s.HeadsChange("id", []string{"head1", "head2"}) + + assert.NotNil(t, s.treeHeads["id"]) + assert.Equal(t, []string{"head1", "head2"}, s.treeHeads["id"].heads) + + s.service.EXPECT().NodeIds("spaceId").Return([]string{"peerId1"}) + + s.HeadsApply("peerId", "id", []string{"head1", "head2"}, true) - // then assert.NotNil(t, s.treeHeads["id"]) assert.Equal(t, StatusNotSynced, s.treeHeads["id"].syncStatus) + assert.Contains(t, s.tempSynced, "id") + assert.Nil(t, s.synced) }) - t.Run("HeadsReceive: object is synced", func(t *testing.T) { - // given - s := newFixture(t) - s.service.EXPECT().NodeIds(s.spaceId).Return([]string{"peerId"}) - s.service.EXPECT().NetworkCompatibilityStatus().Return(nodeconf.NetworkCompatibilityStatusOk) + t.Run("ObjectReceive: responsible", func(t *testing.T) { + s := newFixture(t, "spaceId") + s.service.EXPECT().NodeIds("spaceId").Return([]string{"peerId"}) - // when - s.detailsUpdater.EXPECT().UpdateDetails([]string{"id"}, domain.ObjectSyncing, domain.Null, "spaceId") - s.HeadsChange("id", []string{"head1"}) - s.HeadsReceive("peerId", "id", []string{"head1"}) + s.ObjectReceive("peerId", "id", []string{"head1", "head2"}) - // then - assert.NotNil(t, s.treeHeads["id"]) - assert.Equal(t, StatusSynced, s.treeHeads["id"].syncStatus) + assert.Equal(t, s.synced, []string{"id"}) + }) + t.Run("ObjectReceive: not responsible, but then sync with responsible", func(t *testing.T) { + s := newFixture(t, "spaceId") + s.service.EXPECT().NodeIds("spaceId").Return([]string{"peerId1"}) + + s.ObjectReceive("peerId", "id", []string{"head1", "head2"}) + + require.Contains(t, s.tempSynced, "id") + + s.service.EXPECT().NodeIds("spaceId").Return([]string{"peerId1"}) + + s.RemoveAllExcept("peerId1", []string{}) + + assert.Equal(t, s.synced, []string{"id"}) }) } -func TestSyncStatusService_Watch(t *testing.T) { - t.Run("Watch: object exist", func(t *testing.T) { - // given - s := newFixture(t) +func TestSyncStatusService_Watch_Unwatch(t *testing.T) { + t.Run("watch", func(t *testing.T) { + s := newFixture(t, "spaceId") - // when - s.detailsUpdater.EXPECT().UpdateDetails([]string{"id"}, domain.ObjectSyncing, domain.Null, "spaceId") - s.service.EXPECT().NetworkCompatibilityStatus().Return(nodeconf.NetworkCompatibilityStatusOk) - s.HeadsChange("id", []string{"head1"}) + s.storage.EXPECT().TreeStorage("id").Return(treestorage.NewInMemoryTreeStorage(&treechangeproto.RawTreeChangeWithId{Id: "id"}, []string{"headId"}, nil)) err := s.Watch("id") - - // then assert.Nil(t, err) - _, ok := s.watchers["id"] - assert.True(t, ok) + assert.Contains(t, s.watchers, "id") + assert.Equal(t, []string{"headId"}, s.treeHeads["id"].heads) }) - t.Run("Watch: object not exist", func(t *testing.T) { - // given - s := newFixture(t) - accountKeys, err := accountdata.NewRandom() - assert.Nil(t, err) - acl, err := list.NewTestDerivedAcl("spaceId", accountKeys) - assert.Nil(t, err) + t.Run("unwatch", func(t *testing.T) { + s := newFixture(t, "spaceId") - root, err := objecttree.CreateObjectTreeRoot(objecttree.ObjectTreeCreatePayload{ - PrivKey: accountKeys.SignKey, - ChangeType: "changeType", - ChangePayload: nil, - SpaceId: "spaceId", - IsEncrypted: true, - }, acl) - storage, err := treestorage.NewInMemoryTreeStorage(root, []string{"head1"}, nil) - assert.Nil(t, err) - - s.storage.EXPECT().TreeStorage("id").Return(storage, nil) - - // when - err = s.Watch("id") - - // then - assert.Nil(t, err) - _, ok := s.watchers["id"] - assert.True(t, ok) - assert.NotNil(t, s.treeHeads["id"]) - assert.Equal(t, StatusUnknown, s.treeHeads["id"].syncStatus) - }) -} - -func TestSyncStatusService_Unwatch(t *testing.T) { - t.Run("Unwatch: object exist", func(t *testing.T) { - // given - s := newFixture(t) - - // when - s.detailsUpdater.EXPECT().UpdateDetails([]string{"id"}, domain.ObjectSyncing, domain.Null, "spaceId") - s.service.EXPECT().NetworkCompatibilityStatus().Return(nodeconf.NetworkCompatibilityStatusOk) - s.HeadsChange("id", []string{"head1"}) + s.storage.EXPECT().TreeStorage("id").Return(treestorage.NewInMemoryTreeStorage(&treechangeproto.RawTreeChangeWithId{Id: "id"}, []string{"headId"}, nil)) err := s.Watch("id") assert.Nil(t, err) s.Unwatch("id") - - // then - _, ok := s.watchers["id"] - assert.False(t, ok) + assert.NotContains(t, s.watchers, "id") + assert.Equal(t, []string{"headId"}, s.treeHeads["id"].heads) }) } func TestSyncStatusService_update(t *testing.T) { t.Run("update: got updates on objects", func(t *testing.T) { - // given - s := newFixture(t) + s := newFixture(t, "spaceId") updateReceiver := NewMockUpdateReceiver(t) updateReceiver.EXPECT().UpdateNodeStatus().Return() - updateReceiver.EXPECT().UpdateTree(context.Background(), "id", StatusNotSynced).Return(nil) + updateReceiver.EXPECT().UpdateTree(context.Background(), "id", StatusSynced).Return(nil) + updateReceiver.EXPECT().UpdateTree(context.Background(), "id2", StatusNotSynced).Return(nil) s.SetUpdateReceiver(updateReceiver) - // when - s.detailsUpdater.EXPECT().UpdateDetails([]string{"id"}, domain.ObjectSyncing, domain.Null, "spaceId") - s.service.EXPECT().NetworkCompatibilityStatus().Return(nodeconf.NetworkCompatibilityStatusOk).Times(2) - s.HeadsChange("id", []string{"head1"}) - err := s.Watch("id") - assert.Nil(t, err) - err = s.update(context.Background()) - - // then - assert.Nil(t, err) - updateReceiver.AssertCalled(t, "UpdateTree", context.Background(), "id", StatusNotSynced) + s.detailsUpdater.EXPECT().UpdateDetails("id3", domain.ObjectSynced, "spaceId") + s.synced = []string{"id3"} + s.tempSynced["id4"] = struct{}{} + s.treeHeads["id"] = treeHeadsEntry{syncStatus: StatusSynced, heads: []string{"headId"}} + s.treeHeads["id2"] = treeHeadsEntry{syncStatus: StatusNotSynced, heads: []string{"headId"}} + s.watchers["id"] = struct{}{} + s.watchers["id2"] = struct{}{} + err := s.update(context.Background()) + require.NoError(t, err) }) } func TestSyncStatusService_Run(t *testing.T) { t.Run("successful run", func(t *testing.T) { - // given - s := newFixture(t) + s := newFixture(t, "spaceId") - // when err := s.Run(context.Background()) - // then assert.Nil(t, err) err = s.Close(context.Background()) assert.Nil(t, err) @@ -264,64 +148,31 @@ func TestSyncStatusService_Run(t *testing.T) { func TestSyncStatusService_RemoveAllExcept(t *testing.T) { t.Run("no existing id", func(t *testing.T) { - // given - f := newFixture(t) - f.treeHeads["heads"] = treeHeadsEntry{syncStatus: StatusNotSynced} + f := newFixture(t, "spaceId") + f.treeHeads["id"] = treeHeadsEntry{syncStatus: StatusNotSynced, heads: []string{"heads"}} - // when f.service.EXPECT().NodeIds(f.spaceId).Return([]string{"peerId"}) f.RemoveAllExcept("peerId", nil) - // then - assert.Equal(t, StatusSynced, f.treeHeads["heads"].syncStatus) + assert.Equal(t, StatusSynced, f.treeHeads["id"].syncStatus) }) t.Run("same ids", func(t *testing.T) { - // given - f := newFixture(t) - f.treeHeads["heads1"] = treeHeadsEntry{syncStatus: StatusNotSynced} + f := newFixture(t, "id") + f.treeHeads["id"] = treeHeadsEntry{syncStatus: StatusNotSynced, heads: []string{"heads"}} - // when f.service.EXPECT().NodeIds(f.spaceId).Return([]string{"peerId"}) - f.RemoveAllExcept("peerId", []string{"heads", "heads"}) + f.RemoveAllExcept("peerId", []string{"id"}) - // then - assert.Equal(t, StatusSynced, f.treeHeads["heads1"].syncStatus) + assert.Equal(t, StatusNotSynced, f.treeHeads["id"].syncStatus) }) t.Run("sender not responsible", func(t *testing.T) { - // given - f := newFixture(t) - f.treeHeads["heads1"] = treeHeadsEntry{syncStatus: StatusNotSynced} + f := newFixture(t, "spaceId") + f.treeHeads["id"] = treeHeadsEntry{syncStatus: StatusNotSynced, heads: []string{"heads"}} - // when - f.service.EXPECT().NodeIds(f.spaceId).Return([]string{}) - f.RemoveAllExcept("peerId", []string{"heads"}) + f.service.EXPECT().NodeIds(f.spaceId).Return([]string{"peerId1"}) + f.RemoveAllExcept("peerId", nil) - // then - assert.Equal(t, StatusNotSynced, f.treeHeads["heads1"].syncStatus) - }) - t.Run("current state is outdated", func(t *testing.T) { - // given - f := newFixture(t) - f.treeHeads["heads1"] = treeHeadsEntry{syncStatus: StatusNotSynced, stateCounter: 1} - - // when - f.service.EXPECT().NodeIds(f.spaceId).Return([]string{}) - f.RemoveAllExcept("peerId", []string{"heads"}) - - // then - assert.Equal(t, StatusNotSynced, f.treeHeads["heads1"].syncStatus) - }) - t.Run("tree is not synced", func(t *testing.T) { - // given - f := newFixture(t) - f.treeHeads["heads"] = treeHeadsEntry{syncStatus: StatusNotSynced} - - // when - f.service.EXPECT().NodeIds(f.spaceId).Return([]string{}) - f.RemoveAllExcept("peerId", []string{"heads"}) - - // then - assert.Equal(t, StatusNotSynced, f.treeHeads["heads"].syncStatus) + assert.Equal(t, StatusNotSynced, f.treeHeads["id"].syncStatus) }) } @@ -334,11 +185,11 @@ type fixture struct { nodeStatus nodestatus.NodeStatus } -func newFixture(t *testing.T) *fixture { +func newFixture(t *testing.T, spaceId string) *fixture { ctrl := gomock.NewController(t) service := mock_nodeconf.NewMockService(ctrl) storage := mock_spacestorage.NewMockSpaceStorage(ctrl) - spaceState := &spacestate.SpaceState{SpaceId: "spaceId"} + spaceState := &spacestate.SpaceState{SpaceId: spaceId} config := &config.Config{} detailsUpdater := mock_objectsyncstatus.NewMockUpdater(t) nodeStatus := nodestatus.NewNodeStatus() @@ -354,14 +205,11 @@ func newFixture(t *testing.T) *fixture { err := nodeStatus.Init(a) assert.Nil(t, err) - syncStatusService := &syncStatusService{ - treeHeads: map[string]treeHeadsEntry{}, - watchers: map[string]struct{}{}, - } - err = syncStatusService.Init(a) + statusService := NewSyncStatusService() + err = statusService.Init(a) assert.Nil(t, err) return &fixture{ - syncStatusService: syncStatusService, + syncStatusService: statusService.(*syncStatusService), service: service, storage: storage, config: config, diff --git a/core/syncstatus/spacesyncstatus/mock_spacesyncstatus/mock_Updater.go b/core/syncstatus/spacesyncstatus/mock_spacesyncstatus/mock_Updater.go index 787a6f468..1f8e5c20c 100644 --- a/core/syncstatus/spacesyncstatus/mock_spacesyncstatus/mock_Updater.go +++ b/core/syncstatus/spacesyncstatus/mock_spacesyncstatus/mock_Updater.go @@ -7,8 +7,6 @@ import ( app "github.com/anyproto/any-sync/app" - domain "github.com/anyproto/anytype-heart/core/domain" - mock "github.com/stretchr/testify/mock" ) @@ -162,6 +160,39 @@ func (_c *MockUpdater_Name_Call) RunAndReturn(run func() string) *MockUpdater_Na return _c } +// Refresh provides a mock function with given fields: spaceId +func (_m *MockUpdater) Refresh(spaceId string) { + _m.Called(spaceId) +} + +// MockUpdater_Refresh_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Refresh' +type MockUpdater_Refresh_Call struct { + *mock.Call +} + +// Refresh is a helper method to define mock.On call +// - spaceId string +func (_e *MockUpdater_Expecter) Refresh(spaceId interface{}) *MockUpdater_Refresh_Call { + return &MockUpdater_Refresh_Call{Call: _e.mock.On("Refresh", spaceId)} +} + +func (_c *MockUpdater_Refresh_Call) Run(run func(spaceId string)) *MockUpdater_Refresh_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string)) + }) + return _c +} + +func (_c *MockUpdater_Refresh_Call) Return() *MockUpdater_Refresh_Call { + _c.Call.Return() + return _c +} + +func (_c *MockUpdater_Refresh_Call) RunAndReturn(run func(string)) *MockUpdater_Refresh_Call { + _c.Call.Return(run) + return _c +} + // Run provides a mock function with given fields: ctx func (_m *MockUpdater) Run(ctx context.Context) error { ret := _m.Called(ctx) @@ -208,35 +239,36 @@ func (_c *MockUpdater_Run_Call) RunAndReturn(run func(context.Context) error) *M return _c } -// SendUpdate provides a mock function with given fields: spaceSync -func (_m *MockUpdater) SendUpdate(spaceSync *domain.SpaceSync) { - _m.Called(spaceSync) +// UpdateMissingIds provides a mock function with given fields: spaceId, ids +func (_m *MockUpdater) UpdateMissingIds(spaceId string, ids []string) { + _m.Called(spaceId, ids) } -// MockUpdater_SendUpdate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SendUpdate' -type MockUpdater_SendUpdate_Call struct { +// MockUpdater_UpdateMissingIds_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateMissingIds' +type MockUpdater_UpdateMissingIds_Call struct { *mock.Call } -// SendUpdate is a helper method to define mock.On call -// - spaceSync *domain.SpaceSync -func (_e *MockUpdater_Expecter) SendUpdate(spaceSync interface{}) *MockUpdater_SendUpdate_Call { - return &MockUpdater_SendUpdate_Call{Call: _e.mock.On("SendUpdate", spaceSync)} +// UpdateMissingIds is a helper method to define mock.On call +// - spaceId string +// - ids []string +func (_e *MockUpdater_Expecter) UpdateMissingIds(spaceId interface{}, ids interface{}) *MockUpdater_UpdateMissingIds_Call { + return &MockUpdater_UpdateMissingIds_Call{Call: _e.mock.On("UpdateMissingIds", spaceId, ids)} } -func (_c *MockUpdater_SendUpdate_Call) Run(run func(spaceSync *domain.SpaceSync)) *MockUpdater_SendUpdate_Call { +func (_c *MockUpdater_UpdateMissingIds_Call) Run(run func(spaceId string, ids []string)) *MockUpdater_UpdateMissingIds_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(*domain.SpaceSync)) + run(args[0].(string), args[1].([]string)) }) return _c } -func (_c *MockUpdater_SendUpdate_Call) Return() *MockUpdater_SendUpdate_Call { +func (_c *MockUpdater_UpdateMissingIds_Call) Return() *MockUpdater_UpdateMissingIds_Call { _c.Call.Return() return _c } -func (_c *MockUpdater_SendUpdate_Call) RunAndReturn(run func(*domain.SpaceSync)) *MockUpdater_SendUpdate_Call { +func (_c *MockUpdater_UpdateMissingIds_Call) RunAndReturn(run func(string, []string)) *MockUpdater_UpdateMissingIds_Call { _c.Call.Return(run) return _c } diff --git a/space/spacecore/peermanager/mock_peermanager/mock_Updater.go b/space/spacecore/peermanager/mock_peermanager/mock_Updater.go index c2a21f9df..9d9dcfb08 100644 --- a/space/spacecore/peermanager/mock_peermanager/mock_Updater.go +++ b/space/spacecore/peermanager/mock_peermanager/mock_Updater.go @@ -7,8 +7,6 @@ import ( app "github.com/anyproto/any-sync/app" - domain "github.com/anyproto/anytype-heart/core/domain" - mock "github.com/stretchr/testify/mock" ) @@ -162,6 +160,39 @@ func (_c *MockUpdater_Name_Call) RunAndReturn(run func() string) *MockUpdater_Na return _c } +// Refresh provides a mock function with given fields: spaceId +func (_m *MockUpdater) Refresh(spaceId string) { + _m.Called(spaceId) +} + +// MockUpdater_Refresh_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Refresh' +type MockUpdater_Refresh_Call struct { + *mock.Call +} + +// Refresh is a helper method to define mock.On call +// - spaceId string +func (_e *MockUpdater_Expecter) Refresh(spaceId interface{}) *MockUpdater_Refresh_Call { + return &MockUpdater_Refresh_Call{Call: _e.mock.On("Refresh", spaceId)} +} + +func (_c *MockUpdater_Refresh_Call) Run(run func(spaceId string)) *MockUpdater_Refresh_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string)) + }) + return _c +} + +func (_c *MockUpdater_Refresh_Call) Return() *MockUpdater_Refresh_Call { + _c.Call.Return() + return _c +} + +func (_c *MockUpdater_Refresh_Call) RunAndReturn(run func(string)) *MockUpdater_Refresh_Call { + _c.Call.Return(run) + return _c +} + // Run provides a mock function with given fields: ctx func (_m *MockUpdater) Run(ctx context.Context) error { ret := _m.Called(ctx) @@ -208,39 +239,6 @@ func (_c *MockUpdater_Run_Call) RunAndReturn(run func(context.Context) error) *M return _c } -// SendUpdate provides a mock function with given fields: spaceSync -func (_m *MockUpdater) SendUpdate(spaceSync *domain.SpaceSync) { - _m.Called(spaceSync) -} - -// MockUpdater_SendUpdate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SendUpdate' -type MockUpdater_SendUpdate_Call struct { - *mock.Call -} - -// SendUpdate is a helper method to define mock.On call -// - spaceSync *domain.SpaceSync -func (_e *MockUpdater_Expecter) SendUpdate(spaceSync interface{}) *MockUpdater_SendUpdate_Call { - return &MockUpdater_SendUpdate_Call{Call: _e.mock.On("SendUpdate", spaceSync)} -} - -func (_c *MockUpdater_SendUpdate_Call) Run(run func(spaceSync *domain.SpaceSync)) *MockUpdater_SendUpdate_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(*domain.SpaceSync)) - }) - return _c -} - -func (_c *MockUpdater_SendUpdate_Call) Return() *MockUpdater_SendUpdate_Call { - _c.Call.Return() - return _c -} - -func (_c *MockUpdater_SendUpdate_Call) RunAndReturn(run func(*domain.SpaceSync)) *MockUpdater_SendUpdate_Call { - _c.Call.Return(run) - return _c -} - // NewMockUpdater creates a new instance of MockUpdater. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewMockUpdater(t interface { From 5256e828ae0999fbd398431a9c189434986128ca Mon Sep 17 00:00:00 2001 From: Sergey Date: Wed, 24 Jul 2024 10:48:42 +0200 Subject: [PATCH 27/71] GO-3769: Add fixture for internal subscription testing --- core/subscription/fixture.go | 46 +++++++++++++++++++++++++++ core/subscription/fixture_test.go | 10 ------ core/subscription/internalsub_test.go | 12 +++---- core/subscription/service.go | 6 ++-- 4 files changed, 55 insertions(+), 19 deletions(-) create mode 100644 core/subscription/fixture.go diff --git a/core/subscription/fixture.go b/core/subscription/fixture.go new file mode 100644 index 000000000..9c62c53ac --- /dev/null +++ b/core/subscription/fixture.go @@ -0,0 +1,46 @@ +package subscription + +import ( + "context" + "testing" + + "github.com/anyproto/any-sync/app" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/event/mock_event" + "github.com/anyproto/anytype-heart/core/kanban" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" + "github.com/anyproto/anytype-heart/tests/testutil" +) + +type InternalTestService struct { + Service + *objectstore.StoreFixture +} + +func NewInternalTestService(t *testing.T) *InternalTestService { + s := New() + ctx := context.Background() + + objectStore := objectstore.NewStoreFixture(t) + + a := &app.App{} + a.Register(objectStore) + a.Register(kanban.New()) + a.Register(&collectionServiceMock{MockCollectionService: NewMockCollectionService(t)}) + a.Register(testutil.PrepareMock(ctx, a, mock_event.NewMockSender(t))) + a.Register(s) + err := a.Start(ctx) + require.NoError(t, err) + return &InternalTestService{Service: s, StoreFixture: objectStore} +} + +type collectionServiceMock struct { + *MockCollectionService +} + +func (c *collectionServiceMock) Name() string { + return "collectionService" +} + +func (c *collectionServiceMock) Init(a *app.App) error { return nil } diff --git a/core/subscription/fixture_test.go b/core/subscription/fixture_test.go index 03f91a730..21c42808a 100644 --- a/core/subscription/fixture_test.go +++ b/core/subscription/fixture_test.go @@ -23,16 +23,6 @@ import ( "github.com/anyproto/anytype-heart/util/testMock/mockKanban" ) -type collectionServiceMock struct { - *MockCollectionService -} - -func (c *collectionServiceMock) Name() string { - return "collectionService" -} - -func (c *collectionServiceMock) Init(a *app.App) error { return nil } - type fixture struct { Service a *app.App diff --git a/core/subscription/internalsub_test.go b/core/subscription/internalsub_test.go index 5209f8b58..51d86d36b 100644 --- a/core/subscription/internalsub_test.go +++ b/core/subscription/internalsub_test.go @@ -26,7 +26,7 @@ func wrapToEventMessages(vals []pb.IsEventMessageValue) []*pb.EventMessage { } func TestInternalSubscriptionSingle(t *testing.T) { - fx := newFixtureWithRealObjectStore(t) + fx := NewInternalTestService(t) resp, err := fx.Search(SubscribeRequest{ SubId: "test", Filters: []*model.BlockContentDataviewFilter{ @@ -44,7 +44,7 @@ func TestInternalSubscriptionSingle(t *testing.T) { require.Empty(t, resp.Records) t.Run("amend details not related to filter", func(t *testing.T) { - fx.store.AddObjects(t, []objectstore.TestObject{ + fx.AddObjects(t, []objectstore.TestObject{ { bundle.RelationKeyId: pbtypes.String("id1"), bundle.RelationKeyName: pbtypes.String("task1"), @@ -53,7 +53,7 @@ func TestInternalSubscriptionSingle(t *testing.T) { }, }) time.Sleep(batchTime) - fx.store.AddObjects(t, []objectstore.TestObject{ + fx.AddObjects(t, []objectstore.TestObject{ { bundle.RelationKeyId: pbtypes.String("id1"), bundle.RelationKeyName: pbtypes.String("task1 renamed"), @@ -74,7 +74,7 @@ func TestInternalSubscriptionSingle(t *testing.T) { }) t.Run("amend details related to filter -- remove from subscription", func(t *testing.T) { - fx.store.AddObjects(t, []objectstore.TestObject{ + fx.AddObjects(t, []objectstore.TestObject{ { bundle.RelationKeyId: pbtypes.String("id2"), bundle.RelationKeyName: pbtypes.String("task2"), @@ -83,7 +83,7 @@ func TestInternalSubscriptionSingle(t *testing.T) { }) time.Sleep(batchTime) - fx.store.AddObjects(t, []objectstore.TestObject{ + fx.AddObjects(t, []objectstore.TestObject{ { bundle.RelationKeyId: pbtypes.String("id2"), bundle.RelationKeyName: pbtypes.String("task2"), @@ -112,7 +112,7 @@ func TestInternalSubscriptionSingle(t *testing.T) { t.Run("try to add after close", func(t *testing.T) { time.Sleep(batchTime) - fx.store.AddObjects(t, []objectstore.TestObject{ + fx.AddObjects(t, []objectstore.TestObject{ { bundle.RelationKeyId: pbtypes.String("id3"), bundle.RelationKeyName: pbtypes.String("task2"), diff --git a/core/subscription/service.go b/core/subscription/service.go index 9b244fd65..1f2ecf897 100644 --- a/core/subscription/service.go +++ b/core/subscription/service.go @@ -125,11 +125,11 @@ func (s *service) Init(a *app.App) (err error) { s.ds = newDependencyService(s) s.subscriptions = make(map[string]subscription) s.customOutput = map[string]*mb2.MB[*pb.EventMessage]{} - s.objectStore = a.MustComponent(objectstore.CName).(objectstore.ObjectStore) - s.kanban = a.MustComponent(kanban.CName).(kanban.Service) + s.objectStore = app.MustComponent[objectstore.ObjectStore](a) + s.kanban = app.MustComponent[kanban.Service](a) s.recBatch = mb.New(0) s.collectionService = app.MustComponent[CollectionService](a) - s.eventSender = a.MustComponent(event.CName).(event.Sender) + s.eventSender = app.MustComponent[event.Sender](a) s.ctxBuf = &opCtx{c: s.cache} s.initDebugger() return From 9546a13a30d485a39b0feffcbe6edada2102cbb6 Mon Sep 17 00:00:00 2001 From: Sergey Date: Wed, 24 Jul 2024 11:52:35 +0200 Subject: [PATCH 28/71] GO-3769: Refactor --- .../object/treesyncer/treesyncer_test.go | 6 +- core/domain/syncstatus.go | 38 ++++----- core/files/fileobject/service.go | 4 +- core/indexer/reindex.go | 8 +- core/syncstatus/detailsupdater/updater.go | 38 ++++----- .../syncstatus/detailsupdater/updater_test.go | 56 ++++++------- core/syncstatus/filestatus.go | 39 ++------- core/syncstatus/filestatus_test.go | 12 +-- .../syncstatus/objectsyncstatus/syncstatus.go | 6 +- .../objectsyncstatus/syncstatus_test.go | 8 +- .../spacesyncstatus/spacestatus_test.go | 82 +++++++++---------- .../syncsubscritions/syncingobjects.go | 2 +- space/spacecore/peermanager/manager_test.go | 2 +- 13 files changed, 132 insertions(+), 169 deletions(-) diff --git a/core/block/object/treesyncer/treesyncer_test.go b/core/block/object/treesyncer/treesyncer_test.go index 057980e92..6f0df3de9 100644 --- a/core/block/object/treesyncer/treesyncer_test.go +++ b/core/block/object/treesyncer/treesyncer_test.go @@ -198,7 +198,7 @@ func TestTreeSyncer(t *testing.T) { fx.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{peerId}) fx.checker.EXPECT().IsPeerOffline(peerId).Return(true) fx.syncStatus.EXPECT().RemoveAllExcept(peerId, []string{existingId}).Return() - fx.syncDetailsUpdater.EXPECT().UpdateDetails([]string{"existing"}, domain.ObjectError, domain.NetworkError, "spaceId").Return() + fx.syncDetailsUpdater.EXPECT().UpdateDetails([]string{"existing"}, domain.ObjectSyncStatusError, domain.SyncErrorNetworkError, "spaceId").Return() fx.StartSync() err := fx.SyncAll(context.Background(), peerId, []string{existingId}, []string{missingId}) @@ -218,8 +218,8 @@ func TestTreeSyncer(t *testing.T) { fx.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{peerId}) fx.checker.EXPECT().IsPeerOffline(peerId).Return(false) fx.syncStatus.EXPECT().RemoveAllExcept(peerId, []string{existingId}).Return() - fx.syncDetailsUpdater.EXPECT().UpdateDetails([]string{"existing"}, domain.ObjectSynced, domain.Null, "spaceId").Return() - fx.syncDetailsUpdater.EXPECT().UpdateDetails([]string{"existing"}, domain.ObjectSyncing, domain.Null, "spaceId").Return() + fx.syncDetailsUpdater.EXPECT().UpdateDetails([]string{"existing"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId").Return() + fx.syncDetailsUpdater.EXPECT().UpdateDetails([]string{"existing"}, domain.ObjectSyncStatusSyncing, domain.SyncErrorNull, "spaceId").Return() fx.StartSync() err := fx.SyncAll(context.Background(), peerId, []string{existingId}, []string{missingId}) diff --git a/core/domain/syncstatus.go b/core/domain/syncstatus.go index b5917ca96..e1ac4df01 100644 --- a/core/domain/syncstatus.go +++ b/core/domain/syncstatus.go @@ -1,55 +1,45 @@ package domain -type SyncType int32 - -const ( - Objects SyncType = 0 - Files SyncType = 1 -) - type SpaceSyncStatus int32 const ( - Synced SpaceSyncStatus = 0 - Syncing SpaceSyncStatus = 1 - Error SpaceSyncStatus = 2 - Offline SpaceSyncStatus = 3 - Unknown SpaceSyncStatus = 4 + SpaceSyncStatusSynced SpaceSyncStatus = 0 + SpaceSyncStatusSyncing SpaceSyncStatus = 1 + SpaceSyncStatusError SpaceSyncStatus = 2 + SpaceSyncStatusOffline SpaceSyncStatus = 3 + SpaceSyncStatusUnknown SpaceSyncStatus = 4 ) type ObjectSyncStatus int32 const ( - ObjectSynced ObjectSyncStatus = 0 - ObjectSyncing ObjectSyncStatus = 1 - ObjectError ObjectSyncStatus = 2 - ObjectQueued ObjectSyncStatus = 3 + ObjectSyncStatusSynced ObjectSyncStatus = 0 + ObjectSyncStatusSyncing ObjectSyncStatus = 1 + ObjectSyncStatusError ObjectSyncStatus = 2 + ObjectSyncStatusQueued ObjectSyncStatus = 3 ) type SyncError int32 const ( - Null SyncError = 0 - StorageLimitExceed SyncError = 1 - IncompatibleVersion SyncError = 2 - NetworkError SyncError = 3 - Oversized SyncError = 4 + SyncErrorNull SyncError = 0 + SyncErrorIncompatibleVersion SyncError = 2 + SyncErrorNetworkError SyncError = 3 + SyncErrorOversized SyncError = 4 ) type SpaceSync struct { SpaceId string Status SpaceSyncStatus SyncError SyncError - SyncType SyncType // MissingObjects is a list of object IDs that are missing, it is not set every time MissingObjects []string } -func MakeSyncStatus(spaceId string, status SpaceSyncStatus, syncError SyncError, syncType SyncType) *SpaceSync { +func MakeSyncStatus(spaceId string, status SpaceSyncStatus, syncError SyncError) *SpaceSync { return &SpaceSync{ SpaceId: spaceId, Status: status, SyncError: syncError, - SyncType: syncType, } } diff --git a/core/files/fileobject/service.go b/core/files/fileobject/service.go index 495db5b16..e6719bfc4 100644 --- a/core/files/fileobject/service.go +++ b/core/files/fileobject/service.go @@ -304,8 +304,8 @@ func (s *service) makeInitialDetails(fileId domain.FileId, origin objectorigin.O // Use general file layout. It will be changed for proper layout after indexing bundle.RelationKeyLayout.String(): pbtypes.Int64(int64(model.ObjectType_file)), bundle.RelationKeyFileIndexingStatus.String(): pbtypes.Int64(int64(model.FileIndexingStatus_NotIndexed)), - bundle.RelationKeySyncStatus.String(): pbtypes.Int64(int64(domain.ObjectQueued)), - bundle.RelationKeySyncError.String(): pbtypes.Int64(int64(domain.Null)), + bundle.RelationKeySyncStatus.String(): pbtypes.Int64(int64(domain.ObjectSyncStatusQueued)), + bundle.RelationKeySyncError.String(): pbtypes.Int64(int64(domain.SyncErrorNull)), bundle.RelationKeyFileBackupStatus.String(): pbtypes.Int64(int64(filesyncstatus.Queued)), }, } diff --git a/core/indexer/reindex.go b/core/indexer/reindex.go index 9618ed3df..960c6328b 100644 --- a/core/indexer/reindex.go +++ b/core/indexer/reindex.go @@ -207,11 +207,11 @@ func (i *indexer) ReindexSpace(space clientspace.Space) (err error) { func (i *indexer) addSyncDetails(space clientspace.Space) { typesForSyncRelations := helper.SyncRelationsSmartblockTypes() - syncStatus := domain.ObjectSynced - syncError := domain.Null + syncStatus := domain.ObjectSyncStatusSynced + syncError := domain.SyncErrorNull if i.config.IsLocalOnlyMode() { - syncStatus = domain.ObjectError - syncError = domain.NetworkError + syncStatus = domain.ObjectSyncStatusError + syncError = domain.SyncErrorNetworkError } ids, err := i.getIdsForTypes(space, typesForSyncRelations...) if err != nil { diff --git a/core/syncstatus/detailsupdater/updater.go b/core/syncstatus/detailsupdater/updater.go index 48d414934..c1aecd3d1 100644 --- a/core/syncstatus/detailsupdater/updater.go +++ b/core/syncstatus/detailsupdater/updater.go @@ -133,7 +133,7 @@ func (u *syncStatusUpdater) UpdateSpaceDetails(existing, missing []string, space for _, id := range added { err := u.addToQueue(&syncStatusDetails{ objectId: id, - status: domain.ObjectSynced, + status: domain.ObjectSyncStatusSynced, spaceId: spaceId, }) if err != nil { @@ -143,7 +143,7 @@ func (u *syncStatusUpdater) UpdateSpaceDetails(existing, missing []string, space for _, id := range removed { err := u.addToQueue(&syncStatusDetails{ objectId: id, - status: domain.ObjectSyncing, + status: domain.ObjectSyncStatusSyncing, spaceId: spaceId, }) if err != nil { @@ -166,12 +166,8 @@ func (u *syncStatusUpdater) getSyncingObjects(spaceId string) []string { } func (u *syncStatusUpdater) updateObjectDetails(syncStatusDetails *syncStatusDetails, objectId string) error { - return u.setObjectDetails(syncStatusDetails, objectId) -} - -func (u *syncStatusUpdater) setObjectDetails(syncStatusDetails *syncStatusDetails, objectId string) error { status := syncStatusDetails.status - syncError := domain.Null + syncError := domain.SyncErrorNull spc, err := u.spaceService.Get(u.ctx, syncStatusDetails.spaceId) if err != nil { return err @@ -206,14 +202,14 @@ func (u *syncStatusUpdater) setObjectDetails(syncStatusDetails *syncStatusDetail } func (u *syncStatusUpdater) isLayoutSuitableForSyncRelations(details *types.Struct) bool { - layoutsWithoutSyncRelations := []float64{ - float64(model.ObjectType_participant), - float64(model.ObjectType_dashboard), - float64(model.ObjectType_spaceView), - float64(model.ObjectType_space), - float64(model.ObjectType_date), + layoutsWithoutSyncRelations := []model.ObjectTypeLayout{ + model.ObjectType_participant, + model.ObjectType_dashboard, + model.ObjectType_spaceView, + model.ObjectType_space, + model.ObjectType_date, } - layout := details.Fields[bundle.RelationKeyLayout.String()].GetNumberValue() + layout := model.ObjectTypeLayout(pbtypes.GetInt64(details, bundle.RelationKeyLayout.String())) return !slices.Contains(layoutsWithoutSyncRelations, layout) } @@ -221,17 +217,17 @@ func mapFileStatus(status filesyncstatus.Status) (domain.ObjectSyncStatus, domai var syncError domain.SyncError switch status { case filesyncstatus.Syncing: - return domain.ObjectSyncing, domain.Null + return domain.ObjectSyncStatusSyncing, domain.SyncErrorNull case filesyncstatus.Queued: - return domain.ObjectQueued, domain.Null + return domain.ObjectSyncStatusQueued, domain.SyncErrorNull case filesyncstatus.Limited: - syncError = domain.Oversized - return domain.ObjectError, syncError + syncError = domain.SyncErrorOversized + return domain.ObjectSyncStatusError, syncError case filesyncstatus.Unknown: - syncError = domain.NetworkError - return domain.ObjectError, syncError + syncError = domain.SyncErrorNetworkError + return domain.ObjectSyncStatusError, syncError default: - return domain.ObjectSynced, domain.Null + return domain.ObjectSyncStatusSynced, domain.SyncErrorNull } } diff --git a/core/syncstatus/detailsupdater/updater_test.go b/core/syncstatus/detailsupdater/updater_test.go index 9eb31813c..66e9e3a51 100644 --- a/core/syncstatus/detailsupdater/updater_test.go +++ b/core/syncstatus/detailsupdater/updater_test.go @@ -33,13 +33,13 @@ func TestSyncStatusUpdater_UpdateDetails(t *testing.T) { fixture.storeFixture.AddObjects(t, []objectstore.TestObject{ { bundle.RelationKeyId: pbtypes.String("id"), - bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.Synced)), - bundle.RelationKeySyncError: pbtypes.Int64(int64(domain.Null)), + bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.SpaceSyncStatusSynced)), + bundle.RelationKeySyncError: pbtypes.Int64(int64(domain.SyncErrorNull)), }, }) // when - err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSynced, domain.Null, "spaceId"}, "id") + err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId"}, "id") // then assert.Nil(t, err) @@ -58,8 +58,8 @@ func TestSyncStatusUpdater_UpdateDetails(t *testing.T) { space.EXPECT().DoLockedIfNotExists("id", mock.Anything).Return(nil) // when - fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.Synced, domain.Null, domain.Objects)) - err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSynced, domain.Null, "spaceId"}, "id") + fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull)) + err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId"}, "id") // then assert.Nil(t, err) @@ -72,15 +72,15 @@ func TestSyncStatusUpdater_UpdateDetails(t *testing.T) { fixture.storeFixture.AddObjects(t, []objectstore.TestObject{ { bundle.RelationKeyId: pbtypes.String("id"), - bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.Error)), - bundle.RelationKeySyncError: pbtypes.Int64(int64(domain.NetworkError)), + bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.SpaceSyncStatusError)), + bundle.RelationKeySyncError: pbtypes.Int64(int64(domain.SyncErrorNetworkError)), }, }) space.EXPECT().DoLockedIfNotExists("id", mock.Anything).Return(nil) // when - fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.Synced, domain.Null, domain.Objects)) - err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSynced, domain.Null, "spaceId"}, "id") + fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull)) + err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId"}, "id") // then assert.Nil(t, err) @@ -94,8 +94,8 @@ func TestSyncStatusUpdater_UpdateDetails(t *testing.T) { space.EXPECT().DoCtx(fixture.updater.ctx, "id", mock.Anything).Return(nil) // when - fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.Synced, domain.Null, domain.Objects)) - err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSynced, domain.Null, "spaceId"}, "id") + fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull)) + err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId"}, "id") // then assert.Nil(t, err) @@ -115,8 +115,8 @@ func TestSyncStatusUpdater_UpdateDetails(t *testing.T) { space.EXPECT().DoLockedIfNotExists("id", mock.Anything).Return(nil) // when - fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.Syncing, domain.Null, domain.Objects)) - err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSynced, domain.Null, "spaceId"}, "id") + fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull)) + err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId"}, "id") // then assert.Nil(t, err) @@ -135,8 +135,8 @@ func TestSyncStatusUpdater_UpdateDetails(t *testing.T) { space.EXPECT().DoLockedIfNotExists("id", mock.Anything).Return(nil) // when - fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.Error, domain.NetworkError, domain.Objects)) - err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSynced, domain.Null, "spaceId"}, "id") + fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusError, domain.SyncErrorNetworkError)) + err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId"}, "id") // then assert.Nil(t, err) @@ -155,8 +155,8 @@ func TestSyncStatusUpdater_UpdateDetails(t *testing.T) { space.EXPECT().DoLockedIfNotExists("id", mock.Anything).Return(nil) // when - fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.Syncing, domain.Null, domain.Objects)) - err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncing, domain.Null, "spaceId"}, "id") + fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull)) + err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncStatusSyncing, domain.SyncErrorNull, "spaceId"}, "id") // then assert.Nil(t, err) @@ -175,8 +175,8 @@ func TestSyncStatusUpdater_UpdateDetails(t *testing.T) { space.EXPECT().DoLockedIfNotExists("id", mock.Anything).Return(nil) // when - fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.Synced, domain.Null, domain.Objects)) - err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSynced, domain.Null, "spaceId"}, "id") + fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull)) + err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId"}, "id") // then assert.Nil(t, err) @@ -197,7 +197,7 @@ func TestSyncStatusUpdater_Run(t *testing.T) { err := fixture.updater.Run(context.Background()) fixture.statusUpdater.EXPECT().SendUpdate(mock.Anything).Return().Maybe() assert.Nil(t, err) - fixture.updater.UpdateDetails([]string{"id"}, domain.ObjectSynced, domain.Null, "spaceId") + fixture.updater.UpdateDetails([]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId") // then err = fixture.updater.Close(context.Background()) @@ -210,11 +210,11 @@ func TestSyncStatusUpdater_Run(t *testing.T) { // when fixture.service.EXPECT().TechSpaceId().Return("techSpaceId").Times(2) - fixture.updater.UpdateDetails([]string{"id"}, domain.ObjectSynced, domain.Null, "spaceId") - fixture.updater.UpdateDetails([]string{"id"}, domain.ObjectSyncing, domain.Null, "spaceId") + fixture.updater.UpdateDetails([]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId") + fixture.updater.UpdateDetails([]string{"id"}, domain.ObjectSyncStatusSyncing, domain.SyncErrorNull, "spaceId") // then - assert.Equal(t, &syncStatusDetails{status: domain.ObjectSyncing, syncError: domain.Null, spaceId: "spaceId"}, fixture.updater.entries["id"]) + assert.Equal(t, &syncStatusDetails{status: domain.ObjectSyncStatusSyncing, syncError: domain.SyncErrorNull, spaceId: "spaceId"}, fixture.updater.entries["id"]) }) } @@ -224,14 +224,14 @@ func TestSyncStatusUpdater_setSyncDetails(t *testing.T) { fixture := newFixture(t) // when - err := fixture.updater.setSyncDetails(fixture.sb, domain.ObjectError, domain.NetworkError) + err := fixture.updater.setSyncDetails(fixture.sb, domain.ObjectSyncStatusError, domain.SyncErrorNetworkError) assert.Nil(t, err) // then details := fixture.sb.NewState().CombinedDetails().GetFields() assert.NotNil(t, details) - assert.Equal(t, pbtypes.Int64(int64(domain.Error)), details[bundle.RelationKeySyncStatus.String()]) - assert.Equal(t, pbtypes.Int64(int64(domain.NetworkError)), details[bundle.RelationKeySyncError.String()]) + assert.Equal(t, pbtypes.Int64(int64(domain.SpaceSyncStatusError)), details[bundle.RelationKeySyncStatus.String()]) + assert.Equal(t, pbtypes.Int64(int64(domain.SyncErrorNetworkError)), details[bundle.RelationKeySyncError.String()]) assert.NotNil(t, details[bundle.RelationKeySyncDate.String()]) }) t.Run("not set smartblock details, because it doesn't implement interface DetailsSettable", func(t *testing.T) { @@ -240,7 +240,7 @@ func TestSyncStatusUpdater_setSyncDetails(t *testing.T) { // when fixture.sb.SetType(coresb.SmartBlockTypePage) - err := fixture.updater.setSyncDetails(editor.NewMissingObject(fixture.sb), domain.ObjectError, domain.NetworkError) + err := fixture.updater.setSyncDetails(editor.NewMissingObject(fixture.sb), domain.ObjectSyncStatusError, domain.SyncErrorNetworkError) // then assert.Nil(t, err) @@ -251,7 +251,7 @@ func TestSyncStatusUpdater_setSyncDetails(t *testing.T) { // when fixture.sb.SetType(coresb.SmartBlockTypeHome) - err := fixture.updater.setSyncDetails(fixture.sb, domain.ObjectError, domain.NetworkError) + err := fixture.updater.setSyncDetails(fixture.sb, domain.ObjectSyncStatusError, domain.SyncErrorNetworkError) // then assert.Nil(t, err) diff --git a/core/syncstatus/filestatus.go b/core/syncstatus/filestatus.go index f484e49cb..829620341 100644 --- a/core/syncstatus/filestatus.go +++ b/core/syncstatus/filestatus.go @@ -88,41 +88,18 @@ func getFileObjectStatus(status filesyncstatus.Status) (domain.ObjectSyncStatus, objectError domain.SyncError ) switch status { - case filesyncstatus.Synced: - objectSyncStatus = domain.ObjectSynced + case filesyncstatus.Synced, filesyncstatus.SyncedLegacy: + objectSyncStatus = domain.ObjectSyncStatusSynced case filesyncstatus.Syncing: - objectSyncStatus = domain.ObjectSyncing + objectSyncStatus = domain.ObjectSyncStatusSyncing case filesyncstatus.Queued: - objectSyncStatus = domain.ObjectQueued + objectSyncStatus = domain.ObjectSyncStatusQueued case filesyncstatus.Limited: - objectError = domain.Oversized - objectSyncStatus = domain.ObjectError + objectError = domain.SyncErrorOversized + objectSyncStatus = domain.ObjectSyncStatusError case filesyncstatus.Unknown: - objectSyncStatus = domain.ObjectError - objectError = domain.NetworkError + objectSyncStatus = domain.ObjectSyncStatusError + objectError = domain.SyncErrorNetworkError } return objectSyncStatus, objectError } - -func getSyncStatus(status filesyncstatus.Status, bytesLeftPercentage float64) (domain.SpaceSyncStatus, domain.SyncError) { - var ( - spaceStatus domain.SpaceSyncStatus - spaceError domain.SyncError - ) - switch status { - case filesyncstatus.Synced: - spaceStatus = domain.Synced - case filesyncstatus.Syncing, filesyncstatus.Queued: - spaceStatus = domain.Syncing - case filesyncstatus.Limited: - spaceStatus = domain.Synced - if bytesLeftPercentage <= limitReachErrorPercentage { - spaceStatus = domain.Error - spaceError = domain.StorageLimitExceed - } - case filesyncstatus.Unknown: - spaceStatus = domain.Error - spaceError = domain.NetworkError - } - return spaceStatus, spaceError -} diff --git a/core/syncstatus/filestatus_test.go b/core/syncstatus/filestatus_test.go index 307989402..0a42ae27c 100644 --- a/core/syncstatus/filestatus_test.go +++ b/core/syncstatus/filestatus_test.go @@ -17,7 +17,7 @@ func Test_sendSpaceStatusUpdate(t *testing.T) { } // when - updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.Error, domain.StorageLimitExceed, domain.Files)).Return() + updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusError, domain.SyncErrorStorageLimitExceed)).Return() s.sendSpaceStatusUpdate(filesyncstatus.Limited, "spaceId", 0) }) t.Run("file limited, but over 1% of storage is available", func(t *testing.T) { @@ -28,7 +28,7 @@ func Test_sendSpaceStatusUpdate(t *testing.T) { } // when - updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.Synced, domain.Null, domain.Files)).Return() + updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull)).Return() s.sendSpaceStatusUpdate(filesyncstatus.Limited, "spaceId", 0.9) }) t.Run("file synced", func(t *testing.T) { @@ -39,7 +39,7 @@ func Test_sendSpaceStatusUpdate(t *testing.T) { } // when - updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.Synced, domain.Null, domain.Files)).Return() + updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull)).Return() s.sendSpaceStatusUpdate(filesyncstatus.Synced, "spaceId", 0) }) t.Run("file queued", func(t *testing.T) { @@ -50,7 +50,7 @@ func Test_sendSpaceStatusUpdate(t *testing.T) { } // when - updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.Syncing, domain.Null, domain.Files)).Return() + updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull)).Return() s.sendSpaceStatusUpdate(filesyncstatus.Queued, "spaceId", 0) }) t.Run("file syncing", func(t *testing.T) { @@ -61,7 +61,7 @@ func Test_sendSpaceStatusUpdate(t *testing.T) { } // when - updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.Syncing, domain.Null, domain.Files)).Return() + updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull)).Return() s.sendSpaceStatusUpdate(filesyncstatus.Syncing, "spaceId", 0) }) t.Run("file unknown status", func(t *testing.T) { @@ -72,7 +72,7 @@ func Test_sendSpaceStatusUpdate(t *testing.T) { } // when - updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.Error, domain.NetworkError, domain.Files)).Return() + updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusError, domain.SyncErrorNetworkError)).Return() s.sendSpaceStatusUpdate(filesyncstatus.Unknown, "spaceId", 0) }) diff --git a/core/syncstatus/objectsyncstatus/syncstatus.go b/core/syncstatus/objectsyncstatus/syncstatus.go index 125a8fe77..559acf89c 100644 --- a/core/syncstatus/objectsyncstatus/syncstatus.go +++ b/core/syncstatus/objectsyncstatus/syncstatus.go @@ -144,7 +144,7 @@ func (s *syncStatusService) HeadsChange(treeId string, heads []string) { s.Lock() s.treeHeads[treeId] = treeHeadsEntry{heads: heads, syncStatus: StatusNotSynced} s.Unlock() - s.updateDetails(treeId, domain.ObjectSyncing) + s.updateDetails(treeId, domain.ObjectSyncStatusSyncing) } func (s *syncStatusService) ObjectReceive(senderId, treeId string, heads []string) { @@ -222,9 +222,9 @@ func (s *syncStatusService) update(ctx context.Context) (err error) { func mapStatus(status SyncStatus) domain.ObjectSyncStatus { if status == StatusSynced { - return domain.ObjectSynced + return domain.ObjectSyncStatusSynced } - return domain.ObjectSyncing + return domain.ObjectSyncStatusSyncing } func (s *syncStatusService) HeadsReceive(senderId, treeId string, heads []string) { diff --git a/core/syncstatus/objectsyncstatus/syncstatus_test.go b/core/syncstatus/objectsyncstatus/syncstatus_test.go index 49ddbcca7..4cd2a551c 100644 --- a/core/syncstatus/objectsyncstatus/syncstatus_test.go +++ b/core/syncstatus/objectsyncstatus/syncstatus_test.go @@ -24,7 +24,7 @@ import ( func Test_UseCases(t *testing.T) { t.Run("HeadsChange: new object", func(t *testing.T) { s := newFixture(t, "spaceId") - s.detailsUpdater.EXPECT().UpdateDetails("id", domain.ObjectSyncing, "spaceId") + s.detailsUpdater.EXPECT().UpdateDetails("id", domain.ObjectSyncStatusSyncing, "spaceId") s.HeadsChange("id", []string{"head1", "head2"}) @@ -33,7 +33,7 @@ func Test_UseCases(t *testing.T) { }) t.Run("HeadsChange then HeadsApply: responsible", func(t *testing.T) { s := newFixture(t, "spaceId") - s.detailsUpdater.EXPECT().UpdateDetails("id", domain.ObjectSyncing, "spaceId") + s.detailsUpdater.EXPECT().UpdateDetails("id", domain.ObjectSyncStatusSyncing, "spaceId") s.HeadsChange("id", []string{"head1", "head2"}) @@ -50,7 +50,7 @@ func Test_UseCases(t *testing.T) { }) t.Run("HeadsChange then HeadsApply: not responsible", func(t *testing.T) { s := newFixture(t, "spaceId") - s.detailsUpdater.EXPECT().UpdateDetails("id", domain.ObjectSyncing, "spaceId") + s.detailsUpdater.EXPECT().UpdateDetails("id", domain.ObjectSyncStatusSyncing, "spaceId") s.HeadsChange("id", []string{"head1", "head2"}) @@ -122,7 +122,7 @@ func TestSyncStatusService_update(t *testing.T) { updateReceiver.EXPECT().UpdateTree(context.Background(), "id2", StatusNotSynced).Return(nil) s.SetUpdateReceiver(updateReceiver) - s.detailsUpdater.EXPECT().UpdateDetails("id3", domain.ObjectSynced, "spaceId") + s.detailsUpdater.EXPECT().UpdateDetails("id3", domain.ObjectSyncStatusSynced, "spaceId") s.synced = []string{"id3"} s.tempSynced["id4"] = struct{}{} s.treeHeads["id"] = treeHeadsEntry{syncStatus: StatusSynced, heads: []string{"headId"}} diff --git a/core/syncstatus/spacesyncstatus/spacestatus_test.go b/core/syncstatus/spacesyncstatus/spacestatus_test.go index fc237c05a..6f3db0f0b 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus_test.go +++ b/core/syncstatus/spacesyncstatus/spacestatus_test.go @@ -119,12 +119,12 @@ func TestSpaceSyncStatus_updateSpaceSyncStatus(t *testing.T) { storeFixture.AddObjects(t, []objectstore.TestObject{ { bundle.RelationKeyId: pbtypes.String("id1"), - bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.Syncing)), + bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.SpaceSyncStatusSyncing)), bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, { bundle.RelationKeyId: pbtypes.String("id2"), - bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.Syncing)), + bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.SpaceSyncStatusSyncing)), bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, }) @@ -135,15 +135,15 @@ func TestSpaceSyncStatus_updateSpaceSyncStatus(t *testing.T) { filesState: NewFileState(storeFixture), objectsState: NewObjectState(storeFixture), } - syncStatus := domain.MakeSyncStatus("spaceId", domain.Syncing, domain.Null, domain.Objects) + syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull) // then status.updateSpaceSyncStatus(syncStatus) // when - assert.Equal(t, domain.Syncing, status.objectsState.GetSyncStatus("spaceId")) + assert.Equal(t, domain.SpaceSyncStatusSyncing, status.objectsState.GetSyncStatus("spaceId")) assert.Equal(t, 2, status.objectsState.GetSyncObjectCount("spaceId")) - assert.Equal(t, domain.Syncing, status.getSpaceSyncStatus(syncStatus.SpaceId)) + assert.Equal(t, domain.SpaceSyncStatusSyncing, status.getSpaceSyncStatus(syncStatus.SpaceId)) }) t.Run("syncing event for files", func(t *testing.T) { // given @@ -187,15 +187,15 @@ func TestSpaceSyncStatus_updateSpaceSyncStatus(t *testing.T) { filesState: NewFileState(storeFixture), objectsState: NewObjectState(storeFixture), } - syncStatus := domain.MakeSyncStatus("spaceId", domain.Syncing, domain.Null, domain.Files) + syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull) // then status.updateSpaceSyncStatus(syncStatus) // when - assert.Equal(t, domain.Syncing, status.filesState.GetSyncStatus("spaceId")) + assert.Equal(t, domain.SpaceSyncStatusSyncing, status.filesState.GetSyncStatus("spaceId")) assert.Equal(t, 2, status.filesState.GetSyncObjectCount("spaceId")) - assert.Equal(t, domain.Syncing, status.getSpaceSyncStatus(syncStatus.SpaceId)) + assert.Equal(t, domain.SpaceSyncStatusSyncing, status.getSpaceSyncStatus(syncStatus.SpaceId)) }) t.Run("don't send not needed synced event if files or objects are still syncing", func(t *testing.T) { // given @@ -207,11 +207,11 @@ func TestSpaceSyncStatus_updateSpaceSyncStatus(t *testing.T) { filesState: NewFileState(objectstore.NewStoreFixture(t)), objectsState: NewObjectState(objectstore.NewStoreFixture(t)), } - objectsSyncStatus := domain.MakeSyncStatus("spaceId", domain.Syncing, domain.Null, domain.Objects) - status.objectsState.SetSyncStatusAndErr(objectsSyncStatus.Status, domain.Null, objectsSyncStatus.SpaceId) + objectsSyncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull) + status.objectsState.SetSyncStatusAndErr(objectsSyncStatus.Status, domain.SyncErrorNull, objectsSyncStatus.SpaceId) // then - syncStatus := domain.MakeSyncStatus("spaceId", domain.Synced, domain.Null, domain.Files) + syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull) status.updateSpaceSyncStatus(syncStatus) // when @@ -240,15 +240,15 @@ func TestSpaceSyncStatus_updateSpaceSyncStatus(t *testing.T) { filesState: NewFileState(objectstore.NewStoreFixture(t)), objectsState: NewObjectState(objectstore.NewStoreFixture(t)), } - syncStatus := domain.MakeSyncStatus("spaceId", domain.Error, domain.NetworkError, domain.Objects) + syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusError, domain.SyncErrorNetworkError) // then status.updateSpaceSyncStatus(syncStatus) // when - assert.Equal(t, domain.Error, status.objectsState.GetSyncStatus("spaceId")) + assert.Equal(t, domain.SpaceSyncStatusError, status.objectsState.GetSyncStatus("spaceId")) assert.Equal(t, 0, status.objectsState.GetSyncObjectCount("spaceId")) - assert.Equal(t, domain.Error, status.getSpaceSyncStatus(syncStatus.SpaceId)) + assert.Equal(t, domain.SpaceSyncStatusError, status.getSpaceSyncStatus(syncStatus.SpaceId)) }) t.Run("send storage error event and then reset it", func(t *testing.T) { // given @@ -273,15 +273,15 @@ func TestSpaceSyncStatus_updateSpaceSyncStatus(t *testing.T) { filesState: NewFileState(objectstore.NewStoreFixture(t)), objectsState: NewObjectState(objectstore.NewStoreFixture(t)), } - syncStatus := domain.MakeSyncStatus("spaceId", domain.Error, domain.StorageLimitExceed, domain.Files) + syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusError, domain.SyncErrorStorageLimitExceed) // then status.updateSpaceSyncStatus(syncStatus) // when - assert.Equal(t, domain.Synced, status.filesState.GetSyncStatus("spaceId")) + assert.Equal(t, domain.SpaceSyncStatusSynced, status.filesState.GetSyncStatus("spaceId")) assert.Equal(t, 0, status.filesState.GetSyncObjectCount("spaceId")) - assert.Equal(t, domain.Synced, status.getSpaceSyncStatus(syncStatus.SpaceId)) + assert.Equal(t, domain.SpaceSyncStatusSynced, status.getSpaceSyncStatus(syncStatus.SpaceId)) }) t.Run("send incompatible error event", func(t *testing.T) { // given @@ -306,15 +306,15 @@ func TestSpaceSyncStatus_updateSpaceSyncStatus(t *testing.T) { filesState: NewFileState(objectstore.NewStoreFixture(t)), objectsState: NewObjectState(objectstore.NewStoreFixture(t)), } - syncStatus := domain.MakeSyncStatus("spaceId", domain.Error, domain.IncompatibleVersion, domain.Objects) + syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusError, domain.SyncErrorIncompatibleVersion) // then status.updateSpaceSyncStatus(syncStatus) // when - assert.Equal(t, domain.Error, status.objectsState.GetSyncStatus("spaceId")) + assert.Equal(t, domain.SpaceSyncStatusError, status.objectsState.GetSyncStatus("spaceId")) assert.Equal(t, 0, status.objectsState.GetSyncObjectCount("spaceId")) - assert.Equal(t, domain.Error, status.getSpaceSyncStatus(syncStatus.SpaceId)) + assert.Equal(t, domain.SpaceSyncStatusError, status.getSpaceSyncStatus(syncStatus.SpaceId)) }) t.Run("send offline event", func(t *testing.T) { // given @@ -339,15 +339,15 @@ func TestSpaceSyncStatus_updateSpaceSyncStatus(t *testing.T) { filesState: NewFileState(objectstore.NewStoreFixture(t)), objectsState: NewObjectState(objectstore.NewStoreFixture(t)), } - syncStatus := domain.MakeSyncStatus("spaceId", domain.Offline, domain.Null, domain.Objects) + syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusOffline, domain.SyncErrorNull) // then status.updateSpaceSyncStatus(syncStatus) // when - assert.Equal(t, domain.Offline, status.objectsState.GetSyncStatus("spaceId")) + assert.Equal(t, domain.SpaceSyncStatusOffline, status.objectsState.GetSyncStatus("spaceId")) assert.Equal(t, 0, status.objectsState.GetSyncObjectCount("spaceId")) - assert.Equal(t, domain.Offline, status.getSpaceSyncStatus(syncStatus.SpaceId)) + assert.Equal(t, domain.SpaceSyncStatusOffline, status.getSpaceSyncStatus(syncStatus.SpaceId)) }) t.Run("send synced event", func(t *testing.T) { // given @@ -359,20 +359,20 @@ func TestSpaceSyncStatus_updateSpaceSyncStatus(t *testing.T) { filesState: NewFileState(objectstore.NewStoreFixture(t)), objectsState: NewObjectState(objectstore.NewStoreFixture(t)), } - syncStatus := domain.MakeSyncStatus("spaceId", domain.Syncing, domain.Null, domain.Objects) + syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull) status.objectsState.SetObjectsNumber(syncStatus) - status.objectsState.SetSyncStatusAndErr(syncStatus.Status, domain.Null, syncStatus.SpaceId) + status.objectsState.SetSyncStatusAndErr(syncStatus.Status, domain.SyncErrorNull, syncStatus.SpaceId) // then - syncStatus = domain.MakeSyncStatus("spaceId", domain.Synced, domain.Null, domain.Objects) + syncStatus = domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull) status.updateSpaceSyncStatus(syncStatus) // when - assert.Equal(t, domain.Synced, status.objectsState.GetSyncStatus("spaceId")) + assert.Equal(t, domain.SpaceSyncStatusSynced, status.objectsState.GetSyncStatus("spaceId")) assert.Equal(t, 0, status.objectsState.GetSyncObjectCount("spaceId")) - assert.Equal(t, domain.Unknown, status.filesState.GetSyncStatus("spaceId")) + assert.Equal(t, domain.SpaceSyncStatusUnknown, status.filesState.GetSyncStatus("spaceId")) assert.Equal(t, 0, status.filesState.GetSyncObjectCount("spaceId")) - assert.Equal(t, domain.Synced, status.getSpaceSyncStatus(syncStatus.SpaceId)) + assert.Equal(t, domain.SpaceSyncStatusSynced, status.getSpaceSyncStatus(syncStatus.SpaceId)) }) t.Run("send initial synced event", func(t *testing.T) { // given @@ -398,7 +398,7 @@ func TestSpaceSyncStatus_updateSpaceSyncStatus(t *testing.T) { }}, }) // then - syncStatus := domain.MakeSyncStatus("spaceId", domain.Synced, domain.Null, domain.Objects) + syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull) status.updateSpaceSyncStatus(syncStatus) }) @@ -412,11 +412,11 @@ func TestSpaceSyncStatus_updateSpaceSyncStatus(t *testing.T) { filesState: NewFileState(objectstore.NewStoreFixture(t)), objectsState: NewObjectState(objectstore.NewStoreFixture(t)), } - status.objectsState.SetSyncStatusAndErr(domain.Synced, domain.Null, "spaceId") - status.filesState.SetSyncStatusAndErr(domain.Synced, domain.Null, "spaceId") + status.objectsState.SetSyncStatusAndErr(domain.SpaceSyncStatusSynced, domain.SyncErrorNull, "spaceId") + status.filesState.SetSyncStatusAndErr(domain.SpaceSyncStatusSynced, domain.SyncErrorNull, "spaceId") // then - syncStatus := domain.MakeSyncStatus("spaceId", domain.Synced, domain.Null, domain.Objects) + syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull) status.updateSpaceSyncStatus(syncStatus) // when @@ -462,23 +462,23 @@ func TestSpaceSyncStatus_updateSpaceSyncStatus(t *testing.T) { storeFixture.AddObjects(t, []objectstore.TestObject{ { bundle.RelationKeyId: pbtypes.String("id1"), - bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.Syncing)), + bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.SpaceSyncStatusSyncing)), bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, { bundle.RelationKeyId: pbtypes.String("id2"), - bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.Synced)), + bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.SpaceSyncStatusSynced)), bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, { bundle.RelationKeyId: pbtypes.String("id3"), - bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.Syncing)), + bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.SpaceSyncStatusSyncing)), bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, }) // when - syncStatus := domain.MakeSyncStatus("spaceId", domain.Syncing, domain.Null, domain.Objects) + syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull) status.updateSpaceSyncStatus(syncStatus) status.updateSpaceSyncStatus(syncStatus) }) @@ -492,11 +492,11 @@ func TestSpaceSyncStatus_updateSpaceSyncStatus(t *testing.T) { filesState: NewFileState(objectstore.NewStoreFixture(t)), objectsState: NewObjectState(objectstore.NewStoreFixture(t)), } - status.objectsState.SetSyncStatusAndErr(domain.Synced, domain.Null, "spaceId") - status.filesState.SetSyncStatusAndErr(domain.Synced, domain.Null, "spaceId") + status.objectsState.SetSyncStatusAndErr(domain.SpaceSyncStatusSynced, domain.SyncErrorNull, "spaceId") + status.filesState.SetSyncStatusAndErr(domain.SpaceSyncStatusSynced, domain.SyncErrorNull, "spaceId") // then - syncStatus := domain.MakeSyncStatus("spaceId", domain.Syncing, domain.Null, domain.Objects) + syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull) status.objectsState.SetObjectsNumber(syncStatus) status.updateSpaceSyncStatus(syncStatus) @@ -528,7 +528,7 @@ func TestSpaceSyncStatus_SendUpdate(t *testing.T) { filesState: NewFileState(objectstore.NewStoreFixture(t)), objectsState: NewObjectState(objectstore.NewStoreFixture(t)), } - syncStatus := domain.MakeSyncStatus("spaceId", domain.Synced, domain.Null, domain.Files) + syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull) // then spaceStatus.SendUpdate(syncStatus) diff --git a/core/syncstatus/syncsubscritions/syncingobjects.go b/core/syncstatus/syncsubscritions/syncingobjects.go index 9dbee1b6e..7e5ee2e64 100644 --- a/core/syncstatus/syncsubscritions/syncingobjects.go +++ b/core/syncstatus/syncsubscritions/syncingobjects.go @@ -36,7 +36,7 @@ func (s *syncingObjects) Run() error { { RelationKey: bundle.RelationKeySyncStatus.String(), Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.Int64(int64(domain.Syncing)), + Value: pbtypes.Int64(int64(domain.SpaceSyncStatusSyncing)), }, { RelationKey: bundle.RelationKeyLayout.String(), diff --git a/space/spacecore/peermanager/manager_test.go b/space/spacecore/peermanager/manager_test.go index 0cb35efc5..0e7531d36 100644 --- a/space/spacecore/peermanager/manager_test.go +++ b/space/spacecore/peermanager/manager_test.go @@ -97,7 +97,7 @@ func Test_fetchResponsiblePeers(t *testing.T) { // when f.pool.EXPECT().GetOneOf(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("failed")) - status := domain.MakeSyncStatus(f.cm.spaceId, domain.Offline, domain.Null, domain.Objects) + status := domain.MakeSyncStatus(f.cm.spaceId, domain.SpaceSyncStatusOffline, domain.SyncErrorNull) f.updater.EXPECT().SendUpdate(status) f.cm.fetchResponsiblePeers() From b27a0c2f21d085bb54242ac0df830226d8662267 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 24 Jul 2024 12:29:14 +0200 Subject: [PATCH 29/71] GO-3769 Test id subscription --- .../objectsubscription_test.go | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 core/syncstatus/syncsubscritions/objectsubscription_test.go diff --git a/core/syncstatus/syncsubscritions/objectsubscription_test.go b/core/syncstatus/syncsubscritions/objectsubscription_test.go new file mode 100644 index 000000000..739875015 --- /dev/null +++ b/core/syncstatus/syncsubscritions/objectsubscription_test.go @@ -0,0 +1,130 @@ +package syncsubscritions + +import ( + "context" + "testing" + "time" + + "github.com/cheggaaa/mb/v3" + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/subscription" + "github.com/anyproto/anytype-heart/core/subscription/mock_subscription" + "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/util/pbtypes" +) + +func makeSubscriptionAdd(id string) *pb.EventMessage { + return &pb.EventMessage{ + Value: &pb.EventMessageValueOfSubscriptionAdd{ + SubscriptionAdd: &pb.EventObjectSubscriptionAdd{ + Id: id, + }, + }, + } +} + +func makeSubscriptionRemove(id string) *pb.EventMessage { + return &pb.EventMessage{ + Value: &pb.EventMessageValueOfSubscriptionRemove{ + SubscriptionRemove: &pb.EventObjectSubscriptionRemove{ + Id: id, + }, + }, + } +} + +func makeDetailsSet(id string) *pb.EventMessage { + return &pb.EventMessage{ + Value: &pb.EventMessageValueOfObjectDetailsSet{ + ObjectDetailsSet: &pb.EventObjectDetailsSet{ + Id: id, + Details: &types.Struct{ + Fields: map[string]*types.Value{ + "key1": pbtypes.String("value1"), + }, + }, + }, + }, + } +} + +func makeDetailsUnset(id string) *pb.EventMessage { + return &pb.EventMessage{ + Value: &pb.EventMessageValueOfObjectDetailsUnset{ + ObjectDetailsUnset: &pb.EventObjectDetailsUnset{ + Id: id, + Keys: []string{"key1", "key2"}, + }, + }, + } +} + +func makeDetailsAmend(id string) *pb.EventMessage { + return &pb.EventMessage{ + Value: &pb.EventMessageValueOfObjectDetailsAmend{ + ObjectDetailsAmend: &pb.EventObjectDetailsAmend{ + Id: id, + Details: []*pb.EventObjectDetailsAmendKeyValue{ + { + Key: "key3", + Value: pbtypes.String("value3"), + }, + }, + }, + }, + } +} + +func makeStructs(ids []string) []*types.Struct { + structs := make([]*types.Struct, len(ids)) + for i, id := range ids { + structs[i] = &types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String(id), + }, + } + } + return structs +} + +func TestIdSubscription(t *testing.T) { + subService := mock_subscription.NewMockService(t) + events := mb.New[*pb.EventMessage](0) + records := makeStructs([]string{"1", "2", "3"}) + // for details amend, set and unset we just check that we handle them correctly (i.e. do nothing) + messages := []*pb.EventMessage{ + makeSubscriptionRemove("2"), + makeDetailsSet("1"), + makeDetailsUnset("2"), + makeDetailsAmend("3"), + makeSubscriptionAdd("4"), + makeSubscriptionRemove("1"), + makeSubscriptionAdd("3"), + makeSubscriptionRemove("5"), + } + for _, msg := range messages { + err := events.Add(context.Background(), msg) + require.NoError(t, err) + } + subscribeResponse := &subscription.SubscribeResponse{ + Output: events, + Records: records, + } + subService.EXPECT().Search(mock.Anything).Return(subscribeResponse, nil) + sub := NewIdSubscription(subService, subscription.SubscribeRequest{}) + err := sub.Run() + require.NoError(t, err) + time.Sleep(100 * time.Millisecond) + ids := make(map[string]struct{}) + sub.Iterate(func(id string, _ struct{}) bool { + ids[id] = struct{}{} + return true + }) + require.Len(t, ids, 2) + require.Contains(t, ids, "3") + require.Contains(t, ids, "4") +} From 4280d360c610bebc958aebdec4618f95c5407acc Mon Sep 17 00:00:00 2001 From: Sergey Date: Wed, 24 Jul 2024 12:53:08 +0200 Subject: [PATCH 30/71] GO-3769: Refactor updater --- core/syncstatus/detailsupdater/updater.go | 58 +++++++++++------------ 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/core/syncstatus/detailsupdater/updater.go b/core/syncstatus/detailsupdater/updater.go index c1aecd3d1..be73fb870 100644 --- a/core/syncstatus/detailsupdater/updater.go +++ b/core/syncstatus/detailsupdater/updater.go @@ -12,7 +12,6 @@ import ( "github.com/cheggaaa/mb/v3" "github.com/gogo/protobuf/types" - "github.com/anyproto/anytype-heart/core/block/editor/basic" "github.com/anyproto/anytype-heart/core/block/editor/smartblock" "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/syncstatus/detailsupdater/helper" @@ -201,16 +200,28 @@ func (u *syncStatusUpdater) updateObjectDetails(syncStatusDetails *syncStatusDet }) } +var suitableLayouts = map[model.ObjectTypeLayout]struct{}{ + model.ObjectType_basic: {}, + model.ObjectType_profile: {}, + model.ObjectType_todo: {}, + model.ObjectType_set: {}, + model.ObjectType_objectType: {}, + model.ObjectType_relation: {}, + model.ObjectType_file: {}, + model.ObjectType_image: {}, + model.ObjectType_note: {}, + model.ObjectType_bookmark: {}, + model.ObjectType_relationOption: {}, + model.ObjectType_collection: {}, + model.ObjectType_audio: {}, + model.ObjectType_video: {}, + model.ObjectType_pdf: {}, +} + func (u *syncStatusUpdater) isLayoutSuitableForSyncRelations(details *types.Struct) bool { - layoutsWithoutSyncRelations := []model.ObjectTypeLayout{ - model.ObjectType_participant, - model.ObjectType_dashboard, - model.ObjectType_spaceView, - model.ObjectType_space, - model.ObjectType_date, - } layout := model.ObjectTypeLayout(pbtypes.GetInt64(details, bundle.RelationKeyLayout.String())) - return !slices.Contains(layoutsWithoutSyncRelations, layout) + _, ok := suitableLayouts[layout] + return ok } func mapFileStatus(status filesyncstatus.Status) (domain.ObjectSyncStatus, domain.SyncError) { @@ -235,31 +246,18 @@ func (u *syncStatusUpdater) setSyncDetails(sb smartblock.SmartBlock, status doma if !slices.Contains(helper.SyncRelationsSmartblockTypes(), sb.Type()) { return nil } - details := sb.CombinedDetails() - if !u.isLayoutSuitableForSyncRelations(details) { + if !u.isLayoutSuitableForSyncRelations(sb.Details()) { return nil } - if fileStatus, ok := details.GetFields()[bundle.RelationKeyFileBackupStatus.String()]; ok { + st := sb.NewState() + if fileStatus, ok := st.LocalDetails().GetFields()[bundle.RelationKeyFileBackupStatus.String()]; ok { status, syncError = mapFileStatus(filesyncstatus.Status(int(fileStatus.GetNumberValue()))) } - if d, ok := sb.(basic.DetailsSettable); ok { - syncStatusDetails := []*model.Detail{ - { - Key: bundle.RelationKeySyncStatus.String(), - Value: pbtypes.Int64(int64(status)), - }, - { - Key: bundle.RelationKeySyncError.String(), - Value: pbtypes.Int64(int64(syncError)), - }, - { - Key: bundle.RelationKeySyncDate.String(), - Value: pbtypes.Int64(time.Now().Unix()), - }, - } - return d.SetDetails(nil, syncStatusDetails, false) - } - return nil + st.SetDetailAndBundledRelation(bundle.RelationKeySyncStatus, pbtypes.Int64(int64(status))) + st.SetDetailAndBundledRelation(bundle.RelationKeySyncError, pbtypes.Int64(int64(syncError))) + st.SetDetailAndBundledRelation(bundle.RelationKeySyncDate, pbtypes.Int64(time.Now().Unix())) + + return sb.Apply(st, smartblock.KeepInternalFlags /* do not erase flags */) } func (u *syncStatusUpdater) processEvents() { From 0c8409b8e5fc77c040ae382e12d962c8714d65a7 Mon Sep 17 00:00:00 2001 From: Sergey Date: Wed, 24 Jul 2024 14:03:06 +0200 Subject: [PATCH 31/71] GO-3769: Fixes --- core/subscription/fixture.go | 4 + .../syncstatus/detailsupdater/updater_test.go | 228 ++---------------- .../syncstatus/spacesyncstatus/spacestatus.go | 10 +- 3 files changed, 30 insertions(+), 212 deletions(-) diff --git a/core/subscription/fixture.go b/core/subscription/fixture.go index 9c62c53ac..5f0e3c438 100644 --- a/core/subscription/fixture.go +++ b/core/subscription/fixture.go @@ -18,6 +18,10 @@ type InternalTestService struct { *objectstore.StoreFixture } +func (s *InternalTestService) Init(a *app.App) error { + return s.Service.Init(a) +} + func NewInternalTestService(t *testing.T) *InternalTestService { s := New() ctx := context.Background() diff --git a/core/syncstatus/detailsupdater/updater_test.go b/core/syncstatus/detailsupdater/updater_test.go index 66e9e3a51..3ab6d22fa 100644 --- a/core/syncstatus/detailsupdater/updater_test.go +++ b/core/syncstatus/detailsupdater/updater_test.go @@ -5,217 +5,29 @@ import ( "testing" "github.com/anyproto/any-sync/app" - "github.com/anyproto/any-sync/app/ocache" - "github.com/cheggaaa/mb/v3" "github.com/gogo/protobuf/types" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "github.com/anyproto/anytype-heart/core/block/editor" "github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest" domain "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/core/subscription" "github.com/anyproto/anytype-heart/core/syncstatus/detailsupdater/mock_detailsupdater" - "github.com/anyproto/anytype-heart/core/syncstatus/filesyncstatus" + "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscritions" "github.com/anyproto/anytype-heart/pkg/lib/bundle" coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" - "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" - "github.com/anyproto/anytype-heart/space/clientspace/mock_clientspace" "github.com/anyproto/anytype-heart/space/mock_space" "github.com/anyproto/anytype-heart/tests/testutil" "github.com/anyproto/anytype-heart/util/pbtypes" ) func TestSyncStatusUpdater_UpdateDetails(t *testing.T) { - t.Run("update sync status and date - no changes", func(t *testing.T) { - // given - fixture := newFixture(t) - fixture.storeFixture.AddObjects(t, []objectstore.TestObject{ - { - bundle.RelationKeyId: pbtypes.String("id"), - bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.SpaceSyncStatusSynced)), - bundle.RelationKeySyncError: pbtypes.Int64(int64(domain.SyncErrorNull)), - }, - }) - // when - err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId"}, "id") - - // then - assert.Nil(t, err) - fixture.service.AssertNotCalled(t, "Get") - }) - t.Run("update sync status and date - details exist in store", func(t *testing.T) { - // given - fixture := newFixture(t) - space := mock_clientspace.NewMockSpace(t) - fixture.service.EXPECT().Get(fixture.updater.ctx, "spaceId").Return(space, nil) - fixture.storeFixture.AddObjects(t, []objectstore.TestObject{ - { - bundle.RelationKeyId: pbtypes.String("id"), - }, - }) - space.EXPECT().DoLockedIfNotExists("id", mock.Anything).Return(nil) - - // when - fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull)) - err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId"}, "id") - - // then - assert.Nil(t, err) - }) - t.Run("update sync status and date - object not exist in cache", func(t *testing.T) { - // given - fixture := newFixture(t) - space := mock_clientspace.NewMockSpace(t) - fixture.service.EXPECT().Get(fixture.updater.ctx, "spaceId").Return(space, nil) - fixture.storeFixture.AddObjects(t, []objectstore.TestObject{ - { - bundle.RelationKeyId: pbtypes.String("id"), - bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.SpaceSyncStatusError)), - bundle.RelationKeySyncError: pbtypes.Int64(int64(domain.SyncErrorNetworkError)), - }, - }) - space.EXPECT().DoLockedIfNotExists("id", mock.Anything).Return(nil) - - // when - fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull)) - err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId"}, "id") - - // then - assert.Nil(t, err) - }) - t.Run("update sync status and date - object exist in cache", func(t *testing.T) { - // given - fixture := newFixture(t) - space := mock_clientspace.NewMockSpace(t) - fixture.service.EXPECT().Get(fixture.updater.ctx, "spaceId").Return(space, nil) - space.EXPECT().DoLockedIfNotExists("id", mock.Anything).Return(ocache.ErrExists) - space.EXPECT().DoCtx(fixture.updater.ctx, "id", mock.Anything).Return(nil) - - // when - fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull)) - err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId"}, "id") - - // then - assert.Nil(t, err) - }) - - t.Run("update sync status and date - file status", func(t *testing.T) { - // given - fixture := newFixture(t) - space := mock_clientspace.NewMockSpace(t) - fixture.service.EXPECT().Get(fixture.updater.ctx, "spaceId").Return(space, nil) - fixture.storeFixture.AddObjects(t, []objectstore.TestObject{ - { - bundle.RelationKeyId: pbtypes.String("id"), - bundle.RelationKeyFileBackupStatus: pbtypes.Int64(int64(filesyncstatus.Syncing)), - }, - }) - space.EXPECT().DoLockedIfNotExists("id", mock.Anything).Return(nil) - - // when - fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull)) - err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId"}, "id") - - // then - assert.Nil(t, err) - }) - t.Run("update sync status and date - unknown file status", func(t *testing.T) { - // given - fixture := newFixture(t) - space := mock_clientspace.NewMockSpace(t) - fixture.service.EXPECT().Get(fixture.updater.ctx, "spaceId").Return(space, nil) - fixture.storeFixture.AddObjects(t, []objectstore.TestObject{ - { - bundle.RelationKeyId: pbtypes.String("id"), - bundle.RelationKeyFileBackupStatus: pbtypes.Int64(int64(filesyncstatus.Unknown)), - }, - }) - space.EXPECT().DoLockedIfNotExists("id", mock.Anything).Return(nil) - - // when - fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusError, domain.SyncErrorNetworkError)) - err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId"}, "id") - - // then - assert.Nil(t, err) - }) - t.Run("update sync status and date - queued file status", func(t *testing.T) { - // given - fixture := newFixture(t) - space := mock_clientspace.NewMockSpace(t) - fixture.service.EXPECT().Get(fixture.updater.ctx, "spaceId").Return(space, nil) - fixture.storeFixture.AddObjects(t, []objectstore.TestObject{ - { - bundle.RelationKeyId: pbtypes.String("id"), - bundle.RelationKeyFileBackupStatus: pbtypes.Int64(int64(filesyncstatus.Queued)), - }, - }) - space.EXPECT().DoLockedIfNotExists("id", mock.Anything).Return(nil) - - // when - fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull)) - err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncStatusSyncing, domain.SyncErrorNull, "spaceId"}, "id") - - // then - assert.Nil(t, err) - }) - t.Run("update sync status and date - synced file status", func(t *testing.T) { - // given - fixture := newFixture(t) - space := mock_clientspace.NewMockSpace(t) - fixture.service.EXPECT().Get(fixture.updater.ctx, "spaceId").Return(space, nil) - fixture.storeFixture.AddObjects(t, []objectstore.TestObject{ - { - bundle.RelationKeyId: pbtypes.String("id"), - bundle.RelationKeyFileBackupStatus: pbtypes.Int64(int64(filesyncstatus.Synced)), - }, - }) - space.EXPECT().DoLockedIfNotExists("id", mock.Anything).Return(nil) - - // when - fixture.statusUpdater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull)) - err := fixture.updater.updateObjectDetails(&syncStatusDetails{[]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId"}, "id") - - // then - assert.Nil(t, err) - }) } func TestSyncStatusUpdater_Run(t *testing.T) { - t.Run("run", func(t *testing.T) { - // given - fixture := newFixture(t) - // when - fixture.service.EXPECT().TechSpaceId().Return("techSpaceId") - space := mock_clientspace.NewMockSpace(t) - fixture.service.EXPECT().Get(mock.Anything, mock.Anything).Return(space, nil).Maybe() - space.EXPECT().DoLockedIfNotExists(mock.Anything, mock.Anything).Return(nil).Maybe() - space.EXPECT().DoCtx(mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() - err := fixture.updater.Run(context.Background()) - fixture.statusUpdater.EXPECT().SendUpdate(mock.Anything).Return().Maybe() - assert.Nil(t, err) - fixture.updater.UpdateDetails([]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId") - - // then - err = fixture.updater.Close(context.Background()) - assert.Nil(t, err) - }) - - t.Run("run 2 time for 1 object", func(t *testing.T) { - // given - fixture := newFixture(t) - - // when - fixture.service.EXPECT().TechSpaceId().Return("techSpaceId").Times(2) - fixture.updater.UpdateDetails([]string{"id"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId") - fixture.updater.UpdateDetails([]string{"id"}, domain.ObjectSyncStatusSyncing, domain.SyncErrorNull, "spaceId") - - // then - assert.Equal(t, &syncStatusDetails{status: domain.ObjectSyncStatusSyncing, syncError: domain.SyncErrorNull, spaceId: "spaceId"}, fixture.updater.entries["id"]) - }) } func TestSyncStatusUpdater_setSyncDetails(t *testing.T) { @@ -290,33 +102,39 @@ func TestSyncStatusUpdater_isLayoutSuitableForSyncRelations(t *testing.T) { func newFixture(t *testing.T) *fixture { smartTest := smarttest.New("id") - storeFixture := objectstore.NewStoreFixture(t) service := mock_space.NewMockService(t) updater := &syncStatusUpdater{ - batcher: mb.New[*syncStatusDetails](0), finish: make(chan struct{}), entries: map[string]*syncStatusDetails{}, } statusUpdater := mock_detailsupdater.NewMockSpaceStatusUpdater(t) + + subscriptionService := subscription.NewInternalTestService(t) + + syncSub := syncsubscritions.New() + + ctx := context.Background() + a := &app.App{} - a.Register(storeFixture). - Register(testutil.PrepareMock(context.Background(), a, service)). - Register(testutil.PrepareMock(context.Background(), a, statusUpdater)) + a.Register(subscriptionService) + a.Register(syncSub) + a.Register(testutil.PrepareMock(ctx, a, service)) + a.Register(testutil.PrepareMock(ctx, a, statusUpdater)) err := updater.Init(a) assert.Nil(t, err) return &fixture{ - updater: updater, - sb: smartTest, - storeFixture: storeFixture, - service: service, - statusUpdater: statusUpdater, + updater: updater, + sb: smartTest, + service: service, + statusUpdater: statusUpdater, + subscriptionService: subscriptionService, } } type fixture struct { - sb *smarttest.SmartTest - updater *syncStatusUpdater - storeFixture *objectstore.StoreFixture - service *mock_space.MockService - statusUpdater *mock_detailsupdater.MockSpaceStatusUpdater + sb *smarttest.SmartTest + updater *syncStatusUpdater + service *mock_space.MockService + statusUpdater *mock_detailsupdater.MockSpaceStatusUpdater + subscriptionService *subscription.InternalTestService } diff --git a/core/syncstatus/spacesyncstatus/spacestatus.go b/core/syncstatus/spacesyncstatus/spacestatus.go index b113b08e0..1009bbeb6 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus.go +++ b/core/syncstatus/spacesyncstatus/spacestatus.go @@ -26,13 +26,9 @@ import ( "github.com/anyproto/anytype-heart/util/slice" ) -const service = "core.syncstatus.spacesyncstatus" +const CName = "core.syncstatus.spacesyncstatus" -var log = logging.Logger("anytype-mw-space-status") - -// nodeconfservice -// nodestatus -// GetNodeUsage(ctx context.Context) (*NodeUsageResponse, error) +var log = logging.Logger(CName) type Updater interface { app.ComponentRunnable @@ -112,7 +108,7 @@ func (s *spaceSyncStatus) Init(a *app.App) (err error) { } func (s *spaceSyncStatus) Name() (name string) { - return service + return CName } func (s *spaceSyncStatus) sendSyncEventForNewSession(ctx session.Context) error { From 51ab4478c6e5ae3a16e571767540506bcd3d3749 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Wed, 24 Jul 2024 14:54:28 +0200 Subject: [PATCH 32/71] GO-3817 improve p2p status --- core/peerstatus/status.go | 219 +++++++++++++++++++++----------------- 1 file changed, 121 insertions(+), 98 deletions(-) diff --git a/core/peerstatus/status.go b/core/peerstatus/status.go index b2ea5cc59..bebc33f64 100644 --- a/core/peerstatus/status.go +++ b/core/peerstatus/status.go @@ -2,6 +2,7 @@ package peerstatus import ( "context" + "errors" "sync" "time" @@ -11,13 +12,18 @@ import ( "github.com/anyproto/anytype-heart/core/event" "github.com/anyproto/anytype-heart/core/session" "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/logging" "github.com/anyproto/anytype-heart/space/spacecore/peerstore" ) const CName = "core.syncstatus.p2p" +var log = logging.Logger(CName) + type Status int32 +var ErrClosed = errors.New("component is closing") + const ( Unknown Status = 0 Connected Status = 1 @@ -33,39 +39,38 @@ type LocalDiscoveryHook interface { type PeerToPeerStatus interface { app.ComponentRunnable - SendNotPossibleStatus() - CheckPeerStatus() + RefreshPeerStatus(spaceId string) error + SetNotPossibleStatus() ResetNotPossibleStatus() RegisterSpace(spaceId string) UnregisterSpace(spaceId string) } +type spaceStatus struct { + status Status + connectionsCount int64 +} + type p2pStatus struct { - spaceIds map[string]struct{} + spaceIds map[string]*spaceStatus eventSender event.Sender contextCancel context.CancelFunc ctx context.Context peerStore peerstore.PeerStore sync.Mutex - status Status - connectionsCount int64 - - forceCheckSpace chan struct{} - updateStatus chan Status - resetNotPossibleStatus chan struct{} - finish chan struct{} + p2pNotPossible bool // global flag means p2p is not possible because of network + workerFinished chan struct{} + refreshSpaceId chan string peersConnectionPool pool.Pool } func New() PeerToPeerStatus { p2pStatusService := &p2pStatus{ - forceCheckSpace: make(chan struct{}, 1), - updateStatus: make(chan Status, 1), - resetNotPossibleStatus: make(chan struct{}, 1), - finish: make(chan struct{}), - spaceIds: make(map[string]struct{}), + workerFinished: make(chan struct{}), + refreshSpaceId: make(chan string), + spaceIds: make(map[string]*spaceStatus), } return p2pStatusService @@ -77,20 +82,33 @@ func (p *p2pStatus) Init(a *app.App) (err error) { p.peersConnectionPool = app.MustComponent[pool.Service](a) localDiscoveryHook := app.MustComponent[LocalDiscoveryHook](a) sessionHookRunner := app.MustComponent[session.HookRunner](a) - localDiscoveryHook.RegisterP2PNotPossible(p.SendNotPossibleStatus) + localDiscoveryHook.RegisterP2PNotPossible(p.SetNotPossibleStatus) localDiscoveryHook.RegisterResetNotPossible(p.ResetNotPossibleStatus) sessionHookRunner.RegisterHook(p.sendStatusForNewSession) + p.peerStore.AddObserver(func(peerId string, spaceIds []string) { + for _, spaceId := range spaceIds { + err = p.RefreshPeerStatus(spaceId) + if err == ErrClosed { + return + } + if err != nil { + log.Error("failed to refresh peer status", "peerId", peerId, "spaceId", spaceId, "error", err) + } + } + }) return nil } func (p *p2pStatus) sendStatusForNewSession(ctx session.Context) error { - p.sendStatus(p.status) + for spaceId, space := range p.spaceIds { + p.sendEvent(spaceId, mapStatusToEvent(space.status), space.connectionsCount) + } return nil } func (p *p2pStatus) Run(ctx context.Context) error { p.ctx, p.contextCancel = context.WithCancel(context.Background()) - go p.checkP2PDevices() + go p.worker() return nil } @@ -98,7 +116,7 @@ func (p *p2pStatus) Close(ctx context.Context) error { if p.contextCancel != nil { p.contextCancel() } - <-p.finish + <-p.workerFinished return nil } @@ -106,37 +124,36 @@ func (p *p2pStatus) Name() (name string) { return CName } -func (p *p2pStatus) CheckPeerStatus() { - p.forceCheckSpace <- struct{}{} +func (p *p2pStatus) RefreshPeerStatus(spaceId string) error { + select { + case <-p.ctx.Done(): + return ErrClosed + case p.refreshSpaceId <- spaceId: + + } + return nil } -func (p *p2pStatus) SendNotPossibleStatus() { - p.updateStatus <- NotPossible +func (p *p2pStatus) SetNotPossibleStatus() { + p.Lock() + p.p2pNotPossible = true + p.Unlock() + p.updateAllSpacesP2PStatus() } func (p *p2pStatus) ResetNotPossibleStatus() { - p.resetNotPossibleStatus <- struct{}{} + p.Lock() + p.p2pNotPossible = false + p.Unlock() + p.updateAllSpacesP2PStatus() } func (p *p2pStatus) RegisterSpace(spaceId string) { - p.Lock() - defer p.Unlock() - p.spaceIds[spaceId] = struct{}{} - connection := p.connectionsCount - - p.eventSender.Broadcast(&pb.Event{ - Messages: []*pb.EventMessage{ - { - Value: &pb.EventMessageValueOfP2PStatusUpdate{ - P2PStatusUpdate: &pb.EventP2PStatusUpdate{ - SpaceId: spaceId, - Status: p.mapStatusToEvent(p.status), - DevicesCounter: connection, - }, - }, - }, - }, - }) + select { + case <-p.ctx.Done(): + return + case p.refreshSpaceId <- spaceId: + } } func (p *p2pStatus) UnregisterSpace(spaceId string) { @@ -145,48 +162,71 @@ func (p *p2pStatus) UnregisterSpace(spaceId string) { delete(p.spaceIds, spaceId) } -func (p *p2pStatus) checkP2PDevices() { - defer close(p.finish) - timer := time.NewTicker(10 * time.Second) +func (p *p2pStatus) worker() { + defer close(p.workerFinished) + timer := time.NewTicker(20 * time.Second) defer timer.Stop() - p.updateSpaceP2PStatus() for { select { case <-p.ctx.Done(): return + case spaceId := <-p.refreshSpaceId: + p.updateSpaceP2PStatus(spaceId) case <-timer.C: - p.updateSpaceP2PStatus() - case <-p.forceCheckSpace: - p.updateSpaceP2PStatus() - case newStatus := <-p.updateStatus: - p.sendStatus(newStatus) - case <-p.resetNotPossibleStatus: - p.resetNotPossible() + // todo: looks like we don't need this anymore because we use observer + p.updateAllSpacesP2PStatus() } } } -func (p *p2pStatus) updateSpaceP2PStatus() { +func (p *p2pStatus) updateAllSpacesP2PStatus() { p.Lock() - defer p.Unlock() - connectionCount := p.countOpenConnections() - newStatus, event := p.getResultStatus(connectionCount) - if newStatus == NotPossible { - return + var spaceIds = make([]string, 0, len(p.spaceIds)) + for spaceId, _ := range p.spaceIds { + spaceIds = append(spaceIds, spaceId) } - if p.status != newStatus || p.connectionsCount != connectionCount { - p.sendEvent(event, connectionCount) - p.status = newStatus - p.connectionsCount = connectionCount + p.Unlock() + for _, spaceId := range spaceIds { + select { + case <-p.ctx.Done(): + return + case p.refreshSpaceId <- spaceId: + + } } } -func (p *p2pStatus) getResultStatus(connectionCount int64) (Status, pb.EventP2PStatusStatus) { +// updateSpaceP2PStatus updates status for specific spaceId and sends event if status changed +func (p *p2pStatus) updateSpaceP2PStatus(spaceId string) { + p.Lock() + defer p.Unlock() + var ( + currentStatus *spaceStatus + ok bool + ) + if currentStatus, ok = p.spaceIds[spaceId]; !ok { + p.spaceIds[spaceId] = &spaceStatus{ + status: NotConnected, + connectionsCount: -1, + } + } + connectionCount := p.countOpenConnections(spaceId) + newStatus, event := p.getResultStatus(p.p2pNotPossible, connectionCount) + + if currentStatus.status != newStatus || currentStatus.connectionsCount != connectionCount { + p.sendEvent(spaceId, event, connectionCount) + currentStatus.status = newStatus + currentStatus.connectionsCount = connectionCount + } +} + +func (p *p2pStatus) getResultStatus(notPossible bool, connectionCount int64) (Status, pb.EventP2PStatusStatus) { var ( newStatus Status event pb.EventP2PStatusStatus ) - if p.status == NotPossible && connectionCount == 0 { + + if notPossible && connectionCount == 0 { return NotPossible, pb.EventP2PStatus_NotPossible } if connectionCount == 0 { @@ -198,11 +238,12 @@ func (p *p2pStatus) getResultStatus(connectionCount int64) (Status, pb.EventP2PS } return newStatus, event } -func (p *p2pStatus) countOpenConnections() int64 { + +func (p *p2pStatus) countOpenConnections(spaceId string) int64 { var connectionCount int64 - ctx, cancelFunc := context.WithTimeout(context.Background(), time.Second*20) + ctx, cancelFunc := context.WithTimeout(p.ctx, time.Second*20) defer cancelFunc() - peerIds := p.peerStore.AllLocalPeers() + peerIds := p.peerStore.LocalPeerIds(spaceId) for _, peerId := range peerIds { _, err := p.peersConnectionPool.Pick(ctx, peerId) if err != nil { @@ -213,15 +254,7 @@ func (p *p2pStatus) countOpenConnections() int64 { return connectionCount } -func (p *p2pStatus) sendStatus(status Status) { - p.Lock() - defer p.Unlock() - pbStatus := p.mapStatusToEvent(status) - p.status = status - p.sendEvent(pbStatus, p.connectionsCount) -} - -func (p *p2pStatus) mapStatusToEvent(status Status) pb.EventP2PStatusStatus { +func mapStatusToEvent(status Status) pb.EventP2PStatusStatus { var pbStatus pb.EventP2PStatusStatus switch status { case Connected: @@ -234,28 +267,18 @@ func (p *p2pStatus) mapStatusToEvent(status Status) pb.EventP2PStatusStatus { return pbStatus } -func (p *p2pStatus) sendEvent(status pb.EventP2PStatusStatus, count int64) { - for spaceId := range p.spaceIds { - p.eventSender.Broadcast(&pb.Event{ - Messages: []*pb.EventMessage{ - { - Value: &pb.EventMessageValueOfP2PStatusUpdate{ - P2PStatusUpdate: &pb.EventP2PStatusUpdate{ - SpaceId: spaceId, - Status: status, - DevicesCounter: count, - }, +func (p *p2pStatus) sendEvent(spaceId string, status pb.EventP2PStatusStatus, count int64) { + p.eventSender.Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{ + { + Value: &pb.EventMessageValueOfP2PStatusUpdate{ + P2PStatusUpdate: &pb.EventP2PStatusUpdate{ + SpaceId: spaceId, + Status: status, + DevicesCounter: count, }, }, }, - }) - } -} - -func (p *p2pStatus) resetNotPossible() { - p.Lock() - defer p.Unlock() - if p.status == NotPossible { - p.status = NotConnected - } + }, + }) } From 7176255daf1066bf75e985ab5c52d671ed4f4a0d Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 24 Jul 2024 15:22:34 +0200 Subject: [PATCH 33/71] GO-3769 Add subscription tests --- core/subscription/fixture.go | 13 ++ .../syncsubscritions/syncingobjects_test.go | 23 ++++ .../syncsubscriptions_test.go | 116 ++++++++++++++++++ 3 files changed, 152 insertions(+) create mode 100644 core/syncstatus/syncsubscritions/syncingobjects_test.go create mode 100644 core/syncstatus/syncsubscritions/syncsubscriptions_test.go diff --git a/core/subscription/fixture.go b/core/subscription/fixture.go index 5f0e3c438..99ebd6e6d 100644 --- a/core/subscription/fixture.go +++ b/core/subscription/fixture.go @@ -22,6 +22,19 @@ func (s *InternalTestService) Init(a *app.App) error { return s.Service.Init(a) } +func (s *InternalTestService) Run(ctx context.Context) error { + err := s.StoreFixture.Run(ctx) + if err != nil { + return err + } + return s.Service.Run(ctx) +} + +func (s *InternalTestService) Close(ctx context.Context) (err error) { + _ = s.Service.Close(ctx) + return s.StoreFixture.Close(ctx) +} + func NewInternalTestService(t *testing.T) *InternalTestService { s := New() ctx := context.Background() diff --git a/core/syncstatus/syncsubscritions/syncingobjects_test.go b/core/syncstatus/syncsubscritions/syncingobjects_test.go new file mode 100644 index 000000000..8866a0a2b --- /dev/null +++ b/core/syncstatus/syncsubscritions/syncingobjects_test.go @@ -0,0 +1,23 @@ +package syncsubscritions + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/subscription" +) + +func TestCount(t *testing.T) { + objSubscription := NewIdSubscription(nil, subscription.SubscribeRequest{}) + objSubscription.sub = map[string]*entry[struct{}]{ + "1": newEmptyEntry[struct{}]("1"), + "2": newEmptyEntry[struct{}]("2"), + "4": newEmptyEntry[struct{}]("4"), + } + syncing := &syncingObjects{ + objectSubscription: objSubscription, + } + cnt := syncing.SyncingObjectsCount([]string{"1", "2", "3"}) + require.Equal(t, 4, cnt) +} diff --git a/core/syncstatus/syncsubscritions/syncsubscriptions_test.go b/core/syncstatus/syncsubscritions/syncsubscriptions_test.go new file mode 100644 index 000000000..73f8a232b --- /dev/null +++ b/core/syncstatus/syncsubscritions/syncsubscriptions_test.go @@ -0,0 +1,116 @@ +package syncsubscritions + +import ( + "context" + "fmt" + "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/core/subscription" + "github.com/anyproto/anytype-heart/core/syncstatus/filesyncstatus" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/util/pbtypes" +) + +func mapFileStatus(status filesyncstatus.Status) domain.ObjectSyncStatus { + switch status { + case filesyncstatus.Syncing: + return domain.ObjectSyncStatusSyncing + case filesyncstatus.Queued: + return domain.ObjectSyncStatusSyncing + case filesyncstatus.Limited: + return domain.ObjectSyncStatusError + default: + return domain.ObjectSyncStatusSynced + } +} + +func genFileObject(fileStatus filesyncstatus.Status, spaceId string) objectstore.TestObject { + id := fmt.Sprintf("%d", rand.Int()) + return objectstore.TestObject{ + bundle.RelationKeyId: pbtypes.String(id), + bundle.RelationKeySyncStatus: pbtypes.Int64(int64(mapFileStatus(fileStatus))), + bundle.RelationKeyFileBackupStatus: pbtypes.Int64(int64(fileStatus)), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_file)), + bundle.RelationKeyName: pbtypes.String("name" + id), + bundle.RelationKeySpaceId: pbtypes.String(spaceId), + } +} + +func genObject(syncStatus domain.ObjectSyncStatus, spaceId string) objectstore.TestObject { + id := fmt.Sprintf("%d", rand.Int()) + return objectstore.TestObject{ + bundle.RelationKeyId: pbtypes.String(id), + bundle.RelationKeySyncStatus: pbtypes.Int64(int64(syncStatus)), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_basic)), + bundle.RelationKeyName: pbtypes.String("name" + id), + bundle.RelationKeySpaceId: pbtypes.String(spaceId), + } +} + +func TestSyncSubscriptions(t *testing.T) { + testSubs := subscription.NewInternalTestService(t) + var objects []objectstore.TestObject + fileObjs := map[string]struct{}{} + objs := map[string]struct{}{} + for i := 0; i < 10; i++ { + obj := genObject(domain.ObjectSyncStatusSyncing, "spaceId") + objects = append(objects, obj) + objs[obj[bundle.RelationKeyId].GetStringValue()] = struct{}{} + } + for i := 0; i < 10; i++ { + objects = append(objects, genObject(domain.ObjectSyncStatusSynced, "spaceId")) + } + for i := 0; i < 10; i++ { + obj := genFileObject(filesyncstatus.Syncing, "spaceId") + objects = append(objects, obj) + fileObjs[obj[bundle.RelationKeyId].GetStringValue()] = struct{}{} + } + for i := 0; i < 10; i++ { + obj := genFileObject(filesyncstatus.Queued, "spaceId") + objects = append(objects, obj) + fileObjs[obj[bundle.RelationKeyId].GetStringValue()] = struct{}{} + } + for i := 0; i < 10; i++ { + objects = append(objects, genFileObject(filesyncstatus.Synced, "spaceId")) + } + testSubs.AddObjects(t, objects) + subs := New() + subs.(*syncSubscriptions).service = testSubs + err := subs.Run(context.Background()) + require.NoError(t, err) + time.Sleep(100 * time.Millisecond) + spaceSub, err := subs.GetSubscription("spaceId") + require.NoError(t, err) + syncCnt := spaceSub.SyncingObjectsCount([]string{"1", "2"}) + fileCnt := spaceSub.FileSyncingObjectsCount() + require.Equal(t, 12, syncCnt) + require.Equal(t, 20, fileCnt) + require.Len(t, fileObjs, 20) + require.Len(t, objs, 10) + spaceSub.GetFileSubscription().Iterate(func(id string, data struct{}) bool { + delete(fileObjs, id) + return true + }) + spaceSub.GetObjectSubscription().Iterate(func(id string, data struct{}) bool { + delete(objs, id) + return true + }) + require.Empty(t, fileObjs) + require.Empty(t, objs) + for i := 0; i < 10; i++ { + objects[i][bundle.RelationKeySyncStatus] = pbtypes.Int64(int64(domain.ObjectSyncStatusSynced)) + testSubs.AddObjects(t, []objectstore.TestObject{objects[i]}) + } + time.Sleep(100 * time.Millisecond) + syncCnt = spaceSub.SyncingObjectsCount([]string{"1", "2"}) + require.Equal(t, 2, syncCnt) + err = subs.Close(context.Background()) + require.NoError(t, err) +} From da20e9cd75e29d499de534b70457fa8feb271504 Mon Sep 17 00:00:00 2001 From: Sergey Date: Wed, 24 Jul 2024 15:30:35 +0200 Subject: [PATCH 34/71] GO-3769: Filesync: fix error handling --- core/filestorage/filesync/upload.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/filestorage/filesync/upload.go b/core/filestorage/filesync/upload.go index 9f857a382..6ade7e196 100644 --- a/core/filestorage/filesync/upload.go +++ b/core/filestorage/filesync/upload.go @@ -97,6 +97,9 @@ func (s *fileSync) handleLimitReachedError(err error, it *QueueItem) *errLimitRe func (s *fileSync) uploadingHandler(ctx context.Context, it *QueueItem) (persistentqueue.Action, error) { spaceId, fileId := it.SpaceId, it.FileId err := s.uploadFile(ctx, spaceId, fileId, it.ObjectId) + if errors.Is(err, context.Canceled) { + return persistentqueue.ActionRetry, nil + } if isObjectDeletedError(err) { return persistentqueue.ActionDone, s.DeleteFile(it.ObjectId, it.FullFileId()) } @@ -143,6 +146,9 @@ func (s *fileSync) addToRetryUploadingQueue(it *QueueItem) persistentqueue.Actio func (s *fileSync) retryingHandler(ctx context.Context, it *QueueItem) (persistentqueue.Action, error) { spaceId, fileId := it.SpaceId, it.FileId err := s.uploadFile(ctx, spaceId, fileId, it.ObjectId) + if errors.Is(err, context.Canceled) { + return persistentqueue.ActionRetry, nil + } if isObjectDeletedError(err) { return persistentqueue.ActionDone, s.removeFromUploadingQueues(it.ObjectId) } From 3d6dca58f5ab44ff738f584f12bad3b7477c8f4c Mon Sep 17 00:00:00 2001 From: Sergey Date: Wed, 24 Jul 2024 15:30:59 +0200 Subject: [PATCH 35/71] GO-3769: Fix file status --- core/syncstatus/detailsupdater/updater.go | 9 ++-- core/syncstatus/filestatus.go | 66 +++-------------------- 2 files changed, 10 insertions(+), 65 deletions(-) diff --git a/core/syncstatus/detailsupdater/updater.go b/core/syncstatus/detailsupdater/updater.go index be73fb870..b826df877 100644 --- a/core/syncstatus/detailsupdater/updater.go +++ b/core/syncstatus/detailsupdater/updater.go @@ -225,18 +225,15 @@ func (u *syncStatusUpdater) isLayoutSuitableForSyncRelations(details *types.Stru } func mapFileStatus(status filesyncstatus.Status) (domain.ObjectSyncStatus, domain.SyncError) { - var syncError domain.SyncError switch status { case filesyncstatus.Syncing: return domain.ObjectSyncStatusSyncing, domain.SyncErrorNull case filesyncstatus.Queued: return domain.ObjectSyncStatusQueued, domain.SyncErrorNull case filesyncstatus.Limited: - syncError = domain.SyncErrorOversized - return domain.ObjectSyncStatusError, syncError + return domain.ObjectSyncStatusError, domain.SyncErrorOversized case filesyncstatus.Unknown: - syncError = domain.SyncErrorNetworkError - return domain.ObjectSyncStatusError, syncError + return domain.ObjectSyncStatusError, domain.SyncErrorNetworkError default: return domain.ObjectSyncStatusSynced, domain.SyncErrorNull } @@ -250,7 +247,7 @@ func (u *syncStatusUpdater) setSyncDetails(sb smartblock.SmartBlock, status doma return nil } st := sb.NewState() - if fileStatus, ok := st.LocalDetails().GetFields()[bundle.RelationKeyFileBackupStatus.String()]; ok { + if fileStatus, ok := st.Details().GetFields()[bundle.RelationKeyFileBackupStatus.String()]; ok { status, syncError = mapFileStatus(filesyncstatus.Status(int(fileStatus.GetNumberValue()))) } st.SetDetailAndBundledRelation(bundle.RelationKeySyncStatus, pbtypes.Int64(int64(status))) diff --git a/core/syncstatus/filestatus.go b/core/syncstatus/filestatus.go index 829620341..14c0a8ae5 100644 --- a/core/syncstatus/filestatus.go +++ b/core/syncstatus/filestatus.go @@ -3,37 +3,32 @@ package syncstatus import ( "context" "fmt" - "time" "github.com/anyproto/anytype-heart/core/block/cache" - "github.com/anyproto/anytype-heart/core/block/editor/basic" "github.com/anyproto/anytype-heart/core/block/editor/smartblock" "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/syncstatus/filesyncstatus" "github.com/anyproto/anytype-heart/pkg/lib/bundle" - "github.com/anyproto/anytype-heart/pkg/lib/pb/model" "github.com/anyproto/anytype-heart/util/pbtypes" ) -const limitReachErrorPercentage = 0.01 - func (s *service) onFileUploadStarted(objectId string, _ domain.FullFileId) error { - return s.indexFileSyncStatus(objectId, filesyncstatus.Syncing, 0) + return s.indexFileSyncStatus(objectId, filesyncstatus.Syncing) } func (s *service) onFileUploaded(objectId string, _ domain.FullFileId) error { - return s.indexFileSyncStatus(objectId, filesyncstatus.Synced, 0) + return s.indexFileSyncStatus(objectId, filesyncstatus.Synced) } func (s *service) onFileLimited(objectId string, _ domain.FullFileId, bytesLeftPercentage float64) error { - return s.indexFileSyncStatus(objectId, filesyncstatus.Limited, bytesLeftPercentage) + return s.indexFileSyncStatus(objectId, filesyncstatus.Limited) } func (s *service) OnFileDelete(fileId domain.FullFileId) { s.spaceSyncStatus.Refresh(fileId.SpaceId) } -func (s *service) indexFileSyncStatus(fileObjectId string, status filesyncstatus.Status, bytesLeftPercentage float64) error { +func (s *service) indexFileSyncStatus(fileObjectId string, status filesyncstatus.Status) error { var spaceId string err := cache.Do(s.objectGetter, fileObjectId, func(sb smartblock.SmartBlock) (err error) { spaceId = sb.SpaceID() @@ -42,12 +37,9 @@ func (s *service) indexFileSyncStatus(fileObjectId string, status filesyncstatus if prevStatus == newStatus { return nil } - detailsSetter, ok := sb.(basic.DetailsSettable) - if !ok { - return fmt.Errorf("setting of details is not supported for %T", sb) - } - details := provideFileStatusDetails(status, newStatus) - return detailsSetter.SetDetails(nil, details, true) + st := sb.NewState() + st.SetDetailAndBundledRelation(bundle.RelationKeyFileBackupStatus, pbtypes.Int64(newStatus)) + return sb.Apply(st) }) if err != nil { return fmt.Errorf("get object: %w", err) @@ -59,47 +51,3 @@ func (s *service) indexFileSyncStatus(fileObjectId string, status filesyncstatus s.spaceSyncStatus.Refresh(spaceId) return nil } - -func provideFileStatusDetails(status filesyncstatus.Status, newStatus int64) []*model.Detail { - syncStatus, syncError := getFileObjectStatus(status) - details := make([]*model.Detail, 0, 4) - details = append(details, &model.Detail{ - Key: bundle.RelationKeySyncStatus.String(), - Value: pbtypes.Int64(int64(syncStatus)), - }) - details = append(details, &model.Detail{ - Key: bundle.RelationKeySyncError.String(), - Value: pbtypes.Int64(int64(syncError)), - }) - details = append(details, &model.Detail{ - Key: bundle.RelationKeySyncDate.String(), - Value: pbtypes.Int64(time.Now().Unix()), - }) - details = append(details, &model.Detail{ - Key: bundle.RelationKeyFileBackupStatus.String(), - Value: pbtypes.Int64(newStatus), - }) - return details -} - -func getFileObjectStatus(status filesyncstatus.Status) (domain.ObjectSyncStatus, domain.SyncError) { - var ( - objectSyncStatus domain.ObjectSyncStatus - objectError domain.SyncError - ) - switch status { - case filesyncstatus.Synced, filesyncstatus.SyncedLegacy: - objectSyncStatus = domain.ObjectSyncStatusSynced - case filesyncstatus.Syncing: - objectSyncStatus = domain.ObjectSyncStatusSyncing - case filesyncstatus.Queued: - objectSyncStatus = domain.ObjectSyncStatusQueued - case filesyncstatus.Limited: - objectError = domain.SyncErrorOversized - objectSyncStatus = domain.ObjectSyncStatusError - case filesyncstatus.Unknown: - objectSyncStatus = domain.ObjectSyncStatusError - objectError = domain.SyncErrorNetworkError - } - return objectSyncStatus, objectError -} From a38320fed4fc8a947c89a4ebddc562d3c0b475ef Mon Sep 17 00:00:00 2001 From: Mikhail Iudin Date: Wed, 24 Jul 2024 16:35:24 +0200 Subject: [PATCH 36/71] GO-3815 Fix accountStop sending --- core/application/account_select.go | 1 + metrics/interceptors.go | 145 +++++++++++++++++------------ metrics/service.go | 18 +++- 3 files changed, 106 insertions(+), 58 deletions(-) diff --git a/core/application/account_select.go b/core/application/account_select.go index a4dc6ba0b..02c069568 100644 --- a/core/application/account_select.go +++ b/core/application/account_select.go @@ -71,6 +71,7 @@ func (s *Service) AccountSelect(ctx context.Context, req *pb.RpcAccountSelectReq if err := s.stop(); err != nil { return nil, errors.Join(ErrFailedToStopApplication, err) } + metrics.Service.SetWorkingDir(req.RootPath, req.Id) return s.start(ctx, req.Id, req.RootPath, req.DisableLocalNetworkSync, req.PreferYamuxTransport, req.NetworkMode, req.NetworkCustomConfigFilePath) } diff --git a/metrics/interceptors.go b/metrics/interceptors.go index 3f32a5088..ca12ba36c 100644 --- a/metrics/interceptors.go +++ b/metrics/interceptors.go @@ -9,6 +9,7 @@ import ( "time" "github.com/samber/lo" + "github.com/valyala/fastjson" "go.uber.org/atomic" "google.golang.org/grpc" @@ -21,6 +22,8 @@ const ( unexpectedErrorCode = -1 parsingErrorCode = -2 accountSelect = "AccountSelect" + accountStop = "AccountStop" + accountStopJson = "account_stop.json" ) var ( @@ -61,18 +64,100 @@ func SharedTraceInterceptor(ctx context.Context, req any, methodName string, act start := time.Now().UnixMilli() resp, err := actualCall(ctx, req) delta := time.Now().UnixMilli() - start + var event *MethodEvent if methodName == accountSelect { if hotSync { - SendMethodEvent(methodName+"Hot", err, resp, delta) + event = toEvent(methodName+"Hot", err, resp, delta) } else { - SendMethodEvent(methodName+"Cold", err, resp, delta) + event = toEvent(methodName+"Cold", err, resp, delta) } + _ = trySendAccountStop() } else { - SendMethodEvent(methodName, err, resp, delta) + event = toEvent(methodName, err, resp, delta) + } + + if event != nil { + if methodName == accountStop { + _ = saveAccountStop(event) + } else { + Service.Send(event) + } } return resp, err } +func saveAccountStop(event *MethodEvent) error { + pool := &fastjson.ArenaPool{} + arena := pool.Get() + defer pool.Put(arena) + + json := arena.NewObject() + json.Set("method_name", arena.NewString(event.methodName)) + json.Set("middle_time", arena.NewNumberInt(int(event.middleTime))) + json.Set("error_code", arena.NewNumberInt(int(event.errorCode))) + json.Set("description", arena.NewString(event.description)) + + data := json.MarshalTo(nil) + jsonPath := filepath.Join(Service.getWorkingDir(), accountStopJson) + _ = os.Remove(jsonPath) + return os.WriteFile(jsonPath, data, 0644) +} + +func trySendAccountStop() error { + jsonPath := filepath.Join(Service.getWorkingDir(), accountStopJson) + data, err := os.ReadFile(jsonPath) + if err != nil { + return err + } + + _ = os.Remove(jsonPath) + + parsedJson, err := fastjson.ParseBytes(data) + if err != nil { + return err + } + + Service.Send(&MethodEvent{ + methodName: string(parsedJson.GetStringBytes("method_name")), + middleTime: parsedJson.GetInt64("middle_time"), + errorCode: parsedJson.GetInt64("error_code"), + description: string(parsedJson.GetStringBytes("description")), + }) + + return nil +} + +func toEvent(method string, err error, resp any, delta int64) *MethodEvent { + if !lo.Contains(excludedMethods, method) { + if err != nil { + return &MethodEvent{ + methodName: method, + errorCode: unexpectedErrorCode, + description: err.Error(), + } + } + errorCode, description, err := reflection.GetError(resp) + if err != nil { + return &MethodEvent{ + methodName: method, + errorCode: parsingErrorCode, + } + } + if errorCode > 0 { + return &MethodEvent{ + methodName: method, + errorCode: errorCode, + description: description, + } + } + return &MethodEvent{ + methodName: method, + middleTime: delta, + } + } + return nil +} + func LongMethodsInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { return SharedLongMethodsInterceptor(ctx, req, extractMethodName(info.FullMethod), handler) } @@ -149,57 +234,3 @@ func dirExists(path string) (bool, error) { func stackTraceHasMethod(method string, stackTrace []byte) bool { return bytes.Contains(stackTrace, []byte("core.(*Middleware)."+method+"(")) } - -func SendMethodEvent(method string, err error, resp any, delta int64) { - if !lo.Contains(excludedMethods, method) { - if err != nil { - sendUnexpectedError(method, err.Error()) - } - errorCode, description, err := reflection.GetError(resp) - if err != nil { - sendErrorParsingError(method) - } - if errorCode > 0 { - sendExpectedError(method, errorCode, description) - } - sendSuccess(method, delta) - } -} - -func sendSuccess(method string, delta int64) { - Service.Send( - &MethodEvent{ - methodName: method, - middleTime: delta, - }, - ) -} - -func sendExpectedError(method string, code int64, description string) { - Service.Send( - &MethodEvent{ - methodName: method, - errorCode: code, - description: description, - }, - ) -} - -func sendErrorParsingError(method string) { - Service.Send( - &MethodEvent{ - methodName: method, - errorCode: parsingErrorCode, - }, - ) -} - -func sendUnexpectedError(method string, description string) { - Service.Send( - &MethodEvent{ - methodName: method, - errorCode: unexpectedErrorCode, - description: description, - }, - ) -} diff --git a/metrics/service.go b/metrics/service.go index f41e09d29..b6fd51478 100644 --- a/metrics/service.go +++ b/metrics/service.go @@ -3,6 +3,7 @@ package metrics import ( "context" + "path/filepath" "sync" "time" @@ -37,7 +38,9 @@ type SamplableEvent interface { type MetricsService interface { InitWithKeys(inHouseKey string) - SetAppVersion(v string) + SetWorkingDir(workingDir string, accountId string) + SetAppVersion(path string) + getWorkingDir() string SetStartVersion(v string) SetDeviceId(t string) SetPlatform(p string) @@ -57,6 +60,7 @@ type service struct { userId string deviceId string platform string + workingDir string clients [1]*client alreadyRunning bool } @@ -83,6 +87,18 @@ func NewService() MetricsService { } } +func (s *service) SetWorkingDir(workingDir string, accountId string) { + s.lock.Lock() + defer s.lock.Unlock() + s.workingDir = filepath.Join(workingDir, accountId) +} + +func (s *service) getWorkingDir() string { + s.lock.Lock() + defer s.lock.Unlock() + return s.workingDir +} + func (s *service) InitWithKeys(inHouseKey string) { s.lock.Lock() defer s.lock.Unlock() From 7e1332d40e317b44cdecb6a43be8a9e5ed1463f8 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Wed, 24 Jul 2024 16:53:39 +0200 Subject: [PATCH 37/71] GO-3817 sendStatusForNewSession: send only for the new session --- core/peerstatus/status.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/core/peerstatus/status.go b/core/peerstatus/status.go index bebc33f64..38bea5d2f 100644 --- a/core/peerstatus/status.go +++ b/core/peerstatus/status.go @@ -100,8 +100,10 @@ func (p *p2pStatus) Init(a *app.App) (err error) { } func (p *p2pStatus) sendStatusForNewSession(ctx session.Context) error { + p.Lock() + defer p.Unlock() for spaceId, space := range p.spaceIds { - p.sendEvent(spaceId, mapStatusToEvent(space.status), space.connectionsCount) + p.sendEvent(ctx.ID(), spaceId, mapStatusToEvent(space.status), space.connectionsCount) } return nil } @@ -214,7 +216,7 @@ func (p *p2pStatus) updateSpaceP2PStatus(spaceId string) { newStatus, event := p.getResultStatus(p.p2pNotPossible, connectionCount) if currentStatus.status != newStatus || currentStatus.connectionsCount != connectionCount { - p.sendEvent(spaceId, event, connectionCount) + p.sendEvent("", spaceId, event, connectionCount) currentStatus.status = newStatus currentStatus.connectionsCount = connectionCount } @@ -267,8 +269,9 @@ func mapStatusToEvent(status Status) pb.EventP2PStatusStatus { return pbStatus } -func (p *p2pStatus) sendEvent(spaceId string, status pb.EventP2PStatusStatus, count int64) { - p.eventSender.Broadcast(&pb.Event{ +// sendEvent sends event to session with sessionToken or broadcast if sessionToken is empty +func (p *p2pStatus) sendEvent(sessionToken string, spaceId string, status pb.EventP2PStatusStatus, count int64) { + event := &pb.Event{ Messages: []*pb.EventMessage{ { Value: &pb.EventMessageValueOfP2PStatusUpdate{ @@ -280,5 +283,10 @@ func (p *p2pStatus) sendEvent(spaceId string, status pb.EventP2PStatusStatus, co }, }, }, - }) + } + if sessionToken != "" { + p.eventSender.SendToSession(sessionToken, event) + return + } + p.eventSender.Broadcast(event) } From a323a5dacb497c63a984e126562b75ac22024e4b Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Wed, 24 Jul 2024 17:10:18 +0200 Subject: [PATCH 38/71] GO-3817 unexport statuses --- core/peerstatus/status.go | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/core/peerstatus/status.go b/core/peerstatus/status.go index 38bea5d2f..94ec579c8 100644 --- a/core/peerstatus/status.go +++ b/core/peerstatus/status.go @@ -39,9 +39,6 @@ type LocalDiscoveryHook interface { type PeerToPeerStatus interface { app.ComponentRunnable - RefreshPeerStatus(spaceId string) error - SetNotPossibleStatus() - ResetNotPossibleStatus() RegisterSpace(spaceId string) UnregisterSpace(spaceId string) } @@ -82,12 +79,12 @@ func (p *p2pStatus) Init(a *app.App) (err error) { p.peersConnectionPool = app.MustComponent[pool.Service](a) localDiscoveryHook := app.MustComponent[LocalDiscoveryHook](a) sessionHookRunner := app.MustComponent[session.HookRunner](a) - localDiscoveryHook.RegisterP2PNotPossible(p.SetNotPossibleStatus) - localDiscoveryHook.RegisterResetNotPossible(p.ResetNotPossibleStatus) + localDiscoveryHook.RegisterP2PNotPossible(p.setNotPossibleStatus) + localDiscoveryHook.RegisterResetNotPossible(p.resetNotPossibleStatus) sessionHookRunner.RegisterHook(p.sendStatusForNewSession) p.peerStore.AddObserver(func(peerId string, spaceIds []string) { for _, spaceId := range spaceIds { - err = p.RefreshPeerStatus(spaceId) + err = p.refreshPeerStatus(spaceId) if err == ErrClosed { return } @@ -126,7 +123,7 @@ func (p *p2pStatus) Name() (name string) { return CName } -func (p *p2pStatus) RefreshPeerStatus(spaceId string) error { +func (p *p2pStatus) refreshPeerStatus(spaceId string) error { select { case <-p.ctx.Done(): return ErrClosed @@ -136,14 +133,14 @@ func (p *p2pStatus) RefreshPeerStatus(spaceId string) error { return nil } -func (p *p2pStatus) SetNotPossibleStatus() { +func (p *p2pStatus) setNotPossibleStatus() { p.Lock() p.p2pNotPossible = true p.Unlock() p.updateAllSpacesP2PStatus() } -func (p *p2pStatus) ResetNotPossibleStatus() { +func (p *p2pStatus) resetNotPossibleStatus() { p.Lock() p.p2pNotPossible = false p.Unlock() @@ -207,10 +204,12 @@ func (p *p2pStatus) updateSpaceP2PStatus(spaceId string) { ok bool ) if currentStatus, ok = p.spaceIds[spaceId]; !ok { - p.spaceIds[spaceId] = &spaceStatus{ - status: NotConnected, - connectionsCount: -1, + currentStatus = &spaceStatus{ + status: Unknown, + connectionsCount: 0, } + + p.spaceIds[spaceId] = currentStatus } connectionCount := p.countOpenConnections(spaceId) newStatus, event := p.getResultStatus(p.p2pNotPossible, connectionCount) @@ -269,7 +268,7 @@ func mapStatusToEvent(status Status) pb.EventP2PStatusStatus { return pbStatus } -// sendEvent sends event to session with sessionToken or broadcast if sessionToken is empty +// sendEvent sends event to session with sessionToken or broadcast to all sessions if sessionToken is empty func (p *p2pStatus) sendEvent(sessionToken string, spaceId string, status pb.EventP2PStatusStatus, count int64) { event := &pb.Event{ Messages: []*pb.EventMessage{ From f98a45632a5e876c164e935f3d680f561e977e1a Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Wed, 24 Jul 2024 18:16:20 +0200 Subject: [PATCH 39/71] GO-3817 peerstore observer: call when peer is removed --- core/filestorage/rpcstore/service.go | 2 +- core/peerstatus/status.go | 4 ++-- space/spacecore/peerstore/peerstore.go | 11 +++++++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/core/filestorage/rpcstore/service.go b/core/filestorage/rpcstore/service.go index 4fab4361c..e59ddde81 100644 --- a/core/filestorage/rpcstore/service.go +++ b/core/filestorage/rpcstore/service.go @@ -30,7 +30,7 @@ type service struct { func (s *service) Init(a *app.App) (err error) { s.pool = a.MustComponent(pool.CName).(pool.Pool) s.peerStore = a.MustComponent(peerstore.CName).(peerstore.PeerStore) - s.peerStore.AddObserver(func(peerId string, spaceIds []string) { + s.peerStore.AddObserver(func(peerId string, spaceIds []string, peerRemoved bool) { select { case s.peerUpdateCh <- struct{}{}: default: diff --git a/core/peerstatus/status.go b/core/peerstatus/status.go index 94ec579c8..b85466a20 100644 --- a/core/peerstatus/status.go +++ b/core/peerstatus/status.go @@ -82,7 +82,8 @@ func (p *p2pStatus) Init(a *app.App) (err error) { localDiscoveryHook.RegisterP2PNotPossible(p.setNotPossibleStatus) localDiscoveryHook.RegisterResetNotPossible(p.resetNotPossibleStatus) sessionHookRunner.RegisterHook(p.sendStatusForNewSession) - p.peerStore.AddObserver(func(peerId string, spaceIds []string) { + p.ctx, p.contextCancel = context.WithCancel(context.Background()) + p.peerStore.AddObserver(func(peerId string, spaceIds []string, peerRemoved bool) { for _, spaceId := range spaceIds { err = p.refreshPeerStatus(spaceId) if err == ErrClosed { @@ -106,7 +107,6 @@ func (p *p2pStatus) sendStatusForNewSession(ctx session.Context) error { } func (p *p2pStatus) Run(ctx context.Context) error { - p.ctx, p.contextCancel = context.WithCancel(context.Background()) go p.worker() return nil } diff --git a/space/spacecore/peerstore/peerstore.go b/space/spacecore/peerstore/peerstore.go index 24d648267..ccdf36535 100644 --- a/space/spacecore/peerstore/peerstore.go +++ b/space/spacecore/peerstore/peerstore.go @@ -32,7 +32,8 @@ func New() PeerStore { } } -type Observer func(peerId string, spaceIds []string) +// Observer is a function that will be called when a peer is updated +type Observer func(peerId string, spaceIds []string, peerRemoved bool) type peerStore struct { nodeConf nodeconf.Service @@ -69,7 +70,7 @@ func (p *peerStore) UpdateLocalPeer(peerId string, spaceIds []string) { } for _, ob := range observers { - ob(peerId, spaceIds) + ob(peerId, spaceIds, false) } }() if oldIds, ok := p.spacesByLocalPeerIds[peerId]; ok { @@ -131,6 +132,12 @@ func (p *peerStore) RemoveLocalPeer(peerId string) { if !exists { return } + defer func() { + observers := p.observers + for _, ob := range observers { + ob(peerId, spaceIds, true) + } + }() // TODO: do we need to notify observer here for _, spaceId := range spaceIds { peerIds := p.localPeerIdsBySpace[spaceId] From 0a4b7d2a0fc977b505408e0241d9293d2e0a41b6 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Wed, 24 Jul 2024 18:16:26 +0200 Subject: [PATCH 40/71] GO-3817 fix tests --- core/peerstatus/status_test.go | 347 ++++++++++++++++----------------- 1 file changed, 164 insertions(+), 183 deletions(-) diff --git a/core/peerstatus/status_test.go b/core/peerstatus/status_test.go index 8ca40b79f..c308bdbfd 100644 --- a/core/peerstatus/status_test.go +++ b/core/peerstatus/status_test.go @@ -2,6 +2,7 @@ package peerstatus import ( "context" + "fmt" "testing" "time" @@ -22,7 +23,7 @@ import ( ) type fixture struct { - PeerToPeerStatus + *p2pStatus sender *mock_event.MockSender service *mock_nodeconf.MockService store peerstore.PeerStore @@ -35,9 +36,6 @@ func TestP2PStatus_Init(t *testing.T) { // given f := newFixture(t, "spaceId", pb.EventP2PStatus_NotConnected, 1) - // when - f.Run(nil) - // then f.Close(nil) }) @@ -47,7 +45,6 @@ func TestP2pStatus_SendNewStatus(t *testing.T) { t.Run("send NotPossible status", func(t *testing.T) { // given f := newFixture(t, "spaceId", pb.EventP2PStatus_NotConnected, 1) - f.Run(nil) // when f.sender.EXPECT().Broadcast(&pb.Event{ @@ -63,16 +60,32 @@ func TestP2pStatus_SendNewStatus(t *testing.T) { }, }, }) - f.SendNotPossibleStatus() + + f.setNotPossibleStatus() // then - status := f.PeerToPeerStatus.(*p2pStatus) - assert.NotNil(t, status) - err := waitForStatus(status, NotPossible) + + err := waitForStatus("spaceId", f.p2pStatus, NotPossible) assert.Nil(t, err) - f.CheckPeerStatus() - err = waitForStatus(status, NotPossible) + // when + f.sender.EXPECT().Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{ + { + Value: &pb.EventMessageValueOfP2PStatusUpdate{ + P2PStatusUpdate: &pb.EventP2PStatusUpdate{ + SpaceId: "spaceId", + Status: pb.EventP2PStatus_NotConnected, + DevicesCounter: 0, + }, + }, + }, + }, + }) + f.resetNotPossibleStatus() + + f.refreshPeerStatus("spaceId") + checkStatus(t, "spaceId", f.p2pStatus, NotConnected) assert.Nil(t, err) f.Close(nil) @@ -81,13 +94,10 @@ func TestP2pStatus_SendNewStatus(t *testing.T) { // given f := newFixture(t, "spaceId", pb.EventP2PStatus_NotConnected, 1) - // when - f.Run(nil) - // then - status := f.PeerToPeerStatus.(*p2pStatus) + status := f.p2pStatus assert.NotNil(t, status) - err := waitForStatus(status, NotConnected) + err := waitForStatus("spaceId", status, NotConnected) assert.Nil(t, err) f.Close(nil) }) @@ -104,8 +114,6 @@ func TestP2pStatus_SendPeerUpdate(t *testing.T) { err := f.pool.AddPeer(context.Background(), peer) assert.Nil(t, err) - // when - f.Run(nil) f.sender.EXPECT().Broadcast(&pb.Event{ Messages: []*pb.EventMessage{ { @@ -119,19 +127,108 @@ func TestP2pStatus_SendPeerUpdate(t *testing.T) { }, }, }) - f.CheckPeerStatus() - // then f.Close(nil) - status := f.PeerToPeerStatus.(*p2pStatus) - assert.NotNil(t, status) - err = waitForStatus(status, Connected) - assert.Nil(t, err) + checkStatus(t, "spaceId", f.p2pStatus, Connected) }) t.Run("send NotConnected status, because we have peer were disconnected", func(t *testing.T) { // given f := newFixture(t, "spaceId", pb.EventP2PStatus_Connected, 1) + ctrl := gomock.NewController(t) + peer := mock_peer.NewMockPeer(ctrl) + peer.EXPECT().Id().Return("peerId") + f.sender.EXPECT().Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{ + { + Value: &pb.EventMessageValueOfP2PStatusUpdate{ + P2PStatusUpdate: &pb.EventP2PStatusUpdate{ + SpaceId: "spaceId", + Status: pb.EventP2PStatus_Connected, + DevicesCounter: 1, + }, + }, + }, + }, + }) + err := f.pool.AddPeer(context.Background(), peer) + assert.Nil(t, err) + f.store.UpdateLocalPeer("peerId", []string{"spaceId"}) + checkStatus(t, "spaceId", f.p2pStatus, Connected) + + f.sender.EXPECT().Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{ + { + Value: &pb.EventMessageValueOfP2PStatusUpdate{ + P2PStatusUpdate: &pb.EventP2PStatusUpdate{ + SpaceId: "spaceId", + Status: pb.EventP2PStatus_NotConnected, + DevicesCounter: 0, + }, + }, + }, + }, + }) + f.store.RemoveLocalPeer("peerId") + + checkStatus(t, "spaceId", f.p2pStatus, NotConnected) + + // then + f.Close(nil) + assert.Nil(t, err) + }) + t.Run("connection was not possible, but after a while starts working", func(t *testing.T) { + // given + f := newFixture(t, "spaceId", pb.EventP2PStatus_NotConnected, 1) + + // when + f.sender.EXPECT().Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{ + { + Value: &pb.EventMessageValueOfP2PStatusUpdate{ + P2PStatusUpdate: &pb.EventP2PStatusUpdate{ + SpaceId: "spaceId", + Status: pb.EventP2PStatus_NotPossible, + DevicesCounter: 0, + }, + }, + }, + }, + }) + f.setNotPossibleStatus() + checkStatus(t, "spaceId", f.p2pStatus, NotPossible) + + f.store.UpdateLocalPeer("peerId", []string{"spaceId"}) + ctrl := gomock.NewController(t) + peer := mock_peer.NewMockPeer(ctrl) + peer.EXPECT().Id().Return("peerId") + f.sender.EXPECT().Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{ + { + Value: &pb.EventMessageValueOfP2PStatusUpdate{ + P2PStatusUpdate: &pb.EventP2PStatusUpdate{ + SpaceId: "spaceId", + Status: pb.EventP2PStatus_Connected, + DevicesCounter: 1, + }, + }, + }, + }, + }) + err := f.pool.AddPeer(context.Background(), peer) + assert.Nil(t, err) + + checkStatus(t, "spaceId", f.p2pStatus, Connected) + // then + f.Close(nil) + }) + t.Run("no peers were connected, but after a while one is connected", func(t *testing.T) { + // given + f := newFixture(t, "spaceId", pb.EventP2PStatus_NotConnected, 1) + + // when + checkStatus(t, "spaceId", f.p2pStatus, NotConnected) + f.store.UpdateLocalPeer("peerId", []string{"spaceId"}) ctrl := gomock.NewController(t) peer := mock_peer.NewMockPeer(ctrl) @@ -139,8 +236,6 @@ func TestP2pStatus_SendPeerUpdate(t *testing.T) { err := f.pool.AddPeer(context.Background(), peer) assert.Nil(t, err) - // when - f.Run(nil) f.sender.EXPECT().Broadcast(&pb.Event{ Messages: []*pb.EventMessage{ { @@ -154,139 +249,16 @@ func TestP2pStatus_SendPeerUpdate(t *testing.T) { }, }, }) - err = waitForStatus(f.PeerToPeerStatus.(*p2pStatus), Connected) - assert.Nil(t, err) - - f.store.RemoveLocalPeer("peerId") - f.sender.EXPECT().Broadcast(&pb.Event{ - Messages: []*pb.EventMessage{ - { - Value: &pb.EventMessageValueOfP2PStatusUpdate{ - P2PStatusUpdate: &pb.EventP2PStatusUpdate{ - SpaceId: "spaceId", - Status: pb.EventP2PStatus_NotConnected, - DevicesCounter: 0, - }, - }, - }, - }, - }) - f.CheckPeerStatus() - err = waitForStatus(f.PeerToPeerStatus.(*p2pStatus), NotConnected) - assert.Nil(t, err) + checkStatus(t, "spaceId", f.p2pStatus, Connected) // then f.Close(nil) - assert.Nil(t, err) - - status := f.PeerToPeerStatus.(*p2pStatus) - assert.NotNil(t, status) - err = waitForStatus(status, NotConnected) - }) - t.Run("connection was not possible, but after a while starts working", func(t *testing.T) { - // given - f := newFixture(t, "spaceId", pb.EventP2PStatus_NotConnected, 1) - - // when - f.Run(nil) - - f.sender.EXPECT().Broadcast(&pb.Event{ - Messages: []*pb.EventMessage{ - { - Value: &pb.EventMessageValueOfP2PStatusUpdate{ - P2PStatusUpdate: &pb.EventP2PStatusUpdate{ - SpaceId: "spaceId", - Status: pb.EventP2PStatus_NotPossible, - DevicesCounter: 0, - }, - }, - }, - }, - }) - f.SendNotPossibleStatus() - err := waitForStatus(f.PeerToPeerStatus.(*p2pStatus), NotPossible) - assert.Nil(t, err) - - f.store.UpdateLocalPeer("peerId", []string{"spaceId"}) - ctrl := gomock.NewController(t) - peer := mock_peer.NewMockPeer(ctrl) - peer.EXPECT().Id().Return("peerId") - err = f.pool.AddPeer(context.Background(), peer) - assert.Nil(t, err) - - f.sender.EXPECT().Broadcast(&pb.Event{ - Messages: []*pb.EventMessage{ - { - Value: &pb.EventMessageValueOfP2PStatusUpdate{ - P2PStatusUpdate: &pb.EventP2PStatusUpdate{ - SpaceId: "spaceId", - Status: pb.EventP2PStatus_Connected, - DevicesCounter: 1, - }, - }, - }, - }, - }) - f.CheckPeerStatus() - err = waitForStatus(f.PeerToPeerStatus.(*p2pStatus), Connected) - assert.Nil(t, err) - - // then - f.Close(nil) - assert.Nil(t, err) - - status := f.PeerToPeerStatus.(*p2pStatus) - assert.NotNil(t, status) - checkStatus(t, status, Connected) - }) - t.Run("no peers were connected, but after a while one is connected", func(t *testing.T) { - // given - f := newFixture(t, "spaceId", pb.EventP2PStatus_NotConnected, 1) - - // when - f.Run(nil) - - err := waitForStatus(f.PeerToPeerStatus.(*p2pStatus), NotConnected) - - f.store.UpdateLocalPeer("peerId", []string{"spaceId"}) - ctrl := gomock.NewController(t) - peer := mock_peer.NewMockPeer(ctrl) - peer.EXPECT().Id().Return("peerId") - err = f.pool.AddPeer(context.Background(), peer) - assert.Nil(t, err) - - f.sender.EXPECT().Broadcast(&pb.Event{ - Messages: []*pb.EventMessage{ - { - Value: &pb.EventMessageValueOfP2PStatusUpdate{ - P2PStatusUpdate: &pb.EventP2PStatusUpdate{ - SpaceId: "spaceId", - Status: pb.EventP2PStatus_Connected, - DevicesCounter: 1, - }, - }, - }, - }, - }) - f.CheckPeerStatus() - err = waitForStatus(f.PeerToPeerStatus.(*p2pStatus), Connected) - assert.Nil(t, err) - - // then - f.Close(nil) - assert.Nil(t, err) - - status := f.PeerToPeerStatus.(*p2pStatus) - assert.NotNil(t, status) - checkStatus(t, status, Connected) }) t.Run("reset not possible status", func(t *testing.T) { // given f := newFixture(t, "spaceId", pb.EventP2PStatus_NotConnected, 1) // when - f.Run(nil) - f.sender.EXPECT().Broadcast(&pb.Event{ Messages: []*pb.EventMessage{ { @@ -300,10 +272,13 @@ func TestP2pStatus_SendPeerUpdate(t *testing.T) { }, }, }) - f.SendNotPossibleStatus() - status := f.PeerToPeerStatus.(*p2pStatus) - assert.NotNil(t, status) - err := waitForStatus(status, NotPossible) + f.setNotPossibleStatus() + checkStatus(t, "spaceId", f.p2pStatus, NotPossible) + + // double set should not generate new event + f.setNotPossibleStatus() + checkStatus(t, "spaceId", f.p2pStatus, NotPossible) + f.sender.EXPECT().Broadcast(&pb.Event{ Messages: []*pb.EventMessage{ { @@ -317,32 +292,24 @@ func TestP2pStatus_SendPeerUpdate(t *testing.T) { }, }, }) - f.ResetNotPossibleStatus() - err = waitForStatus(status, NotConnected) - assert.Nil(t, err) + f.resetNotPossibleStatus() + checkStatus(t, "spaceId", f.p2pStatus, NotConnected) // then f.Close(nil) - assert.Nil(t, err) - checkStatus(t, status, NotConnected) }) t.Run("don't reset not possible status, because status != NotPossible", func(t *testing.T) { // given f := newFixture(t, "spaceId", pb.EventP2PStatus_NotConnected, 1) // when - f.Run(nil) - - status := f.PeerToPeerStatus.(*p2pStatus) - - err := waitForStatus(status, NotConnected) - f.ResetNotPossibleStatus() - err = waitForStatus(status, NotConnected) + checkStatus(t, "spaceId", f.p2pStatus, NotConnected) + f.resetNotPossibleStatus() + checkStatus(t, "spaceId", f.p2pStatus, NotConnected) // then f.Close(nil) - assert.Nil(t, err) - checkStatus(t, status, NotConnected) + checkStatus(t, "spaceId", f.p2pStatus, NotConnected) }) } @@ -356,7 +323,7 @@ func TestP2pStatus_UnregisterSpace(t *testing.T) { // then - status := f.PeerToPeerStatus.(*p2pStatus) + status := f.p2pStatus assert.Len(t, status.spaceIds, 0) }) t.Run("delete non existing space", func(t *testing.T) { @@ -367,7 +334,7 @@ func TestP2pStatus_UnregisterSpace(t *testing.T) { f.UnregisterSpace("spaceId1") // then - status := f.PeerToPeerStatus.(*p2pStatus) + status := f.p2pStatus assert.Len(t, status.spaceIds, 1) }) } @@ -423,40 +390,54 @@ func newFixture(t *testing.T, spaceId string, initialStatus pb.EventP2PStatusSta }, }, }).Maybe() - status.RegisterSpace(spaceId) + + err = status.Run(context.Background()) assert.Nil(t, err) + status.RegisterSpace(spaceId) + f := &fixture{ - PeerToPeerStatus: status, - sender: sender, - service: service, - store: store, - pool: pool, - hookRegister: hookRegister, + p2pStatus: status.(*p2pStatus), + sender: sender, + service: service, + store: store, + pool: pool, + hookRegister: hookRegister, } return f } -func waitForStatus(statusSender *p2pStatus, expectedStatus Status) error { +func waitForStatus(spaceId string, statusSender *p2pStatus, expectedStatus Status) error { ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() for { select { case <-ctx.Done(): return ctx.Err() - default: + case <-time.After(time.Millisecond * 10): statusSender.Lock() - if statusSender.status == expectedStatus { + if status, ok := statusSender.spaceIds[spaceId]; !ok { statusSender.Unlock() - return nil + return fmt.Errorf("spaceId %s not found", spaceId) + } else { + if status.status == expectedStatus { + statusSender.Unlock() + return nil + } } statusSender.Unlock() } } + } -func checkStatus(t *testing.T, statusSender *p2pStatus, expectedStatus Status) { +func checkStatus(t *testing.T, spaceId string, statusSender *p2pStatus, expectedStatus Status) { + time.Sleep(time.Millisecond * 300) statusSender.Lock() defer statusSender.Unlock() - assert.Equal(t, expectedStatus, statusSender.status) + if status, ok := statusSender.spaceIds[spaceId]; !ok { + assert.Fail(t, "spaceId %s not found", spaceId) + } else { + assert.Equal(t, expectedStatus, status.status) + } } From 88ce69872c32f8ab49d11ab8ce60e88c5e5bb5ca Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Wed, 24 Jul 2024 18:36:50 +0200 Subject: [PATCH 41/71] GO-3817 cleanup --- core/peerstatus/status.go | 51 +++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/core/peerstatus/status.go b/core/peerstatus/status.go index b85466a20..0bd535cf7 100644 --- a/core/peerstatus/status.go +++ b/core/peerstatus/status.go @@ -31,6 +31,19 @@ const ( NotConnected Status = 3 ) +func (s Status) ToPb() pb.EventP2PStatusStatus { + switch s { + case Connected: + return pb.EventP2PStatus_Connected + case NotConnected: + return pb.EventP2PStatus_NotConnected + case NotPossible: + return pb.EventP2PStatus_NotPossible + } + // default status is NotConnected + return pb.EventP2PStatus_NotConnected +} + type LocalDiscoveryHook interface { app.Component RegisterP2PNotPossible(hook func()) @@ -90,6 +103,7 @@ func (p *p2pStatus) Init(a *app.App) (err error) { return } if err != nil { + // we don't have any other errors for now, added just in case for future log.Error("failed to refresh peer status", "peerId", peerId, "spaceId", spaceId, "error", err) } } @@ -101,7 +115,7 @@ func (p *p2pStatus) sendStatusForNewSession(ctx session.Context) error { p.Lock() defer p.Unlock() for spaceId, space := range p.spaceIds { - p.sendEvent(ctx.ID(), spaceId, mapStatusToEvent(space.status), space.connectionsCount) + p.sendEvent(ctx.ID(), spaceId, space.status.ToPb(), space.connectionsCount) } return nil } @@ -212,37 +226,29 @@ func (p *p2pStatus) updateSpaceP2PStatus(spaceId string) { p.spaceIds[spaceId] = currentStatus } connectionCount := p.countOpenConnections(spaceId) - newStatus, event := p.getResultStatus(p.p2pNotPossible, connectionCount) + newStatus := p.getResultStatus(p.p2pNotPossible, connectionCount) if currentStatus.status != newStatus || currentStatus.connectionsCount != connectionCount { - p.sendEvent("", spaceId, event, connectionCount) + p.sendEvent("", spaceId, newStatus.ToPb(), connectionCount) currentStatus.status = newStatus currentStatus.connectionsCount = connectionCount } } -func (p *p2pStatus) getResultStatus(notPossible bool, connectionCount int64) (Status, pb.EventP2PStatusStatus) { - var ( - newStatus Status - event pb.EventP2PStatusStatus - ) - +func (p *p2pStatus) getResultStatus(notPossible bool, connectionCount int64) Status { if notPossible && connectionCount == 0 { - return NotPossible, pb.EventP2PStatus_NotPossible + return NotPossible } if connectionCount == 0 { - event = pb.EventP2PStatus_NotConnected - newStatus = NotConnected + return NotConnected } else { - event = pb.EventP2PStatus_Connected - newStatus = Connected + return Connected } - return newStatus, event } func (p *p2pStatus) countOpenConnections(spaceId string) int64 { var connectionCount int64 - ctx, cancelFunc := context.WithTimeout(p.ctx, time.Second*20) + ctx, cancelFunc := context.WithTimeout(p.ctx, time.Second*10) defer cancelFunc() peerIds := p.peerStore.LocalPeerIds(spaceId) for _, peerId := range peerIds { @@ -255,19 +261,6 @@ func (p *p2pStatus) countOpenConnections(spaceId string) int64 { return connectionCount } -func mapStatusToEvent(status Status) pb.EventP2PStatusStatus { - var pbStatus pb.EventP2PStatusStatus - switch status { - case Connected: - pbStatus = pb.EventP2PStatus_Connected - case NotConnected: - pbStatus = pb.EventP2PStatus_NotConnected - case NotPossible: - pbStatus = pb.EventP2PStatus_NotPossible - } - return pbStatus -} - // sendEvent sends event to session with sessionToken or broadcast to all sessions if sessionToken is empty func (p *p2pStatus) sendEvent(sessionToken string, spaceId string, status pb.EventP2PStatusStatus, count int64) { event := &pb.Event{ From a71858e336863a7e6838e60b2a14b261e090a37a Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Wed, 24 Jul 2024 18:36:59 +0200 Subject: [PATCH 42/71] GO-3817 add test for new session --- core/peerstatus/status_test.go | 50 +++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/core/peerstatus/status_test.go b/core/peerstatus/status_test.go index c308bdbfd..07d4cfd40 100644 --- a/core/peerstatus/status_test.go +++ b/core/peerstatus/status_test.go @@ -131,8 +131,11 @@ func TestP2pStatus_SendPeerUpdate(t *testing.T) { f.Close(nil) checkStatus(t, "spaceId", f.p2pStatus, Connected) + // should not create a problem, cause we already closed + f.store.RemoveLocalPeer("peerId") + }) - t.Run("send NotConnected status, because we have peer were disconnected", func(t *testing.T) { + t.Run("send NotConnected status, because we have peer and then were disconnected", func(t *testing.T) { // given f := newFixture(t, "spaceId", pb.EventP2PStatus_Connected, 1) ctrl := gomock.NewController(t) @@ -313,6 +316,51 @@ func TestP2pStatus_SendPeerUpdate(t *testing.T) { }) } +func TestP2pStatus_SendToNewSession(t *testing.T) { + t.Run("send event only to new session", func(t *testing.T) { + // given + f := newFixture(t, "spaceId", pb.EventP2PStatus_Connected, 1) + ctrl := gomock.NewController(t) + peer := mock_peer.NewMockPeer(ctrl) + peer.EXPECT().Id().Return("peerId") + f.sender.EXPECT().Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{ + { + Value: &pb.EventMessageValueOfP2PStatusUpdate{ + P2PStatusUpdate: &pb.EventP2PStatusUpdate{ + SpaceId: "spaceId", + Status: pb.EventP2PStatus_Connected, + DevicesCounter: 1, + }, + }, + }, + }, + }) + err := f.pool.AddPeer(context.Background(), peer) + assert.Nil(t, err) + f.store.UpdateLocalPeer("peerId", []string{"spaceId"}) + checkStatus(t, "spaceId", f.p2pStatus, Connected) + + f.sender.EXPECT().SendToSession("token1", &pb.Event{ + Messages: []*pb.EventMessage{ + { + Value: &pb.EventMessageValueOfP2PStatusUpdate{ + P2PStatusUpdate: &pb.EventP2PStatusUpdate{ + SpaceId: "spaceId", + Status: pb.EventP2PStatus_Connected, + DevicesCounter: 1, + }, + }, + }, + }, + }) + err = f.sendStatusForNewSession(session.NewContext(session.WithSession("token1"))) + assert.Nil(t, err) + + // then + f.Close(nil) + }) +} func TestP2pStatus_UnregisterSpace(t *testing.T) { t.Run("success", func(t *testing.T) { // given From 3194aa4b79c17023e235e1a65c151d832ff3f5ad Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Wed, 24 Jul 2024 18:59:31 +0200 Subject: [PATCH 43/71] GO-3817 remove CheckPeerStatus call from peermanager --- core/peerstatus/status.go | 8 ++++---- core/peerstatus/status_test.go | 4 +++- space/spacecore/peermanager/manager.go | 12 +----------- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/core/peerstatus/status.go b/core/peerstatus/status.go index 0bd535cf7..50cc1dadb 100644 --- a/core/peerstatus/status.go +++ b/core/peerstatus/status.go @@ -98,8 +98,8 @@ func (p *p2pStatus) Init(a *app.App) (err error) { p.ctx, p.contextCancel = context.WithCancel(context.Background()) p.peerStore.AddObserver(func(peerId string, spaceIds []string, peerRemoved bool) { for _, spaceId := range spaceIds { - err = p.refreshPeerStatus(spaceId) - if err == ErrClosed { + err = p.refreshSpaceStatus(spaceId) + if errors.Is(err, ErrClosed) { return } if err != nil { @@ -137,7 +137,7 @@ func (p *p2pStatus) Name() (name string) { return CName } -func (p *p2pStatus) refreshPeerStatus(spaceId string) error { +func (p *p2pStatus) refreshSpaceStatus(spaceId string) error { select { case <-p.ctx.Done(): return ErrClosed @@ -195,7 +195,7 @@ func (p *p2pStatus) worker() { func (p *p2pStatus) updateAllSpacesP2PStatus() { p.Lock() var spaceIds = make([]string, 0, len(p.spaceIds)) - for spaceId, _ := range p.spaceIds { + for spaceId := range p.spaceIds { spaceIds = append(spaceIds, spaceId) } p.Unlock() diff --git a/core/peerstatus/status_test.go b/core/peerstatus/status_test.go index 07d4cfd40..0b455455a 100644 --- a/core/peerstatus/status_test.go +++ b/core/peerstatus/status_test.go @@ -84,7 +84,9 @@ func TestP2pStatus_SendNewStatus(t *testing.T) { }) f.resetNotPossibleStatus() - f.refreshPeerStatus("spaceId") + err = f.refreshSpaceStatus("spaceId") + assert.Nil(t, err) + checkStatus(t, "spaceId", f.p2pStatus, NotConnected) assert.Nil(t, err) diff --git a/space/spacecore/peermanager/manager.go b/space/spacecore/peermanager/manager.go index d2065898f..f2bdd8b9e 100644 --- a/space/spacecore/peermanager/manager.go +++ b/space/spacecore/peermanager/manager.go @@ -39,7 +39,6 @@ type Updater interface { } type PeerToPeerStatus interface { - CheckPeerStatus() RegisterSpace(spaceId string) UnregisterSpace(spaceId string) } @@ -170,20 +169,16 @@ func (n *clientPeerManager) getStreamResponsiblePeers(ctx context.Context) (peer peerIds = []string{p.Id()} } peerIds = append(peerIds, n.peerStore.LocalPeerIds(n.spaceId)...) - var needUpdate bool for _, peerId := range peerIds { p, err := n.p.pool.Get(ctx, peerId) if err != nil { n.peerStore.RemoveLocalPeer(peerId) log.Warn("failed to get peer from stream pool", zap.String("peerId", peerId), zap.Error(err)) - needUpdate = true continue } peers = append(peers, p) } - if needUpdate { - n.peerToPeerStatus.CheckPeerStatus() - } + // set node error if no local peers if len(peers) == 0 { err = fmt.Errorf("failed to get peers for stream") @@ -218,20 +213,15 @@ func (n *clientPeerManager) fetchResponsiblePeers() { } peerIds := n.peerStore.LocalPeerIds(n.spaceId) - var needUpdate bool for _, peerId := range peerIds { p, err := n.p.pool.Get(n.ctx, peerId) if err != nil { n.peerStore.RemoveLocalPeer(peerId) log.Warn("failed to get local from net pool", zap.String("peerId", peerId), zap.Error(err)) - needUpdate = true continue } peers = append(peers, p) } - if needUpdate { - n.peerToPeerStatus.CheckPeerStatus() - } n.Lock() defer n.Unlock() From e9df76f3340a298ec4e358792f0b6cfefc57bd63 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Wed, 24 Jul 2024 19:27:16 +0200 Subject: [PATCH 44/71] GO-3817 cleanup naming --- core/peerstatus/status.go | 46 ++++++++++++++++------------------ core/peerstatus/status_test.go | 2 +- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/core/peerstatus/status.go b/core/peerstatus/status.go index 50cc1dadb..7c2aa3434 100644 --- a/core/peerstatus/status.go +++ b/core/peerstatus/status.go @@ -97,16 +97,14 @@ func (p *p2pStatus) Init(a *app.App) (err error) { sessionHookRunner.RegisterHook(p.sendStatusForNewSession) p.ctx, p.contextCancel = context.WithCancel(context.Background()) p.peerStore.AddObserver(func(peerId string, spaceIds []string, peerRemoved bool) { - for _, spaceId := range spaceIds { - err = p.refreshSpaceStatus(spaceId) + go func() { + err := p.refreshSpaces(spaceIds) if errors.Is(err, ErrClosed) { return + } else if err != nil { + log.Errorf("refreshSpaces failed: %v", err) } - if err != nil { - // we don't have any other errors for now, added just in case for future - log.Error("failed to refresh peer status", "peerId", peerId, "spaceId", spaceId, "error", err) - } - } + }() }) return nil } @@ -137,28 +135,18 @@ func (p *p2pStatus) Name() (name string) { return CName } -func (p *p2pStatus) refreshSpaceStatus(spaceId string) error { - select { - case <-p.ctx.Done(): - return ErrClosed - case p.refreshSpaceId <- spaceId: - - } - return nil -} - func (p *p2pStatus) setNotPossibleStatus() { p.Lock() p.p2pNotPossible = true p.Unlock() - p.updateAllSpacesP2PStatus() + p.refreshAllSpaces() } func (p *p2pStatus) resetNotPossibleStatus() { p.Lock() p.p2pNotPossible = false p.Unlock() - p.updateAllSpacesP2PStatus() + p.refreshAllSpaces() } func (p *p2pStatus) RegisterSpace(spaceId string) { @@ -184,33 +172,43 @@ func (p *p2pStatus) worker() { case <-p.ctx.Done(): return case spaceId := <-p.refreshSpaceId: - p.updateSpaceP2PStatus(spaceId) + p.processSpaceStatusUpdate(spaceId) case <-timer.C: // todo: looks like we don't need this anymore because we use observer - p.updateAllSpacesP2PStatus() + p.refreshAllSpaces() } } } -func (p *p2pStatus) updateAllSpacesP2PStatus() { +func (p *p2pStatus) refreshAllSpaces() { p.Lock() var spaceIds = make([]string, 0, len(p.spaceIds)) for spaceId := range p.spaceIds { spaceIds = append(spaceIds, spaceId) } p.Unlock() + err := p.refreshSpaces(spaceIds) + if errors.Is(err, ErrClosed) { + return + } else if err != nil { + log.Errorf("refreshSpaces failed: %v", err) + } +} + +func (p *p2pStatus) refreshSpaces(spaceIds []string) error { for _, spaceId := range spaceIds { select { case <-p.ctx.Done(): - return + return ErrClosed case p.refreshSpaceId <- spaceId: } } + return nil } // updateSpaceP2PStatus updates status for specific spaceId and sends event if status changed -func (p *p2pStatus) updateSpaceP2PStatus(spaceId string) { +func (p *p2pStatus) processSpaceStatusUpdate(spaceId string) { p.Lock() defer p.Unlock() var ( diff --git a/core/peerstatus/status_test.go b/core/peerstatus/status_test.go index 0b455455a..a43c7593b 100644 --- a/core/peerstatus/status_test.go +++ b/core/peerstatus/status_test.go @@ -84,7 +84,7 @@ func TestP2pStatus_SendNewStatus(t *testing.T) { }) f.resetNotPossibleStatus() - err = f.refreshSpaceStatus("spaceId") + err = f.refreshSpaces([]string{"spaceId"}) assert.Nil(t, err) checkStatus(t, "spaceId", f.p2pStatus, NotConnected) From 0af6d4e74327164628a18ca66c0962d61c8fe931 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Wed, 24 Jul 2024 19:35:44 +0200 Subject: [PATCH 45/71] GO-3817 peerstore observer: add before ids --- core/filestorage/rpcstore/service.go | 2 +- core/peerstatus/status.go | 5 +++-- space/spacecore/peerstore/peerstore.go | 12 ++++++++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/core/filestorage/rpcstore/service.go b/core/filestorage/rpcstore/service.go index e59ddde81..f01ee1b0b 100644 --- a/core/filestorage/rpcstore/service.go +++ b/core/filestorage/rpcstore/service.go @@ -30,7 +30,7 @@ type service struct { func (s *service) Init(a *app.App) (err error) { s.pool = a.MustComponent(pool.CName).(pool.Pool) s.peerStore = a.MustComponent(peerstore.CName).(peerstore.PeerStore) - s.peerStore.AddObserver(func(peerId string, spaceIds []string, peerRemoved bool) { + s.peerStore.AddObserver(func(peerId string, _, spaceIds []string, peerRemoved bool) { select { case s.peerUpdateCh <- struct{}{}: default: diff --git a/core/peerstatus/status.go b/core/peerstatus/status.go index 7c2aa3434..3c375dfb6 100644 --- a/core/peerstatus/status.go +++ b/core/peerstatus/status.go @@ -8,6 +8,7 @@ import ( "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/net/pool" + "github.com/samber/lo" "github.com/anyproto/anytype-heart/core/event" "github.com/anyproto/anytype-heart/core/session" @@ -96,9 +97,9 @@ func (p *p2pStatus) Init(a *app.App) (err error) { localDiscoveryHook.RegisterResetNotPossible(p.resetNotPossibleStatus) sessionHookRunner.RegisterHook(p.sendStatusForNewSession) p.ctx, p.contextCancel = context.WithCancel(context.Background()) - p.peerStore.AddObserver(func(peerId string, spaceIds []string, peerRemoved bool) { + p.peerStore.AddObserver(func(peerId string, spaceIdsBefore, spaceIdsAfter []string, peerRemoved bool) { go func() { - err := p.refreshSpaces(spaceIds) + err := p.refreshSpaces(lo.Union(spaceIdsBefore, spaceIdsAfter)) if errors.Is(err, ErrClosed) { return } else if err != nil { diff --git a/space/spacecore/peerstore/peerstore.go b/space/spacecore/peerstore/peerstore.go index ccdf36535..039e2522d 100644 --- a/space/spacecore/peerstore/peerstore.go +++ b/space/spacecore/peerstore/peerstore.go @@ -33,7 +33,7 @@ func New() PeerStore { } // Observer is a function that will be called when a peer is updated -type Observer func(peerId string, spaceIds []string, peerRemoved bool) +type Observer func(peerId string, spaceIdsBefore []string, spaceIdsAfter []string, peerRemoved bool) type peerStore struct { nodeConf nodeconf.Service @@ -62,6 +62,10 @@ func (p *peerStore) AddObserver(observer Observer) { func (p *peerStore) UpdateLocalPeer(peerId string, spaceIds []string) { notify := true p.Lock() + var ( + oldIds []string + ok bool + ) defer func() { observers := p.observers p.Unlock() @@ -70,10 +74,10 @@ func (p *peerStore) UpdateLocalPeer(peerId string, spaceIds []string) { } for _, ob := range observers { - ob(peerId, spaceIds, false) + ob(peerId, oldIds, spaceIds, false) } }() - if oldIds, ok := p.spacesByLocalPeerIds[peerId]; ok { + if oldIds, ok = p.spacesByLocalPeerIds[peerId]; ok { slices.Sort(oldIds) slices.Sort(spaceIds) if slices.Equal(oldIds, spaceIds) { @@ -135,7 +139,7 @@ func (p *peerStore) RemoveLocalPeer(peerId string) { defer func() { observers := p.observers for _, ob := range observers { - ob(peerId, spaceIds, true) + ob(peerId, spaceIds, nil, true) } }() // TODO: do we need to notify observer here From 5d3dea4b685cbb07caaab5ac57d85f326c885641 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Wed, 24 Jul 2024 19:42:27 +0200 Subject: [PATCH 46/71] GO-3817 optimisation --- core/peerstatus/status.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/peerstatus/status.go b/core/peerstatus/status.go index 3c375dfb6..7c0ce5431 100644 --- a/core/peerstatus/status.go +++ b/core/peerstatus/status.go @@ -99,7 +99,10 @@ func (p *p2pStatus) Init(a *app.App) (err error) { p.ctx, p.contextCancel = context.WithCancel(context.Background()) p.peerStore.AddObserver(func(peerId string, spaceIdsBefore, spaceIdsAfter []string, peerRemoved bool) { go func() { - err := p.refreshSpaces(lo.Union(spaceIdsBefore, spaceIdsAfter)) + // we need to update status for all spaces that were either added or removed to some local peer + // because we start this observer on init we can be sure that the spaceIdsBefore is empty on the first run for peer + removed, added := lo.Difference(spaceIdsBefore, spaceIdsAfter) + err := p.refreshSpaces(lo.Union(removed, added)) if errors.Is(err, ErrClosed) { return } else if err != nil { From e65eb88627fc68e28d96581ffba85795151034e6 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Wed, 24 Jul 2024 20:19:38 +0200 Subject: [PATCH 47/71] GO-3817 fix some deadlocks --- core/peerstatus/status.go | 13 ++++++++++++- space/spacecore/peermanager/manager.go | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/core/peerstatus/status.go b/core/peerstatus/status.go index 7c0ce5431..0d2ed2c8e 100644 --- a/core/peerstatus/status.go +++ b/core/peerstatus/status.go @@ -102,6 +102,7 @@ func (p *p2pStatus) Init(a *app.App) (err error) { // we need to update status for all spaces that were either added or removed to some local peer // because we start this observer on init we can be sure that the spaceIdsBefore is empty on the first run for peer removed, added := lo.Difference(spaceIdsBefore, spaceIdsAfter) + log.Warnf("peer %s space observer: removed: %v, added: %v", peerId, removed, added) err := p.refreshSpaces(lo.Union(removed, added)) if errors.Is(err, ErrClosed) { return @@ -153,6 +154,8 @@ func (p *p2pStatus) resetNotPossibleStatus() { p.refreshAllSpaces() } +// RegisterSpace registers spaceId to be monitored for p2p status changes +// must be called only when p2pStatus is Running func (p *p2pStatus) RegisterSpace(spaceId string) { select { case <-p.ctx.Done(): @@ -161,6 +164,8 @@ func (p *p2pStatus) RegisterSpace(spaceId string) { } } +// UnregisterSpace unregisters spaceId from monitoring +// must be called only when p2pStatus is Running func (p *p2pStatus) UnregisterSpace(spaceId string) { p.Lock() defer p.Unlock() @@ -179,7 +184,7 @@ func (p *p2pStatus) worker() { p.processSpaceStatusUpdate(spaceId) case <-timer.C: // todo: looks like we don't need this anymore because we use observer - p.refreshAllSpaces() + go p.refreshAllSpaces() } } } @@ -230,6 +235,8 @@ func (p *p2pStatus) processSpaceStatusUpdate(spaceId string) { connectionCount := p.countOpenConnections(spaceId) newStatus := p.getResultStatus(p.p2pNotPossible, connectionCount) + log.Warnf("processSpaceStatusUpdate: spaceId: %s, currentStatus: %v, newStatus: %v, connectionCount: %d", spaceId, currentStatus.status, newStatus, connectionCount) + if currentStatus.status != newStatus || currentStatus.connectionsCount != connectionCount { p.sendEvent("", spaceId, newStatus.ToPb(), connectionCount) currentStatus.status = newStatus @@ -253,9 +260,12 @@ func (p *p2pStatus) countOpenConnections(spaceId string) int64 { ctx, cancelFunc := context.WithTimeout(p.ctx, time.Second*10) defer cancelFunc() peerIds := p.peerStore.LocalPeerIds(spaceId) + log.Warnf("processSpaceStatusUpdate: spaceId: %s, localPeerIds: %v", spaceId, peerIds) + for _, peerId := range peerIds { _, err := p.peersConnectionPool.Pick(ctx, peerId) if err != nil { + log.Warnf("countOpenConnections space %s failed to get local peer %s from net pool: %v", spaceId, peerId, err) continue } connectionCount++ @@ -265,6 +275,7 @@ func (p *p2pStatus) countOpenConnections(spaceId string) int64 { // sendEvent sends event to session with sessionToken or broadcast to all sessions if sessionToken is empty func (p *p2pStatus) sendEvent(sessionToken string, spaceId string, status pb.EventP2PStatusStatus, count int64) { + log.Warnf("sendEvent: sessionToken: %s, spaceId: %s, status: %v, count: %d", sessionToken, spaceId, status, count) event := &pb.Event{ Messages: []*pb.EventMessage{ { diff --git a/space/spacecore/peermanager/manager.go b/space/spacecore/peermanager/manager.go index f2bdd8b9e..c11567a28 100644 --- a/space/spacecore/peermanager/manager.go +++ b/space/spacecore/peermanager/manager.go @@ -72,7 +72,6 @@ func (n *clientPeerManager) Init(a *app.App) (err error) { n.nodeStatus = app.MustComponent[NodeStatus](a) n.spaceSyncService = app.MustComponent[Updater](a) n.peerToPeerStatus = app.MustComponent[PeerToPeerStatus](a) - n.peerToPeerStatus.RegisterSpace(n.spaceId) return } @@ -81,6 +80,7 @@ func (n *clientPeerManager) Name() (name string) { } func (n *clientPeerManager) Run(ctx context.Context) (err error) { + go n.peerToPeerStatus.RegisterSpace(n.spaceId) go n.manageResponsiblePeers() return } From ad499eae18a2f7da08ad8f81b71492f3216e2685 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Wed, 24 Jul 2024 21:02:48 +0200 Subject: [PATCH 48/71] GO-3769 Test space status --- .mockery.yaml | 5 + core/subscription/fixture.go | 12 + .../mock_nodestatus/mock_NodeStatus.go | 209 ++++ .../mock_NetworkConfig.go | 173 ++++ .../mock_spacesyncstatus/mock_NodeUsage.go | 189 ++++ .../syncstatus/spacesyncstatus/spacestatus.go | 38 +- .../spacesyncstatus/spacestatus_test.go | 948 ++++++++---------- 7 files changed, 1024 insertions(+), 550 deletions(-) create mode 100644 core/syncstatus/nodestatus/mock_nodestatus/mock_NodeStatus.go create mode 100644 core/syncstatus/spacesyncstatus/mock_spacesyncstatus/mock_NetworkConfig.go create mode 100644 core/syncstatus/spacesyncstatus/mock_spacesyncstatus/mock_NodeUsage.go diff --git a/.mockery.yaml b/.mockery.yaml index 3db53eb15..e0b1b83d3 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -192,6 +192,9 @@ packages: interfaces: PeerStatusChecker: SyncDetailsUpdater: + github.com/anyproto/anytype-heart/core/syncstatus/nodestatus: + interfaces: + NodeStatus: github.com/anyproto/anytype-heart/core/syncstatus/objectsyncstatus: interfaces: Updater: @@ -210,4 +213,6 @@ packages: github.com/anyproto/anytype-heart/core/syncstatus/spacesyncstatus: interfaces: SpaceIdGetter: + NodeUsage: + NetworkConfig: Updater: \ No newline at end of file diff --git a/core/subscription/fixture.go b/core/subscription/fixture.go index 99ebd6e6d..0d6b028ef 100644 --- a/core/subscription/fixture.go +++ b/core/subscription/fixture.go @@ -52,6 +52,18 @@ func NewInternalTestService(t *testing.T) *InternalTestService { return &InternalTestService{Service: s, StoreFixture: objectStore} } +func RegisterSubscriptionService(t *testing.T, a *app.App) *InternalTestService { + s := New() + ctx := context.Background() + objectStore := objectstore.NewStoreFixture(t) + a.Register(objectStore). + Register(kanban.New()). + Register(&collectionServiceMock{MockCollectionService: NewMockCollectionService(t)}). + Register(testutil.PrepareMock(ctx, a, mock_event.NewMockSender(t))). + Register(s) + return &InternalTestService{Service: s, StoreFixture: objectStore} +} + type collectionServiceMock struct { *MockCollectionService } diff --git a/core/syncstatus/nodestatus/mock_nodestatus/mock_NodeStatus.go b/core/syncstatus/nodestatus/mock_nodestatus/mock_NodeStatus.go new file mode 100644 index 000000000..c2f1d16ec --- /dev/null +++ b/core/syncstatus/nodestatus/mock_nodestatus/mock_NodeStatus.go @@ -0,0 +1,209 @@ +// Code generated by mockery. DO NOT EDIT. + +package mock_nodestatus + +import ( + app "github.com/anyproto/any-sync/app" + mock "github.com/stretchr/testify/mock" + + nodestatus "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" +) + +// MockNodeStatus is an autogenerated mock type for the NodeStatus type +type MockNodeStatus struct { + mock.Mock +} + +type MockNodeStatus_Expecter struct { + mock *mock.Mock +} + +func (_m *MockNodeStatus) EXPECT() *MockNodeStatus_Expecter { + return &MockNodeStatus_Expecter{mock: &_m.Mock} +} + +// GetNodeStatus provides a mock function with given fields: spaceId +func (_m *MockNodeStatus) GetNodeStatus(spaceId string) nodestatus.ConnectionStatus { + ret := _m.Called(spaceId) + + if len(ret) == 0 { + panic("no return value specified for GetNodeStatus") + } + + var r0 nodestatus.ConnectionStatus + if rf, ok := ret.Get(0).(func(string) nodestatus.ConnectionStatus); ok { + r0 = rf(spaceId) + } else { + r0 = ret.Get(0).(nodestatus.ConnectionStatus) + } + + return r0 +} + +// MockNodeStatus_GetNodeStatus_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetNodeStatus' +type MockNodeStatus_GetNodeStatus_Call struct { + *mock.Call +} + +// GetNodeStatus is a helper method to define mock.On call +// - spaceId string +func (_e *MockNodeStatus_Expecter) GetNodeStatus(spaceId interface{}) *MockNodeStatus_GetNodeStatus_Call { + return &MockNodeStatus_GetNodeStatus_Call{Call: _e.mock.On("GetNodeStatus", spaceId)} +} + +func (_c *MockNodeStatus_GetNodeStatus_Call) Run(run func(spaceId string)) *MockNodeStatus_GetNodeStatus_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string)) + }) + return _c +} + +func (_c *MockNodeStatus_GetNodeStatus_Call) Return(_a0 nodestatus.ConnectionStatus) *MockNodeStatus_GetNodeStatus_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockNodeStatus_GetNodeStatus_Call) RunAndReturn(run func(string) nodestatus.ConnectionStatus) *MockNodeStatus_GetNodeStatus_Call { + _c.Call.Return(run) + return _c +} + +// Init provides a mock function with given fields: a +func (_m *MockNodeStatus) Init(a *app.App) error { + ret := _m.Called(a) + + if len(ret) == 0 { + panic("no return value specified for Init") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*app.App) error); ok { + r0 = rf(a) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockNodeStatus_Init_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Init' +type MockNodeStatus_Init_Call struct { + *mock.Call +} + +// Init is a helper method to define mock.On call +// - a *app.App +func (_e *MockNodeStatus_Expecter) Init(a interface{}) *MockNodeStatus_Init_Call { + return &MockNodeStatus_Init_Call{Call: _e.mock.On("Init", a)} +} + +func (_c *MockNodeStatus_Init_Call) Run(run func(a *app.App)) *MockNodeStatus_Init_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*app.App)) + }) + return _c +} + +func (_c *MockNodeStatus_Init_Call) Return(err error) *MockNodeStatus_Init_Call { + _c.Call.Return(err) + return _c +} + +func (_c *MockNodeStatus_Init_Call) RunAndReturn(run func(*app.App) error) *MockNodeStatus_Init_Call { + _c.Call.Return(run) + return _c +} + +// Name provides a mock function with given fields: +func (_m *MockNodeStatus) Name() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Name") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// MockNodeStatus_Name_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Name' +type MockNodeStatus_Name_Call struct { + *mock.Call +} + +// Name is a helper method to define mock.On call +func (_e *MockNodeStatus_Expecter) Name() *MockNodeStatus_Name_Call { + return &MockNodeStatus_Name_Call{Call: _e.mock.On("Name")} +} + +func (_c *MockNodeStatus_Name_Call) Run(run func()) *MockNodeStatus_Name_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockNodeStatus_Name_Call) Return(name string) *MockNodeStatus_Name_Call { + _c.Call.Return(name) + return _c +} + +func (_c *MockNodeStatus_Name_Call) RunAndReturn(run func() string) *MockNodeStatus_Name_Call { + _c.Call.Return(run) + return _c +} + +// SetNodesStatus provides a mock function with given fields: spaceId, senderId, status +func (_m *MockNodeStatus) SetNodesStatus(spaceId string, senderId string, status nodestatus.ConnectionStatus) { + _m.Called(spaceId, senderId, status) +} + +// MockNodeStatus_SetNodesStatus_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetNodesStatus' +type MockNodeStatus_SetNodesStatus_Call struct { + *mock.Call +} + +// SetNodesStatus is a helper method to define mock.On call +// - spaceId string +// - senderId string +// - status nodestatus.ConnectionStatus +func (_e *MockNodeStatus_Expecter) SetNodesStatus(spaceId interface{}, senderId interface{}, status interface{}) *MockNodeStatus_SetNodesStatus_Call { + return &MockNodeStatus_SetNodesStatus_Call{Call: _e.mock.On("SetNodesStatus", spaceId, senderId, status)} +} + +func (_c *MockNodeStatus_SetNodesStatus_Call) Run(run func(spaceId string, senderId string, status nodestatus.ConnectionStatus)) *MockNodeStatus_SetNodesStatus_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string), args[2].(nodestatus.ConnectionStatus)) + }) + return _c +} + +func (_c *MockNodeStatus_SetNodesStatus_Call) Return() *MockNodeStatus_SetNodesStatus_Call { + _c.Call.Return() + return _c +} + +func (_c *MockNodeStatus_SetNodesStatus_Call) RunAndReturn(run func(string, string, nodestatus.ConnectionStatus)) *MockNodeStatus_SetNodesStatus_Call { + _c.Call.Return(run) + return _c +} + +// NewMockNodeStatus creates a new instance of MockNodeStatus. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockNodeStatus(t interface { + mock.TestingT + Cleanup(func()) +}) *MockNodeStatus { + mock := &MockNodeStatus{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/syncstatus/spacesyncstatus/mock_spacesyncstatus/mock_NetworkConfig.go b/core/syncstatus/spacesyncstatus/mock_spacesyncstatus/mock_NetworkConfig.go new file mode 100644 index 000000000..8f6cdcfbd --- /dev/null +++ b/core/syncstatus/spacesyncstatus/mock_spacesyncstatus/mock_NetworkConfig.go @@ -0,0 +1,173 @@ +// Code generated by mockery. DO NOT EDIT. + +package mock_spacesyncstatus + +import ( + app "github.com/anyproto/any-sync/app" + mock "github.com/stretchr/testify/mock" + + pb "github.com/anyproto/anytype-heart/pb" +) + +// MockNetworkConfig is an autogenerated mock type for the NetworkConfig type +type MockNetworkConfig struct { + mock.Mock +} + +type MockNetworkConfig_Expecter struct { + mock *mock.Mock +} + +func (_m *MockNetworkConfig) EXPECT() *MockNetworkConfig_Expecter { + return &MockNetworkConfig_Expecter{mock: &_m.Mock} +} + +// GetNetworkMode provides a mock function with given fields: +func (_m *MockNetworkConfig) GetNetworkMode() pb.RpcAccountNetworkMode { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetNetworkMode") + } + + var r0 pb.RpcAccountNetworkMode + if rf, ok := ret.Get(0).(func() pb.RpcAccountNetworkMode); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(pb.RpcAccountNetworkMode) + } + + return r0 +} + +// MockNetworkConfig_GetNetworkMode_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetNetworkMode' +type MockNetworkConfig_GetNetworkMode_Call struct { + *mock.Call +} + +// GetNetworkMode is a helper method to define mock.On call +func (_e *MockNetworkConfig_Expecter) GetNetworkMode() *MockNetworkConfig_GetNetworkMode_Call { + return &MockNetworkConfig_GetNetworkMode_Call{Call: _e.mock.On("GetNetworkMode")} +} + +func (_c *MockNetworkConfig_GetNetworkMode_Call) Run(run func()) *MockNetworkConfig_GetNetworkMode_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockNetworkConfig_GetNetworkMode_Call) Return(_a0 pb.RpcAccountNetworkMode) *MockNetworkConfig_GetNetworkMode_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockNetworkConfig_GetNetworkMode_Call) RunAndReturn(run func() pb.RpcAccountNetworkMode) *MockNetworkConfig_GetNetworkMode_Call { + _c.Call.Return(run) + return _c +} + +// Init provides a mock function with given fields: a +func (_m *MockNetworkConfig) Init(a *app.App) error { + ret := _m.Called(a) + + if len(ret) == 0 { + panic("no return value specified for Init") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*app.App) error); ok { + r0 = rf(a) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockNetworkConfig_Init_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Init' +type MockNetworkConfig_Init_Call struct { + *mock.Call +} + +// Init is a helper method to define mock.On call +// - a *app.App +func (_e *MockNetworkConfig_Expecter) Init(a interface{}) *MockNetworkConfig_Init_Call { + return &MockNetworkConfig_Init_Call{Call: _e.mock.On("Init", a)} +} + +func (_c *MockNetworkConfig_Init_Call) Run(run func(a *app.App)) *MockNetworkConfig_Init_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*app.App)) + }) + return _c +} + +func (_c *MockNetworkConfig_Init_Call) Return(err error) *MockNetworkConfig_Init_Call { + _c.Call.Return(err) + return _c +} + +func (_c *MockNetworkConfig_Init_Call) RunAndReturn(run func(*app.App) error) *MockNetworkConfig_Init_Call { + _c.Call.Return(run) + return _c +} + +// Name provides a mock function with given fields: +func (_m *MockNetworkConfig) Name() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Name") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// MockNetworkConfig_Name_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Name' +type MockNetworkConfig_Name_Call struct { + *mock.Call +} + +// Name is a helper method to define mock.On call +func (_e *MockNetworkConfig_Expecter) Name() *MockNetworkConfig_Name_Call { + return &MockNetworkConfig_Name_Call{Call: _e.mock.On("Name")} +} + +func (_c *MockNetworkConfig_Name_Call) Run(run func()) *MockNetworkConfig_Name_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockNetworkConfig_Name_Call) Return(name string) *MockNetworkConfig_Name_Call { + _c.Call.Return(name) + return _c +} + +func (_c *MockNetworkConfig_Name_Call) RunAndReturn(run func() string) *MockNetworkConfig_Name_Call { + _c.Call.Return(run) + return _c +} + +// NewMockNetworkConfig creates a new instance of MockNetworkConfig. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockNetworkConfig(t interface { + mock.TestingT + Cleanup(func()) +}) *MockNetworkConfig { + mock := &MockNetworkConfig{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/syncstatus/spacesyncstatus/mock_spacesyncstatus/mock_NodeUsage.go b/core/syncstatus/spacesyncstatus/mock_spacesyncstatus/mock_NodeUsage.go new file mode 100644 index 000000000..3f8078a95 --- /dev/null +++ b/core/syncstatus/spacesyncstatus/mock_spacesyncstatus/mock_NodeUsage.go @@ -0,0 +1,189 @@ +// Code generated by mockery. DO NOT EDIT. + +package mock_spacesyncstatus + +import ( + context "context" + + app "github.com/anyproto/any-sync/app" + + files "github.com/anyproto/anytype-heart/core/files" + + mock "github.com/stretchr/testify/mock" +) + +// MockNodeUsage is an autogenerated mock type for the NodeUsage type +type MockNodeUsage struct { + mock.Mock +} + +type MockNodeUsage_Expecter struct { + mock *mock.Mock +} + +func (_m *MockNodeUsage) EXPECT() *MockNodeUsage_Expecter { + return &MockNodeUsage_Expecter{mock: &_m.Mock} +} + +// GetNodeUsage provides a mock function with given fields: ctx +func (_m *MockNodeUsage) GetNodeUsage(ctx context.Context) (*files.NodeUsageResponse, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for GetNodeUsage") + } + + var r0 *files.NodeUsageResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (*files.NodeUsageResponse, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) *files.NodeUsageResponse); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*files.NodeUsageResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockNodeUsage_GetNodeUsage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetNodeUsage' +type MockNodeUsage_GetNodeUsage_Call struct { + *mock.Call +} + +// GetNodeUsage is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockNodeUsage_Expecter) GetNodeUsage(ctx interface{}) *MockNodeUsage_GetNodeUsage_Call { + return &MockNodeUsage_GetNodeUsage_Call{Call: _e.mock.On("GetNodeUsage", ctx)} +} + +func (_c *MockNodeUsage_GetNodeUsage_Call) Run(run func(ctx context.Context)) *MockNodeUsage_GetNodeUsage_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockNodeUsage_GetNodeUsage_Call) Return(_a0 *files.NodeUsageResponse, _a1 error) *MockNodeUsage_GetNodeUsage_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockNodeUsage_GetNodeUsage_Call) RunAndReturn(run func(context.Context) (*files.NodeUsageResponse, error)) *MockNodeUsage_GetNodeUsage_Call { + _c.Call.Return(run) + return _c +} + +// Init provides a mock function with given fields: a +func (_m *MockNodeUsage) Init(a *app.App) error { + ret := _m.Called(a) + + if len(ret) == 0 { + panic("no return value specified for Init") + } + + var r0 error + if rf, ok := ret.Get(0).(func(*app.App) error); ok { + r0 = rf(a) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockNodeUsage_Init_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Init' +type MockNodeUsage_Init_Call struct { + *mock.Call +} + +// Init is a helper method to define mock.On call +// - a *app.App +func (_e *MockNodeUsage_Expecter) Init(a interface{}) *MockNodeUsage_Init_Call { + return &MockNodeUsage_Init_Call{Call: _e.mock.On("Init", a)} +} + +func (_c *MockNodeUsage_Init_Call) Run(run func(a *app.App)) *MockNodeUsage_Init_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*app.App)) + }) + return _c +} + +func (_c *MockNodeUsage_Init_Call) Return(err error) *MockNodeUsage_Init_Call { + _c.Call.Return(err) + return _c +} + +func (_c *MockNodeUsage_Init_Call) RunAndReturn(run func(*app.App) error) *MockNodeUsage_Init_Call { + _c.Call.Return(run) + return _c +} + +// Name provides a mock function with given fields: +func (_m *MockNodeUsage) Name() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Name") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// MockNodeUsage_Name_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Name' +type MockNodeUsage_Name_Call struct { + *mock.Call +} + +// Name is a helper method to define mock.On call +func (_e *MockNodeUsage_Expecter) Name() *MockNodeUsage_Name_Call { + return &MockNodeUsage_Name_Call{Call: _e.mock.On("Name")} +} + +func (_c *MockNodeUsage_Name_Call) Run(run func()) *MockNodeUsage_Name_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockNodeUsage_Name_Call) Return(name string) *MockNodeUsage_Name_Call { + _c.Call.Return(name) + return _c +} + +func (_c *MockNodeUsage_Name_Call) RunAndReturn(run func() string) *MockNodeUsage_Name_Call { + _c.Call.Return(run) + return _c +} + +// NewMockNodeUsage creates a new instance of MockNodeUsage. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockNodeUsage(t interface { + mock.TestingT + Cleanup(func()) +}) *MockNodeUsage { + mock := &MockNodeUsage{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/syncstatus/spacesyncstatus/spacestatus.go b/core/syncstatus/spacesyncstatus/spacestatus.go index 1009bbeb6..42febcede 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus.go +++ b/core/syncstatus/spacesyncstatus/spacestatus.go @@ -37,6 +37,7 @@ type Updater interface { } type NodeUsage interface { + app.Component GetNodeUsage(ctx context.Context) (*files.NodeUsageResponse, error) } @@ -56,6 +57,7 @@ type State interface { } type NetworkConfig interface { + app.Component GetNetworkMode() pb.RpcAccountNetworkMode } @@ -80,12 +82,17 @@ type spaceSyncStatus struct { missingIds map[string][]string mx sync.Mutex periodicCall periodicsync.PeriodicSync + loopInterval time.Duration isLocal bool finish chan struct{} } func NewSpaceSyncStatus() Updater { - return &spaceSyncStatus{batcher: mb.New[*domain.SpaceSync](0), finish: make(chan struct{})} + return &spaceSyncStatus{ + batcher: mb.New[*domain.SpaceSync](0), + finish: make(chan struct{}), + loopInterval: time.Second * 1, + } } func (s *spaceSyncStatus) Init(a *app.App) (err error) { @@ -103,7 +110,7 @@ func (s *spaceSyncStatus) Init(a *app.App) (err error) { s.isLocal = s.networkConfig.GetNetworkMode() == pb.RpcAccount_LocalOnly sessionHookRunner := app.MustComponent[session.HookRunner](a) sessionHookRunner.RegisterHook(s.sendSyncEventForNewSession) - s.periodicCall = periodicsync.NewPeriodicSync(1, time.Second*5, s.update, logger.CtxLogger{Logger: log.Desugar()}) + s.periodicCall = periodicsync.NewPeriodicSyncDuration(s.loopInterval, time.Second*5, s.update, logger.CtxLogger{Logger: log.Desugar()}) return } @@ -132,12 +139,14 @@ func (s *spaceSyncStatus) Run(ctx context.Context) (err error) { return } -func (s *spaceSyncStatus) update(ctx context.Context) error { - // TODO: use subscriptions inside middleware instead of this +func (s *spaceSyncStatus) getMissingIds(spaceId string) []string { + s.mx.Lock() + defer s.mx.Unlock() + return slice.Copy(s.missingIds[spaceId]) +} + +func (s *spaceSyncStatus) update(ctx context.Context) error { s.mx.Lock() - missingIds := lo.MapEntries(s.missingIds, func(key string, value []string) (string, []string) { - return key, slice.Copy(value) - }) statuses := lo.MapToSlice(s.curStatuses, func(key string, value struct{}) string { delete(s.curStatuses, key) return key @@ -149,15 +158,12 @@ func (s *spaceSyncStatus) update(ctx context.Context) error { } // if the there are too many updates and this hurts performance, // we may skip some iterations and not do the updates for example - s.updateSpaceSyncStatus(spaceId, missingIds[spaceId]) + s.updateSpaceSyncStatus(spaceId) } return nil } func (s *spaceSyncStatus) sendEventToSession(spaceId, token string) { - s.mx.Lock() - missingObjects := s.missingIds[spaceId] - s.mx.Unlock() if s.isLocal { s.sendLocalOnlyEventToSession(spaceId, token) return @@ -167,7 +173,7 @@ func (s *spaceSyncStatus) sendEventToSession(spaceId, token string) { connectionStatus: s.nodeStatus.GetNodeStatus(spaceId), compatibility: s.nodeConf.NetworkCompatibilityStatus(), filesSyncingCount: s.getFileSyncingObjectsCount(spaceId), - objectsSyncingCount: s.getObjectSyncingObjectsCount(spaceId, missingObjects), + objectsSyncingCount: s.getObjectSyncingObjectsCount(spaceId, s.getMissingIds(spaceId)), } s.eventSender.SendToSession(token, &pb.Event{ Messages: []*pb.EventMessage{{ @@ -180,10 +186,7 @@ func (s *spaceSyncStatus) sendEventToSession(spaceId, token string) { func (s *spaceSyncStatus) sendStartEvent(spaceIds []string) { for _, id := range spaceIds { - s.mx.Lock() - missingObjects := s.missingIds[id] - s.mx.Unlock() - s.updateSpaceSyncStatus(id, missingObjects) + s.updateSpaceSyncStatus(id) } } @@ -248,11 +251,12 @@ func (s *spaceSyncStatus) getBytesLeftPercentage(spaceId string) float64 { return float64(nodeUsage.Usage.BytesLeft) / float64(nodeUsage.Usage.AccountBytesLimit) } -func (s *spaceSyncStatus) updateSpaceSyncStatus(spaceId string, missingObjects []string) { +func (s *spaceSyncStatus) updateSpaceSyncStatus(spaceId string) { if s.isLocal { s.sendLocalOnlyEvent(spaceId) return } + missingObjects := s.getMissingIds(spaceId) params := syncParams{ bytesLeftPercentage: s.getBytesLeftPercentage(spaceId), connectionStatus: s.nodeStatus.GetNodeStatus(spaceId), diff --git a/core/syncstatus/spacesyncstatus/spacestatus_test.go b/core/syncstatus/spacesyncstatus/spacestatus_test.go index 6f3db0f0b..982a5de72 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus_test.go +++ b/core/syncstatus/spacesyncstatus/spacestatus_test.go @@ -2,577 +2,459 @@ package spacesyncstatus import ( "context" + "fmt" + "math/rand" "testing" + "time" "github.com/anyproto/any-sync/app" - "github.com/cheggaaa/mb/v3" - "github.com/stretchr/testify/assert" + "github.com/anyproto/any-sync/nodeconf" + "github.com/anyproto/any-sync/nodeconf/mock_nodeconf" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" - "github.com/anyproto/anytype-heart/core/anytype/config" "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/core/event" "github.com/anyproto/anytype-heart/core/event/mock_event" + "github.com/anyproto/anytype-heart/core/files" + "github.com/anyproto/anytype-heart/core/filestorage/filesync" "github.com/anyproto/anytype-heart/core/session" + "github.com/anyproto/anytype-heart/core/subscription" "github.com/anyproto/anytype-heart/core/syncstatus/filesyncstatus" + "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" + "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus/mock_nodestatus" "github.com/anyproto/anytype-heart/core/syncstatus/spacesyncstatus/mock_spacesyncstatus" + "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscritions" "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/bundle" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" "github.com/anyproto/anytype-heart/tests/testutil" "github.com/anyproto/anytype-heart/util/pbtypes" ) -func TestSpaceSyncStatus_Init(t *testing.T) { - t.Run("init", func(t *testing.T) { - // given - status := NewSpaceSyncStatus() - ctx := context.Background() +var ctx = context.Background() - a := new(app.App) - eventSender := mock_event.NewMockSender(t) - space := mock_spacesyncstatus.NewMockSpaceIdGetter(t) +type mockSessionContext struct { + id string +} - a.Register(testutil.PrepareMock(ctx, a, eventSender)). - Register(objectstore.NewStoreFixture(t)). - Register(&config.Config{NetworkMode: pb.RpcAccount_DefaultConfig}). - Register(testutil.PrepareMock(ctx, a, space)). - Register(session.NewHookRunner()) +func (m mockSessionContext) ID() string { + return m.id +} - // when - err := status.Init(a) +func (m mockSessionContext) ObjectID() string { + panic("implement me") +} - // then - assert.Nil(t, err) +func (m mockSessionContext) TraceID() string { + panic("implement me") +} - space.EXPECT().AllSpaceIds().Return([]string{"personalId"}) - eventSender.EXPECT().Broadcast(&pb.Event{ +func (m mockSessionContext) SetMessages(smartBlockId string, msgs []*pb.EventMessage) { + panic("implement me") +} + +func (m mockSessionContext) GetMessages() []*pb.EventMessage { + return nil +} + +func (m mockSessionContext) GetResponseEvent() *pb.ResponseEvent { + panic("implement me") +} + +type fixture struct { + *spaceSyncStatus + a *app.App + nodeConf *mock_nodeconf.MockService + nodeUsage *mock_spacesyncstatus.MockNodeUsage + nodeStatus *mock_nodestatus.MockNodeStatus + subscriptionService subscription.Service + syncSubs syncsubscritions.SyncSubscriptions + objectStore *objectstore.StoreFixture + spaceIdGetter *mock_spacesyncstatus.MockSpaceIdGetter + eventSender *mock_event.MockSender + session session.HookRunner + networkConfig *mock_spacesyncstatus.MockNetworkConfig + ctrl *gomock.Controller +} + +func mapFileStatus(status filesyncstatus.Status) domain.ObjectSyncStatus { + switch status { + case filesyncstatus.Syncing: + return domain.ObjectSyncStatusSyncing + case filesyncstatus.Queued: + return domain.ObjectSyncStatusSyncing + case filesyncstatus.Limited: + return domain.ObjectSyncStatusError + default: + return domain.ObjectSyncStatusSynced + } +} + +func genFileObject(fileStatus filesyncstatus.Status, spaceId string) objectstore.TestObject { + id := fmt.Sprintf("%d", rand.Int()) + return objectstore.TestObject{ + bundle.RelationKeyId: pbtypes.String(id), + bundle.RelationKeySyncStatus: pbtypes.Int64(int64(mapFileStatus(fileStatus))), + bundle.RelationKeyFileBackupStatus: pbtypes.Int64(int64(fileStatus)), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_file)), + bundle.RelationKeyName: pbtypes.String("name" + id), + bundle.RelationKeySpaceId: pbtypes.String(spaceId), + } +} + +func genObject(syncStatus domain.ObjectSyncStatus, spaceId string) objectstore.TestObject { + id := fmt.Sprintf("%d", rand.Int()) + return objectstore.TestObject{ + bundle.RelationKeyId: pbtypes.String(id), + bundle.RelationKeySyncStatus: pbtypes.Int64(int64(syncStatus)), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_basic)), + bundle.RelationKeyName: pbtypes.String("name" + id), + bundle.RelationKeySpaceId: pbtypes.String(spaceId), + } +} + +func genSyncingObjects(fileObjects, objects int, spaceId string) []objectstore.TestObject { + var res []objectstore.TestObject + for i := 0; i < fileObjects; i++ { + res = append(res, genFileObject(filesyncstatus.Syncing, spaceId)) + } + for i := 0; i < objects; i++ { + res = append(res, genObject(domain.ObjectSyncStatusSyncing, spaceId)) + } + return res +} + +func newFixture(t *testing.T, beforeStart func(fx *fixture)) *fixture { + a := new(app.App) + ctrl := gomock.NewController(t) + internalSubs := subscription.RegisterSubscriptionService(t, a) + networkConfig := mock_spacesyncstatus.NewMockNetworkConfig(t) + sess := session.NewHookRunner() + fx := &fixture{ + a: a, + ctrl: ctrl, + spaceSyncStatus: NewSpaceSyncStatus().(*spaceSyncStatus), + nodeUsage: mock_spacesyncstatus.NewMockNodeUsage(t), + nodeStatus: mock_nodestatus.NewMockNodeStatus(t), + nodeConf: mock_nodeconf.NewMockService(ctrl), + spaceIdGetter: mock_spacesyncstatus.NewMockSpaceIdGetter(t), + objectStore: internalSubs.StoreFixture, + eventSender: app.MustComponent[event.Sender](a).(*mock_event.MockSender), + subscriptionService: internalSubs, + session: sess, + syncSubs: syncsubscritions.New(), + networkConfig: networkConfig, + } + a.Register(fx.syncSubs). + Register(testutil.PrepareMock(ctx, a, networkConfig)). + Register(testutil.PrepareMock(ctx, a, fx.nodeStatus)). + Register(testutil.PrepareMock(ctx, a, fx.spaceIdGetter)). + Register(testutil.PrepareMock(ctx, a, fx.nodeConf)). + Register(testutil.PrepareMock(ctx, a, fx.nodeUsage)). + Register(sess). + Register(fx.spaceSyncStatus) + beforeStart(fx) + err := a.Start(ctx) + require.NoError(t, err) + return fx +} + +func Test(t *testing.T) { + t.Run("empty space synced", func(t *testing.T) { + fx := newFixture(t, func(fx *fixture) { + fx.networkConfig.EXPECT().GetNetworkMode().Return(pb.RpcAccount_DefaultConfig) + fx.spaceIdGetter.EXPECT().AllSpaceIds().Return([]string{"spaceId"}) + fx.nodeStatus.EXPECT().GetNodeStatus("spaceId").Return(nodestatus.Online) + fx.nodeUsage.EXPECT().GetNodeUsage(mock.Anything).Return(&files.NodeUsageResponse{ + Usage: filesync.NodeUsage{ + BytesLeft: 1000, + AccountBytesLimit: 1000, + }, + LocalUsageBytes: 0, + }, nil) + fx.nodeConf.EXPECT().NetworkCompatibilityStatus().Return(nodeconf.NetworkCompatibilityStatusOk) + fx.eventSender.EXPECT().Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{{ + Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ + SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ + Id: "spaceId", + Status: pb.EventSpace_Synced, + Network: pb.EventSpace_Anytype, + }, + }, + }}, + }) + }) + defer fx.ctrl.Finish() + }) + t.Run("objects syncing", func(t *testing.T) { + fx := newFixture(t, func(fx *fixture) { + objs := genSyncingObjects(10, 100, "spaceId") + fx.objectStore.AddObjects(t, objs) + fx.networkConfig.EXPECT().GetNetworkMode().Return(pb.RpcAccount_DefaultConfig) + fx.spaceIdGetter.EXPECT().AllSpaceIds().Return([]string{"spaceId"}) + fx.nodeStatus.EXPECT().GetNodeStatus("spaceId").Return(nodestatus.Online) + fx.nodeUsage.EXPECT().GetNodeUsage(mock.Anything).Return(&files.NodeUsageResponse{ + Usage: filesync.NodeUsage{ + BytesLeft: 1000, + AccountBytesLimit: 1000, + }, + LocalUsageBytes: 0, + }, nil) + fx.nodeConf.EXPECT().NetworkCompatibilityStatus().Return(nodeconf.NetworkCompatibilityStatusOk) + fx.eventSender.EXPECT().Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{{ + Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ + SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ + Id: "spaceId", + SyncingObjectsCounter: 110, + Status: pb.EventSpace_Syncing, + Network: pb.EventSpace_Anytype, + }, + }, + }}, + }) + }) + defer fx.ctrl.Finish() + }) + t.Run("local only", func(t *testing.T) { + fx := newFixture(t, func(fx *fixture) { + objs := genSyncingObjects(10, 100, "spaceId") + fx.objectStore.AddObjects(t, objs) + fx.networkConfig.EXPECT().GetNetworkMode().Return(pb.RpcAccount_LocalOnly) + fx.spaceIdGetter.EXPECT().AllSpaceIds().Return([]string{"spaceId"}) + fx.eventSender.EXPECT().Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{{ + Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ + SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ + Id: "spaceId", + Status: pb.EventSpace_Offline, + Network: pb.EventSpace_LocalOnly, + }, + }, + }}, + }) + }) + defer fx.ctrl.Finish() + }) + t.Run("size exceeded", func(t *testing.T) { + fx := newFixture(t, func(fx *fixture) { + objs := genSyncingObjects(10, 100, "spaceId") + fx.objectStore.AddObjects(t, objs) + fx.networkConfig.EXPECT().GetNetworkMode().Return(pb.RpcAccount_DefaultConfig) + fx.spaceIdGetter.EXPECT().AllSpaceIds().Return([]string{"spaceId"}) + fx.nodeStatus.EXPECT().GetNodeStatus("spaceId").Return(nodestatus.Online) + fx.nodeUsage.EXPECT().GetNodeUsage(mock.Anything).Return(&files.NodeUsageResponse{ + Usage: filesync.NodeUsage{ + BytesLeft: 1, + AccountBytesLimit: 1000, + }, + LocalUsageBytes: 0, + }, nil) + fx.nodeConf.EXPECT().NetworkCompatibilityStatus().Return(nodeconf.NetworkCompatibilityStatusOk) + fx.eventSender.EXPECT().Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{{ + Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ + SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ + Id: "spaceId", + SyncingObjectsCounter: 110, + Status: pb.EventSpace_Error, + Network: pb.EventSpace_Anytype, + Error: pb.EventSpace_StorageLimitExceed, + }, + }, + }}, + }) + }) + defer fx.ctrl.Finish() + }) + t.Run("connection error", func(t *testing.T) { + fx := newFixture(t, func(fx *fixture) { + objs := genSyncingObjects(10, 100, "spaceId") + fx.objectStore.AddObjects(t, objs) + fx.networkConfig.EXPECT().GetNetworkMode().Return(pb.RpcAccount_DefaultConfig) + fx.spaceIdGetter.EXPECT().AllSpaceIds().Return([]string{"spaceId"}) + fx.nodeStatus.EXPECT().GetNodeStatus("spaceId").Return(nodestatus.ConnectionError) + fx.nodeUsage.EXPECT().GetNodeUsage(mock.Anything).Return(&files.NodeUsageResponse{ + Usage: filesync.NodeUsage{ + BytesLeft: 1000, + AccountBytesLimit: 1000, + }, + LocalUsageBytes: 0, + }, nil) + fx.nodeConf.EXPECT().NetworkCompatibilityStatus().Return(nodeconf.NetworkCompatibilityStatusOk) + fx.eventSender.EXPECT().Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{{ + Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ + SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ + Id: "spaceId", + SyncingObjectsCounter: 110, + Status: pb.EventSpace_Offline, + Network: pb.EventSpace_Anytype, + Error: pb.EventSpace_NetworkError, + }, + }, + }}, + }) + }) + defer fx.ctrl.Finish() + }) + t.Run("network incompatible", func(t *testing.T) { + fx := newFixture(t, func(fx *fixture) { + objs := genSyncingObjects(10, 100, "spaceId") + fx.objectStore.AddObjects(t, objs) + fx.networkConfig.EXPECT().GetNetworkMode().Return(pb.RpcAccount_DefaultConfig) + fx.spaceIdGetter.EXPECT().AllSpaceIds().Return([]string{"spaceId"}) + fx.nodeStatus.EXPECT().GetNodeStatus("spaceId").Return(nodestatus.ConnectionError) + fx.nodeUsage.EXPECT().GetNodeUsage(mock.Anything).Return(&files.NodeUsageResponse{ + Usage: filesync.NodeUsage{ + BytesLeft: 1000, + AccountBytesLimit: 1000, + }, + LocalUsageBytes: 0, + }, nil) + fx.nodeConf.EXPECT().NetworkCompatibilityStatus().Return(nodeconf.NetworkCompatibilityStatusIncompatible) + fx.eventSender.EXPECT().Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{{ + Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ + SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ + Id: "spaceId", + SyncingObjectsCounter: 110, + Status: pb.EventSpace_Error, + Network: pb.EventSpace_Anytype, + Error: pb.EventSpace_IncompatibleVersion, + }, + }, + }}, + }) + }) + defer fx.ctrl.Finish() + }) + t.Run("objects syncing, refresh with missing objects", func(t *testing.T) { + fx := newFixture(t, func(fx *fixture) { + fx.spaceSyncStatus.loopInterval = 10 * time.Millisecond + objs := genSyncingObjects(10, 100, "spaceId") + fx.objectStore.AddObjects(t, objs) + fx.networkConfig.EXPECT().GetNetworkMode().Return(pb.RpcAccount_DefaultConfig) + fx.spaceIdGetter.EXPECT().AllSpaceIds().Return([]string{"spaceId"}) + fx.nodeStatus.EXPECT().GetNodeStatus("spaceId").Return(nodestatus.Online) + fx.nodeUsage.EXPECT().GetNodeUsage(mock.Anything).Return(&files.NodeUsageResponse{ + Usage: filesync.NodeUsage{ + BytesLeft: 1000, + AccountBytesLimit: 1000, + }, + LocalUsageBytes: 0, + }, nil) + fx.nodeConf.EXPECT().NetworkCompatibilityStatus().AnyTimes().Return(nodeconf.NetworkCompatibilityStatusOk) + fx.eventSender.EXPECT().Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{{ + Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ + SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ + Id: "spaceId", + SyncingObjectsCounter: 110, + Status: pb.EventSpace_Syncing, + Network: pb.EventSpace_Anytype, + }, + }, + }}, + }) + }) + fx.UpdateMissingIds("spaceId", []string{"missingId"}) + fx.Refresh("spaceId") + fx.spaceIdGetter.EXPECT().TechSpaceId().Return("techSpaceId") + fx.eventSender.EXPECT().Broadcast(&pb.Event{ Messages: []*pb.EventMessage{{ Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ - Id: "personalId", - Status: pb.EventSpace_Synced, - Network: pb.EventSpace_Anytype, + Id: "spaceId", + SyncingObjectsCounter: 111, + Status: pb.EventSpace_Syncing, + Network: pb.EventSpace_Anytype, + }, + }, + }}, + }).Times(1) + time.Sleep(100 * time.Millisecond) + defer fx.ctrl.Finish() + }) + t.Run("hook new session", func(t *testing.T) { + fx := newFixture(t, func(fx *fixture) { + objs := genSyncingObjects(10, 100, "spaceId") + fx.objectStore.AddObjects(t, objs) + fx.networkConfig.EXPECT().GetNetworkMode().Return(pb.RpcAccount_DefaultConfig) + fx.spaceIdGetter.EXPECT().AllSpaceIds().Return([]string{"spaceId"}) + fx.nodeStatus.EXPECT().GetNodeStatus("spaceId").Return(nodestatus.ConnectionError) + fx.nodeUsage.EXPECT().GetNodeUsage(mock.Anything).Return(&files.NodeUsageResponse{ + Usage: filesync.NodeUsage{ + BytesLeft: 1000, + AccountBytesLimit: 1000, + }, + LocalUsageBytes: 0, + }, nil) + fx.nodeConf.EXPECT().NetworkCompatibilityStatus().AnyTimes().Return(nodeconf.NetworkCompatibilityStatusIncompatible) + fx.eventSender.EXPECT().Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{{ + Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ + SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ + Id: "spaceId", + SyncingObjectsCounter: 110, + Status: pb.EventSpace_Error, + Network: pb.EventSpace_Anytype, + Error: pb.EventSpace_IncompatibleVersion, + }, + }, + }}, + }) + }) + fx.eventSender.EXPECT().SendToSession("sessionId", &pb.Event{ + Messages: []*pb.EventMessage{{ + Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ + SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ + Id: "spaceId", + SyncingObjectsCounter: 110, + Status: pb.EventSpace_Error, + Network: pb.EventSpace_Anytype, + Error: pb.EventSpace_IncompatibleVersion, }, }, }}, }) - err = status.Run(ctx) - assert.Nil(t, err) - err = status.Close(ctx) - assert.Nil(t, err) + fx.session.RunHooks(mockSessionContext{"sessionId"}) + defer fx.ctrl.Finish() }) - t.Run("local only mode", func(t *testing.T) { - // given - status := NewSpaceSyncStatus() - ctx := context.Background() - - a := new(app.App) - eventSender := mock_event.NewMockSender(t) - eventSender.EXPECT().Broadcast(&pb.Event{ + t.Run("hook new session local only", func(t *testing.T) { + fx := newFixture(t, func(fx *fixture) { + objs := genSyncingObjects(10, 100, "spaceId") + fx.objectStore.AddObjects(t, objs) + fx.networkConfig.EXPECT().GetNetworkMode().Return(pb.RpcAccount_LocalOnly) + fx.spaceIdGetter.EXPECT().AllSpaceIds().Return([]string{"spaceId"}) + fx.eventSender.EXPECT().Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{{ + Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ + SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ + Id: "spaceId", + Status: pb.EventSpace_Offline, + Network: pb.EventSpace_LocalOnly, + }, + }, + }}, + }) + }) + fx.eventSender.EXPECT().SendToSession("sessionId", &pb.Event{ Messages: []*pb.EventMessage{{ Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ + Id: "spaceId", Status: pb.EventSpace_Offline, Network: pb.EventSpace_LocalOnly, }, }, }}, }) - space := mock_spacesyncstatus.NewMockSpaceIdGetter(t) - - a.Register(testutil.PrepareMock(ctx, a, eventSender)). - Register(objectstore.NewStoreFixture(t)). - Register(&config.Config{NetworkMode: pb.RpcAccount_LocalOnly}). - Register(testutil.PrepareMock(ctx, a, space)). - Register(session.NewHookRunner()) - - // when - err := status.Init(a) - - // then - assert.Nil(t, err) - err = status.Run(ctx) - assert.Nil(t, err) - err = status.Close(ctx) - assert.Nil(t, err) - }) -} - -func TestSpaceSyncStatus_updateSpaceSyncStatus(t *testing.T) { - t.Run("syncing event for objects", func(t *testing.T) { - // given - eventSender := mock_event.NewMockSender(t) - eventSender.EXPECT().Broadcast(&pb.Event{ - Messages: []*pb.EventMessage{{ - Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ - SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ - Id: "spaceId", - Status: pb.EventSpace_Syncing, - Network: pb.EventSpace_Anytype, - Error: pb.EventSpace_Null, - SyncingObjectsCounter: 2, - }, - }, - }}, - }) - storeFixture := objectstore.NewStoreFixture(t) - storeFixture.AddObjects(t, []objectstore.TestObject{ - { - bundle.RelationKeyId: pbtypes.String("id1"), - bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.SpaceSyncStatusSyncing)), - bundle.RelationKeySpaceId: pbtypes.String("spaceId"), - }, - { - bundle.RelationKeyId: pbtypes.String("id2"), - bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.SpaceSyncStatusSyncing)), - bundle.RelationKeySpaceId: pbtypes.String("spaceId"), - }, - }) - status := spaceSyncStatus{ - eventSender: eventSender, - networkConfig: &config.Config{NetworkMode: pb.RpcAccount_DefaultConfig}, - batcher: mb.New[*domain.SpaceSync](0), - filesState: NewFileState(storeFixture), - objectsState: NewObjectState(storeFixture), - } - syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull) - - // then - status.updateSpaceSyncStatus(syncStatus) - - // when - assert.Equal(t, domain.SpaceSyncStatusSyncing, status.objectsState.GetSyncStatus("spaceId")) - assert.Equal(t, 2, status.objectsState.GetSyncObjectCount("spaceId")) - assert.Equal(t, domain.SpaceSyncStatusSyncing, status.getSpaceSyncStatus(syncStatus.SpaceId)) - }) - t.Run("syncing event for files", func(t *testing.T) { - // given - eventSender := mock_event.NewMockSender(t) - eventSender.EXPECT().Broadcast(&pb.Event{ - Messages: []*pb.EventMessage{{ - Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ - SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ - Id: "spaceId", - Status: pb.EventSpace_Syncing, - Network: pb.EventSpace_Anytype, - Error: pb.EventSpace_Null, - SyncingObjectsCounter: 2, - }, - }, - }}, - }) - storeFixture := objectstore.NewStoreFixture(t) - storeFixture.AddObjects(t, []objectstore.TestObject{ - { - bundle.RelationKeyId: pbtypes.String("id1"), - bundle.RelationKeyFileBackupStatus: pbtypes.Int64(int64(filesyncstatus.Syncing)), - bundle.RelationKeySpaceId: pbtypes.String("spaceId"), - }, - { - bundle.RelationKeyId: pbtypes.String("id2"), - bundle.RelationKeyFileBackupStatus: pbtypes.Int64(int64(filesyncstatus.Synced)), - bundle.RelationKeySpaceId: pbtypes.String("spaceId"), - }, - { - bundle.RelationKeyId: pbtypes.String("id3"), - bundle.RelationKeyFileBackupStatus: pbtypes.Int64(int64(filesyncstatus.Syncing)), - bundle.RelationKeySpaceId: pbtypes.String("spaceId"), - }, - }) - - status := spaceSyncStatus{ - eventSender: eventSender, - networkConfig: &config.Config{NetworkMode: pb.RpcAccount_DefaultConfig}, - batcher: mb.New[*domain.SpaceSync](0), - filesState: NewFileState(storeFixture), - objectsState: NewObjectState(storeFixture), - } - syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull) - - // then - status.updateSpaceSyncStatus(syncStatus) - - // when - assert.Equal(t, domain.SpaceSyncStatusSyncing, status.filesState.GetSyncStatus("spaceId")) - assert.Equal(t, 2, status.filesState.GetSyncObjectCount("spaceId")) - assert.Equal(t, domain.SpaceSyncStatusSyncing, status.getSpaceSyncStatus(syncStatus.SpaceId)) - }) - t.Run("don't send not needed synced event if files or objects are still syncing", func(t *testing.T) { - // given - eventSender := mock_event.NewMockSender(t) - status := spaceSyncStatus{ - eventSender: eventSender, - networkConfig: &config.Config{NetworkMode: pb.RpcAccount_DefaultConfig}, - batcher: mb.New[*domain.SpaceSync](0), - filesState: NewFileState(objectstore.NewStoreFixture(t)), - objectsState: NewObjectState(objectstore.NewStoreFixture(t)), - } - objectsSyncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull) - status.objectsState.SetSyncStatusAndErr(objectsSyncStatus.Status, domain.SyncErrorNull, objectsSyncStatus.SpaceId) - - // then - syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull) - status.updateSpaceSyncStatus(syncStatus) - - // when - eventSender.AssertNotCalled(t, "Broadcast") - }) - t.Run("send error event", func(t *testing.T) { - // given - eventSender := mock_event.NewMockSender(t) - eventSender.EXPECT().Broadcast(&pb.Event{ - Messages: []*pb.EventMessage{{ - Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ - SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ - Id: "spaceId", - Status: pb.EventSpace_Error, - Network: pb.EventSpace_Anytype, - Error: pb.EventSpace_NetworkError, - SyncingObjectsCounter: 0, - }, - }, - }}, - }) - status := spaceSyncStatus{ - eventSender: eventSender, - networkConfig: &config.Config{NetworkMode: pb.RpcAccount_DefaultConfig}, - batcher: mb.New[*domain.SpaceSync](0), - filesState: NewFileState(objectstore.NewStoreFixture(t)), - objectsState: NewObjectState(objectstore.NewStoreFixture(t)), - } - syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusError, domain.SyncErrorNetworkError) - - // then - status.updateSpaceSyncStatus(syncStatus) - - // when - assert.Equal(t, domain.SpaceSyncStatusError, status.objectsState.GetSyncStatus("spaceId")) - assert.Equal(t, 0, status.objectsState.GetSyncObjectCount("spaceId")) - assert.Equal(t, domain.SpaceSyncStatusError, status.getSpaceSyncStatus(syncStatus.SpaceId)) - }) - t.Run("send storage error event and then reset it", func(t *testing.T) { - // given - eventSender := mock_event.NewMockSender(t) - eventSender.EXPECT().Broadcast(&pb.Event{ - Messages: []*pb.EventMessage{{ - Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ - SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ - Id: "spaceId", - Status: pb.EventSpace_Error, - Network: pb.EventSpace_Anytype, - Error: pb.EventSpace_StorageLimitExceed, - SyncingObjectsCounter: 0, - }, - }, - }}, - }) - status := spaceSyncStatus{ - eventSender: eventSender, - networkConfig: &config.Config{NetworkMode: pb.RpcAccount_DefaultConfig}, - batcher: mb.New[*domain.SpaceSync](0), - filesState: NewFileState(objectstore.NewStoreFixture(t)), - objectsState: NewObjectState(objectstore.NewStoreFixture(t)), - } - syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusError, domain.SyncErrorStorageLimitExceed) - - // then - status.updateSpaceSyncStatus(syncStatus) - - // when - assert.Equal(t, domain.SpaceSyncStatusSynced, status.filesState.GetSyncStatus("spaceId")) - assert.Equal(t, 0, status.filesState.GetSyncObjectCount("spaceId")) - assert.Equal(t, domain.SpaceSyncStatusSynced, status.getSpaceSyncStatus(syncStatus.SpaceId)) - }) - t.Run("send incompatible error event", func(t *testing.T) { - // given - eventSender := mock_event.NewMockSender(t) - eventSender.EXPECT().Broadcast(&pb.Event{ - Messages: []*pb.EventMessage{{ - Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ - SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ - Id: "spaceId", - Status: pb.EventSpace_Error, - Network: pb.EventSpace_Anytype, - Error: pb.EventSpace_IncompatibleVersion, - SyncingObjectsCounter: 0, - }, - }, - }}, - }) - status := spaceSyncStatus{ - eventSender: eventSender, - networkConfig: &config.Config{NetworkMode: pb.RpcAccount_DefaultConfig}, - batcher: mb.New[*domain.SpaceSync](0), - filesState: NewFileState(objectstore.NewStoreFixture(t)), - objectsState: NewObjectState(objectstore.NewStoreFixture(t)), - } - syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusError, domain.SyncErrorIncompatibleVersion) - - // then - status.updateSpaceSyncStatus(syncStatus) - - // when - assert.Equal(t, domain.SpaceSyncStatusError, status.objectsState.GetSyncStatus("spaceId")) - assert.Equal(t, 0, status.objectsState.GetSyncObjectCount("spaceId")) - assert.Equal(t, domain.SpaceSyncStatusError, status.getSpaceSyncStatus(syncStatus.SpaceId)) - }) - t.Run("send offline event", func(t *testing.T) { - // given - eventSender := mock_event.NewMockSender(t) - eventSender.EXPECT().Broadcast(&pb.Event{ - Messages: []*pb.EventMessage{{ - Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ - SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ - Id: "spaceId", - Status: pb.EventSpace_Offline, - Network: pb.EventSpace_SelfHost, - Error: pb.EventSpace_Null, - SyncingObjectsCounter: 0, - }, - }, - }}, - }) - status := spaceSyncStatus{ - eventSender: eventSender, - networkConfig: &config.Config{NetworkMode: pb.RpcAccount_CustomConfig}, - batcher: mb.New[*domain.SpaceSync](0), - filesState: NewFileState(objectstore.NewStoreFixture(t)), - objectsState: NewObjectState(objectstore.NewStoreFixture(t)), - } - syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusOffline, domain.SyncErrorNull) - - // then - status.updateSpaceSyncStatus(syncStatus) - - // when - assert.Equal(t, domain.SpaceSyncStatusOffline, status.objectsState.GetSyncStatus("spaceId")) - assert.Equal(t, 0, status.objectsState.GetSyncObjectCount("spaceId")) - assert.Equal(t, domain.SpaceSyncStatusOffline, status.getSpaceSyncStatus(syncStatus.SpaceId)) - }) - t.Run("send synced event", func(t *testing.T) { - // given - eventSender := mock_event.NewMockSender(t) - status := spaceSyncStatus{ - eventSender: eventSender, - networkConfig: &config.Config{NetworkMode: pb.RpcAccount_CustomConfig}, - batcher: mb.New[*domain.SpaceSync](0), - filesState: NewFileState(objectstore.NewStoreFixture(t)), - objectsState: NewObjectState(objectstore.NewStoreFixture(t)), - } - syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull) - status.objectsState.SetObjectsNumber(syncStatus) - status.objectsState.SetSyncStatusAndErr(syncStatus.Status, domain.SyncErrorNull, syncStatus.SpaceId) - - // then - syncStatus = domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull) - status.updateSpaceSyncStatus(syncStatus) - - // when - assert.Equal(t, domain.SpaceSyncStatusSynced, status.objectsState.GetSyncStatus("spaceId")) - assert.Equal(t, 0, status.objectsState.GetSyncObjectCount("spaceId")) - assert.Equal(t, domain.SpaceSyncStatusUnknown, status.filesState.GetSyncStatus("spaceId")) - assert.Equal(t, 0, status.filesState.GetSyncObjectCount("spaceId")) - assert.Equal(t, domain.SpaceSyncStatusSynced, status.getSpaceSyncStatus(syncStatus.SpaceId)) - }) - t.Run("send initial synced event", func(t *testing.T) { - // given - eventSender := mock_event.NewMockSender(t) - status := spaceSyncStatus{ - eventSender: eventSender, - networkConfig: &config.Config{NetworkMode: pb.RpcAccount_CustomConfig}, - batcher: mb.New[*domain.SpaceSync](0), - filesState: NewFileState(objectstore.NewStoreFixture(t)), - objectsState: NewObjectState(objectstore.NewStoreFixture(t)), - } - eventSender.EXPECT().Broadcast(&pb.Event{ - Messages: []*pb.EventMessage{{ - Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ - SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ - Id: "spaceId", - Status: pb.EventSpace_Synced, - Network: pb.EventSpace_SelfHost, - Error: pb.EventSpace_Null, - SyncingObjectsCounter: 0, - }, - }, - }}, - }) - // then - syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull) - status.updateSpaceSyncStatus(syncStatus) - }) - - t.Run("not send not needed synced event", func(t *testing.T) { - // given - eventSender := mock_event.NewMockSender(t) - status := spaceSyncStatus{ - eventSender: eventSender, - networkConfig: &config.Config{NetworkMode: pb.RpcAccount_CustomConfig}, - batcher: mb.New[*domain.SpaceSync](0), - filesState: NewFileState(objectstore.NewStoreFixture(t)), - objectsState: NewObjectState(objectstore.NewStoreFixture(t)), - } - status.objectsState.SetSyncStatusAndErr(domain.SpaceSyncStatusSynced, domain.SyncErrorNull, "spaceId") - status.filesState.SetSyncStatusAndErr(domain.SpaceSyncStatusSynced, domain.SyncErrorNull, "spaceId") - - // then - syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull) - status.updateSpaceSyncStatus(syncStatus) - - // when - eventSender.AssertNotCalled(t, "Broadcast", &pb.Event{ - Messages: []*pb.EventMessage{{ - Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ - SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ - Id: "spaceId", - Status: pb.EventSpace_Synced, - Network: pb.EventSpace_SelfHost, - Error: pb.EventSpace_Null, - SyncingObjectsCounter: 0, - }, - }, - }}, - }) - }) - t.Run("not send syncing event if object number not changed", func(t *testing.T) { - // given - eventSender := mock_event.NewMockSender(t) - eventSender.EXPECT().Broadcast(&pb.Event{ - Messages: []*pb.EventMessage{{ - Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ - SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ - Id: "spaceId", - Status: pb.EventSpace_Syncing, - Network: pb.EventSpace_SelfHost, - Error: pb.EventSpace_Null, - SyncingObjectsCounter: 2, - }, - }, - }}, - }).Return().Times(1) - - storeFixture := objectstore.NewStoreFixture(t) - status := spaceSyncStatus{ - eventSender: eventSender, - networkConfig: &config.Config{NetworkMode: pb.RpcAccount_CustomConfig}, - batcher: mb.New[*domain.SpaceSync](0), - filesState: NewFileState(storeFixture), - objectsState: NewObjectState(storeFixture), - } - storeFixture.AddObjects(t, []objectstore.TestObject{ - { - bundle.RelationKeyId: pbtypes.String("id1"), - bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.SpaceSyncStatusSyncing)), - bundle.RelationKeySpaceId: pbtypes.String("spaceId"), - }, - { - bundle.RelationKeyId: pbtypes.String("id2"), - bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.SpaceSyncStatusSynced)), - bundle.RelationKeySpaceId: pbtypes.String("spaceId"), - }, - { - bundle.RelationKeyId: pbtypes.String("id3"), - bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.SpaceSyncStatusSyncing)), - bundle.RelationKeySpaceId: pbtypes.String("spaceId"), - }, - }) - - // when - syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull) - status.updateSpaceSyncStatus(syncStatus) - status.updateSpaceSyncStatus(syncStatus) - }) - t.Run("not send not changed event", func(t *testing.T) { - // given - eventSender := mock_event.NewMockSender(t) - status := spaceSyncStatus{ - eventSender: eventSender, - networkConfig: &config.Config{NetworkMode: pb.RpcAccount_CustomConfig}, - batcher: mb.New[*domain.SpaceSync](0), - filesState: NewFileState(objectstore.NewStoreFixture(t)), - objectsState: NewObjectState(objectstore.NewStoreFixture(t)), - } - status.objectsState.SetSyncStatusAndErr(domain.SpaceSyncStatusSynced, domain.SyncErrorNull, "spaceId") - status.filesState.SetSyncStatusAndErr(domain.SpaceSyncStatusSynced, domain.SyncErrorNull, "spaceId") - - // then - syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull) - status.objectsState.SetObjectsNumber(syncStatus) - status.updateSpaceSyncStatus(syncStatus) - - // when - eventSender.AssertNotCalled(t, "Broadcast", &pb.Event{ - Messages: []*pb.EventMessage{{ - Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ - SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ - Id: "spaceId", - Status: pb.EventSpace_Synced, - Network: pb.EventSpace_SelfHost, - Error: pb.EventSpace_Null, - SyncingObjectsCounter: 0, - }, - }, - }}, - }) - }) -} - -func TestSpaceSyncStatus_SendUpdate(t *testing.T) { - t.Run("SendUpdate success", func(t *testing.T) { - // given - eventSender := mock_event.NewMockSender(t) - spaceStatus := spaceSyncStatus{ - eventSender: eventSender, - networkConfig: &config.Config{NetworkMode: pb.RpcAccount_DefaultConfig}, - batcher: mb.New[*domain.SpaceSync](0), - filesState: NewFileState(objectstore.NewStoreFixture(t)), - objectsState: NewObjectState(objectstore.NewStoreFixture(t)), - } - syncStatus := domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull) - - // then - spaceStatus.SendUpdate(syncStatus) - - // when - status, err := spaceStatus.batcher.WaitOne(context.Background()) - assert.Nil(t, err) - assert.Equal(t, status, syncStatus) - }) -} - -func TestSpaceSyncStatus_Notify(t *testing.T) { - t.Run("Notify success", func(t *testing.T) { - // given - eventSender := mock_event.NewMockSender(t) - spaceIdGetter := mock_spacesyncstatus.NewMockSpaceIdGetter(t) - spaceStatus := spaceSyncStatus{ - eventSender: eventSender, - networkConfig: &config.Config{NetworkMode: pb.RpcAccount_DefaultConfig}, - batcher: mb.New[*domain.SpaceSync](0), - filesState: NewFileState(objectstore.NewStoreFixture(t)), - objectsState: NewObjectState(objectstore.NewStoreFixture(t)), - spaceIdGetter: spaceIdGetter, - } - // then - spaceIdGetter.EXPECT().AllSpaceIds().Return([]string{"id1", "id2"}) - eventSender.EXPECT().SendToSession(mock.Anything, &pb.Event{ - Messages: []*pb.EventMessage{{ - Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ - SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ - Id: "id1", - }, - }, - }}, - }) - eventSender.EXPECT().SendToSession(mock.Anything, &pb.Event{ - Messages: []*pb.EventMessage{{ - Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ - SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ - Id: "id2", - }, - }, - }}, - }) - spaceStatus.sendSyncEventForNewSession(session.NewContext()) + fx.session.RunHooks(mockSessionContext{"sessionId"}) + defer fx.ctrl.Finish() }) } From f1574ca733d23f469e36b40c1c1ff4188e7d60a1 Mon Sep 17 00:00:00 2001 From: kirillston Date: Thu, 25 Jul 2024 01:18:54 +0300 Subject: [PATCH 49/71] GO-3192 Refactor table unit tests --- core/block/editor/table/editor.go | 441 ++--- core/block/editor/table/editor_test.go | 2128 ++++++++++++++++++++++++ core/block/editor/table/table_test.go | 1157 +++---------- 3 files changed, 2548 insertions(+), 1178 deletions(-) create mode 100644 core/block/editor/table/editor_test.go diff --git a/core/block/editor/table/editor.go b/core/block/editor/table/editor.go index 83ac42363..087755637 100644 --- a/core/block/editor/table/editor.go +++ b/core/block/editor/table/editor.go @@ -19,22 +19,27 @@ import ( // nolint:revive,interfacebloat type TableEditor interface { TableCreate(s *state.State, req pb.RpcBlockTableCreateRequest) (string, error) + CellCreate(s *state.State, rowID string, colID string, b *model.Block) (string, error) + RowCreate(s *state.State, req pb.RpcBlockTableRowCreateRequest) (string, error) RowDelete(s *state.State, req pb.RpcBlockTableRowDeleteRequest) error - ColumnDelete(s *state.State, req pb.RpcBlockTableColumnDeleteRequest) error - ColumnMove(s *state.State, req pb.RpcBlockTableColumnMoveRequest) error RowDuplicate(s *state.State, req pb.RpcBlockTableRowDuplicateRequest) (newRowID string, err error) + // RowMove is done via BlockListMoveToExistingObject RowListFill(s *state.State, req pb.RpcBlockTableRowListFillRequest) error RowListClean(s *state.State, req pb.RpcBlockTableRowListCleanRequest) error RowSetHeader(s *state.State, req pb.RpcBlockTableRowSetHeaderRequest) error - ColumnListFill(s *state.State, req pb.RpcBlockTableColumnListFillRequest) error - cleanupTables(_ smartblock.ApplyInfo) error + ColumnCreate(s *state.State, req pb.RpcBlockTableColumnCreateRequest) (string, error) - cloneColumnStyles(s *state.State, srcColID string, targetColID string) error + ColumnDelete(s *state.State, req pb.RpcBlockTableColumnDeleteRequest) error ColumnDuplicate(s *state.State, req pb.RpcBlockTableColumnDuplicateRequest) (id string, err error) + ColumnMove(s *state.State, req pb.RpcBlockTableColumnMoveRequest) error + ColumnListFill(s *state.State, req pb.RpcBlockTableColumnListFillRequest) error + Expand(s *state.State, req pb.RpcBlockTableExpandRequest) error Sort(s *state.State, req pb.RpcBlockTableSortRequest) error - CellCreate(s *state.State, rowID string, colID string, b *model.Block) (string, error) + + cleanupTables(_ smartblock.ApplyInfo) error + cloneColumnStyles(s *state.State, srcColID string, targetColID string) error } type editor struct { @@ -126,6 +131,9 @@ func (t *editor) TableCreate(s *state.State, req pb.RpcBlockTableCreateRequest) tableBlock.Model().ChildrenIds = []string{columnsLayout.Model().Id, rowsLayout.Model().Id} if req.WithHeaderRow { + if len(rowIDs) == 0 { + return "", fmt.Errorf("no rows to make header row") + } headerID := rowIDs[0] if err := t.RowSetHeader(s, pb.RpcBlockTableRowSetHeaderRequest{ @@ -159,6 +167,36 @@ func (t *editor) TableCreate(s *state.State, req pb.RpcBlockTableCreateRequest) return tableBlock.Model().Id, nil } +func (t *editor) CellCreate(s *state.State, rowID string, colID string, b *model.Block) (string, error) { + tb, err := NewTable(s, rowID) + if err != nil { + return "", fmt.Errorf("initialize table state: %w", err) + } + + row, err := getRow(s, rowID) + if err != nil { + return "", fmt.Errorf("get row: %w", err) + } + if _, err = pickColumn(s, colID); err != nil { + return "", fmt.Errorf("pick column: %w", err) + } + + cellID, err := addCell(s, rowID, colID) + if err != nil { + return "", fmt.Errorf("add cell: %w", err) + } + cell := s.Get(cellID) + cell.Model().Content = b.Content + if err := s.InsertTo(rowID, model.Block_Inner, cellID); err != nil { + return "", fmt.Errorf("insert to: %w", err) + } + + colIdx := tb.MakeColumnIndex() + normalizeRow(nil, colIdx, row) + + return cellID, nil +} + func (t *editor) RowCreate(s *state.State, req pb.RpcBlockTableRowCreateRequest) (string, error) { switch req.Position { case model.Block_Top, model.Block_Bottom: @@ -194,93 +232,19 @@ func (t *editor) RowDelete(s *state.State, req pb.RpcBlockTableRowDeleteRequest) return nil } -func (t *editor) ColumnDelete(s *state.State, req pb.RpcBlockTableColumnDeleteRequest) error { - _, err := pickColumn(s, req.TargetId) - if err != nil { - return fmt.Errorf("pick target column: %w", err) - } - - tb, err := NewTable(s, req.TargetId) - if err != nil { - return fmt.Errorf("initialize table state: %w", err) - } - - for _, rowID := range tb.RowIDs() { - row, err := pickRow(s, rowID) - if err != nil { - return fmt.Errorf("pick row %s: %w", rowID, err) - } - - for _, cellID := range row.Model().ChildrenIds { - _, colID, err := ParseCellID(cellID) - if err != nil { - return fmt.Errorf("parse cell id %s: %w", cellID, err) - } - - if colID == req.TargetId { - if !s.Unlink(cellID) { - return fmt.Errorf("unlink cell %s", cellID) - } - break - } - } - } - if !s.Unlink(req.TargetId) { - return fmt.Errorf("unlink column header") - } - - return nil -} - -func (t *editor) ColumnMove(s *state.State, req pb.RpcBlockTableColumnMoveRequest) error { - switch req.Position { - case model.Block_Left: - req.Position = model.Block_Top - case model.Block_Right: - req.Position = model.Block_Bottom - default: - return fmt.Errorf("position is not supported") - } - _, err := pickColumn(s, req.TargetId) - if err != nil { - return fmt.Errorf("get target column: %w", err) - } - _, err = pickColumn(s, req.DropTargetId) - if err != nil { - return fmt.Errorf("get drop target column: %w", err) - } - - tb, err := NewTable(s, req.TargetId) - if err != nil { - return fmt.Errorf("init table block: %w", err) - } - - if !s.Unlink(req.TargetId) { - return fmt.Errorf("unlink target column") - } - if err = s.InsertTo(req.DropTargetId, req.Position, req.TargetId); err != nil { - return fmt.Errorf("insert column: %w", err) - } - - colIdx := tb.MakeColumnIndex() - - for _, id := range tb.RowIDs() { - row, err := getRow(s, id) - if err != nil { - return fmt.Errorf("get row %s: %w", id, err) - } - normalizeRow(nil, colIdx, row) - } - - return nil -} - func (t *editor) RowDuplicate(s *state.State, req pb.RpcBlockTableRowDuplicateRequest) (newRowID string, err error) { + if req.Position != model.Block_Top && req.Position != model.Block_Bottom { + return "", fmt.Errorf("position %s is not supported", model.BlockPosition_name[int32(req.Position)]) + } srcRow, err := pickRow(s, req.BlockId) if err != nil { return "", fmt.Errorf("pick source row: %w", err) } + if _, err = pickRow(s, req.TargetId); err != nil { + return "", fmt.Errorf("pick target row: %w", err) + } + newRow := srcRow.Copy() newRow.Model().Id = t.generateRowID() if !s.Add(newRow) { @@ -390,88 +354,6 @@ func (t *editor) RowSetHeader(s *state.State, req pb.RpcBlockTableRowSetHeaderRe return nil } -func (t *editor) ColumnListFill(s *state.State, req pb.RpcBlockTableColumnListFillRequest) error { - if len(req.BlockIds) == 0 { - return fmt.Errorf("empty row list") - } - - tb, err := NewTable(s, req.BlockIds[0]) - if err != nil { - return fmt.Errorf("init table: %w", err) - } - - rows := tb.RowIDs() - - for _, colID := range req.BlockIds { - for _, rowID := range rows { - id := MakeCellID(rowID, colID) - if s.Exists(id) { - continue - } - _, err := addCell(s, rowID, colID) - if err != nil { - return fmt.Errorf("add cell %s: %w", id, err) - } - - row, err := getRow(s, rowID) - if err != nil { - return fmt.Errorf("get row %s: %w", rowID, err) - } - - row.Model().ChildrenIds = append(row.Model().ChildrenIds, id) - } - } - - colIdx := tb.MakeColumnIndex() - for _, rowID := range rows { - row, err := getRow(s, rowID) - if err != nil { - return fmt.Errorf("get row %s: %w", rowID, err) - } - normalizeRow(nil, colIdx, row) - } - - return nil -} - -func (t *editor) cleanupTables(_ smartblock.ApplyInfo) error { - if t.sb == nil { - return fmt.Errorf("nil smartblock") - } - s := t.sb.NewState() - - err := s.Iterate(func(b simple.Block) bool { - if b.Model().GetTable() == nil { - return true - } - - tb, err := NewTable(s, b.Model().Id) - if err != nil { - log.Errorf("cleanup: init table %s: %s", b.Model().Id, err) - return true - } - err = t.RowListClean(s, pb.RpcBlockTableRowListCleanRequest{ - BlockIds: tb.RowIDs(), - }) - if err != nil { - log.Errorf("cleanup table %s: %s", b.Model().Id, err) - return true - } - return true - }) - if err != nil { - log.Errorf("cleanup iterate: %s", err) - } - - if err = t.sb.Apply(s, smartblock.KeepInternalFlags); err != nil { - if errors.Is(err, source.ErrReadOnly) { - return nil - } - log.Errorf("cleanup apply: %s", err) - } - return nil -} - func (t *editor) ColumnCreate(s *state.State, req pb.RpcBlockTableColumnCreateRequest) (string, error) { switch req.Position { case model.Block_Left: @@ -505,48 +387,40 @@ func (t *editor) ColumnCreate(s *state.State, req pb.RpcBlockTableColumnCreateRe return colID, t.cloneColumnStyles(s, req.TargetId, colID) } -func (t *editor) cloneColumnStyles(s *state.State, srcColID, targetColID string) error { - tb, err := NewTable(s, srcColID) +func (t *editor) ColumnDelete(s *state.State, req pb.RpcBlockTableColumnDeleteRequest) error { + _, err := pickColumn(s, req.TargetId) if err != nil { - return fmt.Errorf("init table block: %w", err) + return fmt.Errorf("pick target column: %w", err) + } + + tb, err := NewTable(s, req.TargetId) + if err != nil { + return fmt.Errorf("initialize table state: %w", err) } - colIdx := tb.MakeColumnIndex() for _, rowID := range tb.RowIDs() { row, err := pickRow(s, rowID) if err != nil { - return fmt.Errorf("pick row: %w", err) + return fmt.Errorf("pick row %s: %w", rowID, err) } - var protoBlock simple.Block for _, cellID := range row.Model().ChildrenIds { _, colID, err := ParseCellID(cellID) if err != nil { - return fmt.Errorf("parse cell id: %w", err) + return fmt.Errorf("parse cell id %s: %w", cellID, err) } - if colID == srcColID { - protoBlock = s.Pick(cellID) - } - } - - if protoBlock != nil && protoBlock.Model().BackgroundColor != "" { - targetCellID := MakeCellID(rowID, targetColID) - - if !s.Exists(targetCellID) { - _, err := addCell(s, rowID, targetColID) - if err != nil { - return fmt.Errorf("add cell: %w", err) + if colID == req.TargetId { + if !s.Unlink(cellID) { + return fmt.Errorf("unlink cell %s", cellID) } + break } - cell := s.Get(targetCellID) - cell.Model().BackgroundColor = protoBlock.Model().BackgroundColor - - row = s.Get(row.Model().Id) - row.Model().ChildrenIds = append(row.Model().ChildrenIds, targetCellID) - normalizeRow(nil, colIdx, row) } } + if !s.Unlink(req.TargetId) { + return fmt.Errorf("unlink column header") + } return nil } @@ -626,6 +500,93 @@ func (t *editor) ColumnDuplicate(s *state.State, req pb.RpcBlockTableColumnDupli return newCol.Model().Id, nil } +func (t *editor) ColumnMove(s *state.State, req pb.RpcBlockTableColumnMoveRequest) error { + switch req.Position { + case model.Block_Left: + req.Position = model.Block_Top + case model.Block_Right: + req.Position = model.Block_Bottom + default: + return fmt.Errorf("position is not supported") + } + _, err := pickColumn(s, req.TargetId) + if err != nil { + return fmt.Errorf("get target column: %w", err) + } + _, err = pickColumn(s, req.DropTargetId) + if err != nil { + return fmt.Errorf("get drop target column: %w", err) + } + + tb, err := NewTable(s, req.TargetId) + if err != nil { + return fmt.Errorf("init table block: %w", err) + } + + if !s.Unlink(req.TargetId) { + return fmt.Errorf("unlink target column") + } + if err = s.InsertTo(req.DropTargetId, req.Position, req.TargetId); err != nil { + return fmt.Errorf("insert column: %w", err) + } + + colIdx := tb.MakeColumnIndex() + + for _, id := range tb.RowIDs() { + row, err := getRow(s, id) + if err != nil { + return fmt.Errorf("get row %s: %w", id, err) + } + normalizeRow(nil, colIdx, row) + } + + return nil +} + +func (t *editor) ColumnListFill(s *state.State, req pb.RpcBlockTableColumnListFillRequest) error { + if len(req.BlockIds) == 0 { + return fmt.Errorf("empty row list") + } + + tb, err := NewTable(s, req.BlockIds[0]) + if err != nil { + return fmt.Errorf("init table: %w", err) + } + + rows := tb.RowIDs() + + for _, colID := range req.BlockIds { + for _, rowID := range rows { + id := MakeCellID(rowID, colID) + if s.Exists(id) { + continue + } + _, err := addCell(s, rowID, colID) + if err != nil { + return fmt.Errorf("add cell %s: %w", id, err) + } + + row, err := getRow(s, rowID) + if err != nil { + return fmt.Errorf("get row %s: %w", rowID, err) + } + + row.Model().ChildrenIds = append(row.Model().ChildrenIds, id) + } + } + + colIdx := tb.MakeColumnIndex() + for _, rowID := range rows { + row, err := getRow(s, rowID) + if err != nil { + return fmt.Errorf("get row %s: %w", rowID, err) + } + normalizeRow(nil, colIdx, row) + } + + return nil +} + func (t *editor) Expand(s *state.State, req pb.RpcBlockTableExpandRequest) error { tb, err := NewTable(s, req.TargetId) if err != nil { @@ -714,34 +675,88 @@ func (t *editor) Sort(s *state.State, req pb.RpcBlockTableSortRequest) error { return nil } -func (t *editor) CellCreate(s *state.State, rowID string, colID string, b *model.Block) (string, error) { - tb, err := NewTable(s, rowID) +func (t *editor) cleanupTables(_ smartblock.ApplyInfo) error { + if t.sb == nil { + return fmt.Errorf("nil smartblock") + } + s := t.sb.NewState() + + err := s.Iterate(func(b simple.Block) bool { + if b.Model().GetTable() == nil { + return true + } + + tb, err := NewTable(s, b.Model().Id) + if err != nil { + log.Errorf("cleanup: init table %s: %s", b.Model().Id, err) + return true + } + err = t.RowListClean(s, pb.RpcBlockTableRowListCleanRequest{ + BlockIds: tb.RowIDs(), + }) + if err != nil { + log.Errorf("cleanup table %s: %s", b.Model().Id, err) + return true + } + return true + }) if err != nil { - return "", fmt.Errorf("initialize table state: %w", err) + log.Errorf("cleanup iterate: %s", err) } - row, err := getRow(s, rowID) - if err != nil { - return "", fmt.Errorf("get row: %w", err) - } - if _, err = pickColumn(s, colID); err != nil { - return "", fmt.Errorf("pick column: %w", err) + if err = t.sb.Apply(s, smartblock.KeepInternalFlags); err != nil { + if errors.Is(err, source.ErrReadOnly) { + return nil + } + log.Errorf("cleanup apply: %s", err) } + return nil +} - cellID, err := addCell(s, rowID, colID) +func (t *editor) cloneColumnStyles(s *state.State, srcColID, targetColID string) error { + tb, err := NewTable(s, srcColID) if err != nil { - return "", fmt.Errorf("add cell: %w", err) + return fmt.Errorf("init table block: %w", err) } - cell := s.Get(cellID) - cell.Model().Content = b.Content - if err := s.InsertTo(rowID, model.Block_Inner, cellID); err != nil { - return "", fmt.Errorf("insert to: %w", err) - } - colIdx := tb.MakeColumnIndex() - normalizeRow(nil, colIdx, row) - return cellID, nil + for _, rowID := range tb.RowIDs() { + row, err := pickRow(s, rowID) + if err != nil { + return fmt.Errorf("pick row: %w", err) + } + + var protoBlock simple.Block + for _, cellID := range row.Model().ChildrenIds { + _, colID, err := ParseCellID(cellID) + if err != nil { + return fmt.Errorf("parse cell id: %w", err) + } + + if colID == srcColID { + protoBlock = s.Pick(cellID) + } + } + + if protoBlock != nil && protoBlock.Model().BackgroundColor != "" { + targetCellID := MakeCellID(rowID, targetColID) + + if !s.Exists(targetCellID) { + _, err := addCell(s, rowID, targetColID) + if err != nil { + return fmt.Errorf("add cell: %w", err) + } + } + cell := s.Get(targetCellID) + cell.Model().BackgroundColor = protoBlock.Model().BackgroundColor + + row = s.Get(row.Model().Id) + row.Model().ChildrenIds = append(row.Model().ChildrenIds, targetCellID) + normalizeRow(nil, colIdx, row) + } + } + + return nil } func (t *editor) addColumnHeader(s *state.State) (string, error) { diff --git a/core/block/editor/table/editor_test.go b/core/block/editor/table/editor_test.go new file mode 100644 index 000000000..4b937016b --- /dev/null +++ b/core/block/editor/table/editor_test.go @@ -0,0 +1,2128 @@ +package table + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/anytype-heart/core/block/editor/smartblock" + "github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest" + "github.com/anyproto/anytype-heart/core/block/editor/state" + "github.com/anyproto/anytype-heart/core/block/restriction" + "github.com/anyproto/anytype-heart/core/block/simple" + "github.com/anyproto/anytype-heart/pb" + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +func TestEditor_TableCreate(t *testing.T) { + t.Run("table create - no error", func(t *testing.T) { + // given + sb := smarttest.New("root") + sb.AddBlock(simple.New(&model.Block{Id: "root"})) + editor := NewEditor(sb) + + s := sb.NewState() + + // when + id, err := editor.TableCreate(s, pb.RpcBlockTableCreateRequest{ + TargetId: "root", + Position: model.Block_Inner, + Columns: 3, + Rows: 4, + }) + + // then + require.NoError(t, err) + assert.NotEmpty(t, id) + + tb, err := NewTable(s, id) + + require.NoError(t, err) + + assert.Len(t, tb.ColumnIDs(), 3) + assert.Len(t, tb.RowIDs(), 4) + + for _, rowID := range tb.RowIDs() { + row, err := pickRow(s, rowID) + + require.NoError(t, err) + assert.Empty(t, row.Model().ChildrenIds) + } + }) + + t.Run("table create - in object with Blocks restriction", func(t *testing.T) { + // given + sb := smarttest.New("root") + sb.AddBlock(simple.New(&model.Block{Id: "root"})) + sb.TestRestrictions = restriction.Restrictions{Object: restriction.ObjectRestrictions{model.Restrictions_Blocks}} + e := NewEditor(sb) + + s := sb.NewState() + + // when + _, err := e.TableCreate(s, pb.RpcBlockTableCreateRequest{}) + + // then + assert.Error(t, err) + assert.True(t, errors.Is(err, restriction.ErrRestricted)) + }) + + t.Run("table create - error on insertion", func(t *testing.T) { + // given + sb := smarttest.New("root") + sb.AddBlock(simple.New(&model.Block{Id: "root"})) + e := NewEditor(sb) + + s := sb.NewState() + + // when + _, err := e.TableCreate(s, pb.RpcBlockTableCreateRequest{ + TargetId: "no_such_block", + Position: model.Block_Inner, + Columns: 3, + Rows: 4, + }) + + // then + assert.Error(t, err) + }) + + t.Run("table create - error on column creation", func(t *testing.T) { + // given + sb := smarttest.New("root") + sb.AddBlock(simple.New(&model.Block{Id: "root", ChildrenIds: []string{"random_column_id"}})).AddBlock(simple.New(&model.Block{Id: "random_column_id"})) + e := editor{sb: sb, generateColID: func() string { + return "random_column_id" + }} + + s := sb.NewState() + + // when + _, err := e.TableCreate(s, pb.RpcBlockTableCreateRequest{ + TargetId: "root", + Position: model.Block_Inner, + Columns: 3, + Rows: 2, + }) + + // then + assert.Error(t, err) + }) + + t.Run("table create - error on row creation", func(t *testing.T) { + // given + sb := smarttest.New("root") + sb.AddBlock(simple.New(&model.Block{Id: "root", ChildrenIds: []string{"random_row_id"}})).AddBlock(simple.New(&model.Block{Id: "random_row_id"})) + e := editor{ + sb: sb, + generateColID: func() string { return "random_col_id" }, + generateRowID: func() string { return "random_row_id" }, + } + + s := sb.NewState() + + // when + _, err := e.TableCreate(s, pb.RpcBlockTableCreateRequest{ + TargetId: "root", + Position: model.Block_Inner, + Columns: 1, + Rows: 3, + }) + + // then + assert.Error(t, err) + }) + + t.Run("table create - with header row", func(t *testing.T) { + // given + sb := smarttest.New("root") + sb.AddBlock(simple.New(&model.Block{Id: "root"})) + editor := NewEditor(sb) + + s := sb.NewState() + + // when + id, err := editor.TableCreate(s, pb.RpcBlockTableCreateRequest{ + TargetId: "root", + Position: model.Block_Inner, + Columns: 3, + Rows: 4, + WithHeaderRow: true, + }) + + // then + require.NoError(t, err) + assert.NotEmpty(t, id) + + tb, err := NewTable(s, id) + require.NoError(t, err) + + assert.Len(t, tb.ColumnIDs(), 3) + assert.Len(t, tb.RowIDs(), 4) + + row, err := tb.PickRow(tb.RowIDs()[0]) + require.NoError(t, err) + headerRowId := row.Model().Id + + headerRow := row.Model().GetTableRow() + require.NotNil(t, headerRow) + assert.True(t, headerRow.IsHeader) + + cells := row.Model().ChildrenIds + assert.Len(t, cells, 3) + + for _, cellID := range cells { + rowID, _, err := ParseCellID(cellID) + require.NoError(t, err) + require.Equal(t, headerRowId, rowID) + + cell := s.Get(cellID) + require.NotNil(t, cell) + + assert.Equal(t, "grey", cell.Model().BackgroundColor) + } + }) + + t.Run("table create - with 0 rows and header row", func(t *testing.T) { + // given + sb := smarttest.New("root") + sb.AddBlock(simple.New(&model.Block{Id: "root"})) + editor := NewEditor(sb) + + s := sb.NewState() + + // when + _, err := editor.TableCreate(s, pb.RpcBlockTableCreateRequest{ + TargetId: "root", + Position: model.Block_Inner, + Columns: 3, + Rows: 0, + WithHeaderRow: true, + }) + + // then + assert.Error(t, err) + }) +} + +func TestEditor_CellCreate(t *testing.T) { + for _, tc := range []struct { + name string + source *state.State + colID, rowID string + block *model.Block + }{ + { + name: "no table in state", + source: state.NewDoc("root", nil).NewState(), + }, + { + name: "failed to find row", + source: mkTestTable([]string{"col1"}, []string{"row1", "row2"}, nil), + colID: "col1", + rowID: "row3", + }, + { + name: "failed to find column", + source: mkTestTable([]string{"col1"}, []string{"row1", "row2"}, nil), + colID: "col2", + rowID: "row2", + }, + { + name: "failed to add a cell", + source: mkTestTable([]string{"col1"}, []string{"row1", "row2"}, [][]string{{"row1-col1"}}), + colID: "col1", + rowID: "row1", + }, + } { + t.Run(tc.name, func(t *testing.T) { + e := editor{} + _, err := e.CellCreate(tc.source, tc.rowID, tc.colID, tc.block) + assert.Error(t, err) + }) + } +} + +func TestEditor_RowCreate(t *testing.T) { + type testCase struct { + name string + source *state.State + newRowId string + req pb.RpcBlockTableRowCreateRequest + want *state.State + } + + for _, tc := range []testCase{ + { + name: "cells are not affected", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), + newRowId: "row3", + req: pb.RpcBlockTableRowCreateRequest{ + TargetId: "row1", + Position: model.Block_Bottom, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row3", "row2"}, [][]string{{"row1-col2"}}), + }, + { + name: "between, bottom position", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), + newRowId: "row3", + req: pb.RpcBlockTableRowCreateRequest{ + TargetId: "row1", + Position: model.Block_Bottom, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row3", "row2"}, nil), + }, + { + name: "between, top position", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), + newRowId: "row3", + req: pb.RpcBlockTableRowCreateRequest{ + TargetId: "row2", + Position: model.Block_Top, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row3", "row2"}, nil), + }, + { + name: "at the beginning", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), + newRowId: "row3", + req: pb.RpcBlockTableRowCreateRequest{ + TargetId: "row1", + Position: model.Block_Top, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row3", "row1", "row2"}, nil), + }, + { + name: "at the end", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), + newRowId: "row3", + req: pb.RpcBlockTableRowCreateRequest{ + TargetId: "row2", + Position: model.Block_Bottom, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, nil), + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{ + generateRowID: idFromSlice([]string{tc.newRowId}), + } + id, err := tb.RowCreate(tc.source, tc.req) + require.NoError(t, err) + assert.Equal(t, tc.newRowId, id) + assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) + }) + } + + for _, tc := range []testCase{ + { + name: "no table in state", + source: state.NewDoc("root", nil).NewState(), + req: pb.RpcBlockTableRowCreateRequest{ + TargetId: "row", + Position: model.Block_Bottom, + }, + }, + { + name: "no table in state on inner creation", + source: state.NewDoc("root", nil).NewState(), + req: pb.RpcBlockTableRowCreateRequest{ + TargetId: "row", + Position: model.Block_Inner, + }, + }, + { + name: "invalid position", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), + req: pb.RpcBlockTableRowCreateRequest{ + TargetId: "row", + Position: model.Block_Replace, + }, + }, + { + name: "failed to add row", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), + req: pb.RpcBlockTableRowCreateRequest{ + TargetId: "row1", + Position: model.Block_Bottom, + }, + newRowId: "row2", + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{ + generateRowID: idFromSlice([]string{tc.newRowId}), + } + _, err := tb.RowCreate(tc.source, tc.req) + assert.Error(t, err) + }) + } +} + +func TestEditor_RowDelete(t *testing.T) { + t.Run("no error", func(t *testing.T) { + // given + s := mkTestTable([]string{"col"}, []string{"row1", "row2"}, [][]string{{"row1-col"}}) + e := editor{} + + // when + err := e.RowDelete(s, pb.RpcBlockTableRowDeleteRequest{TargetId: "row1"}) + + // then + require.NoError(t, err) + tb, err := NewTable(s, "col") + require.NoError(t, err) + assert.Len(t, tb.RowIDs(), 1) + assert.Equal(t, "row2", tb.RowIDs()[0]) + }) + + t.Run("no such row", func(t *testing.T) { + // given + s := mkTestTable([]string{"col"}, []string{"row1", "row2"}, [][]string{{"row1-col"}}) + e := editor{} + + // when + err := e.RowDelete(s, pb.RpcBlockTableRowDeleteRequest{TargetId: "row4"}) + + // then + assert.Error(t, err) + }) + + t.Run("invalid table", func(t *testing.T) { + // given + s := mkTestTable([]string{"col"}, []string{"row1", "row2"}, [][]string{{"row1-col"}}) + s.Unlink("row1") + e := editor{} + + // when + err := e.RowDelete(s, pb.RpcBlockTableRowDeleteRequest{TargetId: "row1"}) + + // then + assert.Error(t, err) + }) +} + +func TestEditor_RowDuplicate(t *testing.T) { + type testCase struct { + name string + source *state.State + newRowId string + req pb.RpcBlockTableRowDuplicateRequest + want *state.State + } + for _, tc := range []testCase{ + { + name: "fully filled", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + }, withBlockContents(map[string]*model.Block{ + "row1-col1": mkTextBlock("test11"), + "row1-col2": mkTextBlock("test12"), + "row2-col1": mkTextBlock("test21"), + })), + newRowId: "row3", + req: pb.RpcBlockTableRowDuplicateRequest{ + BlockId: "row1", + TargetId: "row2", + Position: model.Block_Bottom, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + {"row3-col1", "row3-col2"}, + }, withBlockContents(map[string]*model.Block{ + "row1-col1": mkTextBlock("test11"), + "row1-col2": mkTextBlock("test12"), + "row2-col1": mkTextBlock("test21"), + "row3-col1": mkTextBlock("test11"), + "row3-col2": mkTextBlock("test12"), + })), + }, + { + name: "partially filled", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1"}, + {"row2-col2"}, + }, withBlockContents(map[string]*model.Block{ + "row1-col1": mkTextBlock("test11"), + "row2-col2": mkTextBlock("test22"), + })), + newRowId: "row3", + req: pb.RpcBlockTableRowDuplicateRequest{ + BlockId: "row2", + TargetId: "row1", + Position: model.Block_Top, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row3", "row1", "row2"}, + [][]string{ + {"row3-col2"}, + {"row1-col1"}, + {"row2-col2"}, + }, withBlockContents(map[string]*model.Block{ + "row1-col1": mkTextBlock("test11"), + "row2-col2": mkTextBlock("test22"), + "row3-col2": mkTextBlock("test22"), + })), + }, + { + name: "empty", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, + [][]string{ + {}, + {}, + }), + newRowId: "row3", + req: pb.RpcBlockTableRowDuplicateRequest{ + BlockId: "row2", + TargetId: "row1", + Position: model.Block_Bottom, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row3", "row2"}, + [][]string{ + {}, + {}, + {}, + }), + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{ + generateRowID: idFromSlice([]string{tc.newRowId}), + } + id, err := tb.RowDuplicate(tc.source, tc.req) + require.NoError(t, err) + assert.Equal(t, tc.newRowId, id) + assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) + }) + } + + for _, tc := range []testCase{ + { + name: "invalid position", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), + req: pb.RpcBlockTableRowDuplicateRequest{ + BlockId: "row1", + TargetId: "row1", + Position: model.Block_Inner, + }, + }, + { + name: "no former row", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), + newRowId: "row4", + req: pb.RpcBlockTableRowDuplicateRequest{ + BlockId: "row3", + TargetId: "row1", + Position: model.Block_Bottom, + }, + }, + { + name: "target block is not a row", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), + newRowId: "row4", + req: pb.RpcBlockTableRowDuplicateRequest{ + BlockId: "row1", + TargetId: "rows", + Position: model.Block_Bottom, + }, + }, + { + name: "failed to add new row", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), + newRowId: "row2", + req: pb.RpcBlockTableRowDuplicateRequest{ + BlockId: "row2", + TargetId: "row1", + Position: model.Block_Bottom, + }, + }, + { + name: "cell is not found", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), + func(s *state.State) *state.State { + row := s.Pick("row2") + row.Model().ChildrenIds = []string{"row2-col2"} + s.Set(row) + return s + }), + newRowId: "row3", + req: pb.RpcBlockTableRowDuplicateRequest{ + BlockId: "row2", + TargetId: "row1", + Position: model.Block_Bottom, + }, + }, + { + name: "cell has invalid name", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), + func(s *state.State) *state.State { + row := s.Pick("row1") + row.Model().ChildrenIds = []string{"cell"} + s.Set(row) + s.Add(simple.New(&model.Block{Id: "cell"})) + return s + }), + newRowId: "row3", + req: pb.RpcBlockTableRowDuplicateRequest{ + BlockId: "row1", + TargetId: "row1", + Position: model.Block_Bottom, + }, + }, + { + name: "failed to add new cell", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col1"}}), + func(s *state.State) *state.State { + s.Add(simple.New(&model.Block{Id: "row3-col1"})) + return s + }), + newRowId: "row3", + req: pb.RpcBlockTableRowDuplicateRequest{ + BlockId: "row1", + TargetId: "row1", + Position: model.Block_Bottom, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{ + generateRowID: idFromSlice([]string{tc.newRowId}), + } + _, err := tb.RowDuplicate(tc.source, tc.req) + assert.Error(t, err) + }) + } +} + +func TestEditor_RowListFill(t *testing.T) { + type testCase struct { + name string + source *state.State + req pb.RpcBlockTableRowListFillRequest + want *state.State + } + for _, tc := range []testCase{ + { + name: "empty", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{}), + req: pb.RpcBlockTableRowListFillRequest{ + BlockIds: []string{"row1", "row2"}, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + }), + }, + { + name: "fully filled", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + }), + req: pb.RpcBlockTableRowListFillRequest{ + BlockIds: []string{"row1", "row2"}, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + }), + }, + { + name: "partially filled", + source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2", "row3", "row4", "row5"}, + [][]string{ + {"row1-col1"}, + {"row2-col2"}, + {"row3-col3"}, + {"row4-col1", "row4-col3"}, + {"row5-col2", "row4-col3"}, + }), + req: pb.RpcBlockTableRowListFillRequest{ + BlockIds: []string{"row1", "row2", "row3", "row4", "row5"}, + }, + want: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2", "row3", "row4", "row5"}, + [][]string{ + {"row1-col1", "row1-col2", "row1-col3"}, + {"row2-col1", "row2-col2", "row2-col3"}, + {"row3-col1", "row3-col2", "row3-col3"}, + {"row4-col1", "row4-col2", "row4-col3"}, + {"row5-col1", "row5-col2", "row5-col3"}, + }), + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{} + err := tb.RowListFill(tc.source, tc.req) + require.NoError(t, err) + assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) + }) + } + + for _, tc := range []testCase{ + { + name: "bo block ids", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{}), + req: pb.RpcBlockTableRowListFillRequest{ + BlockIds: nil, + }, + }, + { + name: "no such row", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{}), + req: pb.RpcBlockTableRowListFillRequest{ + BlockIds: []string{"row3"}, + }, + }, + { + name: "ids do not belong to rows", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{}), + req: pb.RpcBlockTableRowListFillRequest{ + BlockIds: []string{"col1", "row1", "root"}, + }, + }, + { + name: "no table in state", + source: state.NewDoc("root", nil).NewState(), + req: pb.RpcBlockTableRowListFillRequest{ + BlockIds: []string{"row1"}, + }, + }, + } { + tb := editor{} + err := tb.RowListFill(tc.source, tc.req) + assert.Error(t, err) + } +} + +func TestEditor_RowListClean(t *testing.T) { + type testCase struct { + name string + source *state.State + req pb.RpcBlockTableRowListCleanRequest + want *state.State + } + + for _, tc := range []testCase{ + { + name: "empty rows", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{ + {}, + {}, + }), + req: pb.RpcBlockTableRowListCleanRequest{ + BlockIds: []string{"row1", "row2"}, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{ + {}, + {}, + }), + }, + { + name: "rows with empty blocks", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col2"}, + }), + req: pb.RpcBlockTableRowListCleanRequest{ + BlockIds: []string{"row1", "row2"}, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{ + {}, + {}, + }), + }, + { + name: "rows with not empty text block", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col2"}, + }, withBlockContents(map[string]*model.Block{ + "row1-col1": mkTextBlock("test11"), + "row2-col1": mkTextBlock(""), + })), + req: pb.RpcBlockTableRowListCleanRequest{ + BlockIds: []string{"row1", "row2"}, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{ + {"row1-col1"}, + {}, + }, withBlockContents(map[string]*model.Block{ + "row1-col1": mkTextBlock("test11"), + })), + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{} + err := tb.RowListClean(tc.source, tc.req) + require.NoError(t, err) + assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) + }) + } + + for _, tc := range []testCase{ + { + name: "block ids list is empty", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), + req: pb.RpcBlockTableRowListCleanRequest{ + BlockIds: nil, + }, + }, + { + name: "no such row", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), + req: pb.RpcBlockTableRowListCleanRequest{ + BlockIds: []string{"row3"}, + }, + }, + { + name: "ids in list do not belong to rows", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), + req: pb.RpcBlockTableRowListCleanRequest{ + BlockIds: []string{"table", "col1", "row2"}, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{} + err := tb.RowListClean(tc.source, tc.req) + assert.Error(t, err) + }) + } +} + +func TestEditor_RowSetHeader(t *testing.T) { + type testCase struct { + name string + source *state.State + req pb.RpcBlockTableRowSetHeaderRequest + want *state.State + } + + for _, tc := range []testCase{ + { + name: "header row moves up", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3", "row4"}, nil), + req: pb.RpcBlockTableRowSetHeaderRequest{ + TargetId: "row3", + IsHeader: true, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row3", "row1", "row2", "row4"}, nil, + withRowBlockContents(map[string]*model.BlockContentTableRow{ + "row3": {IsHeader: true}, + })), + }, + { + name: "non-header row moves down", + source: mkTestTable([]string{"col1", "col2"}, []string{"row2", "row3", "row1", "row4"}, nil, + withRowBlockContents(map[string]*model.BlockContentTableRow{ + "row2": {IsHeader: true}, + "row3": {IsHeader: true}, + })), + req: pb.RpcBlockTableRowSetHeaderRequest{ + TargetId: "row2", + IsHeader: false, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row3", "row2", "row1", "row4"}, nil, + withRowBlockContents(map[string]*model.BlockContentTableRow{ + "row3": {IsHeader: true}, + })), + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{} + err := tb.RowSetHeader(tc.source, tc.req) + require.NoError(t, err) + assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) + }) + } + + for _, tc := range []testCase{ + { + name: "no table in state", + source: state.NewDoc("root", nil).NewState(), + req: pb.RpcBlockTableRowSetHeaderRequest{ + TargetId: "row2", + }, + }, + { + name: "no such row", + source: mkTestTable([]string{"col1", "col2"}, []string{"row2", "row3", "row1", "row4"}, nil), + req: pb.RpcBlockTableRowSetHeaderRequest{ + TargetId: "row0", + IsHeader: true, + }, + }, + { + name: "target block is not a row", + source: mkTestTable([]string{"col1", "col2"}, []string{"row2", "row3", "row1", "row4"}, nil), + req: pb.RpcBlockTableRowSetHeaderRequest{ + TargetId: "col2", + IsHeader: false, + }, + }, + { + name: "rows normalization error", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), + func(s *state.State) *state.State { + rows := s.Pick("rows") + rows.Model().ChildrenIds = []string{"row1", "row2", "row3"} + s.Set(rows) + return s + }), + req: pb.RpcBlockTableRowSetHeaderRequest{ + TargetId: "row2", + IsHeader: true, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{} + err := tb.RowSetHeader(tc.source, tc.req) + assert.Error(t, err) + }) + } +} + +func TestEditor_ColumnCreate(t *testing.T) { + type testCase struct { + name string + source *state.State + newColId string + req pb.RpcBlockTableColumnCreateRequest + want *state.State + } + + for _, tc := range []struct { + name string + source *state.State + newColId string + req pb.RpcBlockTableColumnCreateRequest + want *state.State + }{ + { + name: "between, to the right", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), + newColId: "col3", + req: pb.RpcBlockTableColumnCreateRequest{ + TargetId: "col1", + Position: model.Block_Right, + }, + want: mkTestTable([]string{"col1", "col3", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), + }, + { + name: "between, to the left", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), + newColId: "col3", + req: pb.RpcBlockTableColumnCreateRequest{ + TargetId: "col2", + Position: model.Block_Left, + }, + want: mkTestTable([]string{"col1", "col3", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), + }, + { + name: "at the beginning", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), + newColId: "col3", + req: pb.RpcBlockTableColumnCreateRequest{ + TargetId: "col1", + Position: model.Block_Left, + }, + want: mkTestTable([]string{"col3", "col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), + }, + { + name: "at the end", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), + newColId: "col3", + req: pb.RpcBlockTableColumnCreateRequest{ + TargetId: "col2", + Position: model.Block_Right, + }, + want: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{ + generateColID: idFromSlice([]string{tc.newColId}), + } + id, err := tb.ColumnCreate(tc.source, tc.req) + require.NoError(t, err) + assert.Equal(t, tc.newColId, id) + assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) + }) + } + + for _, tc := range []testCase{ + { + name: "invalid position", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), + req: pb.RpcBlockTableColumnCreateRequest{ + TargetId: "col2", + Position: model.Block_Top, + }, + }, + { + name: "failed to find target column - left", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), + req: pb.RpcBlockTableColumnCreateRequest{ + TargetId: "col0", + Position: model.Block_Left, + }, + }, + { + name: "failed to find target column - right", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), + func(s *state.State) *state.State { + col := s.Pick("col1") + col.Model().Content = nil + s.Set(col) + return s + }), + req: pb.RpcBlockTableColumnCreateRequest{ + TargetId: "col1", + Position: model.Block_Right, + }, + }, + { + name: "no table in state - inner", + source: state.NewDoc("root", nil).NewState(), + req: pb.RpcBlockTableColumnCreateRequest{ + TargetId: "col1", + Position: model.Block_Inner, + }, + }, + { + name: "failed to add column", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), + newColId: "col2", + req: pb.RpcBlockTableColumnCreateRequest{ + TargetId: "col1", + Position: model.Block_Left, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{ + generateColID: idFromSlice([]string{tc.newColId}), + } + _, err := tb.ColumnCreate(tc.source, tc.req) + assert.Error(t, err) + }) + } +} + +func TestEditor_ColumnDelete(t *testing.T) { + type testCase struct { + name string + source *state.State + req pb.RpcBlockTableColumnDeleteRequest + want *state.State + } + for _, tc := range []testCase{ + { + name: "partial table", + source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col2", "row1-col3"}, + {"row2-col1", "row2-col3"}, + }), + req: pb.RpcBlockTableColumnDeleteRequest{ + TargetId: "col2", + }, + want: mkTestTable([]string{"col1", "col3"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col3"}, + {"row2-col1", "row2-col3"}, + }), + }, + { + name: "filled table", + source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col2", "row1-col3"}, + {"row2-col1", "row2-col2", "row2-col3"}, + }), + req: pb.RpcBlockTableColumnDeleteRequest{ + TargetId: "col3", + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + }), + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{} + err := tb.ColumnDelete(tc.source, tc.req) + require.NoError(t, err) + assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) + }) + } + + for _, tc := range []testCase{ + { + name: "no such column", + source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, nil), + req: pb.RpcBlockTableColumnDeleteRequest{ + TargetId: "col4", + }, + }, + { + name: "no table in state", + source: state.NewDoc("root", map[string]simple.Block{ + "col1": simple.New(&model.Block{Id: "col", Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}}), + }).NewState(), + req: pb.RpcBlockTableColumnDeleteRequest{ + TargetId: "col1", + }, + }, + { + name: "invalid cell id", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1"}, nil), + func(s *state.State) *state.State { + row := s.Pick("row1") + row.Model().ChildrenIds = []string{"cell"} + s.Set(row) + s.Set(simple.New(&model.Block{Id: "cell"})) + return s + }), + req: pb.RpcBlockTableColumnDeleteRequest{ + TargetId: "col1", + }, + }, + { + name: "cannot find row", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1"}, nil), + func(s *state.State) *state.State { + rows := s.Pick("rows") + rows.Model().ChildrenIds = []string{"row1", "row2"} + s.Set(rows) + return s + }), + req: pb.RpcBlockTableColumnDeleteRequest{ + TargetId: "col1", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{} + err := tb.ColumnDelete(tc.source, tc.req) + assert.Error(t, err) + }) + } +} + +func TestColumnDuplicate(t *testing.T) { + type testCase struct { + name string + source *state.State + newColId string + req pb.RpcBlockTableColumnDuplicateRequest + want *state.State + } + for _, tc := range []testCase{ + { + name: "fully filled", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + }, withBlockContents(map[string]*model.Block{ + "row1-col1": mkTextBlock("test11"), + "row2-col1": mkTextBlock("test21"), + })), + newColId: "col3", + req: pb.RpcBlockTableColumnDuplicateRequest{ + BlockId: "col1", + TargetId: "col2", + Position: model.Block_Right, + }, + want: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col2", "row1-col3"}, + {"row2-col1", "row2-col2", "row2-col3"}, + }, withBlockContents(map[string]*model.Block{ + "row1-col1": mkTextBlock("test11"), + "row2-col1": mkTextBlock("test21"), + "row1-col3": mkTextBlock("test11"), + "row2-col3": mkTextBlock("test21"), + })), + }, + { + name: "partially filled", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, + [][]string{ + {"row1-col1"}, + {"row2-col2"}, + {}, + }, withBlockContents(map[string]*model.Block{ + "row2-col2": mkTextBlock("test22"), + })), + newColId: "col3", + req: pb.RpcBlockTableColumnDuplicateRequest{ + BlockId: "col2", + TargetId: "col1", + Position: model.Block_Left, + }, + want: mkTestTable([]string{"col3", "col1", "col2"}, []string{"row1", "row2", "row3"}, + [][]string{ + {"row1-col1"}, + {"row2-col3", "row2-col2"}, + {}, + }, withBlockContents(map[string]*model.Block{ + "row2-col2": mkTextBlock("test22"), + "row2-col3": mkTextBlock("test22"), + })), + }, + { + name: "empty", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, + [][]string{ + {"row1-col1"}, + {}, + {}, + }), + newColId: "col3", + req: pb.RpcBlockTableColumnDuplicateRequest{ + BlockId: "col2", + TargetId: "col1", + Position: model.Block_Left, + }, + want: mkTestTable([]string{"col3", "col1", "col2"}, []string{"row1", "row2", "row3"}, + [][]string{ + {"row1-col1"}, + {}, + {}, + }), + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{ + generateColID: idFromSlice([]string{tc.newColId}), + } + id, err := tb.ColumnDuplicate(tc.source, tc.req) + require.NoError(t, err) + assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) + assert.Equal(t, tc.newColId, id) + }) + } + + for _, tc := range []testCase{ + { + name: "invalid position", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, nil), + newColId: "col3", + req: pb.RpcBlockTableColumnDuplicateRequest{ + BlockId: "col2", + TargetId: "col1", + Position: model.Block_Top, + }, + }, + { + name: "failed to find source column", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, nil), + newColId: "col3", + req: pb.RpcBlockTableColumnDuplicateRequest{ + BlockId: "col4", + TargetId: "col1", + Position: model.Block_Left, + }, + }, + { + name: "failed to find target column", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, nil), + newColId: "col3", + req: pb.RpcBlockTableColumnDuplicateRequest{ + BlockId: "col2", + TargetId: "col4", + Position: model.Block_Left, + }, + }, + { + name: "table is broken", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, nil), + func(s *state.State) *state.State { + table := s.Pick("table") + table.Model().ChildrenIds = []string{"rows", "columns", "other"} + s.Set(table) + return s + }), + newColId: "col3", + req: pb.RpcBlockTableColumnDuplicateRequest{ + BlockId: "col2", + TargetId: "col2", + Position: model.Block_Left, + }, + }, + { + name: "failed to add new column", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, nil), + newColId: "col1", + req: pb.RpcBlockTableColumnDuplicateRequest{ + BlockId: "col2", + TargetId: "col2", + Position: model.Block_Right, + }, + }, + { + name: "failed to find a row", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1"}, nil), + func(s *state.State) *state.State { + rows := s.Pick("rows") + rows.Model().ChildrenIds = []string{"row1", "row2"} + s.Set(rows) + return s + }), + newColId: "col3", + req: pb.RpcBlockTableColumnDuplicateRequest{ + BlockId: "col2", + TargetId: "col2", + Position: model.Block_Right, + }, + }, + { + name: "cell with invalid id", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1"}, nil), + func(s *state.State) *state.State { + row := s.Pick("row1") + row.Model().ChildrenIds = []string{"cell"} + s.Set(row) + return s + }), + newColId: "col3", + req: pb.RpcBlockTableColumnDuplicateRequest{ + BlockId: "col2", + TargetId: "col2", + Position: model.Block_Right, + }, + }, + { + name: "failed to find a cell", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1"}, nil), + func(s *state.State) *state.State { + row := s.Pick("row1") + row.Model().ChildrenIds = []string{"row1-col2"} + s.Set(row) + return s + }), + newColId: "col3", + req: pb.RpcBlockTableColumnDuplicateRequest{ + BlockId: "col2", + TargetId: "col2", + Position: model.Block_Right, + }, + }, + { + name: "failed to add new cell", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1"}, [][]string{{"row1-col1"}}), + func(s *state.State) *state.State { + s.Set(simple.New(&model.Block{Id: "row1-col3"})) + return s + }), + newColId: "col3", + req: pb.RpcBlockTableColumnDuplicateRequest{ + BlockId: "col1", + TargetId: "col1", + Position: model.Block_Left, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{ + generateColID: idFromSlice([]string{tc.newColId}), + } + _, err := tb.ColumnDuplicate(tc.source, tc.req) + assert.Error(t, err) + }) + } +} + +func TestEditor_ColumnMove(t *testing.T) { + type testCase struct { + name string + source *state.State + req pb.RpcBlockTableColumnMoveRequest + want *state.State + } + for _, tc := range []testCase{ + { + name: "partial table", + source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col2", "row1-col3"}, + {"row2-col1", "row2-col3"}, + }), + req: pb.RpcBlockTableColumnMoveRequest{ + TargetId: "col1", + DropTargetId: "col3", + Position: model.Block_Left, + }, + want: mkTestTable([]string{"col2", "col1", "col3"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col2", "row1-col1", "row1-col3"}, + {"row2-col1", "row2-col3"}, + }), + }, + { + name: "filled table", + source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col2", "row1-col3"}, + {"row2-col1", "row2-col2", "row2-col3"}, + }), + req: pb.RpcBlockTableColumnMoveRequest{ + TargetId: "col3", + DropTargetId: "col1", + Position: model.Block_Right, + }, + want: mkTestTable([]string{"col1", "col3", "col2"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col3", "row1-col2"}, + {"row2-col1", "row2-col3", "row2-col2"}, + }), + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{} + err := tb.ColumnMove(tc.source, tc.req) + require.NoError(t, err) + assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) + }) + } + + for _, tc := range []testCase{ + { + name: "invalid position of move", + source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, nil), + req: pb.RpcBlockTableColumnMoveRequest{ + TargetId: "col1", + DropTargetId: "col3", + Position: model.Block_Inner, + }, + }, + { + name: "no such column to move", + source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, nil), + req: pb.RpcBlockTableColumnMoveRequest{ + TargetId: "col1", + DropTargetId: "col4", + Position: model.Block_Right, + }, + }, + { + name: "no target column", + source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, nil), + req: pb.RpcBlockTableColumnMoveRequest{ + TargetId: "col5", + DropTargetId: "col3", + Position: model.Block_Left, + }, + }, + { + name: "table is broken", + source: modifyState(mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, nil), + func(s *state.State) *state.State { + s.Unlink("rows") + return s + }), + req: pb.RpcBlockTableColumnMoveRequest{ + TargetId: "col1", + DropTargetId: "col3", + Position: model.Block_Left, + }, + }, + { + name: "failed to insert column", + source: modifyState(mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, nil), + func(s *state.State) *state.State { + s.Unlink("col3") + return s + }), + req: pb.RpcBlockTableColumnMoveRequest{ + TargetId: "col1", + DropTargetId: "col3", + Position: model.Block_Left, + }, + }, + { + name: "failed to find a row", + source: modifyState(mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, nil), + func(s *state.State) *state.State { + rows := s.Pick("rows") + rows.Model().ChildrenIds = []string{"row1", "row2", "row3"} + s.Set(rows) + return s + }), + req: pb.RpcBlockTableColumnMoveRequest{ + TargetId: "col1", + DropTargetId: "col3", + Position: model.Block_Left, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{} + err := tb.ColumnMove(tc.source, tc.req) + assert.Error(t, err) + }) + } +} + +func TestEditor_ColumnListFill(t *testing.T) { + type testCase struct { + name string + source *state.State + req pb.RpcBlockTableColumnListFillRequest + want *state.State + } + for _, tc := range []testCase{ + { + name: "empty", + source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, [][]string{}), + req: pb.RpcBlockTableColumnListFillRequest{ + BlockIds: []string{"col2", "col1"}, + }, + want: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + }), + }, + { + name: "fully filled", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + }), + req: pb.RpcBlockTableColumnListFillRequest{ + BlockIds: []string{"col2", "col1"}, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + }), + }, + { + name: "partially filled", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, [][]string{ + {"row1-col1"}, + {"row2-col2"}, + {"row3-col1", "row3-col2"}, + }), + req: pb.RpcBlockTableColumnListFillRequest{ + BlockIds: []string{"col1", "col2"}, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + {"row3-col1", "row3-col2"}, + }), + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{} + err := tb.ColumnListFill(tc.source, tc.req) + require.NoError(t, err) + assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) + }) + } + + for _, tc := range []testCase{ + { + name: "empty ids list", + source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, [][]string{}), + req: pb.RpcBlockTableColumnListFillRequest{ + BlockIds: nil, + }, + }, + { + name: "no table in state", + source: state.NewDoc("root", nil).NewState(), + req: pb.RpcBlockTableColumnListFillRequest{ + BlockIds: []string{"col1"}, + }, + }, + { + name: "failed to get a row", + source: modifyState(mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, [][]string{}), + func(s *state.State) *state.State { + rows := s.Pick("rows") + rows.Model().ChildrenIds = []string{"row1", "row2", "row3"} + s.Set(rows) + return s + }), + req: pb.RpcBlockTableColumnListFillRequest{ + BlockIds: []string{"col1"}, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{} + err := tb.ColumnListFill(tc.source, tc.req) + assert.Error(t, err) + }) + } +} + +func TestExpand(t *testing.T) { + type testCase struct { + name string + source *state.State + newColIds []string + newRowIds []string + req pb.RpcBlockTableExpandRequest + want *state.State + } + for _, tc := range []testCase{ + { + name: "only rows", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row2-col2"}}), + newRowIds: []string{"row3", "row4"}, + req: pb.RpcBlockTableExpandRequest{ + TargetId: "table", + Rows: 2, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3", "row4"}, [][]string{{"row2-col2"}}), + }, + { + name: "only columns", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row2-col2"}}), + newColIds: []string{"col3", "col4"}, + req: pb.RpcBlockTableExpandRequest{ + TargetId: "table", + Columns: 2, + }, + want: mkTestTable([]string{"col1", "col2", "col3", "col4"}, []string{"row1", "row2"}, [][]string{{"row2-col2"}}), + }, + { + name: "both columns and rows", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row2-col2"}}), + newRowIds: []string{"row3", "row4"}, + newColIds: []string{"col3", "col4"}, + req: pb.RpcBlockTableExpandRequest{ + TargetId: "table", + Rows: 2, + Columns: 2, + }, + want: mkTestTable([]string{"col1", "col2", "col3", "col4"}, []string{"row1", "row2", "row3", "row4"}, [][]string{{"row2-col2"}}), + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{ + generateColID: idFromSlice(tc.newColIds), + generateRowID: idFromSlice(tc.newRowIds), + } + err := tb.Expand(tc.source, tc.req) + require.NoError(t, err) + assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) + }) + } + + for _, tc := range []testCase{ + { + name: "no table in state", + source: state.NewDoc("root", nil).NewState(), + newRowIds: []string{"row3", "row4"}, + newColIds: []string{"col3", "col4"}, + req: pb.RpcBlockTableExpandRequest{ + TargetId: "table", + Rows: 2, + Columns: 2, + }, + }, + { + name: "failed to create column", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row2-col2"}}), + func(s *state.State) *state.State { + s.Set(simple.New(&model.Block{Id: "col3", Content: &model.BlockContentOfTableColumn{TableColumn: &model.BlockContentTableColumn{}}})) + return s + }), + newRowIds: []string{"row3", "row4"}, + newColIds: []string{"col3", "col4"}, + req: pb.RpcBlockTableExpandRequest{ + TargetId: "table", + Rows: 2, + Columns: 2, + }, + }, + { + name: "failed to create row", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row2-col2"}}), + func(s *state.State) *state.State { + s.Set(simple.New(&model.Block{Id: "row4", Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{}}})) + return s + }), + newRowIds: []string{"row3", "row4"}, + newColIds: []string{"col3", "col4"}, + req: pb.RpcBlockTableExpandRequest{ + TargetId: "table", + Rows: 2, + Columns: 2, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{ + generateColID: idFromSlice(tc.newColIds), + generateRowID: idFromSlice(tc.newRowIds), + } + err := tb.Expand(tc.source, tc.req) + assert.Error(t, err) + }) + } +} + +func TestSort(t *testing.T) { + type testCase struct { + name string + source *state.State + req pb.RpcBlockTableSortRequest + want *state.State + } + for _, tc := range []testCase{ + { + name: "asc order", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + {"row3-col1", "row3-col2"}, + }, withBlockContents(map[string]*model.Block{ + "row1-col2": mkTextBlock("Abd"), + "row2-col2": mkTextBlock("bsd"), + "row3-col2": mkTextBlock("abc"), + })), + req: pb.RpcBlockTableSortRequest{ + ColumnId: "col2", + Type: model.BlockContentDataviewSort_Asc, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row3", "row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + {"row3-col1", "row3-col2"}, + }, withBlockContents(map[string]*model.Block{ + "row3-col2": mkTextBlock("abc"), + "row1-col2": mkTextBlock("Abd"), + "row2-col2": mkTextBlock("bsd"), + })), + }, + { + name: "desc order", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + {"row3-col1", "row3-col2"}, + }, withBlockContents(map[string]*model.Block{ + "row1-col2": mkTextBlock("234"), + "row2-col2": mkTextBlock("323"), + "row3-col2": mkTextBlock("123"), + })), + req: pb.RpcBlockTableSortRequest{ + ColumnId: "col2", + Type: model.BlockContentDataviewSort_Desc, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row2", "row1", "row3"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + {"row3-col1", "row3-col2"}, + }, withBlockContents(map[string]*model.Block{ + "row1-col2": mkTextBlock("234"), + "row2-col2": mkTextBlock("323"), + "row3-col2": mkTextBlock("123"), + })), + }, + { + name: "asc order with header rows", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3", "row4", "row5"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + {"row3-col1", "row3-col2"}, + {"row4-col1", "row4-col2"}, + {"row5-col1", "row5-col2"}, + }, withBlockContents(map[string]*model.Block{ + "row1-col2": mkTextBlock("555"), + "row2-col2": mkTextBlock("444"), + "row3-col2": mkTextBlock("333"), + "row4-col2": mkTextBlock("222"), + "row5-col2": mkTextBlock("111"), + }), + withRowBlockContents(map[string]*model.BlockContentTableRow{ + "row1": {IsHeader: true}, + "row3": {IsHeader: true}, + })), + req: pb.RpcBlockTableSortRequest{ + ColumnId: "col2", + Type: model.BlockContentDataviewSort_Asc, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row3", "row5", "row4", "row2"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row3-col1", "row3-col2"}, + {"row5-col1", "row5-col2"}, + {"row4-col1", "row4-col2"}, + {"row2-col1", "row2-col2"}, + }, withBlockContents(map[string]*model.Block{ + "row1-col2": mkTextBlock("555"), + "row2-col2": mkTextBlock("444"), + "row3-col2": mkTextBlock("333"), + "row4-col2": mkTextBlock("222"), + "row5-col2": mkTextBlock("111"), + }), + withRowBlockContents(map[string]*model.BlockContentTableRow{ + "row1": {IsHeader: true}, + "row3": {IsHeader: true}, + })), + }, + { + name: "desc order with header rows", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3", "row4", "row5"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row2-col1", "row2-col2"}, + {"row3-col1", "row3-col2"}, + {"row4-col1", "row4-col2"}, + {"row5-col1", "row5-col2"}, + }, withBlockContents(map[string]*model.Block{ + "row1-col2": mkTextBlock("555"), + "row2-col2": mkTextBlock("444"), + "row3-col2": mkTextBlock("333"), + "row4-col2": mkTextBlock("222"), + "row5-col2": mkTextBlock("111"), + }), + withRowBlockContents(map[string]*model.BlockContentTableRow{ + "row1": {IsHeader: true}, + "row3": {IsHeader: true}, + })), + req: pb.RpcBlockTableSortRequest{ + ColumnId: "col2", + Type: model.BlockContentDataviewSort_Desc, + }, + want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row3", "row2", "row4", "row5"}, + [][]string{ + {"row1-col1", "row1-col2"}, + {"row3-col1", "row3-col2"}, + {"row2-col1", "row2-col2"}, + {"row4-col1", "row4-col2"}, + {"row5-col1", "row5-col2"}, + }, withBlockContents(map[string]*model.Block{ + "row1-col2": mkTextBlock("555"), + "row2-col2": mkTextBlock("444"), + "row3-col2": mkTextBlock("333"), + "row4-col2": mkTextBlock("222"), + "row5-col2": mkTextBlock("111"), + }), + withRowBlockContents(map[string]*model.BlockContentTableRow{ + "row1": {IsHeader: true}, + "row3": {IsHeader: true}, + })), + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{} + err := tb.Sort(tc.source, tc.req) + require.NoError(t, err) + assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) + }) + } + + for _, tc := range []testCase{ + { + name: "failed to find column", + source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3", "row4", "row5"}, nil), + req: pb.RpcBlockTableSortRequest{ + ColumnId: "col3", + Type: model.BlockContentDataviewSort_Desc, + }, + }, + { + name: "table is invalid", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3", "row4", "row5"}, nil), + func(s *state.State) *state.State { + table := s.Pick("table") + table.Model().ChildrenIds = []string{"rows", "columns", "other"} + s.Set(table) + return s + }), + req: pb.RpcBlockTableSortRequest{ + ColumnId: "col2", + Type: model.BlockContentDataviewSort_Desc, + }, + }, + { + name: "failed to find a row", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), + func(s *state.State) *state.State { + rows := s.Pick("rows") + rows.Model().ChildrenIds = []string{"row1", "row2", "row3"} + s.Set(rows) + return s + }), + req: pb.RpcBlockTableSortRequest{ + ColumnId: "col2", + Type: model.BlockContentDataviewSort_Asc, + }, + }, + { + name: "invalid cell id", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, nil), + func(s *state.State) *state.State { + row := s.Pick("row1") + row.Model().ChildrenIds = []string{"cell"} + s.Set(row) + return s + }), + req: pb.RpcBlockTableSortRequest{ + ColumnId: "col2", + Type: model.BlockContentDataviewSort_Asc, + }, + }, + { + name: "failed to find cell", + source: modifyState(mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, nil), + func(s *state.State) *state.State { + row := s.Pick("row1") + row.Model().ChildrenIds = []string{"row1-col2"} + s.Set(row) + return s + }), + req: pb.RpcBlockTableSortRequest{ + ColumnId: "col2", + Type: model.BlockContentDataviewSort_Asc, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + tb := editor{} + err := tb.Sort(tc.source, tc.req) + assert.Error(t, err) + }) + } +} + +func TestEditor_cleanupTables(t *testing.T) { + t.Run("cannot do hook with nil smartblock", func(t *testing.T) { + // given + e := editor{} + + // when + err := e.cleanupTables(smartblock.ApplyInfo{}) + + // then + assert.Error(t, err) + }) + + t.Run("no error", func(t *testing.T) { + // given + sb := mkTestTableSb([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col1"}, {"row2-col2"}}, withBlockContents(map[string]*model.Block{ + "row1-col1": mkTextBlock("test11"), + })) + e := editor{sb: sb} + + // when + err := e.cleanupTables(smartblock.ApplyInfo{}) + + // then + require.NoError(t, err) + assert.Len(t, sb.Pick("row1").Model().ChildrenIds, 1) + assert.Empty(t, sb.Pick("row2").Model().ChildrenIds) + }) + + t.Run("broken table", func(t *testing.T) { + // given + sb := smarttest.New("root") + sb.AddBlock(simple.New(&model.Block{Id: "root", ChildrenIds: []string{"table"}})) + sb.AddBlock(simple.New(&model.Block{Id: "table", ChildrenIds: []string{"columns", "rows"}, Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}})) + sb.AddBlock(simple.New(&model.Block{Id: "columns", Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableColumns}}})) + e := editor{sb: sb} + + // when + err := e.cleanupTables(smartblock.ApplyInfo{}) + + // then + assert.NoError(t, err) + }) + + t.Run("iterate error", func(t *testing.T) { + // given + sb := smarttest.New("root") + sb.AddBlock(simple.New(&model.Block{Id: "root", ChildrenIds: []string{"root"}})) + e := editor{sb: sb} + + // when + err := e.cleanupTables(smartblock.ApplyInfo{}) + + // then + assert.NoError(t, err) + }) + + t.Run("raw list clean error", func(t *testing.T) { + // given + sb := smarttest.New("root") + sb.AddBlock(simple.New(&model.Block{Id: "root", ChildrenIds: []string{"table"}})) + sb.AddBlock(simple.New(&model.Block{Id: "table", ChildrenIds: []string{"columns", "rows"}, Content: &model.BlockContentOfTable{Table: &model.BlockContentTable{}}})) + sb.AddBlock(simple.New(&model.Block{Id: "rows", ChildrenIds: []string{"row"}, Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableRows}}})) + sb.AddBlock(simple.New(&model.Block{Id: "columns", Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_TableColumns}}})) + e := editor{sb: sb} + + // when + err := e.cleanupTables(smartblock.ApplyInfo{}) + + // then + assert.NoError(t, err) + }) +} + +func TestEditor_cloneColumnStyles(t *testing.T) { + for _, tc := range []struct { + name string + source *state.State + srcColId, targetColId string + }{ + { + name: "no table in state", + source: state.NewDoc("root", nil).NewState(), + srcColId: "col1", + }, + { + name: "failed to find a row", + source: modifyState(mkTestTable([]string{"col1"}, []string{"row1"}, nil), + func(s *state.State) *state.State { + rows := s.Pick("rows") + rows.Model().ChildrenIds = []string{"row1", "row2"} + s.Set(rows) + return s + }), + srcColId: "col1", + }, + { + name: "invalid cell id", + source: modifyState(mkTestTable([]string{"col1"}, []string{"row1"}, nil), + func(s *state.State) *state.State { + row := s.Pick("row1") + row.Model().ChildrenIds = []string{"cell"} + s.Set(row) + return s + }), + srcColId: "col1", + }, + } { + t.Run(tc.name, func(t *testing.T) { + e := editor{} + err := e.cloneColumnStyles(tc.source, tc.srcColId, tc.targetColId) + assert.Error(t, err) + }) + } + + t.Run("no error", func(t *testing.T) { + // given + s := mkTestTable([]string{"col1", "col2"}, []string{"row1"}, [][]string{{"row1-col1", "row1-col2"}}, withBlockContents(map[string]*model.Block{ + "row1-col1": {Id: "row1-col1", BackgroundColor: "red"}, + })) + e := editor{} + + // when + err := e.cloneColumnStyles(s, "col1", "col2") + + // then + require.NoError(t, err) + assert.Equal(t, "red", s.Pick("row1-col2").Model().BackgroundColor) + }) +} + +func TestEditorAPI(t *testing.T) { + rawTable := [][]string{ + {"c11", "c12", "c13"}, + {"c21", "c22", "c23"}, + } + + s := state.NewDoc("root", map[string]simple.Block{ + "root": simple.New(&model.Block{ + Content: &model.BlockContentOfSmartblock{ + Smartblock: &model.BlockContentSmartblock{}, + }, + }), + }).(*state.State) + + ed := editor{ + generateColID: idFromSlice([]string{"col1", "col2", "col3"}), + generateRowID: idFromSlice([]string{"row1", "row2"}), + } + + tableID, err := ed.TableCreate(s, pb.RpcBlockTableCreateRequest{ + TargetId: "root", + Position: model.Block_Inner, + }) + require.NoError(t, err) + + err = ed.Expand(s, pb.RpcBlockTableExpandRequest{ + TargetId: tableID, + Columns: 3, + }) + require.NoError(t, err) + + tb, err := NewTable(s, tableID) + require.NoError(t, err) + assert.Equal(t, tableID, tb.Block().Model().Id) + + columnIDs := tb.ColumnIDs() + for _, row := range rawTable { + rowID, err := ed.RowCreate(s, pb.RpcBlockTableRowCreateRequest{ + TargetId: tableID, + Position: model.Block_Inner, + }) + require.NoError(t, err) + + for colIdx, cellTxt := range row { + colID := columnIDs[colIdx] + + _, err := ed.CellCreate(s, rowID, colID, &model.Block{ + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{ + Text: cellTxt, + }, + }, + }) + require.NoError(t, err) + } + } + + want := mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, + [][]string{ + {"row1-col1", "row1-col2", "row1-col3"}, + {"row2-col1", "row2-col2", "row2-col3"}, + }, withBlockContents(map[string]*model.Block{ + "row1-col1": mkTextBlock("c11"), + "row1-col2": mkTextBlock("c12"), + "row1-col3": mkTextBlock("c13"), + "row2-col1": mkTextBlock("c21"), + "row2-col2": mkTextBlock("c22"), + "row2-col3": mkTextBlock("c23"), + })) + + filter := func(bs []*model.Block) []*model.Block { + var res []*model.Block + for _, b := range bs { + if b.GetTableRow() != nil || b.GetTableColumn() != nil || b.GetText() != nil { + res = append(res, b) + } + } + return res + } + assert.Equal(t, filter(want.Blocks()), filter(s.Blocks())) +} diff --git a/core/block/editor/table/table_test.go b/core/block/editor/table/table_test.go index 9fb54c52c..6eaa7eb3a 100644 --- a/core/block/editor/table/table_test.go +++ b/core/block/editor/table/table_test.go @@ -9,997 +9,206 @@ import ( "github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest" "github.com/anyproto/anytype-heart/core/block/editor/state" "github.com/anyproto/anytype-heart/core/block/simple" - "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" ) -func TestTableCreate(t *testing.T) { - sb := smarttest.New("root") - sb.AddBlock(simple.New(&model.Block{Id: "root"})) - editor := NewEditor(sb) +func TestPickTableRootBlock(t *testing.T) { + t.Run("block is not in state", func(t *testing.T) { + // given + s := state.NewDoc("root", nil).NewState() - s := sb.NewState() - id, err := editor.TableCreate(s, pb.RpcBlockTableCreateRequest{ - TargetId: "root", - Position: model.Block_Inner, - Columns: 3, - Rows: 4, + // when + root := PickTableRootBlock(s, "id") + + // then + assert.Nil(t, root) + }) +} + +func TestDestructureDivs(t *testing.T) { + t.Run("remove divs", func(t *testing.T) { + // given + s := mkTestTable([]string{"col1, col2"}, []string{"row1", "row2"}, nil) + s = modifyState(s, func(s *state.State) *state.State { + rows := s.Pick("rows") + rows.Model().ChildrenIds = []string{"div1", "div2"} + s.Set(rows) + s.Set(simple.New(&model.Block{ + Id: "div1", + ChildrenIds: []string{"row1"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_Div}}, + })) + s.Set(simple.New(&model.Block{ + Id: "div2", + ChildrenIds: []string{"row2"}, + Content: &model.BlockContentOfLayout{Layout: &model.BlockContentLayout{Style: model.BlockContentLayout_Div}}, + })) + return s + }) + + // when + destructureDivs(s, "rows") + + // then + assert.Equal(t, []string{"row1", "row2"}, s.Pick("rows").Model().ChildrenIds) + }) +} + +func TestTable_Iterate(t *testing.T) { + t.Run("paint it black", func(t *testing.T) { + // given + colIDs := []string{"col1", "col2"} + rowIDs := []string{"row1", "row2"} + s := mkTestTable(colIDs, rowIDs, [][]string{{"row1-col1", "row1-col2"}, {"row2-col1", "row2-col2"}}) + tb, err := NewTable(s, "rows") + require.NoError(t, err) + + // when + err = tb.Iterate(func(b simple.Block, _ CellPosition) bool { + b.Model().BackgroundColor = "black" + return true + }) + + // then + require.NoError(t, err) + for _, rowId := range rowIDs { + for _, colId := range colIDs { + assert.Equal(t, "black", s.Pick(MakeCellID(rowId, colId)).Model().BackgroundColor) + } + } }) - require.NoError(t, err) - assert.NotEmpty(t, id) - - tb, err := NewTable(s, id) - - require.NoError(t, err) - - assert.Len(t, tb.ColumnIDs(), 3) - assert.Len(t, tb.RowIDs(), 4) - - for _, rowID := range tb.RowIDs() { - row, err := pickRow(s, rowID) - + t.Run("failed to get a row", func(t *testing.T) { + // given + s := mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col1", "row1-col2"}, {"row2-col1", "row2-col2"}}) + s = modifyState(s, func(s *state.State) *state.State { + rows := s.Pick("rows") + rows.Model().ChildrenIds = []string{"row0"} + s.Set(rows) + return s + }) + tb, err := NewTable(s, "rows") require.NoError(t, err) - assert.Empty(t, row.Model().ChildrenIds) - } -} -func TestRowCreate(t *testing.T) { - for _, tc := range []struct { - name string - source *state.State - newRowId string - req pb.RpcBlockTableRowCreateRequest - want *state.State - }{ - { - name: "cells are not affected", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), - newRowId: "row3", - req: pb.RpcBlockTableRowCreateRequest{ - TargetId: "row1", - Position: model.Block_Bottom, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row3", "row2"}, [][]string{{"row1-col2"}}), - }, - { - name: "between, bottom position", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), - newRowId: "row3", - req: pb.RpcBlockTableRowCreateRequest{ - TargetId: "row1", - Position: model.Block_Bottom, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row3", "row2"}, nil), - }, - { - name: "between, top position", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), - newRowId: "row3", - req: pb.RpcBlockTableRowCreateRequest{ - TargetId: "row2", - Position: model.Block_Top, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row3", "row2"}, nil), - }, - { - name: "at the beginning", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), - newRowId: "row3", - req: pb.RpcBlockTableRowCreateRequest{ - TargetId: "row1", - Position: model.Block_Top, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row3", "row1", "row2"}, nil), - }, - { - name: "at the end", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil), - newRowId: "row3", - req: pb.RpcBlockTableRowCreateRequest{ - TargetId: "row2", - Position: model.Block_Bottom, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, nil), - }, - } { - t.Run(tc.name, func(t *testing.T) { - tb := editor{ - generateRowID: idFromSlice([]string{tc.newRowId}), - } - id, err := tb.RowCreate(tc.source, tc.req) - require.NoError(t, err) - assert.Equal(t, tc.newRowId, id) - assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) + // when + err = tb.Iterate(func(b simple.Block, pos CellPosition) bool { + return true }) - } -} -func TestRowListClean(t *testing.T) { - for _, tc := range []struct { - name string - source *state.State - req pb.RpcBlockTableRowListCleanRequest - want *state.State - }{ - { - name: "empty rows", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{ - {}, - {}, - }), - req: pb.RpcBlockTableRowListCleanRequest{ - BlockIds: []string{"row1", "row2"}, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{ - {}, - {}, - }), - }, - { - name: "rows with empty blocks", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col2"}, - }), - req: pb.RpcBlockTableRowListCleanRequest{ - BlockIds: []string{"row1", "row2"}, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{ - {}, - {}, - }), - }, - { - name: "rows with not empty text block", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col2"}, - }, withBlockContents(map[string]*model.Block{ - "row1-col1": mkTextBlock("test11"), - "row2-col1": mkTextBlock(""), - })), - req: pb.RpcBlockTableRowListCleanRequest{ - BlockIds: []string{"row1", "row2"}, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{ - {"row1-col1"}, - {}, - }, withBlockContents(map[string]*model.Block{ - "row1-col1": mkTextBlock("test11"), - })), - }, - } { - t.Run(tc.name, func(t *testing.T) { - tb := editor{} - err := tb.RowListClean(tc.source, tc.req) - require.NoError(t, err) - assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) + // then + assert.Error(t, err) + }) + + t.Run("invalid cell id", func(t *testing.T) { + // given + s := mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, nil) + s = modifyState(s, func(s *state.State) *state.State { + row := s.Pick("row1") + row.Model().ChildrenIds = []string{"cell"} + s.Set(row) + return s }) - } + tb, err := NewTable(s, "rows") + require.NoError(t, err) + + // when + err = tb.Iterate(func(b simple.Block, pos CellPosition) bool { + return true + }) + + // then + assert.Error(t, err) + }) + + t.Run("no iteration", func(t *testing.T) { + // given + s := mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col1"}}) + tb, err := NewTable(s, "rows") + require.NoError(t, err) + + // when + err = tb.Iterate(func(b simple.Block, pos CellPosition) bool { + return false + }) + + // then + assert.NoError(t, err) + }) } -func TestExpand(t *testing.T) { +func TestCheckTableBlocksMove(t *testing.T) { for _, tc := range []struct { name string source *state.State - newColIds []string - newRowIds []string - req pb.RpcBlockTableExpandRequest - want *state.State + target string + pos model.BlockPosition + blockIds []string + resPos model.BlockPosition + resTarget string + shouldErr bool }{ { - name: "only rows", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row2-col2"}}), - newRowIds: []string{"row3", "row4"}, - req: pb.RpcBlockTableExpandRequest{ - TargetId: "table", - Rows: 2, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3", "row4"}, [][]string{{"row2-col2"}}), + name: "no table - no error", + source: state.NewDoc("root", nil).NewState(), + shouldErr: false, }, { - name: "only columns", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row2-col2"}}), - newColIds: []string{"col3", "col4"}, - req: pb.RpcBlockTableExpandRequest{ - TargetId: "table", - Columns: 2, - }, - want: mkTestTable([]string{"col1", "col2", "col3", "col4"}, []string{"row1", "row2"}, [][]string{{"row2-col2"}}), + name: "moving rows between each other", + source: mkTestTable([]string{"col"}, []string{"row1", "row2", "row3"}, nil), + target: "row2", + pos: model.Block_Bottom, + blockIds: []string{"row3", "row1"}, + resTarget: "row2", + resPos: model.Block_Bottom, + shouldErr: false, }, { - name: "both columns and rows", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row2-col2"}}), - newRowIds: []string{"row3", "row4"}, - newColIds: []string{"col3", "col4"}, - req: pb.RpcBlockTableExpandRequest{ - TargetId: "table", - Rows: 2, - Columns: 2, - }, - want: mkTestTable([]string{"col1", "col2", "col3", "col4"}, []string{"row1", "row2", "row3", "row4"}, [][]string{{"row2-col2"}}), + name: "moving rows between each other with invalid position", + source: mkTestTable([]string{"col"}, []string{"row1", "row2", "row3"}, nil), + target: "row2", + pos: model.Block_Replace, + blockIds: []string{"row3", "row1"}, + shouldErr: true, + }, + { + name: "moving inner table blocks is prohibited", + source: mkTestTable([]string{"col"}, []string{"row1", "row2", "row3"}, nil), + target: "root", + pos: model.Block_Bottom, + blockIds: []string{"row3", "cols"}, + shouldErr: true, + }, + { + name: "place blocks under the table", + source: modifyState(mkTestTable([]string{"col"}, []string{"row1", "row2", "row3"}, nil), + func(s *state.State) *state.State { + root := s.Pick("root") + root.Model().ChildrenIds = []string{"table", "text"} + s.Set(root) + s.Add(simple.New(&model.Block{Id: "text"})) + return s + }), + target: "col", + pos: model.Block_Inner, + blockIds: []string{"text"}, + resTarget: "table", + resPos: model.Block_Bottom, + shouldErr: false, }, } { t.Run(tc.name, func(t *testing.T) { - tb := editor{ - generateColID: idFromSlice(tc.newColIds), - generateRowID: idFromSlice(tc.newRowIds), + resTarget, resPos, err := CheckTableBlocksMove(tc.source, tc.target, tc.pos, tc.blockIds) + if tc.shouldErr { + assert.Error(t, err) + return } - err := tb.Expand(tc.source, tc.req) require.NoError(t, err) - assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) + assert.Equal(t, tc.resTarget, resTarget) + assert.Equal(t, tc.resPos, resPos) }) } } -func TestRowListFill(t *testing.T) { - for _, tc := range []struct { - name string - source *state.State - req pb.RpcBlockTableRowListFillRequest - want *state.State - }{ - { - name: "empty", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{}), - req: pb.RpcBlockTableRowListFillRequest{ - BlockIds: []string{"row1", "row2"}, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - }), - }, - { - name: "fully filled", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - }), - req: pb.RpcBlockTableRowListFillRequest{ - BlockIds: []string{"row1", "row2"}, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - }), - }, - { - name: "partially filled", - source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2", "row3", "row4", "row5"}, - [][]string{ - {"row1-col1"}, - {"row2-col2"}, - {"row3-col3"}, - {"row4-col1", "row4-col3"}, - {"row5-col2", "row4-col3"}, - }), - req: pb.RpcBlockTableRowListFillRequest{ - BlockIds: []string{"row1", "row2", "row3", "row4", "row5"}, - }, - want: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2", "row3", "row4", "row5"}, - [][]string{ - {"row1-col1", "row1-col2", "row1-col3"}, - {"row2-col1", "row2-col2", "row2-col3"}, - {"row3-col1", "row3-col2", "row3-col3"}, - {"row4-col1", "row4-col2", "row4-col3"}, - {"row5-col1", "row5-col2", "row5-col3"}, - }), - }, - } { - t.Run(tc.name, func(t *testing.T) { - tb := editor{} - err := tb.RowListFill(tc.source, tc.req) - require.NoError(t, err) - assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) - }) - } -} - -func TestColumnListFill(t *testing.T) { - for _, tc := range []struct { - name string - source *state.State - req pb.RpcBlockTableColumnListFillRequest - want *state.State - }{ - { - name: "empty", - source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, [][]string{}), - req: pb.RpcBlockTableColumnListFillRequest{ - BlockIds: []string{"col2", "col1"}, - }, - want: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - }), - }, - { - name: "fully filled", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - }), - req: pb.RpcBlockTableColumnListFillRequest{ - BlockIds: []string{"col2", "col1"}, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - }), - }, - { - name: "partially filled", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, [][]string{ - {"row1-col1"}, - {"row2-col2"}, - {"row3-col1", "row3-col2"}, - }), - req: pb.RpcBlockTableColumnListFillRequest{ - BlockIds: []string{"col1", "col2"}, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - {"row3-col1", "row3-col2"}, - }), - }, - } { - t.Run(tc.name, func(t *testing.T) { - tb := editor{} - err := tb.ColumnListFill(tc.source, tc.req) - require.NoError(t, err) - assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) - }) - } -} - -func TestColumnCreate(t *testing.T) { - for _, tc := range []struct { - name string - source *state.State - newColId string - req pb.RpcBlockTableColumnCreateRequest - want *state.State - }{ - { - name: "between, to the right", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), - newColId: "col3", - req: pb.RpcBlockTableColumnCreateRequest{ - TargetId: "col1", - Position: model.Block_Right, - }, - want: mkTestTable([]string{"col1", "col3", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), - }, - { - name: "between, to the left", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), - newColId: "col3", - req: pb.RpcBlockTableColumnCreateRequest{ - TargetId: "col2", - Position: model.Block_Left, - }, - want: mkTestTable([]string{"col1", "col3", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), - }, - { - name: "at the beginning", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), - newColId: "col3", - req: pb.RpcBlockTableColumnCreateRequest{ - TargetId: "col1", - Position: model.Block_Left, - }, - want: mkTestTable([]string{"col3", "col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), - }, - { - name: "at the end", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), - newColId: "col3", - req: pb.RpcBlockTableColumnCreateRequest{ - TargetId: "col2", - Position: model.Block_Right, - }, - want: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, [][]string{{"row1-col2"}}), - }, - } { - t.Run(tc.name, func(t *testing.T) { - tb := editor{ - generateColID: idFromSlice([]string{tc.newColId}), - } - id, err := tb.ColumnCreate(tc.source, tc.req) - require.NoError(t, err) - assert.Equal(t, tc.newColId, id) - assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) - }) - } -} - -func TestColumnDuplicate(t *testing.T) { - for _, tc := range []struct { - name string - source *state.State - newColId string - req pb.RpcBlockTableColumnDuplicateRequest - want *state.State - }{ - { - name: "fully filled", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - }, withBlockContents(map[string]*model.Block{ - "row1-col1": mkTextBlock("test11"), - "row2-col1": mkTextBlock("test21"), - })), - newColId: "col3", - req: pb.RpcBlockTableColumnDuplicateRequest{ - BlockId: "col1", - TargetId: "col2", - Position: model.Block_Right, - }, - want: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col2", "row1-col3"}, - {"row2-col1", "row2-col2", "row2-col3"}, - }, withBlockContents(map[string]*model.Block{ - "row1-col1": mkTextBlock("test11"), - "row2-col1": mkTextBlock("test21"), - "row1-col3": mkTextBlock("test11"), - "row2-col3": mkTextBlock("test21"), - })), - }, - { - name: "partially filled", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, - [][]string{ - {"row1-col1"}, - {"row2-col2"}, - {}, - }, withBlockContents(map[string]*model.Block{ - "row2-col2": mkTextBlock("test22"), - })), - newColId: "col3", - req: pb.RpcBlockTableColumnDuplicateRequest{ - BlockId: "col2", - TargetId: "col1", - Position: model.Block_Left, - }, - want: mkTestTable([]string{"col3", "col1", "col2"}, []string{"row1", "row2", "row3"}, - [][]string{ - {"row1-col1"}, - {"row2-col3", "row2-col2"}, - {}, - }, withBlockContents(map[string]*model.Block{ - "row2-col2": mkTextBlock("test22"), - "row2-col3": mkTextBlock("test22"), - })), - }, - { - name: "empty", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, - [][]string{ - {"row1-col1"}, - {}, - {}, - }), - newColId: "col3", - req: pb.RpcBlockTableColumnDuplicateRequest{ - BlockId: "col2", - TargetId: "col1", - Position: model.Block_Left, - }, - want: mkTestTable([]string{"col3", "col1", "col2"}, []string{"row1", "row2", "row3"}, - [][]string{ - {"row1-col1"}, - {}, - {}, - }), - }, - } { - t.Run(tc.name, func(t *testing.T) { - tb := editor{ - generateColID: idFromSlice([]string{tc.newColId}), - } - id, err := tb.ColumnDuplicate(tc.source, tc.req) - require.NoError(t, err) - assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) - assert.Equal(t, tc.newColId, id) - }) - } -} - -func TestRowDuplicate(t *testing.T) { - for _, tc := range []struct { - name string - source *state.State - newRowId string - req pb.RpcBlockTableRowDuplicateRequest - want *state.State - }{ - { - name: "fully filled", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - }, withBlockContents(map[string]*model.Block{ - "row1-col1": mkTextBlock("test11"), - "row1-col2": mkTextBlock("test12"), - "row2-col1": mkTextBlock("test21"), - })), - newRowId: "row3", - req: pb.RpcBlockTableRowDuplicateRequest{ - BlockId: "row1", - TargetId: "row2", - Position: model.Block_Bottom, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - {"row3-col1", "row3-col2"}, - }, withBlockContents(map[string]*model.Block{ - "row1-col1": mkTextBlock("test11"), - "row1-col2": mkTextBlock("test12"), - "row2-col1": mkTextBlock("test21"), - "row3-col1": mkTextBlock("test11"), - "row3-col2": mkTextBlock("test12"), - })), - }, - { - name: "partially filled", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1"}, - {"row2-col2"}, - }, withBlockContents(map[string]*model.Block{ - "row1-col1": mkTextBlock("test11"), - "row2-col2": mkTextBlock("test22"), - })), - newRowId: "row3", - req: pb.RpcBlockTableRowDuplicateRequest{ - BlockId: "row2", - TargetId: "row1", - Position: model.Block_Top, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row3", "row1", "row2"}, - [][]string{ - {"row3-col2"}, - {"row1-col1"}, - {"row2-col2"}, - }, withBlockContents(map[string]*model.Block{ - "row1-col1": mkTextBlock("test11"), - "row2-col2": mkTextBlock("test22"), - "row3-col2": mkTextBlock("test22"), - })), - }, - { - name: "empty", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, - [][]string{ - {}, - {}, - }), - newRowId: "row3", - req: pb.RpcBlockTableRowDuplicateRequest{ - BlockId: "row2", - TargetId: "row1", - Position: model.Block_Bottom, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row3", "row2"}, - [][]string{ - {}, - {}, - {}, - }), - }, - } { - t.Run(tc.name, func(t *testing.T) { - tb := editor{ - generateRowID: idFromSlice([]string{tc.newRowId}), - } - id, err := tb.RowDuplicate(tc.source, tc.req) - require.NoError(t, err) - assert.Equal(t, tc.newRowId, id) - assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) - }) - } -} - -func TestColumnMove(t *testing.T) { - for _, tc := range []struct { - name string - source *state.State - req pb.RpcBlockTableColumnMoveRequest - want *state.State - }{ - { - name: "partial table", - source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col2", "row1-col3"}, - {"row2-col1", "row2-col3"}, - }), - req: pb.RpcBlockTableColumnMoveRequest{ - TargetId: "col1", - DropTargetId: "col3", - Position: model.Block_Left, - }, - want: mkTestTable([]string{"col2", "col1", "col3"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col2", "row1-col1", "row1-col3"}, - {"row2-col1", "row2-col3"}, - }), - }, - { - name: "filled table", - source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col2", "row1-col3"}, - {"row2-col1", "row2-col2", "row2-col3"}, - }), - req: pb.RpcBlockTableColumnMoveRequest{ - TargetId: "col3", - DropTargetId: "col1", - Position: model.Block_Right, - }, - want: mkTestTable([]string{"col1", "col3", "col2"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col3", "row1-col2"}, - {"row2-col1", "row2-col3", "row2-col2"}, - }), - }, - } { - t.Run(tc.name, func(t *testing.T) { - tb := editor{} - err := tb.ColumnMove(tc.source, tc.req) - require.NoError(t, err) - assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) - }) - } -} - -func TestColumnDelete(t *testing.T) { - for _, tc := range []struct { - name string - source *state.State - req pb.RpcBlockTableColumnDeleteRequest - want *state.State - }{ - { - name: "partial table", - source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col2", "row1-col3"}, - {"row2-col1", "row2-col3"}, - }), - req: pb.RpcBlockTableColumnDeleteRequest{ - TargetId: "col2", - }, - want: mkTestTable([]string{"col1", "col3"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col3"}, - {"row2-col1", "row2-col3"}, - }), - }, - { - name: "filled table", - source: mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col2", "row1-col3"}, - {"row2-col1", "row2-col2", "row2-col3"}, - }), - req: pb.RpcBlockTableColumnDeleteRequest{ - TargetId: "col3", - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - }), - }, - } { - t.Run(tc.name, func(t *testing.T) { - tb := editor{} - err := tb.ColumnDelete(tc.source, tc.req) - require.NoError(t, err) - assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) - }) - } -} - -func TestSort(t *testing.T) { - for _, tc := range []struct { - name string - source *state.State - req pb.RpcBlockTableSortRequest - want *state.State - }{ - { - name: "asc order", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - {"row3-col1", "row3-col2"}, - }, withBlockContents(map[string]*model.Block{ - "row1-col2": mkTextBlock("Abd"), - "row2-col2": mkTextBlock("bsd"), - "row3-col2": mkTextBlock("abc"), - })), - req: pb.RpcBlockTableSortRequest{ - ColumnId: "col2", - Type: model.BlockContentDataviewSort_Asc, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row3", "row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - {"row3-col1", "row3-col2"}, - }, withBlockContents(map[string]*model.Block{ - "row3-col2": mkTextBlock("abc"), - "row1-col2": mkTextBlock("Abd"), - "row2-col2": mkTextBlock("bsd"), - })), - }, - { - name: "desc order", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - {"row3-col1", "row3-col2"}, - }, withBlockContents(map[string]*model.Block{ - "row1-col2": mkTextBlock("234"), - "row2-col2": mkTextBlock("323"), - "row3-col2": mkTextBlock("123"), - })), - req: pb.RpcBlockTableSortRequest{ - ColumnId: "col2", - Type: model.BlockContentDataviewSort_Desc, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row2", "row1", "row3"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - {"row3-col1", "row3-col2"}, - }, withBlockContents(map[string]*model.Block{ - "row1-col2": mkTextBlock("234"), - "row2-col2": mkTextBlock("323"), - "row3-col2": mkTextBlock("123"), - })), - }, - { - name: "asc order with header rows", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3", "row4", "row5"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - {"row3-col1", "row3-col2"}, - {"row4-col1", "row4-col2"}, - {"row5-col1", "row5-col2"}, - }, withBlockContents(map[string]*model.Block{ - "row1-col2": mkTextBlock("555"), - "row2-col2": mkTextBlock("444"), - "row3-col2": mkTextBlock("333"), - "row4-col2": mkTextBlock("222"), - "row5-col2": mkTextBlock("111"), - }), - withRowBlockContents(map[string]*model.BlockContentTableRow{ - "row1": {IsHeader: true}, - "row3": {IsHeader: true}, - })), - req: pb.RpcBlockTableSortRequest{ - ColumnId: "col2", - Type: model.BlockContentDataviewSort_Asc, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row3", "row5", "row4", "row2"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row3-col1", "row3-col2"}, - {"row5-col1", "row5-col2"}, - {"row4-col1", "row4-col2"}, - {"row2-col1", "row2-col2"}, - }, withBlockContents(map[string]*model.Block{ - "row1-col2": mkTextBlock("555"), - "row2-col2": mkTextBlock("444"), - "row3-col2": mkTextBlock("333"), - "row4-col2": mkTextBlock("222"), - "row5-col2": mkTextBlock("111"), - }), - withRowBlockContents(map[string]*model.BlockContentTableRow{ - "row1": {IsHeader: true}, - "row3": {IsHeader: true}, - })), - }, - { - name: "desc order with header rows", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3", "row4", "row5"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row2-col1", "row2-col2"}, - {"row3-col1", "row3-col2"}, - {"row4-col1", "row4-col2"}, - {"row5-col1", "row5-col2"}, - }, withBlockContents(map[string]*model.Block{ - "row1-col2": mkTextBlock("555"), - "row2-col2": mkTextBlock("444"), - "row3-col2": mkTextBlock("333"), - "row4-col2": mkTextBlock("222"), - "row5-col2": mkTextBlock("111"), - }), - withRowBlockContents(map[string]*model.BlockContentTableRow{ - "row1": {IsHeader: true}, - "row3": {IsHeader: true}, - })), - req: pb.RpcBlockTableSortRequest{ - ColumnId: "col2", - Type: model.BlockContentDataviewSort_Desc, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row3", "row2", "row4", "row5"}, - [][]string{ - {"row1-col1", "row1-col2"}, - {"row3-col1", "row3-col2"}, - {"row2-col1", "row2-col2"}, - {"row4-col1", "row4-col2"}, - {"row5-col1", "row5-col2"}, - }, withBlockContents(map[string]*model.Block{ - "row1-col2": mkTextBlock("555"), - "row2-col2": mkTextBlock("444"), - "row3-col2": mkTextBlock("333"), - "row4-col2": mkTextBlock("222"), - "row5-col2": mkTextBlock("111"), - }), - withRowBlockContents(map[string]*model.BlockContentTableRow{ - "row1": {IsHeader: true}, - "row3": {IsHeader: true}, - })), - }, - } { - t.Run(tc.name, func(t *testing.T) { - tb := editor{} - err := tb.Sort(tc.source, tc.req) - require.NoError(t, err) - assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) - }) - } -} - -func TestRowSetHeader(t *testing.T) { - for _, tc := range []struct { - name string - source *state.State - req pb.RpcBlockTableRowSetHeaderRequest - want *state.State - }{ - { - name: "header row moves up", - source: mkTestTable([]string{"col1", "col2"}, []string{"row1", "row2", "row3", "row4"}, nil), - req: pb.RpcBlockTableRowSetHeaderRequest{ - TargetId: "row3", - IsHeader: true, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row3", "row1", "row2", "row4"}, nil, - withRowBlockContents(map[string]*model.BlockContentTableRow{ - "row3": {IsHeader: true}, - })), - }, - { - name: "non-header row moves down", - source: mkTestTable([]string{"col1", "col2"}, []string{"row2", "row3", "row1", "row4"}, nil, - withRowBlockContents(map[string]*model.BlockContentTableRow{ - "row2": {IsHeader: true}, - "row3": {IsHeader: true}, - })), - req: pb.RpcBlockTableRowSetHeaderRequest{ - TargetId: "row2", - IsHeader: false, - }, - want: mkTestTable([]string{"col1", "col2"}, []string{"row3", "row2", "row1", "row4"}, nil, - withRowBlockContents(map[string]*model.BlockContentTableRow{ - "row3": {IsHeader: true}, - })), - }, - } { - t.Run(tc.name, func(t *testing.T) { - tb := editor{} - err := tb.RowSetHeader(tc.source, tc.req) - require.NoError(t, err) - assert.Equal(t, tc.want.Blocks(), tc.source.Blocks()) - }) - } -} - -func TestEditorAPI(t *testing.T) { - rawTable := [][]string{ - {"c11", "c12", "c13"}, - {"c21", "c22", "c23"}, - } - - s := state.NewDoc("root", map[string]simple.Block{ - "root": simple.New(&model.Block{ - Content: &model.BlockContentOfSmartblock{ - Smartblock: &model.BlockContentSmartblock{}, - }, - }), - }).(*state.State) - - ed := editor{ - generateColID: idFromSlice([]string{"col1", "col2", "col3"}), - generateRowID: idFromSlice([]string{"row1", "row2"}), - } - - tableID, err := ed.TableCreate(s, pb.RpcBlockTableCreateRequest{ - TargetId: "root", - Position: model.Block_Inner, - }) - require.NoError(t, err) - - err = ed.Expand(s, pb.RpcBlockTableExpandRequest{ - TargetId: tableID, - Columns: 3, - }) - require.NoError(t, err) - - tb, err := NewTable(s, tableID) - require.NoError(t, err) - - columnIDs := tb.ColumnIDs() - for _, row := range rawTable { - rowID, err := ed.RowCreate(s, pb.RpcBlockTableRowCreateRequest{ - TargetId: tableID, - Position: model.Block_Inner, - }) - require.NoError(t, err) - - for colIdx, cellTxt := range row { - colID := columnIDs[colIdx] - - _, err := ed.CellCreate(s, rowID, colID, &model.Block{ - Content: &model.BlockContentOfText{ - Text: &model.BlockContentText{ - Text: cellTxt, - }, - }, - }) - require.NoError(t, err) - } - } - - want := mkTestTable([]string{"col1", "col2", "col3"}, []string{"row1", "row2"}, - [][]string{ - {"row1-col1", "row1-col2", "row1-col3"}, - {"row2-col1", "row2-col2", "row2-col3"}, - }, withBlockContents(map[string]*model.Block{ - "row1-col1": mkTextBlock("c11"), - "row1-col2": mkTextBlock("c12"), - "row1-col3": mkTextBlock("c13"), - "row2-col1": mkTextBlock("c21"), - "row2-col2": mkTextBlock("c22"), - "row2-col3": mkTextBlock("c23"), - })) - - filter := func(bs []*model.Block) []*model.Block { - var res []*model.Block - for _, b := range bs { - if b.GetTableRow() != nil || b.GetTableColumn() != nil || b.GetText() != nil { - res = append(res, b) - } - } - return res - } - assert.Equal(t, filter(want.Blocks()), filter(s.Blocks())) -} - type testTableOptions struct { blocks map[string]*model.Block @@ -1021,12 +230,29 @@ func withRowBlockContents(blocks map[string]*model.BlockContentTableRow) testTab } func mkTestTable(columns []string, rows []string, cells [][]string, opts ...testTableOption) *state.State { + blocks := mkTestTableBlocks(columns, rows, cells, opts...) + s := state.NewDoc("root", nil).NewState() + for _, b := range blocks { + s.Add(simple.New(b)) + } + return s +} + +func mkTestTableSb(columns []string, rows []string, cells [][]string, opts ...testTableOption) *smarttest.SmartTest { + blocks := mkTestTableBlocks(columns, rows, cells, opts...) + sb := smarttest.New("root") + for _, b := range blocks { + sb.AddBlock(simple.New(b)) + } + return sb +} + +func mkTestTableBlocks(columns []string, rows []string, cells [][]string, opts ...testTableOption) []*model.Block { o := testTableOptions{} for _, apply := range opts { apply(&o) } - s := state.NewDoc("root", nil).NewState() blocks := []*model.Block{ { Id: "root", @@ -1099,10 +325,7 @@ func mkTestTable(columns []string, rows []string, cells [][]string, opts ...test }) } - for _, b := range blocks { - s.Add(simple.New(b)) - } - return s + return blocks } func mkTextBlock(txt string) *model.Block { @@ -1121,3 +344,7 @@ func idFromSlice(ids []string) func() string { return id } } + +func modifyState(s *state.State, modifier func(s *state.State) *state.State) *state.State { + return modifier(s) +} From d6c724b41a943fd40d265b0540d018045933243c Mon Sep 17 00:00:00 2001 From: kirillston Date: Thu, 25 Jul 2024 01:30:06 +0300 Subject: [PATCH 50/71] GO-3192 Remove slice.ContainsAll --- core/block/editor/table/editor.go | 60 ++++++++++++++++--------------- core/block/editor/table/table.go | 5 +-- util/slice/slice.go | 9 ----- util/slice/slice_test.go | 11 ------ 4 files changed, 34 insertions(+), 51 deletions(-) diff --git a/core/block/editor/table/editor.go b/core/block/editor/table/editor.go index 087755637..34315c68c 100644 --- a/core/block/editor/table/editor.go +++ b/core/block/editor/table/editor.go @@ -130,38 +130,40 @@ func (t *editor) TableCreate(s *state.State, req pb.RpcBlockTableCreateRequest) tableBlock.Model().ChildrenIds = []string{columnsLayout.Model().Id, rowsLayout.Model().Id} - if req.WithHeaderRow { - if len(rowIDs) == 0 { - return "", fmt.Errorf("no rows to make header row") - } - headerID := rowIDs[0] + if !req.WithHeaderRow { + return tableBlock.Model().Id, nil + } - if err := t.RowSetHeader(s, pb.RpcBlockTableRowSetHeaderRequest{ - TargetId: headerID, - IsHeader: true, - }); err != nil { - return "", fmt.Errorf("row set header: %w", err) + if len(rowIDs) == 0 { + return "", fmt.Errorf("no rows to make header row") + } + headerID := rowIDs[0] + + if err := t.RowSetHeader(s, pb.RpcBlockTableRowSetHeaderRequest{ + TargetId: headerID, + IsHeader: true, + }); err != nil { + return "", fmt.Errorf("row set header: %w", err) + } + + if err := t.RowListFill(s, pb.RpcBlockTableRowListFillRequest{ + BlockIds: []string{headerID}, + }); err != nil { + return "", fmt.Errorf("fill header row: %w", err) + } + + row, err := getRow(s, headerID) + if err != nil { + return "", fmt.Errorf("get header row: %w", err) + } + + for _, cellID := range row.Model().ChildrenIds { + cell := s.Get(cellID) + if cell == nil { + return "", fmt.Errorf("get header cell id %s", cellID) } - if err := t.RowListFill(s, pb.RpcBlockTableRowListFillRequest{ - BlockIds: []string{headerID}, - }); err != nil { - return "", fmt.Errorf("fill header row: %w", err) - } - - row, err := getRow(s, headerID) - if err != nil { - return "", fmt.Errorf("get header row: %w", err) - } - - for _, cellID := range row.Model().ChildrenIds { - cell := s.Get(cellID) - if cell == nil { - return "", fmt.Errorf("get header cell id %s", cellID) - } - - cell.Model().BackgroundColor = "grey" - } + cell.Model().BackgroundColor = "grey" } return tableBlock.Model().Id, nil diff --git a/core/block/editor/table/table.go b/core/block/editor/table/table.go index 311d6c7d5..b554413e2 100644 --- a/core/block/editor/table/table.go +++ b/core/block/editor/table/table.go @@ -4,12 +4,13 @@ import ( "fmt" "strings" + "github.com/samber/lo" + "github.com/anyproto/anytype-heart/core/block/editor/state" "github.com/anyproto/anytype-heart/core/block/simple" "github.com/anyproto/anytype-heart/core/block/simple/table" "github.com/anyproto/anytype-heart/pkg/lib/logging" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" - "github.com/anyproto/anytype-heart/util/slice" ) var log = logging.Logger("anytype-simple-tables") @@ -264,7 +265,7 @@ func (tb Table) Iterate(f func(b simple.Block, pos CellPosition) bool) error { func CheckTableBlocksMove(st *state.State, target string, pos model.BlockPosition, blockIds []string) (string, model.BlockPosition, error) { if t, err := NewTable(st, target); err == nil && t != nil { // we allow moving rows between each other - if slice.ContainsAll(t.RowIDs(), append(blockIds, target)...) { + if lo.Every(t.RowIDs(), append(blockIds, target)) { if pos == model.Block_Bottom || pos == model.Block_Top { return target, pos, nil } diff --git a/util/slice/slice.go b/util/slice/slice.go index 4e7e10cb6..4c87d0084 100644 --- a/util/slice/slice.go +++ b/util/slice/slice.go @@ -237,12 +237,3 @@ func FilterCID(cids []string) []string { return err == nil }) } - -func ContainsAll[T comparable](s []T, items ...T) bool { - for _, e := range items { - if !slices.Contains(s, e) { - return false - } - } - return true -} diff --git a/util/slice/slice_test.go b/util/slice/slice_test.go index 0cb15704a..3d642bd33 100644 --- a/util/slice/slice_test.go +++ b/util/slice/slice_test.go @@ -100,14 +100,3 @@ func TestUnsortedEquals(t *testing.T) { assert.False(t, UnsortedEqual([]string{"a", "b", "c"}, []string{"a", "b"})) assert.False(t, UnsortedEqual([]string{"a", "b", "c"}, []string{"a", "b", "c", "d"})) } - -func TestContainsAll(t *testing.T) { - assert.True(t, ContainsAll([]byte("csabd"), []byte("abc")...)) - assert.True(t, ContainsAll([]string{})) - assert.True(t, ContainsAll([]string{"hello"})) - assert.True(t, ContainsAll([]string{"a", "b", "c", "d"}, "a", "c")) - assert.True(t, ContainsAll([]string{"a", "b", "c", "d"}, "a", "a")) - assert.True(t, ContainsAll([]string{"a", "b", "c"}, "a", "c", "b", "a")) - assert.False(t, ContainsAll([]string{}, "z")) - assert.False(t, ContainsAll([]string{"y", "x"}, "z")) -} From 4b82b407089d2b8be4c441f0fb6b796c7f065afb Mon Sep 17 00:00:00 2001 From: Sergey Date: Thu, 25 Jul 2024 11:04:19 +0200 Subject: [PATCH 51/71] GO-3769: UpdateDetails test --- core/anytype/bootstrap.go | 2 +- core/syncstatus/detailsupdater/updater.go | 101 +++++---- .../syncstatus/detailsupdater/updater_test.go | 208 +++++++++++++++--- 3 files changed, 237 insertions(+), 74 deletions(-) diff --git a/core/anytype/bootstrap.go b/core/anytype/bootstrap.go index 375c0308c..c4d44c67b 100644 --- a/core/anytype/bootstrap.go +++ b/core/anytype/bootstrap.go @@ -264,7 +264,7 @@ func Bootstrap(a *app.App, components ...app.Component) { Register(treemanager.New()). Register(block.New()). Register(indexer.New()). - Register(detailsupdater.NewUpdater()). + Register(detailsupdater.New()). Register(session.NewHookRunner()). Register(spacesyncstatus.NewSpaceSyncStatus()). Register(nodestatus.NewNodeStatus()). diff --git a/core/syncstatus/detailsupdater/updater.go b/core/syncstatus/detailsupdater/updater.go index b826df877..8296755ab 100644 --- a/core/syncstatus/detailsupdater/updater.go +++ b/core/syncstatus/detailsupdater/updater.go @@ -63,7 +63,7 @@ type syncStatusUpdater struct { finish chan struct{} } -func NewUpdater() Updater { +func New() Updater { return &syncStatusUpdater{ batcher: mb.New[string](0), finish: make(chan struct{}), @@ -118,6 +118,32 @@ func (u *syncStatusUpdater) addToQueue(details *syncStatusDetails) error { return u.batcher.TryAdd(details.objectId) } +func (u *syncStatusUpdater) processEvents() { + defer close(u.finish) + + for { + objectId, err := u.batcher.WaitOne(u.ctx) + if err != nil { + return + } + u.updateSpecificObject(objectId) + } +} + +func (u *syncStatusUpdater) updateSpecificObject(objectId string) { + u.mx.Lock() + objectStatus := u.entries[objectId] + delete(u.entries, objectId) + u.mx.Unlock() + + if objectStatus != nil { + err := u.updateObjectDetails(objectStatus, objectId) + if err != nil { + log.Errorf("failed to update details %s", err) + } + } +} + func (u *syncStatusUpdater) UpdateSpaceDetails(existing, missing []string, spaceId string) { if spaceId == u.spaceService.TechSpaceId() { return @@ -181,7 +207,7 @@ func (u *syncStatusUpdater) updateObjectDetails(syncStatusDetails *syncStatusDet return details, nil } if fileStatus, ok := details.GetFields()[bundle.RelationKeyFileBackupStatus.String()]; ok { - status, syncError = mapFileStatus(filesyncstatus.Status(int(fileStatus.GetNumberValue()))) + status, syncError = getSyncStatusForFile(status, syncError, filesyncstatus.Status(int(fileStatus.GetNumberValue()))) } details.Fields[bundle.RelationKeySyncStatus.String()] = pbtypes.Int64(int64(status)) details.Fields[bundle.RelationKeySyncError.String()] = pbtypes.Int64(int64(syncError)) @@ -200,6 +226,24 @@ func (u *syncStatusUpdater) updateObjectDetails(syncStatusDetails *syncStatusDet }) } +func (u *syncStatusUpdater) setSyncDetails(sb smartblock.SmartBlock, status domain.ObjectSyncStatus, syncError domain.SyncError) error { + if !slices.Contains(helper.SyncRelationsSmartblockTypes(), sb.Type()) { + return nil + } + if !u.isLayoutSuitableForSyncRelations(sb.Details()) { + return nil + } + st := sb.NewState() + if fileStatus, ok := st.Details().GetFields()[bundle.RelationKeyFileBackupStatus.String()]; ok { + status, syncError = getSyncStatusForFile(status, syncError, filesyncstatus.Status(int(fileStatus.GetNumberValue()))) + } + st.SetDetailAndBundledRelation(bundle.RelationKeySyncStatus, pbtypes.Int64(int64(status))) + st.SetDetailAndBundledRelation(bundle.RelationKeySyncError, pbtypes.Int64(int64(syncError))) + st.SetDetailAndBundledRelation(bundle.RelationKeySyncDate, pbtypes.Int64(time.Now().Unix())) + + return sb.Apply(st, smartblock.KeepInternalFlags /* do not erase flags */) +} + var suitableLayouts = map[model.ObjectTypeLayout]struct{}{ model.ObjectType_basic: {}, model.ObjectType_profile: {}, @@ -224,6 +268,18 @@ func (u *syncStatusUpdater) isLayoutSuitableForSyncRelations(details *types.Stru return ok } +func getSyncStatusForFile(objectStatus domain.ObjectSyncStatus, objectSyncError domain.SyncError, fileStatus filesyncstatus.Status) (domain.ObjectSyncStatus, domain.SyncError) { + statusFromFile, errFromFile := mapFileStatus(fileStatus) + // If file status is synced, then prioritize object's status, otherwise pick file status + if statusFromFile != domain.ObjectSyncStatusSynced { + objectStatus = statusFromFile + } + if errFromFile != domain.SyncErrorNull { + objectSyncError = errFromFile + } + return objectStatus, objectSyncError +} + func mapFileStatus(status filesyncstatus.Status) (domain.ObjectSyncStatus, domain.SyncError) { switch status { case filesyncstatus.Syncing: @@ -238,44 +294,3 @@ func mapFileStatus(status filesyncstatus.Status) (domain.ObjectSyncStatus, domai return domain.ObjectSyncStatusSynced, domain.SyncErrorNull } } - -func (u *syncStatusUpdater) setSyncDetails(sb smartblock.SmartBlock, status domain.ObjectSyncStatus, syncError domain.SyncError) error { - if !slices.Contains(helper.SyncRelationsSmartblockTypes(), sb.Type()) { - return nil - } - if !u.isLayoutSuitableForSyncRelations(sb.Details()) { - return nil - } - st := sb.NewState() - if fileStatus, ok := st.Details().GetFields()[bundle.RelationKeyFileBackupStatus.String()]; ok { - status, syncError = mapFileStatus(filesyncstatus.Status(int(fileStatus.GetNumberValue()))) - } - st.SetDetailAndBundledRelation(bundle.RelationKeySyncStatus, pbtypes.Int64(int64(status))) - st.SetDetailAndBundledRelation(bundle.RelationKeySyncError, pbtypes.Int64(int64(syncError))) - st.SetDetailAndBundledRelation(bundle.RelationKeySyncDate, pbtypes.Int64(time.Now().Unix())) - - return sb.Apply(st, smartblock.KeepInternalFlags /* do not erase flags */) -} - -func (u *syncStatusUpdater) processEvents() { - defer close(u.finish) - updateSpecificObject := func(objectId string) { - u.mx.Lock() - objectStatus := u.entries[objectId] - delete(u.entries, objectId) - u.mx.Unlock() - if objectStatus != nil { - err := u.updateObjectDetails(objectStatus, objectId) - if err != nil { - log.Errorf("failed to update details %s", err) - } - } - } - for { - objectId, err := u.batcher.WaitOne(u.ctx) - if err != nil { - return - } - updateSpecificObject(objectId) - } -} diff --git a/core/syncstatus/detailsupdater/updater_test.go b/core/syncstatus/detailsupdater/updater_test.go index 3ab6d22fa..6b613cc6a 100644 --- a/core/syncstatus/detailsupdater/updater_test.go +++ b/core/syncstatus/detailsupdater/updater_test.go @@ -3,44 +3,195 @@ package detailsupdater import ( "context" "testing" + "time" "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/app/ocache" "github.com/gogo/protobuf/types" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/anyproto/anytype-heart/core/block/editor" + "github.com/anyproto/anytype-heart/core/block/editor/smartblock" "github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest" + "github.com/anyproto/anytype-heart/core/block/editor/state" domain "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/subscription" "github.com/anyproto/anytype-heart/core/syncstatus/detailsupdater/mock_detailsupdater" + "github.com/anyproto/anytype-heart/core/syncstatus/filesyncstatus" "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscritions" "github.com/anyproto/anytype-heart/pkg/lib/bundle" coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/space/clientspace/mock_clientspace" "github.com/anyproto/anytype-heart/space/mock_space" "github.com/anyproto/anytype-heart/tests/testutil" "github.com/anyproto/anytype-heart/util/pbtypes" ) -func TestSyncStatusUpdater_UpdateDetails(t *testing.T) { - +type updateTester struct { + t *testing.T + waitCh chan struct{} + eventsCount int } -func TestSyncStatusUpdater_Run(t *testing.T) { +func newUpdateTester(t *testing.T, eventsCount int) *updateTester { + return &updateTester{t: t, eventsCount: eventsCount, waitCh: make(chan struct{}, eventsCount)} +} + +func (t *updateTester) done() { + t.waitCh <- struct{}{} +} + +// wait waits for at least one event up to t.eventsCount events +func (t *updateTester) wait() { + timeout := time.After(1 * time.Second) + minReceivedTimer := time.After(10 * time.Millisecond) + var eventsReceived int + for i := 0; i < t.eventsCount; i++ { + select { + case <-minReceivedTimer: + if eventsReceived > 0 { + return + } + case <-t.waitCh: + eventsReceived++ + case <-timeout: + t.t.Fatal("timeout") + } + } +} + +func newUpdateDetailsFixture(t *testing.T) *fixture { + fx := newFixture(t) + fx.spaceService.EXPECT().TechSpaceId().Return("techSpace") + err := fx.Run(context.Background()) + require.NoError(t, err) + t.Cleanup(func() { + err := fx.Close(context.Background()) + require.NoError(t, err) + }) + return fx +} + +func TestSyncStatusUpdater_UpdateDetails(t *testing.T) { + t.Run("ignore tech space", func(t *testing.T) { + fx := newUpdateDetailsFixture(t) + + fx.UpdateDetails("spaceView1", domain.ObjectSyncStatusSynced, "techSpace") + }) + + t.Run("updates to the same object", func(t *testing.T) { + fx := newUpdateDetailsFixture(t) + updTester := newUpdateTester(t, 4) + + space := mock_clientspace.NewMockSpace(t) + fx.spaceService.EXPECT().Get(mock.Anything, "space1").Return(space, nil) + space.EXPECT().DoLockedIfNotExists(mock.Anything, mock.Anything).Return(ocache.ErrExists).Times(0) + space.EXPECT().DoCtx(mock.Anything, mock.Anything, mock.Anything).Run(func(ctx context.Context, objectId string, apply func(smartblock.SmartBlock) error) { + sb := smarttest.New(objectId) + st := sb.Doc.(*state.State) + st.SetDetailAndBundledRelation(bundle.RelationKeyLayout, pbtypes.Int64(int64(model.ObjectType_basic))) + err := apply(sb) + require.NoError(t, err) + + det := sb.Doc.LocalDetails() + assert.Contains(t, det.GetFields(), bundle.RelationKeySyncStatus.String()) + assert.Contains(t, det.GetFields(), bundle.RelationKeySyncDate.String()) + assert.Contains(t, det.GetFields(), bundle.RelationKeySyncError.String()) + + fx.spaceStatusUpdater.EXPECT().Refresh("space1") + + updTester.done() + }).Return(nil).Times(0) + + fx.UpdateDetails("id1", domain.ObjectSyncStatusSyncing, "space1") + fx.UpdateDetails("id1", domain.ObjectSyncStatusError, "space1") + fx.UpdateDetails("id1", domain.ObjectSyncStatusSyncing, "space1") + fx.UpdateDetails("id1", domain.ObjectSyncStatusSynced, "space1") + + updTester.wait() + }) + + t.Run("updates in file object", func(t *testing.T) { + t.Run("file backup status limited", func(t *testing.T) { + fx := newUpdateDetailsFixture(t) + updTester := newUpdateTester(t, 1) + + space := mock_clientspace.NewMockSpace(t) + fx.spaceService.EXPECT().Get(mock.Anything, "space1").Return(space, nil) + space.EXPECT().DoLockedIfNotExists(mock.Anything, mock.Anything).Return(ocache.ErrExists) + space.EXPECT().DoCtx(mock.Anything, mock.Anything, mock.Anything).Run(func(ctx context.Context, objectId string, apply func(smartblock.SmartBlock) error) { + sb := smarttest.New(objectId) + st := sb.Doc.(*state.State) + st.SetDetailAndBundledRelation(bundle.RelationKeyLayout, pbtypes.Int64(int64(model.ObjectType_file))) + st.SetDetailAndBundledRelation(bundle.RelationKeyFileBackupStatus, pbtypes.Int64(int64(filesyncstatus.Limited))) + err := apply(sb) + require.NoError(t, err) + + det := sb.Doc.LocalDetails() + assert.True(t, pbtypes.GetInt64(det, bundle.RelationKeySyncStatus.String()) == int64(domain.ObjectSyncStatusError)) + assert.True(t, pbtypes.GetInt64(det, bundle.RelationKeySyncError.String()) == int64(domain.SyncErrorOversized)) + assert.Contains(t, det.GetFields(), bundle.RelationKeySyncDate.String()) + + fx.spaceStatusUpdater.EXPECT().Refresh("space1") + + updTester.done() + }).Return(nil) + + fx.UpdateDetails("id2", domain.ObjectSyncStatusSynced, "space1") + + updTester.wait() + }) + t.Run("prioritize object status", func(t *testing.T) { + fx := newUpdateDetailsFixture(t) + updTester := newUpdateTester(t, 1) + + space := mock_clientspace.NewMockSpace(t) + fx.spaceService.EXPECT().Get(mock.Anything, "space1").Return(space, nil) + space.EXPECT().DoLockedIfNotExists(mock.Anything, mock.Anything).Return(ocache.ErrExists) + space.EXPECT().DoCtx(mock.Anything, mock.Anything, mock.Anything).Run(func(ctx context.Context, objectId string, apply func(smartblock.SmartBlock) error) { + sb := smarttest.New(objectId) + st := sb.Doc.(*state.State) + st.SetDetailAndBundledRelation(bundle.RelationKeyLayout, pbtypes.Int64(int64(model.ObjectType_file))) + st.SetDetailAndBundledRelation(bundle.RelationKeyFileBackupStatus, pbtypes.Int64(int64(filesyncstatus.Synced))) + err := apply(sb) + require.NoError(t, err) + + det := sb.Doc.LocalDetails() + assert.True(t, pbtypes.GetInt64(det, bundle.RelationKeySyncStatus.String()) == int64(domain.ObjectSyncStatusSyncing)) + assert.Contains(t, det.GetFields(), bundle.RelationKeySyncError.String()) + assert.Contains(t, det.GetFields(), bundle.RelationKeySyncDate.String()) + + fx.spaceStatusUpdater.EXPECT().Refresh("space1") + + updTester.done() + }).Return(nil) + + fx.UpdateDetails("id3", domain.ObjectSyncStatusSyncing, "space1") + + updTester.wait() + }) + }) +} + +func TestSyncStatusUpdater_UpdateSpaceDetails(t *testing.T) { } func TestSyncStatusUpdater_setSyncDetails(t *testing.T) { t.Run("set smartblock details", func(t *testing.T) { // given - fixture := newFixture(t) + fx := newFixture(t) + sb := smarttest.New("id") // when - err := fixture.updater.setSyncDetails(fixture.sb, domain.ObjectSyncStatusError, domain.SyncErrorNetworkError) + err := fx.setSyncDetails(sb, domain.ObjectSyncStatusError, domain.SyncErrorNetworkError) assert.Nil(t, err) // then - details := fixture.sb.NewState().CombinedDetails().GetFields() + details := sb.NewState().CombinedDetails().GetFields() assert.NotNil(t, details) assert.Equal(t, pbtypes.Int64(int64(domain.SpaceSyncStatusError)), details[bundle.RelationKeySyncStatus.String()]) assert.Equal(t, pbtypes.Int64(int64(domain.SyncErrorNetworkError)), details[bundle.RelationKeySyncError.String()]) @@ -48,22 +199,24 @@ func TestSyncStatusUpdater_setSyncDetails(t *testing.T) { }) t.Run("not set smartblock details, because it doesn't implement interface DetailsSettable", func(t *testing.T) { // given - fixture := newFixture(t) + fx := newFixture(t) + sb := smarttest.New("id") // when - fixture.sb.SetType(coresb.SmartBlockTypePage) - err := fixture.updater.setSyncDetails(editor.NewMissingObject(fixture.sb), domain.ObjectSyncStatusError, domain.SyncErrorNetworkError) + sb.SetType(coresb.SmartBlockTypePage) + err := fx.setSyncDetails(editor.NewMissingObject(sb), domain.ObjectSyncStatusError, domain.SyncErrorNetworkError) // then assert.Nil(t, err) }) t.Run("not set smartblock details, because it doesn't need details", func(t *testing.T) { // given - fixture := newFixture(t) + fx := newFixture(t) + sb := smarttest.New("id") // when - fixture.sb.SetType(coresb.SmartBlockTypeHome) - err := fixture.updater.setSyncDetails(fixture.sb, domain.ObjectSyncStatusError, domain.SyncErrorNetworkError) + sb.SetType(coresb.SmartBlockTypeHome) + err := fx.setSyncDetails(sb, domain.ObjectSyncStatusError, domain.SyncErrorNetworkError) // then assert.Nil(t, err) @@ -73,13 +226,13 @@ func TestSyncStatusUpdater_setSyncDetails(t *testing.T) { func TestSyncStatusUpdater_isLayoutSuitableForSyncRelations(t *testing.T) { t.Run("isLayoutSuitableForSyncRelations - participant details", func(t *testing.T) { // given - fixture := newFixture(t) + fx := newFixture(t) // when details := &types.Struct{Fields: map[string]*types.Value{ bundle.RelationKeyLayout.String(): pbtypes.Float64(float64(model.ObjectType_participant)), }} - isSuitable := fixture.updater.isLayoutSuitableForSyncRelations(details) + isSuitable := fx.isLayoutSuitableForSyncRelations(details) // then assert.False(t, isSuitable) @@ -87,13 +240,13 @@ func TestSyncStatusUpdater_isLayoutSuitableForSyncRelations(t *testing.T) { t.Run("isLayoutSuitableForSyncRelations - basic details", func(t *testing.T) { // given - fixture := newFixture(t) + fx := newFixture(t) // when details := &types.Struct{Fields: map[string]*types.Value{ bundle.RelationKeyLayout.String(): pbtypes.Float64(float64(model.ObjectType_basic)), }} - isSuitable := fixture.updater.isLayoutSuitableForSyncRelations(details) + isSuitable := fx.isLayoutSuitableForSyncRelations(details) // then assert.True(t, isSuitable) @@ -101,12 +254,8 @@ func TestSyncStatusUpdater_isLayoutSuitableForSyncRelations(t *testing.T) { } func newFixture(t *testing.T) *fixture { - smartTest := smarttest.New("id") service := mock_space.NewMockService(t) - updater := &syncStatusUpdater{ - finish: make(chan struct{}), - entries: map[string]*syncStatusDetails{}, - } + updater := New() statusUpdater := mock_detailsupdater.NewMockSpaceStatusUpdater(t) subscriptionService := subscription.NewInternalTestService(t) @@ -121,20 +270,19 @@ func newFixture(t *testing.T) *fixture { a.Register(testutil.PrepareMock(ctx, a, service)) a.Register(testutil.PrepareMock(ctx, a, statusUpdater)) err := updater.Init(a) - assert.Nil(t, err) + assert.NoError(t, err) + return &fixture{ - updater: updater, - sb: smartTest, - service: service, - statusUpdater: statusUpdater, + syncStatusUpdater: updater.(*syncStatusUpdater), + spaceService: service, + spaceStatusUpdater: statusUpdater, subscriptionService: subscriptionService, } } type fixture struct { - sb *smarttest.SmartTest - updater *syncStatusUpdater - service *mock_space.MockService - statusUpdater *mock_detailsupdater.MockSpaceStatusUpdater + *syncStatusUpdater + spaceService *mock_space.MockService + spaceStatusUpdater *mock_detailsupdater.MockSpaceStatusUpdater subscriptionService *subscription.InternalTestService } From da11d4bbfbf1ef27e1ef985f09879b81c0882741 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Thu, 25 Jul 2024 11:03:09 +0200 Subject: [PATCH 52/71] GO-3817 remove peer pick and debug logs --- core/peerstatus/status.go | 51 ++++++++------------------ space/spacecore/peerstore/peerstore.go | 3 +- 2 files changed, 18 insertions(+), 36 deletions(-) diff --git a/core/peerstatus/status.go b/core/peerstatus/status.go index 0d2ed2c8e..f281ab7ed 100644 --- a/core/peerstatus/status.go +++ b/core/peerstatus/status.go @@ -4,7 +4,6 @@ import ( "context" "errors" "sync" - "time" "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/net/pool" @@ -98,18 +97,15 @@ func (p *p2pStatus) Init(a *app.App) (err error) { sessionHookRunner.RegisterHook(p.sendStatusForNewSession) p.ctx, p.contextCancel = context.WithCancel(context.Background()) p.peerStore.AddObserver(func(peerId string, spaceIdsBefore, spaceIdsAfter []string, peerRemoved bool) { - go func() { - // we need to update status for all spaces that were either added or removed to some local peer - // because we start this observer on init we can be sure that the spaceIdsBefore is empty on the first run for peer - removed, added := lo.Difference(spaceIdsBefore, spaceIdsAfter) - log.Warnf("peer %s space observer: removed: %v, added: %v", peerId, removed, added) - err := p.refreshSpaces(lo.Union(removed, added)) - if errors.Is(err, ErrClosed) { - return - } else if err != nil { - log.Errorf("refreshSpaces failed: %v", err) - } - }() + // we need to update status for all spaces that were either added or removed to some local peer + // because we start this observer on init we can be sure that the spaceIdsBefore is empty on the first run for peer + removed, added := lo.Difference(spaceIdsBefore, spaceIdsAfter) + err := p.refreshSpaces(lo.Union(removed, added)) + if errors.Is(err, ErrClosed) { + return + } else if err != nil { + log.Errorf("refreshSpaces failed: %v", err) + } }) return nil } @@ -142,6 +138,9 @@ func (p *p2pStatus) Name() (name string) { func (p *p2pStatus) setNotPossibleStatus() { p.Lock() + if p.p2pNotPossible { + return + } p.p2pNotPossible = true p.Unlock() p.refreshAllSpaces() @@ -149,6 +148,9 @@ func (p *p2pStatus) setNotPossibleStatus() { func (p *p2pStatus) resetNotPossibleStatus() { p.Lock() + if !p.p2pNotPossible { + return + } p.p2pNotPossible = false p.Unlock() p.refreshAllSpaces() @@ -174,17 +176,12 @@ func (p *p2pStatus) UnregisterSpace(spaceId string) { func (p *p2pStatus) worker() { defer close(p.workerFinished) - timer := time.NewTicker(20 * time.Second) - defer timer.Stop() for { select { case <-p.ctx.Done(): return case spaceId := <-p.refreshSpaceId: p.processSpaceStatusUpdate(spaceId) - case <-timer.C: - // todo: looks like we don't need this anymore because we use observer - go p.refreshAllSpaces() } } } @@ -235,8 +232,6 @@ func (p *p2pStatus) processSpaceStatusUpdate(spaceId string) { connectionCount := p.countOpenConnections(spaceId) newStatus := p.getResultStatus(p.p2pNotPossible, connectionCount) - log.Warnf("processSpaceStatusUpdate: spaceId: %s, currentStatus: %v, newStatus: %v, connectionCount: %d", spaceId, currentStatus.status, newStatus, connectionCount) - if currentStatus.status != newStatus || currentStatus.connectionsCount != connectionCount { p.sendEvent("", spaceId, newStatus.ToPb(), connectionCount) currentStatus.status = newStatus @@ -256,26 +251,12 @@ func (p *p2pStatus) getResultStatus(notPossible bool, connectionCount int64) Sta } func (p *p2pStatus) countOpenConnections(spaceId string) int64 { - var connectionCount int64 - ctx, cancelFunc := context.WithTimeout(p.ctx, time.Second*10) - defer cancelFunc() peerIds := p.peerStore.LocalPeerIds(spaceId) - log.Warnf("processSpaceStatusUpdate: spaceId: %s, localPeerIds: %v", spaceId, peerIds) - - for _, peerId := range peerIds { - _, err := p.peersConnectionPool.Pick(ctx, peerId) - if err != nil { - log.Warnf("countOpenConnections space %s failed to get local peer %s from net pool: %v", spaceId, peerId, err) - continue - } - connectionCount++ - } - return connectionCount + return int64(len(peerIds)) } // sendEvent sends event to session with sessionToken or broadcast to all sessions if sessionToken is empty func (p *p2pStatus) sendEvent(sessionToken string, spaceId string, status pb.EventP2PStatusStatus, count int64) { - log.Warnf("sendEvent: sessionToken: %s, spaceId: %s, status: %v, count: %d", sessionToken, spaceId, status, count) event := &pb.Event{ Messages: []*pb.EventMessage{ { diff --git a/space/spacecore/peerstore/peerstore.go b/space/spacecore/peerstore/peerstore.go index 039e2522d..d5d4b459e 100644 --- a/space/spacecore/peerstore/peerstore.go +++ b/space/spacecore/peerstore/peerstore.go @@ -131,13 +131,14 @@ func (p *peerStore) LocalPeerIds(spaceId string) []string { func (p *peerStore) RemoveLocalPeer(peerId string) { p.Lock() - defer p.Unlock() spaceIds, exists := p.spacesByLocalPeerIds[peerId] if !exists { + p.Unlock() return } defer func() { observers := p.observers + p.Unlock() for _, ob := range observers { ob(peerId, spaceIds, nil, true) } From 6cc307da3a61062274326d823f495a75ee7ca41c Mon Sep 17 00:00:00 2001 From: Sergey Date: Thu, 25 Jul 2024 11:11:09 +0200 Subject: [PATCH 53/71] GO-3769: Some test fixes --- core/syncstatus/filestatus.go | 3 -- core/syncstatus/filestatus_test.go | 79 ------------------------------ core/syncstatus/service.go | 4 -- 3 files changed, 86 deletions(-) delete mode 100644 core/syncstatus/filestatus_test.go diff --git a/core/syncstatus/filestatus.go b/core/syncstatus/filestatus.go index 14c0a8ae5..9b6cb41b5 100644 --- a/core/syncstatus/filestatus.go +++ b/core/syncstatus/filestatus.go @@ -29,9 +29,7 @@ func (s *service) OnFileDelete(fileId domain.FullFileId) { } func (s *service) indexFileSyncStatus(fileObjectId string, status filesyncstatus.Status) error { - var spaceId string err := cache.Do(s.objectGetter, fileObjectId, func(sb smartblock.SmartBlock) (err error) { - spaceId = sb.SpaceID() prevStatus := pbtypes.GetInt64(sb.Details(), bundle.RelationKeyFileBackupStatus.String()) newStatus := int64(status) if prevStatus == newStatus { @@ -48,6 +46,5 @@ func (s *service) indexFileSyncStatus(fileObjectId string, status filesyncstatus if err != nil { return fmt.Errorf("update tree: %w", err) } - s.spaceSyncStatus.Refresh(spaceId) return nil } diff --git a/core/syncstatus/filestatus_test.go b/core/syncstatus/filestatus_test.go deleted file mode 100644 index 0a42ae27c..000000000 --- a/core/syncstatus/filestatus_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package syncstatus - -import ( - "testing" - - "github.com/anyproto/anytype-heart/core/domain" - "github.com/anyproto/anytype-heart/core/syncstatus/filesyncstatus" - "github.com/anyproto/anytype-heart/core/syncstatus/spacesyncstatus/mock_spacesyncstatus" -) - -func Test_sendSpaceStatusUpdate(t *testing.T) { - t.Run("file limited", func(t *testing.T) { - // given - updater := mock_spacesyncstatus.NewMockUpdater(t) - s := &service{ - spaceSyncStatus: updater, - } - - // when - updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusError, domain.SyncErrorStorageLimitExceed)).Return() - s.sendSpaceStatusUpdate(filesyncstatus.Limited, "spaceId", 0) - }) - t.Run("file limited, but over 1% of storage is available", func(t *testing.T) { - // given - updater := mock_spacesyncstatus.NewMockUpdater(t) - s := &service{ - spaceSyncStatus: updater, - } - - // when - updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull)).Return() - s.sendSpaceStatusUpdate(filesyncstatus.Limited, "spaceId", 0.9) - }) - t.Run("file synced", func(t *testing.T) { - // given - updater := mock_spacesyncstatus.NewMockUpdater(t) - s := &service{ - spaceSyncStatus: updater, - } - - // when - updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSynced, domain.SyncErrorNull)).Return() - s.sendSpaceStatusUpdate(filesyncstatus.Synced, "spaceId", 0) - }) - t.Run("file queued", func(t *testing.T) { - // given - updater := mock_spacesyncstatus.NewMockUpdater(t) - s := &service{ - spaceSyncStatus: updater, - } - - // when - updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull)).Return() - s.sendSpaceStatusUpdate(filesyncstatus.Queued, "spaceId", 0) - }) - t.Run("file syncing", func(t *testing.T) { - // given - updater := mock_spacesyncstatus.NewMockUpdater(t) - s := &service{ - spaceSyncStatus: updater, - } - - // when - updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusSyncing, domain.SyncErrorNull)).Return() - s.sendSpaceStatusUpdate(filesyncstatus.Syncing, "spaceId", 0) - }) - t.Run("file unknown status", func(t *testing.T) { - // given - updater := mock_spacesyncstatus.NewMockUpdater(t) - s := &service{ - spaceSyncStatus: updater, - } - - // when - updater.EXPECT().SendUpdate(domain.MakeSyncStatus("spaceId", domain.SpaceSyncStatusError, domain.SyncErrorNetworkError)).Return() - s.sendSpaceStatusUpdate(filesyncstatus.Unknown, "spaceId", 0) - }) - -} diff --git a/core/syncstatus/service.go b/core/syncstatus/service.go index d816097a2..d868c2bc3 100644 --- a/core/syncstatus/service.go +++ b/core/syncstatus/service.go @@ -14,7 +14,6 @@ import ( "github.com/anyproto/anytype-heart/core/filestorage/filesync" "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" "github.com/anyproto/anytype-heart/core/syncstatus/objectsyncstatus" - "github.com/anyproto/anytype-heart/core/syncstatus/spacesyncstatus" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" ) @@ -39,8 +38,6 @@ type service struct { objectStore objectstore.ObjectStore objectGetter cache.ObjectGetter - - spaceSyncStatus spacesyncstatus.Updater } func New() Service { @@ -52,7 +49,6 @@ func New() Service { func (s *service) Init(a *app.App) (err error) { s.fileSyncService = app.MustComponent[filesync.FileSync](a) s.objectStore = app.MustComponent[objectstore.ObjectStore](a) - s.spaceSyncStatus = app.MustComponent[spacesyncstatus.Updater](a) s.objectGetter = app.MustComponent[cache.ObjectGetter](a) s.fileSyncService.OnUploaded(s.onFileUploaded) s.fileSyncService.OnUploadStarted(s.onFileUploadStarted) From 8d266a946d819deff65e8e1d5357e8de49e63fae Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Thu, 25 Jul 2024 12:50:19 +0200 Subject: [PATCH 54/71] GO-3817 fix missed unlock --- core/peerstatus/status.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/peerstatus/status.go b/core/peerstatus/status.go index f281ab7ed..3ab5cfb3d 100644 --- a/core/peerstatus/status.go +++ b/core/peerstatus/status.go @@ -139,6 +139,7 @@ func (p *p2pStatus) Name() (name string) { func (p *p2pStatus) setNotPossibleStatus() { p.Lock() if p.p2pNotPossible { + p.Unlock() return } p.p2pNotPossible = true @@ -149,6 +150,7 @@ func (p *p2pStatus) setNotPossibleStatus() { func (p *p2pStatus) resetNotPossibleStatus() { p.Lock() if !p.p2pNotPossible { + p.Unlock() return } p.p2pNotPossible = false From b8dd955188c936847ab7f168d9a5fe2d215c006a Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Thu, 25 Jul 2024 12:59:28 +0200 Subject: [PATCH 55/71] GO-3817 fix peermanager tests --- space/spacecore/peermanager/manager_test.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/space/spacecore/peermanager/manager_test.go b/space/spacecore/peermanager/manager_test.go index 0cb35efc5..171f56eee 100644 --- a/space/spacecore/peermanager/manager_test.go +++ b/space/spacecore/peermanager/manager_test.go @@ -112,9 +112,6 @@ func Test_fetchResponsiblePeers(t *testing.T) { f.conf.EXPECT().NodeIds(f.cm.spaceId).Return([]string{"id"}) f.pool.EXPECT().GetOneOf(gomock.Any(), gomock.Any()).Return(newTestPeer("id"), nil) f.cm.fetchResponsiblePeers() - - // then - f.peerToPeerStatus.AssertNotCalled(t, "CheckPeerStatus") }) t.Run("local peers connected", func(t *testing.T) { // given @@ -127,23 +124,17 @@ func Test_fetchResponsiblePeers(t *testing.T) { f.pool.EXPECT().Get(f.cm.ctx, "peerId").Return(newTestPeer("id1"), nil) f.cm.fetchResponsiblePeers() - // then - f.peerToPeerStatus.AssertNotCalled(t, "CheckPeerStatus") }) t.Run("local peer not connected", func(t *testing.T) { // given f := newFixtureManager(t, spaceId) f.store.UpdateLocalPeer("peerId", []string{spaceId}) - f.peerToPeerStatus.EXPECT().CheckPeerStatus().Return() // when f.conf.EXPECT().NodeIds(f.cm.spaceId).Return([]string{"id"}) f.pool.EXPECT().GetOneOf(gomock.Any(), gomock.Any()).Return(newTestPeer("id"), nil) f.pool.EXPECT().Get(f.cm.ctx, "peerId").Return(nil, fmt.Errorf("error")) f.cm.fetchResponsiblePeers() - - // then - f.peerToPeerStatus.AssertCalled(t, "CheckPeerStatus") }) } @@ -161,7 +152,6 @@ func Test_getStreamResponsiblePeers(t *testing.T) { // then assert.Nil(t, err) assert.Len(t, peers, 1) - f.peerToPeerStatus.AssertNotCalled(t, "CheckPeerStatus") }) t.Run("local peers connected", func(t *testing.T) { // given @@ -177,13 +167,11 @@ func Test_getStreamResponsiblePeers(t *testing.T) { // then assert.Nil(t, err) assert.Len(t, peers, 2) - f.peerToPeerStatus.AssertNotCalled(t, "CheckPeerStatus") }) t.Run("local peer not connected", func(t *testing.T) { // given f := newFixtureManager(t, spaceId) f.store.UpdateLocalPeer("peerId", []string{spaceId}) - f.peerToPeerStatus.EXPECT().CheckPeerStatus().Return() // when f.pool.EXPECT().GetOneOf(gomock.Any(), gomock.Any()).Return(newTestPeer("id"), nil) @@ -194,7 +182,6 @@ func Test_getStreamResponsiblePeers(t *testing.T) { // then assert.Nil(t, err) assert.Len(t, peers, 1) - f.peerToPeerStatus.AssertCalled(t, "CheckPeerStatus") }) } From 07237ebb531783815d7b5521aef9f504e7c6dba0 Mon Sep 17 00:00:00 2001 From: Sergey Date: Thu, 25 Jul 2024 13:07:59 +0200 Subject: [PATCH 56/71] GO-3769: Add UpdateSpaceDetails test --- core/syncstatus/detailsupdater/updater.go | 3 + .../syncstatus/detailsupdater/updater_test.go | 87 +++++++++++++++---- 2 files changed, 75 insertions(+), 15 deletions(-) diff --git a/core/syncstatus/detailsupdater/updater.go b/core/syncstatus/detailsupdater/updater.go index 8296755ab..38148b265 100644 --- a/core/syncstatus/detailsupdater/updater.go +++ b/core/syncstatus/detailsupdater/updater.go @@ -150,6 +150,9 @@ func (u *syncStatusUpdater) UpdateSpaceDetails(existing, missing []string, space } u.spaceSyncStatus.UpdateMissingIds(spaceId, missing) ids := u.getSyncingObjects(spaceId) + + // removed contains ids that are not yet marked as syncing + // added contains ids that were syncing, but appeared as synced, because they are not in existing list removed, added := slice.DifferenceRemovedAdded(existing, ids) if len(removed)+len(added) == 0 { u.spaceSyncStatus.Refresh(spaceId) diff --git a/core/syncstatus/detailsupdater/updater_test.go b/core/syncstatus/detailsupdater/updater_test.go index 6b613cc6a..2c4df6145 100644 --- a/core/syncstatus/detailsupdater/updater_test.go +++ b/core/syncstatus/detailsupdater/updater_test.go @@ -23,6 +23,7 @@ import ( "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscritions" "github.com/anyproto/anytype-heart/pkg/lib/bundle" coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" + "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" "github.com/anyproto/anytype-heart/space/clientspace/mock_clientspace" "github.com/anyproto/anytype-heart/space/mock_space" @@ -31,28 +32,34 @@ import ( ) type updateTester struct { - t *testing.T - waitCh chan struct{} - eventsCount int + t *testing.T + waitCh chan struct{} + minEventsCount int + maxEventsCount int } -func newUpdateTester(t *testing.T, eventsCount int) *updateTester { - return &updateTester{t: t, eventsCount: eventsCount, waitCh: make(chan struct{}, eventsCount)} +func newUpdateTester(t *testing.T, minEventsCount int, maxEventsCount int) *updateTester { + return &updateTester{ + t: t, + minEventsCount: minEventsCount, + maxEventsCount: maxEventsCount, + waitCh: make(chan struct{}, maxEventsCount), + } } func (t *updateTester) done() { t.waitCh <- struct{}{} } -// wait waits for at least one event up to t.eventsCount events +// wait waits for at least one event up to t.maxEventsCount events func (t *updateTester) wait() { timeout := time.After(1 * time.Second) minReceivedTimer := time.After(10 * time.Millisecond) var eventsReceived int - for i := 0; i < t.eventsCount; i++ { + for i := 0; i < t.maxEventsCount; i++ { select { case <-minReceivedTimer: - if eventsReceived > 0 { + if eventsReceived >= t.minEventsCount { return } case <-t.waitCh: @@ -84,7 +91,7 @@ func TestSyncStatusUpdater_UpdateDetails(t *testing.T) { t.Run("updates to the same object", func(t *testing.T) { fx := newUpdateDetailsFixture(t) - updTester := newUpdateTester(t, 4) + updTester := newUpdateTester(t, 1, 4) space := mock_clientspace.NewMockSpace(t) fx.spaceService.EXPECT().Get(mock.Anything, "space1").Return(space, nil) @@ -117,7 +124,7 @@ func TestSyncStatusUpdater_UpdateDetails(t *testing.T) { t.Run("updates in file object", func(t *testing.T) { t.Run("file backup status limited", func(t *testing.T) { fx := newUpdateDetailsFixture(t) - updTester := newUpdateTester(t, 1) + updTester := newUpdateTester(t, 1, 1) space := mock_clientspace.NewMockSpace(t) fx.spaceService.EXPECT().Get(mock.Anything, "space1").Return(space, nil) @@ -146,7 +153,7 @@ func TestSyncStatusUpdater_UpdateDetails(t *testing.T) { }) t.Run("prioritize object status", func(t *testing.T) { fx := newUpdateDetailsFixture(t) - updTester := newUpdateTester(t, 1) + updTester := newUpdateTester(t, 1, 1) space := mock_clientspace.NewMockSpace(t) fx.spaceService.EXPECT().Get(mock.Anything, "space1").Return(space, nil) @@ -174,10 +181,59 @@ func TestSyncStatusUpdater_UpdateDetails(t *testing.T) { updTester.wait() }) }) + + // TODO Test DoLockedIfNotExists } func TestSyncStatusUpdater_UpdateSpaceDetails(t *testing.T) { + fx := newUpdateDetailsFixture(t) + updTester := newUpdateTester(t, 3, 3) + fx.subscriptionService.StoreFixture.AddObjects(t, []objectstore.TestObject{ + { + bundle.RelationKeyId: pbtypes.String("id1"), + bundle.RelationKeySpaceId: pbtypes.String("space1"), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_basic)), + bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.ObjectSyncStatusSyncing)), + }, + { + bundle.RelationKeyId: pbtypes.String("id4"), + bundle.RelationKeySpaceId: pbtypes.String("space1"), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_basic)), + bundle.RelationKeySyncStatus: pbtypes.Int64(int64(domain.ObjectSyncStatusSyncing)), + }, + }) + + space := mock_clientspace.NewMockSpace(t) + fx.spaceService.EXPECT().Get(mock.Anything, "space1").Return(space, nil) + space.EXPECT().DoLockedIfNotExists(mock.Anything, mock.Anything).Return(ocache.ErrExists).Times(0) + + assertUpdate := func(objectId string, status domain.ObjectSyncStatus) { + space.EXPECT().DoCtx(mock.Anything, objectId, mock.Anything).Run(func(ctx context.Context, objectId string, apply func(smartblock.SmartBlock) error) { + sb := smarttest.New(objectId) + st := sb.Doc.(*state.State) + st.SetDetailAndBundledRelation(bundle.RelationKeyLayout, pbtypes.Int64(int64(model.ObjectType_basic))) + err := apply(sb) + require.NoError(t, err) + + det := sb.Doc.LocalDetails() + assert.True(t, pbtypes.GetInt64(det, bundle.RelationKeySyncStatus.String()) == int64(status)) + assert.Contains(t, det.GetFields(), bundle.RelationKeySyncDate.String()) + assert.Contains(t, det.GetFields(), bundle.RelationKeySyncError.String()) + + fx.spaceStatusUpdater.EXPECT().Refresh("space1") + + updTester.done() + }).Return(nil).Times(0) + } + + assertUpdate("id2", domain.ObjectSyncStatusSyncing) + assertUpdate("id4", domain.ObjectSyncStatusSynced) + + fx.spaceStatusUpdater.EXPECT().UpdateMissingIds("space1", []string{"id3"}) + fx.UpdateSpaceDetails([]string{"id1", "id2"}, []string{"id3"}, "space1") + + updTester.wait() } func TestSyncStatusUpdater_setSyncDetails(t *testing.T) { @@ -258,19 +314,20 @@ func newFixture(t *testing.T) *fixture { updater := New() statusUpdater := mock_detailsupdater.NewMockSpaceStatusUpdater(t) - subscriptionService := subscription.NewInternalTestService(t) - syncSub := syncsubscritions.New() ctx := context.Background() a := &app.App{} - a.Register(subscriptionService) + subscriptionService := subscription.RegisterSubscriptionService(t, a) a.Register(syncSub) a.Register(testutil.PrepareMock(ctx, a, service)) a.Register(testutil.PrepareMock(ctx, a, statusUpdater)) err := updater.Init(a) - assert.NoError(t, err) + require.NoError(t, err) + + err = a.Start(ctx) + require.NoError(t, err) return &fixture{ syncStatusUpdater: updater.(*syncStatusUpdater), From 229cbbef32fde43fa995f9f3134a6477bb9acde0 Mon Sep 17 00:00:00 2001 From: Sergey Date: Thu, 25 Jul 2024 13:21:18 +0200 Subject: [PATCH 57/71] GO-3769: Update test --- .../syncstatus/detailsupdater/updater_test.go | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/core/syncstatus/detailsupdater/updater_test.go b/core/syncstatus/detailsupdater/updater_test.go index 2c4df6145..f2b9392aa 100644 --- a/core/syncstatus/detailsupdater/updater_test.go +++ b/core/syncstatus/detailsupdater/updater_test.go @@ -121,6 +121,40 @@ func TestSyncStatusUpdater_UpdateDetails(t *testing.T) { updTester.wait() }) + t.Run("updates to object not in cache", func(t *testing.T) { + fx := newUpdateDetailsFixture(t) + updTester := newUpdateTester(t, 1, 1) + + fx.subscriptionService.StoreFixture.AddObjects(t, []objectstore.TestObject{ + { + bundle.RelationKeyId: pbtypes.String("id1"), + bundle.RelationKeySpaceId: pbtypes.String("space1"), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_basic)), + }, + }) + + space := mock_clientspace.NewMockSpace(t) + fx.spaceService.EXPECT().Get(mock.Anything, "space1").Return(space, nil) + space.EXPECT().DoLockedIfNotExists(mock.Anything, mock.Anything).Run(func(objectId string, proc func() error) { + err := proc() + require.NoError(t, err) + + details, err := fx.objectStore.GetDetails(objectId) + require.NoError(t, err) + + assert.True(t, pbtypes.GetInt64(details.Details, bundle.RelationKeySyncStatus.String()) == int64(domain.ObjectSyncStatusError)) + assert.True(t, pbtypes.GetInt64(details.Details, bundle.RelationKeySyncError.String()) == int64(domain.SyncErrorNull)) + assert.Contains(t, details.Details.GetFields(), bundle.RelationKeySyncDate.String()) + updTester.done() + }).Return(nil).Times(0) + + fx.UpdateDetails("id1", domain.ObjectSyncStatusError, "space1") + + fx.spaceStatusUpdater.EXPECT().Refresh("space1") + + updTester.wait() + }) + t.Run("updates in file object", func(t *testing.T) { t.Run("file backup status limited", func(t *testing.T) { fx := newUpdateDetailsFixture(t) @@ -233,6 +267,10 @@ func TestSyncStatusUpdater_UpdateSpaceDetails(t *testing.T) { fx.spaceStatusUpdater.EXPECT().UpdateMissingIds("space1", []string{"id3"}) fx.UpdateSpaceDetails([]string{"id1", "id2"}, []string{"id3"}, "space1") + fx.spaceStatusUpdater.EXPECT().UpdateMissingIds("space1", []string{"id3"}) + fx.spaceStatusUpdater.EXPECT().Refresh("space1") + fx.UpdateSpaceDetails([]string{"id1", "id2"}, []string{"id3"}, "space1") + updTester.wait() } From c613545c9361ac59ed3ad1828e1dc5a09c3e1376 Mon Sep 17 00:00:00 2001 From: Sergey Date: Thu, 25 Jul 2024 13:39:25 +0200 Subject: [PATCH 58/71] GO-3769: Fix filestatus --- core/syncstatus/filestatus.go | 4 ---- core/syncstatus/service.go | 1 - 2 files changed, 5 deletions(-) diff --git a/core/syncstatus/filestatus.go b/core/syncstatus/filestatus.go index 9b6cb41b5..e03e18d5b 100644 --- a/core/syncstatus/filestatus.go +++ b/core/syncstatus/filestatus.go @@ -24,10 +24,6 @@ func (s *service) onFileLimited(objectId string, _ domain.FullFileId, bytesLeftP return s.indexFileSyncStatus(objectId, filesyncstatus.Limited) } -func (s *service) OnFileDelete(fileId domain.FullFileId) { - s.spaceSyncStatus.Refresh(fileId.SpaceId) -} - func (s *service) indexFileSyncStatus(fileObjectId string, status filesyncstatus.Status) error { err := cache.Do(s.objectGetter, fileObjectId, func(sb smartblock.SmartBlock) (err error) { prevStatus := pbtypes.GetInt64(sb.Details(), bundle.RelationKeyFileBackupStatus.String()) diff --git a/core/syncstatus/service.go b/core/syncstatus/service.go index d868c2bc3..98b432df3 100644 --- a/core/syncstatus/service.go +++ b/core/syncstatus/service.go @@ -53,7 +53,6 @@ func (s *service) Init(a *app.App) (err error) { s.fileSyncService.OnUploaded(s.onFileUploaded) s.fileSyncService.OnUploadStarted(s.onFileUploadStarted) s.fileSyncService.OnLimited(s.onFileLimited) - s.fileSyncService.OnDelete(s.OnFileDelete) nodeConfService := app.MustComponent[nodeconf.Service](a) cfg := app.MustComponent[*config.Config](a) From dd87311ab00bab6c60d83f2d9a542f31b786214c Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 25 Jul 2024 13:47:08 +0200 Subject: [PATCH 59/71] GO-3769 Fix treesyncer tests --- core/block/object/treesyncer/treesyncer.go | 1 + .../object/treesyncer/treesyncer_test.go | 60 ++++++------------- go.mod | 2 +- go.sum | 2 + 4 files changed, 23 insertions(+), 42 deletions(-) diff --git a/core/block/object/treesyncer/treesyncer.go b/core/block/object/treesyncer/treesyncer.go index 03d1d39b2..a56b18365 100644 --- a/core/block/object/treesyncer/treesyncer.go +++ b/core/block/object/treesyncer/treesyncer.go @@ -237,6 +237,7 @@ func (t *treeSyncer) updateTree(peerId, id string) { syncTree, ok := tr.(synctree.SyncTree) if !ok { log.Warn("not a sync tree") + return } if err = syncTree.SyncWithPeer(ctx, peerId); err != nil { log.Warn("synctree.SyncWithPeer error", zap.Error(err)) diff --git a/core/block/object/treesyncer/treesyncer_test.go b/core/block/object/treesyncer/treesyncer_test.go index 6f0df3de9..04be6fdf7 100644 --- a/core/block/object/treesyncer/treesyncer_test.go +++ b/core/block/object/treesyncer/treesyncer_test.go @@ -16,7 +16,6 @@ import ( "go.uber.org/mock/gomock" "github.com/anyproto/anytype-heart/core/block/object/treesyncer/mock_treesyncer" - "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/tests/testutil" ) @@ -91,6 +90,25 @@ func TestTreeSyncer(t *testing.T) { fx.Close(ctx) }) + t.Run("delayed sync notify sync status", func(t *testing.T) { + ctx := context.Background() + fx := newFixture(t, spaceId) + fx.treeManager.EXPECT().GetTree(gomock.Any(), spaceId, existingId).Return(fx.existingMock, nil) + fx.existingMock.EXPECT().SyncWithPeer(gomock.Any(), peerId).Return(nil) + fx.treeManager.EXPECT().GetTree(gomock.Any(), spaceId, missingId).Return(fx.missingMock, nil) + fx.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{peerId}) + fx.syncDetailsUpdater.EXPECT().UpdateSpaceDetails([]string{existingId}, []string{missingId}, spaceId) + fx.syncStatus.EXPECT().RemoveAllExcept(peerId, []string{existingId}).Return() + err := fx.SyncAll(context.Background(), peerId, []string{existingId}, []string{missingId}) + require.NoError(t, err) + require.NotNil(t, fx.requestPools[peerId]) + require.NotNil(t, fx.headPools[peerId]) + + fx.StartSync() + time.Sleep(100 * time.Millisecond) + fx.Close(ctx) + }) + t.Run("sync after run", func(t *testing.T) { ctx := context.Background() fx := newFixture(t, spaceId) @@ -189,45 +207,5 @@ func TestTreeSyncer(t *testing.T) { require.Equal(t, []string{"before close", "after done"}, events) mutex.Unlock() }) - t.Run("send offline event", func(t *testing.T) { - ctx := context.Background() - fx := newFixture(t, spaceId) - fx.treeManager.EXPECT().GetTree(gomock.Any(), spaceId, existingId).Return(fx.existingMock, nil) - fx.existingMock.EXPECT().SyncWithPeer(gomock.Any(), peerId).Return(nil) - fx.treeManager.EXPECT().GetTree(gomock.Any(), spaceId, missingId).Return(fx.missingMock, nil) - fx.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{peerId}) - fx.checker.EXPECT().IsPeerOffline(peerId).Return(true) - fx.syncStatus.EXPECT().RemoveAllExcept(peerId, []string{existingId}).Return() - fx.syncDetailsUpdater.EXPECT().UpdateDetails([]string{"existing"}, domain.ObjectSyncStatusError, domain.SyncErrorNetworkError, "spaceId").Return() - fx.StartSync() - err := fx.SyncAll(context.Background(), peerId, []string{existingId}, []string{missingId}) - require.NoError(t, err) - require.NotNil(t, fx.requestPools[peerId]) - require.NotNil(t, fx.headPools[peerId]) - - time.Sleep(100 * time.Millisecond) - fx.Close(ctx) - }) - t.Run("send syncing and synced event", func(t *testing.T) { - ctx := context.Background() - fx := newFixture(t, spaceId) - fx.treeManager.EXPECT().GetTree(gomock.Any(), spaceId, existingId).Return(fx.existingMock, nil) - fx.existingMock.EXPECT().SyncWithPeer(gomock.Any(), peerId).Return(nil) - fx.treeManager.EXPECT().GetTree(gomock.Any(), spaceId, missingId).Return(fx.missingMock, nil) - fx.nodeConf.EXPECT().NodeIds(spaceId).Return([]string{peerId}) - fx.checker.EXPECT().IsPeerOffline(peerId).Return(false) - fx.syncStatus.EXPECT().RemoveAllExcept(peerId, []string{existingId}).Return() - fx.syncDetailsUpdater.EXPECT().UpdateDetails([]string{"existing"}, domain.ObjectSyncStatusSynced, domain.SyncErrorNull, "spaceId").Return() - fx.syncDetailsUpdater.EXPECT().UpdateDetails([]string{"existing"}, domain.ObjectSyncStatusSyncing, domain.SyncErrorNull, "spaceId").Return() - - fx.StartSync() - err := fx.SyncAll(context.Background(), peerId, []string{existingId}, []string{missingId}) - require.NoError(t, err) - require.NotNil(t, fx.requestPools[peerId]) - require.NotNil(t, fx.headPools[peerId]) - - time.Sleep(100 * time.Millisecond) - fx.Close(ctx) - }) } diff --git a/go.mod b/go.mod index 468c27f5d..694b5b559 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/PuerkitoBio/goquery v1.9.2 github.com/VividCortex/ewma v1.2.0 github.com/adrium/goheif v0.0.0-20230113233934-ca402e77a786 - github.com/anyproto/any-sync v0.4.22-alpha.2 + github.com/anyproto/any-sync v0.4.22-alpha.2.0.20240725113756-193a1ce441f1 github.com/anyproto/go-naturaldate/v2 v2.0.2-0.20230524105841-9829cfd13438 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de github.com/avast/retry-go/v4 v4.6.0 diff --git a/go.sum b/go.sum index 58cefa808..cf9e64e86 100644 --- a/go.sum +++ b/go.sum @@ -91,6 +91,8 @@ github.com/anyproto/any-sync v0.4.22-alpha.1.0.20240717122315-3a22302fda91 h1:Wk github.com/anyproto/any-sync v0.4.22-alpha.1.0.20240717122315-3a22302fda91/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= github.com/anyproto/any-sync v0.4.22-alpha.2 h1:q9eAASQU3VEYYQH8VQAPZwzWHwcAku1mXZlWWs96VMk= github.com/anyproto/any-sync v0.4.22-alpha.2/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= +github.com/anyproto/any-sync v0.4.22-alpha.2.0.20240725113756-193a1ce441f1 h1:GPwjbv7dpW29zPVHBtFcsGOUM/YF1XIJq9bEq4cTW8Q= +github.com/anyproto/any-sync v0.4.22-alpha.2.0.20240725113756-193a1ce441f1/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580 h1:Ba80IlCCxkZ9H1GF+7vFu/TSpPvbpDCxXJ5ogc4euYc= github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580/go.mod h1:T/uWAYxrXdaXw64ihI++9RMbKTCpKd/yE9+saARew7k= github.com/anyproto/go-chash v0.1.0 h1:I9meTPjXFRfXZHRJzjOHC/XF7Q5vzysKkiT/grsogXY= From a3969b01d2b87fde3f71428bf065a99c2e44fbdf Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 25 Jul 2024 14:30:31 +0200 Subject: [PATCH 60/71] GO-3769 Not send duplicate space status events, fix manager tests --- core/domain/syncstatus.go | 16 ---- .../mock_nodestatus/mock_NodeStatus.go | 17 ++-- core/syncstatus/nodestatus/status.go | 21 +---- core/syncstatus/nodestatus/status_test.go | 80 ++----------------- .../syncstatus/spacesyncstatus/spacestatus.go | 65 ++++++++------- .../spacesyncstatus/spacestatus_test.go | 34 ++++++++ space/spacecore/peermanager/manager.go | 12 +-- space/spacecore/peermanager/manager_test.go | 38 ++++----- .../mock_peermanager/mock_PeerToPeerStatus.go | 32 -------- 9 files changed, 105 insertions(+), 210 deletions(-) diff --git a/core/domain/syncstatus.go b/core/domain/syncstatus.go index e1ac4df01..134351667 100644 --- a/core/domain/syncstatus.go +++ b/core/domain/syncstatus.go @@ -27,19 +27,3 @@ const ( SyncErrorNetworkError SyncError = 3 SyncErrorOversized SyncError = 4 ) - -type SpaceSync struct { - SpaceId string - Status SpaceSyncStatus - SyncError SyncError - // MissingObjects is a list of object IDs that are missing, it is not set every time - MissingObjects []string -} - -func MakeSyncStatus(spaceId string, status SpaceSyncStatus, syncError SyncError) *SpaceSync { - return &SpaceSync{ - SpaceId: spaceId, - Status: status, - SyncError: syncError, - } -} diff --git a/core/syncstatus/nodestatus/mock_nodestatus/mock_NodeStatus.go b/core/syncstatus/nodestatus/mock_nodestatus/mock_NodeStatus.go index c2f1d16ec..94e66595f 100644 --- a/core/syncstatus/nodestatus/mock_nodestatus/mock_NodeStatus.go +++ b/core/syncstatus/nodestatus/mock_nodestatus/mock_NodeStatus.go @@ -159,9 +159,9 @@ func (_c *MockNodeStatus_Name_Call) RunAndReturn(run func() string) *MockNodeSta return _c } -// SetNodesStatus provides a mock function with given fields: spaceId, senderId, status -func (_m *MockNodeStatus) SetNodesStatus(spaceId string, senderId string, status nodestatus.ConnectionStatus) { - _m.Called(spaceId, senderId, status) +// SetNodesStatus provides a mock function with given fields: spaceId, status +func (_m *MockNodeStatus) SetNodesStatus(spaceId string, status nodestatus.ConnectionStatus) { + _m.Called(spaceId, status) } // MockNodeStatus_SetNodesStatus_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetNodesStatus' @@ -171,15 +171,14 @@ type MockNodeStatus_SetNodesStatus_Call struct { // SetNodesStatus is a helper method to define mock.On call // - spaceId string -// - senderId string // - status nodestatus.ConnectionStatus -func (_e *MockNodeStatus_Expecter) SetNodesStatus(spaceId interface{}, senderId interface{}, status interface{}) *MockNodeStatus_SetNodesStatus_Call { - return &MockNodeStatus_SetNodesStatus_Call{Call: _e.mock.On("SetNodesStatus", spaceId, senderId, status)} +func (_e *MockNodeStatus_Expecter) SetNodesStatus(spaceId interface{}, status interface{}) *MockNodeStatus_SetNodesStatus_Call { + return &MockNodeStatus_SetNodesStatus_Call{Call: _e.mock.On("SetNodesStatus", spaceId, status)} } -func (_c *MockNodeStatus_SetNodesStatus_Call) Run(run func(spaceId string, senderId string, status nodestatus.ConnectionStatus)) *MockNodeStatus_SetNodesStatus_Call { +func (_c *MockNodeStatus_SetNodesStatus_Call) Run(run func(spaceId string, status nodestatus.ConnectionStatus)) *MockNodeStatus_SetNodesStatus_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(string), args[1].(string), args[2].(nodestatus.ConnectionStatus)) + run(args[0].(string), args[1].(nodestatus.ConnectionStatus)) }) return _c } @@ -189,7 +188,7 @@ func (_c *MockNodeStatus_SetNodesStatus_Call) Return() *MockNodeStatus_SetNodesS return _c } -func (_c *MockNodeStatus_SetNodesStatus_Call) RunAndReturn(run func(string, string, nodestatus.ConnectionStatus)) *MockNodeStatus_SetNodesStatus_Call { +func (_c *MockNodeStatus_SetNodesStatus_Call) RunAndReturn(run func(string, nodestatus.ConnectionStatus)) *MockNodeStatus_SetNodesStatus_Call { _c.Call.Return(run) return _c } diff --git a/core/syncstatus/nodestatus/status.go b/core/syncstatus/nodestatus/status.go index 775c92579..0fb20c9e3 100644 --- a/core/syncstatus/nodestatus/status.go +++ b/core/syncstatus/nodestatus/status.go @@ -1,19 +1,16 @@ package nodestatus import ( - "slices" "sync" "github.com/anyproto/any-sync/app" - "github.com/anyproto/any-sync/nodeconf" ) const CName = "core.syncstatus.nodestatus" type nodeStatus struct { sync.Mutex - configuration nodeconf.NodeConf - nodeStatus map[string]ConnectionStatus + nodeStatus map[string]ConnectionStatus } type ConnectionStatus int @@ -26,16 +23,15 @@ const ( type NodeStatus interface { app.Component - SetNodesStatus(spaceId string, senderId string, status ConnectionStatus) + SetNodesStatus(spaceId string, status ConnectionStatus) GetNodeStatus(spaceId string) ConnectionStatus } func NewNodeStatus() NodeStatus { - return &nodeStatus{nodeStatus: make(map[string]ConnectionStatus, 0)} + return &nodeStatus{nodeStatus: make(map[string]ConnectionStatus)} } func (n *nodeStatus) Init(a *app.App) (err error) { - n.configuration = app.MustComponent[nodeconf.NodeConf](a) return } @@ -49,17 +45,8 @@ func (n *nodeStatus) GetNodeStatus(spaceId string) ConnectionStatus { return n.nodeStatus[spaceId] } -func (n *nodeStatus) SetNodesStatus(spaceId string, senderId string, status ConnectionStatus) { - if !n.isSenderResponsible(senderId, spaceId) { - return - } - +func (n *nodeStatus) SetNodesStatus(spaceId string, status ConnectionStatus) { n.Lock() defer n.Unlock() - n.nodeStatus[spaceId] = status } - -func (n *nodeStatus) isSenderResponsible(senderId string, spaceId string) bool { - return slices.Contains(n.configuration.NodeIds(spaceId), senderId) -} diff --git a/core/syncstatus/nodestatus/status_test.go b/core/syncstatus/nodestatus/status_test.go index a58272af7..dfaf05258 100644 --- a/core/syncstatus/nodestatus/status_test.go +++ b/core/syncstatus/nodestatus/status_test.go @@ -3,79 +3,13 @@ package nodestatus import ( "testing" - "github.com/anyproto/any-sync/app" - "github.com/anyproto/any-sync/nodeconf/mock_nodeconf" - "github.com/stretchr/testify/assert" - "go.uber.org/mock/gomock" + "github.com/stretchr/testify/require" ) -type fixture struct { - *nodeStatus - nodeConf *mock_nodeconf.MockService -} - -func TestNodeStatus_SetNodesStatus(t *testing.T) { - t.Run("peer is responsible", func(t *testing.T) { - // given - f := newFixture(t) - f.nodeConf.EXPECT().NodeIds("spaceId").Return([]string{"peerId"}) - - // when - f.SetNodesStatus("spaceId", "peerId", Online) - - // then - assert.Equal(t, Online, f.nodeStatus.nodeStatus["spaceId"]) - }) - t.Run("peer is not responsible", func(t *testing.T) { - // given - f := newFixture(t) - f.nodeConf.EXPECT().NodeIds("spaceId").Return([]string{"peerId2"}) - - // when - f.SetNodesStatus("spaceId", "peerId", ConnectionError) - - // then - assert.NotEqual(t, ConnectionError, f.nodeStatus.nodeStatus["spaceId"]) - }) -} - -func TestNodeStatus_GetNodeStatus(t *testing.T) { - t.Run("get default status", func(t *testing.T) { - // given - f := newFixture(t) - - // when - status := f.GetNodeStatus("") - - // then - assert.Equal(t, Online, status) - }) - t.Run("get updated status", func(t *testing.T) { - // given - f := newFixture(t) - f.nodeConf.EXPECT().NodeIds("spaceId").Return([]string{"peerId"}) - - // when - f.SetNodesStatus("spaceId", "peerId", ConnectionError) - status := f.GetNodeStatus("spaceId") - - // then - assert.Equal(t, ConnectionError, status) - }) -} - -func newFixture(t *testing.T) *fixture { - ctrl := gomock.NewController(t) - nodeConf := mock_nodeconf.NewMockService(ctrl) - nodeStatus := &nodeStatus{ - nodeStatus: map[string]ConnectionStatus{}, - } - a := &app.App{} - a.Register(nodeConf) - err := nodeStatus.Init(a) - assert.Nil(t, err) - return &fixture{ - nodeStatus: nodeStatus, - nodeConf: nodeConf, - } +func TestNodeStatus(t *testing.T) { + st := NewNodeStatus() + st.SetNodesStatus("spaceId", Online) + require.Equal(t, Online, st.GetNodeStatus("spaceId")) + st.SetNodesStatus("spaceId", ConnectionError) + require.Equal(t, ConnectionError, st.GetNodeStatus("spaceId")) } diff --git a/core/syncstatus/spacesyncstatus/spacestatus.go b/core/syncstatus/spacesyncstatus/spacestatus.go index 42febcede..b84f9d317 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus.go +++ b/core/syncstatus/spacesyncstatus/spacestatus.go @@ -2,7 +2,6 @@ package spacesyncstatus import ( "context" - "fmt" "sync" "time" @@ -10,10 +9,8 @@ import ( "github.com/anyproto/any-sync/app/logger" "github.com/anyproto/any-sync/nodeconf" "github.com/anyproto/any-sync/util/periodicsync" - "github.com/cheggaaa/mb/v3" "github.com/samber/lo" - "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/event" "github.com/anyproto/anytype-heart/core/files" "github.com/anyproto/anytype-heart/core/session" @@ -47,15 +44,6 @@ type SpaceIdGetter interface { AllSpaceIds() []string } -type State interface { - SetObjectsNumber(status *domain.SpaceSync) - SetSyncStatusAndErr(status domain.SpaceSyncStatus, syncError domain.SyncError, spaceId string) - GetSyncStatus(spaceId string) domain.SpaceSyncStatus - GetSyncObjectCount(spaceId string) int - GetSyncErr(spaceId string) domain.SyncError - ResetSpaceErrorStatus(spaceId string, syncError domain.SyncError) -} - type NetworkConfig interface { app.Component GetNetworkMode() pb.RpcAccountNetworkMode @@ -68,29 +56,24 @@ type spaceSyncStatus struct { nodeConf nodeconf.Service nodeUsage NodeUsage store objectstore.ObjectStore - batcher *mb.MB[*domain.SpaceSync] subscriptionService subscription.Service subs syncsubscritions.SyncSubscriptions - filesState State - objectsState State - - ctx context.Context - ctxCancel context.CancelFunc - spaceIdGetter SpaceIdGetter - curStatuses map[string]struct{} - missingIds map[string][]string - mx sync.Mutex - periodicCall periodicsync.PeriodicSync - loopInterval time.Duration - isLocal bool - finish chan struct{} + ctx context.Context + ctxCancel context.CancelFunc + spaceIdGetter SpaceIdGetter + curStatuses map[string]struct{} + missingIds map[string][]string + lastSentEvents map[string]pb.EventSpaceSyncStatusUpdate + mx sync.Mutex + periodicCall periodicsync.PeriodicSync + loopInterval time.Duration + isLocal bool + finish chan struct{} } func NewSpaceSyncStatus() Updater { return &spaceSyncStatus{ - batcher: mb.New[*domain.SpaceSync](0), - finish: make(chan struct{}), loopInterval: time.Second * 1, } } @@ -106,6 +89,7 @@ func (s *spaceSyncStatus) Init(a *app.App) (err error) { s.curStatuses = make(map[string]struct{}) s.subs = app.MustComponent[syncsubscritions.SyncSubscriptions](a) s.missingIds = make(map[string][]string) + s.lastSentEvents = make(map[string]pb.EventSpaceSyncStatusUpdate) s.spaceIdGetter = app.MustComponent[SpaceIdGetter](a) s.isLocal = s.networkConfig.GetNetworkMode() == pb.RpcAccount_LocalOnly sessionHookRunner := app.MustComponent[session.HookRunner](a) @@ -191,7 +175,7 @@ func (s *spaceSyncStatus) sendStartEvent(spaceIds []string) { } func (s *spaceSyncStatus) sendLocalOnlyEvent(spaceId string) { - s.eventSender.Broadcast(&pb.Event{ + s.broadcast(&pb.Event{ Messages: []*pb.EventMessage{{ Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ @@ -204,6 +188,26 @@ func (s *spaceSyncStatus) sendLocalOnlyEvent(spaceId string) { }) } +func eventsEqual(a, b pb.EventSpaceSyncStatusUpdate) bool { + return a.Id == b.Id && + a.Status == b.Status && + a.Network == b.Network && + a.Error == b.Error && + a.SyncingObjectsCounter == b.SyncingObjectsCounter +} + +func (s *spaceSyncStatus) broadcast(event *pb.Event) { + s.mx.Lock() + val := *event.Messages[0].Value.(*pb.EventMessageValueOfSpaceSyncStatusUpdate).SpaceSyncStatusUpdate + ev, ok := s.lastSentEvents[val.Id] + s.lastSentEvents[val.Id] = val + s.mx.Unlock() + if ok && eventsEqual(ev, val) { + return + } + s.eventSender.Broadcast(event) +} + func (s *spaceSyncStatus) sendLocalOnlyEventToSession(spaceId, token string) { s.eventSender.SendToSession(token, &pb.Event{ Messages: []*pb.EventMessage{{ @@ -264,7 +268,7 @@ func (s *spaceSyncStatus) updateSpaceSyncStatus(spaceId string) { filesSyncingCount: s.getFileSyncingObjectsCount(spaceId), objectsSyncingCount: s.getObjectSyncingObjectsCount(spaceId, missingObjects), } - s.eventSender.Broadcast(&pb.Event{ + s.broadcast(&pb.Event{ Messages: []*pb.EventMessage{{ Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ SpaceSyncStatusUpdate: s.makeSyncEvent(spaceId, params), @@ -305,7 +309,6 @@ func (s *spaceSyncStatus) makeSyncEvent(spaceId string, params syncParams) *pb.E status = pb.EventSpace_Error err = pb.EventSpace_IncompatibleVersion } - fmt.Println("[x]: status: connection", params.connectionStatus, ", space id", spaceId, ", compatibility", params.compatibility, ", object number", syncingObjectsCount, ", bytes left", params.bytesLeftPercentage) return &pb.EventSpaceSyncStatusUpdate{ Id: spaceId, Status: status, diff --git a/core/syncstatus/spacesyncstatus/spacestatus_test.go b/core/syncstatus/spacesyncstatus/spacestatus_test.go index 982a5de72..253eb3073 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus_test.go +++ b/core/syncstatus/spacesyncstatus/spacestatus_test.go @@ -220,6 +220,40 @@ func Test(t *testing.T) { }) defer fx.ctrl.Finish() }) + t.Run("objects syncing, not sending same event", func(t *testing.T) { + fx := newFixture(t, func(fx *fixture) { + fx.spaceSyncStatus.loopInterval = 10 * time.Millisecond + objs := genSyncingObjects(10, 100, "spaceId") + fx.objectStore.AddObjects(t, objs) + fx.spaceIdGetter.EXPECT().TechSpaceId().Return("techSpaceId") + fx.networkConfig.EXPECT().GetNetworkMode().Return(pb.RpcAccount_DefaultConfig) + fx.spaceIdGetter.EXPECT().AllSpaceIds().Return([]string{"spaceId"}) + fx.nodeStatus.EXPECT().GetNodeStatus("spaceId").Return(nodestatus.Online) + fx.nodeUsage.EXPECT().GetNodeUsage(mock.Anything).Return(&files.NodeUsageResponse{ + Usage: filesync.NodeUsage{ + BytesLeft: 1000, + AccountBytesLimit: 1000, + }, + LocalUsageBytes: 0, + }, nil) + fx.nodeConf.EXPECT().NetworkCompatibilityStatus().AnyTimes().Return(nodeconf.NetworkCompatibilityStatusOk) + fx.eventSender.EXPECT().Broadcast(&pb.Event{ + Messages: []*pb.EventMessage{{ + Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ + SpaceSyncStatusUpdate: &pb.EventSpaceSyncStatusUpdate{ + Id: "spaceId", + SyncingObjectsCounter: 110, + Status: pb.EventSpace_Syncing, + Network: pb.EventSpace_Anytype, + }, + }, + }}, + }).Times(1) + }) + fx.Refresh("spaceId") + time.Sleep(100 * time.Millisecond) + defer fx.ctrl.Finish() + }) t.Run("local only", func(t *testing.T) { fx := newFixture(t, func(fx *fixture) { objs := genSyncingObjects(10, 100, "spaceId") diff --git a/space/spacecore/peermanager/manager.go b/space/spacecore/peermanager/manager.go index 9a00654ae..b68ee8afd 100644 --- a/space/spacecore/peermanager/manager.go +++ b/space/spacecore/peermanager/manager.go @@ -28,7 +28,7 @@ var ( type NodeStatus interface { app.Component - SetNodesStatus(spaceId string, senderId string, status nodestatus.ConnectionStatus) + SetNodesStatus(spaceId string, status nodestatus.ConnectionStatus) GetNodeStatus(string) nodestatus.ConnectionStatus } @@ -202,12 +202,10 @@ func (n *clientPeerManager) fetchResponsiblePeers() { p, err := n.p.pool.GetOneOf(n.ctx, n.responsibleNodeIds) if err == nil { peers = []peer.Peer{p} - n.nodeStatus.SetNodesStatus(n.spaceId, p.Id(), nodestatus.Online) + n.nodeStatus.SetNodesStatus(n.spaceId, nodestatus.Online) } else { log.Info("can't get node peers", zap.Error(err)) - for _, p := range n.responsiblePeers { - n.nodeStatus.SetNodesStatus(n.spaceId, p.Id(), nodestatus.ConnectionError) - } + n.nodeStatus.SetNodesStatus(n.spaceId, nodestatus.ConnectionError) } n.spaceSyncService.Refresh(n.spaceId) peerIds := n.peerStore.LocalPeerIds(n.spaceId) @@ -263,7 +261,3 @@ func (n *clientPeerManager) Close(ctx context.Context) (err error) { n.peerToPeerStatus.UnregisterSpace(n.spaceId) return } - -func (n *clientPeerManager) IsPeerOffline(senderId string) bool { - return n.nodeStatus.GetNodeStatus(n.spaceId) != nodestatus.Online -} diff --git a/space/spacecore/peermanager/manager_test.go b/space/spacecore/peermanager/manager_test.go index 3dd3cabb6..d44aa1482 100644 --- a/space/spacecore/peermanager/manager_test.go +++ b/space/spacecore/peermanager/manager_test.go @@ -17,7 +17,6 @@ import ( "go.uber.org/mock/gomock" "storj.io/drpc" - "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" "github.com/anyproto/anytype-heart/space/spacecore/peermanager/mock_peermanager" "github.com/anyproto/anytype-heart/space/spacecore/peerstore" @@ -92,26 +91,18 @@ func TestClientPeerManager_GetResponsiblePeers_Deadline(t *testing.T) { func Test_fetchResponsiblePeers(t *testing.T) { spaceId := "spaceId" t.Run("node offline", func(t *testing.T) { - // given f := newFixtureManager(t, spaceId) - - // when f.pool.EXPECT().GetOneOf(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("failed")) - status := domain.MakeSyncStatus(f.cm.spaceId, domain.SpaceSyncStatusOffline, domain.SyncErrorNull) - f.updater.EXPECT().SendUpdate(status) + f.updater.EXPECT().Refresh(spaceId) f.cm.fetchResponsiblePeers() - - // then - f.updater.AssertCalled(t, "SendUpdate", status) + require.Equal(t, f.cm.nodeStatus.GetNodeStatus("spaceId"), nodestatus.ConnectionError) }) t.Run("no local peers", func(t *testing.T) { - // given f := newFixtureManager(t, spaceId) - - // when - f.conf.EXPECT().NodeIds(f.cm.spaceId).Return([]string{"id"}) f.pool.EXPECT().GetOneOf(gomock.Any(), gomock.Any()).Return(newTestPeer("id"), nil) + f.updater.EXPECT().Refresh(spaceId) f.cm.fetchResponsiblePeers() + require.Equal(t, f.cm.nodeStatus.GetNodeStatus("spaceId"), nodestatus.Online) }) t.Run("local peers connected", func(t *testing.T) { // given @@ -119,9 +110,9 @@ func Test_fetchResponsiblePeers(t *testing.T) { f.store.UpdateLocalPeer("peerId", []string{spaceId}) // when - f.conf.EXPECT().NodeIds(f.cm.spaceId).Return([]string{"id"}) f.pool.EXPECT().GetOneOf(gomock.Any(), gomock.Any()).Return(newTestPeer("id"), nil) f.pool.EXPECT().Get(f.cm.ctx, "peerId").Return(newTestPeer("id1"), nil) + f.updater.EXPECT().Refresh(spaceId) f.cm.fetchResponsiblePeers() }) @@ -131,9 +122,9 @@ func Test_fetchResponsiblePeers(t *testing.T) { f.store.UpdateLocalPeer("peerId", []string{spaceId}) // when - f.conf.EXPECT().NodeIds(f.cm.spaceId).Return([]string{"id"}) f.pool.EXPECT().GetOneOf(gomock.Any(), gomock.Any()).Return(newTestPeer("id"), nil) f.pool.EXPECT().Get(f.cm.ctx, "peerId").Return(nil, fmt.Errorf("error")) + f.updater.EXPECT().Refresh(spaceId) f.cm.fetchResponsiblePeers() }) } @@ -284,14 +275,15 @@ func newFixtureManager(t *testing.T, spaceId string) *fixture { updater := mock_peermanager.NewMockUpdater(t) peerToPeerStatus := mock_peermanager.NewMockPeerToPeerStatus(t) cm := &clientPeerManager{ - p: provider, - spaceId: spaceId, - peerStore: store, - watchingPeers: map[string]struct{}{}, - ctx: context.Background(), - nodeStatus: ns, - spaceSyncService: updater, - peerToPeerStatus: peerToPeerStatus, + responsibleNodeIds: []string{"nodeId"}, + p: provider, + spaceId: spaceId, + peerStore: store, + watchingPeers: map[string]struct{}{}, + ctx: context.Background(), + nodeStatus: ns, + spaceSyncService: updater, + peerToPeerStatus: peerToPeerStatus, } return &fixture{ cm: cm, diff --git a/space/spacecore/peermanager/mock_peermanager/mock_PeerToPeerStatus.go b/space/spacecore/peermanager/mock_peermanager/mock_PeerToPeerStatus.go index 4fb87b006..eaddd36b6 100644 --- a/space/spacecore/peermanager/mock_peermanager/mock_PeerToPeerStatus.go +++ b/space/spacecore/peermanager/mock_peermanager/mock_PeerToPeerStatus.go @@ -17,38 +17,6 @@ func (_m *MockPeerToPeerStatus) EXPECT() *MockPeerToPeerStatus_Expecter { return &MockPeerToPeerStatus_Expecter{mock: &_m.Mock} } -// CheckPeerStatus provides a mock function with given fields: -func (_m *MockPeerToPeerStatus) CheckPeerStatus() { - _m.Called() -} - -// MockPeerToPeerStatus_CheckPeerStatus_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CheckPeerStatus' -type MockPeerToPeerStatus_CheckPeerStatus_Call struct { - *mock.Call -} - -// CheckPeerStatus is a helper method to define mock.On call -func (_e *MockPeerToPeerStatus_Expecter) CheckPeerStatus() *MockPeerToPeerStatus_CheckPeerStatus_Call { - return &MockPeerToPeerStatus_CheckPeerStatus_Call{Call: _e.mock.On("CheckPeerStatus")} -} - -func (_c *MockPeerToPeerStatus_CheckPeerStatus_Call) Run(run func()) *MockPeerToPeerStatus_CheckPeerStatus_Call { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *MockPeerToPeerStatus_CheckPeerStatus_Call) Return() *MockPeerToPeerStatus_CheckPeerStatus_Call { - _c.Call.Return() - return _c -} - -func (_c *MockPeerToPeerStatus_CheckPeerStatus_Call) RunAndReturn(run func()) *MockPeerToPeerStatus_CheckPeerStatus_Call { - _c.Call.Return(run) - return _c -} - // RegisterSpace provides a mock function with given fields: spaceId func (_m *MockPeerToPeerStatus) RegisterSpace(spaceId string) { _m.Called(spaceId) From 02c3fb65b0264372242b5fac167b74a507881762 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 25 Jul 2024 14:46:00 +0200 Subject: [PATCH 61/71] GO-3769 Fix treesyncer --- core/block/object/treesyncer/treesyncer.go | 7 ------- core/block/object/treesyncer/treesyncer_test.go | 5 ----- 2 files changed, 12 deletions(-) diff --git a/core/block/object/treesyncer/treesyncer.go b/core/block/object/treesyncer/treesyncer.go index a56b18365..56c1a2aca 100644 --- a/core/block/object/treesyncer/treesyncer.go +++ b/core/block/object/treesyncer/treesyncer.go @@ -60,11 +60,6 @@ type SyncedTreeRemover interface { RemoveAllExcept(senderId string, differentRemoteIds []string) } -type PeerStatusChecker interface { - app.Component - IsPeerOffline(peerId string) bool -} - type SyncDetailsUpdater interface { app.Component UpdateSpaceDetails(existing, missing []string, spaceId string) @@ -82,7 +77,6 @@ type treeSyncer struct { treeManager treemanager.TreeManager isRunning bool isSyncing bool - peerManager PeerStatusChecker nodeConf nodeconf.NodeConf syncedTreeRemover SyncedTreeRemover syncDetailsUpdater SyncDetailsUpdater @@ -104,7 +98,6 @@ func NewTreeSyncer(spaceId string) treesyncer.TreeSyncer { func (t *treeSyncer) Init(a *app.App) (err error) { t.isSyncing = true t.treeManager = app.MustComponent[treemanager.TreeManager](a) - t.peerManager = app.MustComponent[PeerStatusChecker](a) t.nodeConf = app.MustComponent[nodeconf.NodeConf](a) t.syncedTreeRemover = app.MustComponent[SyncedTreeRemover](a) t.syncDetailsUpdater = app.MustComponent[SyncDetailsUpdater](a) diff --git a/core/block/object/treesyncer/treesyncer_test.go b/core/block/object/treesyncer/treesyncer_test.go index 04be6fdf7..09ed6a032 100644 --- a/core/block/object/treesyncer/treesyncer_test.go +++ b/core/block/object/treesyncer/treesyncer_test.go @@ -25,7 +25,6 @@ type fixture struct { missingMock *mock_objecttree.MockObjectTree existingMock *mock_synctree.MockSyncTree treeManager *mock_treemanager.MockTreeManager - checker *mock_treesyncer.MockPeerStatusChecker nodeConf *mock_nodeconf.MockService syncStatus *mock_treesyncer.MockSyncedTreeRemover syncDetailsUpdater *mock_treesyncer.MockSyncDetailsUpdater @@ -36,8 +35,6 @@ func newFixture(t *testing.T, spaceId string) *fixture { treeManager := mock_treemanager.NewMockTreeManager(ctrl) missingMock := mock_objecttree.NewMockObjectTree(ctrl) existingMock := mock_synctree.NewMockSyncTree(ctrl) - checker := mock_treesyncer.NewMockPeerStatusChecker(t) - checker.EXPECT().Name().Return("checker").Maybe() nodeConf := mock_nodeconf.NewMockService(ctrl) nodeConf.EXPECT().Name().Return("nodeConf").AnyTimes() syncStatus := mock_treesyncer.NewMockSyncedTreeRemover(t) @@ -45,7 +42,6 @@ func newFixture(t *testing.T, spaceId string) *fixture { a := new(app.App) a.Register(testutil.PrepareMock(context.Background(), a, treeManager)). - Register(testutil.PrepareMock(context.Background(), a, checker)). Register(testutil.PrepareMock(context.Background(), a, syncStatus)). Register(testutil.PrepareMock(context.Background(), a, nodeConf)). Register(testutil.PrepareMock(context.Background(), a, syncDetailsUpdater)) @@ -58,7 +54,6 @@ func newFixture(t *testing.T, spaceId string) *fixture { missingMock: missingMock, existingMock: existingMock, treeManager: treeManager, - checker: checker, nodeConf: nodeConf, syncStatus: syncStatus, syncDetailsUpdater: syncDetailsUpdater, From e1b0fa3e15c95f0e25518428ae7e170fdf68440b Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 25 Jul 2024 15:23:47 +0200 Subject: [PATCH 62/71] GO-3769 Fix lint --- core/anytype/bootstrap.go | 4 ++-- core/syncstatus/detailsupdater/updater.go | 6 +++--- core/syncstatus/detailsupdater/updater_test.go | 4 ++-- core/syncstatus/objectsyncstatus/syncstatus.go | 8 ++++---- core/syncstatus/spacesyncstatus/spacestatus.go | 6 +++--- .../spacesyncstatus/spacestatus_test.go | 6 +++--- .../objectsubscription.go | 16 +++++++--------- .../objectsubscription_test.go | 2 +- .../syncingobjects.go | 4 ++-- .../syncingobjects_test.go | 8 ++++---- .../syncsubscriptions.go | 4 ++-- .../syncsubscriptions_test.go | 2 +- 12 files changed, 34 insertions(+), 36 deletions(-) rename core/syncstatus/{syncsubscritions => syncsubscriptions}/objectsubscription.go (91%) rename core/syncstatus/{syncsubscritions => syncsubscriptions}/objectsubscription_test.go (99%) rename core/syncstatus/{syncsubscritions => syncsubscriptions}/syncingobjects.go (97%) rename core/syncstatus/{syncsubscritions => syncsubscriptions}/syncingobjects_test.go (76%) rename core/syncstatus/{syncsubscritions => syncsubscriptions}/syncsubscriptions.go (95%) rename core/syncstatus/{syncsubscritions => syncsubscriptions}/syncsubscriptions_test.go (99%) diff --git a/core/anytype/bootstrap.go b/core/anytype/bootstrap.go index c4d44c67b..8fdd9f761 100644 --- a/core/anytype/bootstrap.go +++ b/core/anytype/bootstrap.go @@ -81,7 +81,7 @@ import ( "github.com/anyproto/anytype-heart/core/syncstatus/detailsupdater" "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" "github.com/anyproto/anytype-heart/core/syncstatus/spacesyncstatus" - "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscritions" + "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscriptions" "github.com/anyproto/anytype-heart/core/wallet" "github.com/anyproto/anytype-heart/metrics" "github.com/anyproto/anytype-heart/pkg/lib/core" @@ -278,7 +278,7 @@ func Bootstrap(a *app.App, components ...app.Component) { Register(debug.New()). Register(collection.New()). Register(subscription.New()). - Register(syncsubscritions.New()). + Register(syncsubscriptions.New()). Register(builtinobjects.New()). Register(bookmark.New()). Register(importer.New()). diff --git a/core/syncstatus/detailsupdater/updater.go b/core/syncstatus/detailsupdater/updater.go index 38148b265..385fab540 100644 --- a/core/syncstatus/detailsupdater/updater.go +++ b/core/syncstatus/detailsupdater/updater.go @@ -16,7 +16,7 @@ import ( "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/syncstatus/detailsupdater/helper" "github.com/anyproto/anytype-heart/core/syncstatus/filesyncstatus" - "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscritions" + "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscriptions" "github.com/anyproto/anytype-heart/pkg/lib/bundle" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/logging" @@ -55,7 +55,7 @@ type syncStatusUpdater struct { batcher *mb.MB[string] spaceService space.Service spaceSyncStatus SpaceStatusUpdater - syncSubscriptions syncsubscritions.SyncSubscriptions + syncSubscriptions syncsubscriptions.SyncSubscriptions entries map[string]*syncStatusDetails mx sync.Mutex @@ -89,7 +89,7 @@ func (u *syncStatusUpdater) Init(a *app.App) (err error) { u.objectStore = app.MustComponent[objectstore.ObjectStore](a) u.spaceService = app.MustComponent[space.Service](a) u.spaceSyncStatus = app.MustComponent[SpaceStatusUpdater](a) - u.syncSubscriptions = app.MustComponent[syncsubscritions.SyncSubscriptions](a) + u.syncSubscriptions = app.MustComponent[syncsubscriptions.SyncSubscriptions](a) return nil } diff --git a/core/syncstatus/detailsupdater/updater_test.go b/core/syncstatus/detailsupdater/updater_test.go index f2b9392aa..9becf1d7b 100644 --- a/core/syncstatus/detailsupdater/updater_test.go +++ b/core/syncstatus/detailsupdater/updater_test.go @@ -20,7 +20,7 @@ import ( "github.com/anyproto/anytype-heart/core/subscription" "github.com/anyproto/anytype-heart/core/syncstatus/detailsupdater/mock_detailsupdater" "github.com/anyproto/anytype-heart/core/syncstatus/filesyncstatus" - "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscritions" + "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscriptions" "github.com/anyproto/anytype-heart/pkg/lib/bundle" coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" @@ -352,7 +352,7 @@ func newFixture(t *testing.T) *fixture { updater := New() statusUpdater := mock_detailsupdater.NewMockSpaceStatusUpdater(t) - syncSub := syncsubscritions.New() + syncSub := syncsubscriptions.New() ctx := context.Background() diff --git a/core/syncstatus/objectsyncstatus/syncstatus.go b/core/syncstatus/objectsyncstatus/syncstatus.go index 559acf89c..32a96ae84 100644 --- a/core/syncstatus/objectsyncstatus/syncstatus.go +++ b/core/syncstatus/objectsyncstatus/syncstatus.go @@ -186,11 +186,11 @@ func (s *syncStatusService) HeadsApply(senderId, treeId string, heads []string, } func (s *syncStatusService) update(ctx context.Context) (err error) { - var ( - updateDetailsStatuses []treeStatus - updateThreadStatuses []treeStatus - ) s.Lock() + var ( + updateDetailsStatuses = make([]treeStatus, 0, len(s.synced)) + updateThreadStatuses = make([]treeStatus, 0, len(s.watchers)) + ) if s.updateReceiver == nil { s.Unlock() return diff --git a/core/syncstatus/spacesyncstatus/spacestatus.go b/core/syncstatus/spacesyncstatus/spacestatus.go index b84f9d317..041ee1740 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus.go +++ b/core/syncstatus/spacesyncstatus/spacestatus.go @@ -16,7 +16,7 @@ import ( "github.com/anyproto/anytype-heart/core/session" "github.com/anyproto/anytype-heart/core/subscription" "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" - "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscritions" + "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscriptions" "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/logging" @@ -57,7 +57,7 @@ type spaceSyncStatus struct { nodeUsage NodeUsage store objectstore.ObjectStore subscriptionService subscription.Service - subs syncsubscritions.SyncSubscriptions + subs syncsubscriptions.SyncSubscriptions ctx context.Context ctxCancel context.CancelFunc @@ -87,7 +87,7 @@ func (s *spaceSyncStatus) Init(a *app.App) (err error) { s.subscriptionService = app.MustComponent[subscription.Service](a) s.store = app.MustComponent[objectstore.ObjectStore](a) s.curStatuses = make(map[string]struct{}) - s.subs = app.MustComponent[syncsubscritions.SyncSubscriptions](a) + s.subs = app.MustComponent[syncsubscriptions.SyncSubscriptions](a) s.missingIds = make(map[string][]string) s.lastSentEvents = make(map[string]pb.EventSpaceSyncStatusUpdate) s.spaceIdGetter = app.MustComponent[SpaceIdGetter](a) diff --git a/core/syncstatus/spacesyncstatus/spacestatus_test.go b/core/syncstatus/spacesyncstatus/spacestatus_test.go index 253eb3073..0eab6e966 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus_test.go +++ b/core/syncstatus/spacesyncstatus/spacestatus_test.go @@ -25,7 +25,7 @@ import ( "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus/mock_nodestatus" "github.com/anyproto/anytype-heart/core/syncstatus/spacesyncstatus/mock_spacesyncstatus" - "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscritions" + "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscriptions" "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/bundle" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" @@ -71,7 +71,7 @@ type fixture struct { nodeUsage *mock_spacesyncstatus.MockNodeUsage nodeStatus *mock_nodestatus.MockNodeStatus subscriptionService subscription.Service - syncSubs syncsubscritions.SyncSubscriptions + syncSubs syncsubscriptions.SyncSubscriptions objectStore *objectstore.StoreFixture spaceIdGetter *mock_spacesyncstatus.MockSpaceIdGetter eventSender *mock_event.MockSender @@ -145,7 +145,7 @@ func newFixture(t *testing.T, beforeStart func(fx *fixture)) *fixture { eventSender: app.MustComponent[event.Sender](a).(*mock_event.MockSender), subscriptionService: internalSubs, session: sess, - syncSubs: syncsubscritions.New(), + syncSubs: syncsubscriptions.New(), networkConfig: networkConfig, } a.Register(fx.syncSubs). diff --git a/core/syncstatus/syncsubscritions/objectsubscription.go b/core/syncstatus/syncsubscriptions/objectsubscription.go similarity index 91% rename from core/syncstatus/syncsubscritions/objectsubscription.go rename to core/syncstatus/syncsubscriptions/objectsubscription.go index ab823ad89..cd0942f19 100644 --- a/core/syncstatus/syncsubscritions/objectsubscription.go +++ b/core/syncstatus/syncsubscriptions/objectsubscription.go @@ -1,4 +1,4 @@ -package syncsubscritions +package syncsubscriptions import ( "context" @@ -14,16 +14,15 @@ import ( ) type entry[T any] struct { - id string data T } -func newEmptyEntry[T any](id string) *entry[T] { - return &entry[T]{id: id} +func newEmptyEntry[T any]() *entry[T] { + return &entry[T]{} } -func newEntry[T any](id string, data T) *entry[T] { - return &entry[T]{id: id, data: data} +func newEntry[T any](data T) *entry[T] { + return &entry[T]{data: data} } type ( @@ -91,7 +90,7 @@ func (o *ObjectSubscription[T]) Run() error { o.sub = map[string]*entry[T]{} for _, rec := range resp.Records { id, data := o.extract(rec) - o.sub[id] = newEntry(id, data) + o.sub[id] = newEntry(data) } go o.read() return nil @@ -100,7 +99,6 @@ func (o *ObjectSubscription[T]) Run() error { func (o *ObjectSubscription[T]) Close() { o.cancel() <-o.ch - return } func (o *ObjectSubscription[T]) Len() int { @@ -127,7 +125,7 @@ func (o *ObjectSubscription[T]) read() { defer o.mx.Unlock() switch v := event.Value.(type) { case *pb.EventMessageValueOfSubscriptionAdd: - o.sub[v.SubscriptionAdd.Id] = newEmptyEntry[T](v.SubscriptionAdd.Id) + o.sub[v.SubscriptionAdd.Id] = newEmptyEntry[T]() case *pb.EventMessageValueOfSubscriptionRemove: delete(o.sub, v.SubscriptionRemove.Id) case *pb.EventMessageValueOfObjectDetailsAmend: diff --git a/core/syncstatus/syncsubscritions/objectsubscription_test.go b/core/syncstatus/syncsubscriptions/objectsubscription_test.go similarity index 99% rename from core/syncstatus/syncsubscritions/objectsubscription_test.go rename to core/syncstatus/syncsubscriptions/objectsubscription_test.go index 739875015..8dabe872b 100644 --- a/core/syncstatus/syncsubscritions/objectsubscription_test.go +++ b/core/syncstatus/syncsubscriptions/objectsubscription_test.go @@ -1,4 +1,4 @@ -package syncsubscritions +package syncsubscriptions import ( "context" diff --git a/core/syncstatus/syncsubscritions/syncingobjects.go b/core/syncstatus/syncsubscriptions/syncingobjects.go similarity index 97% rename from core/syncstatus/syncsubscritions/syncingobjects.go rename to core/syncstatus/syncsubscriptions/syncingobjects.go index 7e5ee2e64..c889de317 100644 --- a/core/syncstatus/syncsubscritions/syncingobjects.go +++ b/core/syncstatus/syncsubscriptions/syncingobjects.go @@ -1,4 +1,4 @@ -package syncsubscritions +package syncsubscriptions import ( "fmt" @@ -78,7 +78,7 @@ func (s *syncingObjects) Run() error { errFiles := s.fileSubscription.Run() errObjects := s.objectSubscription.Run() if errFiles != nil || errObjects != nil { - return fmt.Errorf("error running syncing objects: %v %v", errFiles, errObjects) + return fmt.Errorf("error running syncing objects: %w %w", errFiles, errObjects) } return nil } diff --git a/core/syncstatus/syncsubscritions/syncingobjects_test.go b/core/syncstatus/syncsubscriptions/syncingobjects_test.go similarity index 76% rename from core/syncstatus/syncsubscritions/syncingobjects_test.go rename to core/syncstatus/syncsubscriptions/syncingobjects_test.go index 8866a0a2b..a7b599140 100644 --- a/core/syncstatus/syncsubscritions/syncingobjects_test.go +++ b/core/syncstatus/syncsubscriptions/syncingobjects_test.go @@ -1,4 +1,4 @@ -package syncsubscritions +package syncsubscriptions import ( "testing" @@ -11,9 +11,9 @@ import ( func TestCount(t *testing.T) { objSubscription := NewIdSubscription(nil, subscription.SubscribeRequest{}) objSubscription.sub = map[string]*entry[struct{}]{ - "1": newEmptyEntry[struct{}]("1"), - "2": newEmptyEntry[struct{}]("2"), - "4": newEmptyEntry[struct{}]("4"), + "1": newEmptyEntry[struct{}](), + "2": newEmptyEntry[struct{}](), + "4": newEmptyEntry[struct{}](), } syncing := &syncingObjects{ objectSubscription: objSubscription, diff --git a/core/syncstatus/syncsubscritions/syncsubscriptions.go b/core/syncstatus/syncsubscriptions/syncsubscriptions.go similarity index 95% rename from core/syncstatus/syncsubscritions/syncsubscriptions.go rename to core/syncstatus/syncsubscriptions/syncsubscriptions.go index 65fec81ea..90408be63 100644 --- a/core/syncstatus/syncsubscritions/syncsubscriptions.go +++ b/core/syncstatus/syncsubscriptions/syncsubscriptions.go @@ -1,4 +1,4 @@ -package syncsubscritions +package syncsubscriptions import ( "context" @@ -10,7 +10,7 @@ import ( "github.com/anyproto/anytype-heart/core/subscription" ) -const CName = "client.syncstatus.syncsubscritions" +const CName = "client.syncstatus.syncsubscriptions" type SyncSubscription interface { Run() error diff --git a/core/syncstatus/syncsubscritions/syncsubscriptions_test.go b/core/syncstatus/syncsubscriptions/syncsubscriptions_test.go similarity index 99% rename from core/syncstatus/syncsubscritions/syncsubscriptions_test.go rename to core/syncstatus/syncsubscriptions/syncsubscriptions_test.go index 73f8a232b..b989c6f97 100644 --- a/core/syncstatus/syncsubscritions/syncsubscriptions_test.go +++ b/core/syncstatus/syncsubscriptions/syncsubscriptions_test.go @@ -1,4 +1,4 @@ -package syncsubscritions +package syncsubscriptions import ( "context" From 5672ba8f86c88c5674f7ce34836225260c42d6c9 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 25 Jul 2024 16:09:31 +0200 Subject: [PATCH 63/71] GO-3769 Fix lint --- .golangci.yml | 1 + .../syncstatus/objectsyncstatus/syncstatus.go | 8 +++---- .../syncstatus/spacesyncstatus/spacestatus.go | 22 +++++-------------- .../syncsubscriptions/objectsubscription.go | 1 - 4 files changed, 11 insertions(+), 21 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 8004986a4..e1ae4a969 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -13,6 +13,7 @@ issues: - pb exclude-files: - '.*_test.go' + - 'mock*' - 'testMock/*' - 'clientlibrary/service/service.pb.go' diff --git a/core/syncstatus/objectsyncstatus/syncstatus.go b/core/syncstatus/objectsyncstatus/syncstatus.go index 32a96ae84..d88cba729 100644 --- a/core/syncstatus/objectsyncstatus/syncstatus.go +++ b/core/syncstatus/objectsyncstatus/syncstatus.go @@ -174,10 +174,9 @@ func (s *syncStatusService) HeadsApply(senderId, treeId string, heads []string, // checking if we received the head that we are interested in for _, head := range heads { if idx, found := slices.BinarySearch(curTreeHeads.heads, head); found { - curTreeHeads.heads[idx] = "" + curTreeHeads.heads = slice.RemoveIndex(curTreeHeads.heads, idx) } } - curTreeHeads.heads = slice.RemoveMut(curTreeHeads.heads, "") if len(curTreeHeads.heads) == 0 { curTreeHeads.syncStatus = StatusSynced } @@ -247,9 +246,10 @@ func (s *syncStatusService) Watch(treeId string) (err error) { if err != nil { return } - slices.Sort(heads) + headsCopy := slice.Copy(heads) + slices.Sort(headsCopy) s.treeHeads[treeId] = treeHeadsEntry{ - heads: heads, + heads: headsCopy, syncStatus: StatusUnknown, } } diff --git a/core/syncstatus/spacesyncstatus/spacestatus.go b/core/syncstatus/spacesyncstatus/spacestatus.go index 041ee1740..7d03db2b9 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus.go +++ b/core/syncstatus/spacesyncstatus/spacestatus.go @@ -14,11 +14,9 @@ import ( "github.com/anyproto/anytype-heart/core/event" "github.com/anyproto/anytype-heart/core/files" "github.com/anyproto/anytype-heart/core/session" - "github.com/anyproto/anytype-heart/core/subscription" "github.com/anyproto/anytype-heart/core/syncstatus/nodestatus" "github.com/anyproto/anytype-heart/core/syncstatus/syncsubscriptions" "github.com/anyproto/anytype-heart/pb" - "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/logging" "github.com/anyproto/anytype-heart/util/slice" ) @@ -50,17 +48,13 @@ type NetworkConfig interface { } type spaceSyncStatus struct { - eventSender event.Sender - networkConfig NetworkConfig - nodeStatus nodestatus.NodeStatus - nodeConf nodeconf.Service - nodeUsage NodeUsage - store objectstore.ObjectStore - subscriptionService subscription.Service - subs syncsubscriptions.SyncSubscriptions + eventSender event.Sender + networkConfig NetworkConfig + nodeStatus nodestatus.NodeStatus + nodeConf nodeconf.Service + nodeUsage NodeUsage + subs syncsubscriptions.SyncSubscriptions - ctx context.Context - ctxCancel context.CancelFunc spaceIdGetter SpaceIdGetter curStatuses map[string]struct{} missingIds map[string][]string @@ -69,7 +63,6 @@ type spaceSyncStatus struct { periodicCall periodicsync.PeriodicSync loopInterval time.Duration isLocal bool - finish chan struct{} } func NewSpaceSyncStatus() Updater { @@ -84,8 +77,6 @@ func (s *spaceSyncStatus) Init(a *app.App) (err error) { s.nodeStatus = app.MustComponent[nodestatus.NodeStatus](a) s.nodeConf = app.MustComponent[nodeconf.Service](a) s.nodeUsage = app.MustComponent[NodeUsage](a) - s.subscriptionService = app.MustComponent[subscription.Service](a) - s.store = app.MustComponent[objectstore.ObjectStore](a) s.curStatuses = make(map[string]struct{}) s.subs = app.MustComponent[syncsubscriptions.SyncSubscriptions](a) s.missingIds = make(map[string][]string) @@ -118,7 +109,6 @@ func (s *spaceSyncStatus) UpdateMissingIds(spaceId string, ids []string) { func (s *spaceSyncStatus) Run(ctx context.Context) (err error) { s.sendStartEvent(s.spaceIdGetter.AllSpaceIds()) - s.ctx, s.ctxCancel = context.WithCancel(context.Background()) s.periodicCall.Run() return } diff --git a/core/syncstatus/syncsubscriptions/objectsubscription.go b/core/syncstatus/syncsubscriptions/objectsubscription.go index cd0942f19..a3d5e4654 100644 --- a/core/syncstatus/syncsubscriptions/objectsubscription.go +++ b/core/syncstatus/syncsubscriptions/objectsubscription.go @@ -115,7 +115,6 @@ func (o *ObjectSubscription[T]) Iterate(iter func(id string, data T) bool) { return } } - return } func (o *ObjectSubscription[T]) read() { From c824adf39a696bce0a558b79b4ebf4a695b78e5a Mon Sep 17 00:00:00 2001 From: Mikhail Iudin Date: Thu, 25 Jul 2024 17:45:43 +0200 Subject: [PATCH 64/71] GO-3815 Change to 600 --- metrics/interceptors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/interceptors.go b/metrics/interceptors.go index ca12ba36c..226f3f38a 100644 --- a/metrics/interceptors.go +++ b/metrics/interceptors.go @@ -100,7 +100,7 @@ func saveAccountStop(event *MethodEvent) error { data := json.MarshalTo(nil) jsonPath := filepath.Join(Service.getWorkingDir(), accountStopJson) _ = os.Remove(jsonPath) - return os.WriteFile(jsonPath, data, 0644) + return os.WriteFile(jsonPath, data, 0600) } func trySendAccountStop() error { From b7e74a86be2d194f139a3ee005cf0d95ca1105f6 Mon Sep 17 00:00:00 2001 From: Sergey Date: Thu, 25 Jul 2024 17:47:33 +0200 Subject: [PATCH 65/71] Fix data races --- core/subscription/context.go | 2 +- core/subscription/service.go | 8 +++--- .../components/spaceloader/loadingspace.go | 28 +++++++++++++++---- .../components/spaceloader/spaceloader.go | 4 +-- 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/core/subscription/context.go b/core/subscription/context.go index 1483a1703..e36f698fd 100644 --- a/core/subscription/context.go +++ b/core/subscription/context.go @@ -350,7 +350,7 @@ func (ctx *opCtx) reset() { ctx.groups = ctx.groups[:0] if ctx.outputs == nil { ctx.outputs = map[string][]*pb.EventMessage{ - defaultOutput: make([]*pb.EventMessage, 0, 10), + defaultOutput: nil, } } } diff --git a/core/subscription/service.go b/core/subscription/service.go index 9b244fd65..cfb037c80 100644 --- a/core/subscription/service.go +++ b/core/subscription/service.go @@ -574,18 +574,18 @@ func (s *service) onChange(entries []*entry) time.Duration { handleTime := time.Since(st) // Reset output buffer - for subId, msgs := range s.ctxBuf.outputs { + for subId := range s.ctxBuf.outputs { if subId == defaultOutput { - s.ctxBuf.outputs[subId] = msgs[:0] + s.ctxBuf.outputs[subId] = nil } else if _, ok := s.customOutput[subId]; ok { - s.ctxBuf.outputs[subId] = msgs[:0] + s.ctxBuf.outputs[subId] = nil } else { delete(s.ctxBuf.outputs, subId) } } for subId := range s.customOutput { if _, ok := s.ctxBuf.outputs[subId]; !ok { - s.ctxBuf.outputs[subId] = make([]*pb.EventMessage, 0, 10) + s.ctxBuf.outputs[subId] = nil } } diff --git a/space/internal/components/spaceloader/loadingspace.go b/space/internal/components/spaceloader/loadingspace.go index 2ad1c0005..a9a803377 100644 --- a/space/internal/components/spaceloader/loadingspace.go +++ b/space/internal/components/spaceloader/loadingspace.go @@ -3,6 +3,7 @@ package spaceloader import ( "context" "errors" + "sync" "time" "github.com/anyproto/any-sync/app/logger" @@ -36,8 +37,11 @@ type loadingSpace struct { disableRemoteLoad bool latestAclHeadId string space clientspace.Space - loadErr error - loadCh chan struct{} + + loadCh chan struct{} + + lock sync.Mutex + loadErr error } func (s *spaceLoader) newLoadingSpace(ctx context.Context, stopIfMandatoryFail, disableRemoteLoad bool, aclHeadId string) *loadingSpace { @@ -53,9 +57,21 @@ func (s *spaceLoader) newLoadingSpace(ctx context.Context, stopIfMandatoryFail, return ls } +func (ls *loadingSpace) getLoadErr() error { + ls.lock.Lock() + defer ls.lock.Unlock() + return ls.loadErr +} + +func (ls *loadingSpace) setLoadErr(err error) { + ls.lock.Lock() + defer ls.lock.Unlock() + ls.loadErr = err +} + func (ls *loadingSpace) loadRetry(ctx context.Context) { defer func() { - if err := ls.spaceServiceProvider.onLoad(ls.space, ls.loadErr); err != nil { + if err := ls.spaceServiceProvider.onLoad(ls.space, ls.getLoadErr()); err != nil { log.WarnCtx(ctx, "space onLoad error", zap.Error(err)) } close(ls.loadCh) @@ -67,7 +83,7 @@ func (ls *loadingSpace) loadRetry(ctx context.Context) { for { select { case <-ctx.Done(): - ls.loadErr = ctx.Err() + ls.setLoadErr(ctx.Err()) return case <-time.After(timeout): if ls.load(ctx) { @@ -90,7 +106,7 @@ func (ls *loadingSpace) load(ctx context.Context) (ok bool) { err = sp.WaitMandatoryObjects(ctx) if errors.Is(err, treechangeproto.ErrGetTree) || errors.Is(err, objecttree.ErrHasInvalidChanges) || errors.Is(err, list.ErrNoReadKey) { if ls.stopIfMandatoryFail { - ls.loadErr = err + ls.setLoadErr(err) return true } return ls.disableRemoteLoad @@ -103,7 +119,7 @@ func (ls *loadingSpace) load(ctx context.Context) (ok bool) { log.WarnCtx(ctx, "space close error", zap.Error(closeErr)) } } - ls.loadErr = err + ls.setLoadErr(err) } else { if ls.latestAclHeadId != "" && !ls.disableRemoteLoad { acl := sp.CommonSpace().Acl() diff --git a/space/internal/components/spaceloader/spaceloader.go b/space/internal/components/spaceloader/spaceloader.go index 0145887f0..f1074f6ce 100644 --- a/space/internal/components/spaceloader/spaceloader.go +++ b/space/internal/components/spaceloader/spaceloader.go @@ -129,7 +129,7 @@ func (s *spaceLoader) WaitLoad(ctx context.Context) (sp clientspace.Space, err e case spaceinfo.LocalStatusLoading: // loading in progress, wait channel and retry waitCh := s.loading.loadCh - loadErr := s.loading.loadErr + loadErr := s.loading.getLoadErr() s.mx.Unlock() if loadErr != nil { return nil, loadErr @@ -142,7 +142,7 @@ func (s *spaceLoader) WaitLoad(ctx context.Context) (sp clientspace.Space, err e return s.WaitLoad(ctx) case spaceinfo.LocalStatusMissing: // local missing state means the loader ended with an error - err = s.loading.loadErr + err = s.loading.getLoadErr() case spaceinfo.LocalStatusOk: sp = s.space default: From 683923f3a4de216feaa60e1608aefd59e0e58f38 Mon Sep 17 00:00:00 2001 From: Sergey Date: Thu, 25 Jul 2024 18:05:08 +0200 Subject: [PATCH 66/71] GO-3769: Fix adding tree heads --- core/syncstatus/objectsyncstatus/syncstatus.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/core/syncstatus/objectsyncstatus/syncstatus.go b/core/syncstatus/objectsyncstatus/syncstatus.go index d88cba729..90c9f77a1 100644 --- a/core/syncstatus/objectsyncstatus/syncstatus.go +++ b/core/syncstatus/objectsyncstatus/syncstatus.go @@ -142,7 +142,7 @@ func (s *syncStatusService) Run(ctx context.Context) error { func (s *syncStatusService) HeadsChange(treeId string, heads []string) { s.Lock() - s.treeHeads[treeId] = treeHeadsEntry{heads: heads, syncStatus: StatusNotSynced} + s.addTreeHead(treeId, heads, StatusNotSynced) s.Unlock() s.updateDetails(treeId, domain.ObjectSyncStatusSyncing) } @@ -229,6 +229,15 @@ func mapStatus(status SyncStatus) domain.ObjectSyncStatus { func (s *syncStatusService) HeadsReceive(senderId, treeId string, heads []string) { } +func (s *syncStatusService) addTreeHead(treeId string, heads []string, status SyncStatus) { + headsCopy := slice.Copy(heads) + slices.Sort(headsCopy) + s.treeHeads[treeId] = treeHeadsEntry{ + heads: headsCopy, + syncStatus: status, + } +} + func (s *syncStatusService) Watch(treeId string) (err error) { s.Lock() defer s.Unlock() @@ -246,12 +255,7 @@ func (s *syncStatusService) Watch(treeId string) (err error) { if err != nil { return } - headsCopy := slice.Copy(heads) - slices.Sort(headsCopy) - s.treeHeads[treeId] = treeHeadsEntry{ - heads: headsCopy, - syncStatus: StatusUnknown, - } + s.addTreeHead(treeId, heads, StatusUnknown) } s.watchers[treeId] = struct{}{} From 36b28d9316f09b3c7e6723843817960f4da93459 Mon Sep 17 00:00:00 2001 From: Mikhail Iudin Date: Thu, 25 Jul 2024 18:40:36 +0200 Subject: [PATCH 67/71] GO-3815 Use arena properly --- metrics/interceptors.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metrics/interceptors.go b/metrics/interceptors.go index 226f3f38a..4794fa5ee 100644 --- a/metrics/interceptors.go +++ b/metrics/interceptors.go @@ -87,9 +87,7 @@ func SharedTraceInterceptor(ctx context.Context, req any, methodName string, act } func saveAccountStop(event *MethodEvent) error { - pool := &fastjson.ArenaPool{} - arena := pool.Get() - defer pool.Put(arena) + arena := &fastjson.Arena{} json := arena.NewObject() json.Set("method_name", arena.NewString(event.methodName)) From 4de92a469f8f5dd44ca000fd5d7da0ba6162f140 Mon Sep 17 00:00:00 2001 From: kirillston Date: Thu, 25 Jul 2024 22:20:48 +0300 Subject: [PATCH 68/71] GO-3778 Fix turning blocks into obj --- core/block/editor/basic/extract_objects.go | 52 ++++++++----- .../editor/basic/extract_objects_test.go | 78 +++++++++++++++++++ 2 files changed, 110 insertions(+), 20 deletions(-) diff --git a/core/block/editor/basic/extract_objects.go b/core/block/editor/basic/extract_objects.go index 265e51f08..768e612bb 100644 --- a/core/block/editor/basic/extract_objects.go +++ b/core/block/editor/basic/extract_objects.go @@ -2,7 +2,6 @@ package basic import ( "context" - "errors" "fmt" "github.com/globalsign/mgo/bson" @@ -119,31 +118,44 @@ func insertBlocksToState( } func (bs *basic) changeToBlockWithLink(newState *state.State, blockToReplace simple.Block, objectID string, linkBlock *model.Block) (string, error) { - if linkBlock == nil { - linkBlock = &model.Block{ - Content: &model.BlockContentOfLink{ - Link: &model.BlockContentLink{ - TargetBlockId: objectID, - Style: model.BlockContentLink_Page, - }, - }, - } - } else { - link := linkBlock.GetLink() - if link == nil { - return "", errors.New("linkBlock content is not a link") - } else { - link.TargetBlockId = objectID - } - } - linkBlockCopy := pbtypes.CopyBlock(linkBlock) return bs.CreateBlock(newState, pb.RpcBlockCreateRequest{ TargetId: blockToReplace.Model().Id, - Block: linkBlockCopy, + Block: buildBlock(linkBlock, objectID), Position: model.Block_Replace, }) } +func buildBlock(b *model.Block, targetID string) (result *model.Block) { + fallback := &model.Block{ + Content: &model.BlockContentOfLink{ + Link: &model.BlockContentLink{ + TargetBlockId: targetID, + Style: model.BlockContentLink_Page, + }, + }, + } + + if b == nil { + return fallback + } + result = pbtypes.CopyBlock(b) + + switch v := result.Content.(type) { + case *model.BlockContentOfLink: + v.Link.TargetBlockId = targetID + case *model.BlockContentOfBookmark: + v.Bookmark.TargetObjectId = targetID + case *model.BlockContentOfFile: + v.File.TargetObjectId = targetID + case *model.BlockContentOfDataview: + v.Dataview.TargetObjectId = targetID + default: + result = fallback + } + + return +} + func removeBlocks(state *state.State, descendants []simple.Block) { for _, b := range descendants { state.Unlink(b.Model().Id) diff --git a/core/block/editor/basic/extract_objects_test.go b/core/block/editor/basic/extract_objects_test.go index 369162f8b..6b90fb450 100644 --- a/core/block/editor/basic/extract_objects_test.go +++ b/core/block/editor/basic/extract_objects_test.go @@ -394,6 +394,84 @@ func TestExtractObjects(t *testing.T) { }) } +func TestBuildBlock(t *testing.T) { + const target = "target" + + for _, tc := range []struct { + name string + input, output *model.Block + }{ + { + name: "nil", + input: nil, + output: &model.Block{Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + TargetBlockId: target, + Style: model.BlockContentLink_Page, + }}}, + }, + { + name: "link", + input: &model.Block{Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + Style: model.BlockContentLink_Dashboard, + CardStyle: model.BlockContentLink_Card, + }}}, + output: &model.Block{Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + TargetBlockId: target, + Style: model.BlockContentLink_Dashboard, + CardStyle: model.BlockContentLink_Card, + }}}, + }, + { + name: "bookmark", + input: &model.Block{Content: &model.BlockContentOfBookmark{Bookmark: &model.BlockContentBookmark{ + Type: model.LinkPreview_Image, + State: model.BlockContentBookmark_Fetching, + }}}, + output: &model.Block{Content: &model.BlockContentOfBookmark{Bookmark: &model.BlockContentBookmark{ + TargetObjectId: target, + Type: model.LinkPreview_Image, + State: model.BlockContentBookmark_Fetching, + }}}, + }, + { + name: "file", + input: &model.Block{Content: &model.BlockContentOfFile{File: &model.BlockContentFile{ + Type: model.BlockContentFile_Image, + }}}, + output: &model.Block{Content: &model.BlockContentOfFile{File: &model.BlockContentFile{ + TargetObjectId: target, + Type: model.BlockContentFile_Image, + }}}, + }, + { + name: "dataview", + input: &model.Block{Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + IsCollection: true, + Source: []string{"ot-note"}, + }}}, + output: &model.Block{Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + TargetObjectId: target, + IsCollection: true, + Source: []string{"ot-note"}, + }}}, + }, + { + name: "other", + input: &model.Block{Content: &model.BlockContentOfTableRow{TableRow: &model.BlockContentTableRow{ + IsHeader: true, + }}}, + output: &model.Block{Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{ + TargetBlockId: target, + Style: model.BlockContentLink_Page, + }}}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.output, buildBlock(tc.input, target)) + }) + } +} + type fixture struct { t *testing.T ctrl *gomock.Controller From b22efb2b6f3a02d7ee36d289df83e7df16d78540 Mon Sep 17 00:00:00 2001 From: Sergey Date: Fri, 26 Jul 2024 10:46:49 +0200 Subject: [PATCH 69/71] GO-3769: Fix tests and review --- .../objectsyncstatus/syncstatus_test.go | 71 ++++++++++++------- .../spacesyncstatus/spacestatus_test.go | 4 +- .../syncsubscriptions/syncsubscriptions.go | 4 +- 3 files changed, 48 insertions(+), 31 deletions(-) diff --git a/core/syncstatus/objectsyncstatus/syncstatus_test.go b/core/syncstatus/objectsyncstatus/syncstatus_test.go index 4cd2a551c..c570c7984 100644 --- a/core/syncstatus/objectsyncstatus/syncstatus_test.go +++ b/core/syncstatus/objectsyncstatus/syncstatus_test.go @@ -24,7 +24,7 @@ import ( func Test_UseCases(t *testing.T) { t.Run("HeadsChange: new object", func(t *testing.T) { s := newFixture(t, "spaceId") - s.detailsUpdater.EXPECT().UpdateDetails("id", domain.ObjectSyncStatusSyncing, "spaceId") + s.syncDetailsUpdater.EXPECT().UpdateDetails("id", domain.ObjectSyncStatusSyncing, "spaceId") s.HeadsChange("id", []string{"head1", "head2"}) @@ -33,14 +33,14 @@ func Test_UseCases(t *testing.T) { }) t.Run("HeadsChange then HeadsApply: responsible", func(t *testing.T) { s := newFixture(t, "spaceId") - s.detailsUpdater.EXPECT().UpdateDetails("id", domain.ObjectSyncStatusSyncing, "spaceId") + s.syncDetailsUpdater.EXPECT().UpdateDetails("id", domain.ObjectSyncStatusSyncing, "spaceId") s.HeadsChange("id", []string{"head1", "head2"}) assert.NotNil(t, s.treeHeads["id"]) assert.Equal(t, []string{"head1", "head2"}, s.treeHeads["id"].heads) - s.service.EXPECT().NodeIds("spaceId").Return([]string{"peerId"}) + s.nodeConfService.EXPECT().NodeIds("spaceId").Return([]string{"peerId"}) s.HeadsApply("peerId", "id", []string{"head1", "head2"}, true) @@ -50,14 +50,14 @@ func Test_UseCases(t *testing.T) { }) t.Run("HeadsChange then HeadsApply: not responsible", func(t *testing.T) { s := newFixture(t, "spaceId") - s.detailsUpdater.EXPECT().UpdateDetails("id", domain.ObjectSyncStatusSyncing, "spaceId") + s.syncDetailsUpdater.EXPECT().UpdateDetails("id", domain.ObjectSyncStatusSyncing, "spaceId") s.HeadsChange("id", []string{"head1", "head2"}) assert.NotNil(t, s.treeHeads["id"]) assert.Equal(t, []string{"head1", "head2"}, s.treeHeads["id"].heads) - s.service.EXPECT().NodeIds("spaceId").Return([]string{"peerId1"}) + s.nodeConfService.EXPECT().NodeIds("spaceId").Return([]string{"peerId1"}) s.HeadsApply("peerId", "id", []string{"head1", "head2"}, true) @@ -68,7 +68,7 @@ func Test_UseCases(t *testing.T) { }) t.Run("ObjectReceive: responsible", func(t *testing.T) { s := newFixture(t, "spaceId") - s.service.EXPECT().NodeIds("spaceId").Return([]string{"peerId"}) + s.nodeConfService.EXPECT().NodeIds("spaceId").Return([]string{"peerId"}) s.ObjectReceive("peerId", "id", []string{"head1", "head2"}) @@ -76,13 +76,13 @@ func Test_UseCases(t *testing.T) { }) t.Run("ObjectReceive: not responsible, but then sync with responsible", func(t *testing.T) { s := newFixture(t, "spaceId") - s.service.EXPECT().NodeIds("spaceId").Return([]string{"peerId1"}) + s.nodeConfService.EXPECT().NodeIds("spaceId").Return([]string{"peerId1"}) s.ObjectReceive("peerId", "id", []string{"head1", "head2"}) require.Contains(t, s.tempSynced, "id") - s.service.EXPECT().NodeIds("spaceId").Return([]string{"peerId1"}) + s.nodeConfService.EXPECT().NodeIds("spaceId").Return([]string{"peerId1"}) s.RemoveAllExcept("peerId1", []string{}) @@ -94,16 +94,16 @@ func TestSyncStatusService_Watch_Unwatch(t *testing.T) { t.Run("watch", func(t *testing.T) { s := newFixture(t, "spaceId") - s.storage.EXPECT().TreeStorage("id").Return(treestorage.NewInMemoryTreeStorage(&treechangeproto.RawTreeChangeWithId{Id: "id"}, []string{"headId"}, nil)) + s.spaceStorage.EXPECT().TreeStorage("id").Return(treestorage.NewInMemoryTreeStorage(&treechangeproto.RawTreeChangeWithId{Id: "id"}, []string{"head3", "head2", "head1"}, nil)) err := s.Watch("id") assert.Nil(t, err) assert.Contains(t, s.watchers, "id") - assert.Equal(t, []string{"headId"}, s.treeHeads["id"].heads) + assert.Equal(t, []string{"head1", "head2", "head3"}, s.treeHeads["id"].heads, "should be sorted") }) t.Run("unwatch", func(t *testing.T) { s := newFixture(t, "spaceId") - s.storage.EXPECT().TreeStorage("id").Return(treestorage.NewInMemoryTreeStorage(&treechangeproto.RawTreeChangeWithId{Id: "id"}, []string{"headId"}, nil)) + s.spaceStorage.EXPECT().TreeStorage("id").Return(treestorage.NewInMemoryTreeStorage(&treechangeproto.RawTreeChangeWithId{Id: "id"}, []string{"headId"}, nil)) err := s.Watch("id") assert.Nil(t, err) @@ -122,7 +122,7 @@ func TestSyncStatusService_update(t *testing.T) { updateReceiver.EXPECT().UpdateTree(context.Background(), "id2", StatusNotSynced).Return(nil) s.SetUpdateReceiver(updateReceiver) - s.detailsUpdater.EXPECT().UpdateDetails("id3", domain.ObjectSyncStatusSynced, "spaceId") + s.syncDetailsUpdater.EXPECT().UpdateDetails("id3", domain.ObjectSyncStatusSynced, "spaceId") s.synced = []string{"id3"} s.tempSynced["id4"] = struct{}{} s.treeHeads["id"] = treeHeadsEntry{syncStatus: StatusSynced, heads: []string{"headId"}} @@ -151,7 +151,7 @@ func TestSyncStatusService_RemoveAllExcept(t *testing.T) { f := newFixture(t, "spaceId") f.treeHeads["id"] = treeHeadsEntry{syncStatus: StatusNotSynced, heads: []string{"heads"}} - f.service.EXPECT().NodeIds(f.spaceId).Return([]string{"peerId"}) + f.nodeConfService.EXPECT().NodeIds(f.spaceId).Return([]string{"peerId"}) f.RemoveAllExcept("peerId", nil) assert.Equal(t, StatusSynced, f.treeHeads["id"].syncStatus) @@ -160,7 +160,7 @@ func TestSyncStatusService_RemoveAllExcept(t *testing.T) { f := newFixture(t, "id") f.treeHeads["id"] = treeHeadsEntry{syncStatus: StatusNotSynced, heads: []string{"heads"}} - f.service.EXPECT().NodeIds(f.spaceId).Return([]string{"peerId"}) + f.nodeConfService.EXPECT().NodeIds(f.spaceId).Return([]string{"peerId"}) f.RemoveAllExcept("peerId", []string{"id"}) assert.Equal(t, StatusNotSynced, f.treeHeads["id"].syncStatus) @@ -169,20 +169,39 @@ func TestSyncStatusService_RemoveAllExcept(t *testing.T) { f := newFixture(t, "spaceId") f.treeHeads["id"] = treeHeadsEntry{syncStatus: StatusNotSynced, heads: []string{"heads"}} - f.service.EXPECT().NodeIds(f.spaceId).Return([]string{"peerId1"}) + f.nodeConfService.EXPECT().NodeIds(f.spaceId).Return([]string{"peerId1"}) f.RemoveAllExcept("peerId", nil) assert.Equal(t, StatusNotSynced, f.treeHeads["id"].syncStatus) }) } +func TestHeadsChange(t *testing.T) { + fx := newFixture(t, "space1") + fx.syncDetailsUpdater.EXPECT().UpdateDetails("obj1", domain.ObjectSyncStatusSyncing, "space1") + inputHeads := []string{"b", "c", "a"} + + fx.HeadsChange("obj1", inputHeads) + + got, ok := fx.treeHeads["obj1"] + require.True(t, ok) + + want := treeHeadsEntry{ + heads: []string{"a", "b", "c"}, + syncStatus: StatusNotSynced, + } + assert.Equal(t, want, got) + assert.Equal(t, []string{"b", "c", "a"}, inputHeads, "heads should be copied") + +} + type fixture struct { *syncStatusService - service *mock_nodeconf.MockService - storage *mock_spacestorage.MockSpaceStorage - config *config.Config - detailsUpdater *mock_objectsyncstatus.MockUpdater - nodeStatus nodestatus.NodeStatus + nodeConfService *mock_nodeconf.MockService + spaceStorage *mock_spacestorage.MockSpaceStorage + config *config.Config + syncDetailsUpdater *mock_objectsyncstatus.MockUpdater + nodeStatus nodestatus.NodeStatus } func newFixture(t *testing.T, spaceId string) *fixture { @@ -209,11 +228,11 @@ func newFixture(t *testing.T, spaceId string) *fixture { err = statusService.Init(a) assert.Nil(t, err) return &fixture{ - syncStatusService: statusService.(*syncStatusService), - service: service, - storage: storage, - config: config, - detailsUpdater: detailsUpdater, - nodeStatus: nodeStatus, + syncStatusService: statusService.(*syncStatusService), + nodeConfService: service, + spaceStorage: storage, + config: config, + syncDetailsUpdater: detailsUpdater, + nodeStatus: nodeStatus, } } diff --git a/core/syncstatus/spacesyncstatus/spacestatus_test.go b/core/syncstatus/spacesyncstatus/spacestatus_test.go index 0eab6e966..5d56d821f 100644 --- a/core/syncstatus/spacesyncstatus/spacestatus_test.go +++ b/core/syncstatus/spacesyncstatus/spacestatus_test.go @@ -148,6 +148,8 @@ func newFixture(t *testing.T, beforeStart func(fx *fixture)) *fixture { syncSubs: syncsubscriptions.New(), networkConfig: networkConfig, } + fx.spaceIdGetter.EXPECT().TechSpaceId().Return("techSpaceId").Maybe() + a.Register(fx.syncSubs). Register(testutil.PrepareMock(ctx, a, networkConfig)). Register(testutil.PrepareMock(ctx, a, fx.nodeStatus)). @@ -225,7 +227,6 @@ func Test(t *testing.T) { fx.spaceSyncStatus.loopInterval = 10 * time.Millisecond objs := genSyncingObjects(10, 100, "spaceId") fx.objectStore.AddObjects(t, objs) - fx.spaceIdGetter.EXPECT().TechSpaceId().Return("techSpaceId") fx.networkConfig.EXPECT().GetNetworkMode().Return(pb.RpcAccount_DefaultConfig) fx.spaceIdGetter.EXPECT().AllSpaceIds().Return([]string{"spaceId"}) fx.nodeStatus.EXPECT().GetNodeStatus("spaceId").Return(nodestatus.Online) @@ -398,7 +399,6 @@ func Test(t *testing.T) { }) fx.UpdateMissingIds("spaceId", []string{"missingId"}) fx.Refresh("spaceId") - fx.spaceIdGetter.EXPECT().TechSpaceId().Return("techSpaceId") fx.eventSender.EXPECT().Broadcast(&pb.Event{ Messages: []*pb.EventMessage{{ Value: &pb.EventMessageValueOfSpaceSyncStatusUpdate{ diff --git a/core/syncstatus/syncsubscriptions/syncsubscriptions.go b/core/syncstatus/syncsubscriptions/syncsubscriptions.go index 90408be63..f57390d0a 100644 --- a/core/syncstatus/syncsubscriptions/syncsubscriptions.go +++ b/core/syncstatus/syncsubscriptions/syncsubscriptions.go @@ -71,9 +71,7 @@ func (s *syncSubscriptions) GetSubscription(id string) (SyncSubscription, error) func (s *syncSubscriptions) Close(ctx context.Context) (err error) { s.Lock() - subs := lo.MapToSlice(s.subs, func(key string, value SyncSubscription) SyncSubscription { - return value - }) + subs := lo.Values(s.subs) s.Unlock() for _, sub := range subs { sub.Close() From 15d2d3985affe07f9a779b1ec5b0fef418402153 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Fri, 26 Jul 2024 11:11:47 +0200 Subject: [PATCH 70/71] GO-3769 Update any-sync --- go.mod | 2 +- go.sum | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 694b5b559..8380f9e4f 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/PuerkitoBio/goquery v1.9.2 github.com/VividCortex/ewma v1.2.0 github.com/adrium/goheif v0.0.0-20230113233934-ca402e77a786 - github.com/anyproto/any-sync v0.4.22-alpha.2.0.20240725113756-193a1ce441f1 + github.com/anyproto/any-sync v0.4.22 github.com/anyproto/go-naturaldate/v2 v2.0.2-0.20230524105841-9829cfd13438 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de github.com/avast/retry-go/v4 v4.6.0 diff --git a/go.sum b/go.sum index cf9e64e86..8a512f876 100644 --- a/go.sum +++ b/go.sum @@ -83,16 +83,8 @@ github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxB github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= -github.com/anyproto/any-sync v0.4.22-0.20240716173953-cc827ebbc8b2 h1:y2+207FGo5gMqcWXPbTfZGRSRkf2cZt+M5AtikInFzE= -github.com/anyproto/any-sync v0.4.22-0.20240716173953-cc827ebbc8b2/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= -github.com/anyproto/any-sync v0.4.22-alpha.1 h1:8hgiiW2fAPLwY5kcwKBfYvat3jpk8Hfw5h+QwdZCHvg= -github.com/anyproto/any-sync v0.4.22-alpha.1/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= -github.com/anyproto/any-sync v0.4.22-alpha.1.0.20240717122315-3a22302fda91 h1:WkyVWywVGi7P72yo5ykpNhZxoNeYCLaNdEJUyGCVr7c= -github.com/anyproto/any-sync v0.4.22-alpha.1.0.20240717122315-3a22302fda91/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= -github.com/anyproto/any-sync v0.4.22-alpha.2 h1:q9eAASQU3VEYYQH8VQAPZwzWHwcAku1mXZlWWs96VMk= -github.com/anyproto/any-sync v0.4.22-alpha.2/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= -github.com/anyproto/any-sync v0.4.22-alpha.2.0.20240725113756-193a1ce441f1 h1:GPwjbv7dpW29zPVHBtFcsGOUM/YF1XIJq9bEq4cTW8Q= -github.com/anyproto/any-sync v0.4.22-alpha.2.0.20240725113756-193a1ce441f1/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= +github.com/anyproto/any-sync v0.4.22 h1:f9iAbCv/clTzYtzOzkX1IOXahVM/Art1WkUtIgnwl8U= +github.com/anyproto/any-sync v0.4.22/go.mod h1:qHIG2zMvGIthEb2FmcjQN5YZZwV8kPHv7/T0ib7YSDg= github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580 h1:Ba80IlCCxkZ9H1GF+7vFu/TSpPvbpDCxXJ5ogc4euYc= github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580/go.mod h1:T/uWAYxrXdaXw64ihI++9RMbKTCpKd/yE9+saARew7k= github.com/anyproto/go-chash v0.1.0 h1:I9meTPjXFRfXZHRJzjOHC/XF7Q5vzysKkiT/grsogXY= From 27722959334b3dbffdfcb1cf4d165b8a856735b7 Mon Sep 17 00:00:00 2001 From: Sergey Date: Fri, 26 Jul 2024 12:07:41 +0200 Subject: [PATCH 71/71] GO-3769: Fix data race --- core/syncstatus/service.go | 2 +- core/syncstatus/updatereceiver.go | 25 ++++++++++++++++--------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/core/syncstatus/service.go b/core/syncstatus/service.go index 98b432df3..5c86885d8 100644 --- a/core/syncstatus/service.go +++ b/core/syncstatus/service.go @@ -76,7 +76,7 @@ func (s *service) RegisterSpace(space commonspace.Space, sw objectsyncstatus.Sta sw.SetUpdateReceiver(s.updateReceiver) s.objectWatchers[space.Id()] = sw - s.updateReceiver.spaceId = space.Id() + s.updateReceiver.setSpaceId(space.Id()) } func (s *service) UnregisterSpace(space commonspace.Space) { diff --git a/core/syncstatus/updatereceiver.go b/core/syncstatus/updatereceiver.go index 0175c4b16..09b55a599 100644 --- a/core/syncstatus/updatereceiver.go +++ b/core/syncstatus/updatereceiver.go @@ -21,11 +21,11 @@ type updateReceiver struct { eventSender event.Sender nodeConfService nodeconf.Service - sync.Mutex - nodeConnected bool - objectStore objectstore.ObjectStore - nodeStatus nodestatus.NodeStatus - spaceId string + lock sync.Mutex + nodeConnected bool + objectStore objectstore.ObjectStore + nodeStatus nodestatus.NodeStatus + spaceId string } func newUpdateReceiver( @@ -90,14 +90,21 @@ func (r *updateReceiver) getObjectSyncStatus(objectId string, status objectsyncs } func (r *updateReceiver) isNodeConnected() bool { - r.Lock() - defer r.Unlock() + r.lock.Lock() + defer r.lock.Unlock() return r.nodeConnected } +func (r *updateReceiver) setSpaceId(spaceId string) { + r.lock.Lock() + defer r.lock.Unlock() + + r.spaceId = spaceId +} + func (r *updateReceiver) UpdateNodeStatus() { - r.Lock() - defer r.Unlock() + r.lock.Lock() + defer r.lock.Unlock() r.nodeConnected = r.nodeStatus.GetNodeStatus(r.spaceId) == nodestatus.Online }