From 74ba1fe6d24e24a1324429afcab1c79e894dd146 Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Sat, 22 Jun 2024 16:27:18 +0200 Subject: [PATCH 1/5] Add flusher logic --- commonspace/object/tree/objecttree/change.go | 5 +- commonspace/object/tree/objecttree/flusher.go | 43 +++++++++++ .../object/tree/objecttree/objecttree.go | 18 ++++- .../object/tree/objecttree/objecttree_test.go | 71 +++++++++++++++++++ .../tree/objecttree/objecttreefactory.go | 8 +++ 5 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 commonspace/object/tree/objecttree/flusher.go diff --git a/commonspace/object/tree/objecttree/change.go b/commonspace/object/tree/objecttree/change.go index 74a97a1f..83ac9026 100644 --- a/commonspace/object/tree/objecttree/change.go +++ b/commonspace/object/tree/objecttree/change.go @@ -2,9 +2,11 @@ package objecttree import ( "errors" + + "github.com/gogo/protobuf/proto" + "github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto" "github.com/anyproto/any-sync/util/crypto" - "github.com/gogo/protobuf/proto" ) var ( @@ -28,6 +30,7 @@ type Change struct { DataType string IsSnapshot bool IsDerived bool + IsNew bool // iterator helpers visited bool diff --git a/commonspace/object/tree/objecttree/flusher.go b/commonspace/object/tree/objecttree/flusher.go new file mode 100644 index 00000000..53c576d8 --- /dev/null +++ b/commonspace/object/tree/objecttree/flusher.go @@ -0,0 +1,43 @@ +package objecttree + +type flusher interface { + markNewChange(ch *Change) + flushAfterBuild(t *objectTree) error + flush(t *objectTree) error +} + +type defaultFlusher struct { +} + +func (d *defaultFlusher) markNewChange(ch *Change) { +} + +func (d *defaultFlusher) flushAfterBuild(t *objectTree) error { + t.tree.reduceTree() + return nil +} + +func (d *defaultFlusher) flush(t *objectTree) error { + return nil +} + +type newChangeFlusher struct { + newChanges []*Change +} + +func (n *newChangeFlusher) markNewChange(ch *Change) { + ch.IsNew = true + n.newChanges = append(n.newChanges, ch) +} + +func (n *newChangeFlusher) flushAfterBuild(t *objectTree) error { + return nil +} + +func (n *newChangeFlusher) flush(t *objectTree) error { + for _, ch := range n.newChanges { + ch.IsNew = false + } + t.tree.reduceTree() + return nil +} diff --git a/commonspace/object/tree/objecttree/objecttree.go b/commonspace/object/tree/objecttree/objecttree.go index 63dad661..54d5f4ad 100644 --- a/commonspace/object/tree/objecttree/objecttree.go +++ b/commonspace/object/tree/objecttree/objecttree.go @@ -90,6 +90,7 @@ type ObjectTree interface { Delete() error Close() error + Flush() error TryClose(objectTTL time.Duration) (bool, error) } @@ -100,6 +101,7 @@ type objectTree struct { rawChangeLoader *rawChangeLoader treeBuilder *treeBuilder aclList list.AclList + flusher flusher id string rawRoot *treechangeproto.RawTreeChangeWithId @@ -332,7 +334,10 @@ func (ot *objectTree) AddRawChanges(ctx context.Context, changesPayload RawChang } // reducing tree if we have new roots - ot.tree.reduceTree() + err = ot.flusher.flushAfterBuild(ot) + if err != nil { + return + } // that means that we removed the ids while reducing if _, exists := ot.tree.attached[lastHeadId]; !exists { @@ -504,6 +509,13 @@ func (ot *objectTree) addRawChanges(ctx context.Context, changesPayload RawChang } } +func (ot *objectTree) Flush() error { + if ot.isDeleted { + return ErrDeleted + } + return ot.flusher.flush(ot) +} + func (ot *objectTree) createAddResult(oldHeads []string, mode Mode, treeChangesAdded []*Change, rawChanges []*treechangeproto.RawTreeChangeWithId) (addResult AddResult, err error) { headsCopy := func() []string { newHeads := make([]string, 0, len(ot.tree.Heads())) @@ -521,6 +533,8 @@ func (ot *objectTree) createAddResult(oldHeads []string, mode Mode, treeChangesA for _, idx := range ot.notSeenIdxBuf { rawChange := rawChanges[idx] if ch, exists := ot.tree.attached[rawChange.Id]; exists { + // this marks the change as new if needed + ot.flusher.markNewChange(ch) if len(toConvert) != 0 { alreadyConverted[ch] = struct{}{} } @@ -539,6 +553,8 @@ func (ot *objectTree) createAddResult(oldHeads []string, mode Mode, treeChangesA for _, ch := range toConvert { // if we got some changes that we need to convert to raw if _, exists := alreadyConverted[ch]; !exists { + // this marks the change as new if needed + ot.flusher.markNewChange(ch) var raw *treechangeproto.RawTreeChangeWithId raw, err = ot.changeBuilder.Marshall(ch) if err != nil { diff --git a/commonspace/object/tree/objecttree/objecttree_test.go b/commonspace/object/tree/objecttree/objecttree_test.go index c280c6aa..225fe614 100644 --- a/commonspace/object/tree/objecttree/objecttree_test.go +++ b/commonspace/object/tree/objecttree/objecttree_test.go @@ -40,6 +40,7 @@ func genBuildFilterableTestableTree(filterFunc func(ch *Change) bool) func(treeS rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder), validator: &noOpTreeValidator{filterFunc: filterFunc}, aclList: aclList, + flusher: &defaultFlusher{}, } return buildObjectTree(deps) @@ -703,6 +704,76 @@ func TestObjectTree(t *testing.T) { } }) + t.Run("add new snapshot simple with newChangeFlusher", func(t *testing.T) { + ctx := prepareTreeContext(t, aclList) + treeStorage := ctx.treeStorage + changeCreator := ctx.changeCreator + objTree := ctx.objTree.(*objectTree) + objTree.flusher = &newChangeFlusher{} + + rawChanges := []*treechangeproto.RawTreeChangeWithId{ + changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"), + changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"), + changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"), + changeCreator.CreateRaw("4", aclList.Head().Id, "3", false, "3"), + } + payload := RawChangesPayload{ + NewHeads: []string{rawChanges[len(rawChanges)-1].Id}, + RawChanges: rawChanges, + } + + res, err := objTree.AddRawChanges(context.Background(), payload) + require.NoError(t, err, "adding changes should be without error") + + // check result + assert.Equal(t, []string{"0"}, res.OldHeads) + assert.Equal(t, []string{"4"}, res.Heads) + assert.Equal(t, len(rawChanges), len(res.Added)) + assert.Equal(t, Append, res.Mode) + + // check tree heads + assert.Equal(t, []string{"4"}, objTree.Heads()) + + // check tree iterate + var iterChangesId []string + err = objTree.IterateRoot(nil, func(change *Change) bool { + iterChangesId = append(iterChangesId, change.Id) + return true + }) + require.NoError(t, err, "iterate should be without error") + assert.Equal(t, []string{"0", "1", "2", "3", "4"}, iterChangesId) + // before flush + assert.Equal(t, "0", objTree.Root().Id) + + // check storage + heads, _ := treeStorage.Heads() + assert.Equal(t, []string{"4"}, heads) + + for _, ch := range rawChanges { + treeCh, err := objTree.GetChange(ch.Id) + require.NoError(t, err) + require.True(t, treeCh.IsNew) + raw, err := treeStorage.GetRawChange(context.Background(), ch.Id) + assert.NoError(t, err, "storage should have all the changes") + assert.Equal(t, ch, raw, "the changes in the storage should be the same") + } + + err = objTree.Flush() + require.NoError(t, err) + + // after flush + assert.Equal(t, "3", objTree.Root().Id) + for _, ch := range rawChanges { + treeCh, err := objTree.GetChange(ch.Id) + if ch.Id == "3" || ch.Id == "4" { + require.NoError(t, err) + require.False(t, treeCh.IsNew) + continue + } + require.Error(t, err) + } + }) + t.Run("snapshot path", func(t *testing.T) { ctx := prepareTreeContext(t, aclList) changeCreator := ctx.changeCreator diff --git a/commonspace/object/tree/objecttree/objecttreefactory.go b/commonspace/object/tree/objecttree/objecttreefactory.go index a96992ba..226cc3b0 100644 --- a/commonspace/object/tree/objecttree/objecttreefactory.go +++ b/commonspace/object/tree/objecttree/objecttreefactory.go @@ -39,6 +39,7 @@ type objectTreeDeps struct { validator ObjectTreeValidator rawChangeLoader *rawChangeLoader aclList list.AclList + flusher flusher } type BuildObjectTreeFunc = func(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) @@ -58,6 +59,7 @@ func verifiableTreeDeps( validator: newTreeValidator(false, false), rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder), aclList: aclList, + flusher: &defaultFlusher{}, } } @@ -74,6 +76,7 @@ func emptyDataTreeDeps( validator: newTreeValidator(false, false), rawChangeLoader: newStorageLoader(treeStorage, changeBuilder), aclList: aclList, + flusher: &defaultFlusher{}, } } @@ -90,6 +93,7 @@ func nonVerifiableTreeDeps( validator: &noOpTreeValidator{}, rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder), aclList: aclList, + flusher: &defaultFlusher{}, } } @@ -114,6 +118,7 @@ func BuildTestableTree(treeStorage treestorage.TreeStorage, aclList list.AclList rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder), validator: &noOpTreeValidator{}, aclList: aclList, + flusher: &defaultFlusher{}, } return buildObjectTree(deps) @@ -131,6 +136,7 @@ func BuildEmptyDataTestableTree(treeStorage treestorage.TreeStorage, aclList lis rawChangeLoader: newStorageLoader(treeStorage, changeBuilder), validator: &noOpTreeValidator{}, aclList: aclList, + flusher: &defaultFlusher{}, } return buildObjectTree(deps) @@ -219,6 +225,7 @@ func buildObjectTree(deps objectTreeDeps) (ObjectTree, error) { difSnapshotBuf: make([]*treechangeproto.RawTreeChangeWithId, 0, 10), notSeenIdxBuf: make([]int, 0, 10), newSnapshotsBuf: make([]*Change, 0, 10), + flusher: deps.flusher, } err := objTree.rebuildFromStorage(nil, nil) @@ -255,6 +262,7 @@ func buildHistoryTree(deps objectTreeDeps, params HistoryTreeParams) (ht History difSnapshotBuf: make([]*treechangeproto.RawTreeChangeWithId, 0, 10), notSeenIdxBuf: make([]int, 0, 10), newSnapshotsBuf: make([]*Change, 0, 10), + flusher: deps.flusher, } hTree := &historyTree{objectTree: objTree} From 7aa784a9858eb77ac868dd50412c21f831e8359d Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Sat, 22 Jun 2024 16:43:38 +0200 Subject: [PATCH 2/5] Add flush logic --- commonspace/object/tree/objecttree/flusher.go | 24 +++++++++++-------- .../object/tree/objecttree/objecttree.go | 15 ++++++++---- .../object/tree/objecttree/objecttree_test.go | 4 ++-- .../tree/objecttree/objecttreefactory.go | 2 +- commonspace/object/tree/synctree/synctree.go | 9 +++++++ 5 files changed, 36 insertions(+), 18 deletions(-) diff --git a/commonspace/object/tree/objecttree/flusher.go b/commonspace/object/tree/objecttree/flusher.go index 53c576d8..e169a791 100644 --- a/commonspace/object/tree/objecttree/flusher.go +++ b/commonspace/object/tree/objecttree/flusher.go @@ -1,40 +1,44 @@ package objecttree -type flusher interface { - markNewChange(ch *Change) - flushAfterBuild(t *objectTree) error - flush(t *objectTree) error +type Flusher interface { + MarkNewChange(ch *Change) + FlushAfterBuild(t *objectTree) error + Flush(t *objectTree) error } type defaultFlusher struct { } -func (d *defaultFlusher) markNewChange(ch *Change) { +func (d *defaultFlusher) MarkNewChange(ch *Change) { } -func (d *defaultFlusher) flushAfterBuild(t *objectTree) error { +func (d *defaultFlusher) FlushAfterBuild(t *objectTree) error { t.tree.reduceTree() return nil } -func (d *defaultFlusher) flush(t *objectTree) error { +func (d *defaultFlusher) Flush(t *objectTree) error { return nil } +func MarkNewChangeFlusher() Flusher { + return &newChangeFlusher{} +} + type newChangeFlusher struct { newChanges []*Change } -func (n *newChangeFlusher) markNewChange(ch *Change) { +func (n *newChangeFlusher) MarkNewChange(ch *Change) { ch.IsNew = true n.newChanges = append(n.newChanges, ch) } -func (n *newChangeFlusher) flushAfterBuild(t *objectTree) error { +func (n *newChangeFlusher) FlushAfterBuild(t *objectTree) error { return nil } -func (n *newChangeFlusher) flush(t *objectTree) error { +func (n *newChangeFlusher) Flush(t *objectTree) error { for _, ch := range n.newChanges { ch.IsNew = false } diff --git a/commonspace/object/tree/objecttree/objecttree.go b/commonspace/object/tree/objecttree/objecttree.go index 54d5f4ad..44c81e08 100644 --- a/commonspace/object/tree/objecttree/objecttree.go +++ b/commonspace/object/tree/objecttree/objecttree.go @@ -90,6 +90,7 @@ type ObjectTree interface { Delete() error Close() error + SetFlusher(flusher Flusher) Flush() error TryClose(objectTTL time.Duration) (bool, error) } @@ -101,7 +102,7 @@ type objectTree struct { rawChangeLoader *rawChangeLoader treeBuilder *treeBuilder aclList list.AclList - flusher flusher + flusher Flusher id string rawRoot *treechangeproto.RawTreeChangeWithId @@ -173,6 +174,10 @@ func (ot *objectTree) Header() *treechangeproto.RawTreeChangeWithId { return ot.rawRoot } +func (ot *objectTree) SetFlusher(flusher Flusher) { + ot.flusher = flusher +} + func (ot *objectTree) UnmarshalledHeader() *Change { return ot.root } @@ -334,7 +339,7 @@ func (ot *objectTree) AddRawChanges(ctx context.Context, changesPayload RawChang } // reducing tree if we have new roots - err = ot.flusher.flushAfterBuild(ot) + err = ot.flusher.FlushAfterBuild(ot) if err != nil { return } @@ -513,7 +518,7 @@ func (ot *objectTree) Flush() error { if ot.isDeleted { return ErrDeleted } - return ot.flusher.flush(ot) + return ot.flusher.Flush(ot) } func (ot *objectTree) createAddResult(oldHeads []string, mode Mode, treeChangesAdded []*Change, rawChanges []*treechangeproto.RawTreeChangeWithId) (addResult AddResult, err error) { @@ -534,7 +539,7 @@ func (ot *objectTree) createAddResult(oldHeads []string, mode Mode, treeChangesA rawChange := rawChanges[idx] if ch, exists := ot.tree.attached[rawChange.Id]; exists { // this marks the change as new if needed - ot.flusher.markNewChange(ch) + ot.flusher.MarkNewChange(ch) if len(toConvert) != 0 { alreadyConverted[ch] = struct{}{} } @@ -554,7 +559,7 @@ func (ot *objectTree) createAddResult(oldHeads []string, mode Mode, treeChangesA // if we got some changes that we need to convert to raw if _, exists := alreadyConverted[ch]; !exists { // this marks the change as new if needed - ot.flusher.markNewChange(ch) + ot.flusher.MarkNewChange(ch) var raw *treechangeproto.RawTreeChangeWithId raw, err = ot.changeBuilder.Marshall(ch) if err != nil { diff --git a/commonspace/object/tree/objecttree/objecttree_test.go b/commonspace/object/tree/objecttree/objecttree_test.go index 225fe614..6bbbe979 100644 --- a/commonspace/object/tree/objecttree/objecttree_test.go +++ b/commonspace/object/tree/objecttree/objecttree_test.go @@ -742,7 +742,7 @@ func TestObjectTree(t *testing.T) { }) require.NoError(t, err, "iterate should be without error") assert.Equal(t, []string{"0", "1", "2", "3", "4"}, iterChangesId) - // before flush + // before Flush assert.Equal(t, "0", objTree.Root().Id) // check storage @@ -761,7 +761,7 @@ func TestObjectTree(t *testing.T) { err = objTree.Flush() require.NoError(t, err) - // after flush + // after Flush assert.Equal(t, "3", objTree.Root().Id) for _, ch := range rawChanges { treeCh, err := objTree.GetChange(ch.Id) diff --git a/commonspace/object/tree/objecttree/objecttreefactory.go b/commonspace/object/tree/objecttree/objecttreefactory.go index 226cc3b0..488bc5b3 100644 --- a/commonspace/object/tree/objecttree/objecttreefactory.go +++ b/commonspace/object/tree/objecttree/objecttreefactory.go @@ -39,7 +39,7 @@ type objectTreeDeps struct { validator ObjectTreeValidator rawChangeLoader *rawChangeLoader aclList list.AclList - flusher flusher + flusher Flusher } type BuildObjectTreeFunc = func(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index ca3f2cc3..f80bd539 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -179,6 +179,7 @@ func (s *syncTree) AddRawChanges(ctx context.Context, changesPayload objecttree. s.listener.Rebuild(s) } } + s.flush() if res.Mode != objecttree.Nothing { if s.notifiable != nil { s.notifiable.UpdateHeads(s.Id(), res.Heads) @@ -258,8 +259,16 @@ func (s *syncTree) SyncWithPeer(ctx context.Context, peerId string) (err error) func (s *syncTree) afterBuild() { if s.listener != nil { s.listener.Rebuild(s) + s.flush() } if s.notifiable != nil { s.notifiable.UpdateHeads(s.Id(), s.Heads()) } } + +func (s *syncTree) flush() { + err := s.Flush() + if err != nil { + log.Warn("flush error", zap.Error(err)) + } +} From cbcc034ad15603531f39982ca379c9bfbf080dca Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Mon, 24 Jun 2024 12:06:36 +0200 Subject: [PATCH 3/5] Fix tests --- .../mock_objecttree/mock_objecttree.go | 26 +++++++++++++++++++ .../synctree/mock_synctree/mock_synctree.go | 26 +++++++++++++++++++ .../object/tree/synctree/synctree_test.go | 10 ++++--- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/commonspace/object/tree/objecttree/mock_objecttree/mock_objecttree.go b/commonspace/object/tree/objecttree/mock_objecttree/mock_objecttree.go index ad2e0fea..b56a325c 100644 --- a/commonspace/object/tree/objecttree/mock_objecttree/mock_objecttree.go +++ b/commonspace/object/tree/objecttree/mock_objecttree/mock_objecttree.go @@ -160,6 +160,20 @@ func (mr *MockObjectTreeMockRecorder) Delete() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockObjectTree)(nil).Delete)) } +// Flush mocks base method. +func (m *MockObjectTree) Flush() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Flush") + ret0, _ := ret[0].(error) + return ret0 +} + +// Flush indicates an expected call of Flush. +func (mr *MockObjectTreeMockRecorder) Flush() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Flush", reflect.TypeOf((*MockObjectTree)(nil).Flush)) +} + // GetChange mocks base method. func (m *MockObjectTree) GetChange(arg0 string) (*objecttree.Change, error) { m.ctrl.T.Helper() @@ -332,6 +346,18 @@ func (mr *MockObjectTreeMockRecorder) Root() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Root", reflect.TypeOf((*MockObjectTree)(nil).Root)) } +// SetFlusher mocks base method. +func (m *MockObjectTree) SetFlusher(arg0 objecttree.Flusher) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetFlusher", arg0) +} + +// SetFlusher indicates an expected call of SetFlusher. +func (mr *MockObjectTreeMockRecorder) SetFlusher(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetFlusher", reflect.TypeOf((*MockObjectTree)(nil).SetFlusher), arg0) +} + // SnapshotPath mocks base method. func (m *MockObjectTree) SnapshotPath() []string { m.ctrl.T.Helper() diff --git a/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go b/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go index 15b5b689..0c7af9f9 100644 --- a/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go +++ b/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go @@ -162,6 +162,20 @@ func (mr *MockSyncTreeMockRecorder) Delete() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockSyncTree)(nil).Delete)) } +// Flush mocks base method. +func (m *MockSyncTree) Flush() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Flush") + ret0, _ := ret[0].(error) + return ret0 +} + +// Flush indicates an expected call of Flush. +func (mr *MockSyncTreeMockRecorder) Flush() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Flush", reflect.TypeOf((*MockSyncTree)(nil).Flush)) +} + // GetChange mocks base method. func (m *MockSyncTree) GetChange(arg0 string) (*objecttree.Change, error) { m.ctrl.T.Helper() @@ -363,6 +377,18 @@ func (mr *MockSyncTreeMockRecorder) Root() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Root", reflect.TypeOf((*MockSyncTree)(nil).Root)) } +// SetFlusher mocks base method. +func (m *MockSyncTree) SetFlusher(arg0 objecttree.Flusher) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetFlusher", arg0) +} + +// SetFlusher indicates an expected call of SetFlusher. +func (mr *MockSyncTreeMockRecorder) SetFlusher(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetFlusher", reflect.TypeOf((*MockSyncTree)(nil).SetFlusher), arg0) +} + // SetListener mocks base method. func (m *MockSyncTree) SetListener(arg0 updatelistener.UpdateListener) { m.ctrl.T.Helper() diff --git a/commonspace/object/tree/synctree/synctree_test.go b/commonspace/object/tree/synctree/synctree_test.go index 3bba33b3..4634d4f7 100644 --- a/commonspace/object/tree/synctree/synctree_test.go +++ b/commonspace/object/tree/synctree/synctree_test.go @@ -2,6 +2,11 @@ package synctree import ( "context" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "github.com/anyproto/any-sync/commonspace/object/tree/objecttree" "github.com/anyproto/any-sync/commonspace/object/tree/objecttree/mock_objecttree" "github.com/anyproto/any-sync/commonspace/object/tree/synctree/mock_synctree" @@ -11,9 +16,6 @@ import ( "github.com/anyproto/any-sync/commonspace/objectsync" "github.com/anyproto/any-sync/commonspace/syncstatus" "github.com/anyproto/any-sync/nodeconf" - "github.com/stretchr/testify/require" - "go.uber.org/mock/gomock" - "testing" ) type syncTreeMatcher struct { @@ -74,6 +76,7 @@ func Test_BuildSyncTree(t *testing.T) { syncClientMock.EXPECT().CreateHeadUpdate(gomock.Eq(tr), gomock.Eq(changes)).Return(headUpdate) syncClientMock.EXPECT().Broadcast(gomock.Eq(headUpdate)) + objTreeMock.EXPECT().Flush() res, err := tr.AddRawChanges(ctx, payload) require.NoError(t, err) require.Equal(t, expectedRes, res) @@ -96,6 +99,7 @@ func Test_BuildSyncTree(t *testing.T) { syncClientMock.EXPECT().CreateHeadUpdate(gomock.Eq(tr), gomock.Eq(changes)).Return(headUpdate) syncClientMock.EXPECT().Broadcast(gomock.Eq(headUpdate)) + objTreeMock.EXPECT().Flush() res, err := tr.AddRawChanges(ctx, payload) require.NoError(t, err) require.Equal(t, expectedRes, res) From 5b31ec812a2850108cf219afba027c4f340dfd9d Mon Sep 17 00:00:00 2001 From: Sergey Date: Fri, 9 Aug 2024 13:13:24 +0200 Subject: [PATCH 4/5] Object Tree: Add method AddContentWithValidator --- commonspace/object/tree/objecttree/objecttree.go | 13 +++++++++++++ commonspace/object/tree/synctree/synctree.go | 7 ++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/commonspace/object/tree/objecttree/objecttree.go b/commonspace/object/tree/objecttree/objecttree.go index 44c81e08..d2c555f6 100644 --- a/commonspace/object/tree/objecttree/objecttree.go +++ b/commonspace/object/tree/objecttree/objecttree.go @@ -83,6 +83,7 @@ type ObjectTree interface { Storage() treestorage.TreeStorage AddContent(ctx context.Context, content SignableChangeContent) (AddResult, error) + AddContentWithValidator(ctx context.Context, content SignableChangeContent, validate func(change *treechangeproto.RawTreeChangeWithId) error) (AddResult, error) AddRawChanges(ctx context.Context, changes RawChangesPayload) (AddResult, error) UnpackChange(raw *treechangeproto.RawTreeChangeWithId) (data []byte, err error) @@ -209,6 +210,10 @@ func (ot *objectTree) logUseWhenUnlocked() { } func (ot *objectTree) AddContent(ctx context.Context, content SignableChangeContent) (res AddResult, err error) { + return ot.AddContentWithValidator(ctx, content, nil) +} + +func (ot *objectTree) AddContentWithValidator(ctx context.Context, content SignableChangeContent, validator func(change *treechangeproto.RawTreeChangeWithId) error) (res AddResult, err error) { if ot.isDeleted { err = ErrDeleted return @@ -228,6 +233,14 @@ func (ot *objectTree) AddContent(ctx context.Context, content SignableChangeCont // clearing tree, because we already saved everything in the last snapshot ot.tree = &Tree{} } + + if validator != nil { + err = validator(rawChange) + if err != nil { + return + } + } + err = ot.tree.AddMergedHead(objChange) if err != nil { panic(err) diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index f80bd539..d4cd7936 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -12,6 +12,7 @@ import ( "github.com/anyproto/any-sync/commonspace/object/acl/list" "github.com/anyproto/any-sync/commonspace/object/tree/objecttree" "github.com/anyproto/any-sync/commonspace/object/tree/synctree/updatelistener" + "github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto" "github.com/anyproto/any-sync/commonspace/object/tree/treestorage" "github.com/anyproto/any-sync/commonspace/objectsync/synchandler" "github.com/anyproto/any-sync/commonspace/spacestorage" @@ -145,10 +146,14 @@ func (s *syncTree) IterateRoot(convert objecttree.ChangeConvertFunc, iterate obj } func (s *syncTree) AddContent(ctx context.Context, content objecttree.SignableChangeContent) (res objecttree.AddResult, err error) { + return s.AddContentWithValidator(ctx, content, nil) +} + +func (s *syncTree) AddContentWithValidator(ctx context.Context, content objecttree.SignableChangeContent, validate func(change *treechangeproto.RawTreeChangeWithId) error) (res objecttree.AddResult, err error) { if err = s.checkAlive(); err != nil { return } - res, err = s.ObjectTree.AddContent(ctx, content) + res, err = s.ObjectTree.AddContentWithValidator(ctx, content, validate) if err != nil { return } From c988e39b55bebedf14fa6e0d4b0d43c882b6d47a Mon Sep 17 00:00:00 2001 From: Sergey Date: Tue, 13 Aug 2024 15:56:01 +0200 Subject: [PATCH 5/5] Object Tree: add missing mocks --- .../objecttree/mock_objecttree/mock_objecttree.go | 15 +++++++++++++++ .../tree/synctree/mock_synctree/mock_synctree.go | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/commonspace/object/tree/objecttree/mock_objecttree/mock_objecttree.go b/commonspace/object/tree/objecttree/mock_objecttree/mock_objecttree.go index b56a325c..1ea8a43a 100644 --- a/commonspace/object/tree/objecttree/mock_objecttree/mock_objecttree.go +++ b/commonspace/object/tree/objecttree/mock_objecttree/mock_objecttree.go @@ -73,6 +73,21 @@ func (mr *MockObjectTreeMockRecorder) AddContent(arg0, arg1 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddContent", reflect.TypeOf((*MockObjectTree)(nil).AddContent), arg0, arg1) } +// AddContentWithValidator mocks base method. +func (m *MockObjectTree) AddContentWithValidator(arg0 context.Context, arg1 objecttree.SignableChangeContent, arg2 func(*treechangeproto.RawTreeChangeWithId) error) (objecttree.AddResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddContentWithValidator", arg0, arg1, arg2) + ret0, _ := ret[0].(objecttree.AddResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AddContentWithValidator indicates an expected call of AddContentWithValidator. +func (mr *MockObjectTreeMockRecorder) AddContentWithValidator(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddContentWithValidator", reflect.TypeOf((*MockObjectTree)(nil).AddContentWithValidator), arg0, arg1, arg2) +} + // AddRawChanges mocks base method. func (m *MockObjectTree) AddRawChanges(arg0 context.Context, arg1 objecttree.RawChangesPayload) (objecttree.AddResult, error) { m.ctrl.T.Helper() diff --git a/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go b/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go index 0c7af9f9..1dbef02e 100644 --- a/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go +++ b/commonspace/object/tree/synctree/mock_synctree/mock_synctree.go @@ -75,6 +75,21 @@ func (mr *MockSyncTreeMockRecorder) AddContent(arg0, arg1 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddContent", reflect.TypeOf((*MockSyncTree)(nil).AddContent), arg0, arg1) } +// AddContentWithValidator mocks base method. +func (m *MockSyncTree) AddContentWithValidator(arg0 context.Context, arg1 objecttree.SignableChangeContent, arg2 func(*treechangeproto.RawTreeChangeWithId) error) (objecttree.AddResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddContentWithValidator", arg0, arg1, arg2) + ret0, _ := ret[0].(objecttree.AddResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AddContentWithValidator indicates an expected call of AddContentWithValidator. +func (mr *MockSyncTreeMockRecorder) AddContentWithValidator(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddContentWithValidator", reflect.TypeOf((*MockSyncTree)(nil).AddContentWithValidator), arg0, arg1, arg2) +} + // AddRawChanges mocks base method. func (m *MockSyncTree) AddRawChanges(arg0 context.Context, arg1 objecttree.RawChangesPayload) (objecttree.AddResult, error) { m.ctrl.T.Helper()