anyproto/any-sync
1
0
Fork 0
mirror of https://github.com/anyproto/any-sync.git synced 2025-06-09 17:45:03 +09:00
Code Activity

Fix settings state rebuild

Browse source
This commit is contained in:
mcrakhman 2023-05-21 23:50:14 +02:00 • committed by Mikhail Iudin
parent 6e48b2dcff
commit 5532c2c482
No known key found for this signature in database
GPG key ID: FAAAA8BAABDFF1C0
10 changed files with 126 additions and 109 deletions
Download patch file Download diff file Expand all files Collapse all files

10
commonspace/settings/deletionmanager.go
View file

@ -47,10 +47,7 @@ type deletionManager struct {
func (d *deletionManager) UpdateState(ctx context.Context, state *settingsstate.State) error {
log := log.With(zap.String("spaceId", d.spaceId))
err := d.deletionState.Add(state.DeletedIds)
if err != nil {
log.Debug("failed to add deleted ids to deletion state")
}
d.deletionState.Add(state.DeletedIds)
if state.DeleterId == "" {
return nil
}
@ -59,10 +56,7 @@ func (d *deletionManager) UpdateState(ctx context.Context, state *settingsstate.
allIds := slice.DiscardFromSlice(d.provider.AllIds(), func(id string) bool {
return id == d.settingsId
})
err := d.deletionState.Add(allIds)
if err != nil {
log.Debug("failed to add all ids to deletion state")
}
d.deletionState.Add(allIds)
}
d.onSpaceDelete()
return nil

6
commonspace/settings/deletionmanager_test.go
View file

@ -29,7 +29,7 @@ func TestDeletionManager_UpdateState_NotResponsible(t *testing.T) {
delState := mock_settingsstate.NewMockObjectDeletionState(ctrl)
treeManager := mock_treemanager.NewMockTreeManager(ctrl)
delState.EXPECT().Add(state.DeletedIds).Return(nil)
delState.EXPECT().Add(state.DeletedIds)
delManager := newDeletionManager(spaceId,
settingsId,
@ -62,9 +62,9 @@ func TestDeletionManager_UpdateState_Responsible(t *testing.T) {
treeManager := mock_treemanager.NewMockTreeManager(ctrl)
provider := mock_settings.NewMockSpaceIdsProvider(ctrl)
delState.EXPECT().Add(state.DeletedIds).Return(nil)
delState.EXPECT().Add(state.DeletedIds)
provider.EXPECT().AllIds().Return([]string{"id", "otherId", settingsId})
delState.EXPECT().Add([]string{"id", "otherId"}).Return(nil)
delState.EXPECT().Add([]string{"id", "otherId"})
delManager := newDeletionManager(spaceId,
settingsId,
true,

49
commonspace/settings/settings.go
View file

@ -40,7 +40,16 @@ var (
ErrCantDeleteSpace = errors.New("not able to delete space")
)
var doSnapshot = objecttree.DoSnapshot
var (
doSnapshot = objecttree.DoSnapshot
buildHistoryTree = func(objTree objecttree.ObjectTree) (objecttree.ReadableObjectTree, error) {
return objecttree.BuildHistoryTree(objecttree.HistoryTreeParams{
TreeStorage: objTree.Storage(),
AclList: objTree.AclList(),
BuildFullTree: true,
})
}
)
type BuildTreeFunc func(ctx context.Context, id string, listener updatelistener.UpdateListener) (t synctree.SyncTree, err error)
@ -166,11 +175,36 @@ func (s *settingsObject) Init(ctx context.Context) (err error) {
if err != nil {
return
}
// TODO: remove this check when everybody updates
if err = s.checkHistoryState(ctx); err != nil {
return
}
s.loop.Run()
return
}
func (s *settingsObject) checkHistoryState(ctx context.Context) (err error) {
historyTree, err := buildHistoryTree(s.SyncTree)
if err != nil {
return
}
fullState, err := s.builder.Build(historyTree, nil)
if err != nil {
return
}
if len(fullState.DeletedIds) != len(s.state.DeletedIds) {
log.WarnCtx(ctx, "state does not have all deleted ids",
zap.Int("fullstate ids", len(fullState.DeletedIds)),
zap.Int("state ids", len(fullState.DeletedIds)))
s.state = fullState
err = s.deletionManager.UpdateState(context.Background(), s.state)
if err != nil {
return
}
}
return
}
func (s *settingsObject) Close() error {
s.loop.Close()
return s.SyncTree.Close()
@ -221,7 +255,7 @@ func (s *settingsObject) DeleteObject(id string) (err error) {
err = ErrDeleteSelf
return
}
if s.deletionState.Exists(id) {
if s.state.Exists(id) {
err = ErrAlreadyDeleted
return nil
}
@ -249,7 +283,7 @@ func (s *settingsObject) verifyDeleteSpace(raw *treechangeproto.RawTreeChangeWit
func (s *settingsObject) addContent(data []byte, isSnapshot bool) (err error) {
accountData := s.account.Account()
_, err = s.AddContent(context.Background(), objecttree.SignableChangeContent{
res, err := s.AddContent(context.Background(), objecttree.SignableChangeContent{
Data: data,
Key: accountData.SignKey,
IsSnapshot: isSnapshot,
@ -258,8 +292,11 @@ func (s *settingsObject) addContent(data []byte, isSnapshot bool) (err error) {
if err != nil {
return
}
s.Update(s)
if res.Mode == objecttree.Rebuild {
s.Rebuild(s)
} else {
s.Update(s)
}
return
}

103
commonspace/settings/settings_test.go
View file

@ -5,6 +5,7 @@ import (
"github.com/anytypeio/any-sync/accountservice/mock_accountservice"
"github.com/anytypeio/any-sync/commonspace/object/accountdata"
"github.com/anytypeio/any-sync/commonspace/object/tree/objecttree"
"github.com/anytypeio/any-sync/commonspace/object/tree/objecttree/mock_objecttree"
"github.com/anytypeio/any-sync/commonspace/object/tree/synctree"
"github.com/anytypeio/any-sync/commonspace/object/tree/synctree/mock_synctree"
"github.com/anytypeio/any-sync/commonspace/object/tree/synctree/updatelistener"
@ -52,6 +53,7 @@ type settingsFixture struct {
changeFactory *mock_settingsstate.MockChangeFactory
deleter *mock_settings.MockDeleter
syncTree *mock_synctree.MockSyncTree
historyTree *mock_objecttree.MockObjectTree
delState *mock_settingsstate.MockObjectDeletionState
account *mock_accountservice.MockService
}
@ -69,6 +71,7 @@ func newSettingsFixture(t *testing.T) *settingsFixture {
stateBuilder := mock_settingsstate.NewMockStateBuilder(ctrl)
changeFactory := mock_settingsstate.NewMockChangeFactory(ctrl)
syncTree := mock_synctree.NewMockSyncTree(ctrl)
historyTree := mock_objecttree.NewMockObjectTree(ctrl)
del := mock_settings.NewMockDeleter(ctrl)
delState.EXPECT().AddObserver(gomock.Any())
@ -77,6 +80,9 @@ func newSettingsFixture(t *testing.T) *settingsFixture {
require.Equal(t, objectId, id)
return newTestObjMock(syncTree), nil
})
buildHistoryTree = func(objTree objecttree.ObjectTree) (objecttree.ReadableObjectTree, error) {
return historyTree, nil
}
deps := Deps{
BuildFunc: buildFunc,
@ -104,36 +110,40 @@ func newSettingsFixture(t *testing.T) *settingsFixture {
syncTree: syncTree,
account: acc,
delState: delState,
historyTree: historyTree,
}
}
func (fx *settingsFixture) stop() {
func (fx *settingsFixture) init(t *testing.T) {
fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId)
fx.deleter.EXPECT().Delete()
fx.stateBuilder.EXPECT().Build(fx.historyTree, nil).Return(&settingsstate.State{}, nil)
fx.doc.state = &settingsstate.State{}
err := fx.doc.Init(context.Background())
require.NoError(t, err)
}
func (fx *settingsFixture) stop(t *testing.T) {
fx.syncTree.EXPECT().Close().Return(nil)
err := fx.doc.Close()
require.NoError(t, err)
fx.ctrl.Finish()
}
func TestSettingsObject_Init(t *testing.T) {
fx := newSettingsFixture(t)
defer fx.stop()
defer fx.stop(t)
fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId)
fx.deleter.EXPECT().Delete()
fx.syncTree.EXPECT().Close().Return(nil)
err := fx.doc.Init(context.Background())
require.NoError(t, err)
err = fx.doc.Close()
require.NoError(t, err)
fx.init(t)
}
func TestSettingsObject_DeleteObject_NoSnapshot(t *testing.T) {
fx := newSettingsFixture(t)
defer fx.stop()
defer fx.stop(t)
fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId)
fx.deleter.EXPECT().Delete()
err := fx.doc.Init(context.Background())
require.NoError(t, err)
fx.init(t)
delId := "delId"
doSnapshot = func(len int) bool {
@ -142,7 +152,6 @@ func TestSettingsObject_DeleteObject_NoSnapshot(t *testing.T) {
fx.syncTree.EXPECT().Id().Return("syncId")
fx.syncTree.EXPECT().Len().Return(10)
fx.delState.EXPECT().Exists(delId).Return(false)
fx.spaceStorage.EXPECT().TreeStorage(delId).Return(nil, nil)
res := []byte("settingsData")
fx.doc.state = &settingsstate.State{LastIteratedId: "someId"}
@ -162,22 +171,13 @@ func TestSettingsObject_DeleteObject_NoSnapshot(t *testing.T) {
fx.deletionManager.EXPECT().UpdateState(gomock.Any(), fx.doc.state).Return(nil)
err = fx.doc.DeleteObject(delId)
require.NoError(t, err)
fx.syncTree.EXPECT().Close().Return(nil)
err = fx.doc.Close()
require.NoError(t, err)
}
func TestSettingsObject_DeleteObject_WithSnapshot(t *testing.T) {
fx := newSettingsFixture(t)
defer fx.stop()
fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId)
fx.deleter.EXPECT().Delete()
err := fx.doc.Init(context.Background())
require.NoError(t, err)
defer fx.stop(t)
fx.init(t)
delId := "delId"
doSnapshot = func(len int) bool {
return true
@ -185,7 +185,6 @@ func TestSettingsObject_DeleteObject_WithSnapshot(t *testing.T) {
fx.syncTree.EXPECT().Id().Return("syncId")
fx.syncTree.EXPECT().Len().Return(10)
fx.delState.EXPECT().Exists(delId).Return(false)
fx.spaceStorage.EXPECT().TreeStorage(delId).Return(nil, nil)
res := []byte("settingsData")
fx.doc.state = &settingsstate.State{LastIteratedId: "someId"}
@ -199,27 +198,19 @@ func TestSettingsObject_DeleteObject_WithSnapshot(t *testing.T) {
Key: accountData.SignKey,
IsSnapshot: true,
IsEncrypted: false,
}).Return(objecttree.AddResult{}, nil)
}).Return(objecttree.AddResult{Mode: objecttree.Rebuild}, nil)
fx.stateBuilder.EXPECT().Build(fx.doc, fx.doc.state).Return(fx.doc.state, nil)
fx.stateBuilder.EXPECT().Build(fx.doc, nil).Return(fx.doc.state, nil)
fx.deletionManager.EXPECT().UpdateState(gomock.Any(), fx.doc.state).Return(nil)
err = fx.doc.DeleteObject(delId)
require.NoError(t, err)
fx.syncTree.EXPECT().Close().Return(nil)
err = fx.doc.Close()
require.NoError(t, err)
}
func TestSettingsObject_Rebuild(t *testing.T) {
fx := newSettingsFixture(t)
defer fx.stop()
defer fx.stop(t)
fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId)
fx.deleter.EXPECT().Delete()
err := fx.doc.Init(context.Background())
require.NoError(t, err)
fx.init(t)
time.Sleep(100 * time.Millisecond)
newSt := &settingsstate.State{}
@ -232,13 +223,9 @@ func TestSettingsObject_Rebuild(t *testing.T) {
func TestSettingsObject_Update(t *testing.T) {
fx := newSettingsFixture(t)
defer fx.stop()
defer fx.stop(t)
fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId)
fx.deleter.EXPECT().Delete()
err := fx.doc.Init(context.Background())
require.NoError(t, err)
fx.init(t)
time.Sleep(100 * time.Millisecond)
fx.doc.state = &settingsstate.State{}
@ -250,13 +237,9 @@ func TestSettingsObject_Update(t *testing.T) {
func TestSettingsObject_DeleteSpace(t *testing.T) {
fx := newSettingsFixture(t)
defer fx.stop()
defer fx.stop(t)
fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId)
fx.deleter.EXPECT().Delete()
err := fx.doc.Init(context.Background())
require.NoError(t, err)
fx.init(t)
time.Sleep(100 * time.Millisecond)
deleterId := "delId"
@ -275,19 +258,15 @@ func TestSettingsObject_DeleteSpace(t *testing.T) {
Heads: []string{rawCh.Id},
}, nil)
err = fx.doc.DeleteSpace(context.Background(), rawCh)
err := fx.doc.DeleteSpace(context.Background(), rawCh)
require.NoError(t, err)
}
func TestSettingsObject_DeleteSpaceIncorrectChange(t *testing.T) {
fx := newSettingsFixture(t)
defer fx.stop()
defer fx.stop(t)
fx.spaceStorage.EXPECT().SpaceSettingsId().Return(fx.docId)
fx.deleter.EXPECT().Delete()
err := fx.doc.Init(context.Background())
require.NoError(t, err)
fx.init(t)
time.Sleep(100 * time.Millisecond)
t.Run("incorrect change type", func(t *testing.T) {
@ -299,7 +278,7 @@ func TestSettingsObject_DeleteSpaceIncorrectChange(t *testing.T) {
delChange, _ := changeFactory.CreateObjectDeleteChange("otherId", &settingsstate.State{}, false)
fx.syncTree.EXPECT().UnpackChange(rawCh).Return(delChange, nil)
err = fx.doc.DeleteSpace(context.Background(), rawCh)
err := fx.doc.DeleteSpace(context.Background(), rawCh)
require.NotNil(t, err)
})
@ -312,7 +291,7 @@ func TestSettingsObject_DeleteSpaceIncorrectChange(t *testing.T) {
delChange, _ := changeFactory.CreateSpaceDeleteChange("", &settingsstate.State{}, false)
fx.syncTree.EXPECT().UnpackChange(rawCh).Return(delChange, nil)
err = fx.doc.DeleteSpace(context.Background(), rawCh)
err := fx.doc.DeleteSpace(context.Background(), rawCh)
require.NotNil(t, err)
})
}

30
commonspace/settings/settingsstate/deletionstate.go
View file

@ -2,7 +2,9 @@
package settingsstate
import (
"github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/spacestorage"
"go.uber.org/zap"
"sync"
)
@ -10,7 +12,7 @@ type StateUpdateObserver func(ids []string)
type ObjectDeletionState interface {
AddObserver(observer StateUpdateObserver)
Add(ids []string) (err error)
Add(ids []string)
GetQueued() (ids []string)
Delete(id string) (err error)
Exists(id string) bool
@ -19,14 +21,16 @@ type ObjectDeletionState interface {
type objectDeletionState struct {
sync.RWMutex
log logger.CtxLogger
queued map[string]struct{}
deleted map[string]struct{}
stateUpdateObservers []StateUpdateObserver
storage spacestorage.SpaceStorage
}
func NewObjectDeletionState(storage spacestorage.SpaceStorage) ObjectDeletionState {
func NewObjectDeletionState(log logger.CtxLogger, storage spacestorage.SpaceStorage) ObjectDeletionState {
return &objectDeletionState{
log: log,
queued: map[string]struct{}{},
deleted: map[string]struct{}{},
storage: storage,
@ -39,15 +43,13 @@ func (st *objectDeletionState) AddObserver(observer StateUpdateObserver) {
st.stateUpdateObservers = append(st.stateUpdateObservers, observer)
}
func (st *objectDeletionState) Add(ids []string) (err error) {
func (st *objectDeletionState) Add(ids []string) {
var added []string
st.Lock()
defer func() {
st.Unlock()
if err != nil {
return
}
for _, ob := range st.stateUpdateObservers {
ob(ids)
ob(added)
}
}()
@ -60,9 +62,10 @@ func (st *objectDeletionState) Add(ids []string) (err error) {
}
var status string
status, err = st.storage.TreeDeletedStatus(id)
status, err := st.storage.TreeDeletedStatus(id)
if err != nil {
return
st.log.Warn("failed to get deleted status", zap.String("treeId", id), zap.Error(err))
continue
}
switch status {
@ -71,14 +74,15 @@ func (st *objectDeletionState) Add(ids []string) (err error) {
case spacestorage.TreeDeletedStatusDeleted:
st.deleted[id] = struct{}{}
default:
st.queued[id] = struct{}{}
err = st.storage.SetTreeDeletedStatus(id, spacestorage.TreeDeletedStatusQueued)
err := st.storage.SetTreeDeletedStatus(id, spacestorage.TreeDeletedStatusQueued)
if err != nil {
return
st.log.Warn("failed to set deleted status", zap.String("treeId", id), zap.Error(err))
continue
}
st.queued[id] = struct{}{}
}
added = append(added, id)
}
return
}
func (st *objectDeletionState) GetQueued() (ids []string) {

15
commonspace/settings/settingsstate/deletionstate_test.go
View file

@ -1,6 +1,7 @@
package settingsstate
import (
"github.com/anytypeio/any-sync/app/logger"
"github.com/anytypeio/any-sync/commonspace/spacestorage"
"github.com/anytypeio/any-sync/commonspace/spacestorage/mock_spacestorage"
"github.com/golang/mock/gomock"
@ -18,7 +19,7 @@ type fixture struct {
func newFixture(t *testing.T) *fixture {
ctrl := gomock.NewController(t)
spaceStorage := mock_spacestorage.NewMockSpaceStorage(ctrl)
delState := NewObjectDeletionState(spaceStorage).(*objectDeletionState)
delState := NewObjectDeletionState(logger.NewNamed("test"), spaceStorage).(*objectDeletionState)
return &fixture{
ctrl: ctrl,
delState: delState,
@ -37,8 +38,7 @@ func TestDeletionState_Add(t *testing.T) {
id := "newId"
fx.spaceStorage.EXPECT().TreeDeletedStatus(id).Return("", nil)
fx.spaceStorage.EXPECT().SetTreeDeletedStatus(id, spacestorage.TreeDeletedStatusQueued).Return(nil)
err := fx.delState.Add([]string{id})
require.NoError(t, err)
fx.delState.Add([]string{id})
require.Contains(t, fx.delState.queued, id)
})
@ -47,8 +47,7 @@ func TestDeletionState_Add(t *testing.T) {
defer fx.stop()
id := "newId"
fx.spaceStorage.EXPECT().TreeDeletedStatus(id).Return(spacestorage.TreeDeletedStatusQueued, nil)
err := fx.delState.Add([]string{id})
require.NoError(t, err)
fx.delState.Add([]string{id})
require.Contains(t, fx.delState.queued, id)
})
@ -57,8 +56,7 @@ func TestDeletionState_Add(t *testing.T) {
defer fx.stop()
id := "newId"
fx.spaceStorage.EXPECT().TreeDeletedStatus(id).Return(spacestorage.TreeDeletedStatusDeleted, nil)
err := fx.delState.Add([]string{id})
require.NoError(t, err)
fx.delState.Add([]string{id})
require.Contains(t, fx.delState.deleted, id)
})
}
@ -98,8 +96,7 @@ func TestDeletionState_AddObserver(t *testing.T) {
id := "newId"
fx.spaceStorage.EXPECT().TreeDeletedStatus(id).Return("", nil)
fx.spaceStorage.EXPECT().SetTreeDeletedStatus(id, spacestorage.TreeDeletedStatusQueued).Return(nil)
err := fx.delState.Add([]string{id})
require.NoError(t, err)
fx.delState.Add([]string{id})
require.Contains(t, fx.delState.queued, id)
require.Equal(t, []string{id}, queued)
}

8
commonspace/settings/settingsstate/mock_settingsstate/mock_settingsstate.go
View file

@ -36,11 +36,9 @@ func (m *MockObjectDeletionState) EXPECT() *MockObjectDeletionStateMockRecorder
}
// Add mocks base method.
func (m *MockObjectDeletionState) Add(arg0 []string) error {
func (m *MockObjectDeletionState) Add(arg0 []string) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Add", arg0)
ret0, _ := ret[0].(error)
return ret0
m.ctrl.Call(m, "Add", arg0)
}
// Add indicates an expected call of Add.
@ -145,7 +143,7 @@ func (m *MockStateBuilder) EXPECT() *MockStateBuilderMockRecorder {
}
// Build mocks base method.
func (m *MockStateBuilder) Build(arg0 objecttree.ObjectTree, arg1 *settingsstate.State) (*settingsstate.State, error) {
func (m *MockStateBuilder) Build(arg0 objecttree.ReadableObjectTree, arg1 *settingsstate.State) (*settingsstate.State, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Build", arg0, arg1)
ret0, _ := ret[0].(*settingsstate.State)

8
commonspace/settings/settingsstate/settingsstate.go
View file

@ -1,7 +1,15 @@
package settingsstate
import "golang.org/x/exp/slices"
type State struct {
DeletedIds []string
DeleterId string
LastIteratedId string
}
func (s *State) Exists(id string) bool {
// using map here will not give a lot of benefit, because this thing should be called only
// when we are adding content, thus it doesn't matter
return slices.Contains(s.DeletedIds, id)
}

4
commonspace/settings/settingsstate/statebuilder.go
View file

@ -7,7 +7,7 @@ import (
)
type StateBuilder interface {
Build(tree objecttree.ObjectTree, state *State) (*State, error)
Build(tree objecttree.ReadableObjectTree, state *State) (*State, error)
}
func NewStateBuilder() StateBuilder {
@ -17,7 +17,7 @@ func NewStateBuilder() StateBuilder {
type stateBuilder struct {
}
func (s *stateBuilder) Build(tr objecttree.ObjectTree, oldState *State) (state *State, err error) {
func (s *stateBuilder) Build(tr objecttree.ReadableObjectTree, oldState *State) (state *State, err error) {
var (
rootId = tr.Root().Id
startId = rootId