anyproto/any-sync
1
0
Fork 0
mirror of https://github.com/anyproto/any-sync.git synced 2025-06-08 05:57:03 +09:00
Code Activity

Merge branch 'main' of github.com:anyproto/any-sync into go-2828-add-debug-diagnostics-for-spaces-and-connections

Browse source
This commit is contained in:
AnastasiaShemyakinskaya 2024-11-08 17:21:46 +01:00
commit 87f038c457
No known key found for this signature in database
GPG key ID: CCD60ED83B103281
21 changed files with 354 additions and 102 deletions
Download patch file Download diff file Expand all files Collapse all files

10
commonspace/deletionmanager/deleteloop.go
View file

@ -11,11 +11,11 @@ type deleteLoop struct {
deleteCtx context.Context
deleteCancel context.CancelFunc
deleteChan chan struct{}
deleteFunc func()
deleteFunc func(ctx context.Context)
loopDone chan struct{}
}
func newDeleteLoop(deleteFunc func()) *deleteLoop {
func newDeleteLoop(deleteFunc func(ctx context.Context)) *deleteLoop {
ctx, cancel := context.WithCancel(context.Background())
return &deleteLoop{
deleteCtx: ctx,
@ -32,7 +32,7 @@ func (dl *deleteLoop) Run() {
func (dl *deleteLoop) loop() {
defer close(dl.loopDone)
dl.deleteFunc()
dl.deleteFunc(dl.deleteCtx)
ticker := time.NewTicker(deleteLoopInterval)
defer ticker.Stop()
for {
@ -40,10 +40,10 @@ func (dl *deleteLoop) loop() {
case <-dl.deleteCtx.Done():
return
case <-dl.deleteChan:
dl.deleteFunc()
dl.deleteFunc(dl.deleteCtx)
ticker.Reset(deleteLoopInterval)
case <-ticker.C:
dl.deleteFunc()
dl.deleteFunc(dl.deleteCtx)
}
}
}

10
commonspace/deletionmanager/deleter.go
View file

@ -2,16 +2,18 @@ package deletionmanager
import (
"context"
"go.uber.org/zap"
"github.com/anyproto/any-sync/app/logger"
"github.com/anyproto/any-sync/commonspace/deletionstate"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/commonspace/object/treemanager"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"go.uber.org/zap"
)
type Deleter interface {
Delete()
Delete(ctx context.Context)
}
type deleter struct {
@ -25,7 +27,7 @@ func newDeleter(st spacestorage.SpaceStorage, state deletionstate.ObjectDeletion
return &deleter{st, state, getter, log}
}
func (d *deleter) Delete() {
func (d *deleter) Delete(ctx context.Context) {
var (
allQueued = d.state.GetQueued()
spaceId = d.st.Id()
@ -39,7 +41,7 @@ func (d *deleter) Delete() {
continue
}
} else {
err = d.getter.DeleteTree(context.Background(), spaceId, id)
err = d.getter.DeleteTree(ctx, spaceId, id)
if err != nil && err != spacestorage.ErrTreeStorageAlreadyDeleted {
log.Error("failed to delete object", zap.Error(err))
continue

17
commonspace/deletionmanager/deleter_test.go
View file

@ -1,13 +1,16 @@
package deletionmanager
import (
"context"
"fmt"
"testing"
"go.uber.org/mock/gomock"
"github.com/anyproto/any-sync/commonspace/deletionstate/mock_deletionstate"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/commonspace/object/treemanager/mock_treemanager"
"github.com/anyproto/any-sync/commonspace/spacestorage/mock_spacestorage"
"go.uber.org/mock/gomock"
"testing"
)
func TestDeleter_Delete(t *testing.T) {
@ -27,7 +30,7 @@ func TestDeleter_Delete(t *testing.T) {
treeManager.EXPECT().MarkTreeDeleted(gomock.Any(), spaceId, id).Return(nil)
delState.EXPECT().Delete(id).Return(nil)
deleter.Delete()
deleter.Delete(context.TODO())
})
t.Run("deleter delete mark deleted other error", func(t *testing.T) {
@ -37,7 +40,7 @@ func TestDeleter_Delete(t *testing.T) {
st.EXPECT().Id().Return(spaceId)
st.EXPECT().TreeStorage(id).Return(nil, fmt.Errorf("unknown error"))
deleter.Delete()
deleter.Delete(context.TODO())
})
t.Run("deleter delete mark deleted fail", func(t *testing.T) {
@ -48,7 +51,7 @@ func TestDeleter_Delete(t *testing.T) {
st.EXPECT().TreeStorage(id).Return(nil, treestorage.ErrUnknownTreeId)
treeManager.EXPECT().MarkTreeDeleted(gomock.Any(), spaceId, id).Return(fmt.Errorf("mark error"))
deleter.Delete()
deleter.Delete(context.TODO())
})
t.Run("deleter delete success", func(t *testing.T) {
@ -60,7 +63,7 @@ func TestDeleter_Delete(t *testing.T) {
treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(nil)
delState.EXPECT().Delete(id).Return(nil)
deleter.Delete()
deleter.Delete(context.TODO())
})
t.Run("deleter delete error", func(t *testing.T) {
@ -71,6 +74,6 @@ func TestDeleter_Delete(t *testing.T) {
st.EXPECT().TreeStorage(id).Return(nil, nil)
treeManager.EXPECT().DeleteTree(gomock.Any(), spaceId, id).Return(fmt.Errorf("some error"))
deleter.Delete()
deleter.Delete(context.TODO())
})
}

8
commonspace/deletionmanager/mock_deletionmanager/mock_deletionmanager.go
View file

@ -135,13 +135,13 @@ func (m *MockDeleter) EXPECT() *MockDeleterMockRecorder {
}
// Delete mocks base method.
func (m *MockDeleter) Delete() {
func (m *MockDeleter) Delete(arg0 context.Context) {
m.ctrl.T.Helper()
m.ctrl.Call(m, "Delete")
m.ctrl.Call(m, "Delete", arg0)
}
// Delete indicates an expected call of Delete.
func (mr *MockDeleterMockRecorder) Delete() *gomock.Call {
func (mr *MockDeleterMockRecorder) Delete(arg0 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockDeleter)(nil).Delete))
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockDeleter)(nil).Delete), arg0)
}

1
commonspace/object/tree/objecttree/change.go
View file

@ -18,6 +18,7 @@ var (
type Change struct {
Next []*Change
PreviousIds []string
Previous []*Change
AclHeadId string
Id string
SnapshotId string

8
commonspace/object/tree/objecttree/loaditerator.go
View file

@ -57,7 +57,9 @@ func (l *loadIterator) NextBatch(maxSize int) (batch IteratorBatch, err error) {
batch.Heads = slice.DiscardFromSlice(batch.Heads, func(s string) bool {
return slices.Contains(c.PreviousIds, s)
})
batch.Heads = append(batch.Heads, c.Id)
if !slices.Contains(batch.Heads, c.Id) {
batch.Heads = append(batch.Heads, c.Id)
}
return true
}
if curSize+rawEntry.size >= maxSize && len(batch.Batch) != 0 {
@ -75,7 +77,9 @@ func (l *loadIterator) NextBatch(maxSize int) (batch IteratorBatch, err error) {
batch.Heads = slice.DiscardFromSlice(batch.Heads, func(s string) bool {
return slices.Contains(c.PreviousIds, s)
})
batch.Heads = append(batch.Heads, c.Id)
if !slices.Contains(batch.Heads, c.Id) {
batch.Heads = append(batch.Heads, c.Id)
}
return true
})
l.lastHeads = batch.Heads

15
commonspace/object/tree/objecttree/objecttree.go
View file

@ -460,11 +460,24 @@ func (ot *objectTree) addChangesToTree(ctx context.Context, changesPayload RawCh
headsToUse = []string{}
}
rollback := func(changes []*Change) {
var visited []*Change
for _, ch := range changes {
if _, exists := ot.tree.attached[ch.Id]; exists {
if ex, exists := ot.tree.attached[ch.Id]; exists {
ex.visited = true
visited = append(visited, ex)
delete(ot.tree.attached, ch.Id)
}
}
for _, ch := range ot.tree.attached {
// deleting all visited changes from next
ch.Next = slice.DiscardFromSlice(ch.Next, func(change *Change) bool {
return change.visited
})
}
// doing this just in case
for _, ch := range visited {
ch.visited = false
}
ot.tree.headIds = headsCopy(prevHeadsCopy)
ot.tree.lastIteratedHeadId = lastIteratedId
}

39
commonspace/object/tree/objecttree/objecttree_test.go
View file

@ -323,13 +323,13 @@ func TestObjectTree(t *testing.T) {
bStore = aTree.Storage().(*treestorage.InMemoryTreeStorage).Copy()
root, _ = bStore.Root()
heads, _ := bStore.Heads()
filteredPayload, err := ValidateFilterRawTree(treestorage.TreeStorageCreatePayload{
newTree, err := ValidateFilterRawTree(treestorage.TreeStorageCreatePayload{
RootRawChange: root,
Changes: bStore.AllChanges(),
Heads: heads,
}, bAccount.Acl)
}, InMemoryStorageCreator{}, bAccount.Acl)
require.NoError(t, err)
require.Equal(t, 2, len(filteredPayload.Changes))
require.Equal(t, 2, len(newTree.Storage().(*treestorage.InMemoryTreeStorage).AllChanges()))
err = aTree.IterateRoot(func(change *Change, decrypted []byte) (any, error) {
return nil, nil
}, func(change *Change) bool {
@ -497,6 +497,7 @@ func TestObjectTree(t *testing.T) {
store, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root})
oTree, err := BuildObjectTree(store, aclList)
require.NoError(t, err)
emptyDataTreeDeps = nonVerifiableTreeDeps
err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
RootRawChange: oTree.Header(),
Heads: []string{root.Id},
@ -516,6 +517,7 @@ func TestObjectTree(t *testing.T) {
store, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root})
oTree, err := BuildObjectTree(store, aclList)
require.NoError(t, err)
emptyDataTreeDeps = nonVerifiableTreeDeps
err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
RootRawChange: oTree.Header(),
Heads: []string{root.Id},
@ -558,6 +560,7 @@ func TestObjectTree(t *testing.T) {
})
require.NoError(t, err)
allChanges := oTree.Storage().(*treestorage.InMemoryTreeStorage).AllChanges()
emptyDataTreeDeps = nonVerifiableTreeDeps
err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
RootRawChange: oTree.Header(),
Heads: []string{oTree.Heads()[0]},
@ -587,6 +590,7 @@ func TestObjectTree(t *testing.T) {
}, aclList)
require.NoError(t, err)
store, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root})
emptyDataTreeDeps = nonVerifiableTreeDeps
oTree, err := BuildObjectTree(store, aclList)
require.NoError(t, err)
_, err = oTree.AddContent(ctx, SignableChangeContent{
@ -629,7 +633,7 @@ func TestObjectTree(t *testing.T) {
changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"),
changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"),
}
defaultObjectTreeDeps = nonVerifiableTreeDeps
emptyDataTreeDeps = nonVerifiableTreeDeps
err := ValidateRawTree(treestorage.TreeStorageCreatePayload{
RootRawChange: ctx.objTree.Header(),
Heads: []string{"3"},
@ -814,6 +818,29 @@ func TestObjectTree(t *testing.T) {
}
})
t.Run("add with rollback", func(t *testing.T) {
ctx := prepareTreeContext(t, aclList)
changeCreator := ctx.changeCreator
objTree := ctx.objTree
rawChanges := []*treechangeproto.RawTreeChangeWithId{
changeCreator.CreateRaw("1", aclList.Head().Id, "0", false, "0"),
changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"),
changeCreator.CreateRaw("3", aclList.Head().Id, "0", false, "2"),
changeCreator.CreateRaw("4", aclList.Head().Id, "0", false, "3"),
}
payload := RawChangesPayload{
NewHeads: []string{rawChanges[len(rawChanges)-1].Id},
RawChanges: rawChanges,
}
tr := objTree.(*objectTree)
tr.validator.(*noOpTreeValidator).fail = true
_, err := objTree.AddRawChanges(context.Background(), payload)
require.Error(t, err)
require.Len(t, tr.tree.attached, 1)
require.Empty(t, tr.tree.attached["0"].Next)
})
t.Run("add new snapshot simple with newChangeFlusher", func(t *testing.T) {
ctx := prepareTreeContext(t, aclList)
treeStorage := ctx.treeStorage
@ -1476,7 +1503,7 @@ func TestObjectTree(t *testing.T) {
changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"),
changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"),
}
defaultObjectTreeDeps = nonVerifiableTreeDeps
emptyDataTreeDeps = nonVerifiableTreeDeps
err := ValidateRawTree(treestorage.TreeStorageCreatePayload{
RootRawChange: ctx.objTree.Header(),
Heads: []string{"3"},
@ -1493,7 +1520,7 @@ func TestObjectTree(t *testing.T) {
ctx.objTree.Header(),
changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"),
}
defaultObjectTreeDeps = nonVerifiableTreeDeps
emptyDataTreeDeps = nonVerifiableTreeDeps
err := ValidateRawTree(treestorage.TreeStorageCreatePayload{
RootRawChange: ctx.objTree.Header(),
Heads: []string{"3"},

14
commonspace/object/tree/objecttree/objecttreefactory.go
View file

@ -63,7 +63,9 @@ func verifiableTreeDeps(
}
}
func emptyDataTreeDeps(
var emptyDataTreeDeps = verifiableEmptyDataTreeDeps
func verifiableEmptyDataTreeDeps(
rootChange *treechangeproto.RawTreeChangeWithId,
treeStorage treestorage.TreeStorage,
aclList list.AclList) objectTreeDeps {
@ -156,6 +158,16 @@ func BuildKeyFilterableObjectTree(treeStorage treestorage.TreeStorage, aclList l
return buildObjectTree(deps)
}
func BuildEmptyDataKeyFilterableObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) {
rootChange, err := treeStorage.Root()
if err != nil {
return nil, err
}
deps := emptyDataTreeDeps(rootChange, treeStorage, aclList)
deps.validator = newTreeValidator(true, true)
return buildObjectTree(deps)
}
func BuildObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) {
rootChange, err := treeStorage.Root()
if err != nil {

57
commonspace/object/tree/objecttree/objecttreevalidator.go
View file

@ -9,7 +9,17 @@ import (
"github.com/anyproto/any-sync/util/slice"
)
type ValidatorFunc func(payload treestorage.TreeStorageCreatePayload, buildFunc BuildObjectTreeFunc, aclList list.AclList) (retPayload treestorage.TreeStorageCreatePayload, err error)
type TreeStorageCreator interface {
CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (treestorage.TreeStorage, error)
}
type InMemoryStorageCreator struct{}
func (i InMemoryStorageCreator) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (treestorage.TreeStorage, error) {
return treestorage.NewInMemoryTreeStorage(payload.RootRawChange, payload.Heads, payload.Changes)
}
type ValidatorFunc func(payload treestorage.TreeStorageCreatePayload, storageCreator TreeStorageCreator, aclList list.AclList) (ret ObjectTree, err error)
type ObjectTreeValidator interface {
// ValidateFullTree should always be entered while holding a read lock on AclList
@ -21,13 +31,20 @@ type ObjectTreeValidator interface {
type noOpTreeValidator struct {
filterFunc func(ch *Change) bool
fail bool
}
func (n *noOpTreeValidator) ValidateFullTree(tree *Tree, aclList list.AclList) error {
if n.fail {
return fmt.Errorf("failed")
}
return nil
}
func (n *noOpTreeValidator) ValidateNewChanges(tree *Tree, aclList list.AclList, newChanges []*Change) error {
if n.fail {
return fmt.Errorf("failed")
}
return nil
}
@ -160,12 +177,15 @@ func (v *objectTreeValidator) validateChange(tree *Tree, aclList list.AclList, c
return
}
func ValidateRawTreeBuildFunc(payload treestorage.TreeStorageCreatePayload, buildFunc BuildObjectTreeFunc, aclList list.AclList) (newPayload treestorage.TreeStorageCreatePayload, err error) {
treeStorage, err := treestorage.NewInMemoryTreeStorage(payload.RootRawChange, []string{payload.RootRawChange.Id}, nil)
func ValidateRawTreeDefault(payload treestorage.TreeStorageCreatePayload, storageCreator TreeStorageCreator, aclList list.AclList) (objTree ObjectTree, err error) {
treeStorage, err := storageCreator.CreateTreeStorage(treestorage.TreeStorageCreatePayload{
RootRawChange: payload.RootRawChange,
Heads: []string{payload.RootRawChange.Id},
})
if err != nil {
return
}
tree, err := buildFunc(treeStorage, aclList)
tree, err := BuildEmptyDataObjectTree(treeStorage, aclList)
if err != nil {
return
}
@ -179,33 +199,36 @@ func ValidateRawTreeBuildFunc(payload treestorage.TreeStorageCreatePayload, buil
return
}
if !slice.UnsortedEquals(res.Heads, payload.Heads) {
return payload, fmt.Errorf("heads mismatch: %v != %v, %w", res.Heads, payload.Heads, ErrHasInvalidChanges)
return nil, fmt.Errorf("heads mismatch: %v != %v, %w", res.Heads, payload.Heads, ErrHasInvalidChanges)
}
// if tree has only one change we still should check if the snapshot id is same as root
if IsEmptyDerivedTree(tree) {
return payload, ErrDerived
return nil, ErrDerived
}
return payload, nil
return tree, nil
}
func ValidateFilterRawTree(payload treestorage.TreeStorageCreatePayload, aclList list.AclList) (retPayload treestorage.TreeStorageCreatePayload, err error) {
func ValidateFilterRawTree(payload treestorage.TreeStorageCreatePayload, storageCreator TreeStorageCreator, aclList list.AclList) (objTree ObjectTree, err error) {
aclList.RLock()
if !aclList.AclState().HadReadPermissions(aclList.AclState().Identity()) {
aclList.RUnlock()
return payload, list.ErrNoReadKey
return nil, list.ErrNoReadKey
}
aclList.RUnlock()
treeStorage, err := treestorage.NewInMemoryTreeStorage(payload.RootRawChange, []string{payload.RootRawChange.Id}, nil)
treeStorage, err := storageCreator.CreateTreeStorage(treestorage.TreeStorageCreatePayload{
RootRawChange: payload.RootRawChange,
Heads: []string{payload.RootRawChange.Id},
})
if err != nil {
return
}
tree, err := BuildKeyFilterableObjectTree(treeStorage, aclList)
tree, err := BuildEmptyDataKeyFilterableObjectTree(treeStorage, aclList)
if err != nil {
return
}
tree.Lock()
defer tree.Unlock()
res, err := tree.AddRawChanges(context.Background(), RawChangesPayload{
_, err = tree.AddRawChanges(context.Background(), RawChangesPayload{
NewHeads: payload.Heads,
RawChanges: payload.Changes,
})
@ -213,16 +236,12 @@ func ValidateFilterRawTree(payload treestorage.TreeStorageCreatePayload, aclList
return
}
if IsEmptyTree(tree) {
return payload, ErrNoChangeInTree
return nil, ErrNoChangeInTree
}
return treestorage.TreeStorageCreatePayload{
RootRawChange: payload.RootRawChange,
Heads: res.Heads,
Changes: treeStorage.(*treestorage.InMemoryTreeStorage).AllChanges(),
}, nil
return tree, nil
}
func ValidateRawTree(payload treestorage.TreeStorageCreatePayload, aclList list.AclList) (err error) {
_, err = ValidateRawTreeBuildFunc(payload, BuildObjectTree, aclList)
_, err = ValidateRawTreeDefault(payload, InMemoryStorageCreator{}, aclList)
return
}

7
commonspace/object/tree/objecttree/tree.go
View file

@ -266,6 +266,7 @@ func (t *Tree) attach(c *Change, newEl bool) {
for _, id := range c.PreviousIds {
// prev id must already be attached if we attach this id, so we don't need to check if it exists
prev := t.attached[id]
c.Previous = append(c.Previous, prev)
// appending c to next changes of all previous changes
if len(prev.Next) == 0 || prev.Next[len(prev.Next)-1].Id <= c.Id {
prev.Next = append(prev.Next, c)
@ -341,10 +342,8 @@ func (t *Tree) dfsPrev(stack []*Change, breakpoints []string, visit func(ch *Cha
ch.visited = true
t.visitedBuf = append(t.visitedBuf, ch)
for _, prevId := range ch.PreviousIds {
prevCh, exists := t.attached[prevId]
// here the only time it wouldn't exist if we are at the tree root
if exists && !prevCh.visited {
for _, prevCh := range ch.Previous {
if !prevCh.visited {
stack = append(stack, prevCh)
}
}

25
commonspace/object/tree/objecttree/tree_test.go
View file

@ -200,6 +200,31 @@ func TestTree_CheckRootReduce(t *testing.T) {
assert.Equal(t, []string{"10", "last"}, res)
})
})
t.Run("snapshots in line", func(t *testing.T) {
tr := new(Tree)
tr.Add(
newSnapshot("0", ""),
newSnapshot("0.1", "0", "0"),
newSnapshot("0.2", "0", "0"),
newSnapshot("1", "0", "0.1", "0.2"),
newSnapshot("2", "1", "1"),
newSnapshot("3", "2", "2"),
)
t.Run("check root", func(t *testing.T) {
total := tr.checkRoot(tr.attached["3"])
assert.Equal(t, 0, total)
})
t.Run("reduce", func(t *testing.T) {
tr.reduceTree()
assert.Equal(t, "3", tr.RootId())
var res []string
tr.IterateSkip(tr.RootId(), func(c *Change) (isContinue bool) {
res = append(res, c.Id)
return true
})
assert.Equal(t, []string{"3"}, res)
})
})
t.Run("check root many", func(t *testing.T) {
tr := new(Tree)
tr.Add(

20
commonspace/object/tree/objecttree/treereduce.go
View file

@ -1,6 +1,10 @@
package objecttree
import "math"
import (
"math"
"github.com/anyproto/any-sync/util/slice"
)
// clearPossibleRoots force removes any snapshots which can further be deemed as roots
func (t *Tree) clearPossibleRoots() {
@ -70,6 +74,20 @@ func (t *Tree) reduceTree() (res bool) {
minRoot *Change
minTotal = math.MaxInt
)
for _, r := range t.possibleRoots {
// if this is snapshot and next is also snapshot, then we don't need to take this one into account
if len(r.Next) == 1 && r.Next[0].IsSnapshot {
r.visited = true
}
}
t.possibleRoots = slice.DiscardFromSlice(t.possibleRoots, func(change *Change) bool {
if change.visited {
change.visited = false
return true
}
return false
})
// TODO: this can be further optimized by iterating the tree and checking the roots from top to bottom
// checking if we can reduce tree to other root
for _, root := range t.possibleRoots {

45
commonspace/object/tree/synctree/responsecollector.go
View file

@ -3,19 +3,25 @@ package synctree
import (
"context"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
"github.com/anyproto/any-sync/commonspace/object/tree/synctree/response"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/commonspace/sync/syncdeps"
)
type fullResponseCollector struct {
heads []string
root *treechangeproto.RawTreeChangeWithId
changes []*treechangeproto.RawTreeChangeWithId
deps BuildDeps
heads []string
root *treechangeproto.RawTreeChangeWithId
changes []*treechangeproto.RawTreeChangeWithId
objectTree objecttree.ObjectTree
}
func newFullResponseCollector() *fullResponseCollector {
return &fullResponseCollector{}
func newFullResponseCollector(deps BuildDeps) *fullResponseCollector {
return &fullResponseCollector{
deps: deps,
}
}
func (r *fullResponseCollector) CollectResponse(ctx context.Context, peerId, objectId string, resp syncdeps.Response) error {
@ -23,9 +29,32 @@ func (r *fullResponseCollector) CollectResponse(ctx context.Context, peerId, obj
if !ok {
return ErrUnexpectedResponseType
}
r.heads = treeResp.Heads
r.root = treeResp.Root
r.changes = append(r.changes, treeResp.Changes...)
if r.objectTree == nil {
createPayload := treestorage.TreeStorageCreatePayload{
RootRawChange: treeResp.Root,
Changes: treeResp.Changes,
Heads: treeResp.Heads,
}
validator := r.deps.ValidateObjectTree
if validator == nil {
validator = objecttree.ValidateRawTreeDefault
}
objTree, err := validator(createPayload, r.deps.SpaceStorage, r.deps.AclList)
if err != nil {
return err
}
r.objectTree = objTree
return nil
}
r.objectTree.Lock()
defer r.objectTree.Unlock()
_, err := r.objectTree.AddRawChanges(ctx, objecttree.RawChangesPayload{
NewHeads: treeResp.Heads,
RawChanges: treeResp.Changes,
})
if err != nil {
return err
}
return nil
}

72
commonspace/object/tree/synctree/responsecollector_test.go Normal file
View file

@ -0,0 +1,72 @@
package synctree
import (
"testing"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"github.com/anyproto/any-sync/commonspace/object/acl/list"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree/mock_objecttree"
"github.com/anyproto/any-sync/commonspace/object/tree/synctree/response"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
)
func TestFullResponseCollector_CollectResponse(t *testing.T) {
t.Run("no object tree", func(t *testing.T) {
ctrl := gomock.NewController(t)
objTree := mock_objecttree.NewMockObjectTree(ctrl)
defer ctrl.Finish()
testPayload := treestorage.TreeStorageCreatePayload{
RootRawChange: &treechangeproto.RawTreeChangeWithId{Id: "root"},
Changes: []*treechangeproto.RawTreeChangeWithId{{Id: "change"}},
Heads: []string{"head"},
}
resp := &response.Response{
SpaceId: "spaceId",
ObjectId: "objectId",
Heads: testPayload.Heads,
Changes: testPayload.Changes,
Root: testPayload.RootRawChange,
}
var validator objecttree.ValidatorFunc = func(payload treestorage.TreeStorageCreatePayload, storageCreator objecttree.TreeStorageCreator, aclList list.AclList) (ret objecttree.ObjectTree, err error) {
require.Equal(t, testPayload, payload)
return objTree, nil
}
coll := newFullResponseCollector(BuildDeps{
ValidateObjectTree: validator,
})
err := coll.CollectResponse(nil, "peerId", "objectId", resp)
require.NoError(t, err)
require.Equal(t, objTree, coll.objectTree)
})
t.Run("object tree exists", func(t *testing.T) {
ctrl := gomock.NewController(t)
objTree := mock_objecttree.NewMockObjectTree(ctrl)
defer ctrl.Finish()
testPayload := treestorage.TreeStorageCreatePayload{
RootRawChange: &treechangeproto.RawTreeChangeWithId{Id: "root"},
Changes: []*treechangeproto.RawTreeChangeWithId{{Id: "change"}},
Heads: []string{"head"},
}
resp := &response.Response{
SpaceId: "spaceId",
ObjectId: "objectId",
Heads: testPayload.Heads,
Changes: testPayload.Changes,
Root: testPayload.RootRawChange,
}
coll := newFullResponseCollector(BuildDeps{})
coll.objectTree = objTree
objTree.EXPECT().Lock()
objTree.EXPECT().Unlock()
objTree.EXPECT().AddRawChanges(nil, objecttree.RawChangesPayload{
NewHeads: testPayload.Heads,
RawChanges: testPayload.Changes,
}).Return(objecttree.AddResult{}, nil)
err := coll.CollectResponse(nil, "peerId", "objectId", resp)
require.NoError(t, err)
})
}

25
commonspace/object/tree/synctree/treeremotegetter.go
View file

@ -4,9 +4,6 @@ import (
"context"
"errors"
"go.uber.org/zap"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/net/peer"
)
@ -52,7 +49,7 @@ func (t treeRemoteGetter) getPeers(ctx context.Context) (peerIds []string, err e
}
func (t treeRemoteGetter) treeRequest(ctx context.Context, peerId string) (collector *fullResponseCollector, err error) {
collector = createCollector()
collector = createCollector(t.deps)
req := t.deps.SyncClient.CreateNewTreeRequest(peerId, t.treeId)
err = t.deps.SyncClient.SendTreeRequest(ctx, req, collector)
if err != nil {
@ -90,23 +87,5 @@ func (t treeRemoteGetter) getTree(ctx context.Context) (treeStorage treestorage.
return
}
payload := treestorage.TreeStorageCreatePayload{
RootRawChange: collector.root,
Changes: collector.changes,
Heads: collector.heads,
}
validatorFunc := t.deps.ValidateObjectTree
if validatorFunc == nil {
validatorFunc = objecttree.ValidateRawTreeBuildFunc
}
// basically building tree with in-memory storage and validating that it was without errors
log.With(zap.String("id", t.treeId)).DebugCtx(ctx, "validating tree")
newPayload, err := validatorFunc(payload, t.deps.BuildObjectTree, t.deps.AclList)
if err != nil {
return
}
// now we are sure that we can save it to the storage
treeStorage, err = t.deps.SpaceStorage.CreateTreeStorage(newPayload)
return
return collector.objectTree.Storage(), peerId, nil
}

12
commonspace/object/tree/synctree/treeremotegetter_test.go
View file

@ -55,8 +55,8 @@ func TestTreeRemoteGetter(t *testing.T) {
fx := newTreeRemoteGetterFixture(t)
defer fx.stop()
mockPeer := mock_peer.NewMockPeer(fx.ctrl)
coll := newFullResponseCollector()
createCollector = func() *fullResponseCollector {
coll := newFullResponseCollector(BuildDeps{})
createCollector = func(deps BuildDeps) *fullResponseCollector {
return coll
}
tCtx := peer.CtxWithPeerId(ctx, "*")
@ -74,8 +74,8 @@ func TestTreeRemoteGetter(t *testing.T) {
fx := newTreeRemoteGetterFixture(t)
defer fx.stop()
mockPeer := mock_peer.NewMockPeer(fx.ctrl)
coll := newFullResponseCollector()
createCollector = func() *fullResponseCollector {
coll := newFullResponseCollector(BuildDeps{})
createCollector = func(deps BuildDeps) *fullResponseCollector {
return coll
}
tCtx := peer.CtxWithPeerId(ctx, "*")
@ -91,8 +91,8 @@ func TestTreeRemoteGetter(t *testing.T) {
fx := newTreeRemoteGetterFixture(t)
defer fx.stop()
mockPeer := mock_peer.NewMockPeer(fx.ctrl)
coll := newFullResponseCollector()
createCollector = func() *fullResponseCollector {
coll := newFullResponseCollector(BuildDeps{})
createCollector = func(deps BuildDeps) *fullResponseCollector {
return coll
}
tCtx := peer.CtxWithPeerId(ctx, "*")

6
nameservice/nameserviceclient/nameserviceclient.go
View file

@ -71,15 +71,15 @@ func New() AnyNsClientService {
func (s *service) doClient(ctx context.Context, fn func(cl nsp.DRPCAnynsClient) error) error {
if len(s.nodeconf.NamingNodePeers()) == 0 {
log.Error("no namingNode peers configured")
return errors.New("no namingNode peers configured. Node config ID: " + s.nodeconf.Id())
log.Error("no ns peers configured. Maybe you're on a custom network. Node config ID: " + s.nodeconf.Id())
return errors.New("no namingNode peers configured. Maybe you're on a custom network. Node config ID: " + s.nodeconf.Id())
}
// it will try to connect to the Naming Node
// please enable "namingNode" type of node in the config (in the network.nodes array)
peer, err := s.pool.GetOneOf(ctx, s.nodeconf.NamingNodePeers())
if err != nil {
log.Error("failed to get a namingnode peer. maybe you're on a custom network", zap.Error(err))
log.Error("failed to get a namingnode peer", zap.Error(err))
return err
}

17
net/secureservice/credential.go
View file

@ -1,12 +1,15 @@
package secureservice
import (
"strings"
"go.uber.org/zap"
"golang.org/x/exp/slices"
"github.com/anyproto/any-sync/commonspace/object/accountdata"
"github.com/anyproto/any-sync/net/secureservice/handshake"
"github.com/anyproto/any-sync/net/secureservice/handshake/handshakeproto"
"github.com/anyproto/any-sync/util/crypto"
"go.uber.org/zap"
"golang.org/x/exp/slices"
)
func newNoVerifyChecker(protoVersion uint32, compatibleProtoVersions []uint32, clientVersion string) handshake.CredentialChecker {
@ -34,6 +37,11 @@ func (n noVerifyChecker) CheckCredential(remotePeerId string, cred *handshakepro
err = handshake.ErrIncompatibleVersion
return
}
// Hotfix for a bad version
if strings.Contains(cred.ClientVersion, "middle:v0.36.6") {
err = handshake.ErrIncompatibleVersion
return
}
return handshake.Result{
ProtoVersion: cred.Version,
ClientVersion: cred.ClientVersion,
@ -103,6 +111,11 @@ func (p *peerSignVerifier) CheckCredential(remotePeerId string, cred *handshakep
err = handshake.ErrInvalidCredentials
return
}
// Hotfix for a bad version
if strings.Contains(cred.ClientVersion, "middle:v0.36.6") {
err = handshake.ErrIncompatibleVersion
return
}
return handshake.Result{
Identity: msg.Identity,
ProtoVersion: cred.Version,

29
net/secureservice/credential_test.go
View file

@ -1,12 +1,14 @@
package secureservice
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anyproto/any-sync/commonspace/object/accountdata"
"github.com/anyproto/any-sync/net/secureservice/handshake"
"github.com/anyproto/any-sync/testutil/accounttest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"testing"
)
func TestPeerSignVerifier_CheckCredential(t *testing.T) {
@ -58,6 +60,27 @@ func TestIncompatibleVersion(t *testing.T) {
assert.EqualError(t, err, handshake.ErrInvalidCredentials.Error())
}
func TestIncompatibleVersion_Issue4423(t *testing.T) {
a1 := newTestAccData(t)
a2 := newTestAccData(t)
identity2, _ := a2.SignKey.GetPublic().Marshall()
cc1 := newPeerSignVerifier(1, []uint32{1}, "Linux:0.43.3/middle:v0.36.6/any-sync:v0.5.11", a1)
cc2 := newPeerSignVerifier(1, []uint32{1}, "test:v1", a2)
c1 := a2.PeerId
c2 := a1.PeerId
cr1 := cc1.MakeCredentials(c1)
cr2 := cc2.MakeCredentials(c2)
res, err := cc1.CheckCredential(c1, cr2)
assert.NoError(t, err)
assert.Equal(t, identity2, res.Identity)
_, err = cc2.CheckCredential(c2, cr1)
assert.ErrorIs(t, err, handshake.ErrIncompatibleVersion)
}
func newTestAccData(t *testing.T) *accountdata.AccountKeys {
as := accounttest.AccountTestService{}
require.NoError(t, as.Init(nil))

19
paymentservice/paymentserviceclient/paymentserviceclient.go
View file

@ -55,17 +55,30 @@ func New() AnyPpClientService {
return new(service)
}
/*
* Case 1: Custom network, no paymentProcessingNode peers ->
* This should not be called in custom networks! Please see what client of this service is doing.
* Otherwise will return: { "no payment processing peers configured. Maybe you're on a custom network" }
*
* Case 2: Anytype network, no paymentProcessingNode peers in config ->
* !!! This is a big issue, probably because of problems with nodeconf. Should be logged!
*
* Case 3: Anytype network, paymentProcessingNode peers in config, no connectivity ->
* This can happen due to network connectivity issues and it is OK.
* Will return:
* { "failed to get a paymentnode peer. maybe you're on a custom network","error":"unable to connect” }
*/
func (s *service) doClient(ctx context.Context, fn func(cl pp.DRPCAnyPaymentProcessingClient) error) error {
if len(s.nodeconf.PaymentProcessingNodePeers()) == 0 {
log.Error("no payment processing peers configured")
return errors.New("no paymentProcessingNode peers configured. Node config ID: " + s.nodeconf.Id())
log.Error("no payment processing peers configured. Maybe you're on a custom network. Node config ID: " + s.nodeconf.Id())
return errors.New("no paymentProcessingNode peers configured. Maybe you're on a custom network. Node config ID: " + s.nodeconf.Id())
}
// it will try to connect to the Payment Node
// please use "paymentProcessingNode" type of node in the config (in the network.nodes array)
peer, err := s.pool.GetOneOf(ctx, s.nodeconf.PaymentProcessingNodePeers())
if err != nil {
log.Error("failed to get a paymentnode peer. maybe you're on a custom network", zap.Error(err))
log.Error("failed to get a paymentnode peer", zap.Error(err))
return err
}