mirror of
https://github.com/anyproto/any-sync.git
synced 2025-06-08 14:07:02 +09:00
Merge pull request #337 from anyproto/GO-4433-optimize-tree-reduce
GO-4433: Optimize tree reduce/validation
This commit is contained in:
commit
b82af41535
12 changed files with 221 additions and 70 deletions
|
@ -18,6 +18,7 @@ var (
|
||||||
type Change struct {
|
type Change struct {
|
||||||
Next []*Change
|
Next []*Change
|
||||||
PreviousIds []string
|
PreviousIds []string
|
||||||
|
Previous []*Change
|
||||||
AclHeadId string
|
AclHeadId string
|
||||||
Id string
|
Id string
|
||||||
SnapshotId string
|
SnapshotId string
|
||||||
|
|
|
@ -57,7 +57,9 @@ func (l *loadIterator) NextBatch(maxSize int) (batch IteratorBatch, err error) {
|
||||||
batch.Heads = slice.DiscardFromSlice(batch.Heads, func(s string) bool {
|
batch.Heads = slice.DiscardFromSlice(batch.Heads, func(s string) bool {
|
||||||
return slices.Contains(c.PreviousIds, s)
|
return slices.Contains(c.PreviousIds, s)
|
||||||
})
|
})
|
||||||
|
if !slices.Contains(batch.Heads, c.Id) {
|
||||||
batch.Heads = append(batch.Heads, c.Id)
|
batch.Heads = append(batch.Heads, c.Id)
|
||||||
|
}
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
if curSize+rawEntry.size >= maxSize && len(batch.Batch) != 0 {
|
if curSize+rawEntry.size >= maxSize && len(batch.Batch) != 0 {
|
||||||
|
@ -75,7 +77,9 @@ func (l *loadIterator) NextBatch(maxSize int) (batch IteratorBatch, err error) {
|
||||||
batch.Heads = slice.DiscardFromSlice(batch.Heads, func(s string) bool {
|
batch.Heads = slice.DiscardFromSlice(batch.Heads, func(s string) bool {
|
||||||
return slices.Contains(c.PreviousIds, s)
|
return slices.Contains(c.PreviousIds, s)
|
||||||
})
|
})
|
||||||
|
if !slices.Contains(batch.Heads, c.Id) {
|
||||||
batch.Heads = append(batch.Heads, c.Id)
|
batch.Heads = append(batch.Heads, c.Id)
|
||||||
|
}
|
||||||
return true
|
return true
|
||||||
})
|
})
|
||||||
l.lastHeads = batch.Heads
|
l.lastHeads = batch.Heads
|
||||||
|
|
|
@ -323,13 +323,13 @@ func TestObjectTree(t *testing.T) {
|
||||||
bStore = aTree.Storage().(*treestorage.InMemoryTreeStorage).Copy()
|
bStore = aTree.Storage().(*treestorage.InMemoryTreeStorage).Copy()
|
||||||
root, _ = bStore.Root()
|
root, _ = bStore.Root()
|
||||||
heads, _ := bStore.Heads()
|
heads, _ := bStore.Heads()
|
||||||
filteredPayload, err := ValidateFilterRawTree(treestorage.TreeStorageCreatePayload{
|
newTree, err := ValidateFilterRawTree(treestorage.TreeStorageCreatePayload{
|
||||||
RootRawChange: root,
|
RootRawChange: root,
|
||||||
Changes: bStore.AllChanges(),
|
Changes: bStore.AllChanges(),
|
||||||
Heads: heads,
|
Heads: heads,
|
||||||
}, bAccount.Acl)
|
}, InMemoryStorageCreator{}, bAccount.Acl)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, 2, len(filteredPayload.Changes))
|
require.Equal(t, 2, len(newTree.Storage().(*treestorage.InMemoryTreeStorage).AllChanges()))
|
||||||
err = aTree.IterateRoot(func(change *Change, decrypted []byte) (any, error) {
|
err = aTree.IterateRoot(func(change *Change, decrypted []byte) (any, error) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}, func(change *Change) bool {
|
}, func(change *Change) bool {
|
||||||
|
@ -497,6 +497,7 @@ func TestObjectTree(t *testing.T) {
|
||||||
store, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root})
|
store, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root})
|
||||||
oTree, err := BuildObjectTree(store, aclList)
|
oTree, err := BuildObjectTree(store, aclList)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
emptyDataTreeDeps = nonVerifiableTreeDeps
|
||||||
err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
|
err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
|
||||||
RootRawChange: oTree.Header(),
|
RootRawChange: oTree.Header(),
|
||||||
Heads: []string{root.Id},
|
Heads: []string{root.Id},
|
||||||
|
@ -516,6 +517,7 @@ func TestObjectTree(t *testing.T) {
|
||||||
store, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root})
|
store, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root})
|
||||||
oTree, err := BuildObjectTree(store, aclList)
|
oTree, err := BuildObjectTree(store, aclList)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
emptyDataTreeDeps = nonVerifiableTreeDeps
|
||||||
err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
|
err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
|
||||||
RootRawChange: oTree.Header(),
|
RootRawChange: oTree.Header(),
|
||||||
Heads: []string{root.Id},
|
Heads: []string{root.Id},
|
||||||
|
@ -558,6 +560,7 @@ func TestObjectTree(t *testing.T) {
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
allChanges := oTree.Storage().(*treestorage.InMemoryTreeStorage).AllChanges()
|
allChanges := oTree.Storage().(*treestorage.InMemoryTreeStorage).AllChanges()
|
||||||
|
emptyDataTreeDeps = nonVerifiableTreeDeps
|
||||||
err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
|
err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
|
||||||
RootRawChange: oTree.Header(),
|
RootRawChange: oTree.Header(),
|
||||||
Heads: []string{oTree.Heads()[0]},
|
Heads: []string{oTree.Heads()[0]},
|
||||||
|
@ -587,6 +590,7 @@ func TestObjectTree(t *testing.T) {
|
||||||
}, aclList)
|
}, aclList)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
store, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root})
|
store, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root})
|
||||||
|
emptyDataTreeDeps = nonVerifiableTreeDeps
|
||||||
oTree, err := BuildObjectTree(store, aclList)
|
oTree, err := BuildObjectTree(store, aclList)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
_, err = oTree.AddContent(ctx, SignableChangeContent{
|
_, err = oTree.AddContent(ctx, SignableChangeContent{
|
||||||
|
@ -629,7 +633,7 @@ func TestObjectTree(t *testing.T) {
|
||||||
changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"),
|
changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"),
|
||||||
changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"),
|
changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"),
|
||||||
}
|
}
|
||||||
defaultObjectTreeDeps = nonVerifiableTreeDeps
|
emptyDataTreeDeps = nonVerifiableTreeDeps
|
||||||
err := ValidateRawTree(treestorage.TreeStorageCreatePayload{
|
err := ValidateRawTree(treestorage.TreeStorageCreatePayload{
|
||||||
RootRawChange: ctx.objTree.Header(),
|
RootRawChange: ctx.objTree.Header(),
|
||||||
Heads: []string{"3"},
|
Heads: []string{"3"},
|
||||||
|
@ -1476,7 +1480,7 @@ func TestObjectTree(t *testing.T) {
|
||||||
changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"),
|
changeCreator.CreateRaw("2", aclList.Head().Id, "0", false, "1"),
|
||||||
changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"),
|
changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"),
|
||||||
}
|
}
|
||||||
defaultObjectTreeDeps = nonVerifiableTreeDeps
|
emptyDataTreeDeps = nonVerifiableTreeDeps
|
||||||
err := ValidateRawTree(treestorage.TreeStorageCreatePayload{
|
err := ValidateRawTree(treestorage.TreeStorageCreatePayload{
|
||||||
RootRawChange: ctx.objTree.Header(),
|
RootRawChange: ctx.objTree.Header(),
|
||||||
Heads: []string{"3"},
|
Heads: []string{"3"},
|
||||||
|
@ -1493,7 +1497,7 @@ func TestObjectTree(t *testing.T) {
|
||||||
ctx.objTree.Header(),
|
ctx.objTree.Header(),
|
||||||
changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"),
|
changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"),
|
||||||
}
|
}
|
||||||
defaultObjectTreeDeps = nonVerifiableTreeDeps
|
emptyDataTreeDeps = nonVerifiableTreeDeps
|
||||||
err := ValidateRawTree(treestorage.TreeStorageCreatePayload{
|
err := ValidateRawTree(treestorage.TreeStorageCreatePayload{
|
||||||
RootRawChange: ctx.objTree.Header(),
|
RootRawChange: ctx.objTree.Header(),
|
||||||
Heads: []string{"3"},
|
Heads: []string{"3"},
|
||||||
|
|
|
@ -63,7 +63,9 @@ func verifiableTreeDeps(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func emptyDataTreeDeps(
|
var emptyDataTreeDeps = verifiableEmptyDataTreeDeps
|
||||||
|
|
||||||
|
func verifiableEmptyDataTreeDeps(
|
||||||
rootChange *treechangeproto.RawTreeChangeWithId,
|
rootChange *treechangeproto.RawTreeChangeWithId,
|
||||||
treeStorage treestorage.TreeStorage,
|
treeStorage treestorage.TreeStorage,
|
||||||
aclList list.AclList) objectTreeDeps {
|
aclList list.AclList) objectTreeDeps {
|
||||||
|
@ -156,6 +158,16 @@ func BuildKeyFilterableObjectTree(treeStorage treestorage.TreeStorage, aclList l
|
||||||
return buildObjectTree(deps)
|
return buildObjectTree(deps)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func BuildEmptyDataKeyFilterableObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) {
|
||||||
|
rootChange, err := treeStorage.Root()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
deps := emptyDataTreeDeps(rootChange, treeStorage, aclList)
|
||||||
|
deps.validator = newTreeValidator(true, true)
|
||||||
|
return buildObjectTree(deps)
|
||||||
|
}
|
||||||
|
|
||||||
func BuildObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) {
|
func BuildObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) {
|
||||||
rootChange, err := treeStorage.Root()
|
rootChange, err := treeStorage.Root()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -9,7 +9,17 @@ import (
|
||||||
"github.com/anyproto/any-sync/util/slice"
|
"github.com/anyproto/any-sync/util/slice"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ValidatorFunc func(payload treestorage.TreeStorageCreatePayload, buildFunc BuildObjectTreeFunc, aclList list.AclList) (retPayload treestorage.TreeStorageCreatePayload, err error)
|
type TreeStorageCreator interface {
|
||||||
|
CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (treestorage.TreeStorage, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type InMemoryStorageCreator struct{}
|
||||||
|
|
||||||
|
func (i InMemoryStorageCreator) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (treestorage.TreeStorage, error) {
|
||||||
|
return treestorage.NewInMemoryTreeStorage(payload.RootRawChange, payload.Heads, payload.Changes)
|
||||||
|
}
|
||||||
|
|
||||||
|
type ValidatorFunc func(payload treestorage.TreeStorageCreatePayload, storageCreator TreeStorageCreator, aclList list.AclList) (ret ObjectTree, err error)
|
||||||
|
|
||||||
type ObjectTreeValidator interface {
|
type ObjectTreeValidator interface {
|
||||||
// ValidateFullTree should always be entered while holding a read lock on AclList
|
// ValidateFullTree should always be entered while holding a read lock on AclList
|
||||||
|
@ -160,12 +170,15 @@ func (v *objectTreeValidator) validateChange(tree *Tree, aclList list.AclList, c
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func ValidateRawTreeBuildFunc(payload treestorage.TreeStorageCreatePayload, buildFunc BuildObjectTreeFunc, aclList list.AclList) (newPayload treestorage.TreeStorageCreatePayload, err error) {
|
func ValidateRawTreeDefault(payload treestorage.TreeStorageCreatePayload, storageCreator TreeStorageCreator, aclList list.AclList) (objTree ObjectTree, err error) {
|
||||||
treeStorage, err := treestorage.NewInMemoryTreeStorage(payload.RootRawChange, []string{payload.RootRawChange.Id}, nil)
|
treeStorage, err := storageCreator.CreateTreeStorage(treestorage.TreeStorageCreatePayload{
|
||||||
|
RootRawChange: payload.RootRawChange,
|
||||||
|
Heads: []string{payload.RootRawChange.Id},
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
tree, err := buildFunc(treeStorage, aclList)
|
tree, err := BuildEmptyDataObjectTree(treeStorage, aclList)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -179,33 +192,36 @@ func ValidateRawTreeBuildFunc(payload treestorage.TreeStorageCreatePayload, buil
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if !slice.UnsortedEquals(res.Heads, payload.Heads) {
|
if !slice.UnsortedEquals(res.Heads, payload.Heads) {
|
||||||
return payload, fmt.Errorf("heads mismatch: %v != %v, %w", res.Heads, payload.Heads, ErrHasInvalidChanges)
|
return nil, fmt.Errorf("heads mismatch: %v != %v, %w", res.Heads, payload.Heads, ErrHasInvalidChanges)
|
||||||
}
|
}
|
||||||
// if tree has only one change we still should check if the snapshot id is same as root
|
// if tree has only one change we still should check if the snapshot id is same as root
|
||||||
if IsEmptyDerivedTree(tree) {
|
if IsEmptyDerivedTree(tree) {
|
||||||
return payload, ErrDerived
|
return nil, ErrDerived
|
||||||
}
|
}
|
||||||
return payload, nil
|
return tree, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func ValidateFilterRawTree(payload treestorage.TreeStorageCreatePayload, aclList list.AclList) (retPayload treestorage.TreeStorageCreatePayload, err error) {
|
func ValidateFilterRawTree(payload treestorage.TreeStorageCreatePayload, storageCreator TreeStorageCreator, aclList list.AclList) (objTree ObjectTree, err error) {
|
||||||
aclList.RLock()
|
aclList.RLock()
|
||||||
if !aclList.AclState().HadReadPermissions(aclList.AclState().Identity()) {
|
if !aclList.AclState().HadReadPermissions(aclList.AclState().Identity()) {
|
||||||
aclList.RUnlock()
|
aclList.RUnlock()
|
||||||
return payload, list.ErrNoReadKey
|
return nil, list.ErrNoReadKey
|
||||||
}
|
}
|
||||||
aclList.RUnlock()
|
aclList.RUnlock()
|
||||||
treeStorage, err := treestorage.NewInMemoryTreeStorage(payload.RootRawChange, []string{payload.RootRawChange.Id}, nil)
|
treeStorage, err := storageCreator.CreateTreeStorage(treestorage.TreeStorageCreatePayload{
|
||||||
|
RootRawChange: payload.RootRawChange,
|
||||||
|
Heads: []string{payload.RootRawChange.Id},
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
tree, err := BuildKeyFilterableObjectTree(treeStorage, aclList)
|
tree, err := BuildEmptyDataKeyFilterableObjectTree(treeStorage, aclList)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
tree.Lock()
|
tree.Lock()
|
||||||
defer tree.Unlock()
|
defer tree.Unlock()
|
||||||
res, err := tree.AddRawChanges(context.Background(), RawChangesPayload{
|
_, err = tree.AddRawChanges(context.Background(), RawChangesPayload{
|
||||||
NewHeads: payload.Heads,
|
NewHeads: payload.Heads,
|
||||||
RawChanges: payload.Changes,
|
RawChanges: payload.Changes,
|
||||||
})
|
})
|
||||||
|
@ -213,16 +229,12 @@ func ValidateFilterRawTree(payload treestorage.TreeStorageCreatePayload, aclList
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if IsEmptyTree(tree) {
|
if IsEmptyTree(tree) {
|
||||||
return payload, ErrNoChangeInTree
|
return nil, ErrNoChangeInTree
|
||||||
}
|
}
|
||||||
return treestorage.TreeStorageCreatePayload{
|
return tree, nil
|
||||||
RootRawChange: payload.RootRawChange,
|
|
||||||
Heads: res.Heads,
|
|
||||||
Changes: treeStorage.(*treestorage.InMemoryTreeStorage).AllChanges(),
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func ValidateRawTree(payload treestorage.TreeStorageCreatePayload, aclList list.AclList) (err error) {
|
func ValidateRawTree(payload treestorage.TreeStorageCreatePayload, aclList list.AclList) (err error) {
|
||||||
_, err = ValidateRawTreeBuildFunc(payload, BuildObjectTree, aclList)
|
_, err = ValidateRawTreeDefault(payload, InMemoryStorageCreator{}, aclList)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -266,6 +266,7 @@ func (t *Tree) attach(c *Change, newEl bool) {
|
||||||
for _, id := range c.PreviousIds {
|
for _, id := range c.PreviousIds {
|
||||||
// prev id must already be attached if we attach this id, so we don't need to check if it exists
|
// prev id must already be attached if we attach this id, so we don't need to check if it exists
|
||||||
prev := t.attached[id]
|
prev := t.attached[id]
|
||||||
|
c.Previous = append(c.Previous, prev)
|
||||||
// appending c to next changes of all previous changes
|
// appending c to next changes of all previous changes
|
||||||
if len(prev.Next) == 0 || prev.Next[len(prev.Next)-1].Id <= c.Id {
|
if len(prev.Next) == 0 || prev.Next[len(prev.Next)-1].Id <= c.Id {
|
||||||
prev.Next = append(prev.Next, c)
|
prev.Next = append(prev.Next, c)
|
||||||
|
@ -341,10 +342,8 @@ func (t *Tree) dfsPrev(stack []*Change, breakpoints []string, visit func(ch *Cha
|
||||||
ch.visited = true
|
ch.visited = true
|
||||||
t.visitedBuf = append(t.visitedBuf, ch)
|
t.visitedBuf = append(t.visitedBuf, ch)
|
||||||
|
|
||||||
for _, prevId := range ch.PreviousIds {
|
for _, prevCh := range ch.Previous {
|
||||||
prevCh, exists := t.attached[prevId]
|
if !prevCh.visited {
|
||||||
// here the only time it wouldn't exist if we are at the tree root
|
|
||||||
if exists && !prevCh.visited {
|
|
||||||
stack = append(stack, prevCh)
|
stack = append(stack, prevCh)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -200,6 +200,31 @@ func TestTree_CheckRootReduce(t *testing.T) {
|
||||||
assert.Equal(t, []string{"10", "last"}, res)
|
assert.Equal(t, []string{"10", "last"}, res)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
t.Run("snapshots in line", func(t *testing.T) {
|
||||||
|
tr := new(Tree)
|
||||||
|
tr.Add(
|
||||||
|
newSnapshot("0", ""),
|
||||||
|
newSnapshot("0.1", "0", "0"),
|
||||||
|
newSnapshot("0.2", "0", "0"),
|
||||||
|
newSnapshot("1", "0", "0.1", "0.2"),
|
||||||
|
newSnapshot("2", "1", "1"),
|
||||||
|
newSnapshot("3", "2", "2"),
|
||||||
|
)
|
||||||
|
t.Run("check root", func(t *testing.T) {
|
||||||
|
total := tr.checkRoot(tr.attached["3"])
|
||||||
|
assert.Equal(t, 0, total)
|
||||||
|
})
|
||||||
|
t.Run("reduce", func(t *testing.T) {
|
||||||
|
tr.reduceTree()
|
||||||
|
assert.Equal(t, "3", tr.RootId())
|
||||||
|
var res []string
|
||||||
|
tr.IterateSkip(tr.RootId(), func(c *Change) (isContinue bool) {
|
||||||
|
res = append(res, c.Id)
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
assert.Equal(t, []string{"3"}, res)
|
||||||
|
})
|
||||||
|
})
|
||||||
t.Run("check root many", func(t *testing.T) {
|
t.Run("check root many", func(t *testing.T) {
|
||||||
tr := new(Tree)
|
tr := new(Tree)
|
||||||
tr.Add(
|
tr.Add(
|
||||||
|
|
|
@ -1,6 +1,10 @@
|
||||||
package objecttree
|
package objecttree
|
||||||
|
|
||||||
import "math"
|
import (
|
||||||
|
"math"
|
||||||
|
|
||||||
|
"github.com/anyproto/any-sync/util/slice"
|
||||||
|
)
|
||||||
|
|
||||||
// clearPossibleRoots force removes any snapshots which can further be deemed as roots
|
// clearPossibleRoots force removes any snapshots which can further be deemed as roots
|
||||||
func (t *Tree) clearPossibleRoots() {
|
func (t *Tree) clearPossibleRoots() {
|
||||||
|
@ -70,6 +74,20 @@ func (t *Tree) reduceTree() (res bool) {
|
||||||
minRoot *Change
|
minRoot *Change
|
||||||
minTotal = math.MaxInt
|
minTotal = math.MaxInt
|
||||||
)
|
)
|
||||||
|
for _, r := range t.possibleRoots {
|
||||||
|
// if this is snapshot and next is also snapshot, then we don't need to take this one into account
|
||||||
|
if len(r.Next) == 1 && r.Next[0].IsSnapshot {
|
||||||
|
r.visited = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.possibleRoots = slice.DiscardFromSlice(t.possibleRoots, func(change *Change) bool {
|
||||||
|
if change.visited {
|
||||||
|
change.visited = false
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
})
|
||||||
|
// TODO: this can be further optimized by iterating the tree and checking the roots from top to bottom
|
||||||
|
|
||||||
// checking if we can reduce tree to other root
|
// checking if we can reduce tree to other root
|
||||||
for _, root := range t.possibleRoots {
|
for _, root := range t.possibleRoots {
|
||||||
|
|
|
@ -3,19 +3,25 @@ package synctree
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
|
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
|
||||||
"github.com/anyproto/any-sync/commonspace/object/tree/synctree/response"
|
"github.com/anyproto/any-sync/commonspace/object/tree/synctree/response"
|
||||||
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||||
|
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||||
"github.com/anyproto/any-sync/commonspace/sync/syncdeps"
|
"github.com/anyproto/any-sync/commonspace/sync/syncdeps"
|
||||||
)
|
)
|
||||||
|
|
||||||
type fullResponseCollector struct {
|
type fullResponseCollector struct {
|
||||||
|
deps BuildDeps
|
||||||
heads []string
|
heads []string
|
||||||
root *treechangeproto.RawTreeChangeWithId
|
root *treechangeproto.RawTreeChangeWithId
|
||||||
changes []*treechangeproto.RawTreeChangeWithId
|
changes []*treechangeproto.RawTreeChangeWithId
|
||||||
|
objectTree objecttree.ObjectTree
|
||||||
}
|
}
|
||||||
|
|
||||||
func newFullResponseCollector() *fullResponseCollector {
|
func newFullResponseCollector(deps BuildDeps) *fullResponseCollector {
|
||||||
return &fullResponseCollector{}
|
return &fullResponseCollector{
|
||||||
|
deps: deps,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *fullResponseCollector) CollectResponse(ctx context.Context, peerId, objectId string, resp syncdeps.Response) error {
|
func (r *fullResponseCollector) CollectResponse(ctx context.Context, peerId, objectId string, resp syncdeps.Response) error {
|
||||||
|
@ -23,9 +29,30 @@ func (r *fullResponseCollector) CollectResponse(ctx context.Context, peerId, obj
|
||||||
if !ok {
|
if !ok {
|
||||||
return ErrUnexpectedResponseType
|
return ErrUnexpectedResponseType
|
||||||
}
|
}
|
||||||
r.heads = treeResp.Heads
|
if r.objectTree == nil {
|
||||||
r.root = treeResp.Root
|
createPayload := treestorage.TreeStorageCreatePayload{
|
||||||
r.changes = append(r.changes, treeResp.Changes...)
|
RootRawChange: treeResp.Root,
|
||||||
|
Changes: treeResp.Changes,
|
||||||
|
Heads: treeResp.Heads,
|
||||||
|
}
|
||||||
|
validator := r.deps.ValidateObjectTree
|
||||||
|
if validator == nil {
|
||||||
|
validator = objecttree.ValidateRawTreeDefault
|
||||||
|
}
|
||||||
|
objTree, err := validator(createPayload, r.deps.SpaceStorage, r.deps.AclList)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
r.objectTree = objTree
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
_, err := r.objectTree.AddRawChanges(ctx, objecttree.RawChangesPayload{
|
||||||
|
NewHeads: treeResp.Heads,
|
||||||
|
RawChanges: treeResp.Changes,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
70
commonspace/object/tree/synctree/responsecollector_test.go
Normal file
70
commonspace/object/tree/synctree/responsecollector_test.go
Normal file
|
@ -0,0 +1,70 @@
|
||||||
|
package synctree
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"go.uber.org/mock/gomock"
|
||||||
|
|
||||||
|
"github.com/anyproto/any-sync/commonspace/object/acl/list"
|
||||||
|
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
|
||||||
|
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree/mock_objecttree"
|
||||||
|
"github.com/anyproto/any-sync/commonspace/object/tree/synctree/response"
|
||||||
|
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||||
|
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFullResponseCollector_CollectResponse(t *testing.T) {
|
||||||
|
t.Run("no object tree", func(t *testing.T) {
|
||||||
|
ctrl := gomock.NewController(t)
|
||||||
|
objTree := mock_objecttree.NewMockObjectTree(ctrl)
|
||||||
|
defer ctrl.Finish()
|
||||||
|
testPayload := treestorage.TreeStorageCreatePayload{
|
||||||
|
RootRawChange: &treechangeproto.RawTreeChangeWithId{Id: "root"},
|
||||||
|
Changes: []*treechangeproto.RawTreeChangeWithId{{Id: "change"}},
|
||||||
|
Heads: []string{"head"},
|
||||||
|
}
|
||||||
|
resp := &response.Response{
|
||||||
|
SpaceId: "spaceId",
|
||||||
|
ObjectId: "objectId",
|
||||||
|
Heads: testPayload.Heads,
|
||||||
|
Changes: testPayload.Changes,
|
||||||
|
Root: testPayload.RootRawChange,
|
||||||
|
}
|
||||||
|
var validator objecttree.ValidatorFunc = func(payload treestorage.TreeStorageCreatePayload, storageCreator objecttree.TreeStorageCreator, aclList list.AclList) (ret objecttree.ObjectTree, err error) {
|
||||||
|
require.Equal(t, testPayload, payload)
|
||||||
|
return objTree, nil
|
||||||
|
}
|
||||||
|
coll := newFullResponseCollector(BuildDeps{
|
||||||
|
ValidateObjectTree: validator,
|
||||||
|
})
|
||||||
|
err := coll.CollectResponse(nil, "peerId", "objectId", resp)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, objTree, coll.objectTree)
|
||||||
|
})
|
||||||
|
t.Run("object tree exists", func(t *testing.T) {
|
||||||
|
ctrl := gomock.NewController(t)
|
||||||
|
objTree := mock_objecttree.NewMockObjectTree(ctrl)
|
||||||
|
defer ctrl.Finish()
|
||||||
|
testPayload := treestorage.TreeStorageCreatePayload{
|
||||||
|
RootRawChange: &treechangeproto.RawTreeChangeWithId{Id: "root"},
|
||||||
|
Changes: []*treechangeproto.RawTreeChangeWithId{{Id: "change"}},
|
||||||
|
Heads: []string{"head"},
|
||||||
|
}
|
||||||
|
resp := &response.Response{
|
||||||
|
SpaceId: "spaceId",
|
||||||
|
ObjectId: "objectId",
|
||||||
|
Heads: testPayload.Heads,
|
||||||
|
Changes: testPayload.Changes,
|
||||||
|
Root: testPayload.RootRawChange,
|
||||||
|
}
|
||||||
|
coll := newFullResponseCollector(BuildDeps{})
|
||||||
|
coll.objectTree = objTree
|
||||||
|
objTree.EXPECT().AddRawChanges(nil, objecttree.RawChangesPayload{
|
||||||
|
NewHeads: testPayload.Heads,
|
||||||
|
RawChanges: testPayload.Changes,
|
||||||
|
}).Return(objecttree.AddResult{}, nil)
|
||||||
|
err := coll.CollectResponse(nil, "peerId", "objectId", resp)
|
||||||
|
require.NoError(t, err)
|
||||||
|
})
|
||||||
|
}
|
|
@ -4,9 +4,6 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
"go.uber.org/zap"
|
|
||||||
|
|
||||||
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
|
|
||||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||||
"github.com/anyproto/any-sync/net/peer"
|
"github.com/anyproto/any-sync/net/peer"
|
||||||
)
|
)
|
||||||
|
@ -52,7 +49,7 @@ func (t treeRemoteGetter) getPeers(ctx context.Context) (peerIds []string, err e
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t treeRemoteGetter) treeRequest(ctx context.Context, peerId string) (collector *fullResponseCollector, err error) {
|
func (t treeRemoteGetter) treeRequest(ctx context.Context, peerId string) (collector *fullResponseCollector, err error) {
|
||||||
collector = createCollector()
|
collector = createCollector(t.deps)
|
||||||
req := t.deps.SyncClient.CreateNewTreeRequest(peerId, t.treeId)
|
req := t.deps.SyncClient.CreateNewTreeRequest(peerId, t.treeId)
|
||||||
err = t.deps.SyncClient.SendTreeRequest(ctx, req, collector)
|
err = t.deps.SyncClient.SendTreeRequest(ctx, req, collector)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -90,23 +87,5 @@ func (t treeRemoteGetter) getTree(ctx context.Context) (treeStorage treestorage.
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
payload := treestorage.TreeStorageCreatePayload{
|
return collector.objectTree.Storage(), peerId, nil
|
||||||
RootRawChange: collector.root,
|
|
||||||
Changes: collector.changes,
|
|
||||||
Heads: collector.heads,
|
|
||||||
}
|
|
||||||
|
|
||||||
validatorFunc := t.deps.ValidateObjectTree
|
|
||||||
if validatorFunc == nil {
|
|
||||||
validatorFunc = objecttree.ValidateRawTreeBuildFunc
|
|
||||||
}
|
|
||||||
// basically building tree with in-memory storage and validating that it was without errors
|
|
||||||
log.With(zap.String("id", t.treeId)).DebugCtx(ctx, "validating tree")
|
|
||||||
newPayload, err := validatorFunc(payload, t.deps.BuildObjectTree, t.deps.AclList)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// now we are sure that we can save it to the storage
|
|
||||||
treeStorage, err = t.deps.SpaceStorage.CreateTreeStorage(newPayload)
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -55,8 +55,8 @@ func TestTreeRemoteGetter(t *testing.T) {
|
||||||
fx := newTreeRemoteGetterFixture(t)
|
fx := newTreeRemoteGetterFixture(t)
|
||||||
defer fx.stop()
|
defer fx.stop()
|
||||||
mockPeer := mock_peer.NewMockPeer(fx.ctrl)
|
mockPeer := mock_peer.NewMockPeer(fx.ctrl)
|
||||||
coll := newFullResponseCollector()
|
coll := newFullResponseCollector(BuildDeps{})
|
||||||
createCollector = func() *fullResponseCollector {
|
createCollector = func(deps BuildDeps) *fullResponseCollector {
|
||||||
return coll
|
return coll
|
||||||
}
|
}
|
||||||
tCtx := peer.CtxWithPeerId(ctx, "*")
|
tCtx := peer.CtxWithPeerId(ctx, "*")
|
||||||
|
@ -74,8 +74,8 @@ func TestTreeRemoteGetter(t *testing.T) {
|
||||||
fx := newTreeRemoteGetterFixture(t)
|
fx := newTreeRemoteGetterFixture(t)
|
||||||
defer fx.stop()
|
defer fx.stop()
|
||||||
mockPeer := mock_peer.NewMockPeer(fx.ctrl)
|
mockPeer := mock_peer.NewMockPeer(fx.ctrl)
|
||||||
coll := newFullResponseCollector()
|
coll := newFullResponseCollector(BuildDeps{})
|
||||||
createCollector = func() *fullResponseCollector {
|
createCollector = func(deps BuildDeps) *fullResponseCollector {
|
||||||
return coll
|
return coll
|
||||||
}
|
}
|
||||||
tCtx := peer.CtxWithPeerId(ctx, "*")
|
tCtx := peer.CtxWithPeerId(ctx, "*")
|
||||||
|
@ -91,8 +91,8 @@ func TestTreeRemoteGetter(t *testing.T) {
|
||||||
fx := newTreeRemoteGetterFixture(t)
|
fx := newTreeRemoteGetterFixture(t)
|
||||||
defer fx.stop()
|
defer fx.stop()
|
||||||
mockPeer := mock_peer.NewMockPeer(fx.ctrl)
|
mockPeer := mock_peer.NewMockPeer(fx.ctrl)
|
||||||
coll := newFullResponseCollector()
|
coll := newFullResponseCollector(BuildDeps{})
|
||||||
createCollector = func() *fullResponseCollector {
|
createCollector = func(deps BuildDeps) *fullResponseCollector {
|
||||||
return coll
|
return coll
|
||||||
}
|
}
|
||||||
tCtx := peer.CtxWithPeerId(ctx, "*")
|
tCtx := peer.CtxWithPeerId(ctx, "*")
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue