diff --git a/commonspace/object/tree/objecttree/changebuilder.go b/commonspace/object/tree/objecttree/changebuilder.go index cb22d249..2f10fca9 100644 --- a/commonspace/object/tree/objecttree/changebuilder.go +++ b/commonspace/object/tree/objecttree/changebuilder.go @@ -52,6 +52,18 @@ func (c *nonVerifiableChangeBuilder) Marshall(ch *Change) (raw *treechangeproto. return c.ChangeBuilder.Marshall(ch) } +type emptyDataChangeBuilder struct { + ChangeBuilder +} + +func (c *emptyDataChangeBuilder) Build(payload BuilderContent) (ch *Change, raw *treechangeproto.RawTreeChangeWithId, err error) { + panic("should not be called") +} + +func (c *emptyDataChangeBuilder) Marshall(ch *Change) (raw *treechangeproto.RawTreeChangeWithId, err error) { + panic("should not be called") +} + type ChangeBuilder interface { Unmarshall(rawIdChange *treechangeproto.RawTreeChangeWithId, verify bool) (ch *Change, err error) Build(payload BuilderContent) (ch *Change, raw *treechangeproto.RawTreeChangeWithId, err error) @@ -59,13 +71,28 @@ type ChangeBuilder interface { Marshall(ch *Change) (*treechangeproto.RawTreeChangeWithId, error) } +type newChangeFunc = func(id string, identity crypto.PubKey, ch *treechangeproto.TreeChange, signature []byte) *Change + type changeBuilder struct { rootChange *treechangeproto.RawTreeChangeWithId keys crypto.KeyStorage + newChange newChangeFunc +} + +func NewEmptyDataBuilder(keys crypto.KeyStorage, rootChange *treechangeproto.RawTreeChangeWithId) ChangeBuilder { + return &emptyDataChangeBuilder{&changeBuilder{ + rootChange: rootChange, + keys: keys, + newChange: func(id string, identity crypto.PubKey, ch *treechangeproto.TreeChange, signature []byte) *Change { + c := NewChange(id, identity, ch, signature) + c.Data = nil + return c + }, + }} } func NewChangeBuilder(keys crypto.KeyStorage, rootChange *treechangeproto.RawTreeChangeWithId) ChangeBuilder { - return &changeBuilder{keys: keys, rootChange: rootChange} + return &changeBuilder{keys: keys, rootChange: rootChange, newChange: NewChange} } func (c *changeBuilder) Unmarshall(rawIdChange *treechangeproto.RawTreeChangeWithId, verify bool) (ch *Change, err error) { @@ -197,7 +224,7 @@ func (c *changeBuilder) Build(payload BuilderContent) (ch *Change, rawIdChange * if err != nil { return } - ch = NewChange(id, payload.PrivKey.GetPublic(), change, signature) + ch = c.newChange(id, payload.PrivKey.GetPublic(), change, signature) rawIdChange = &treechangeproto.RawTreeChangeWithId{ RawChange: marshalledRawChange, Id: id, @@ -268,7 +295,7 @@ func (c *changeBuilder) unmarshallRawChange(raw *treechangeproto.RawTreeChange, if err != nil { return } - ch = NewChange(id, key, unmarshalled, raw.Signature) + ch = c.newChange(id, key, unmarshalled, raw.Signature) return } diff --git a/commonspace/object/tree/objecttree/objecttree.go b/commonspace/object/tree/objecttree/objecttree.go index 430329c2..5f2d98e4 100644 --- a/commonspace/object/tree/objecttree/objecttree.go +++ b/commonspace/object/tree/objecttree/objecttree.go @@ -621,19 +621,7 @@ func (ot *objectTree) ChangesAfterCommonSnapshot(theirPath, theirHeads []string) } } - if commonSnapshot == ot.tree.RootId() { - return ot.getChangesFromTree(theirHeads) - } else { - return ot.getChangesFromDB(commonSnapshot, theirHeads) - } -} - -func (ot *objectTree) getChangesFromTree(theirHeads []string) (rawChanges []*treechangeproto.RawTreeChangeWithId, err error) { - return ot.rawChangeLoader.LoadFromTree(ot.tree, theirHeads) -} - -func (ot *objectTree) getChangesFromDB(commonSnapshot string, theirHeads []string) (rawChanges []*treechangeproto.RawTreeChangeWithId, err error) { - return ot.rawChangeLoader.LoadFromStorage(commonSnapshot, ot.tree.headIds, theirHeads) + return ot.rawChangeLoader.Load(commonSnapshot, ot.tree, theirHeads) } func (ot *objectTree) snapshotPathIsActual() bool { diff --git a/commonspace/object/tree/objecttree/objecttreefactory.go b/commonspace/object/tree/objecttree/objecttreefactory.go index 0ab51ec1..5b9196e4 100644 --- a/commonspace/object/tree/objecttree/objecttreefactory.go +++ b/commonspace/object/tree/objecttree/objecttreefactory.go @@ -51,6 +51,22 @@ func verifiableTreeDeps( } } +func emptyDataTreeDeps( + rootChange *treechangeproto.RawTreeChangeWithId, + treeStorage treestorage.TreeStorage, + aclList list.AclList) objectTreeDeps { + changeBuilder := NewEmptyDataBuilder(crypto.NewKeyStorage(), rootChange) + treeBuilder := newTreeBuilder(treeStorage, changeBuilder) + return objectTreeDeps{ + changeBuilder: changeBuilder, + treeBuilder: treeBuilder, + treeStorage: treeStorage, + validator: newTreeValidator(), + rawChangeLoader: newStorageLoader(treeStorage, changeBuilder), + aclList: aclList, + } +} + func nonVerifiableTreeDeps( rootChange *treechangeproto.RawTreeChangeWithId, treeStorage treestorage.TreeStorage, @@ -80,6 +96,49 @@ func DeriveObjectTreeRoot(payload ObjectTreeCreatePayload, aclList list.AclList) return createObjectTreeRoot(payload, 0, nil, aclList) } +func BuildEmptyDataObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) { + rootChange, err := treeStorage.Root() + if err != nil { + return nil, err + } + deps := emptyDataTreeDeps(rootChange, treeStorage, aclList) + return buildObjectTree(deps) +} + +func BuildTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) { + root, _ := treeStorage.Root() + changeBuilder := &nonVerifiableChangeBuilder{ + ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), root), + } + deps := objectTreeDeps{ + changeBuilder: changeBuilder, + treeBuilder: newTreeBuilder(treeStorage, changeBuilder), + treeStorage: treeStorage, + rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder), + validator: &noOpTreeValidator{}, + aclList: aclList, + } + + return buildObjectTree(deps) +} + +func BuildEmptyDataTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) { + root, _ := treeStorage.Root() + changeBuilder := &nonVerifiableChangeBuilder{ + ChangeBuilder: NewEmptyDataBuilder(newMockKeyStorage(), root), + } + deps := objectTreeDeps{ + changeBuilder: changeBuilder, + treeBuilder: newTreeBuilder(treeStorage, changeBuilder), + treeStorage: treeStorage, + rawChangeLoader: newStorageLoader(treeStorage, changeBuilder), + validator: &noOpTreeValidator{}, + aclList: aclList, + } + + return buildObjectTree(deps) +} + func BuildObjectTree(treeStorage treestorage.TreeStorage, aclList list.AclList) (ObjectTree, error) { rootChange, err := treeStorage.Root() if err != nil { diff --git a/commonspace/object/tree/objecttree/rawloader.go b/commonspace/object/tree/objecttree/rawloader.go index cae32cb3..c9234b5b 100644 --- a/commonspace/object/tree/objecttree/rawloader.go +++ b/commonspace/object/tree/objecttree/rawloader.go @@ -9,8 +9,9 @@ import ( ) type rawChangeLoader struct { - treeStorage treestorage.TreeStorage - changeBuilder ChangeBuilder + treeStorage treestorage.TreeStorage + changeBuilder ChangeBuilder + alwaysFromStorage bool // buffers idStack []string @@ -23,6 +24,12 @@ type rawCacheEntry struct { position int } +func newStorageLoader(treeStorage treestorage.TreeStorage, changeBuilder ChangeBuilder) *rawChangeLoader { + loader := newRawChangeLoader(treeStorage, changeBuilder) + loader.alwaysFromStorage = true + return loader +} + func newRawChangeLoader(treeStorage treestorage.TreeStorage, changeBuilder ChangeBuilder) *rawChangeLoader { return &rawChangeLoader{ treeStorage: treeStorage, @@ -30,7 +37,15 @@ func newRawChangeLoader(treeStorage treestorage.TreeStorage, changeBuilder Chang } } -func (r *rawChangeLoader) LoadFromTree(t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { +func (r *rawChangeLoader) Load(commonSnapshot string, t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { + if commonSnapshot == t.root.Id && !r.alwaysFromStorage { + return r.loadFromTree(t, breakpoints) + } else { + return r.loadFromStorage(commonSnapshot, t.Heads(), breakpoints) + } +} + +func (r *rawChangeLoader) loadFromTree(t *Tree, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { var stack []*Change for _, h := range t.headIds { stack = append(stack, t.attached[h]) @@ -98,7 +113,7 @@ func (r *rawChangeLoader) LoadFromTree(t *Tree, breakpoints []string) ([]*treech return convert(results) } -func (r *rawChangeLoader) LoadFromStorage(commonSnapshot string, heads, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { +func (r *rawChangeLoader) loadFromStorage(commonSnapshot string, heads, breakpoints []string) ([]*treechangeproto.RawTreeChangeWithId, error) { // resetting cache r.cache = make(map[string]rawCacheEntry) defer func() { diff --git a/commonspace/object/tree/objecttree/testutils.go b/commonspace/object/tree/objecttree/testutils.go index 6385b9f7..fdcd4a9d 100644 --- a/commonspace/object/tree/objecttree/testutils.go +++ b/commonspace/object/tree/objecttree/testutils.go @@ -2,7 +2,6 @@ package objecttree import ( "fmt" - "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" "github.com/anytypeio/any-sync/commonspace/object/tree/treestorage" "github.com/anytypeio/any-sync/util/crypto" @@ -112,20 +111,3 @@ func (c *MockChangeCreator) CreateNewTreeStorage(treeId, aclHeadId string) trees treeStorage, _ := treestorage.NewInMemoryTreeStorage(root, []string{root.Id}, []*treechangeproto.RawTreeChangeWithId{root}) return treeStorage } - -func BuildTestableTree(aclList list.AclList, treeStorage treestorage.TreeStorage) (ObjectTree, error) { - root, _ := treeStorage.Root() - changeBuilder := &nonVerifiableChangeBuilder{ - ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), root), - } - deps := objectTreeDeps{ - changeBuilder: changeBuilder, - treeBuilder: newTreeBuilder(treeStorage, changeBuilder), - treeStorage: treeStorage, - rawChangeLoader: newRawChangeLoader(treeStorage, changeBuilder), - validator: &noOpTreeValidator{}, - aclList: aclList, - } - - return buildObjectTree(deps) -} diff --git a/commonspace/object/tree/synctree/syncclient_test.go b/commonspace/object/tree/synctree/syncprotocol_test.go similarity index 75% rename from commonspace/object/tree/synctree/syncclient_test.go rename to commonspace/object/tree/synctree/syncprotocol_test.go index ed6dd710..4b81b166 100644 --- a/commonspace/object/tree/synctree/syncclient_test.go +++ b/commonspace/object/tree/synctree/syncprotocol_test.go @@ -2,7 +2,6 @@ package synctree import ( "context" - "fmt" "github.com/anytypeio/any-sync/commonspace/object/accountdata" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" @@ -60,7 +59,24 @@ func TestEmptyClientGetsFullHistory(t *testing.T) { require.Len(t, fullResponseMsg.changes, 2) } -func TestRandomTreeMerge(t *testing.T) { +func TestRandomMerge(t *testing.T) { + var ( + rnd = rand.New(rand.NewSource(time.Now().Unix())) + levels = 20 + perLevel = 20 + ) + testTreeMerge(t, levels, perLevel, func() bool { + return true + }) + testTreeMerge(t, levels, perLevel, func() bool { + return false + }) + testTreeMerge(t, levels, perLevel, func() bool { + return rnd.Intn(10) > 8 + }) +} + +func testTreeMerge(t *testing.T, levels, perlevel int, isSnapshot func() bool) { treeId := "treeId" spaceId := "spaceId" keys, err := accountdata.NewRandom() @@ -68,17 +84,15 @@ func TestRandomTreeMerge(t *testing.T) { aclList, err := list.NewTestDerivedAcl(spaceId, keys) storage := createStorage(treeId, aclList) changeCreator := objecttree.NewMockChangeCreator() - rnd := rand.New(rand.NewSource(time.Now().Unix())) params := genParams{ prefix: "peer1", aclId: aclList.Id(), startIdx: 0, - levels: 10, + levels: levels, + perLevel: perlevel, snapshotId: treeId, prevHeads: []string{treeId}, - isSnapshot: func() bool { - return rnd.Intn(100) > 80 - }, + isSnapshot: isSnapshot, } initialRes := genChanges(changeCreator, params) err = storage.TransactionAdd(initialRes.changes, initialRes.heads) @@ -99,20 +113,20 @@ func TestRandomTreeMerge(t *testing.T) { NewHeads: initialRes.heads, RawChanges: initialRes.changes, }) - time.Sleep(1000 * time.Millisecond) + time.Sleep(100 * time.Millisecond) firstHeads := fx.handlers["peer1"].tree().Heads() secondHeads := fx.handlers["peer2"].tree().Heads() require.True(t, slice.UnsortedEquals(firstHeads, secondHeads)) params = genParams{ - prefix: "peer1", - aclId: aclList.Id(), - startIdx: 11, - levels: 10, + prefix: "peer1", + aclId: aclList.Id(), + startIdx: levels, + levels: levels, + perLevel: perlevel, + snapshotId: initialRes.snapshotId, prevHeads: initialRes.heads, - isSnapshot: func() bool { - return rnd.Intn(100) > 80 - }, + isSnapshot: isSnapshot, } peer1Res := genChanges(changeCreator, params) params.prefix = "peer2" @@ -125,10 +139,15 @@ func TestRandomTreeMerge(t *testing.T) { NewHeads: peer2Res.heads, RawChanges: peer2Res.changes, }) - time.Sleep(1000 * time.Millisecond) + time.Sleep(100 * time.Millisecond) fx.stop() - firstHeads = fx.handlers["peer1"].tree().Heads() - secondHeads = fx.handlers["peer2"].tree().Heads() - fmt.Println(firstHeads) - fmt.Println(secondHeads) + firstTree := fx.handlers["peer1"].tree() + secondTree := fx.handlers["peer2"].tree() + firstHeads = firstTree.Heads() + secondHeads = secondTree.Heads() + require.True(t, slice.UnsortedEquals(firstHeads, secondHeads)) + require.True(t, slice.UnsortedEquals(firstHeads, append(peer1Res.heads, peer2Res.heads...))) + firstStorage := firstTree.Storage().(*treestorage.InMemoryTreeStorage) + secondStorage := secondTree.Storage().(*treestorage.InMemoryTreeStorage) + require.True(t, firstStorage.Equal(secondStorage)) } diff --git a/commonspace/object/tree/synctree/utils_test.go b/commonspace/object/tree/synctree/utils_test.go index 0399d239..ff2f5c06 100644 --- a/commonspace/object/tree/synctree/utils_test.go +++ b/commonspace/object/tree/synctree/utils_test.go @@ -3,7 +3,6 @@ package synctree import ( "context" "fmt" - "github.com/anytypeio/any-sync/commonspace/object/accountdata" "github.com/anytypeio/any-sync/commonspace/object/acl/list" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" @@ -275,7 +274,7 @@ func createStorage(treeId string, aclList list.AclList) treestorage.TreeStorage } func createTestTree(aclList list.AclList, storage treestorage.TreeStorage) (objecttree.ObjectTree, error) { - return objecttree.BuildTestableTree(aclList, storage) + return objecttree.BuildEmptyDataTestableTree(aclList, storage) } type fixtureDeps struct { @@ -345,6 +344,7 @@ type genParams struct { aclId string startIdx int levels int + perLevel int snapshotId string prevHeads []string isSnapshot func() bool @@ -374,7 +374,7 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge snapshotId = newId continue } - perLevel := rnd.Intn(10) + perLevel := rnd.Intn(params.perLevel) if perLevel == 0 { perLevel = 1 } @@ -383,7 +383,6 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge usedIds = map[string]struct{}{} ) for j := 0; j < perLevel; j++ { - // if we didn't connect with all prev ones prevConns := rnd.Intn(len(prevHeads)) if prevConns == 0 { prevConns = 1 @@ -391,6 +390,7 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge rnd.Shuffle(len(prevHeads), func(i, j int) { prevHeads[i], prevHeads[j] = prevHeads[j], prevHeads[i] }) + // if we didn't connect with all prev ones if j == perLevel-1 && len(usedIds) != len(prevHeads) { var unusedIds []string for _, id := range prevHeads { @@ -417,30 +417,3 @@ func genChanges(creator *objecttree.MockChangeCreator, params genParams) (res ge res.snapshotId = snapshotId return } - -func TestGenChanges(t *testing.T) { - treeId := "treeId" - spaceId := "spaceId" - keys, err := accountdata.NewRandom() - require.NoError(t, err) - aclList, err := list.NewTestDerivedAcl(spaceId, keys) - storage := createStorage(treeId, aclList) - creator := objecttree.NewMockChangeCreator() - rnd := rand.New(rand.NewSource(time.Now().Unix())) - params := genParams{ - prefix: "peerId", - aclId: aclList.Id(), - startIdx: 0, - levels: 10, - snapshotId: treeId, - prevHeads: []string{treeId}, - isSnapshot: func() bool { - return rnd.Intn(100) > 80 - }, - } - res := genChanges(creator, params) - storage.TransactionAdd(res.changes, res.heads) - tr, err := createTestTree(aclList, storage) - require.NoError(t, err) - fmt.Println(tr.Debug(objecttree.NoOpDescriptionParser)) -} diff --git a/commonspace/object/tree/treestorage/inmemory.go b/commonspace/object/tree/treestorage/inmemory.go index 0f22037e..f5b192ba 100644 --- a/commonspace/object/tree/treestorage/inmemory.go +++ b/commonspace/object/tree/treestorage/inmemory.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anytypeio/any-sync/util/slice" "sync" ) @@ -105,3 +106,21 @@ func (t *InMemoryTreeStorage) Copy() *InMemoryTreeStorage { other, _ := NewInMemoryTreeStorage(t.root, t.heads, changes) return other.(*InMemoryTreeStorage) } + +func (t *InMemoryTreeStorage) Equal(other *InMemoryTreeStorage) bool { + if !slice.UnsortedEquals(t.heads, other.heads) { + return false + } + if len(t.changes) != len(other.changes) { + return false + } + for k, v := range t.changes { + if otherV, exists := other.changes[k]; exists { + if otherV.Id == v.Id { + continue + } + } + return false + } + return true +}