anyproto/any-sync
1
0
Fork 0
mirror of https://github.com/anyproto/any-sync.git synced 2025-06-09 17:45:03 +09:00
Code Activity

Make better dependencies for object tree

Browse source
This commit is contained in:
mcrakhman 2022-09-06 18:08:24 +02:00 • committed by Mikhail Iudin
parent d3e62b418a
commit 266fd9436c
No known key found for this signature in database
GPG key ID: FAAAA8BAABDFF1C0
7 changed files with 238 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

119
pkg/acl/tree/changebuilder.go Normal file
View file

@ -0,0 +1,119 @@
package tree
import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/cid"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/asymmetric/signingkey"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/keys/symmetric"
"github.com/gogo/protobuf/proto"
"time"
)
const componentBuilder = "tree.changebuilder"
type BuilderContent struct {
treeHeadIds []string
aclHeadId string
snapshotBaseId string
currentReadKeyHash uint64
identity string
isSnapshot bool
signingKey signingkey.PrivKey
readKey *symmetric.Key
content proto.Marshaler
}
type ChangeBuilder interface {
ConvertFromRaw(rawChange *aclpb.RawChange) (ch *Change, err error)
ConvertFromRawAndVerify(rawChange *aclpb.RawChange) (ch *Change, err error)
BuildContent(payload BuilderContent) (ch *Change, raw *aclpb.RawChange, err error)
}
type changeBuilder struct {
keys *keychain
}
func newChangeBuilder(keys *keychain) *changeBuilder {
return &changeBuilder{keys: keys}
}
func (c *changeBuilder) ConvertFromRaw(rawChange *aclpb.RawChange) (ch *Change, err error) {
unmarshalled := &aclpb.Change{}
err = proto.Unmarshal(rawChange.Payload, unmarshalled)
if err != nil {
return nil, err
}
ch = NewChange(rawChange.Id, unmarshalled, rawChange.Signature)
return
}
func (c *changeBuilder) ConvertFromRawAndVerify(rawChange *aclpb.RawChange) (ch *Change, err error) {
unmarshalled := &aclpb.Change{}
ch, err = c.ConvertFromRaw(rawChange)
if err != nil {
return nil, err
}
identityKey, err := c.keys.getOrAdd(unmarshalled.Identity)
if err != nil {
return
}
res, err := identityKey.Verify(rawChange.Payload, rawChange.Signature)
if err != nil {
return
}
if !res {
err = ErrIncorrectSignature
}
return
}
func (c *changeBuilder) BuildContent(payload BuilderContent) (ch *Change, raw *aclpb.RawChange, err error) {
aclChange := &aclpb.Change{
TreeHeadIds: payload.treeHeadIds,
AclHeadId: payload.aclHeadId,
SnapshotBaseId: payload.snapshotBaseId,
CurrentReadKeyHash: payload.currentReadKeyHash,
Timestamp: int64(time.Now().Nanosecond()),
Identity: payload.identity,
IsSnapshot: payload.isSnapshot,
}
marshalledData, err := payload.content.Marshal()
if err != nil {
return
}
encrypted, err := payload.readKey.Encrypt(marshalledData)
if err != nil {
return
}
aclChange.ChangesData = encrypted
fullMarshalledChange, err := proto.Marshal(aclChange)
if err != nil {
return
}
signature, err := payload.signingKey.Sign(fullMarshalledChange)
if err != nil {
return
}
id, err := cid.NewCIDFromBytes(fullMarshalledChange)
if err != nil {
return
}
ch = NewChange(id, aclChange, signature)
ch.ParsedModel = payload.content
raw = &aclpb.RawChange{
Payload: fullMarshalledChange,
Signature: signature,
Id: id,
}
return
}

14
pkg/acl/tree/changevalidator.go
View file

@ -6,19 +6,19 @@ import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/list"
)
type DocTreeValidator interface {
type ObjectTreeValidator interface {
ValidateTree(tree *Tree, aclList list.ACLList) error
}
type docTreeValidator struct{}
type objectTreeValidator struct{}
func newTreeValidator() DocTreeValidator {
return &docTreeValidator{}
func newTreeValidator() ObjectTreeValidator {
return &objectTreeValidator{}
}
func (v *docTreeValidator) ValidateTree(tree *Tree, aclList list.ACLList) (err error) {
// TODO: add validation logic where we check that the change refers to correct acl heads
// that means that more recent changes should refer to more recent acl heads
func (v *objectTreeValidator) ValidateTree(tree *Tree, aclList list.ACLList) (err error) {
aclList.RLock()
defer aclList.RUnlock()
var (
perm list.UserPermissionPair
state = aclList.ACLState()

1
pkg/acl/tree/keychain.go
View file

@ -13,6 +13,7 @@ type keychain struct {
func newKeychain() *keychain {
return &keychain{
decoder: signingkey.NewEDPubKeyDecoder(),
keys: make(map[string]signingkey.PubKey),
}
}

161
pkg/acl/tree/objecttree.go
View file

@ -6,12 +6,9 @@ import (
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/aclchanges/aclpb"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/list"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/cid"
"github.com/anytypeio/go-anytype-infrastructure-experiments/util/slice"
"github.com/gogo/protobuf/proto"
"go.uber.org/zap"
"sync"
"time"
)
type ObjectTreeUpdateListener interface {
@ -64,24 +61,24 @@ type ObjectTree interface {
Storage() storage.TreeStorage
DebugDump() (string, error)
AddContent(ctx context.Context, aclList list.ACLList, content SignableChangeContent) (*aclpb.RawChange, error)
AddRawChanges(ctx context.Context, aclList list.ACLList, changes ...*aclpb.RawChange) (AddResult, error)
AddContent(ctx context.Context, content SignableChangeContent) (*aclpb.RawChange, error)
AddRawChanges(ctx context.Context, changes ...*aclpb.RawChange) (AddResult, error)
Close() error
}
type objectTree struct {
treeStorage storage.TreeStorage
changeBuilder ChangeBuilder
updateListener ObjectTreeUpdateListener
validator ObjectTreeValidator
treeBuilder *treeBuilder
aclList list.ACLList
id string
header *aclpb.Header
tree *Tree
treeBuilder *treeBuilder
validator DocTreeValidator
kch *keychain
// buffers
difSnapshotBuf []*aclpb.RawChange
tmpChangesBuf []*Change
@ -93,26 +90,52 @@ type objectTree struct {
sync.RWMutex
}
func BuildObjectTree(treeStorage storage.TreeStorage, listener ObjectTreeUpdateListener, aclList list.ACLList) (ObjectTree, error) {
treeBuilder := newTreeBuilder(treeStorage)
validator := newTreeValidator()
type objectTreeDeps struct {
changeBuilder ChangeBuilder
treeBuilder *treeBuilder
treeStorage storage.TreeStorage
updateListener ObjectTreeUpdateListener
validator ObjectTreeValidator
aclList list.ACLList
}
objTree := &objectTree{
treeStorage: treeStorage,
tree: nil,
func defaultObjectTreeDeps(
treeStorage storage.TreeStorage,
listener ObjectTreeUpdateListener,
aclList list.ACLList) objectTreeDeps {
keychain := newKeychain()
changeBuilder := newChangeBuilder(keychain)
treeBuilder := newTreeBuilder(treeStorage, changeBuilder)
return objectTreeDeps{
changeBuilder: changeBuilder,
treeBuilder: treeBuilder,
validator: validator,
treeStorage: treeStorage,
updateListener: listener,
validator: newTreeValidator(),
aclList: aclList,
}
}
func buildObjectTree(deps objectTreeDeps) (ObjectTree, error) {
objTree := &objectTree{
treeStorage: deps.treeStorage,
updateListener: deps.updateListener,
treeBuilder: deps.treeBuilder,
validator: deps.validator,
aclList: deps.aclList,
changeBuilder: deps.changeBuilder,
tree: nil,
tmpChangesBuf: make([]*Change, 0, 10),
difSnapshotBuf: make([]*aclpb.RawChange, 0, 10),
notSeenIdxBuf: make([]int, 0, 10),
kch: newKeychain(),
}
err := objTree.rebuildFromStorage(aclList, nil)
err := objTree.rebuildFromStorage(nil)
if err != nil {
return nil, err
}
storageHeads, err := treeStorage.Heads()
storageHeads, err := objTree.treeStorage.Heads()
if err != nil {
return nil, err
}
@ -123,30 +146,35 @@ func BuildObjectTree(treeStorage storage.TreeStorage, listener ObjectTreeUpdateL
if !slice.UnsortedEquals(storageHeads, objTree.tree.Heads()) {
log.With(zap.Strings("storage", storageHeads), zap.Strings("rebuilt", objTree.tree.Heads())).
Errorf("the heads in storage and objTree are different")
err = treeStorage.SetHeads(objTree.tree.Heads())
err = objTree.treeStorage.SetHeads(objTree.tree.Heads())
if err != nil {
return nil, err
}
}
objTree.id, err = treeStorage.ID()
objTree.id, err = objTree.treeStorage.ID()
if err != nil {
return nil, err
}
objTree.header, err = treeStorage.Header()
objTree.header, err = objTree.treeStorage.Header()
if err != nil {
return nil, err
}
if listener != nil {
listener.Rebuild(objTree)
if objTree.updateListener != nil {
objTree.updateListener.Rebuild(objTree)
}
return objTree, nil
}
func (ot *objectTree) rebuildFromStorage(aclList list.ACLList, newChanges []*Change) (err error) {
ot.treeBuilder.Init(ot.kch)
func BuildObjectTree(treeStorage storage.TreeStorage, listener ObjectTreeUpdateListener, aclList list.ACLList) (ObjectTree, error) {
deps := defaultObjectTreeDeps(treeStorage, listener, aclList)
return buildObjectTree(deps)
}
func (ot *objectTree) rebuildFromStorage(newChanges []*Change) (err error) {
ot.treeBuilder.Reset()
ot.tree, err = ot.treeBuilder.Build(newChanges)
if err != nil {
@ -157,7 +185,7 @@ func (ot *objectTree) rebuildFromStorage(aclList list.ACLList, newChanges []*Cha
// but obviously they are not roots, because of the way how we construct the tree
ot.tree.clearPossibleRoots()
return ot.validator.ValidateTree(ot.tree, aclList)
return ot.validator.ValidateTree(ot.tree, ot.aclList)
}
func (ot *objectTree) ID() string {
@ -172,83 +200,54 @@ func (ot *objectTree) Storage() storage.TreeStorage {
return ot.treeStorage
}
func (ot *objectTree) AddContent(ctx context.Context, aclList list.ACLList, content SignableChangeContent) (rawChange *aclpb.RawChange, err error) {
func (ot *objectTree) AddContent(ctx context.Context, content SignableChangeContent) (rawChange *aclpb.RawChange, err error) {
ot.aclList.Lock()
defer func() {
ot.aclList.Unlock()
if ot.updateListener != nil {
ot.updateListener.Update(ot)
}
}()
state := aclList.ACLState() // special method for own keys
aclChange := &aclpb.Change{
TreeHeadIds: ot.tree.Heads(),
AclHeadId: aclList.Head().Id,
SnapshotBaseId: ot.tree.RootId(),
CurrentReadKeyHash: state.CurrentReadKeyHash(),
Timestamp: int64(time.Now().Nanosecond()),
Identity: content.Identity,
IsSnapshot: content.IsSnapshot,
}
marshalledData, err := content.Proto.Marshal()
if err != nil {
return nil, err
}
state := ot.aclList.ACLState() // special method for own keys
readKey, err := state.CurrentReadKey()
if err != nil {
return nil, err
}
encrypted, err := readKey.Encrypt(marshalledData)
if err != nil {
return nil, err
payload := BuilderContent{
treeHeadIds: ot.tree.Heads(),
aclHeadId: ot.aclList.Head().Id,
snapshotBaseId: ot.tree.RootId(),
currentReadKeyHash: state.CurrentReadKeyHash(),
identity: content.Identity,
isSnapshot: content.IsSnapshot,
signingKey: content.Key,
readKey: readKey,
content: content.Proto,
}
aclChange.ChangesData = encrypted
fullMarshalledChange, err := proto.Marshal(aclChange)
if err != nil {
return nil, err
}
signature, err := content.Key.Sign(fullMarshalledChange)
if err != nil {
return nil, err
}
id, err := cid.NewCIDFromBytes(fullMarshalledChange)
if err != nil {
return nil, err
}
docChange := NewChange(id, aclChange, signature)
docChange.ParsedModel = content
objChange, rawChange, err := ot.changeBuilder.BuildContent(payload)
if content.IsSnapshot {
// clearing tree, because we already fixed everything in the last snapshot
ot.tree = &Tree{}
}
err = ot.tree.AddMergedHead(docChange)
err = ot.tree.AddMergedHead(objChange)
if err != nil {
panic(err)
}
rawChange = &aclpb.RawChange{
Payload: fullMarshalledChange,
Signature: docChange.Signature(),
Id: docChange.Id,
}
err = ot.treeStorage.AddRawChange(rawChange)
if err != nil {
return
}
err = ot.treeStorage.SetHeads([]string{docChange.Id})
err = ot.treeStorage.SetHeads([]string{objChange.Id})
return
}
func (ot *objectTree) AddRawChanges(ctx context.Context, aclList list.ACLList, rawChanges ...*aclpb.RawChange) (addResult AddResult, err error) {
func (ot *objectTree) AddRawChanges(ctx context.Context, rawChanges ...*aclpb.RawChange) (addResult AddResult, err error) {
var mode Mode
mode, addResult, err = ot.addRawChanges(ctx, aclList, rawChanges...)
mode, addResult, err = ot.addRawChanges(ctx, rawChanges...)
if err != nil {
return
}
@ -285,14 +284,14 @@ func (ot *objectTree) AddRawChanges(ctx context.Context, aclList list.ACLList, r
return
}
func (ot *objectTree) addRawChanges(ctx context.Context, aclList list.ACLList, rawChanges ...*aclpb.RawChange) (mode Mode, addResult AddResult, err error) {
func (ot *objectTree) addRawChanges(ctx context.Context, rawChanges ...*aclpb.RawChange) (mode Mode, addResult AddResult, err error) {
// resetting buffers
ot.tmpChangesBuf = ot.tmpChangesBuf[:0]
ot.notSeenIdxBuf = ot.notSeenIdxBuf[:0]
ot.difSnapshotBuf = ot.difSnapshotBuf[:0]
ot.newSnapshotsBuf = ot.newSnapshotsBuf[:0]
// this will be returned to client so we shouldn't use buffer here
// this will be returned to client, so we shouldn't use buffer here
prevHeadsCopy := make([]string, 0, len(ot.tree.Heads()))
copy(prevHeadsCopy, ot.tree.Heads())
@ -303,7 +302,7 @@ func (ot *objectTree) addRawChanges(ctx context.Context, aclList list.ACLList, r
}
var change *Change
change, err = newVerifiedChangeFromRaw(ch, ot.kch)
change, err = ot.changeBuilder.ConvertFromRawAndVerify(ch)
if err != nil {
return
}
@ -370,10 +369,10 @@ func (ot *objectTree) addRawChanges(ctx context.Context, aclList list.ACLList, r
// checking if we have some changes with different snapshot and then rebuilding
for _, ch := range ot.tmpChangesBuf {
if isOldSnapshot(ch) {
err = ot.rebuildFromStorage(aclList, ot.tmpChangesBuf)
err = ot.rebuildFromStorage(ot.tmpChangesBuf)
if err != nil {
// rebuilding without new changes
ot.rebuildFromStorage(aclList, nil)
ot.rebuildFromStorage(nil)
return
}
@ -401,7 +400,7 @@ func (ot *objectTree) addRawChanges(ctx context.Context, aclList list.ACLList, r
default:
// just rebuilding the state from start without reloading everything from tree storage
// as an optimization we could've started from current heads, but I didn't implement that
err = ot.validator.ValidateTree(ot.tree, aclList)
err = ot.validator.ValidateTree(ot.tree, ot.aclList)
if err != nil {
rollback()
err = ErrHasInvalidChanges

21
pkg/acl/tree/objecttree_test.go Normal file
View file

@ -0,0 +1,21 @@
package tree
import (
"context"
"github.com/anytypeio/go-anytype-infrastructure-experiments/app"
"github.com/anytypeio/go-anytype-infrastructure-experiments/pkg/acl/storage"
"testing"
)
func TestObjectTree(t *testing.T) {
a := &app.App{}
inmemory := storage.NewInMemoryTreeStorage(...)
app.RegisterWithType[storage.TreeStorage](a, inmemory)
app.RegisterWithType[]()
a.Start(context.Background())
objectTree := app.MustComponentWithType[ObjectTree](a).(ObjectTree)
}

1
pkg/acl/tree/tree.go
View file

@ -115,6 +115,7 @@ func (t *Tree) Add(changes ...*Change) (mode Mode) {
return Append
}
// RemoveInvalidChange removes all the changes that are descendants of id
func (t *Tree) RemoveInvalidChange(id string) {
stack := []string{id}
// removing all children of this id (either next or unattached)

17
pkg/acl/tree/treebuilder.go
View file

@ -17,25 +17,26 @@ var (
)
type treeBuilder struct {
cache map[string]*Change
kch *keychain
tree *Tree
treeStorage storage.TreeStorage
builder ChangeBuilder
cache map[string]*Change
tree *Tree
// buffers
idStack []string
loadBuffer []*Change
}
func newTreeBuilder(t storage.TreeStorage) *treeBuilder {
func newTreeBuilder(storage storage.TreeStorage, builder ChangeBuilder) *treeBuilder {
return &treeBuilder{
treeStorage: t,
treeStorage: storage,
builder: builder,
}
}
func (tb *treeBuilder) Init(kch *keychain) {
func (tb *treeBuilder) Reset() {
tb.cache = make(map[string]*Change)
tb.kch = kch
tb.tree = &Tree{}
}
@ -131,7 +132,7 @@ func (tb *treeBuilder) loadChange(id string) (ch *Change, err error) {
return nil, err
}
ch, err = newVerifiedChangeFromRaw(change, tb.kch)
ch, err = tb.builder.ConvertFromRawAndVerify(change)
if err != nil {
return nil, err
}