anyproto/any-sync
1
0
Fork 0
mirror of https://github.com/anyproto/any-sync.git synced 2025-06-08 05:57:03 +09:00
Code Activity

Add inmemory storage

Browse source
This commit is contained in:
mcrakhman 2024-12-02 13:10:48 +01:00
parent 0fb7b3a3ed
commit 76db272232
No known key found for this signature in database
GPG key ID: DED12CFEF5B8396B
10 changed files with 166 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

10
acl/acl_test.go
View file

@ -24,7 +24,7 @@ func TestAclService_AddRecord(t *testing.T) {
ownerKeys, err := accountdata.NewRandom()
require.NoError(t, err)
spaceId := "spaceId"
ownerAcl, err := list.NewTestDerivedAcl(spaceId, ownerKeys)
ownerAcl, err := list.NewInMemoryDerivedAcl(spaceId, ownerKeys)
require.NoError(t, err)
inv, err := ownerAcl.RecordBuilder().BuildInvite()
require.NoError(t, err)
@ -96,7 +96,7 @@ func TestAclService_RecordsAfter(t *testing.T) {
ownerKeys, err := accountdata.NewRandom()
require.NoError(t, err)
spaceId := "spaceId"
ownerAcl, err := list.NewTestDerivedAcl(spaceId, ownerKeys)
ownerAcl, err := list.NewInMemoryDerivedAcl(spaceId, ownerKeys)
require.NoError(t, err)
fx := newFixture(t)
@ -121,7 +121,7 @@ func TestAclService(t *testing.T) {
ownerKeys, err := accountdata.NewRandom()
require.NoError(t, err)
spaceId := "spaceId"
ownerAcl, err := list.NewTestDerivedAcl(spaceId, ownerKeys)
ownerAcl, err := list.NewInMemoryDerivedAcl(spaceId, ownerKeys)
require.NoError(t, err)
fx := newFixture(t)
@ -153,7 +153,7 @@ func TestAclService_ReadState(t *testing.T) {
ownerKeys, err := accountdata.NewRandom()
require.NoError(t, err)
spaceId := "spaceId"
ownerAcl, err := list.NewTestDerivedAcl(spaceId, ownerKeys)
ownerAcl, err := list.NewInMemoryDerivedAcl(spaceId, ownerKeys)
require.NoError(t, err)
fx := newFixture(t)
@ -179,7 +179,7 @@ func TestAclService_HasRecord(t *testing.T) {
ownerKeys, err := accountdata.NewRandom()
require.NoError(t, err)
spaceId := "spaceId"
ownerAcl, err := list.NewTestDerivedAcl(spaceId, ownerKeys)
ownerAcl, err := list.NewInMemoryDerivedAcl(spaceId, ownerKeys)
require.NoError(t, err)
fx := newFixture(t)

4
commonspace/object/acl/list/acltestsuite.go
View file

@ -245,14 +245,14 @@ func (a *AclTestExecutor) Execute(cmd string) (err error) {
meta := []byte(account)
var acl AclList
if a.ownerKeys == nil {
acl, err = NewTestDerivedAclMetadata(a.spaceId, keys, meta)
acl, err = newInMemoryDerivedAclMetadata(a.spaceId, keys, meta)
if err != nil {
return err
}
} else {
keys = a.ownerKeys
meta = a.ownerMeta
acl, err = NewTestAclWithRoot(keys, a.root)
acl, err = newInMemoryAclWithRoot(keys, a.root)
if err != nil {
return err
}

114
commonspace/object/acl/list/inmemorystorage.go Normal file
View file

@ -0,0 +1,114 @@
package list
import (
"context"
"fmt"
"sync"
"github.com/anyproto/any-sync/consensus/consensusproto"
)
type inMemoryStorage struct {
id string
root StorageRecord
head string
recordsToIndex map[string]int
records []StorageRecord
sync.RWMutex
}
func NewInMemoryStorage(
id string,
records []*consensusproto.RawRecordWithId) (Storage, error) {
if len(records) == 0 {
return nil, fmt.Errorf("empty records")
}
newRecs := make([]StorageRecord, 0, len(records))
recordsToIndex := make(map[string]int, len(records))
newRecs = append(newRecs, StorageRecord{
RawRecord: records[0].Payload,
PrevId: "",
Id: records[0].Id,
Order: 1,
})
recordsToIndex[newRecs[0].Id] = 0
for i := 1; i < len(records)-1; i++ {
prevRec := newRecs[i-1]
rec := records[i]
newRecs = append(newRecs, StorageRecord{
RawRecord: rec.Payload,
PrevId: prevRec.Id,
Id: rec.Id,
Order: prevRec.Order + 1,
ChangeSize: len(rec.Payload),
})
recordsToIndex[rec.Id] = i
}
root := newRecs[0]
head := records[len(records)-1]
return &inMemoryStorage{
id: root.Id,
root: root,
head: head.Id,
records: newRecs,
recordsToIndex: recordsToIndex,
}, nil
}
func (t *inMemoryStorage) Root(ctx context.Context) (StorageRecord, error) {
return t.root, nil
}
func (t *inMemoryStorage) Head(ctx context.Context) (string, error) {
t.RLock()
defer t.RUnlock()
return t.head, nil
}
func (t *inMemoryStorage) Has(ctx context.Context, id string) (bool, error) {
t.RLock()
defer t.RUnlock()
_, exists := t.recordsToIndex[id]
return exists, nil
}
func (t *inMemoryStorage) Get(ctx context.Context, id string) (StorageRecord, error) {
t.RLock()
defer t.RUnlock()
if idx, exists := t.recordsToIndex[id]; exists {
return t.records[idx], nil
}
return StorageRecord{}, ErrNoSuchRecord
}
func (t *inMemoryStorage) GetAfterOrder(ctx context.Context, order int, iter StorageIterator) error {
t.RLock()
defer t.RUnlock()
if order >= len(t.records) {
return nil
}
for i := order; i < len(t.records); i++ {
if shouldContinue, err := iter(ctx, t.records[i]); !shouldContinue || err != nil {
return err
}
}
return nil
}
func (t *inMemoryStorage) AddAll(ctx context.Context, records []StorageRecord) error {
t.Lock()
defer t.Unlock()
for _, rec := range records {
t.records = append(t.records, rec)
t.recordsToIndex[rec.Id] = len(t.records) - 1
}
t.head = records[len(records)-1].Id
return nil
}
func (t *inMemoryStorage) Id() string {
return t.id
}

6
commonspace/object/acl/list/list_test.go
View file

@ -29,9 +29,9 @@ func newFixture(t *testing.T) *aclFixture {
accountKeys, err := accountdata.NewRandom()
require.NoError(t, err)
spaceId := "spaceId"
ownerAcl, err := NewTestDerivedAcl(spaceId, ownerKeys)
ownerAcl, err := NewInMemoryDerivedAcl(spaceId, ownerKeys)
require.NoError(t, err)
accountAcl, err := NewTestAclWithRoot(accountKeys, ownerAcl.Root())
accountAcl, err := newInMemoryAclWithRoot(accountKeys, ownerAcl.Root())
require.NoError(t, err)
require.Equal(t, ownerAcl.AclState().lastRecordId, ownerAcl.Id())
require.Equal(t, ownerAcl.AclState().lastRecordId, accountAcl.AclState().lastRecordId)
@ -114,7 +114,7 @@ func (fx *aclFixture) inviteAccount(t *testing.T, perms AclPermissions) {
func TestAclList_BuildRoot(t *testing.T) {
randomKeys, err := accountdata.NewRandom()
require.NoError(t, err)
randomAcl, err := NewTestDerivedAcl("spaceId", randomKeys)
randomAcl, err := NewInMemoryDerivedAcl("spaceId", randomKeys)
require.NoError(t, err)
fmt.Println(randomAcl.Id())
}

8
commonspace/object/acl/list/listutils.go
View file

@ -9,11 +9,11 @@ import (
"github.com/anyproto/any-sync/util/crypto"
)
func NewTestDerivedAcl(spaceId string, keys *accountdata.AccountKeys) (AclList, error) {
return NewTestDerivedAclMetadata(spaceId, keys, []byte("metadata"))
func NewInMemoryDerivedAcl(spaceId string, keys *accountdata.AccountKeys) (AclList, error) {
return newInMemoryDerivedAclMetadata(spaceId, keys, []byte("metadata"))
}
func NewTestDerivedAclMetadata(spaceId string, keys *accountdata.AccountKeys, metadata []byte) (AclList, error) {
func newInMemoryDerivedAclMetadata(spaceId string, keys *accountdata.AccountKeys, metadata []byte) (AclList, error) {
builder := NewAclRecordBuilder("", crypto.NewKeyStorage(), keys, NoOpAcceptorVerifier{})
masterKey, _, err := crypto.GenerateRandomEd25519KeyPair()
if err != nil {
@ -46,7 +46,7 @@ func NewTestDerivedAclMetadata(spaceId string, keys *accountdata.AccountKeys, me
return BuildAclListWithIdentity(keys, st, NoOpAcceptorVerifier{})
}
func NewTestAclWithRoot(keys *accountdata.AccountKeys, root *consensusproto.RawRecordWithId) (AclList, error) {
func newInMemoryAclWithRoot(keys *accountdata.AccountKeys, root *consensusproto.RawRecordWithId) (AclList, error) {
st, err := liststorage.NewInMemoryAclListStorage(root.Id, []*consensusproto.RawRecordWithId{
root,
})

4
commonspace/object/acl/list/storage.go
View file

@ -57,7 +57,7 @@ type storage struct {
arena *anyenc.Arena
}
func createStorage(ctx context.Context, root *consensusproto.RawRecordWithId, store anystore.DB) (Storage, error) {
func CreateStorage(ctx context.Context, root *consensusproto.RawRecordWithId, store anystore.DB) (Storage, error) {
st := &storage{
id: root.Id,
store: store,
@ -110,7 +110,7 @@ func createStorage(ctx context.Context, root *consensusproto.RawRecordWithId, st
return st, tx.Commit()
}
func newStorage(ctx context.Context, id string, store anystore.DB) (Storage, error) {
func NewStorage(ctx context.Context, id string, store anystore.DB) (Storage, error) {
st := &storage{
id: id,
store: store,

64
commonspace/object/tree/objecttree/objecttree_test.go
View file

@ -130,11 +130,11 @@ type testStore struct {
errAdd bool
}
func newStore(ctx context.Context, t *testing.T) anystore.DB {
return newNamedStore(ctx, t, "changes.db")
func createStore(ctx context.Context, t *testing.T) anystore.DB {
return createNamedStore(ctx, t, "changes.db")
}
func newNamedStore(ctx context.Context, t *testing.T, name string) anystore.DB {
func createNamedStore(ctx context.Context, t *testing.T, name string) anystore.DB {
path := filepath.Join(t.TempDir(), name)
db, err := anystore.Open(ctx, path, nil)
require.NoError(t, err)
@ -222,7 +222,7 @@ func copyStore(ctx context.Context, t *testing.T, store testStore, name string)
func prepareAclList(t *testing.T) (list.AclList, *accountdata.AccountKeys) {
randKeys, err := accountdata.NewRandom()
require.NoError(t, err)
aclList, err := list.NewTestDerivedAcl("spaceId", randKeys)
aclList, err := list.NewInMemoryDerivedAcl("spaceId", randKeys)
require.NoError(t, err, "building acl list should be without error")
return aclList, randKeys
@ -230,7 +230,7 @@ func prepareAclList(t *testing.T) (list.AclList, *accountdata.AccountKeys) {
func prepareHistoryTreeDeps(t *testing.T, aclList list.AclList) (*MockChangeCreator, objectTreeDeps) {
changeCreator := NewMockChangeCreator(func() anystore.DB {
return newStore(ctx, t)
return createStore(ctx, t)
})
treeStorage := changeCreator.CreateNewTreeStorage(t, "0", aclList.Head().Id, false)
root, _ := treeStorage.Root(ctx)
@ -258,7 +258,7 @@ func prepareContext(
isDerived bool,
additionalChanges func(changeCreator *MockChangeCreator) RawChangesPayload) testTreeContext {
changeCreator := NewMockChangeCreator(func() anystore.DB {
return newStore(ctx, t)
return createStore(ctx, t)
})
treeStorage := changeCreator.CreateNewTreeStorage(t, "0", aclList.Head().Id, isDerived)
objTree, err := objTreeBuilder(treeStorage, aclList)
@ -287,7 +287,7 @@ func TestObjectTree(t *testing.T) {
ctx := context.Background()
t.Run("delete object tree", func(t *testing.T) {
store := newStore(ctx, t)
store := createStore(ctx, t)
exec := list.NewAclExecutor("spaceId")
type cmdErr struct {
cmd string
@ -309,7 +309,7 @@ func TestObjectTree(t *testing.T) {
IsEncrypted: true,
}, aAccount.Acl)
require.NoError(t, err)
aStore, err := createStorage(ctx, root, store)
aStore, err := CreateStorage(ctx, root, store)
require.NoError(t, err)
aTree, err := BuildKeyFilterableObjectTree(aStore, aAccount.Acl)
require.NoError(t, err)
@ -334,7 +334,7 @@ func TestObjectTree(t *testing.T) {
})
t.Run("user delete logic: validation, key change, decryption", func(t *testing.T) {
storeA := newNamedStore(ctx, t, "a")
storeA := createNamedStore(ctx, t, "a")
exec := list.NewAclExecutor("spaceId")
type cmdErr struct {
cmd string
@ -360,7 +360,7 @@ func TestObjectTree(t *testing.T) {
IsEncrypted: true,
}, aAccount.Acl)
require.NoError(t, err)
aStore, err := createStorage(ctx, root, storeA)
aStore, err := CreateStorage(ctx, root, storeA)
require.NoError(t, err)
aTree, err := BuildKeyFilterableObjectTree(aStore, aAccount.Acl)
require.NoError(t, err)
@ -373,7 +373,7 @@ func TestObjectTree(t *testing.T) {
})
require.NoError(t, err)
storeB := copyStore(ctx, t, storeA.(testStore), "b")
bStore, err := newStorage(ctx, root.Id, storeB)
bStore, err := NewStorage(ctx, root.Id, storeB)
require.NoError(t, err)
bTree, err := BuildKeyFilterableObjectTree(bStore, bAccount.Acl)
require.NoError(t, err)
@ -399,7 +399,7 @@ func TestObjectTree(t *testing.T) {
require.Equal(t, oldHeads, bTree.Heads())
rawAdded = allChanges(ctx, t, bStore)
require.NoError(t, err)
validateStore := newStore(ctx, t)
validateStore := createStore(ctx, t)
newTree, err := ValidateFilterRawTree(treestorage.TreeStorageCreatePayload{
RootRawChange: root,
Changes: rawAdded,
@ -423,7 +423,7 @@ func TestObjectTree(t *testing.T) {
t.Run("filter changes when no aclHeadId", func(t *testing.T) {
exec := list.NewAclExecutor("spaceId")
storeA := newStore(ctx, t)
storeA := createStore(ctx, t)
type cmdErr struct {
cmd string
err error
@ -448,7 +448,7 @@ func TestObjectTree(t *testing.T) {
IsEncrypted: true,
}, aAccount.Acl)
require.NoError(t, err)
aStore, err := createStorage(ctx, root, storeA)
aStore, err := CreateStorage(ctx, root, storeA)
require.NoError(t, err)
aTree, err := BuildKeyFilterableObjectTree(aStore, aAccount.Acl)
require.NoError(t, err)
@ -461,7 +461,7 @@ func TestObjectTree(t *testing.T) {
})
require.NoError(t, err)
storeB := copyStore(ctx, t, storeA.(testStore), "b")
bStore, err := newStorage(ctx, root.Id, storeB)
bStore, err := NewStorage(ctx, root.Id, storeB)
require.NoError(t, err)
// copying old version of storage
prevAclRecs, err := bAccount.Acl.RecordsAfter(ctx, "")
@ -512,8 +512,8 @@ func TestObjectTree(t *testing.T) {
IsEncrypted: true,
}, aclList)
require.NoError(t, err)
store := newStore(ctx, t)
storage, _ := createStorage(ctx, root, store)
store := createStore(ctx, t)
storage, _ := CreateStorage(ctx, root, store)
oTree, err := BuildObjectTree(storage, aclList)
require.NoError(t, err)
@ -582,12 +582,12 @@ func TestObjectTree(t *testing.T) {
IsEncrypted: true,
}, aclList)
require.NoError(t, err)
store := newStore(ctx, t)
storage, _ := createStorage(ctx, root, store)
store := createStore(ctx, t)
storage, _ := CreateStorage(ctx, root, store)
oTree, err := BuildObjectTree(storage, aclList)
require.NoError(t, err)
emptyDataTreeDeps = nonVerifiableTreeDeps
validateStore := newStore(ctx, t)
validateStore := createStore(ctx, t)
err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
RootRawChange: oTree.Header(),
Heads: []string{root.Id},
@ -604,11 +604,11 @@ func TestObjectTree(t *testing.T) {
IsEncrypted: true,
}, aclList)
require.NoError(t, err)
store := newStore(ctx, t)
storage, _ := createStorage(ctx, root, store)
store := createStore(ctx, t)
storage, _ := CreateStorage(ctx, root, store)
oTree, err := BuildObjectTree(storage, aclList)
require.NoError(t, err)
validateStore := newStore(ctx, t)
validateStore := createStore(ctx, t)
err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
RootRawChange: oTree.Header(),
Heads: []string{root.Id},
@ -638,8 +638,8 @@ func TestObjectTree(t *testing.T) {
IsEncrypted: true,
}, aclList)
require.NoError(t, err)
store := newStore(ctx, t)
storage, _ := createStorage(ctx, root, store)
store := createStore(ctx, t)
storage, _ := CreateStorage(ctx, root, store)
oTree, err := BuildObjectTree(storage, aclList)
require.NoError(t, err)
_, err = oTree.AddContent(ctx, SignableChangeContent{
@ -653,7 +653,7 @@ func TestObjectTree(t *testing.T) {
require.NoError(t, err)
chs := allChanges(ctx, t, storage)
emptyDataTreeDeps = nonVerifiableTreeDeps
validateStore := newStore(ctx, t)
validateStore := createStore(ctx, t)
err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
RootRawChange: oTree.Header(),
Heads: []string{oTree.Heads()[0]},
@ -685,8 +685,8 @@ func TestObjectTree(t *testing.T) {
}, aclList)
require.NoError(t, err)
emptyDataTreeDeps = nonVerifiableTreeDeps
store := newStore(ctx, t)
storage, _ := createStorage(ctx, root, store)
store := createStore(ctx, t)
storage, _ := CreateStorage(ctx, root, store)
oTree, err := BuildObjectTree(storage, aclList)
require.NoError(t, err)
_, err = oTree.AddContent(ctx, SignableChangeContent{
@ -699,7 +699,7 @@ func TestObjectTree(t *testing.T) {
})
require.NoError(t, err)
chs := allChanges(ctx, t, storage)
validateStore := newStore(ctx, t)
validateStore := createStore(ctx, t)
err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
RootRawChange: oTree.Header(),
Heads: []string{oTree.Heads()[0]},
@ -733,7 +733,7 @@ func TestObjectTree(t *testing.T) {
changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"),
}
emptyDataTreeDeps = nonVerifiableTreeDeps
validateStore := newStore(ctx, t)
validateStore := createStore(ctx, t)
err := ValidateRawTree(treestorage.TreeStorageCreatePayload{
RootRawChange: treeCtx.objTree.Header(),
Heads: []string{"3"},
@ -1601,7 +1601,7 @@ func TestObjectTree(t *testing.T) {
changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"),
}
emptyDataTreeDeps = nonVerifiableTreeDeps
validateStore := newStore(ctx, t)
validateStore := createStore(ctx, t)
err := ValidateRawTree(treestorage.TreeStorageCreatePayload{
RootRawChange: treeCtx.objTree.Header(),
Heads: []string{"3"},
@ -1619,7 +1619,7 @@ func TestObjectTree(t *testing.T) {
changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"),
}
emptyDataTreeDeps = nonVerifiableTreeDeps
validateStore := newStore(ctx, t)
validateStore := createStore(ctx, t)
err := ValidateRawTree(treestorage.TreeStorageCreatePayload{
RootRawChange: treeCtx.objTree.Header(),
Heads: []string{"3"},

2
commonspace/object/tree/objecttree/objecttreevalidator.go
View file

@ -20,7 +20,7 @@ type tempTreeStorageCreator struct {
}
func (t *tempTreeStorageCreator) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (Storage, error) {
return createStorage(context.Background(), payload.RootRawChange, t.store)
return CreateStorage(context.Background(), payload.RootRawChange, t.store)
}
type ValidatorFunc func(payload treestorage.TreeStorageCreatePayload, storageCreator TreeStorageCreator, aclList list.AclList) (ret ObjectTree, err error)

4
commonspace/object/tree/objecttree/storage.go
View file

@ -67,7 +67,7 @@ type storage struct {
var storageChangeBuilder = NewChangeBuilder
func createStorage(ctx context.Context, root *treechangeproto.RawTreeChangeWithId, store anystore.DB) (Storage, error) {
func CreateStorage(ctx context.Context, root *treechangeproto.RawTreeChangeWithId, store anystore.DB) (Storage, error) {
st := &storage{
id: root.Id,
store: store,
@ -129,7 +129,7 @@ func createStorage(ctx context.Context, root *treechangeproto.RawTreeChangeWithI
return st, tx.Commit()
}
func newStorage(ctx context.Context, id string, store anystore.DB) (Storage, error) {
func NewStorage(ctx context.Context, id string, store anystore.DB) (Storage, error) {
// TODO: use spacestorage to set heads
st := &storage{
id: id,

2
commonspace/object/tree/objecttree/testutils.go
View file

@ -164,7 +164,7 @@ func (c *MockChangeCreator) CreateNewTreeStorage(t *testing.T, treeId, aclHeadId
ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), rootChange),
}
}
storage, err := createStorage(context.Background(), root, c.storeCreator())
storage, err := CreateStorage(context.Background(), root, c.storeCreator())
require.NoError(t, err)
return &testStorage{
Storage: storage,