Add inmemory storage

acl/acl_test.go

@@ -24,7 +24,7 @@ func TestAclService_AddRecord(t *testing.T) {
 	ownerKeys, err := accountdata.NewRandom()
 	require.NoError(t, err)
 	spaceId := "spaceId"
-	ownerAcl, err := list.NewTestDerivedAcl(spaceId, ownerKeys)
+	ownerAcl, err := list.NewInMemoryDerivedAcl(spaceId, ownerKeys)
 	require.NoError(t, err)
 	inv, err := ownerAcl.RecordBuilder().BuildInvite()
 	require.NoError(t, err)
@@ -96,7 +96,7 @@ func TestAclService_RecordsAfter(t *testing.T) {
 	ownerKeys, err := accountdata.NewRandom()
 	require.NoError(t, err)
 	spaceId := "spaceId"
-	ownerAcl, err := list.NewTestDerivedAcl(spaceId, ownerKeys)
+	ownerAcl, err := list.NewInMemoryDerivedAcl(spaceId, ownerKeys)
 	require.NoError(t, err)
 
 	fx := newFixture(t)
@@ -121,7 +121,7 @@ func TestAclService(t *testing.T) {
 	ownerKeys, err := accountdata.NewRandom()
 	require.NoError(t, err)
 	spaceId := "spaceId"
-	ownerAcl, err := list.NewTestDerivedAcl(spaceId, ownerKeys)
+	ownerAcl, err := list.NewInMemoryDerivedAcl(spaceId, ownerKeys)
 	require.NoError(t, err)
 
 	fx := newFixture(t)
@@ -153,7 +153,7 @@ func TestAclService_ReadState(t *testing.T) {
 	ownerKeys, err := accountdata.NewRandom()
 	require.NoError(t, err)
 	spaceId := "spaceId"
-	ownerAcl, err := list.NewTestDerivedAcl(spaceId, ownerKeys)
+	ownerAcl, err := list.NewInMemoryDerivedAcl(spaceId, ownerKeys)
 	require.NoError(t, err)
 
 	fx := newFixture(t)
@@ -179,7 +179,7 @@ func TestAclService_HasRecord(t *testing.T) {
 	ownerKeys, err := accountdata.NewRandom()
 	require.NoError(t, err)
 	spaceId := "spaceId"
-	ownerAcl, err := list.NewTestDerivedAcl(spaceId, ownerKeys)
+	ownerAcl, err := list.NewInMemoryDerivedAcl(spaceId, ownerKeys)
 	require.NoError(t, err)
 
 	fx := newFixture(t)

commonspace/object/acl/list/acltestsuite.go

@@ -245,14 +245,14 @@ func (a *AclTestExecutor) Execute(cmd string) (err error) {
 			meta := []byte(account)
 			var acl AclList
 			if a.ownerKeys == nil {
-				acl, err = NewTestDerivedAclMetadata(a.spaceId, keys, meta)
+				acl, err = newInMemoryDerivedAclMetadata(a.spaceId, keys, meta)
 				if err != nil {
 					return err
 				}
 			} else {
 				keys = a.ownerKeys
 				meta = a.ownerMeta
-				acl, err = NewTestAclWithRoot(keys, a.root)
+				acl, err = newInMemoryAclWithRoot(keys, a.root)
 				if err != nil {
 					return err
 				}

commonspace/object/acl/list/inmemorystorage.go

@@ -0,0 +1,114 @@
+package list
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/anyproto/any-sync/consensus/consensusproto"
+)
+
+type inMemoryStorage struct {
+	id             string
+	root           StorageRecord
+	head           string
+	recordsToIndex map[string]int
+	records        []StorageRecord
+
+	sync.RWMutex
+}
+
+func NewInMemoryStorage(
+	id string,
+	records []*consensusproto.RawRecordWithId) (Storage, error) {
+	if len(records) == 0 {
+		return nil, fmt.Errorf("empty records")
+	}
+	newRecs := make([]StorageRecord, 0, len(records))
+	recordsToIndex := make(map[string]int, len(records))
+	newRecs = append(newRecs, StorageRecord{
+		RawRecord: records[0].Payload,
+		PrevId:    "",
+		Id:        records[0].Id,
+		Order:     1,
+	})
+	recordsToIndex[newRecs[0].Id] = 0
+	for i := 1; i < len(records)-1; i++ {
+		prevRec := newRecs[i-1]
+		rec := records[i]
+		newRecs = append(newRecs, StorageRecord{
+			RawRecord:  rec.Payload,
+			PrevId:     prevRec.Id,
+			Id:         rec.Id,
+			Order:      prevRec.Order + 1,
+			ChangeSize: len(rec.Payload),
+		})
+		recordsToIndex[rec.Id] = i
+	}
+	root := newRecs[0]
+	head := records[len(records)-1]
+
+	return &inMemoryStorage{
+		id:             root.Id,
+		root:           root,
+		head:           head.Id,
+		records:        newRecs,
+		recordsToIndex: recordsToIndex,
+	}, nil
+}
+
+func (t *inMemoryStorage) Root(ctx context.Context) (StorageRecord, error) {
+	return t.root, nil
+}
+
+func (t *inMemoryStorage) Head(ctx context.Context) (string, error) {
+	t.RLock()
+	defer t.RUnlock()
+	return t.head, nil
+}
+
+func (t *inMemoryStorage) Has(ctx context.Context, id string) (bool, error) {
+	t.RLock()
+	defer t.RUnlock()
+	_, exists := t.recordsToIndex[id]
+	return exists, nil
+}
+
+func (t *inMemoryStorage) Get(ctx context.Context, id string) (StorageRecord, error) {
+	t.RLock()
+	defer t.RUnlock()
+	if idx, exists := t.recordsToIndex[id]; exists {
+		return t.records[idx], nil
+	}
+	return StorageRecord{}, ErrNoSuchRecord
+}
+
+func (t *inMemoryStorage) GetAfterOrder(ctx context.Context, order int, iter StorageIterator) error {
+	t.RLock()
+	defer t.RUnlock()
+	if order >= len(t.records) {
+		return nil
+	}
+	for i := order; i < len(t.records); i++ {
+		if shouldContinue, err := iter(ctx, t.records[i]); !shouldContinue || err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (t *inMemoryStorage) AddAll(ctx context.Context, records []StorageRecord) error {
+	t.Lock()
+	defer t.Unlock()
+
+	for _, rec := range records {
+		t.records = append(t.records, rec)
+		t.recordsToIndex[rec.Id] = len(t.records) - 1
+	}
+	t.head = records[len(records)-1].Id
+	return nil
+}
+
+func (t *inMemoryStorage) Id() string {
+	return t.id
+}

commonspace/object/acl/list/list_test.go

@@ -29,9 +29,9 @@ func newFixture(t *testing.T) *aclFixture {
 	accountKeys, err := accountdata.NewRandom()
 	require.NoError(t, err)
 	spaceId := "spaceId"
-	ownerAcl, err := NewTestDerivedAcl(spaceId, ownerKeys)
+	ownerAcl, err := NewInMemoryDerivedAcl(spaceId, ownerKeys)
 	require.NoError(t, err)
-	accountAcl, err := NewTestAclWithRoot(accountKeys, ownerAcl.Root())
+	accountAcl, err := newInMemoryAclWithRoot(accountKeys, ownerAcl.Root())
 	require.NoError(t, err)
 	require.Equal(t, ownerAcl.AclState().lastRecordId, ownerAcl.Id())
 	require.Equal(t, ownerAcl.AclState().lastRecordId, accountAcl.AclState().lastRecordId)
@@ -114,7 +114,7 @@ func (fx *aclFixture) inviteAccount(t *testing.T, perms AclPermissions) {
 func TestAclList_BuildRoot(t *testing.T) {
 	randomKeys, err := accountdata.NewRandom()
 	require.NoError(t, err)
-	randomAcl, err := NewTestDerivedAcl("spaceId", randomKeys)
+	randomAcl, err := NewInMemoryDerivedAcl("spaceId", randomKeys)
 	require.NoError(t, err)
 	fmt.Println(randomAcl.Id())
 }

commonspace/object/acl/list/listutils.go

@@ -9,11 +9,11 @@ import (
 	"github.com/anyproto/any-sync/util/crypto"
 )
 
-func NewTestDerivedAcl(spaceId string, keys *accountdata.AccountKeys) (AclList, error) {
+func NewInMemoryDerivedAcl(spaceId string, keys *accountdata.AccountKeys) (AclList, error) {
-	return NewTestDerivedAclMetadata(spaceId, keys, []byte("metadata"))
+	return newInMemoryDerivedAclMetadata(spaceId, keys, []byte("metadata"))
 }
 
-func NewTestDerivedAclMetadata(spaceId string, keys *accountdata.AccountKeys, metadata []byte) (AclList, error) {
+func newInMemoryDerivedAclMetadata(spaceId string, keys *accountdata.AccountKeys, metadata []byte) (AclList, error) {
 	builder := NewAclRecordBuilder("", crypto.NewKeyStorage(), keys, NoOpAcceptorVerifier{})
 	masterKey, _, err := crypto.GenerateRandomEd25519KeyPair()
 	if err != nil {
@@ -46,7 +46,7 @@ func NewTestDerivedAclMetadata(spaceId string, keys *accountdata.AccountKeys, me
 	return BuildAclListWithIdentity(keys, st, NoOpAcceptorVerifier{})
 }
 
-func NewTestAclWithRoot(keys *accountdata.AccountKeys, root *consensusproto.RawRecordWithId) (AclList, error) {
+func newInMemoryAclWithRoot(keys *accountdata.AccountKeys, root *consensusproto.RawRecordWithId) (AclList, error) {
 	st, err := liststorage.NewInMemoryAclListStorage(root.Id, []*consensusproto.RawRecordWithId{
 		root,
 	})

commonspace/object/acl/list/storage.go

@@ -57,7 +57,7 @@ type storage struct {
 	arena       *anyenc.Arena
 }
 
-func createStorage(ctx context.Context, root *consensusproto.RawRecordWithId, store anystore.DB) (Storage, error) {
+func CreateStorage(ctx context.Context, root *consensusproto.RawRecordWithId, store anystore.DB) (Storage, error) {
 	st := &storage{
 		id:    root.Id,
 		store: store,
@@ -110,7 +110,7 @@ func createStorage(ctx context.Context, root *consensusproto.RawRecordWithId, st
 	return st, tx.Commit()
 }
 
-func newStorage(ctx context.Context, id string, store anystore.DB) (Storage, error) {
+func NewStorage(ctx context.Context, id string, store anystore.DB) (Storage, error) {
 	st := &storage{
 		id:    id,
 		store: store,

commonspace/object/tree/objecttree/objecttree_test.go

@@ -130,11 +130,11 @@ type testStore struct {
 	errAdd bool
 }
 
-func newStore(ctx context.Context, t *testing.T) anystore.DB {
+func createStore(ctx context.Context, t *testing.T) anystore.DB {
-	return newNamedStore(ctx, t, "changes.db")
+	return createNamedStore(ctx, t, "changes.db")
 }
 
-func newNamedStore(ctx context.Context, t *testing.T, name string) anystore.DB {
+func createNamedStore(ctx context.Context, t *testing.T, name string) anystore.DB {
 	path := filepath.Join(t.TempDir(), name)
 	db, err := anystore.Open(ctx, path, nil)
 	require.NoError(t, err)
@@ -222,7 +222,7 @@ func copyStore(ctx context.Context, t *testing.T, store testStore, name string)
 func prepareAclList(t *testing.T) (list.AclList, *accountdata.AccountKeys) {
 	randKeys, err := accountdata.NewRandom()
 	require.NoError(t, err)
-	aclList, err := list.NewTestDerivedAcl("spaceId", randKeys)
+	aclList, err := list.NewInMemoryDerivedAcl("spaceId", randKeys)
 	require.NoError(t, err, "building acl list should be without error")
 
 	return aclList, randKeys
@@ -230,7 +230,7 @@ func prepareAclList(t *testing.T) (list.AclList, *accountdata.AccountKeys) {
 
 func prepareHistoryTreeDeps(t *testing.T, aclList list.AclList) (*MockChangeCreator, objectTreeDeps) {
 	changeCreator := NewMockChangeCreator(func() anystore.DB {
-		return newStore(ctx, t)
+		return createStore(ctx, t)
 	})
 	treeStorage := changeCreator.CreateNewTreeStorage(t, "0", aclList.Head().Id, false)
 	root, _ := treeStorage.Root(ctx)
@@ -258,7 +258,7 @@ func prepareContext(
 	isDerived bool,
 	additionalChanges func(changeCreator *MockChangeCreator) RawChangesPayload) testTreeContext {
 	changeCreator := NewMockChangeCreator(func() anystore.DB {
-		return newStore(ctx, t)
+		return createStore(ctx, t)
 	})
 	treeStorage := changeCreator.CreateNewTreeStorage(t, "0", aclList.Head().Id, isDerived)
 	objTree, err := objTreeBuilder(treeStorage, aclList)
@@ -287,7 +287,7 @@ func TestObjectTree(t *testing.T) {
 	ctx := context.Background()
 
 	t.Run("delete object tree", func(t *testing.T) {
-		store := newStore(ctx, t)
+		store := createStore(ctx, t)
 		exec := list.NewAclExecutor("spaceId")
 		type cmdErr struct {
 			cmd string
@@ -309,7 +309,7 @@ func TestObjectTree(t *testing.T) {
 			IsEncrypted:   true,
 		}, aAccount.Acl)
 		require.NoError(t, err)
-		aStore, err := createStorage(ctx, root, store)
+		aStore, err := CreateStorage(ctx, root, store)
 		require.NoError(t, err)
 		aTree, err := BuildKeyFilterableObjectTree(aStore, aAccount.Acl)
 		require.NoError(t, err)
@@ -334,7 +334,7 @@ func TestObjectTree(t *testing.T) {
 	})
 
 	t.Run("user delete logic: validation, key change, decryption", func(t *testing.T) {
-		storeA := newNamedStore(ctx, t, "a")
+		storeA := createNamedStore(ctx, t, "a")
 		exec := list.NewAclExecutor("spaceId")
 		type cmdErr struct {
 			cmd string
@@ -360,7 +360,7 @@ func TestObjectTree(t *testing.T) {
 			IsEncrypted:   true,
 		}, aAccount.Acl)
 		require.NoError(t, err)
-		aStore, err := createStorage(ctx, root, storeA)
+		aStore, err := CreateStorage(ctx, root, storeA)
 		require.NoError(t, err)
 		aTree, err := BuildKeyFilterableObjectTree(aStore, aAccount.Acl)
 		require.NoError(t, err)
@@ -373,7 +373,7 @@ func TestObjectTree(t *testing.T) {
 		})
 		require.NoError(t, err)
 		storeB := copyStore(ctx, t, storeA.(testStore), "b")
-		bStore, err := newStorage(ctx, root.Id, storeB)
+		bStore, err := NewStorage(ctx, root.Id, storeB)
 		require.NoError(t, err)
 		bTree, err := BuildKeyFilterableObjectTree(bStore, bAccount.Acl)
 		require.NoError(t, err)
@@ -399,7 +399,7 @@ func TestObjectTree(t *testing.T) {
 		require.Equal(t, oldHeads, bTree.Heads())
 		rawAdded = allChanges(ctx, t, bStore)
 		require.NoError(t, err)
-		validateStore := newStore(ctx, t)
+		validateStore := createStore(ctx, t)
 		newTree, err := ValidateFilterRawTree(treestorage.TreeStorageCreatePayload{
 			RootRawChange: root,
 			Changes:       rawAdded,
@@ -423,7 +423,7 @@ func TestObjectTree(t *testing.T) {
 
 	t.Run("filter changes when no aclHeadId", func(t *testing.T) {
 		exec := list.NewAclExecutor("spaceId")
-		storeA := newStore(ctx, t)
+		storeA := createStore(ctx, t)
 		type cmdErr struct {
 			cmd string
 			err error
@@ -448,7 +448,7 @@ func TestObjectTree(t *testing.T) {
 			IsEncrypted:   true,
 		}, aAccount.Acl)
 		require.NoError(t, err)
-		aStore, err := createStorage(ctx, root, storeA)
+		aStore, err := CreateStorage(ctx, root, storeA)
 		require.NoError(t, err)
 		aTree, err := BuildKeyFilterableObjectTree(aStore, aAccount.Acl)
 		require.NoError(t, err)
@@ -461,7 +461,7 @@ func TestObjectTree(t *testing.T) {
 		})
 		require.NoError(t, err)
 		storeB := copyStore(ctx, t, storeA.(testStore), "b")
-		bStore, err := newStorage(ctx, root.Id, storeB)
+		bStore, err := NewStorage(ctx, root.Id, storeB)
 		require.NoError(t, err)
 		// copying old version of storage
 		prevAclRecs, err := bAccount.Acl.RecordsAfter(ctx, "")
@@ -512,8 +512,8 @@ func TestObjectTree(t *testing.T) {
 			IsEncrypted:   true,
 		}, aclList)
 		require.NoError(t, err)
-		store := newStore(ctx, t)
+		store := createStore(ctx, t)
-		storage, _ := createStorage(ctx, root, store)
+		storage, _ := CreateStorage(ctx, root, store)
 		oTree, err := BuildObjectTree(storage, aclList)
 		require.NoError(t, err)
 
@@ -582,12 +582,12 @@ func TestObjectTree(t *testing.T) {
 				IsEncrypted:   true,
 			}, aclList)
 			require.NoError(t, err)
-			store := newStore(ctx, t)
+			store := createStore(ctx, t)
-			storage, _ := createStorage(ctx, root, store)
+			storage, _ := CreateStorage(ctx, root, store)
 			oTree, err := BuildObjectTree(storage, aclList)
 			require.NoError(t, err)
 			emptyDataTreeDeps = nonVerifiableTreeDeps
-			validateStore := newStore(ctx, t)
+			validateStore := createStore(ctx, t)
 			err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
 				RootRawChange: oTree.Header(),
 				Heads:         []string{root.Id},
@@ -604,11 +604,11 @@ func TestObjectTree(t *testing.T) {
 				IsEncrypted:   true,
 			}, aclList)
 			require.NoError(t, err)
-			store := newStore(ctx, t)
+			store := createStore(ctx, t)
-			storage, _ := createStorage(ctx, root, store)
+			storage, _ := CreateStorage(ctx, root, store)
 			oTree, err := BuildObjectTree(storage, aclList)
 			require.NoError(t, err)
-			validateStore := newStore(ctx, t)
+			validateStore := createStore(ctx, t)
 			err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
 				RootRawChange: oTree.Header(),
 				Heads:         []string{root.Id},
@@ -638,8 +638,8 @@ func TestObjectTree(t *testing.T) {
 				IsEncrypted:   true,
 			}, aclList)
 			require.NoError(t, err)
-			store := newStore(ctx, t)
+			store := createStore(ctx, t)
-			storage, _ := createStorage(ctx, root, store)
+			storage, _ := CreateStorage(ctx, root, store)
 			oTree, err := BuildObjectTree(storage, aclList)
 			require.NoError(t, err)
 			_, err = oTree.AddContent(ctx, SignableChangeContent{
@@ -653,7 +653,7 @@ func TestObjectTree(t *testing.T) {
 			require.NoError(t, err)
 			chs := allChanges(ctx, t, storage)
 			emptyDataTreeDeps = nonVerifiableTreeDeps
-			validateStore := newStore(ctx, t)
+			validateStore := createStore(ctx, t)
 			err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
 				RootRawChange: oTree.Header(),
 				Heads:         []string{oTree.Heads()[0]},
@@ -685,8 +685,8 @@ func TestObjectTree(t *testing.T) {
 			}, aclList)
 			require.NoError(t, err)
 			emptyDataTreeDeps = nonVerifiableTreeDeps
-			store := newStore(ctx, t)
+			store := createStore(ctx, t)
-			storage, _ := createStorage(ctx, root, store)
+			storage, _ := CreateStorage(ctx, root, store)
 			oTree, err := BuildObjectTree(storage, aclList)
 			require.NoError(t, err)
 			_, err = oTree.AddContent(ctx, SignableChangeContent{
@@ -699,7 +699,7 @@ func TestObjectTree(t *testing.T) {
 			})
 			require.NoError(t, err)
 			chs := allChanges(ctx, t, storage)
-			validateStore := newStore(ctx, t)
+			validateStore := createStore(ctx, t)
 			err = ValidateRawTree(treestorage.TreeStorageCreatePayload{
 				RootRawChange: oTree.Header(),
 				Heads:         []string{oTree.Heads()[0]},
@@ -733,7 +733,7 @@ func TestObjectTree(t *testing.T) {
 				changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"),
 			}
 			emptyDataTreeDeps = nonVerifiableTreeDeps
-			validateStore := newStore(ctx, t)
+			validateStore := createStore(ctx, t)
 			err := ValidateRawTree(treestorage.TreeStorageCreatePayload{
 				RootRawChange: treeCtx.objTree.Header(),
 				Heads:         []string{"3"},
@@ -1601,7 +1601,7 @@ func TestObjectTree(t *testing.T) {
 			changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"),
 		}
 		emptyDataTreeDeps = nonVerifiableTreeDeps
-		validateStore := newStore(ctx, t)
+		validateStore := createStore(ctx, t)
 		err := ValidateRawTree(treestorage.TreeStorageCreatePayload{
 			RootRawChange: treeCtx.objTree.Header(),
 			Heads:         []string{"3"},
@@ -1619,7 +1619,7 @@ func TestObjectTree(t *testing.T) {
 			changeCreator.CreateRaw("3", aclList.Head().Id, "0", true, "2"),
 		}
 		emptyDataTreeDeps = nonVerifiableTreeDeps
-		validateStore := newStore(ctx, t)
+		validateStore := createStore(ctx, t)
 		err := ValidateRawTree(treestorage.TreeStorageCreatePayload{
 			RootRawChange: treeCtx.objTree.Header(),
 			Heads:         []string{"3"},

commonspace/object/tree/objecttree/objecttreevalidator.go

@@ -20,7 +20,7 @@ type tempTreeStorageCreator struct {
 }
 
 func (t *tempTreeStorageCreator) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (Storage, error) {
-	return createStorage(context.Background(), payload.RootRawChange, t.store)
+	return CreateStorage(context.Background(), payload.RootRawChange, t.store)
 }
 
 type ValidatorFunc func(payload treestorage.TreeStorageCreatePayload, storageCreator TreeStorageCreator, aclList list.AclList) (ret ObjectTree, err error)

commonspace/object/tree/objecttree/storage.go

@@ -67,7 +67,7 @@ type storage struct {
 
 var storageChangeBuilder = NewChangeBuilder
 
-func createStorage(ctx context.Context, root *treechangeproto.RawTreeChangeWithId, store anystore.DB) (Storage, error) {
+func CreateStorage(ctx context.Context, root *treechangeproto.RawTreeChangeWithId, store anystore.DB) (Storage, error) {
 	st := &storage{
 		id:    root.Id,
 		store: store,
@@ -129,7 +129,7 @@ func createStorage(ctx context.Context, root *treechangeproto.RawTreeChangeWithI
 	return st, tx.Commit()
 }
 
-func newStorage(ctx context.Context, id string, store anystore.DB) (Storage, error) {
+func NewStorage(ctx context.Context, id string, store anystore.DB) (Storage, error) {
 	// TODO: use spacestorage to set heads
 	st := &storage{
 		id:    id,

commonspace/object/tree/objecttree/testutils.go

@@ -164,7 +164,7 @@ func (c *MockChangeCreator) CreateNewTreeStorage(t *testing.T, treeId, aclHeadId
 			ChangeBuilder: NewChangeBuilder(newMockKeyStorage(), rootChange),
 		}
 	}
-	storage, err := createStorage(context.Background(), root, c.storeCreator())
+	storage, err := CreateStorage(context.Background(), root, c.storeCreator())
 	require.NoError(t, err)
 	return &testStorage{
 		Storage: storage,