anyproto/any-sync
1
0
Fork 0
mirror of https://github.com/anyproto/any-sync.git synced 2025-06-08 05:57:03 +09:00
Code Activity

WIP fix tests and key value

Browse source
This commit is contained in:
Mikhail Rakhmanov 2025-04-11 22:31:55 +02:00
parent 5d2e7ce5fe
commit 21b0b9e835
No known key found for this signature in database
GPG key ID: DED12CFEF5B8396B
7 changed files with 152 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

118
commonspace/object/keyvalue/keyvalue_test.go
View file

@ -6,6 +6,7 @@ import (
"fmt"
"math/rand"
"path/filepath"
"sort"
"strconv"
"strings"
"testing"
@ -121,6 +122,111 @@ func TestKeyValueService(t *testing.T) {
fxServer.check(t, key, clientValue)
fxServer.check(t, key, serverValue)
}
foundClientKeys := make(map[string]bool)
foundServerKeys := make(map[string]bool)
err = fxClient.defaultStore.Iterate(context.Background(), func(decryptor keyvaluestorage.Decryptor, key string, values []innerstorage.KeyValue) (bool, error) {
foundClientKeys[key] = true
return true, nil
})
require.NoError(t, err)
err = fxServer.defaultStore.Iterate(context.Background(), func(decryptor keyvaluestorage.Decryptor, key string, values []innerstorage.KeyValue) (bool, error) {
foundServerKeys[key] = true
return true, nil
})
require.NoError(t, err)
require.True(t, mapEqual(allKeys, foundServerKeys), "expected all client keys to be found")
require.True(t, mapEqual(foundClientKeys, foundServerKeys), "expected all client keys to be found")
})
}
func TestKeyValueServiceIterate(t *testing.T) {
t.Run("empty storage", func(t *testing.T) {
fxClient, _, _ := prepareFixtures(t)
var keys []string
err := fxClient.defaultStore.Iterate(context.Background(), func(decryptor keyvaluestorage.Decryptor, key string, values []innerstorage.KeyValue) (bool, error) {
keys = append(keys, key)
return true, nil
})
require.NoError(t, err)
require.Empty(t, keys, "expected no keys in empty storage")
})
t.Run("single key later value", func(t *testing.T) {
fxClient, _, _ := prepareFixtures(t)
err := fxClient.defaultStore.Set(context.Background(), "test-key", []byte("value1"))
require.NoError(t, err)
err = fxClient.defaultStore.Set(context.Background(), "test-key", []byte("value2"))
require.NoError(t, err)
var keys []string
valueCount := 0
err = fxClient.defaultStore.Iterate(context.Background(), func(decryptor keyvaluestorage.Decryptor, key string, values []innerstorage.KeyValue) (bool, error) {
keys = append(keys, key)
valueCount = len(values)
for _, kv := range values {
val, err := decryptor(kv)
require.NoError(t, err)
require.Equal(t, "value2", string(val))
}
return true, nil
})
require.NoError(t, err)
require.Equal(t, 1, len(keys), "expected one key")
require.Equal(t, "test-key", keys[0], "expected key to be 'test-key'")
require.Equal(t, 1, valueCount, "expected one value for key")
})
t.Run("multiple keys", func(t *testing.T) {
fxClient, _, _ := prepareFixtures(t)
testKeys := []string{"key1", "key2", "key3"}
for _, key := range testKeys {
err := fxClient.defaultStore.Set(context.Background(), key, []byte("value-"+key))
require.NoError(t, err)
}
var foundKeys []string
err := fxClient.defaultStore.Iterate(context.Background(), func(decryptor keyvaluestorage.Decryptor, key string, values []innerstorage.KeyValue) (bool, error) {
foundKeys = append(foundKeys, key)
require.Equal(t, 1, len(values), "Expected one value for key: "+key)
val, err := decryptor(values[0])
require.NoError(t, err)
require.Equal(t, "value-"+key, string(val), "Value doesn't match for key: "+key)
return true, nil
})
require.NoError(t, err)
sort.Strings(foundKeys)
sort.Strings(testKeys)
require.Equal(t, testKeys, foundKeys, "Expected all keys to be found")
})
t.Run("early termination", func(t *testing.T) {
fxClient, _, _ := prepareFixtures(t)
testKeys := []string{"key1", "key2", "key3", "key4", "key5"}
for _, key := range testKeys {
err := fxClient.defaultStore.Set(context.Background(), key, []byte("value-"+key))
require.NoError(t, err)
}
var foundKeys []string
err := fxClient.defaultStore.Iterate(context.Background(), func(decryptor keyvaluestorage.Decryptor, key string, values []innerstorage.KeyValue) (bool, error) {
foundKeys = append(foundKeys, key)
return len(foundKeys) < 2, nil
})
require.NoError(t, err)
require.Equal(t, 2, len(foundKeys), "expected to find exactly 2 keys before stopping")
})
t.Run("error during iteration", func(t *testing.T) {
fxClient, _, _ := prepareFixtures(t)
err := fxClient.defaultStore.Set(context.Background(), "test-key", []byte("test-value"))
require.NoError(t, err)
expectedErr := context.Canceled
err = fxClient.defaultStore.Iterate(context.Background(), func(decryptor keyvaluestorage.Decryptor, key string, values []innerstorage.KeyValue) (bool, error) {
return false, expectedErr
})
require.Equal(t, expectedErr, err, "expected error to be propagated")
})
}
@ -141,6 +247,18 @@ func prepareFixtures(t *testing.T) (fxClient *fixture, fxServer *fixture, server
return
}
func mapEqual[K comparable, V comparable](map1, map2 map[K]V) bool {
if len(map1) != len(map2) {
return false
}
for key, val1 := range map1 {
if val2, ok := map2[key]; !ok || val1 != val2 {
return false
}
}
return true
}
var ctx = context.Background()
type noOpSyncClient struct{}

2
commonspace/object/keyvalue/keyvaluestorage/innerstorage/element.go
View file

@ -37,7 +37,7 @@ func KeyValueFromProto(proto *spacesyncproto.StoreKeyValue, verify bool) (kv Key
if err = innerValue.Unmarshal(proto.Value); err != nil {
return kv, err
}
kv.TimestampMilli = int(innerValue.TimestampMilli)
kv.TimestampMilli = int(innerValue.TimestampMicro)
identity, err := crypto.UnmarshalEd25519PublicKeyProto(innerValue.Identity)
if err != nil {
return kv, err

4
commonspace/object/keyvalue/keyvaluestorage/innerstorage/keyvaluestorage.go
View file

@ -121,7 +121,7 @@ func (s *storage) IterateValues(ctx context.Context, iterFunc func(kv KeyValue)
func (s *storage) IteratePrefix(ctx context.Context, prefix string, iterFunc func(kv KeyValue) error) (err error) {
filter := query.Key{Path: []string{"id"}, Filter: query.NewComp(query.CompOpGte, prefix)}
qry := s.collection.Find(filter).Sort("id")
iter, err := s.collection.Find(qry).Iter(ctx)
iter, err := qry.Iter(ctx)
if err != nil {
return
}
@ -134,7 +134,7 @@ func (s *storage) IteratePrefix(ctx context.Context, prefix string, iterFunc fun
return
}
if !strings.Contains(doc.Value().GetString("id"), prefix) {
continue
break
}
err := iterFunc(s.keyValueFromDoc(doc))
if err != nil {

24
commonspace/object/keyvalue/keyvaluestorage/storage.go
View file

@ -54,7 +54,7 @@ type Storage interface {
Set(ctx context.Context, key string, value []byte) error
SetRaw(ctx context.Context, keyValue ...*spacesyncproto.StoreKeyValue) error
GetAll(ctx context.Context, key string, get func(decryptor Decryptor, values []innerstorage.KeyValue) error) error
Iterate(ctx context.Context, f func(key string, values []innerstorage.KeyValue) (bool, error)) error
Iterate(ctx context.Context, f func(decryptor Decryptor, key string, values []innerstorage.KeyValue) (bool, error)) error
InnerStorage() innerstorage.KeyValueStorage
}
@ -139,12 +139,12 @@ func (s *storage) Set(ctx context.Context, key string, value []byte) error {
if err != nil {
return err
}
timestampMilli := time.Now().UnixMilli()
timestampMicro := time.Now().UnixMicro()
inner := spacesyncproto.StoreKeyInner{
Peer: protoPeerKey,
Identity: protoIdentityKey,
Value: value,
TimestampMilli: timestampMilli,
TimestampMicro: timestampMicro,
AclHeadId: headId,
Key: key,
}
@ -164,7 +164,7 @@ func (s *storage) Set(ctx context.Context, key string, value []byte) error {
keyValue := innerstorage.KeyValue{
KeyPeerId: keyPeerId,
Key: key,
TimestampMilli: int(timestampMilli),
TimestampMilli: int(timestampMicro),
Identity: identityKey.GetPublic().Account(),
PeerId: peerIdKey.GetPublic().PeerId(),
AclId: headId,
@ -251,6 +251,9 @@ func (s *storage) SetRaw(ctx context.Context, keyValue ...*spacesyncproto.StoreK
func (s *storage) GetAll(ctx context.Context, key string, get func(decryptor Decryptor, values []innerstorage.KeyValue) error) (err error) {
var values []innerstorage.KeyValue
err = s.inner.IteratePrefix(ctx, key, func(kv innerstorage.KeyValue) error {
bytes := make([]byte, len(kv.Value.Value))
copy(bytes, kv.Value.Value)
kv.Value.Value = bytes
values = append(values, kv)
return nil
})
@ -297,25 +300,32 @@ func (s *storage) readKeysFromAclState(state *list.AclState) (err error) {
return err
}
func (s *storage) Iterate(ctx context.Context, f func(key string, values []innerstorage.KeyValue) (bool, error)) (err error) {
func (s *storage) Iterate(ctx context.Context, f func(decryptor Decryptor, key string, values []innerstorage.KeyValue) (bool, error)) (err error) {
s.mx.Lock()
defer s.mx.Unlock()
var (
curKey = ""
// TODO: reuse buffer
values []innerstorage.KeyValue
)
err = s.inner.IterateValues(ctx, func(kv innerstorage.KeyValue) (bool, error) {
if kv.Key != curKey {
if curKey != "" {
iter, err := f(curKey, values)
iter, err := f(s.decrypt, curKey, values)
if err != nil {
return false, err
}
if !iter {
values = nil
return false, nil
}
}
curKey = kv.Key
values = values[:0]
}
bytes := make([]byte, len(kv.Value.Value))
copy(bytes, kv.Value.Value)
kv.Value.Value = bytes
values = append(values, kv)
return true, nil
})
@ -323,7 +333,7 @@ func (s *storage) Iterate(ctx context.Context, f func(key string, values []inner
return err
}
if len(values) > 0 {
_, err = f(curKey, values)
_, err = f(s.decrypt, curKey, values)
}
return err
}

2
commonspace/spacesyncproto/protos/spacesync.proto
View file

@ -232,7 +232,7 @@ message StoreKeyInner {
bytes peer = 1;
bytes identity = 2;
bytes value = 3;
int64 timestampMilli = 4;
int64 timestampMicro = 4;
string aclHeadId = 5;
string key = 6;
}

26
commonspace/spacesyncproto/spacesync.pb.go
View file

@ -2033,7 +2033,7 @@ type StoreKeyInner struct {
Peer []byte `protobuf:"bytes,1,opt,name=peer,proto3" json:"peer,omitempty"`
Identity []byte `protobuf:"bytes,2,opt,name=identity,proto3" json:"identity,omitempty"`
Value []byte `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"`
TimestampMilli int64 `protobuf:"varint,4,opt,name=timestampMilli,proto3" json:"timestampMilli,omitempty"`
TimestampMicro int64 `protobuf:"varint,4,opt,name=timestampMicro,proto3" json:"timestampMicro,omitempty"`
AclHeadId string `protobuf:"bytes,5,opt,name=aclHeadId,proto3" json:"aclHeadId,omitempty"`
Key string `protobuf:"bytes,6,opt,name=key,proto3" json:"key,omitempty"`
}
@ -2100,9 +2100,9 @@ func (m *StoreKeyInner) GetValue() []byte {
return nil
}
func (m *StoreKeyInner) GetTimestampMilli() int64 {
func (m *StoreKeyInner) GetTimestampMicro() int64 {
if m != nil {
return m.TimestampMilli
return m.TimestampMicro
}
return 0
}
@ -2293,8 +2293,8 @@ var fileDescriptor_80e49f1f4ac27799 = []byte{
0x4e, 0x99, 0x6e, 0xa6, 0x22, 0xd2, 0x8c, 0xd7, 0x5c, 0x21, 0x5e, 0x78, 0x1f, 0xea, 0x05, 0x93,
0x19, 0xba, 0x25, 0x6d, 0x4e, 0x01, 0xe5, 0xbc, 0x19, 0xc4, 0x02, 0x35, 0xc9, 0x49, 0xf1, 0x8f,
0x86, 0xf7, 0x9d, 0x28, 0xa2, 0x89, 0x58, 0x20, 0xc2, 0x8c, 0xec, 0x05, 0x2d, 0xce, 0x85, 0xa5,
0x66, 0x8f, 0x2c, 0x35, 0x1d, 0x8f, 0x92, 0x19, 0x8f, 0xab, 0x50, 0xd7, 0x9b, 0xed, 0x20, 0x08,
0xc3, 0x40, 0xba, 0x58, 0x22, 0x23, 0x58, 0x11, 0x6b, 0x55, 0x65, 0xda, 0xcb, 0x1c, 0x81, 0x1c,
0x66, 0x8f, 0x2c, 0x35, 0x1d, 0x8f, 0x92, 0x19, 0x8f, 0xab, 0x50, 0xd7, 0x9b, 0xed, 0x20, 0xf0,
0x92, 0x58, 0xba, 0x58, 0x22, 0x23, 0x58, 0x11, 0x6b, 0x55, 0x65, 0xda, 0xcb, 0x1c, 0x81, 0x1c,
0x28, 0x3d, 0xa3, 0x43, 0xb9, 0xa9, 0xaa, 0x44, 0x1c, 0xf1, 0x83, 0xd4, 0x5c, 0xf7, 0xf4, 0x1f,
0x98, 0xa3, 0x1b, 0x7f, 0x58, 0x50, 0xd9, 0x4b, 0x92, 0x9d, 0xd8, 0xa7, 0x0c, 0xd5, 0x01, 0x1e,
0x47, 0xf4, 0xbc, 0x4f, 0x3d, 0x4e, 0x7d, 0x67, 0x06, 0x39, 0xea, 0x6d, 0x73, 0x10, 0x30, 0x16,
@ -2322,7 +2322,7 @@ var fileDescriptor_80e49f1f4ac27799 = []byte{
0xd7, 0x71, 0x5c, 0x2c, 0xec, 0x45, 0x34, 0xc2, 0x31, 0xb6, 0x7d, 0x9b, 0xed, 0xe9, 0x04, 0xa9,
0xcc, 0xbb, 0xef, 0xff, 0xfc, 0xaa, 0x65, 0xbd, 0x7c, 0xd5, 0xb2, 0x7e, 0x7f, 0xd5, 0xb2, 0xbe,
0x7f, 0xdd, 0x9a, 0x79, 0xf9, 0xba, 0x35, 0xf3, 0xdb, 0xeb, 0xd6, 0xcc, 0x67, 0xcd, 0xe9, 0xff,
0x91, 0x39, 0x2e, 0xcb, 0x9f, 0x9b, 0x7f, 0x06, 0x00, 0x00, 0xff, 0xff, 0xf9, 0xe8, 0x68, 0xd7,
0x91, 0x39, 0x2e, 0xcb, 0x9f, 0x9b, 0x7f, 0x06, 0x00, 0x00, 0xff, 0xff, 0xdc, 0x8f, 0x50, 0x36,
0xb6, 0x11, 0x00, 0x00,
}
@ -3591,8 +3591,8 @@ func (m *StoreKeyInner) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i--
dAtA[i] = 0x2a
}
if m.TimestampMilli != 0 {
i = encodeVarintSpacesync(dAtA, i, uint64(m.TimestampMilli))
if m.TimestampMicro != 0 {
i = encodeVarintSpacesync(dAtA, i, uint64(m.TimestampMicro))
i--
dAtA[i] = 0x20
}
@ -4243,8 +4243,8 @@ func (m *StoreKeyInner) Size() (n int) {
if l > 0 {
n += 1 + l + sovSpacesync(uint64(l))
}
if m.TimestampMilli != 0 {
n += 1 + sovSpacesync(uint64(m.TimestampMilli))
if m.TimestampMicro != 0 {
n += 1 + sovSpacesync(uint64(m.TimestampMicro))
}
l = len(m.AclHeadId)
if l > 0 {
@ -7933,9 +7933,9 @@ func (m *StoreKeyInner) Unmarshal(dAtA []byte) error {
iNdEx = postIndex
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field TimestampMilli", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field TimestampMicro", wireType)
}
m.TimestampMilli = 0
m.TimestampMicro = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowSpacesync
@ -7945,7 +7945,7 @@ func (m *StoreKeyInner) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
m.TimestampMilli |= int64(b&0x7F) << shift
m.TimestampMicro |= int64(b&0x7F) << shift
if b < 0x80 {
break
}

8
commonspace/sync/sync_test.go
View file

@ -115,14 +115,6 @@ func TestSyncService(t *testing.T) {
}
require.Equal(t, headUpdate, f.syncHandler.headUpdate)
})
t.Run("handle key value message no op", func(t *testing.T) {
f := newFixture(t)
headUpdate := &testMessage{objectId: "objectId", objectType: spacesyncproto.ObjectType_KeyValue}
err := f.HandleMessage(ctx, headUpdate)
require.NoError(t, err)
f.Close(t)
require.Nil(t, f.syncHandler.headUpdate)
})
t.Run("handle message", func(t *testing.T) {
f := newFixture(t)
f.syncHandler.toReceiveData = map[string][]*testResponse{