anyproto/any-sync
1
0
Fork 0
mirror of https://github.com/anyproto/any-sync.git synced 2025-06-07 21:47:02 +09:00
Code Activity

merge main

Browse source
This commit is contained in:
Sergey Cherepanov 2025-05-13 13:31:52 +02:00
commit c61950bb5f
No known key found for this signature in database
GPG key ID: 87F8EDE8FBDF637C
52 changed files with 5057 additions and 502 deletions
Download patch file Download diff file Expand all files Collapse all files

86
app/ldiff/diff.go
View file

@ -42,6 +42,11 @@ type Diff interface {
DiffType() spacesyncproto.DiffType
}
type CompareDiff interface {
CompareDiff(ctx context.Context, dl Remote) (newIds, ourChangedIds, theirChangedIds, removedIds []string, err error)
Diff
}
// New creates precalculated Diff container
//
// divideFactor - means how many hashes you want to ask for once
@ -273,17 +278,18 @@ func (d *diff) Ranges(ctx context.Context, ranges []Range, resBuf []RangeResult)
}
type diffCtx struct {
newIds, changedIds, removedIds []string
newIds, changedIds, theirChangedIds, removedIds []string
toSend, prepare []Range
myRes, otherRes []RangeResult
compareFunc func(dctx *diffCtx, my, other []Element)
}
var errMismatched = errors.New("query and results mismatched")
// Diff makes diff with remote container
func (d *diff) Diff(ctx context.Context, dl Remote) (newIds, changedIds, removedIds []string, err error) {
dctx := &diffCtx{}
dctx := &diffCtx{compareFunc: d.compareElementsEqual}
dctx.toSend = append(dctx.toSend, Range{
From: 0,
To: math.MaxUint64,
@ -314,18 +320,51 @@ func (d *diff) Diff(ctx context.Context, dl Remote) (newIds, changedIds, removed
return dctx.newIds, dctx.changedIds, dctx.removedIds, nil
}
func (d *diff) CompareDiff(ctx context.Context, dl Remote) (newIds, ourChangedIds, theirChangedIds, removedIds []string, err error) {
dctx := &diffCtx{compareFunc: d.compareElementsGreater}
dctx.toSend = append(dctx.toSend, Range{
From: 0,
To: math.MaxUint64,
})
for len(dctx.toSend) > 0 {
select {
case <-ctx.Done():
err = ctx.Err()
return
default:
}
if dctx.otherRes, err = dl.Ranges(ctx, dctx.toSend, dctx.otherRes); err != nil {
return
}
if dctx.myRes, err = d.Ranges(ctx, dctx.toSend, dctx.myRes); err != nil {
return
}
if len(dctx.otherRes) != len(dctx.toSend) || len(dctx.myRes) != len(dctx.toSend) {
err = errMismatched
return
}
for i, r := range dctx.toSend {
d.compareResults(dctx, r, dctx.myRes[i], dctx.otherRes[i])
}
dctx.toSend, dctx.prepare = dctx.prepare, dctx.toSend
dctx.prepare = dctx.prepare[:0]
}
return dctx.newIds, dctx.changedIds, dctx.theirChangedIds, dctx.removedIds, nil
}
func (d *diff) compareResults(dctx *diffCtx, r Range, myRes, otherRes RangeResult) {
// both hash equals - do nothing
if bytes.Equal(myRes.Hash, otherRes.Hash) {
return
}
// other has elements
if len(otherRes.Elements) == otherRes.Count {
if len(myRes.Elements) == myRes.Count {
d.compareElements(dctx, myRes.Elements, otherRes.Elements)
dctx.compareFunc(dctx, myRes.Elements, otherRes.Elements)
} else {
r.Elements = true
d.compareElements(dctx, d.getRange(r).Elements, otherRes.Elements)
dctx.compareFunc(dctx, d.getRange(r).Elements, otherRes.Elements)
}
return
}
@ -341,7 +380,7 @@ func (d *diff) compareResults(dctx *diffCtx, r Range, myRes, otherRes RangeResul
return
}
func (d *diff) compareElements(dctx *diffCtx, my, other []Element) {
func (d *diff) compareElementsEqual(dctx *diffCtx, my, other []Element) {
find := func(list []Element, targetEl Element) (has, eq bool) {
for _, el := range list {
if el.Id == targetEl.Id {
@ -369,3 +408,40 @@ func (d *diff) compareElements(dctx *diffCtx, my, other []Element) {
}
}
}
func (d *diff) compareElementsGreater(dctx *diffCtx, my, other []Element) {
find := func(list []Element, targetEl Element) (has, equal, greater bool) {
for _, el := range list {
if el.Id == targetEl.Id {
if el.Head == targetEl.Head {
return true, true, false
}
return true, false, el.Head > targetEl.Head
}
}
return false, false, false
}
for _, el := range my {
has, eq, theirGreater := find(other, el)
if !has {
dctx.removedIds = append(dctx.removedIds, el.Id)
continue
} else {
if eq {
continue
}
if theirGreater {
dctx.theirChangedIds = append(dctx.theirChangedIds, el.Id)
} else {
dctx.changedIds = append(dctx.changedIds, el.Id)
}
}
}
for _, el := range other {
if has, _, _ := find(my, el); !has {
dctx.newIds = append(dctx.newIds, el.Id)
}
}
}

29
app/ldiff/diff_test.go
View file

@ -139,6 +139,35 @@ func TestDiff_Diff(t *testing.T) {
assert.Len(t, changedIds, length/2)
assert.Len(t, removedIds, 0)
})
t.Run("compare diff", func(t *testing.T) {
d1 := New(16, 128).(CompareDiff)
d2 := New(16, 128)
length := 10000
for i := 0; i < length; i++ {
id := fmt.Sprint(i)
head := "a" + uuid.NewString()
d1.Set(Element{
Id: id,
Head: head,
})
}
for i := 0; i < length; i++ {
id := fmt.Sprint(i)
head := "b" + uuid.NewString()
d2.Set(Element{
Id: id,
Head: head,
})
}
newIds, changedIds, theirChangedIds, removedIds, err := d1.CompareDiff(ctx, d2)
require.NoError(t, err)
assert.Len(t, newIds, 0)
assert.Len(t, changedIds, 0)
assert.Len(t, theirChangedIds, length)
assert.Len(t, removedIds, 0)
})
t.Run("empty", func(t *testing.T) {
d1 := New(16, 16)
d2 := New(16, 16)

5
commonspace/deletion_test.go
View file

@ -11,6 +11,7 @@ import (
"github.com/anyproto/any-sync/commonspace/object/accountdata"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
"github.com/anyproto/any-sync/commonspace/spacepayloads"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/any-sync/commonspace/syncstatus"
"github.com/anyproto/any-sync/util/crypto"
@ -47,7 +48,7 @@ func TestSpaceDeleteIdsMarkDeleted(t *testing.T) {
totalObjs := 1000
// creating space
sp, err := fx.spaceService.CreateSpace(ctx, SpaceCreatePayload{
sp, err := fx.spaceService.CreateSpace(ctx, spacepayloads.SpaceCreatePayload{
SigningKey: acc.SignKey,
SpaceType: "type",
ReadKey: rk,
@ -140,7 +141,7 @@ func TestSpaceDeleteIds(t *testing.T) {
totalObjs := 1000
// creating space
sp, err := fx.spaceService.CreateSpace(ctx, SpaceCreatePayload{
sp, err := fx.spaceService.CreateSpace(ctx, spacepayloads.SpaceCreatePayload{
SigningKey: acc.SignKey,
SpaceType: "type",
ReadKey: rk,

43
commonspace/headsync/diffsyncer.go
View file

@ -8,6 +8,7 @@ import (
"github.com/quic-go/quic-go"
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces"
"github.com/anyproto/any-sync/commonspace/object/treesyncer"
"github.com/anyproto/any-sync/net/rpc/rpcerr"
@ -42,6 +43,7 @@ func newDiffSyncer(hs *headSync) DiffSyncer {
peerManager: hs.peerManager,
clientFactory: spacesyncproto.ClientFactoryFunc(spacesyncproto.NewDRPCSpaceSyncClient),
credentialProvider: hs.credentialProvider,
keyValue: hs.keyValue,
log: newSyncLogger(hs.log, logPeriodSecs),
deletionState: hs.deletionState,
syncAcl: hs.syncAcl,
@ -63,6 +65,7 @@ type diffSyncer struct {
cancel context.CancelFunc
deletionState deletionstate.ObjectDeletionState
credentialProvider credentialprovider.CredentialProvider
keyValue kvinterfaces.KeyValueService
syncAcl syncacl.SyncAcl
}
@ -90,10 +93,17 @@ func (d *diffSyncer) updateHeads(update headstorage.HeadsUpdate) {
if update.IsDerived != nil && *update.IsDerived && len(update.Heads) == 1 && update.Heads[0] == update.Id {
return
}
d.diffContainer.Set(ldiff.Element{
Id: update.Id,
Head: concatStrings(update.Heads),
})
if update.Id == d.keyValue.DefaultStore().Id() {
d.diffContainer.NewDiff().Set(ldiff.Element{
Id: update.Id,
Head: concatStrings(update.Heads),
})
} else {
d.diffContainer.Set(ldiff.Element{
Id: update.Id,
Head: concatStrings(update.Heads),
})
}
}
// probably we should somehow batch the updates
oldHash := d.diffContainer.OldDiff().Hash()
@ -126,7 +136,6 @@ func (d *diffSyncer) Sync(ctx context.Context) error {
}
}
d.log.DebugCtx(ctx, "diff done", zap.String("spaceId", d.spaceId), zap.Duration("dur", time.Since(st)))
d.peerManager.KeepAlive(ctx)
return nil
}
@ -153,6 +162,7 @@ func (d *diffSyncer) syncWithPeer(ctx context.Context, p peer.Peer) (err error)
syncAclId = d.syncAcl.Id()
newIds, changedIds, removedIds []string
)
storageId := d.keyValue.DefaultStore().Id()
needsSync, diff, err := d.diffContainer.DiffTypeCheck(ctx, rdiff)
err = rpcerr.Unwrap(err)
if err != nil {
@ -169,17 +179,32 @@ func (d *diffSyncer) syncWithPeer(ctx context.Context, p peer.Peer) (err error)
// not syncing ids which were removed through settings document
missingIds := d.deletionState.Filter(newIds)
existingIds := append(d.deletionState.Filter(removedIds), d.deletionState.Filter(changedIds)...)
prevExistingLen := len(existingIds)
var (
isStorage = false
isAcl = false
)
existingIds = slice.DiscardFromSlice(existingIds, func(s string) bool {
return s == syncAclId
if s == storageId {
isStorage = true
return true
}
if s == syncAclId {
isAcl = true
return true
}
return false
})
// if we removed acl head from the list
if len(existingIds) < prevExistingLen {
if isAcl {
if syncErr := d.syncAcl.SyncWithPeer(ctx, p); syncErr != nil {
log.Warn("failed to send acl sync message to peer", zap.String("aclId", syncAclId))
}
}
if isStorage {
if err = d.keyValue.SyncWithPeer(p); err != nil {
log.Warn("failed to send storage sync message to peer", zap.String("storageId", storageId))
}
}
// treeSyncer should not get acl id, that's why we filter existing ids before
err = d.treeSyncer.SyncAll(ctx, p, existingIds, missingIds)

25
commonspace/headsync/diffsyncer_test.go
View file

@ -118,6 +118,31 @@ func TestDiffSyncer(t *testing.T) {
require.NoError(t, fx.diffSyncer.Sync(ctx))
})
t.Run("diff syncer sync, store changed", func(t *testing.T) {
fx := newHeadSyncFixture(t)
fx.initDiffSyncer(t)
defer fx.stop()
mPeer := rpctest.MockPeer{}
remDiff := NewRemoteDiff(fx.spaceState.SpaceId, fx.clientMock)
fx.treeSyncerMock.EXPECT().ShouldSync(gomock.Any()).Return(true)
fx.aclMock.EXPECT().Id().AnyTimes().Return("aclId")
fx.peerManagerMock.EXPECT().
GetResponsiblePeers(gomock.Any()).
Return([]peer.Peer{mPeer}, nil)
fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, nil)
fx.diffMock.EXPECT().
Diff(gomock.Any(), gomock.Eq(remDiff)).
Return([]string{"new"}, []string{"changed"}, nil, nil)
fx.deletionStateMock.EXPECT().Filter([]string{"new"}).Return([]string{"new"}).Times(1)
fx.deletionStateMock.EXPECT().Filter([]string{"changed"}).Return([]string{"changed", "store"}).Times(1)
fx.deletionStateMock.EXPECT().Filter(nil).Return(nil).Times(1)
fx.treeSyncerMock.EXPECT().SyncAll(gomock.Any(), mPeer, []string{"changed"}, []string{"new"}).Return(nil)
fx.kvMock.EXPECT().SyncWithPeer(mPeer).Return(nil)
fx.peerManagerMock.EXPECT().KeepAlive(gomock.Any())
require.NoError(t, fx.diffSyncer.Sync(ctx))
})
t.Run("diff syncer sync conf error", func(t *testing.T) {
fx := newHeadSyncFixture(t)
fx.initDiffSyncer(t)

25
commonspace/headsync/headsync.go
View file

@ -16,6 +16,7 @@ import (
"github.com/anyproto/any-sync/commonspace/deletionstate"
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
"github.com/anyproto/any-sync/commonspace/object/acl/syncacl"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces"
"github.com/anyproto/any-sync/commonspace/object/treesyncer"
"github.com/anyproto/any-sync/commonspace/peermanager"
"github.com/anyproto/any-sync/commonspace/spacestate"
@ -58,6 +59,7 @@ type headSync struct {
credentialProvider credentialprovider.CredentialProvider
deletionState deletionstate.ObjectDeletionState
syncAcl syncacl.SyncAcl
keyValue kvinterfaces.KeyValueService
}
func New() HeadSync {
@ -80,6 +82,7 @@ func (h *headSync) Init(a *app.App) (err error) {
h.credentialProvider = a.MustComponent(credentialprovider.CName).(credentialprovider.CredentialProvider)
h.treeSyncer = a.MustComponent(treesyncer.CName).(treesyncer.TreeSyncer)
h.deletionState = a.MustComponent(deletionstate.CName).(deletionstate.ObjectDeletionState)
h.keyValue = a.MustComponent(kvinterfaces.CName).(kvinterfaces.KeyValueService)
h.syncer = createDiffSyncer(h)
sync := func(ctx context.Context) (err error) {
return h.syncer.Sync(ctx)
@ -117,8 +120,9 @@ func (h *headSync) AllIds() []string {
func (h *headSync) ExternalIds() []string {
settingsId := h.storage.StateStorage().SettingsId()
aclId := h.syncAcl.Id()
keyValueId := h.keyValue.DefaultStore().Id()
return slice.DiscardFromSlice(h.AllIds(), func(id string) bool {
return id == settingsId || id == aclId
return id == settingsId || id == aclId || id == keyValueId
})
}
@ -130,14 +134,23 @@ func (h *headSync) Close(ctx context.Context) (err error) {
func (h *headSync) fillDiff(ctx context.Context) error {
var els = make([]ldiff.Element, 0, 100)
var aclOrStorage []ldiff.Element
err := h.storage.HeadStorage().IterateEntries(ctx, headstorage.IterOpts{}, func(entry headstorage.HeadsEntry) (bool, error) {
if entry.IsDerived && entry.Heads[0] == entry.Id {
return true, nil
}
els = append(els, ldiff.Element{
Id: entry.Id,
Head: concatStrings(entry.Heads),
})
if entry.CommonSnapshot != "" {
els = append(els, ldiff.Element{
Id: entry.Id,
Head: concatStrings(entry.Heads),
})
} else {
// this whole stuff is done to prevent storage hash from being set to old diff
aclOrStorage = append(aclOrStorage, ldiff.Element{
Id: entry.Id,
Head: concatStrings(entry.Heads),
})
}
return true, nil
})
if err != nil {
@ -149,6 +162,8 @@ func (h *headSync) fillDiff(ctx context.Context) error {
})
log.Debug("setting acl", zap.String("aclId", h.syncAcl.Id()), zap.String("headId", h.syncAcl.Head().Id))
h.diffContainer.Set(els...)
// acl will be set twice to the diff but it doesn't matter
h.diffContainer.NewDiff().Set(aclOrStorage...)
oldHash := h.diffContainer.OldDiff().Hash()
newHash := h.diffContainer.NewDiff().Hash()
if err := h.storage.StateStorage().SetHash(ctx, oldHash, newHash); err != nil {

52
commonspace/headsync/headsync_test.go
View file

@ -22,6 +22,9 @@ import (
"github.com/anyproto/any-sync/commonspace/object/acl/list"
"github.com/anyproto/any-sync/commonspace/object/acl/syncacl"
"github.com/anyproto/any-sync/commonspace/object/acl/syncacl/mock_syncacl"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage/mock_keyvaluestorage"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces/mock_kvinterfaces"
"github.com/anyproto/any-sync/commonspace/object/treemanager"
"github.com/anyproto/any-sync/commonspace/object/treemanager/mock_treemanager"
"github.com/anyproto/any-sync/commonspace/object/treesyncer"
@ -34,6 +37,7 @@ import (
"github.com/anyproto/any-sync/commonspace/spacesyncproto/mock_spacesyncproto"
"github.com/anyproto/any-sync/nodeconf"
"github.com/anyproto/any-sync/nodeconf/mock_nodeconf"
"github.com/anyproto/any-sync/testutil/anymock"
)
type mockConfig struct {
@ -57,6 +61,8 @@ type headSyncFixture struct {
app *app.App
configurationMock *mock_nodeconf.MockService
kvMock *mock_kvinterfaces.MockKeyValueService
defStoreMock *mock_keyvaluestorage.MockStorage
storageMock *mock_spacestorage.MockSpaceStorage
peerManagerMock *mock_peermanager.MockPeerManager
credentialProviderMock *mock_credentialprovider.MockCredentialProvider
@ -96,6 +102,11 @@ func newHeadSyncFixture(t *testing.T) *headSyncFixture {
treeSyncerMock := mock_treesyncer.NewMockTreeSyncer(ctrl)
headStorage := mock_headstorage.NewMockHeadStorage(ctrl)
stateStorage := mock_statestorage.NewMockStateStorage(ctrl)
kvMock := mock_kvinterfaces.NewMockKeyValueService(ctrl)
anymock.ExpectComp(kvMock.EXPECT(), kvinterfaces.CName)
defStore := mock_keyvaluestorage.NewMockStorage(ctrl)
kvMock.EXPECT().DefaultStore().Return(defStore).AnyTimes()
defStore.EXPECT().Id().Return("store").AnyTimes()
storageMock.EXPECT().HeadStorage().AnyTimes().Return(headStorage)
storageMock.EXPECT().StateStorage().AnyTimes().Return(stateStorage)
treeSyncerMock.EXPECT().Name().AnyTimes().Return(treesyncer.CName)
@ -108,6 +119,7 @@ func newHeadSyncFixture(t *testing.T) *headSyncFixture {
a := &app.App{}
a.Register(spaceState).
Register(aclMock).
Register(kvMock).
Register(mockConfig{}).
Register(configurationMock).
Register(storageMock).
@ -121,6 +133,8 @@ func newHeadSyncFixture(t *testing.T) *headSyncFixture {
spaceState: spaceState,
ctrl: ctrl,
app: a,
kvMock: kvMock,
defStoreMock: defStore,
configurationMock: configurationMock,
storageMock: storageMock,
diffContainerMock: diffContainerMock,
@ -164,14 +178,16 @@ func TestHeadSync(t *testing.T) {
headEntries := []headstorage.HeadsEntry{
{
Id: "id1",
Heads: []string{"h1", "h2"},
IsDerived: false,
Id: "id1",
Heads: []string{"h1", "h2"},
CommonSnapshot: "id1",
IsDerived: false,
},
{
Id: "id2",
Heads: []string{"h3", "h4"},
IsDerived: false,
Id: "id2",
Heads: []string{"h3", "h4"},
CommonSnapshot: "id2",
IsDerived: false,
},
}
fx.headStorage.EXPECT().IterateEntries(gomock.Any(), gomock.Any(), gomock.Any()).
@ -196,8 +212,9 @@ func TestHeadSync(t *testing.T) {
Id: "aclId",
Head: "headId",
})
fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock)
fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock)
fx.diffMock.EXPECT().Set([]ldiff.Element{})
fx.diffContainerMock.EXPECT().NewDiff().AnyTimes().Return(fx.diffMock)
fx.diffContainerMock.EXPECT().OldDiff().AnyTimes().Return(fx.diffMock)
fx.diffMock.EXPECT().Hash().AnyTimes().Return("hash")
fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil)
fx.diffSyncerMock.EXPECT().Sync(gomock.Any()).Return(nil)
@ -215,14 +232,16 @@ func TestHeadSync(t *testing.T) {
headEntries := []headstorage.HeadsEntry{
{
Id: "id1",
Heads: []string{"id1"},
IsDerived: true,
Id: "id1",
Heads: []string{"id1"},
CommonSnapshot: "id1",
IsDerived: true,
},
{
Id: "id2",
Heads: []string{"h3", "h4"},
IsDerived: false,
Id: "id2",
Heads: []string{"h3", "h4"},
CommonSnapshot: "id2",
IsDerived: false,
},
}
fx.headStorage.EXPECT().IterateEntries(gomock.Any(), gomock.Any(), gomock.Any()).
@ -244,8 +263,9 @@ func TestHeadSync(t *testing.T) {
Id: "aclId",
Head: "headId",
})
fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock)
fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock)
fx.diffMock.EXPECT().Set([]ldiff.Element{})
fx.diffContainerMock.EXPECT().NewDiff().AnyTimes().Return(fx.diffMock)
fx.diffContainerMock.EXPECT().OldDiff().AnyTimes().Return(fx.diffMock)
fx.diffMock.EXPECT().Hash().AnyTimes().Return("hash")
fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil)
fx.diffSyncerMock.EXPECT().Sync(gomock.Any()).Return(nil)

20
commonspace/headsync/statestorage/statestorage.go
View file

@ -33,6 +33,7 @@ const (
idKey = "id"
oldHashKey = "oh"
newHashKey = "nh"
legacyHashKey = "h"
headerKey = "e"
aclIdKey = "a"
settingsIdKey = "s"
@ -108,12 +109,8 @@ func Create(ctx context.Context, state State, store anystore.DB) (st StateStorag
return nil, err
}
storage, err := CreateTx(tx.Context(), state, store)
defer func() {
if err != nil {
tx.Rollback()
}
}()
if err != nil {
tx.Rollback()
return nil, err
}
return storage, tx.Commit()
@ -149,12 +146,21 @@ func (s *stateStorage) SettingsId() string {
}
func (s *stateStorage) stateFromDoc(doc anystore.Doc) State {
var (
oldHash = doc.Value().GetString(oldHashKey)
newHash = doc.Value().GetString(newHashKey)
)
// legacy hash is used for backward compatibility, which was due to a mistake in key names
if oldHash == "" || newHash == "" {
oldHash = doc.Value().GetString(legacyHashKey)
newHash = oldHash
}
return State{
SpaceId: doc.Value().GetString(idKey),
SettingsId: doc.Value().GetString(settingsIdKey),
AclId: doc.Value().GetString(aclIdKey),
OldHash: doc.Value().GetString(newHashKey),
NewHash: doc.Value().GetString(oldHashKey),
OldHash: oldHash,
NewHash: newHash,
SpaceHeader: doc.Value().GetBytes(headerKey),
}
}

15
commonspace/mock_commonspace/mock_commonspace.go
View file

@ -18,6 +18,7 @@ import (
aclclient "github.com/anyproto/any-sync/commonspace/acl/aclclient"
headsync "github.com/anyproto/any-sync/commonspace/headsync"
syncacl "github.com/anyproto/any-sync/commonspace/object/acl/syncacl"
kvinterfaces "github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces"
treesyncer "github.com/anyproto/any-sync/commonspace/object/treesyncer"
objecttreebuilder "github.com/anyproto/any-sync/commonspace/objecttreebuilder"
spacestorage "github.com/anyproto/any-sync/commonspace/spacestorage"
@ -224,6 +225,20 @@ func (mr *MockSpaceMockRecorder) Init(ctx any) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockSpace)(nil).Init), ctx)
}
// KeyValue mocks base method.
func (m *MockSpace) KeyValue() kvinterfaces.KeyValueService {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "KeyValue")
ret0, _ := ret[0].(kvinterfaces.KeyValueService)
return ret0
}
// KeyValue indicates an expected call of KeyValue.
func (mr *MockSpaceMockRecorder) KeyValue() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeyValue", reflect.TypeOf((*MockSpace)(nil).KeyValue))
}
// Storage mocks base method.
func (m *MockSpace) Storage() spacestorage.SpaceStorage {
m.ctrl.T.Helper()

21
commonspace/object/acl/list/aclstate.go
View file

@ -130,6 +130,20 @@ func (st *AclState) CurrentReadKeyId() string {
return st.readKeyChanges[len(st.readKeyChanges)-1]
}
func (st *AclState) ReadKeyForAclId(id string) (string, error) {
recIdx, ok := st.list.indexes[id]
if !ok {
return "", ErrNoSuchRecord
}
for i := len(st.readKeyChanges) - 1; i >= 0; i-- {
recId := st.readKeyChanges[i]
if recIdx >= st.list.indexes[recId] {
return recId, nil
}
}
return "", ErrNoSuchRecord
}
func (st *AclState) AccountKey() crypto.PrivKey {
return st.key
}
@ -150,6 +164,13 @@ func (st *AclState) CurrentMetadataKey() (crypto.PubKey, error) {
return curKeys.MetadataPubKey, nil
}
func (st *AclState) FirstMetadataKey() (crypto.PrivKey, error) {
if firstKey, ok := st.keys[st.id]; ok && firstKey.MetadataPrivKey != nil {
return firstKey.MetadataPrivKey, nil
}
return nil, ErrNoMetadataKey
}
func (st *AclState) Keys() map[string]AclKeys {
return st.keys
}

43
commonspace/object/acl/list/aclstate_test.go
View file

@ -1,8 +1,11 @@
package list
import (
"crypto/rand"
"testing"
"github.com/anyproto/any-sync/util/crypto"
"github.com/stretchr/testify/require"
)
@ -68,3 +71,43 @@ func TestAclStateIsEmpty(t *testing.T) {
require.True(t, st.IsEmpty())
})
}
func TestAclState_FirstMetadataKey(t *testing.T) {
t.Run("returns first metadata key successfully", func(t *testing.T) {
privKey, _, err := crypto.GenerateEd25519Key(rand.Reader)
require.NoError(t, err)
pubKey := privKey.GetPublic()
readKey := crypto.NewAES()
state := &AclState{
id: "recordId",
keys: map[string]AclKeys{
"recordId": {
ReadKey: readKey,
MetadataPrivKey: privKey,
MetadataPubKey: pubKey,
},
},
}
key, err := state.FirstMetadataKey()
require.NoError(t, err)
require.Equal(t, privKey, key)
})
t.Run("first metadata is nil", func(t *testing.T) {
state := &AclState{
id: "recordId",
keys: map[string]AclKeys{
"recordId": {
ReadKey: crypto.NewAES(),
},
},
}
key, err := state.FirstMetadataKey()
require.ErrorIs(t, err, ErrNoMetadataKey)
require.Nil(t, key)
})
t.Run("returns error when no read key changes", func(t *testing.T) {
state := &AclState{}
_, err := state.FirstMetadataKey()
require.ErrorIs(t, err, ErrNoMetadataKey)
})
}

23
commonspace/object/acl/list/storage.go
View file

@ -66,12 +66,8 @@ func CreateStorage(ctx context.Context, root *consensusproto.RawRecordWithId, he
return nil, err
}
storage, err := CreateStorageTx(tx.Context(), root, headStorage, store)
defer func() {
if err != nil {
tx.Rollback()
}
}()
if err != nil {
tx.Rollback()
return nil, err
}
return storage, tx.Commit()
@ -210,6 +206,13 @@ func (s *storage) AddAll(ctx context.Context, records []StorageRecord) error {
if err != nil {
return fmt.Errorf("failed to create write tx: %w", err)
}
defer func() {
if err != nil {
_ = tx.Rollback()
} else {
err = tx.Commit()
}
}()
vals := make([]*anyenc.Value, 0, len(records))
for _, ch := range records {
newVal := newStorageRecordValue(ch, arena)
@ -217,20 +220,14 @@ func (s *storage) AddAll(ctx context.Context, records []StorageRecord) error {
}
err = s.recordsColl.Insert(tx.Context(), vals...)
if err != nil {
tx.Rollback()
return nil
return err
}
head := records[len(records)-1].Id
update := headstorage.HeadsUpdate{
Id: s.id,
Heads: []string{head},
}
err = s.headStorage.UpdateEntryTx(tx.Context(), update)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
return s.headStorage.UpdateEntryTx(tx.Context(), update)
}
func (s *storage) Id() string {

5
commonspace/object/acl/syncacl/headupdate.go
View file

@ -1,6 +1,7 @@
package syncacl
import (
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
"github.com/anyproto/any-sync/commonspace/sync/objectsync/objectmessages"
"github.com/anyproto/any-sync/consensus/consensusproto"
)
@ -24,6 +25,10 @@ func (h *InnerHeadUpdate) MsgSize() uint64 {
return size + uint64(len(h.head)) + uint64(len(h.root.Id)) + uint64(len(h.root.Payload))
}
func (h *InnerHeadUpdate) ObjectType() spacesyncproto.ObjectType {
return spacesyncproto.ObjectType_Acl
}
func (h *InnerHeadUpdate) Prepare() error {
logMsg := consensusproto.WrapHeadUpdate(&consensusproto.LogHeadUpdate{
Head: h.head,

245
commonspace/object/keyvalue/keyvalue.go Normal file
View file

@ -0,0 +1,245 @@
package keyvalue
import (
"context"
"errors"
"go.uber.org/zap"
"storj.io/drpc"
"github.com/anyproto/any-sync/accountservice"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/app/logger"
"github.com/anyproto/any-sync/commonspace/object/acl/list"
"github.com/anyproto/any-sync/commonspace/object/acl/syncacl"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage/syncstorage"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces"
"github.com/anyproto/any-sync/commonspace/spacestate"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
"github.com/anyproto/any-sync/commonspace/sync"
"github.com/anyproto/any-sync/commonspace/sync/objectsync/objectmessages"
"github.com/anyproto/any-sync/net/peer"
"github.com/anyproto/any-sync/net/rpc/rpcerr"
"github.com/anyproto/any-sync/util/cidutil"
)
var ErrUnexpectedMessageType = errors.New("unexpected message type")
var log = logger.NewNamed(kvinterfaces.CName)
type keyValueService struct {
storageId string
spaceId string
ctx context.Context
cancel context.CancelFunc
limiter *concurrentLimiter
defaultStore keyvaluestorage.Storage
clientFactory spacesyncproto.ClientFactory
}
func New() kvinterfaces.KeyValueService {
return &keyValueService{}
}
func (k *keyValueService) DefaultStore() keyvaluestorage.Storage {
return k.defaultStore
}
func (k *keyValueService) SyncWithPeer(p peer.Peer) (err error) {
k.limiter.ScheduleRequest(k.ctx, p.Id(), func() {
err = k.syncWithPeer(k.ctx, p)
if err != nil {
log.Error("failed to sync with peer", zap.String("peerId", p.Id()), zap.Error(err))
}
})
return nil
}
func (k *keyValueService) syncWithPeer(ctx context.Context, p peer.Peer) (err error) {
conn, err := p.AcquireDrpcConn(ctx)
if err != nil {
return
}
defer p.ReleaseDrpcConn(conn)
var (
client = k.clientFactory.Client(conn)
rdiff = NewRemoteDiff(k.spaceId, client)
diff = k.defaultStore.InnerStorage().Diff()
)
newIds, changedIds, theirChangedIds, removedIds, err := diff.CompareDiff(ctx, rdiff)
err = rpcerr.Unwrap(err)
if err != nil {
return err
}
innerStorage := k.defaultStore.InnerStorage()
stream, err := client.StoreElements(ctx)
if err != nil {
return err
}
defer stream.CloseSend()
err = stream.Send(&spacesyncproto.StoreKeyValue{SpaceId: k.spaceId})
if err != nil {
return err
}
for _, id := range append(removedIds, changedIds...) {
kv, err := innerStorage.GetKeyPeerId(ctx, id)
if err != nil {
return err
}
err = stream.Send(kv.Proto())
if err != nil {
return err
}
}
for _, id := range append(theirChangedIds, newIds...) {
kv := &spacesyncproto.StoreKeyValue{
KeyPeerId: id,
}
err := stream.Send(kv)
if err != nil {
return err
}
}
err = stream.Send(&spacesyncproto.StoreKeyValue{})
if err != nil {
return err
}
var messages []*spacesyncproto.StoreKeyValue
for {
msg, err := stream.Recv()
if err != nil {
return err
}
if msg.KeyPeerId == "" {
break
}
messages = append(messages, msg)
}
return k.defaultStore.SetRaw(ctx, messages...)
}
func (k *keyValueService) HandleStoreDiffRequest(ctx context.Context, req *spacesyncproto.StoreDiffRequest) (resp *spacesyncproto.StoreDiffResponse, err error) {
return HandleRangeRequest(ctx, k.defaultStore.InnerStorage().Diff(), req)
}
func (k *keyValueService) HandleStoreElementsRequest(ctx context.Context, stream spacesyncproto.DRPCSpaceSync_StoreElementsStream) (err error) {
var (
messagesToSave []*spacesyncproto.StoreKeyValue
messagesToSend []string
)
for {
msg, err := stream.Recv()
if err != nil {
return err
}
if msg.KeyPeerId == "" {
break
}
if msg.Value != nil {
messagesToSave = append(messagesToSave, msg)
} else {
messagesToSend = append(messagesToSend, msg.KeyPeerId)
}
}
innerStorage := k.defaultStore.InnerStorage()
isError := false
for _, msg := range messagesToSend {
kv, err := innerStorage.GetKeyPeerId(ctx, msg)
if err != nil {
log.Warn("failed to get key value", zap.String("key", msg), zap.Error(err))
continue
}
err = stream.Send(kv.Proto())
if err != nil {
log.Warn("failed to send key value", zap.String("key", msg), zap.Error(err))
isError = true
break
}
}
if !isError {
err = stream.Send(&spacesyncproto.StoreKeyValue{})
if err != nil {
return err
}
}
return k.defaultStore.SetRaw(ctx, messagesToSave...)
}
func (k *keyValueService) HandleMessage(ctx context.Context, headUpdate drpc.Message) (err error) {
update, ok := headUpdate.(*objectmessages.HeadUpdate)
if !ok {
return ErrUnexpectedMessageType
}
keyValueMsg := &spacesyncproto.StoreKeyValues{}
err = keyValueMsg.UnmarshalVT(update.Bytes)
if err != nil {
objectmessages.FreeHeadUpdate(update)
return err
}
objectmessages.FreeHeadUpdate(update)
return k.defaultStore.SetRaw(ctx, keyValueMsg.KeyValues...)
}
func (k *keyValueService) Init(a *app.App) (err error) {
k.ctx, k.cancel = context.WithCancel(context.Background())
spaceState := a.MustComponent(spacestate.CName).(*spacestate.SpaceState)
k.spaceId = spaceState.SpaceId
k.clientFactory = spacesyncproto.ClientFactoryFunc(spacesyncproto.NewDRPCSpaceSyncClient)
k.limiter = newConcurrentLimiter()
accountService := a.MustComponent(accountservice.CName).(accountservice.Service)
aclList := a.MustComponent(syncacl.CName).(list.AclList)
spaceStorage := a.MustComponent(spacestorage.CName).(spacestorage.SpaceStorage)
syncService := a.MustComponent(sync.CName).(sync.SyncService)
k.storageId, err = storageIdFromSpace(k.spaceId)
if err != nil {
return err
}
indexer := a.Component(keyvaluestorage.IndexerCName).(keyvaluestorage.Indexer)
if indexer == nil {
indexer = keyvaluestorage.NoOpIndexer{}
}
syncClient := syncstorage.New(spaceState.SpaceId, syncService)
k.defaultStore, err = keyvaluestorage.New(
k.ctx,
k.storageId,
spaceStorage.AnyStore(),
spaceStorage.HeadStorage(),
accountService.Account(),
syncClient,
aclList,
indexer)
return
}
func (k *keyValueService) Name() (name string) {
return kvinterfaces.CName
}
func (k *keyValueService) Run(ctx context.Context) (err error) {
return k.defaultStore.Prepare()
}
func (k *keyValueService) Close(ctx context.Context) (err error) {
k.cancel()
k.limiter.Close()
return nil
}
func storageIdFromSpace(spaceId string) (storageId string, err error) {
header := &spacesyncproto.StorageHeader{
SpaceId: spaceId,
StorageName: "default",
}
data, err := header.MarshalVT()
if err != nil {
return "", err
}
cid, err := cidutil.NewCidFromBytes(data)
if err != nil {
return "", err
}
return cid, nil
}

371
commonspace/object/keyvalue/keyvalue_test.go Normal file
View file

@ -0,0 +1,371 @@
package keyvalue
import (
"bytes"
"context"
"fmt"
"math/rand"
"path/filepath"
"sort"
"strconv"
"strings"
"testing"
"time"
anystore "github.com/anyproto/any-store"
"github.com/stretchr/testify/require"
"github.com/anyproto/any-sync/commonspace/object/accountdata"
"github.com/anyproto/any-sync/commonspace/object/acl/list"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage/innerstorage"
"github.com/anyproto/any-sync/commonspace/spacepayloads"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
"github.com/anyproto/any-sync/net/peer"
"github.com/anyproto/any-sync/net/rpc/rpctest"
"github.com/anyproto/any-sync/util/crypto"
)
func TestKeyValueService(t *testing.T) {
t.Run("different keys", func(t *testing.T) {
fxClient, fxServer, serverPeer := prepareFixtures(t)
fxClient.add(t, "key1", []byte("value1"))
fxClient.add(t, "key2", []byte("value2"))
fxServer.add(t, "key3", []byte("value3"))
fxServer.add(t, "key4", []byte("value4"))
err := fxClient.SyncWithPeer(serverPeer)
require.NoError(t, err)
fxClient.limiter.Close()
fxClient.check(t, "key3", []byte("value3"))
fxClient.check(t, "key4", []byte("value4"))
fxServer.check(t, "key1", []byte("value1"))
fxServer.check(t, "key2", []byte("value2"))
})
t.Run("change same keys, different values", func(t *testing.T) {
fxClient, fxServer, serverPeer := prepareFixtures(t)
fxClient.add(t, "key1", []byte("value1"))
fxServer.add(t, "key1", []byte("value2"))
err := fxClient.SyncWithPeer(serverPeer)
require.NoError(t, err)
fxClient.limiter.Close()
fxClient.check(t, "key1", []byte("value1"))
fxClient.check(t, "key1", []byte("value2"))
fxServer.check(t, "key1", []byte("value1"))
fxServer.check(t, "key1", []byte("value2"))
fxClient.add(t, "key1", []byte("value1-2"))
fxServer.add(t, "key1", []byte("value2-2"))
err = fxClient.SyncWithPeer(serverPeer)
require.NoError(t, err)
fxClient.limiter.Close()
fxClient.check(t, "key1", []byte("value1-2"))
fxClient.check(t, "key1", []byte("value2-2"))
fxServer.check(t, "key1", []byte("value1-2"))
fxServer.check(t, "key1", []byte("value2-2"))
})
t.Run("random keys and values", func(t *testing.T) {
rand.Seed(time.Now().UnixNano())
diffEntries := 100
ovelappingEntries := 10
fxClient, fxServer, serverPeer := prepareFixtures(t)
numClientEntries := 5 + rand.Intn(diffEntries)
numServerEntries := 5 + rand.Intn(diffEntries)
allKeys := make(map[string]bool)
for i := 0; i < numClientEntries; i++ {
key := fmt.Sprintf("client-key-%d", i)
value := []byte(fmt.Sprintf("client-value-%d", i))
fxClient.add(t, key, value)
allKeys[key] = true
}
for i := 0; i < numServerEntries; i++ {
key := fmt.Sprintf("server-key-%d", i)
value := []byte(fmt.Sprintf("server-value-%d", i))
fxServer.add(t, key, value)
allKeys[key] = true
}
numOverlappingKeys := 3 + rand.Intn(ovelappingEntries)
for i := 0; i < numOverlappingKeys; i++ {
key := fmt.Sprintf("overlap-key-%d", i)
clientValue := []byte(fmt.Sprintf("client-overlap-value-%d", i))
serverValue := []byte(fmt.Sprintf("server-overlap-value-%d", i))
fxClient.add(t, key, clientValue)
fxServer.add(t, key, serverValue)
allKeys[key] = true
}
err := fxClient.SyncWithPeer(serverPeer)
require.NoError(t, err)
fxClient.limiter.Close()
for key := range allKeys {
if strings.HasPrefix(key, "client-key-") {
i, _ := strconv.Atoi(strings.TrimPrefix(key, "client-key-"))
value := []byte(fmt.Sprintf("client-value-%d", i))
fxClient.check(t, key, value)
fxServer.check(t, key, value)
}
if strings.HasPrefix(key, "server-key-") {
i, _ := strconv.Atoi(strings.TrimPrefix(key, "server-key-"))
value := []byte(fmt.Sprintf("server-value-%d", i))
fxClient.check(t, key, value)
fxServer.check(t, key, value)
}
}
for i := 0; i < numOverlappingKeys; i++ {
key := fmt.Sprintf("overlap-key-%d", i)
clientValue := []byte(fmt.Sprintf("client-overlap-value-%d", i))
serverValue := []byte(fmt.Sprintf("server-overlap-value-%d", i))
fxClient.check(t, key, clientValue)
fxClient.check(t, key, serverValue)
fxServer.check(t, key, clientValue)
fxServer.check(t, key, serverValue)
}
foundClientKeys := make(map[string]bool)
foundServerKeys := make(map[string]bool)
err = fxClient.defaultStore.Iterate(context.Background(), func(decryptor keyvaluestorage.Decryptor, key string, values []innerstorage.KeyValue) (bool, error) {
foundClientKeys[key] = true
return true, nil
})
require.NoError(t, err)
err = fxServer.defaultStore.Iterate(context.Background(), func(decryptor keyvaluestorage.Decryptor, key string, values []innerstorage.KeyValue) (bool, error) {
foundServerKeys[key] = true
return true, nil
})
require.NoError(t, err)
require.True(t, mapEqual(allKeys, foundServerKeys), "expected all client keys to be found")
require.True(t, mapEqual(foundClientKeys, foundServerKeys), "expected all client keys to be found")
})
}
func TestKeyValueServiceIterate(t *testing.T) {
t.Run("empty storage", func(t *testing.T) {
fxClient, _, _ := prepareFixtures(t)
var keys []string
err := fxClient.defaultStore.Iterate(context.Background(), func(decryptor keyvaluestorage.Decryptor, key string, values []innerstorage.KeyValue) (bool, error) {
keys = append(keys, key)
return true, nil
})
require.NoError(t, err)
require.Empty(t, keys, "expected no keys in empty storage")
})
t.Run("single key later value", func(t *testing.T) {
fxClient, _, _ := prepareFixtures(t)
err := fxClient.defaultStore.Set(context.Background(), "test-key", []byte("value1"))
require.NoError(t, err)
err = fxClient.defaultStore.Set(context.Background(), "test-key", []byte("value2"))
require.NoError(t, err)
var keys []string
valueCount := 0
err = fxClient.defaultStore.Iterate(context.Background(), func(decryptor keyvaluestorage.Decryptor, key string, values []innerstorage.KeyValue) (bool, error) {
keys = append(keys, key)
valueCount = len(values)
for _, kv := range values {
val, err := decryptor(kv)
require.NoError(t, err)
require.Equal(t, "value2", string(val))
}
return true, nil
})
require.NoError(t, err)
require.Equal(t, 1, len(keys), "expected one key")
require.Equal(t, "test-key", keys[0], "expected key to be 'test-key'")
require.Equal(t, 1, valueCount, "expected one value for key")
})
t.Run("multiple keys", func(t *testing.T) {
fxClient, _, _ := prepareFixtures(t)
testKeys := []string{"key1", "key2", "key3"}
for _, key := range testKeys {
err := fxClient.defaultStore.Set(context.Background(), key, []byte("value-"+key))
require.NoError(t, err)
}
var foundKeys []string
err := fxClient.defaultStore.Iterate(context.Background(), func(decryptor keyvaluestorage.Decryptor, key string, values []innerstorage.KeyValue) (bool, error) {
foundKeys = append(foundKeys, key)
require.Equal(t, 1, len(values), "Expected one value for key: "+key)
val, err := decryptor(values[0])
require.NoError(t, err)
require.Equal(t, "value-"+key, string(val), "Value doesn't match for key: "+key)
return true, nil
})
require.NoError(t, err)
sort.Strings(foundKeys)
sort.Strings(testKeys)
require.Equal(t, testKeys, foundKeys, "Expected all keys to be found")
})
t.Run("early termination", func(t *testing.T) {
fxClient, _, _ := prepareFixtures(t)
testKeys := []string{"key1", "key2", "key3", "key4", "key5"}
for _, key := range testKeys {
err := fxClient.defaultStore.Set(context.Background(), key, []byte("value-"+key))
require.NoError(t, err)
}
var foundKeys []string
err := fxClient.defaultStore.Iterate(context.Background(), func(decryptor keyvaluestorage.Decryptor, key string, values []innerstorage.KeyValue) (bool, error) {
foundKeys = append(foundKeys, key)
return len(foundKeys) < 2, nil
})
require.NoError(t, err)
require.Equal(t, 2, len(foundKeys), "expected to find exactly 2 keys before stopping")
})
t.Run("error during iteration", func(t *testing.T) {
fxClient, _, _ := prepareFixtures(t)
err := fxClient.defaultStore.Set(context.Background(), "test-key", []byte("test-value"))
require.NoError(t, err)
expectedErr := context.Canceled
err = fxClient.defaultStore.Iterate(context.Background(), func(decryptor keyvaluestorage.Decryptor, key string, values []innerstorage.KeyValue) (bool, error) {
return false, expectedErr
})
require.Equal(t, expectedErr, err, "expected error to be propagated")
})
}
func prepareFixtures(t *testing.T) (fxClient *fixture, fxServer *fixture, serverPeer peer.Peer) {
firstKeys, err := accountdata.NewRandom()
require.NoError(t, err)
secondKeys, err := accountdata.NewRandom()
require.NoError(t, err)
secondKeys.SignKey = firstKeys.SignKey
payload := newStorageCreatePayload(t, firstKeys)
fxClient = newFixture(t, firstKeys, payload)
fxServer = newFixture(t, secondKeys, payload)
serverConn, clientConn := rpctest.MultiConnPair(firstKeys.PeerId, secondKeys.PeerId)
serverPeer, err = peer.NewPeer(serverConn, fxClient.server)
require.NoError(t, err)
_, err = peer.NewPeer(clientConn, fxServer.server)
require.NoError(t, err)
return
}
func mapEqual[K comparable, V comparable](map1, map2 map[K]V) bool {
if len(map1) != len(map2) {
return false
}
for key, val1 := range map1 {
if val2, ok := map2[key]; !ok || val1 != val2 {
return false
}
}
return true
}
var ctx = context.Background()
type noOpSyncClient struct{}
func (n noOpSyncClient) Broadcast(ctx context.Context, objectId string, keyValues ...innerstorage.KeyValue) error {
return nil
}
type fixture struct {
*keyValueService
server *rpctest.TestServer
}
func newFixture(t *testing.T, keys *accountdata.AccountKeys, spacePayload spacestorage.SpaceStorageCreatePayload) *fixture {
storePath := filepath.Join(t.TempDir(), "store.db")
anyStore, err := anystore.Open(ctx, storePath, nil)
require.NoError(t, err)
storage, err := spacestorage.Create(ctx, anyStore, spacePayload)
require.NoError(t, err)
aclStorage, err := storage.AclStorage()
require.NoError(t, err)
aclList, err := list.BuildAclListWithIdentity(keys, aclStorage, list.NoOpAcceptorVerifier{})
require.NoError(t, err)
storageId := "kv.storage"
rpcHandler := rpctest.NewTestServer()
defaultStorage, err := keyvaluestorage.New(ctx,
storageId,
anyStore,
storage.HeadStorage(),
keys,
noOpSyncClient{},
aclList,
keyvaluestorage.NoOpIndexer{})
require.NoError(t, err)
ctx, cancel := context.WithCancel(ctx)
service := &keyValueService{
spaceId: storage.Id(),
storageId: storageId,
limiter: newConcurrentLimiter(),
ctx: ctx,
cancel: cancel,
clientFactory: spacesyncproto.ClientFactoryFunc(spacesyncproto.NewDRPCSpaceSyncClient),
defaultStore: defaultStorage,
}
require.NoError(t, spacesyncproto.DRPCRegisterSpaceSync(rpcHandler, &testServer{service: service, t: t}))
return &fixture{
keyValueService: service,
server: rpcHandler,
}
}
func (fx *fixture) add(t *testing.T, key string, value []byte) {
err := fx.defaultStore.Set(ctx, key, value)
require.NoError(t, err)
}
func (fx *fixture) check(t *testing.T, key string, value []byte) (isFound bool) {
err := fx.defaultStore.GetAll(ctx, key, func(decryptor keyvaluestorage.Decryptor, values []innerstorage.KeyValue) error {
for _, v := range values {
decryptedValue, err := decryptor(v)
require.NoError(t, err)
if bytes.Equal(value, decryptedValue) {
isFound = true
break
}
}
return nil
})
require.NoError(t, err)
return
}
func newStorageCreatePayload(t *testing.T, keys *accountdata.AccountKeys) spacestorage.SpaceStorageCreatePayload {
masterKey, _, err := crypto.GenerateRandomEd25519KeyPair()
require.NoError(t, err)
metaKey, _, err := crypto.GenerateRandomEd25519KeyPair()
require.NoError(t, err)
readKey := crypto.NewAES()
meta := []byte("account")
payload := spacepayloads.SpaceCreatePayload{
SigningKey: keys.SignKey,
SpaceType: "space",
ReplicationKey: 10,
SpacePayload: nil,
MasterKey: masterKey,
ReadKey: readKey,
MetadataKey: metaKey,
Metadata: meta,
}
createSpace, err := spacepayloads.StoragePayloadForSpaceCreate(payload)
require.NoError(t, err)
return createSpace
}
type testServer struct {
spacesyncproto.DRPCSpaceSyncUnimplementedServer
service *keyValueService
t *testing.T
}
func (t *testServer) StoreDiff(ctx context.Context, req *spacesyncproto.StoreDiffRequest) (*spacesyncproto.StoreDiffResponse, error) {
return t.service.HandleStoreDiffRequest(ctx, req)
}
func (t *testServer) StoreElements(stream spacesyncproto.DRPCSpaceSync_StoreElementsStream) error {
msg, err := stream.Recv()
require.NoError(t.t, err)
require.NotEmpty(t.t, msg.SpaceId)
return t.service.HandleStoreElementsRequest(ctx, stream)
}

92
commonspace/object/keyvalue/keyvaluestorage/innerstorage/element.go Normal file
View file

@ -0,0 +1,92 @@
package innerstorage
import (
"errors"
"github.com/anyproto/any-store/anyenc"
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
"github.com/anyproto/any-sync/util/crypto"
)
var ErrInvalidSignature = errors.New("invalid signature")
type KeyValue struct {
KeyPeerId string
ReadKeyId string
Key string
Value Value
TimestampMilli int
Identity string
PeerId string
AclId string
}
type Value struct {
Value []byte
PeerSignature []byte
IdentitySignature []byte
}
func KeyValueFromProto(proto *spacesyncproto.StoreKeyValue, verify bool) (kv KeyValue, err error) {
kv.KeyPeerId = proto.KeyPeerId
kv.Value.Value = proto.Value
kv.Value.PeerSignature = proto.PeerSignature
kv.Value.IdentitySignature = proto.IdentitySignature
innerValue := &spacesyncproto.StoreKeyInner{}
if err = innerValue.UnmarshalVT(proto.Value); err != nil {
return kv, err
}
kv.TimestampMilli = int(innerValue.TimestampMicro)
identity, err := crypto.UnmarshalEd25519PublicKeyProto(innerValue.Identity)
if err != nil {
return kv, err
}
peerId, err := crypto.UnmarshalEd25519PublicKeyProto(innerValue.Peer)
if err != nil {
return kv, err
}
kv.Identity = identity.Account()
kv.PeerId = peerId.PeerId()
kv.Key = innerValue.Key
kv.AclId = innerValue.AclHeadId
// TODO: check that key-peerId is equal to key+peerId?
if verify {
if verify, _ = identity.Verify(proto.Value, proto.IdentitySignature); !verify {
return kv, ErrInvalidSignature
}
if verify, _ = peerId.Verify(proto.Value, proto.PeerSignature); !verify {
return kv, ErrInvalidSignature
}
}
return kv, nil
}
func (v Value) AnyEnc(a *anyenc.Arena) *anyenc.Value {
obj := a.NewObject()
obj.Set("v", a.NewBinary(v.Value))
obj.Set("p", a.NewBinary(v.PeerSignature))
obj.Set("i", a.NewBinary(v.IdentitySignature))
return obj
}
func (kv KeyValue) AnyEnc(a *anyenc.Arena) *anyenc.Value {
obj := a.NewObject()
obj.Set("id", a.NewString(kv.KeyPeerId))
obj.Set("k", a.NewString(kv.Key))
obj.Set("r", a.NewString(kv.ReadKeyId))
obj.Set("v", kv.Value.AnyEnc(a))
obj.Set("t", a.NewNumberInt(kv.TimestampMilli))
obj.Set("i", a.NewString(kv.Identity))
obj.Set("p", a.NewString(kv.PeerId))
return obj
}
func (kv KeyValue) Proto() *spacesyncproto.StoreKeyValue {
return &spacesyncproto.StoreKeyValue{
KeyPeerId: kv.KeyPeerId,
Value: kv.Value.Value,
PeerSignature: kv.Value.PeerSignature,
IdentitySignature: kv.Value.IdentitySignature,
}
}

249
commonspace/object/keyvalue/keyvaluestorage/innerstorage/keyvaluestorage.go Normal file
View file

@ -0,0 +1,249 @@
package innerstorage
import (
"context"
"encoding/binary"
"errors"
"strings"
anystore "github.com/anyproto/any-store"
"github.com/anyproto/any-store/anyenc"
"github.com/anyproto/any-store/query"
"github.com/anyproto/any-sync/app/ldiff"
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
)
var (
parserPool = &anyenc.ParserPool{}
arenaPool = &anyenc.ArenaPool{}
)
type KeyValueStorage interface {
Set(ctx context.Context, keyValues ...KeyValue) (err error)
Diff() ldiff.CompareDiff
GetKeyPeerId(ctx context.Context, keyPeerId string) (keyValue KeyValue, err error)
IterateValues(context.Context, func(kv KeyValue) (bool, error)) (err error)
IteratePrefix(context.Context, string, func(kv KeyValue) error) (err error)
}
type storage struct {
diff ldiff.CompareDiff
headStorage headstorage.HeadStorage
collection anystore.Collection
store anystore.DB
storageName string
}
func New(ctx context.Context, storageName string, headStorage headstorage.HeadStorage, store anystore.DB) (kv KeyValueStorage, err error) {
collection, err := store.Collection(ctx, storageName)
if err != nil {
return nil, err
}
tx, err := store.WriteTx(ctx)
if err != nil {
return nil, err
}
defer func() {
if err != nil {
_ = tx.Rollback()
} else {
err = tx.Commit()
}
}()
storage := &storage{
storageName: storageName,
headStorage: headStorage,
collection: collection,
store: store,
diff: ldiff.New(32, 256).(ldiff.CompareDiff),
}
iter, err := storage.collection.Find(nil).Iter(ctx)
if err != nil {
return
}
defer func() {
_ = iter.Close()
}()
var (
doc anystore.Doc
elements []ldiff.Element
)
for iter.Next() {
if doc, err = iter.Doc(); err != nil {
return
}
elements = append(elements, anyEncToElement(doc.Value()))
}
storage.diff.Set(elements...)
hash := storage.diff.Hash()
err = headStorage.UpdateEntryTx(tx.Context(), headstorage.HeadsUpdate{
Id: storageName,
Heads: []string{hash},
})
return storage, err
}
func (s *storage) Diff() ldiff.CompareDiff {
return s.diff
}
func (s *storage) GetKeyPeerId(ctx context.Context, keyPeerId string) (value KeyValue, err error) {
doc, err := s.collection.FindId(ctx, keyPeerId)
if err != nil {
return
}
return s.keyValueFromDoc(doc), nil
}
func (s *storage) IterateValues(ctx context.Context, iterFunc func(kv KeyValue) (bool, error)) (err error) {
iter, err := s.collection.Find(nil).Iter(ctx)
if err != nil {
return
}
defer func() {
_ = iter.Close()
}()
var doc anystore.Doc
for iter.Next() {
if doc, err = iter.Doc(); err != nil {
return
}
continueIteration, err := iterFunc(s.keyValueFromDoc(doc))
if err != nil {
return err
}
if !continueIteration {
break
}
}
return nil
}
func (s *storage) IteratePrefix(ctx context.Context, prefix string, iterFunc func(kv KeyValue) error) (err error) {
filter := query.Key{Path: []string{"id"}, Filter: query.NewComp(query.CompOpGte, prefix)}
qry := s.collection.Find(filter).Sort("id")
iter, err := qry.Iter(ctx)
if err != nil {
return
}
defer func() {
_ = iter.Close()
}()
var doc anystore.Doc
for iter.Next() {
if doc, err = iter.Doc(); err != nil {
return
}
if !strings.Contains(doc.Value().GetString("id"), prefix) {
break
}
err := iterFunc(s.keyValueFromDoc(doc))
if err != nil {
return err
}
}
return nil
}
func (s *storage) keyValueFromDoc(doc anystore.Doc) KeyValue {
valueObj := doc.Value().GetObject("v")
value := Value{
Value: valueObj.Get("v").GetBytes(),
PeerSignature: valueObj.Get("p").GetBytes(),
IdentitySignature: valueObj.Get("i").GetBytes(),
}
return KeyValue{
KeyPeerId: doc.Value().GetString("id"),
ReadKeyId: doc.Value().GetString("r"),
Value: value,
TimestampMilli: doc.Value().GetInt("t"),
Identity: doc.Value().GetString("i"),
PeerId: doc.Value().GetString("p"),
Key: doc.Value().GetString("k"),
}
}
func (s *storage) init(ctx context.Context) (err error) {
s.diff = ldiff.New(32, 256).(ldiff.CompareDiff)
iter, err := s.collection.Find(nil).Iter(ctx)
if err != nil {
return
}
defer func() {
_ = iter.Close()
}()
var doc anystore.Doc
var elements []ldiff.Element
for iter.Next() {
if doc, err = iter.Doc(); err != nil {
return
}
elements = append(elements, anyEncToElement(doc.Value()))
}
s.diff.Set(elements...)
return
}
func (s *storage) Set(ctx context.Context, values ...KeyValue) (err error) {
tx, err := s.collection.WriteTx(ctx)
if err != nil {
return
}
defer func() {
if err != nil {
_ = tx.Rollback()
} else {
err = tx.Commit()
}
}()
ctx = tx.Context()
elements, err := s.updateValues(ctx, values...)
if err != nil {
return
}
s.diff.Set(elements...)
err = s.headStorage.UpdateEntryTx(ctx, headstorage.HeadsUpdate{
Id: s.storageName,
Heads: []string{s.diff.Hash()},
})
return
}
func (s *storage) updateValues(ctx context.Context, values ...KeyValue) (elements []ldiff.Element, err error) {
parser := parserPool.Get()
defer parserPool.Put(parser)
arena := arenaPool.Get()
defer arenaPool.Put(arena)
elements = make([]ldiff.Element, 0, len(values))
var doc anystore.Doc
for _, value := range values {
doc, err = s.collection.FindIdWithParser(ctx, parser, value.KeyPeerId)
isNotFound := errors.Is(err, anystore.ErrDocNotFound)
if err != nil && !isNotFound {
return
}
if !isNotFound {
if doc.Value().GetInt("t") >= value.TimestampMilli {
continue
}
}
arena.Reset()
val := value.AnyEnc(arena)
if err = s.collection.UpsertOne(ctx, val); err != nil {
return
}
elements = append(elements, anyEncToElement(val))
}
return
}
func anyEncToElement(val *anyenc.Value) ldiff.Element {
byteRepr := make([]byte, 8)
binary.BigEndian.PutUint64(byteRepr, uint64(val.GetInt("t")))
return ldiff.Element{
Id: val.GetString("id"),
Head: string(byteRepr),
}
}

144
commonspace/object/keyvalue/keyvaluestorage/mock_keyvaluestorage/mock_keyvaluestorage.go Normal file
View file

@ -0,0 +1,144 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage (interfaces: Storage)
//
// Generated by this command:
//
// mockgen -destination mock_keyvaluestorage/mock_keyvaluestorage.go github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage Storage
//
// Package mock_keyvaluestorage is a generated GoMock package.
package mock_keyvaluestorage
import (
context "context"
reflect "reflect"
innerstorage "github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage/innerstorage"
spacesyncproto "github.com/anyproto/any-sync/commonspace/spacesyncproto"
gomock "go.uber.org/mock/gomock"
)
// MockStorage is a mock of Storage interface.
type MockStorage struct {
ctrl *gomock.Controller
recorder *MockStorageMockRecorder
}
// MockStorageMockRecorder is the mock recorder for MockStorage.
type MockStorageMockRecorder struct {
mock *MockStorage
}
// NewMockStorage creates a new mock instance.
func NewMockStorage(ctrl *gomock.Controller) *MockStorage {
mock := &MockStorage{ctrl: ctrl}
mock.recorder = &MockStorageMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use.
func (m *MockStorage) EXPECT() *MockStorageMockRecorder {
return m.recorder
}
// GetAll mocks base method.
func (m *MockStorage) GetAll(arg0 context.Context, arg1 string, arg2 func(func(innerstorage.KeyValue) ([]byte, error), []innerstorage.KeyValue) error) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetAll", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
return ret0
}
// GetAll indicates an expected call of GetAll.
func (mr *MockStorageMockRecorder) GetAll(arg0, arg1, arg2 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAll", reflect.TypeOf((*MockStorage)(nil).GetAll), arg0, arg1, arg2)
}
// Id mocks base method.
func (m *MockStorage) Id() string {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Id")
ret0, _ := ret[0].(string)
return ret0
}
// Id indicates an expected call of Id.
func (mr *MockStorageMockRecorder) Id() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Id", reflect.TypeOf((*MockStorage)(nil).Id))
}
// InnerStorage mocks base method.
func (m *MockStorage) InnerStorage() innerstorage.KeyValueStorage {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "InnerStorage")
ret0, _ := ret[0].(innerstorage.KeyValueStorage)
return ret0
}
// InnerStorage indicates an expected call of InnerStorage.
func (mr *MockStorageMockRecorder) InnerStorage() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InnerStorage", reflect.TypeOf((*MockStorage)(nil).InnerStorage))
}
// Iterate mocks base method.
func (m *MockStorage) Iterate(arg0 context.Context, arg1 func(func(innerstorage.KeyValue) ([]byte, error), string, []innerstorage.KeyValue) (bool, error)) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Iterate", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// Iterate indicates an expected call of Iterate.
func (mr *MockStorageMockRecorder) Iterate(arg0, arg1 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Iterate", reflect.TypeOf((*MockStorage)(nil).Iterate), arg0, arg1)
}
// Prepare mocks base method.
func (m *MockStorage) Prepare() error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Prepare")
ret0, _ := ret[0].(error)
return ret0
}
// Prepare indicates an expected call of Prepare.
func (mr *MockStorageMockRecorder) Prepare() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Prepare", reflect.TypeOf((*MockStorage)(nil).Prepare))
}
// Set mocks base method.
func (m *MockStorage) Set(arg0 context.Context, arg1 string, arg2 []byte) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Set", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
return ret0
}
// Set indicates an expected call of Set.
func (mr *MockStorageMockRecorder) Set(arg0, arg1, arg2 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockStorage)(nil).Set), arg0, arg1, arg2)
}
// SetRaw mocks base method.
func (m *MockStorage) SetRaw(arg0 context.Context, arg1 ...*spacesyncproto.StoreKeyValue) error {
m.ctrl.T.Helper()
varargs := []any{arg0}
for _, a := range arg1 {
varargs = append(varargs, a)
}
ret := m.ctrl.Call(m, "SetRaw", varargs...)
ret0, _ := ret[0].(error)
return ret0
}
// SetRaw indicates an expected call of SetRaw.
func (mr *MockStorageMockRecorder) SetRaw(arg0 any, arg1 ...any) *gomock.Call {
mr.mock.ctrl.T.Helper()
varargs := append([]any{arg0}, arg1...)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetRaw", reflect.TypeOf((*MockStorage)(nil).SetRaw), varargs...)
}

367
commonspace/object/keyvalue/keyvaluestorage/storage.go Normal file
View file

@ -0,0 +1,367 @@
//go:generate mockgen -destination mock_keyvaluestorage/mock_keyvaluestorage.go github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage Storage
package keyvaluestorage
import (
"context"
"encoding/binary"
"fmt"
"sync"
"time"
anystore "github.com/anyproto/any-store"
"go.uber.org/zap"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/app/logger"
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
"github.com/anyproto/any-sync/commonspace/object/accountdata"
"github.com/anyproto/any-sync/commonspace/object/acl/list"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage/innerstorage"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage/syncstorage"
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
"github.com/anyproto/any-sync/util/crypto"
"github.com/anyproto/any-sync/util/slice"
)
var log = logger.NewNamed("common.keyvalue.keyvaluestorage")
const IndexerCName = "common.keyvalue.indexer"
type Indexer interface {
app.Component
Index(decryptor Decryptor, keyValue ...innerstorage.KeyValue) error
}
type Decryptor = func(kv innerstorage.KeyValue) (value []byte, err error)
type NoOpIndexer struct{}
func (n NoOpIndexer) Init(a *app.App) (err error) {
return nil
}
func (n NoOpIndexer) Name() (name string) {
return IndexerCName
}
func (n NoOpIndexer) Index(decryptor Decryptor, keyValue ...innerstorage.KeyValue) error {
return nil
}
type Storage interface {
Id() string
Prepare() error
Set(ctx context.Context, key string, value []byte) error
SetRaw(ctx context.Context, keyValue ...*spacesyncproto.StoreKeyValue) error
GetAll(ctx context.Context, key string, get func(decryptor Decryptor, values []innerstorage.KeyValue) error) error
Iterate(ctx context.Context, f func(decryptor Decryptor, key string, values []innerstorage.KeyValue) (bool, error)) error
InnerStorage() innerstorage.KeyValueStorage
}
type storage struct {
inner innerstorage.KeyValueStorage
keys *accountdata.AccountKeys
aclList list.AclList
syncClient syncstorage.SyncClient
indexer Indexer
storageId string
byteRepr []byte
readKeys map[string]crypto.SymKey
currentReadKey crypto.SymKey
mx sync.Mutex
}
func New(
ctx context.Context,
storageId string,
store anystore.DB,
headStorage headstorage.HeadStorage,
keys *accountdata.AccountKeys,
syncClient syncstorage.SyncClient,
aclList list.AclList,
indexer Indexer,
) (Storage, error) {
inner, err := innerstorage.New(ctx, storageId, headStorage, store)
if err != nil {
return nil, err
}
s := &storage{
inner: inner,
keys: keys,
storageId: storageId,
aclList: aclList,
indexer: indexer,
syncClient: syncClient,
byteRepr: make([]byte, 8),
readKeys: make(map[string]crypto.SymKey),
}
return s, nil
}
func (s *storage) Prepare() error {
s.aclList.RLock()
defer s.aclList.RUnlock()
return s.readKeysFromAclState(s.aclList.AclState())
}
func (s *storage) Id() string {
return s.storageId
}
func (s *storage) Set(ctx context.Context, key string, value []byte) error {
s.mx.Lock()
defer s.mx.Unlock()
s.aclList.RLock()
headId := s.aclList.Head().Id
state := s.aclList.AclState()
if !s.aclList.AclState().Permissions(state.Identity()).CanWrite() {
s.aclList.RUnlock()
return list.ErrInsufficientPermissions
}
readKeyId := state.CurrentReadKeyId()
err := s.readKeysFromAclState(state)
if err != nil {
s.aclList.RUnlock()
return err
}
s.aclList.RUnlock()
value, err = s.currentReadKey.Encrypt(value)
if err != nil {
return err
}
peerIdKey := s.keys.PeerKey
identityKey := s.keys.SignKey
protoPeerKey, err := peerIdKey.GetPublic().Marshall()
if err != nil {
return err
}
protoIdentityKey, err := identityKey.GetPublic().Marshall()
if err != nil {
return err
}
timestampMicro := time.Now().UnixMicro()
inner := spacesyncproto.StoreKeyInner{
Peer: protoPeerKey,
Identity: protoIdentityKey,
Value: value,
TimestampMicro: timestampMicro,
AclHeadId: headId,
Key: key,
}
innerBytes, err := inner.MarshalVT()
if err != nil {
return err
}
peerSig, err := peerIdKey.Sign(innerBytes)
if err != nil {
return err
}
identitySig, err := identityKey.Sign(innerBytes)
if err != nil {
return err
}
keyPeerId := key + "-" + peerIdKey.GetPublic().PeerId()
keyValue := innerstorage.KeyValue{
KeyPeerId: keyPeerId,
Key: key,
TimestampMilli: int(timestampMicro),
Identity: identityKey.GetPublic().Account(),
PeerId: peerIdKey.GetPublic().PeerId(),
AclId: headId,
ReadKeyId: readKeyId,
Value: innerstorage.Value{
Value: innerBytes,
PeerSignature: peerSig,
IdentitySignature: identitySig,
},
}
err = s.inner.Set(ctx, keyValue)
if err != nil {
return err
}
indexErr := s.indexer.Index(s.decrypt, keyValue)
if indexErr != nil {
log.Warn("failed to index for key", zap.String("key", key), zap.Error(indexErr))
}
sendErr := s.syncClient.Broadcast(ctx, s.storageId, keyValue)
if sendErr != nil {
log.Warn("failed to send key value", zap.String("key", key), zap.Error(sendErr))
}
return nil
}
func (s *storage) SetRaw(ctx context.Context, keyValue ...*spacesyncproto.StoreKeyValue) (err error) {
if len(keyValue) == 0 {
return nil
}
s.mx.Lock()
defer s.mx.Unlock()
keyValues := make([]innerstorage.KeyValue, 0, len(keyValue))
for _, kv := range keyValue {
innerKv, err := innerstorage.KeyValueFromProto(kv, true)
if err != nil {
return err
}
keyValues = append(keyValues, innerKv)
}
s.aclList.RLock()
state := s.aclList.AclState()
err = s.readKeysFromAclState(state)
if err != nil {
s.aclList.RUnlock()
return err
}
for i := range keyValues {
el, err := s.inner.Diff().Element(keyValues[i].KeyPeerId)
if err == nil {
binary.BigEndian.PutUint64(s.byteRepr, uint64(keyValues[i].TimestampMilli))
if el.Head >= string(s.byteRepr) {
keyValues[i].KeyPeerId = ""
continue
}
}
keyValues[i].ReadKeyId, err = state.ReadKeyForAclId(keyValues[i].AclId)
if err != nil {
keyValues[i].KeyPeerId = ""
continue
}
}
s.aclList.RUnlock()
keyValues = slice.DiscardFromSlice(keyValues, func(value innerstorage.KeyValue) bool {
return value.KeyPeerId == ""
})
if len(keyValues) == 0 {
return nil
}
err = s.inner.Set(ctx, keyValues...)
if err != nil {
return err
}
sendErr := s.syncClient.Broadcast(ctx, s.storageId, keyValues...)
if sendErr != nil {
log.Warn("failed to send key values", zap.Error(sendErr))
}
indexErr := s.indexer.Index(s.decrypt, keyValues...)
if indexErr != nil {
log.Warn("failed to index for keys", zap.Error(indexErr))
}
return nil
}
func (s *storage) GetAll(ctx context.Context, key string, get func(decryptor Decryptor, values []innerstorage.KeyValue) error) (err error) {
var values []innerstorage.KeyValue
err = s.inner.IteratePrefix(ctx, key, func(kv innerstorage.KeyValue) error {
bytes := make([]byte, len(kv.Value.Value))
copy(bytes, kv.Value.Value)
kv.Value.Value = bytes
values = append(values, kv)
return nil
})
if err != nil {
return err
}
s.mx.Lock()
defer s.mx.Unlock()
return get(s.decrypt, values)
}
func (s *storage) InnerStorage() innerstorage.KeyValueStorage {
return s.inner
}
func (s *storage) readKeysFromAclState(state *list.AclState) (err error) {
if len(s.readKeys) == len(state.Keys()) {
return nil
}
if state.AccountKey() == nil || !state.HadReadPermissions(state.AccountKey().GetPublic()) {
return nil
}
for key, value := range state.Keys() {
if _, exists := s.readKeys[key]; exists {
continue
}
if value.ReadKey == nil {
continue
}
treeKey, err := deriveKey(value.ReadKey, s.storageId)
if err != nil {
return err
}
s.readKeys[key] = treeKey
}
curKey, err := state.CurrentReadKey()
if err != nil {
return err
}
if curKey == nil {
return nil
}
s.currentReadKey, err = deriveKey(curKey, s.storageId)
return err
}
func (s *storage) Iterate(ctx context.Context, f func(decryptor Decryptor, key string, values []innerstorage.KeyValue) (bool, error)) (err error) {
s.mx.Lock()
defer s.mx.Unlock()
var (
curKey = ""
// TODO: reuse buffer
values []innerstorage.KeyValue
)
err = s.inner.IterateValues(ctx, func(kv innerstorage.KeyValue) (bool, error) {
if kv.Key != curKey {
if curKey != "" {
iter, err := f(s.decrypt, curKey, values)
if err != nil {
return false, err
}
if !iter {
values = nil
return false, nil
}
}
curKey = kv.Key
values = values[:0]
}
bytes := make([]byte, len(kv.Value.Value))
copy(bytes, kv.Value.Value)
kv.Value.Value = bytes
values = append(values, kv)
return true, nil
})
if err != nil {
return err
}
if len(values) > 0 {
_, err = f(s.decrypt, curKey, values)
}
return err
}
func (s *storage) decrypt(kv innerstorage.KeyValue) (value []byte, err error) {
if kv.ReadKeyId == "" {
return nil, fmt.Errorf("no read key id")
}
key := s.readKeys[kv.ReadKeyId]
if key == nil {
return nil, fmt.Errorf("no read key for %s", kv.ReadKeyId)
}
msg := &spacesyncproto.StoreKeyInner{}
err = msg.UnmarshalVT(kv.Value.Value)
if err != nil {
return nil, err
}
value, err = key.Decrypt(msg.Value)
if err != nil {
return nil, err
}
return value, nil
}
func deriveKey(key crypto.SymKey, id string) (crypto.SymKey, error) {
raw, err := key.Raw()
if err != nil {
return nil, err
}
return crypto.DeriveSymmetricKey(raw, fmt.Sprintf(crypto.AnysyncKeyValuePath, id))
}

83
commonspace/object/keyvalue/keyvaluestorage/syncstorage/syncclient.go Normal file
View file

@ -0,0 +1,83 @@
package syncstorage
import (
"context"
"fmt"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage/innerstorage"
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
"github.com/anyproto/any-sync/commonspace/sync"
"github.com/anyproto/any-sync/commonspace/sync/objectsync/objectmessages"
)
type innerUpdate struct {
prepared []byte
keyValues []innerstorage.KeyValue
}
func (i *innerUpdate) Marshall(data objectmessages.ObjectMeta) ([]byte, error) {
if i.prepared != nil {
return i.prepared, nil
}
return nil, fmt.Errorf("no prepared data")
}
func (i *innerUpdate) Prepare() error {
// TODO: Add peer to ignored peers list
var (
protoKeyValues []*spacesyncproto.StoreKeyValue
err error
)
for _, kv := range i.keyValues {
protoKeyValues = append(protoKeyValues, kv.Proto())
}
keyValues := &spacesyncproto.StoreKeyValues{KeyValues: protoKeyValues}
i.prepared, err = keyValues.MarshalVT()
return err
}
func (i *innerUpdate) Heads() []string {
return nil
}
func (i *innerUpdate) MsgSize() uint64 {
return uint64(len(i.prepared))
}
func (i *innerUpdate) ObjectType() spacesyncproto.ObjectType {
return spacesyncproto.ObjectType_KeyValue
}
type SyncClient interface {
Broadcast(ctx context.Context, objectId string, keyValues ...innerstorage.KeyValue) error
}
type syncClient struct {
spaceId string
syncService sync.SyncService
}
func New(spaceId string, syncService sync.SyncService) SyncClient {
return &syncClient{
spaceId: spaceId,
syncService: syncService,
}
}
func (s *syncClient) Broadcast(ctx context.Context, objectId string, keyValue ...innerstorage.KeyValue) error {
inner := &innerUpdate{
keyValues: keyValue,
}
err := inner.Prepare()
if err != nil {
return err
}
headUpdate := &objectmessages.HeadUpdate{
Meta: objectmessages.ObjectMeta{
ObjectId: objectId,
SpaceId: s.spaceId,
},
Update: inner,
}
return s.syncService.BroadcastMessage(ctx, headUpdate)
}

24
commonspace/object/keyvalue/kvinterfaces/interfaces.go Normal file
View file

@ -0,0 +1,24 @@
//go:generate mockgen -destination mock_kvinterfaces/mock_kvinterfaces.go github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces KeyValueService
package kvinterfaces
import (
"context"
"storj.io/drpc"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage"
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
"github.com/anyproto/any-sync/net/peer"
)
const CName = "common.object.keyvalue"
type KeyValueService interface {
app.ComponentRunnable
DefaultStore() keyvaluestorage.Storage
HandleMessage(ctx context.Context, msg drpc.Message) (err error)
SyncWithPeer(p peer.Peer) (err error)
HandleStoreDiffRequest(ctx context.Context, req *spacesyncproto.StoreDiffRequest) (resp *spacesyncproto.StoreDiffResponse, err error)
HandleStoreElementsRequest(ctx context.Context, stream spacesyncproto.DRPCSpaceSync_StoreElementsStream) (err error)
}

171
commonspace/object/keyvalue/kvinterfaces/mock_kvinterfaces/mock_kvinterfaces.go Normal file
View file

@ -0,0 +1,171 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces (interfaces: KeyValueService)
//
// Generated by this command:
//
// mockgen -destination mock_kvinterfaces/mock_kvinterfaces.go github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces KeyValueService
//
// Package mock_kvinterfaces is a generated GoMock package.
package mock_kvinterfaces
import (
context "context"
reflect "reflect"
app "github.com/anyproto/any-sync/app"
keyvaluestorage "github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage"
spacesyncproto "github.com/anyproto/any-sync/commonspace/spacesyncproto"
peer "github.com/anyproto/any-sync/net/peer"
gomock "go.uber.org/mock/gomock"
drpc "storj.io/drpc"
)
// MockKeyValueService is a mock of KeyValueService interface.
type MockKeyValueService struct {
ctrl *gomock.Controller
recorder *MockKeyValueServiceMockRecorder
}
// MockKeyValueServiceMockRecorder is the mock recorder for MockKeyValueService.
type MockKeyValueServiceMockRecorder struct {
mock *MockKeyValueService
}
// NewMockKeyValueService creates a new mock instance.
func NewMockKeyValueService(ctrl *gomock.Controller) *MockKeyValueService {
mock := &MockKeyValueService{ctrl: ctrl}
mock.recorder = &MockKeyValueServiceMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use.
func (m *MockKeyValueService) EXPECT() *MockKeyValueServiceMockRecorder {
return m.recorder
}
// Close mocks base method.
func (m *MockKeyValueService) Close(arg0 context.Context) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Close", arg0)
ret0, _ := ret[0].(error)
return ret0
}
// Close indicates an expected call of Close.
func (mr *MockKeyValueServiceMockRecorder) Close(arg0 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockKeyValueService)(nil).Close), arg0)
}
// DefaultStore mocks base method.
func (m *MockKeyValueService) DefaultStore() keyvaluestorage.Storage {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "DefaultStore")
ret0, _ := ret[0].(keyvaluestorage.Storage)
return ret0
}
// DefaultStore indicates an expected call of DefaultStore.
func (mr *MockKeyValueServiceMockRecorder) DefaultStore() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DefaultStore", reflect.TypeOf((*MockKeyValueService)(nil).DefaultStore))
}
// HandleMessage mocks base method.
func (m *MockKeyValueService) HandleMessage(arg0 context.Context, arg1 drpc.Message) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "HandleMessage", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// HandleMessage indicates an expected call of HandleMessage.
func (mr *MockKeyValueServiceMockRecorder) HandleMessage(arg0, arg1 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HandleMessage", reflect.TypeOf((*MockKeyValueService)(nil).HandleMessage), arg0, arg1)
}
// HandleStoreDiffRequest mocks base method.
func (m *MockKeyValueService) HandleStoreDiffRequest(arg0 context.Context, arg1 *spacesyncproto.StoreDiffRequest) (*spacesyncproto.StoreDiffResponse, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "HandleStoreDiffRequest", arg0, arg1)
ret0, _ := ret[0].(*spacesyncproto.StoreDiffResponse)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// HandleStoreDiffRequest indicates an expected call of HandleStoreDiffRequest.
func (mr *MockKeyValueServiceMockRecorder) HandleStoreDiffRequest(arg0, arg1 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HandleStoreDiffRequest", reflect.TypeOf((*MockKeyValueService)(nil).HandleStoreDiffRequest), arg0, arg1)
}
// HandleStoreElementsRequest mocks base method.
func (m *MockKeyValueService) HandleStoreElementsRequest(arg0 context.Context, arg1 spacesyncproto.DRPCSpaceSync_StoreElementsStream) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "HandleStoreElementsRequest", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// HandleStoreElementsRequest indicates an expected call of HandleStoreElementsRequest.
func (mr *MockKeyValueServiceMockRecorder) HandleStoreElementsRequest(arg0, arg1 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HandleStoreElementsRequest", reflect.TypeOf((*MockKeyValueService)(nil).HandleStoreElementsRequest), arg0, arg1)
}
// Init mocks base method.
func (m *MockKeyValueService) Init(arg0 *app.App) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Init", arg0)
ret0, _ := ret[0].(error)
return ret0
}
// Init indicates an expected call of Init.
func (mr *MockKeyValueServiceMockRecorder) Init(arg0 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockKeyValueService)(nil).Init), arg0)
}
// Name mocks base method.
func (m *MockKeyValueService) Name() string {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Name")
ret0, _ := ret[0].(string)
return ret0
}
// Name indicates an expected call of Name.
func (mr *MockKeyValueServiceMockRecorder) Name() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockKeyValueService)(nil).Name))
}
// Run mocks base method.
func (m *MockKeyValueService) Run(arg0 context.Context) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Run", arg0)
ret0, _ := ret[0].(error)
return ret0
}
// Run indicates an expected call of Run.
func (mr *MockKeyValueServiceMockRecorder) Run(arg0 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockKeyValueService)(nil).Run), arg0)
}
// SyncWithPeer mocks base method.
func (m *MockKeyValueService) SyncWithPeer(arg0 peer.Peer) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SyncWithPeer", arg0)
ret0, _ := ret[0].(error)
return ret0
}
// SyncWithPeer indicates an expected call of SyncWithPeer.
func (mr *MockKeyValueServiceMockRecorder) SyncWithPeer(arg0 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyncWithPeer", reflect.TypeOf((*MockKeyValueService)(nil).SyncWithPeer), arg0)
}

52
commonspace/object/keyvalue/limiter.go Normal file
View file

@ -0,0 +1,52 @@
package keyvalue
import (
"context"
"sync"
)
type concurrentLimiter struct {
mu sync.Mutex
inProgress map[string]bool
wg sync.WaitGroup
}
func newConcurrentLimiter() *concurrentLimiter {
return &concurrentLimiter{
inProgress: make(map[string]bool),
}
}
func (cl *concurrentLimiter) ScheduleRequest(ctx context.Context, id string, action func()) bool {
cl.mu.Lock()
if cl.inProgress[id] {
cl.mu.Unlock()
return false
}
cl.inProgress[id] = true
cl.wg.Add(1)
cl.mu.Unlock()
go func() {
defer func() {
cl.mu.Lock()
delete(cl.inProgress, id)
cl.mu.Unlock()
cl.wg.Done()
}()
select {
case <-ctx.Done():
return
default:
action()
}
}()
return true
}
func (cl *concurrentLimiter) Close() {
cl.wg.Wait()
}

106
commonspace/object/keyvalue/remotediff.go Normal file
View file

@ -0,0 +1,106 @@
package keyvalue
import (
"context"
"github.com/anyproto/any-sync/app/ldiff"
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
)
type Client interface {
StoreDiff(context.Context, *spacesyncproto.StoreDiffRequest) (*spacesyncproto.StoreDiffResponse, error)
}
type RemoteDiff interface {
ldiff.Remote
}
func NewRemoteDiff(spaceId string, client Client) RemoteDiff {
return &remote{
spaceId: spaceId,
client: client,
}
}
type remote struct {
spaceId string
client Client
}
func (r *remote) Ranges(ctx context.Context, ranges []ldiff.Range, resBuf []ldiff.RangeResult) (results []ldiff.RangeResult, err error) {
results = resBuf[:0]
pbRanges := make([]*spacesyncproto.HeadSyncRange, 0, len(ranges))
for _, rg := range ranges {
pbRanges = append(pbRanges, &spacesyncproto.HeadSyncRange{
From: rg.From,
To: rg.To,
Elements: rg.Elements,
Limit: uint32(rg.Limit),
})
}
req := &spacesyncproto.StoreDiffRequest{
SpaceId: r.spaceId,
Ranges: pbRanges,
}
resp, err := r.client.StoreDiff(ctx, req)
if err != nil {
return
}
for _, rr := range resp.Results {
var elms []ldiff.Element
if len(rr.Elements) > 0 {
elms = make([]ldiff.Element, 0, len(rr.Elements))
}
for _, e := range rr.Elements {
elms = append(elms, ldiff.Element{
Id: e.Id,
Head: e.Head,
})
}
results = append(results, ldiff.RangeResult{
Hash: rr.Hash,
Elements: elms,
Count: int(rr.Count),
})
}
return
}
func HandleRangeRequest(ctx context.Context, d ldiff.Diff, req *spacesyncproto.StoreDiffRequest) (resp *spacesyncproto.StoreDiffResponse, err error) {
ranges := make([]ldiff.Range, 0, len(req.Ranges))
// basically we gather data applicable for both diffs
for _, reqRange := range req.Ranges {
ranges = append(ranges, ldiff.Range{
From: reqRange.From,
To: reqRange.To,
Limit: int(reqRange.Limit),
Elements: reqRange.Elements,
})
}
res, err := d.Ranges(ctx, ranges, nil)
if err != nil {
return
}
resp = &spacesyncproto.StoreDiffResponse{
Results: make([]*spacesyncproto.HeadSyncResult, 0, len(res)),
}
for _, rangeRes := range res {
var elements []*spacesyncproto.HeadSyncResultElement
if len(rangeRes.Elements) > 0 {
elements = make([]*spacesyncproto.HeadSyncResultElement, 0, len(rangeRes.Elements))
for _, el := range rangeRes.Elements {
elements = append(elements, &spacesyncproto.HeadSyncResultElement{
Id: el.Id,
Head: el.Head,
})
}
}
resp.Results = append(resp.Results, &spacesyncproto.HeadSyncResult{
Hash: rangeRes.Hash,
Elements: elements,
Count: uint32(rangeRes.Count),
})
}
return
}

36
commonspace/object/tree/objecttree/storage.go
View file

@ -82,12 +82,8 @@ func CreateStorage(ctx context.Context, root *treechangeproto.RawTreeChangeWithI
return nil, err
}
storage, err := CreateStorageTx(tx.Context(), root, headStorage, store)
defer func() {
if err != nil {
tx.Rollback()
}
}()
if err != nil {
tx.Rollback()
return nil, err
}
return storage, tx.Commit()
@ -225,13 +221,19 @@ func (s *storage) AddAll(ctx context.Context, changes []StorageChange, heads []s
if err != nil {
return fmt.Errorf("failed to create write tx: %w", err)
}
defer func() {
if err != nil {
tx.Rollback()
} else {
err = tx.Commit()
}
}()
for _, ch := range changes {
ch.TreeId = s.id
newVal := newStorageChangeValue(ch, arena)
err = s.changesColl.Insert(tx.Context(), newVal)
arena.Reset()
if err != nil {
tx.Rollback()
return err
}
}
@ -240,12 +242,7 @@ func (s *storage) AddAll(ctx context.Context, changes []StorageChange, heads []s
Heads: heads,
CommonSnapshot: &commonSnapshot,
}
err = s.headStorage.UpdateEntryTx(tx.Context(), update)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
return s.headStorage.UpdateEntryTx(tx.Context(), update)
}
func (s *storage) AddAllNoError(ctx context.Context, changes []StorageChange, heads []string, commonSnapshot string) error {
@ -255,13 +252,19 @@ func (s *storage) AddAllNoError(ctx context.Context, changes []StorageChange, he
if err != nil {
return fmt.Errorf("failed to create write tx: %w", err)
}
defer func() {
if err != nil {
tx.Rollback()
} else {
err = tx.Commit()
}
}()
for _, ch := range changes {
ch.TreeId = s.id
newVal := newStorageChangeValue(ch, arena)
err = s.changesColl.Insert(tx.Context(), newVal)
arena.Reset()
if err != nil && !errors.Is(err, anystore.ErrDocExists) {
tx.Rollback()
return err
}
}
@ -270,12 +273,7 @@ func (s *storage) AddAllNoError(ctx context.Context, changes []StorageChange, he
Heads: heads,
CommonSnapshot: &commonSnapshot,
}
err = s.headStorage.UpdateEntryTx(tx.Context(), update)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
return s.headStorage.UpdateEntryTx(tx.Context(), update)
}
func (s *storage) Delete(ctx context.Context) error {

5
commonspace/object/tree/synctree/headupdate.go
View file

@ -4,6 +4,7 @@ import (
"slices"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
"github.com/anyproto/any-sync/commonspace/sync/objectsync/objectmessages"
)
@ -21,6 +22,10 @@ func (h *InnerHeadUpdate) MsgSize() (size uint64) {
return uint64(len(h.prepared))
}
func (h *InnerHeadUpdate) ObjectType() spacesyncproto.ObjectType {
return spacesyncproto.ObjectType_Tree
}
func (h *InnerHeadUpdate) Prepare() error {
treeMsg := treechangeproto.WrapHeadUpdate(&treechangeproto.TreeHeadUpdate{
Heads: h.heads,

37
commonspace/space.go
View file

@ -16,6 +16,7 @@ import (
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
"github.com/anyproto/any-sync/commonspace/object/acl/list"
"github.com/anyproto/any-sync/commonspace/object/acl/syncacl"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces"
"github.com/anyproto/any-sync/commonspace/object/treesyncer"
"github.com/anyproto/any-sync/commonspace/objecttreebuilder"
"github.com/anyproto/any-sync/commonspace/peermanager"
@ -28,35 +29,8 @@ import (
"github.com/anyproto/any-sync/commonspace/syncstatus"
"github.com/anyproto/any-sync/net/peer"
"github.com/anyproto/any-sync/net/streampool"
"github.com/anyproto/any-sync/util/crypto"
)
type SpaceCreatePayload struct {
// SigningKey is the signing key of the owner
SigningKey crypto.PrivKey
// SpaceType is an arbitrary string
SpaceType string
// ReplicationKey is a key which is to be used to determine the node where the space should be held
ReplicationKey uint64
// SpacePayload is an arbitrary payload related to space type
SpacePayload []byte
// MasterKey is the master key of the owner
MasterKey crypto.PrivKey
// ReadKey is the first read key of space
ReadKey crypto.SymKey
// MetadataKey is the first metadata key of space
MetadataKey crypto.PrivKey
// Metadata is the metadata of the owner
Metadata []byte
}
type SpaceDerivePayload struct {
SigningKey crypto.PrivKey
MasterKey crypto.PrivKey
SpaceType string
SpacePayload []byte
}
type SpaceDescription struct {
SpaceHeader *spacesyncproto.RawSpaceHeaderWithId
AclId string
@ -83,6 +57,7 @@ type Space interface {
AclClient() aclclient.AclSpaceClient
SyncStatus() syncstatus.StatusUpdater
Storage() spacestorage.SpaceStorage
KeyValue() kvinterfaces.KeyValueService
DeleteTree(ctx context.Context, id string) (err error)
GetNodePeers(ctx context.Context) (peer []peer.Peer, err error)
@ -110,6 +85,7 @@ type space struct {
settings settings.Settings
storage spacestorage.SpaceStorage
aclClient aclclient.AclSpaceClient
keyValue kvinterfaces.KeyValueService
aclList list.AclList
creationTime time.Time
}
@ -150,7 +126,7 @@ func (s *space) DebugAllHeads() (heads []headsync.TreeHeads) {
s.storage.HeadStorage().IterateEntries(context.Background(), headstorage.IterOpts{}, func(entry headstorage.HeadsEntry) (bool, error) {
if entry.CommonSnapshot != "" {
heads = append(heads, headsync.TreeHeads{
Id: entry.Id,
Id: entry.Id,
Heads: entry.Heads,
})
}
@ -221,6 +197,7 @@ func (s *space) Init(ctx context.Context) (err error) {
s.streamPool = s.app.MustComponent(streampool.CName).(streampool.StreamPool)
s.treeSyncer = s.app.MustComponent(treesyncer.CName).(treesyncer.TreeSyncer)
s.aclClient = s.app.MustComponent(aclclient.CName).(aclclient.AclSpaceClient)
s.keyValue = s.app.MustComponent(kvinterfaces.CName).(kvinterfaces.KeyValueService)
return
}
@ -228,6 +205,10 @@ func (s *space) SyncStatus() syncstatus.StatusUpdater {
return s.syncStatus
}
func (s *space) KeyValue() kvinterfaces.KeyValueService {
return s.keyValue
}
func (s *space) Storage() spacestorage.SpaceStorage {
return s.storage
}

38
commonspace/payloads.go → commonspace/spacepayloads/payloads.go
View file

@ -1,9 +1,9 @@
package commonspace
package spacepayloads
import (
"crypto/rand"
"errors"
"hash/fnv"
"math/rand"
"strconv"
"strings"
"time"
@ -19,6 +19,32 @@ import (
"github.com/anyproto/any-sync/util/crypto"
)
type SpaceCreatePayload struct {
// SigningKey is the signing key of the owner
SigningKey crypto.PrivKey
// SpaceType is an arbitrary string
SpaceType string
// ReplicationKey is a key which is to be used to determine the node where the space should be held
ReplicationKey uint64
// SpacePayload is an arbitrary payload related to space type
SpacePayload []byte
// MasterKey is the master key of the owner
MasterKey crypto.PrivKey
// ReadKey is the first read key of space
ReadKey crypto.SymKey
// MetadataKey is the first metadata key of space
MetadataKey crypto.PrivKey
// Metadata is the metadata of the owner
Metadata []byte
}
type SpaceDerivePayload struct {
SigningKey crypto.PrivKey
MasterKey crypto.PrivKey
SpaceType string
SpacePayload []byte
}
const (
SpaceReserved = "any-sync.space"
)
@ -111,7 +137,7 @@ func StoragePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload sp
return
}
func storagePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload spacestorage.SpaceStorageCreatePayload, err error) {
func StoragePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload spacestorage.SpaceStorageCreatePayload, err error) {
// marshalling keys
identity, err := payload.SigningKey.GetPublic().Marshall()
if err != nil {
@ -190,7 +216,7 @@ func storagePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload sp
return
}
func validateSpaceStorageCreatePayload(payload spacestorage.SpaceStorageCreatePayload) (err error) {
func ValidateSpaceStorageCreatePayload(payload spacestorage.SpaceStorageCreatePayload) (err error) {
err = ValidateSpaceHeader(payload.SpaceHeaderWithId, nil)
if err != nil {
return
@ -326,3 +352,7 @@ func validateCreateSpaceSettingsPayload(rawWithId *treechangeproto.RawTreeChange
return
}
func NewSpaceId(id string, repKey uint64) string {
return strings.Join([]string{id, strconv.FormatUint(repKey, 36)}, ".")
}

15
commonspace/payloads_test.go → commonspace/spacepayloads/payloads_test.go
View file

@ -1,4 +1,4 @@
package commonspace
package spacepayloads
import (
"fmt"
@ -7,6 +7,9 @@ import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anyproto/any-sync/commonspace/object/accountdata"
"github.com/anyproto/any-sync/commonspace/object/acl/aclrecordproto"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
@ -16,8 +19,6 @@ import (
"github.com/anyproto/any-sync/consensus/consensusproto"
"github.com/anyproto/any-sync/util/cidutil"
"github.com/anyproto/any-sync/util/crypto"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSuccessHeaderPayloadForSpaceCreate(t *testing.T) {
@ -438,7 +439,7 @@ func TestSuccessSameIds(t *testing.T) {
SpaceHeaderWithId: rawHeaderWithId,
SpaceSettingsWithId: rawSettingsPayload,
}
err = validateSpaceStorageCreatePayload(spacePayload)
err = ValidateSpaceStorageCreatePayload(spacePayload)
require.NoError(t, err)
}
@ -455,7 +456,7 @@ func TestFailWithAclWrongSpaceId(t *testing.T) {
SpaceHeaderWithId: rawHeaderWithId,
SpaceSettingsWithId: rawSettingsPayload,
}
err = validateSpaceStorageCreatePayload(spacePayload)
err = ValidateSpaceStorageCreatePayload(spacePayload)
assert.EqualErrorf(t, err, spacestorage.ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", spacestorage.ErrIncorrectSpaceHeader, err)
}
@ -472,7 +473,7 @@ func TestFailWithSettingsWrongSpaceId(t *testing.T) {
SpaceHeaderWithId: rawHeaderWithId,
SpaceSettingsWithId: rawSettingsPayload,
}
err = validateSpaceStorageCreatePayload(spacePayload)
err = ValidateSpaceStorageCreatePayload(spacePayload)
assert.EqualErrorf(t, err, spacestorage.ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", spacestorage.ErrIncorrectSpaceHeader, err)
}
@ -489,7 +490,7 @@ func TestFailWithWrongAclHeadIdInSettingsPayload(t *testing.T) {
SpaceHeaderWithId: rawHeaderWithId,
SpaceSettingsWithId: rawSettingsPayload,
}
err = validateSpaceStorageCreatePayload(spacePayload)
err = ValidateSpaceStorageCreatePayload(spacePayload)
assert.EqualErrorf(t, err, spacestorage.ErrIncorrectSpaceHeader.Error(), "Error should be: %v, got: %v", spacestorage.ErrIncorrectSpaceHeader, err)
}

10
commonspace/spacerpc_test.go
View file

@ -93,6 +93,16 @@ type RpcServer struct {
sync.Mutex
}
func (r *RpcServer) StoreDiff(ctx2 context.Context, request *spacesyncproto.StoreDiffRequest) (*spacesyncproto.StoreDiffResponse, error) {
//TODO implement me
panic("implement me")
}
func (r *RpcServer) StoreElements(stream spacesyncproto.DRPCSpaceSync_StoreElementsStream) error {
//TODO implement me
panic("implement me")
}
func NewRpcServer() *RpcServer {
return &RpcServer{
spaces: make(map[string]Space),

30
commonspace/spaceservice.go
View file

@ -13,7 +13,10 @@ import (
"github.com/anyproto/any-sync/commonspace/acl/aclclient"
"github.com/anyproto/any-sync/commonspace/deletionmanager"
"github.com/anyproto/any-sync/commonspace/object/keyvalue"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage"
"github.com/anyproto/any-sync/commonspace/object/treesyncer"
"github.com/anyproto/any-sync/commonspace/spacepayloads"
"github.com/anyproto/any-sync/commonspace/sync"
"github.com/anyproto/any-sync/commonspace/sync/objectsync"
"github.com/anyproto/any-sync/net"
@ -58,9 +61,9 @@ type ctxKey int
const AddSpaceCtxKey ctxKey = 0
type SpaceService interface {
DeriveSpace(ctx context.Context, payload SpaceDerivePayload) (string, error)
DeriveId(ctx context.Context, payload SpaceDerivePayload) (string, error)
CreateSpace(ctx context.Context, payload SpaceCreatePayload) (string, error)
DeriveSpace(ctx context.Context, payload spacepayloads.SpaceDerivePayload) (string, error)
DeriveId(ctx context.Context, payload spacepayloads.SpaceDerivePayload) (string, error)
CreateSpace(ctx context.Context, payload spacepayloads.SpaceCreatePayload) (string, error)
NewSpace(ctx context.Context, id string, deps Deps) (sp Space, err error)
app.Component
}
@ -69,6 +72,7 @@ type Deps struct {
SyncStatus syncstatus.StatusUpdater
TreeSyncer treesyncer.TreeSyncer
AccountService accountservice.Service
Indexer keyvaluestorage.Indexer
}
type spaceService struct {
@ -101,8 +105,8 @@ func (s *spaceService) Name() (name string) {
return CName
}
func (s *spaceService) CreateSpace(ctx context.Context, payload SpaceCreatePayload) (id string, err error) {
storageCreate, err := StoragePayloadForSpaceCreate(payload)
func (s *spaceService) CreateSpace(ctx context.Context, payload spacepayloads.SpaceCreatePayload) (id string, err error) {
storageCreate, err := spacepayloads.StoragePayloadForSpaceCreate(payload)
if err != nil {
return
}
@ -117,8 +121,8 @@ func (s *spaceService) CreateSpace(ctx context.Context, payload SpaceCreatePaylo
return store.Id(), store.Close(ctx)
}
func (s *spaceService) DeriveId(ctx context.Context, payload SpaceDerivePayload) (id string, err error) {
storageCreate, err := storagePayloadForSpaceDerive(payload)
func (s *spaceService) DeriveId(ctx context.Context, payload spacepayloads.SpaceDerivePayload) (id string, err error) {
storageCreate, err := spacepayloads.StoragePayloadForSpaceDerive(payload)
if err != nil {
return
}
@ -126,8 +130,8 @@ func (s *spaceService) DeriveId(ctx context.Context, payload SpaceDerivePayload)
return
}
func (s *spaceService) DeriveSpace(ctx context.Context, payload SpaceDerivePayload) (id string, err error) {
storageCreate, err := storagePayloadForSpaceDerive(payload)
func (s *spaceService) DeriveSpace(ctx context.Context, payload spacepayloads.SpaceDerivePayload) (id string, err error) {
storageCreate, err := spacepayloads.StoragePayloadForSpaceDerive(payload)
if err != nil {
return
}
@ -180,13 +184,19 @@ func (s *spaceService) NewSpace(ctx context.Context, id string, deps Deps) (Spac
if deps.AccountService != nil {
spaceApp.Register(deps.AccountService)
}
var keyValueIndexer keyvaluestorage.Indexer = keyvaluestorage.NoOpIndexer{}
if deps.Indexer != nil {
keyValueIndexer = deps.Indexer
}
spaceApp.Register(state).
Register(deps.SyncStatus).
Register(peerManager).
Register(st).
Register(keyValueIndexer).
Register(objectsync.New()).
Register(sync.NewSyncService()).
Register(syncacl.New()).
Register(keyvalue.New()).
Register(deletionstate.New()).
Register(deletionmanager.New()).
Register(settings.New()).
@ -304,7 +314,7 @@ func (s *spaceService) spacePullWithPeer(ctx context.Context, p peer.Peer, id st
}
func (s *spaceService) createSpaceStorage(ctx context.Context, payload spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error) {
err := validateSpaceStorageCreatePayload(payload)
err := spacepayloads.ValidateSpaceStorageCreatePayload(payload)
if err != nil {
return nil, err
}

40
commonspace/spacestorage/migration/spacemigrator.go
View file

@ -4,8 +4,6 @@ import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"time"
anystore "github.com/anyproto/any-store"
@ -30,14 +28,13 @@ var ErrAlreadyMigrated = errors.New("already migrated")
type SpaceMigrator interface {
MigrateId(ctx context.Context, id string, progress Progress) error
CheckMigrated(ctx context.Context, id string) (bool, error)
}
type Progress interface {
AddDone(done int64)
}
type RemoveFunc func(newStorage spacestorage.SpaceStorage, rootPath string) error
type RemoveFunc func(newStorage spacestorage.SpaceStorage, id, rootPath string) error
type spaceMigrator struct {
oldProvider oldstorage.SpaceStorageProvider
@ -47,11 +44,7 @@ type spaceMigrator struct {
removeFunc RemoveFunc
}
func NewSpaceMigrator(oldProvider oldstorage.SpaceStorageProvider, newProvider spacestorage.SpaceStorageProvider, numParallel int, rootPath string) SpaceMigrator {
return NewSpaceMigratorWithRemoveFunc(oldProvider, newProvider, numParallel, rootPath, nil)
}
func NewSpaceMigratorWithRemoveFunc(oldProvider oldstorage.SpaceStorageProvider, newProvider spacestorage.SpaceStorageProvider, numParallel int, rootPath string, removeFunc RemoveFunc) SpaceMigrator {
func NewSpaceMigrator(oldProvider oldstorage.SpaceStorageProvider, newProvider spacestorage.SpaceStorageProvider, numParallel int, rootPath string, removeFunc RemoveFunc) SpaceMigrator {
return &spaceMigrator{
oldProvider: oldProvider,
newProvider: newProvider,
@ -61,33 +54,15 @@ func NewSpaceMigratorWithRemoveFunc(oldProvider oldstorage.SpaceStorageProvider,
}
}
func (s *spaceMigrator) CheckMigrated(ctx context.Context, id string) (bool, error) {
migrated, storage := s.checkMigrated(ctx, id)
if storage != nil {
return migrated, storage.Close(ctx)
}
return false, nil
}
func (s *spaceMigrator) MigrateId(ctx context.Context, id string, progress Progress) error {
migrated, storage := s.checkMigrated(ctx, id)
if migrated {
storage.Close(ctx)
return ErrAlreadyMigrated
}
if storage != nil {
if s.removeFunc != nil {
err := s.removeFunc(storage, s.rootPath)
if err != nil {
return fmt.Errorf("migration: failed to remove new storage: %w", err)
}
} else {
err := storage.Close(ctx)
if err != nil {
return fmt.Errorf("migration: failed to close old storage: %w", err)
}
os.RemoveAll(filepath.Join(s.rootPath, id))
}
err := s.removeFunc(storage, id, s.rootPath)
if err != nil {
return fmt.Errorf("migration: failed to remove new storage: %w", err)
}
oldStorage, err := s.oldProvider.WaitSpaceStorage(ctx, id)
if err != nil {
@ -146,7 +121,6 @@ func (s *spaceMigrator) MigrateId(ctx context.Context, id string, progress Progr
treeMigrators = append(treeMigrators, objecttree.NewTreeMigrator(crypto.NewKeyStorage(), aclList))
ch <- treeMigrators[i]
}
var allErrors []error
slices.Sort(storedIds)
storedIds = slice.DiscardDuplicatesSorted(storedIds)
for _, id := range storedIds {
@ -163,7 +137,6 @@ func (s *spaceMigrator) MigrateId(ctx context.Context, id string, progress Progr
}
err = tm.MigrateTreeStorage(ctx, treeStorage, newStorage.HeadStorage(), newStorage.AnyStore())
if err != nil {
allErrors = append(allErrors, fmt.Errorf("migration: failed to migrate tree storage: %w", err))
return
}
})
@ -176,9 +149,6 @@ func (s *spaceMigrator) MigrateId(ctx context.Context, id string, progress Progr
if err != nil {
return fmt.Errorf("migration: failed to wait for executor: %w", err)
}
if len(allErrors) > 0 {
return fmt.Errorf("migration failed: %w", errors.Join(allErrors...))
}
if err := s.migrateHash(ctx, oldStorage, newStorage); err != nil {
log.Warn("migration: failed to migrate hash", zap.Error(err))
}

6
commonspace/spacestorage/spacestorage.go
View file

@ -66,7 +66,9 @@ func Create(ctx context.Context, store anystore.DB, payload SpaceStorageCreatePa
}
defer func() {
if err != nil {
tx.Rollback()
_ = tx.Rollback()
} else {
err = tx.Commit()
}
}()
changesColl, err := store.Collection(tx.Context(), objecttree.CollName)
@ -110,7 +112,7 @@ func Create(ctx context.Context, store anystore.DB, payload SpaceStorageCreatePa
headStorage: headStorage,
stateStorage: stateStorage,
aclStorage: aclStorage,
}, tx.Commit()
}, nil
}
func New(ctx context.Context, spaceId string, store anystore.DB) (SpaceStorage, error) {

5
commonspace/spacestorage_test.go
View file

@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/anyproto/any-sync/commonspace/object/accountdata"
"github.com/anyproto/any-sync/commonspace/spacepayloads"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/any-sync/util/crypto"
)
@ -22,7 +23,7 @@ func newStorageCreatePayload(t *testing.T) spacestorage.SpaceStorageCreatePayloa
require.NoError(t, err)
readKey := crypto.NewAES()
meta := []byte("account")
payload := SpaceCreatePayload{
payload := spacepayloads.SpaceCreatePayload{
SigningKey: keys.SignKey,
SpaceType: "space",
ReplicationKey: 10,
@ -32,7 +33,7 @@ func newStorageCreatePayload(t *testing.T) spacestorage.SpaceStorageCreatePayloa
MetadataKey: metaKey,
Metadata: meta,
}
createSpace, err := StoragePayloadForSpaceCreate(payload)
createSpace, err := spacepayloads.StoragePayloadForSpaceCreate(payload)
require.NoError(t, err)
return createSpace
}

30
commonspace/spacesyncproto/mock_spacesyncproto/mock_spacesyncproto.go
View file

@ -175,3 +175,33 @@ func (mr *MockDRPCSpaceSyncClientMockRecorder) SpacePush(ctx, in any) *gomock.Ca
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SpacePush", reflect.TypeOf((*MockDRPCSpaceSyncClient)(nil).SpacePush), ctx, in)
}
// StoreDiff mocks base method.
func (m *MockDRPCSpaceSyncClient) StoreDiff(arg0 context.Context, arg1 *spacesyncproto.StoreDiffRequest) (*spacesyncproto.StoreDiffResponse, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StoreDiff", arg0, arg1)
ret0, _ := ret[0].(*spacesyncproto.StoreDiffResponse)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StoreDiff indicates an expected call of StoreDiff.
func (mr *MockDRPCSpaceSyncClientMockRecorder) StoreDiff(arg0, arg1 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StoreDiff", reflect.TypeOf((*MockDRPCSpaceSyncClient)(nil).StoreDiff), arg0, arg1)
}
// StoreElements mocks base method.
func (m *MockDRPCSpaceSyncClient) StoreElements(arg0 context.Context) (spacesyncproto.DRPCSpaceSync_StoreElementsClient, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StoreElements", arg0)
ret0, _ := ret[0].(spacesyncproto.DRPCSpaceSync_StoreElementsClient)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StoreElements indicates an expected call of StoreElements.
func (mr *MockDRPCSpaceSyncClientMockRecorder) StoreElements(arg0 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StoreElements", reflect.TypeOf((*MockDRPCSpaceSyncClient)(nil).StoreElements), arg0)
}

55
commonspace/spacesyncproto/protos/spacesync.proto
View file

@ -20,6 +20,10 @@ enum ErrCodes {
service SpaceSync {
// HeadSync compares all objects and their hashes in a space
rpc HeadSync(HeadSyncRequest) returns (HeadSyncResponse);
// StoreDiff compares all objects and their hashes in a space
rpc StoreDiff(StoreDiffRequest) returns (StoreDiffResponse);
// StoreElements exchanges elements between peers
rpc StoreElements(stream StoreKeyValue) returns (stream StoreKeyValue);
// SpacePush sends new space to the node
rpc SpacePush(SpacePushRequest) returns (SpacePushResponse);
// SpacePull gets space from the remote peer
@ -79,6 +83,7 @@ message ObjectSyncMessage {
string replyId = 3;
bytes payload = 4;
string objectId = 5;
ObjectType objectType = 6;
}
// SpacePushRequest is a request to add space on a node containing only one acl record
@ -144,6 +149,12 @@ message ObjectDelete {
string id = 1;
}
// StoreHeader is a header for a store
message StoreHeader {
string spaceId = 1;
string storageName = 2;
}
// SpaceDelete is a message containing deleter peer id
message SpaceDelete {
string deleterPeerId = 1;
@ -196,9 +207,51 @@ message AclGetRecordsResponse {
repeated bytes records = 1;
}
message StoreDiffRequest {
string spaceId = 1;
repeated HeadSyncRange ranges = 2;
}
message StoreDiffResponse {
repeated HeadSyncResult results = 1;
}
message StoreKeyValue {
string keyPeerId = 1;
bytes value = 2;
bytes identitySignature = 3;
bytes peerSignature = 4;
string spaceId = 5;
}
message StoreKeyValues {
repeated StoreKeyValue keyValues = 1;
}
message StoreKeyInner {
bytes peer = 1;
bytes identity = 2;
bytes value = 3;
int64 timestampMicro = 4;
string aclHeadId = 5;
string key = 6;
}
message StorageHeader {
string spaceId = 1;
string storageName = 2;
}
// DiffType is a type of diff
enum DiffType {
Initial = 0;
V1 = 1;
V2 = 2;
}
}
// ObjectType is a type of object
enum ObjectType {
Tree = 0;
Acl = 1;
KeyValue = 2;
}

1045
commonspace/spacesyncproto/spacesync.pb.go
View file

File diff suppressed because it is too large Load diff

133
commonspace/spacesyncproto/spacesync_drpc.pb.go
View file

@ -34,6 +34,8 @@ type DRPCSpaceSyncClient interface {
DRPCConn() drpc.Conn
HeadSync(ctx context.Context, in *HeadSyncRequest) (*HeadSyncResponse, error)
StoreDiff(ctx context.Context, in *StoreDiffRequest) (*StoreDiffResponse, error)
StoreElements(ctx context.Context) (DRPCSpaceSync_StoreElementsClient, error)
SpacePush(ctx context.Context, in *SpacePushRequest) (*SpacePushResponse, error)
SpacePull(ctx context.Context, in *SpacePullRequest) (*SpacePullResponse, error)
ObjectSyncStream(ctx context.Context) (DRPCSpaceSync_ObjectSyncStreamClient, error)
@ -62,6 +64,54 @@ func (c *drpcSpaceSyncClient) HeadSync(ctx context.Context, in *HeadSyncRequest)
return out, nil
}
func (c *drpcSpaceSyncClient) StoreDiff(ctx context.Context, in *StoreDiffRequest) (*StoreDiffResponse, error) {
out := new(StoreDiffResponse)
err := c.cc.Invoke(ctx, "/spacesync.SpaceSync/StoreDiff", drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{}, in, out)
if err != nil {
return nil, err
}
return out, nil
}
func (c *drpcSpaceSyncClient) StoreElements(ctx context.Context) (DRPCSpaceSync_StoreElementsClient, error) {
stream, err := c.cc.NewStream(ctx, "/spacesync.SpaceSync/StoreElements", drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{})
if err != nil {
return nil, err
}
x := &drpcSpaceSync_StoreElementsClient{stream}
return x, nil
}
type DRPCSpaceSync_StoreElementsClient interface {
drpc.Stream
Send(*StoreKeyValue) error
Recv() (*StoreKeyValue, error)
}
type drpcSpaceSync_StoreElementsClient struct {
drpc.Stream
}
func (x *drpcSpaceSync_StoreElementsClient) GetStream() drpc.Stream {
return x.Stream
}
func (x *drpcSpaceSync_StoreElementsClient) Send(m *StoreKeyValue) error {
return x.MsgSend(m, drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{})
}
func (x *drpcSpaceSync_StoreElementsClient) Recv() (*StoreKeyValue, error) {
m := new(StoreKeyValue)
if err := x.MsgRecv(m, drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{}); err != nil {
return nil, err
}
return m, nil
}
func (x *drpcSpaceSync_StoreElementsClient) RecvMsg(m *StoreKeyValue) error {
return x.MsgRecv(m, drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{})
}
func (c *drpcSpaceSyncClient) SpacePush(ctx context.Context, in *SpacePushRequest) (*SpacePushResponse, error) {
out := new(SpacePushResponse)
err := c.cc.Invoke(ctx, "/spacesync.SpaceSync/SpacePush", drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{}, in, out)
@ -188,6 +238,8 @@ func (c *drpcSpaceSyncClient) AclGetRecords(ctx context.Context, in *AclGetRecor
type DRPCSpaceSyncServer interface {
HeadSync(context.Context, *HeadSyncRequest) (*HeadSyncResponse, error)
StoreDiff(context.Context, *StoreDiffRequest) (*StoreDiffResponse, error)
StoreElements(DRPCSpaceSync_StoreElementsStream) error
SpacePush(context.Context, *SpacePushRequest) (*SpacePushResponse, error)
SpacePull(context.Context, *SpacePullRequest) (*SpacePullResponse, error)
ObjectSyncStream(DRPCSpaceSync_ObjectSyncStreamStream) error
@ -203,6 +255,14 @@ func (s *DRPCSpaceSyncUnimplementedServer) HeadSync(context.Context, *HeadSyncRe
return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
}
func (s *DRPCSpaceSyncUnimplementedServer) StoreDiff(context.Context, *StoreDiffRequest) (*StoreDiffResponse, error) {
return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
}
func (s *DRPCSpaceSyncUnimplementedServer) StoreElements(DRPCSpaceSync_StoreElementsStream) error {
return drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
}
func (s *DRPCSpaceSyncUnimplementedServer) SpacePush(context.Context, *SpacePushRequest) (*SpacePushResponse, error) {
return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented)
}
@ -233,7 +293,7 @@ func (s *DRPCSpaceSyncUnimplementedServer) AclGetRecords(context.Context, *AclGe
type DRPCSpaceSyncDescription struct{}
func (DRPCSpaceSyncDescription) NumMethods() int { return 8 }
func (DRPCSpaceSyncDescription) NumMethods() int { return 10 }
func (DRPCSpaceSyncDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) {
switch n {
@ -247,6 +307,23 @@ func (DRPCSpaceSyncDescription) Method(n int) (string, drpc.Encoding, drpc.Recei
)
}, DRPCSpaceSyncServer.HeadSync, true
case 1:
return "/spacesync.SpaceSync/StoreDiff", drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return srv.(DRPCSpaceSyncServer).
StoreDiff(
ctx,
in1.(*StoreDiffRequest),
)
}, DRPCSpaceSyncServer.StoreDiff, true
case 2:
return "/spacesync.SpaceSync/StoreElements", drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return nil, srv.(DRPCSpaceSyncServer).
StoreElements(
&drpcSpaceSync_StoreElementsStream{in1.(drpc.Stream)},
)
}, DRPCSpaceSyncServer.StoreElements, true
case 3:
return "/spacesync.SpaceSync/SpacePush", drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return srv.(DRPCSpaceSyncServer).
@ -255,7 +332,7 @@ func (DRPCSpaceSyncDescription) Method(n int) (string, drpc.Encoding, drpc.Recei
in1.(*SpacePushRequest),
)
}, DRPCSpaceSyncServer.SpacePush, true
case 2:
case 4:
return "/spacesync.SpaceSync/SpacePull", drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return srv.(DRPCSpaceSyncServer).
@ -264,7 +341,7 @@ func (DRPCSpaceSyncDescription) Method(n int) (string, drpc.Encoding, drpc.Recei
in1.(*SpacePullRequest),
)
}, DRPCSpaceSyncServer.SpacePull, true
case 3:
case 5:
return "/spacesync.SpaceSync/ObjectSyncStream", drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return nil, srv.(DRPCSpaceSyncServer).
@ -272,7 +349,7 @@ func (DRPCSpaceSyncDescription) Method(n int) (string, drpc.Encoding, drpc.Recei
&drpcSpaceSync_ObjectSyncStreamStream{in1.(drpc.Stream)},
)
}, DRPCSpaceSyncServer.ObjectSyncStream, true
case 4:
case 6:
return "/spacesync.SpaceSync/ObjectSync", drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return srv.(DRPCSpaceSyncServer).
@ -281,7 +358,7 @@ func (DRPCSpaceSyncDescription) Method(n int) (string, drpc.Encoding, drpc.Recei
in1.(*ObjectSyncMessage),
)
}, DRPCSpaceSyncServer.ObjectSync, true
case 5:
case 7:
return "/spacesync.SpaceSync/ObjectSyncRequestStream", drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return nil, srv.(DRPCSpaceSyncServer).
@ -290,7 +367,7 @@ func (DRPCSpaceSyncDescription) Method(n int) (string, drpc.Encoding, drpc.Recei
&drpcSpaceSync_ObjectSyncRequestStreamStream{in2.(drpc.Stream)},
)
}, DRPCSpaceSyncServer.ObjectSyncRequestStream, true
case 6:
case 8:
return "/spacesync.SpaceSync/AclAddRecord", drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return srv.(DRPCSpaceSyncServer).
@ -299,7 +376,7 @@ func (DRPCSpaceSyncDescription) Method(n int) (string, drpc.Encoding, drpc.Recei
in1.(*AclAddRecordRequest),
)
}, DRPCSpaceSyncServer.AclAddRecord, true
case 7:
case 9:
return "/spacesync.SpaceSync/AclGetRecords", drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{},
func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) {
return srv.(DRPCSpaceSyncServer).
@ -333,6 +410,48 @@ func (x *drpcSpaceSync_HeadSyncStream) SendAndClose(m *HeadSyncResponse) error {
return x.CloseSend()
}
type DRPCSpaceSync_StoreDiffStream interface {
drpc.Stream
SendAndClose(*StoreDiffResponse) error
}
type drpcSpaceSync_StoreDiffStream struct {
drpc.Stream
}
func (x *drpcSpaceSync_StoreDiffStream) SendAndClose(m *StoreDiffResponse) error {
if err := x.MsgSend(m, drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{}); err != nil {
return err
}
return x.CloseSend()
}
type DRPCSpaceSync_StoreElementsStream interface {
drpc.Stream
Send(*StoreKeyValue) error
Recv() (*StoreKeyValue, error)
}
type drpcSpaceSync_StoreElementsStream struct {
drpc.Stream
}
func (x *drpcSpaceSync_StoreElementsStream) Send(m *StoreKeyValue) error {
return x.MsgSend(m, drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{})
}
func (x *drpcSpaceSync_StoreElementsStream) Recv() (*StoreKeyValue, error) {
m := new(StoreKeyValue)
if err := x.MsgRecv(m, drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{}); err != nil {
return nil, err
}
return m, nil
}
func (x *drpcSpaceSync_StoreElementsStream) RecvMsg(m *StoreKeyValue) error {
return x.MsgRecv(m, drpcEncoding_File_commonspace_spacesyncproto_protos_spacesync_proto{})
}
type DRPCSpaceSync_SpacePushStream interface {
drpc.Stream
SendAndClose(*SpacePushResponse) error

1525
commonspace/spacesyncproto/spacesync_vtproto.pb.go
View file

File diff suppressed because it is too large Load diff

10
commonspace/spaceutils_test.go
View file

@ -2,12 +2,14 @@ package commonspace
import (
"context"
"errors"
"fmt"
"math/rand"
"sync"
"testing"
"time"
anystore "github.com/anyproto/any-store"
"github.com/anyproto/go-chash"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
@ -26,6 +28,7 @@ import (
"github.com/anyproto/any-sync/commonspace/object/treesyncer"
"github.com/anyproto/any-sync/commonspace/objecttreebuilder"
"github.com/anyproto/any-sync/commonspace/peermanager"
"github.com/anyproto/any-sync/commonspace/spacepayloads"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
"github.com/anyproto/any-sync/commonspace/sync/objectsync/objectmessages"
@ -751,7 +754,7 @@ func newMultiPeerFixture(t *testing.T, peerNum int) *multiPeerFixture {
require.NoError(t, err)
readKey := crypto.NewAES()
meta := []byte("account")
payload := SpaceCreatePayload{
payload := spacepayloads.SpaceCreatePayload{
SigningKey: keys.SignKey,
SpaceType: "space",
ReplicationKey: 10,
@ -761,7 +764,7 @@ func newMultiPeerFixture(t *testing.T, peerNum int) *multiPeerFixture {
MetadataKey: metaKey,
Metadata: meta,
}
createSpace, err := StoragePayloadForSpaceCreate(payload)
createSpace, err := spacepayloads.StoragePayloadForSpaceCreate(payload)
require.NoError(t, err)
executor := list.NewExternalKeysAclExecutor(createSpace.SpaceHeaderWithId.Id, keys, meta, createSpace.AclWithId)
cmds := []string{
@ -802,6 +805,9 @@ func newMultiPeerFixture(t *testing.T, peerNum int) *multiPeerFixture {
err := listStorage.AddAll(ctx, []list.StorageRecord{
{RawRecord: rec.Payload, Id: rec.Id, PrevId: prevRec, Order: i + 1, ChangeSize: len(rec.Payload)},
})
if errors.Is(err, anystore.ErrDocExists) {
continue
}
require.NoError(t, err)
}
}

31
commonspace/sync/objectsync/objectmessages/headupdate.go
View file

@ -39,6 +39,7 @@ type InnerHeadUpdate interface {
Prepare() error
Heads() []string
MsgSize() uint64
ObjectType() spacesyncproto.ObjectType
}
type ObjectMeta struct {
@ -48,10 +49,11 @@ type ObjectMeta struct {
}
type HeadUpdate struct {
Meta ObjectMeta
Bytes []byte
Update InnerHeadUpdate
msg *spacesyncproto.ObjectSyncMessage
Meta ObjectMeta
Bytes []byte
Update InnerHeadUpdate
objectType spacesyncproto.ObjectType
msg *spacesyncproto.ObjectSyncMessage
}
func (h *HeadUpdate) MsgSize() uint64 {
@ -84,6 +86,7 @@ func (h *HeadUpdate) SetProtoMessage(message protobuf.Message) error {
h.Bytes = msg.GetPayload()
h.Meta.SpaceId = msg.SpaceId
h.Meta.ObjectId = msg.ObjectId
h.objectType = msg.GetObjectType()
return nil
}
@ -94,14 +97,19 @@ func (h *HeadUpdate) ProtoMessage() (protobuf.Message, error) {
return nil, err
}
return &spacesyncproto.ObjectSyncMessage{
SpaceId: h.Meta.SpaceId,
Payload: payload,
ObjectId: h.Meta.ObjectId,
SpaceId: h.Meta.SpaceId,
Payload: payload,
ObjectId: h.Meta.ObjectId,
ObjectType: h.Update.ObjectType(),
}, nil
}
return NewMessage(), nil
}
func (h *HeadUpdate) ObjectType() spacesyncproto.ObjectType {
return h.objectType
}
func (h *HeadUpdate) SpaceId() string {
return h.Meta.SpaceId
}
@ -116,9 +124,10 @@ func (h *HeadUpdate) ObjectId() string {
func (h *HeadUpdate) Copy() drpc.Message {
return &HeadUpdate{
Meta: h.Meta,
Bytes: h.Bytes,
Update: h.Update,
msg: h.msg,
Meta: h.Meta,
Bytes: h.Bytes,
Update: h.Update,
msg: h.msg,
objectType: h.objectType,
}
}

6
commonspace/sync/objectsync/objectsync_test.go
View file

@ -10,6 +10,8 @@ import (
"go.uber.org/mock/gomock"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces/mock_kvinterfaces"
"github.com/anyproto/any-sync/commonspace/object/tree/synctree"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/object/treemanager"
@ -159,6 +161,7 @@ func TestObjectSync_ApplyRequest(t *testing.T) {
type fixture struct {
*objectSync
objectManager *mock_objectmanager.MockObjectManager
keyValue *mock_kvinterfaces.MockKeyValueService
pool *mock_pool.MockService
a *app.App
ctrl *gomock.Controller
@ -171,13 +174,16 @@ func newFixture(t *testing.T) *fixture {
fx.ctrl = gomock.NewController(t)
fx.objectManager = mock_objectmanager.NewMockObjectManager(fx.ctrl)
fx.pool = mock_pool.NewMockService(fx.ctrl)
fx.keyValue = mock_kvinterfaces.NewMockKeyValueService(fx.ctrl)
anymock.ExpectComp(fx.objectManager.EXPECT(), treemanager.CName)
anymock.ExpectComp(fx.pool.EXPECT(), pool.CName)
anymock.ExpectComp(fx.keyValue.EXPECT(), kvinterfaces.CName)
fx.objectSync = &objectSync{}
spaceState := &spacestate.SpaceState{SpaceId: "spaceId"}
fx.a.Register(fx.objectManager).
Register(spaceState).
Register(fx.pool).
Register(fx.keyValue).
Register(syncstatus.NewNoOpSyncStatus()).
Register(fx.objectSync)
require.NoError(t, fx.a.Start(context.Background()))

14
commonspace/sync/objectsync/synchandler.go
View file

@ -11,6 +11,7 @@ import (
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/app/logger"
"github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces"
"github.com/anyproto/any-sync/commonspace/object/tree/synctree"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/object/treemanager"
@ -30,10 +31,11 @@ var ErrUnexpectedHeadUpdateType = errors.New("unexpected head update type")
var log = logger.NewNamed(syncdeps.CName)
type objectSync struct {
spaceId string
pool pool.Service
manager objectmanager.ObjectManager
status syncstatus.StatusUpdater
spaceId string
pool pool.Service
manager objectmanager.ObjectManager
status syncstatus.StatusUpdater
keyValue kvinterfaces.KeyValueService
}
func New() syncdeps.SyncHandler {
@ -43,6 +45,7 @@ func New() syncdeps.SyncHandler {
func (o *objectSync) Init(a *app.App) (err error) {
o.manager = a.MustComponent(treemanager.CName).(objectmanager.ObjectManager)
o.pool = a.MustComponent(pool.CName).(pool.Service)
o.keyValue = a.MustComponent(kvinterfaces.CName).(kvinterfaces.KeyValueService)
o.status = a.MustComponent(syncstatus.CName).(syncstatus.StatusUpdater)
o.spaceId = a.MustComponent(spacestate.CName).(*spacestate.SpaceState).SpaceId
return
@ -57,6 +60,9 @@ func (o *objectSync) HandleHeadUpdate(ctx context.Context, headUpdate drpc.Messa
if !ok {
return nil, ErrUnexpectedHeadUpdateType
}
if update.ObjectType() == spacesyncproto.ObjectType_KeyValue {
return nil, o.keyValue.HandleMessage(ctx, update)
}
peerId, err := peer.CtxPeerId(ctx)
if err != nil {
return nil, err

7
commonspace/sync/sync_test.go
View file

@ -391,7 +391,12 @@ func (r *testRequest) MsgSize() uint64 {
}
type testMessage struct {
objectId string
objectId string
objectType spacesyncproto.ObjectType
}
func (t *testMessage) ObjectType() spacesyncproto.ObjectType {
return t.objectType
}
func (t *testMessage) ObjectId() string {

3
commonspace/sync/syncdeps/message.go
View file

@ -1,6 +1,9 @@
package syncdeps
import "github.com/anyproto/any-sync/commonspace/spacesyncproto"
type Message interface {
ObjectId() string
MsgSize() uint64
ObjectType() spacesyncproto.ObjectType
}

25
go.mod
View file

@ -1,6 +1,8 @@
module github.com/anyproto/any-sync
go 1.23.2
go 1.23.0
toolchain go1.24.1
require (
filippo.io/edwards25519 v1.1.0
@ -23,25 +25,25 @@ require (
github.com/ipfs/go-block-format v0.2.0
github.com/ipfs/go-cid v0.5.0
github.com/ipfs/go-ipld-format v0.6.0
github.com/libp2p/go-libp2p v0.41.0
github.com/libp2p/go-libp2p v0.41.1
github.com/mr-tron/base58 v1.2.0
github.com/multiformats/go-multibase v0.2.0
github.com/multiformats/go-multihash v0.2.3
github.com/planetscale/vtprotobuf v0.6.0
github.com/prometheus/client_golang v1.21.1
github.com/quic-go/quic-go v0.50.1
github.com/prometheus/client_golang v1.22.0
github.com/quic-go/quic-go v0.51.0
github.com/stretchr/testify v1.10.0
github.com/tyler-smith/go-bip39 v1.1.0
github.com/zeebo/blake3 v0.2.4
go.uber.org/atomic v1.11.0
go.uber.org/mock v0.5.0
go.uber.org/mock v0.5.2
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.36.0
golang.org/x/crypto v0.38.0
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394
golang.org/x/net v0.37.0
golang.org/x/sys v0.31.0
golang.org/x/net v0.40.0
golang.org/x/sys v0.33.0
golang.org/x/time v0.11.0
golang.org/x/tools v0.31.0
golang.org/x/tools v0.33.0
google.golang.org/protobuf v1.36.5
gopkg.in/yaml.v3 v3.0.1
storj.io/drpc v0.0.34
@ -77,7 +79,6 @@ require (
github.com/ipld/go-codec-dagpb v1.6.0 // indirect
github.com/ipld/go-ipld-prime v0.21.0 // indirect
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
@ -109,8 +110,8 @@ require (
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/image v0.21.0 // indirect
golang.org/x/mod v0.24.0 // indirect
golang.org/x/sync v0.12.0 // indirect
golang.org/x/text v0.23.0 // indirect
golang.org/x/sync v0.14.0 // indirect
golang.org/x/text v0.25.0 // indirect
lukechampine.com/blake3 v1.4.0 // indirect
modernc.org/libc v1.61.13 // indirect
modernc.org/mathutil v1.7.1 // indirect

52
go.sum
View file

@ -96,8 +96,8 @@ github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGw
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
github.com/google/pprof v0.0.0-20250208200701-d0013a598941 h1:43XjGa6toxLpeksjcxs1jIoIyr+vUfOqY2c6HB4bpoc=
@ -182,8 +182,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw=
github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:st3qqfu8+pMfh+9Mzqb2GTiwrAGjIPszEjZmtksN8Jc=
github.com/libp2p/go-libp2p v0.41.0 h1:JRaD39dqf/tBBGapJ0T38N73vOaDCsWgcx3mE6HgXWk=
github.com/libp2p/go-libp2p v0.41.0/go.mod h1:Be8QYqC4JW6Xq8buukNeoZJjyT1XUDcGoIooCHm1ye4=
github.com/libp2p/go-libp2p v0.41.1 h1:8ecNQVT5ev/jqALTvisSJeVNvXYJyK4NhQx1nNRXQZE=
github.com/libp2p/go-libp2p v0.41.1/go.mod h1:DcGTovJzQl/I7HMrby5ZRjeD0kQkGiy+9w6aEkSZpRI=
github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94=
github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8=
github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg=
@ -244,8 +244,8 @@ github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk=
github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE=
github.com/pion/dtls/v3 v3.0.4 h1:44CZekewMzfrn9pmGrj5BNnTMDCFwr+6sLH+cCuLM7U=
github.com/pion/dtls/v3 v3.0.4/go.mod h1:R373CsjxWqNPf6MEkfdy3aSe9niZvL/JaKlGeFphtMg=
github.com/pion/ice/v4 v4.0.6 h1:jmM9HwI9lfetQV/39uD0nY4y++XZNPhvzIPCb8EwxUM=
github.com/pion/ice/v4 v4.0.6/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw=
github.com/pion/ice/v4 v4.0.8 h1:ajNx0idNG+S+v9Phu4LSn2cs8JEfTsA1/tEjkkAVpFY=
github.com/pion/ice/v4 v4.0.8/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw=
github.com/pion/interceptor v0.1.37 h1:aRA8Zpab/wE7/c0O3fh1PqY0AJI3fCSEM5lRWJVorwI=
github.com/pion/interceptor v0.1.37/go.mod h1:JzxbJ4umVTlZAf+/utHzNesY8tmRkM2lVmkS82TTj8Y=
github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI=
@ -258,8 +258,8 @@ github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo=
github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0=
github.com/pion/rtp v1.8.11 h1:17xjnY5WO5hgO6SD3/NTIUPvSFw/PbLsIJyz1r1yNIk=
github.com/pion/rtp v1.8.11/go.mod h1:8uMBJj32Pa1wwx8Fuv/AsFhn8jsgw+3rUC2PfoBZ8p4=
github.com/pion/sctp v1.8.36 h1:owNudmnz1xmhfYje5L/FCav3V9wpPRePHle3Zi+P+M0=
github.com/pion/sctp v1.8.36/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE=
github.com/pion/sctp v1.8.37 h1:ZDmGPtRPX9mKCiVXtMbTWybFw3z/hVKAZgU81wcOrqs=
github.com/pion/sctp v1.8.37/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE=
github.com/pion/sdp/v3 v3.0.10 h1:6MChLE/1xYB+CjumMw+gZ9ufp2DPApuVSnDT8t5MIgA=
github.com/pion/sdp/v3 v3.0.10/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E=
github.com/pion/srtp/v3 v3.0.4 h1:2Z6vDVxzrX3UHEgrUyIGM4rRouoC7v+NiF1IHtp9B5M=
@ -283,8 +283,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4=
github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw=
github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk=
github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg=
github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
@ -293,8 +293,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
github.com/quic-go/quic-go v0.50.1 h1:unsgjFIUqW8a2oopkY7YNONpV1gYND6Nt9hnt1PN94Q=
github.com/quic-go/quic-go v0.50.1/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E=
github.com/quic-go/quic-go v0.51.0 h1:K8exxe9zXxeRKxaXxi/GpUqYiTrtdiWP8bo1KFya6Wc=
github.com/quic-go/quic-go v0.51.0/go.mod h1:MFlGGpcpJqRAfmYi6NC2cptDPSxRWTOGNuP4wqrWmzQ=
github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg=
github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
@ -361,8 +361,8 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0
go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
@ -375,8 +375,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw=
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@ -391,13 +391,13 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c=
golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -407,13 +407,13 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@ -421,8 +421,8 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU=
golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ=
golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

4
net/secureservice/secureservice.go
View file

@ -30,12 +30,14 @@ var (
// ProtoVersion 3 - acl with breaking changes / multiplayer
// ProtoVersion 4 - new sync compatible version
// ProtoVersion 5 - sync with no entry space
// ProtoVersion 6 - sync with key value messages
CompatibleVersion = uint32(4)
ProtoVersion = uint32(5)
KeyValueVersion = uint32(6)
)
var (
compatibleVersions = []uint32{CompatibleVersion, ProtoVersion}
compatibleVersions = []uint32{CompatibleVersion, ProtoVersion, KeyValueVersion}
)
func New() SecureService {

48
paymentservice/paymentserviceproto/paymentservice_tiers.pb.go
View file

@ -272,8 +272,10 @@ type TierData struct {
// Android platform-specific data:
AndroidProductId string `protobuf:"bytes,18,opt,name=androidProductId,proto3" json:"androidProductId,omitempty"`
AndroidManageUrl string `protobuf:"bytes,19,opt,name=androidManageUrl,proto3" json:"androidManageUrl,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
// something like "limited offer", etc
Offer string `protobuf:"bytes,20,opt,name=offer,proto3" json:"offer,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *TierData) Reset() {
@ -439,6 +441,13 @@ func (x *TierData) GetAndroidManageUrl() string {
return ""
}
func (x *TierData) GetOffer() string {
if x != nil {
return x.Offer
}
return ""
}
type GetTiersResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
// list of all available tiers
@ -504,7 +513,7 @@ var file_paymentservice_paymentserviceproto_protos_paymentservice_tiers_proto_ra
0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61,
0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02,
0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22,
0xc1, 0x05, 0x0a, 0x08, 0x54, 0x69, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02,
0xd7, 0x05, 0x0a, 0x08, 0x54, 0x69, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02,
0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04,
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65,
0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18,
@ -548,22 +557,23 @@ var file_paymentservice_paymentserviceproto_protos_paymentservice_tiers_proto_ra
0x6f, 0x64, 0x75, 0x63, 0x74, 0x49, 0x64, 0x12, 0x2a, 0x0a, 0x10, 0x61, 0x6e, 0x64, 0x72, 0x6f,
0x69, 0x64, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x55, 0x72, 0x6c, 0x18, 0x13, 0x20, 0x01, 0x28,
0x09, 0x52, 0x10, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65,
0x55, 0x72, 0x6c, 0x22, 0x33, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x54, 0x69, 0x65, 0x72, 0x73, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x05, 0x74, 0x69, 0x65, 0x72, 0x73,
0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x54, 0x69, 0x65, 0x72, 0x44, 0x61, 0x74,
0x61, 0x52, 0x05, 0x74, 0x69, 0x65, 0x72, 0x73, 0x2a, 0x90, 0x01, 0x0a, 0x0a, 0x50, 0x65, 0x72,
0x69, 0x6f, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x15, 0x0a, 0x11, 0x50, 0x65, 0x72, 0x69, 0x6f,
0x64, 0x54, 0x79, 0x70, 0x65, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x17,
0x0a, 0x13, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x54, 0x79, 0x70, 0x65, 0x55, 0x6e, 0x6c, 0x69,
0x6d, 0x69, 0x74, 0x65, 0x64, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x50, 0x65, 0x72, 0x69, 0x6f,
0x64, 0x54, 0x79, 0x70, 0x65, 0x44, 0x61, 0x79, 0x73, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x50,
0x65, 0x72, 0x69, 0x6f, 0x64, 0x54, 0x79, 0x70, 0x65, 0x57, 0x65, 0x65, 0x6b, 0x73, 0x10, 0x03,
0x12, 0x14, 0x0a, 0x10, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x54, 0x79, 0x70, 0x65, 0x4d, 0x6f,
0x6e, 0x74, 0x68, 0x73, 0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64,
0x54, 0x79, 0x70, 0x65, 0x59, 0x65, 0x61, 0x72, 0x73, 0x10, 0x05, 0x42, 0x24, 0x5a, 0x22, 0x70,
0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x70, 0x61,
0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x55, 0x72, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x6f, 0x66, 0x66, 0x65, 0x72, 0x18, 0x14, 0x20, 0x01,
0x28, 0x09, 0x52, 0x05, 0x6f, 0x66, 0x66, 0x65, 0x72, 0x22, 0x33, 0x0a, 0x10, 0x47, 0x65, 0x74,
0x54, 0x69, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a,
0x05, 0x74, 0x69, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x54,
0x69, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x05, 0x74, 0x69, 0x65, 0x72, 0x73, 0x2a, 0x90,
0x01, 0x0a, 0x0a, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x15, 0x0a,
0x11, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x54, 0x79, 0x70, 0x65, 0x55, 0x6e, 0x6b, 0x6e, 0x6f,
0x77, 0x6e, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x54, 0x79,
0x70, 0x65, 0x55, 0x6e, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x64, 0x10, 0x01, 0x12, 0x12, 0x0a,
0x0e, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x54, 0x79, 0x70, 0x65, 0x44, 0x61, 0x79, 0x73, 0x10,
0x02, 0x12, 0x13, 0x0a, 0x0f, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x54, 0x79, 0x70, 0x65, 0x57,
0x65, 0x65, 0x6b, 0x73, 0x10, 0x03, 0x12, 0x14, 0x0a, 0x10, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64,
0x54, 0x79, 0x70, 0x65, 0x4d, 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f,
0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x54, 0x79, 0x70, 0x65, 0x59, 0x65, 0x61, 0x72, 0x73, 0x10,
0x05, 0x42, 0x24, 0x5a, 0x22, 0x70, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x65, 0x72, 0x76,
0x69, 0x63, 0x65, 0x2f, 0x70, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
})
var (

45
paymentservice/paymentserviceproto/paymentservice_tiers_vtproto.pb.go
View file

@ -182,6 +182,15 @@ func (m *TierData) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
i -= len(m.unknownFields)
copy(dAtA[i:], m.unknownFields)
}
if len(m.Offer) > 0 {
i -= len(m.Offer)
copy(dAtA[i:], m.Offer)
i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Offer)))
i--
dAtA[i] = 0x1
i--
dAtA[i] = 0xa2
}
if len(m.AndroidManageUrl) > 0 {
i -= len(m.AndroidManageUrl)
copy(dAtA[i:], m.AndroidManageUrl)
@ -498,6 +507,10 @@ func (m *TierData) SizeVT() (n int) {
if l > 0 {
n += 2 + l + protohelpers.SizeOfVarint(uint64(l))
}
l = len(m.Offer)
if l > 0 {
n += 2 + l + protohelpers.SizeOfVarint(uint64(l))
}
n += len(m.unknownFields)
return n
}
@ -1360,6 +1373,38 @@ func (m *TierData) UnmarshalVT(dAtA []byte) error {
}
m.AndroidManageUrl = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 20:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Offer", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return protohelpers.ErrIntOverflow
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return protohelpers.ErrInvalidLength
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return protohelpers.ErrInvalidLength
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Offer = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := protohelpers.Skip(dAtA[iNdEx:])

3
paymentservice/paymentserviceproto/protos/paymentservice_tiers.proto
View file

@ -73,6 +73,9 @@ message TierData {
// Android platform-specific data:
string androidProductId = 18;
string androidManageUrl = 19;
// something like "limited offer", etc
string offer = 20;
}
message GetTiersResponse {

5
util/crypto/derived.go
View file

@ -5,8 +5,9 @@ import (
)
const (
AnysyncSpacePath = "m/SLIP-0021/anysync/space"
AnysyncTreePath = "m/SLIP-0021/anysync/tree/%s"
AnysyncSpacePath = "m/SLIP-0021/anysync/space"
AnysyncTreePath = "m/SLIP-0021/anysync/tree/%s"
AnysyncKeyValuePath = "m/SLIP-0021/anysync/keyvalue/%s"
)
// DeriveSymmetricKey derives a symmetric key from seed and path using slip-21