anyproto/any-sync
1
0
Fork 0
mirror of https://github.com/anyproto/any-sync.git synced 2025-06-08 05:57:03 +09:00
Code Activity

WIP object deletion etc

Browse source
This commit is contained in:
mcrakhman 2024-12-09 08:45:35 +01:00
parent f1754c89cd
commit 817c088781
No known key found for this signature in database
GPG key ID: DED12CFEF5B8396B
11 changed files with 191 additions and 130 deletions
Download patch file Download diff file Expand all files Collapse all files

5
commonspace/deletionmanager/deleter.go
View file

@ -2,6 +2,7 @@ package deletionmanager
import (
"context"
"errors"
"go.uber.org/zap"
@ -42,7 +43,7 @@ func (d *deleter) Delete(ctx context.Context) {
}
} else {
err = d.getter.DeleteTree(ctx, spaceId, id)
if err != nil && err != spacestorage.ErrTreeStorageAlreadyDeleted {
if err != nil && !errors.Is(err, spacestorage.ErrTreeStorageAlreadyDeleted) {
log.Error("failed to delete object", zap.Error(err))
continue
}
@ -60,7 +61,7 @@ func (d *deleter) tryMarkDeleted(spaceId, treeId string) (bool, error) {
if err == nil {
return true, nil
}
if err != treestorage.ErrUnknownTreeId {
if !errors.Is(err, treestorage.ErrUnknownTreeId) {
return false, err
}
return false, d.getter.MarkTreeDeleted(context.Background(), spaceId, treeId)

2
commonspace/deletionmanager/deletionmanager.go
View file

@ -40,7 +40,7 @@ type deletionManager struct {
func (d *deletionManager) Init(a *app.App) (err error) {
state := a.MustComponent(spacestate.CName).(*spacestate.SpaceState)
storage := a.MustComponent(spacestorage.CName).(spacestorage.SpaceStorage)
d.log = log.With(zap.String("spaceId", state.SpaceId), zap.String("settingsId", storage.SpaceSettingsId()))
d.log = log.With(zap.String("spaceId", state.SpaceId))
d.deletionState = a.MustComponent(deletionstate.CName).(deletionstate.ObjectDeletionState)
treeManager := a.MustComponent(treemanager.CName).(treemanager.TreeManager)
d.deleter = newDeleter(storage, d.deletionState, treeManager, d.log)

68
commonspace/deletionstate/deletionstate.go
View file

@ -2,11 +2,16 @@
package deletionstate
import (
"context"
"sync"
"time"
"go.uber.org/zap"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/app/logger"
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"go.uber.org/zap"
"sync"
)
var log = logger.NewNamed(CName)
@ -25,17 +30,36 @@ type ObjectDeletionState interface {
Filter(ids []string) (filtered []string)
}
const setTimeout = 5 * time.Second
type objectDeletionState struct {
sync.RWMutex
log logger.CtxLogger
queued map[string]struct{}
deleted map[string]struct{}
stateUpdateObservers []StateUpdateObserver
storage spacestorage.SpaceStorage
storage headstorage.HeadStorage
}
func (st *objectDeletionState) Run(ctx context.Context) (err error) {
return st.storage.IterateEntries(ctx, headstorage.IterOpts{Deleted: true}, func(entry headstorage.HeadsEntry) (bool, error) {
switch entry.DeletedStatus {
case headstorage.DeletedStatusQueued:
st.queued[entry.Id] = struct{}{}
case headstorage.DeletedStatusDeleted:
st.deleted[entry.Id] = struct{}{}
default:
}
return true, nil
})
}
func (st *objectDeletionState) Close(ctx context.Context) (err error) {
return nil
}
func (st *objectDeletionState) Init(a *app.App) (err error) {
st.storage = a.MustComponent(spacestorage.CName).(spacestorage.SpaceStorage)
st.storage = a.MustComponent(spacestorage.CName).(spacestorage.SpaceStorage).HeadStorage()
return nil
}
@ -74,27 +98,12 @@ func (st *objectDeletionState) Add(ids map[string]struct{}) {
if _, exists := st.queued[id]; exists {
continue
}
var status string
status, err := st.storage.TreeDeletedStatus(id)
err := st.updateStatus(id, headstorage.DeletedStatusQueued)
if err != nil {
st.log.Warn("failed to get deleted status", zap.String("treeId", id), zap.Error(err))
st.log.Warn("failed to set deleted status", zap.String("treeId", id), zap.Error(err))
continue
}
switch status {
case spacestorage.TreeDeletedStatusQueued:
st.queued[id] = struct{}{}
case spacestorage.TreeDeletedStatusDeleted:
st.deleted[id] = struct{}{}
default:
err := st.storage.SetTreeDeletedStatus(id, spacestorage.TreeDeletedStatusQueued)
if err != nil {
st.log.Warn("failed to set deleted status", zap.String("treeId", id), zap.Error(err))
continue
}
st.queued[id] = struct{}{}
}
st.queued[id] = struct{}{}
added = append(added, id)
}
}
@ -109,16 +118,21 @@ func (st *objectDeletionState) GetQueued() (ids []string) {
return
}
func (st *objectDeletionState) updateStatus(id string, status headstorage.DeletedStatus) (err error) {
ctx, cancel := context.WithTimeout(context.Background(), setTimeout)
defer cancel()
return st.storage.UpdateEntry(ctx, headstorage.HeadsUpdate{
Id: id,
DeletedStatus: &status,
})
}
func (st *objectDeletionState) Delete(id string) (err error) {
st.Lock()
defer st.Unlock()
delete(st.queued, id)
st.deleted[id] = struct{}{}
err = st.storage.SetTreeDeletedStatus(id, spacestorage.TreeDeletedStatusDeleted)
if err != nil {
return
}
return
return st.updateStatus(id, headstorage.DeletedStatusDeleted)
}
func (st *objectDeletionState) Exists(id string) bool {

15
commonspace/headsync/diffsyncer.go
View file

@ -180,32 +180,33 @@ func (d *diffSyncer) sendPushSpaceRequest(ctx context.Context, peerId string, cl
return
}
root, err := aclStorage.Root()
root, err := aclStorage.Root(ctx)
if err != nil {
return
}
header, err := d.storage.SpaceHeader()
state, err := d.storage.StateStorage().GetState(ctx)
if err != nil {
return
}
settingsStorage, err := d.storage.TreeStorage(d.storage.SpaceSettingsId())
settingsStorage, err := d.storage.TreeStorage(ctx, state.SettingsId)
if err != nil {
return
}
spaceSettingsRoot, err := settingsStorage.Root()
spaceSettingsRoot, err := settingsStorage.Root(ctx)
if err != nil {
return
}
cred, err := d.credentialProvider.GetCredential(ctx, header)
raw := &spacesyncproto.RawSpaceHeaderWithId{RawHeader: state.SpaceHeader, Id: state.SpaceId}
cred, err := d.credentialProvider.GetCredential(ctx, raw)
if err != nil {
return
}
spacePayload := &spacesyncproto.SpacePayload{
SpaceHeader: header,
AclPayload: root.Payload,
SpaceHeader: raw,
AclPayload: root.RawRecord,
AclPayloadId: root.Id,
SpaceSettingsPayload: spaceSettingsRoot.RawChange,
SpaceSettingsPayloadId: spaceSettingsRoot.Id,

65
commonspace/headsync/headstorage/headstorage.go
View file

@ -16,30 +16,46 @@ const (
commonSnapshotKey = "s"
idKey = "id"
deletedStatusKey = "d"
derivedStatusKey = "r"
headsCollectionName = "heads"
)
type DeletedStatus int
const (
DeletedStatusNotDeleted DeletedStatus = iota
DeletedStatusQueued
DeletedStatusDeleted
)
type HeadsEntry struct {
Id string
Heads []string
CommonSnapshot string
DeletedStatus string
DeletedStatus DeletedStatus
IsDerived bool
}
type HeadsUpdate struct {
Id string
Heads []string
CommonSnapshot *string
DeletedStatus *string
DeletedStatus *DeletedStatus
IsDerived *bool
}
type EntryIterator func(entry HeadsEntry) (bool, error)
type IterOpts struct {
Deleted bool
}
type HeadStorage interface {
IterateEntries(ctx context.Context, iter EntryIterator) error
IterateEntries(ctx context.Context, iterOpts IterOpts, iter EntryIterator) error
GetEntry(ctx context.Context, id string) (HeadsEntry, error)
DeleteEntryTx(txCtx context.Context, id string) error
UpdateEntryTx(txCtx context.Context, update HeadsUpdate) error
UpdateEntry(ctx context.Context, update HeadsUpdate) error
}
type headStorage struct {
@ -58,11 +74,23 @@ func New(ctx context.Context, store anystore.DB) (HeadStorage, error) {
headsColl: headsColl,
arena: &anyenc.Arena{},
}
return st, nil
deletedIdx := anystore.IndexInfo{
Name: deletedStatusKey,
Fields: []string{deletedStatusKey},
Unique: true,
Sparse: true,
}
return st, st.headsColl.EnsureIndex(ctx, deletedIdx)
}
func (h *headStorage) IterateEntries(ctx context.Context, entryIter EntryIterator) error {
iter, err := h.headsColl.Find(nil).Sort(idKey).Iter(ctx)
func (h *headStorage) IterateEntries(ctx context.Context, opts IterOpts, entryIter EntryIterator) error {
var qry any
if opts.Deleted {
qry = query.Key{Path: []string{deletedStatusKey}, Filter: query.NewComp(query.CompOpGte, DeletedStatusQueued)}
} else {
qry = query.Key{Path: []string{deletedStatusKey}, Filter: query.NewComp(query.CompOpLt, DeletedStatusQueued)}
}
iter, err := h.headsColl.Find(qry).Sort(idKey).Iter(ctx)
if err != nil {
return fmt.Errorf("find iter: %w", err)
}
@ -89,10 +117,23 @@ func (h *headStorage) GetEntry(ctx context.Context, id string) (HeadsEntry, erro
return h.entryFromDoc(doc), nil
}
func (h *headStorage) UpdateEntry(ctx context.Context, update HeadsUpdate) (err error) {
tx, err := h.headsColl.WriteTx(ctx)
if err != nil {
return
}
err = h.UpdateEntryTx(tx.Context(), update)
if err != nil {
tx.Rollback()
return
}
return tx.Commit()
}
func (h *headStorage) UpdateEntryTx(ctx context.Context, update HeadsUpdate) (err error) {
mod := query.ModifyFunc(func(a *anyenc.Arena, v *anyenc.Value) (result *anyenc.Value, modified bool, err error) {
if update.DeletedStatus != nil {
v.Set(deletedStatusKey, a.NewString(*update.DeletedStatus))
v.Set(deletedStatusKey, a.NewNumberInt(int(*update.DeletedStatus)))
}
if update.CommonSnapshot != nil {
v.Set(commonSnapshotKey, a.NewString(*update.CommonSnapshot))
@ -100,6 +141,13 @@ func (h *headStorage) UpdateEntryTx(ctx context.Context, update HeadsUpdate) (er
if update.Heads != nil {
v.Set(headsKey, storeutil.NewStringArrayValue(update.Heads, a))
}
if update.IsDerived != nil {
if *update.IsDerived {
v.Set(derivedStatusKey, a.NewTrue())
} else {
v.Set(derivedStatusKey, a.NewFalse())
}
}
return v, true, nil
})
_, err = h.headsColl.UpsertId(ctx, update.Id, mod)
@ -115,6 +163,7 @@ func (h *headStorage) entryFromDoc(doc anystore.Doc) HeadsEntry {
Id: doc.Value().GetString(idKey),
Heads: storeutil.StringsFromArrayValue(doc.Value(), headsKey),
CommonSnapshot: doc.Value().GetString(commonSnapshotKey),
DeletedStatus: doc.Value().GetString(deletedStatusKey),
DeletedStatus: DeletedStatus(doc.Value().GetInt(deletedStatusKey)),
IsDerived: doc.Value().GetBool(derivedStatusKey),
}
}

36
commonspace/headsync/headsync.go
View file

@ -13,6 +13,7 @@ import (
"github.com/anyproto/any-sync/commonspace/config"
"github.com/anyproto/any-sync/commonspace/credentialprovider"
"github.com/anyproto/any-sync/commonspace/deletionstate"
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
"github.com/anyproto/any-sync/commonspace/object/acl/syncacl"
"github.com/anyproto/any-sync/commonspace/object/treesyncer"
"github.com/anyproto/any-sync/commonspace/peermanager"
@ -46,6 +47,7 @@ type HeadSync interface {
type headSync struct {
spaceId string
syncPeriod int
settingsId string
periodicSync periodicsync.PeriodicSync
storage spacestorage.SpaceStorage
@ -96,11 +98,9 @@ func (h *headSync) Name() (name string) {
}
func (h *headSync) Run(ctx context.Context) (err error) {
initialIds, err := h.storage.StoredIds()
if err != nil {
return
if err := h.fillDiff(ctx); err != nil {
return err
}
h.fillDiff(initialIds)
h.periodicSync.Run()
return
}
@ -124,7 +124,7 @@ func (h *headSync) AllIds() []string {
}
func (h *headSync) ExternalIds() []string {
settingsId := h.storage.SpaceSettingsId()
settingsId := h.storage.StateStorage().SettingsId()
aclId := h.syncAcl.Id()
return slice.DiscardFromSlice(h.AllIds(), func(id string) bool {
return id == settingsId || id == aclId
@ -152,21 +152,17 @@ func (h *headSync) Close(ctx context.Context) (err error) {
return
}
func (h *headSync) fillDiff(objectIds []string) {
var els = make([]ldiff.Element, 0, len(objectIds))
for _, id := range objectIds {
st, err := h.storage.TreeStorage(id)
if err != nil {
continue
}
heads, err := st.Heads()
if err != nil {
continue
}
func (h *headSync) fillDiff(ctx context.Context) error {
var els = make([]ldiff.Element, 0, 100)
err := h.storage.HeadStorage().IterateEntries(ctx, headstorage.IterOpts{}, func(entry headstorage.HeadsEntry) (bool, error) {
els = append(els, ldiff.Element{
Id: id,
Head: concatStrings(heads),
Id: entry.Id,
Head: concatStrings(entry.Heads),
})
return true, nil
})
if err != nil {
return err
}
els = append(els, ldiff.Element{
Id: h.syncAcl.Id(),
@ -174,7 +170,9 @@ func (h *headSync) fillDiff(objectIds []string) {
})
log.Debug("setting acl", zap.String("aclId", h.syncAcl.Id()), zap.String("headId", h.syncAcl.Head().Id))
h.diff.Set(els...)
if err := h.storage.WriteSpaceHash(h.diff.Hash()); err != nil {
if err := h.storage.StateStorage().SetHash(ctx, h.diff.Hash()); err != nil {
h.log.Error("can't write space hash", zap.Error(err))
return err
}
return nil
}

54
commonspace/headsync/statestorage/statestorage.go
View file

@ -8,14 +8,16 @@ import (
)
type State struct {
Hash string
AclId string
SettingsId string
SpaceId string
Hash string
AclId string
SettingsId string
SpaceId string
SpaceHeader []byte
}
type StateStorage interface {
GetState(ctx context.Context) (State, error)
SettingsId() string
SetHash(ctx context.Context, hash string) error
}
@ -23,15 +25,18 @@ const (
stateCollectionKey = "state"
idKey = "id"
hashKey = "h"
headerKey = "e"
aclIdKey = "a"
settingsIdKey = "s"
)
type stateStorage struct {
spaceId string
store anystore.DB
stateColl anystore.Collection
arena *anyenc.Arena
spaceId string
settingsId string
aclId string
store anystore.DB
stateColl anystore.Collection
arena *anyenc.Arena
}
func (s *stateStorage) GetState(ctx context.Context) (State, error) {
@ -56,12 +61,18 @@ func New(ctx context.Context, spaceId string, store anystore.DB) (StateStorage,
if err != nil {
return nil, err
}
return &stateStorage{
storage := &stateStorage{
store: store,
spaceId: spaceId,
stateColl: stateCollection,
arena: &anyenc.Arena{},
}, nil
}
st, err := storage.GetState(ctx)
if err != nil {
return nil, err
}
storage.settingsId = st.SettingsId
return storage, nil
}
func Create(ctx context.Context, state State, store anystore.DB) (StateStorage, error) {
@ -78,6 +89,7 @@ func Create(ctx context.Context, state State, store anystore.DB) (StateStorage,
doc := arena.NewObject()
doc.Set(idKey, arena.NewString(state.SpaceId))
doc.Set(settingsIdKey, arena.NewString(state.SettingsId))
doc.Set(headerKey, arena.NewBinary(state.SpaceHeader))
doc.Set(aclIdKey, arena.NewString(state.AclId))
err = stateCollection.Insert(tx.Context(), doc)
if err != nil {
@ -85,18 +97,24 @@ func Create(ctx context.Context, state State, store anystore.DB) (StateStorage,
return nil, err
}
return &stateStorage{
spaceId: state.SpaceId,
store: store,
stateColl: stateCollection,
arena: arena,
spaceId: state.SpaceId,
store: store,
settingsId: state.SettingsId,
stateColl: stateCollection,
arena: arena,
}, tx.Commit()
}
func (s *stateStorage) SettingsId() string {
return s.settingsId
}
func (s *stateStorage) stateFromDoc(doc anystore.Doc) State {
return State{
SpaceId: doc.Value().GetString(idKey),
SettingsId: doc.Value().GetString(settingsIdKey),
AclId: doc.Value().GetString(aclIdKey),
Hash: doc.Value().GetString(hashKey),
SpaceId: doc.Value().GetString(idKey),
SettingsId: doc.Value().GetString(settingsIdKey),
AclId: doc.Value().GetString(aclIdKey),
Hash: doc.Value().GetString(hashKey),
SpaceHeader: doc.Value().GetBytes(headerKey),
}
}

8
commonspace/object/tree/objecttree/storage.go
View file

@ -76,7 +76,7 @@ func CreateStorage(ctx context.Context, root *treechangeproto.RawTreeChangeWithI
headStorage: headStorage,
}
builder := storageChangeBuilder(crypto.NewKeyStorage(), root)
_, err := builder.Unmarshall(root, true)
unmarshalled, err := builder.Unmarshall(root, true)
if err != nil {
return nil, err
}
@ -119,6 +119,7 @@ func CreateStorage(ctx context.Context, root *treechangeproto.RawTreeChangeWithI
Id: root.Id,
Heads: []string{root.Id},
CommonSnapshot: &root.Id,
IsDerived: &unmarshalled.IsDerived,
})
if err != nil {
tx.Rollback()
@ -231,11 +232,6 @@ func (s *storage) Delete(ctx context.Context) error {
tx.Rollback()
return fmt.Errorf("failed to delete changes collection: %w", err)
}
err = s.headStorage.DeleteEntryTx(tx.Context(), s.id)
if err != nil {
tx.Rollback()
return fmt.Errorf("failed to remove document from heads collection: %w", err)
}
return tx.Commit()
}

6
commonspace/settings/settings.go
View file

@ -2,6 +2,9 @@ package settings
import (
"context"
"go.uber.org/zap"
"github.com/anyproto/any-sync/accountservice"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/deletionmanager"
@ -12,7 +15,6 @@ import (
"github.com/anyproto/any-sync/commonspace/spacestate"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/any-sync/nodeconf"
"go.uber.org/zap"
)
const CName = "common.commonspace.settings"
@ -79,7 +81,7 @@ func (s *settings) Close(ctx context.Context) (err error) {
}
func (s *settings) DeleteTree(ctx context.Context, id string) (err error) {
return s.settingsObject.DeleteObject(id)
return s.settingsObject.DeleteObject(ctx, id)
}
func (s *settings) SettingsObject() SettingsObject {

33
commonspace/settings/settingsobject.go
View file

@ -29,7 +29,7 @@ var log = logger.NewNamed("common.commonspace.settings")
type SettingsObject interface {
synctree.SyncTree
Init(ctx context.Context) (err error)
DeleteObject(id string) (err error)
DeleteObject(ctx context.Context, id string) (err error)
}
var (
@ -44,8 +44,8 @@ var (
DoSnapshot = objecttree.DoSnapshot
buildHistoryTree = func(objTree objecttree.ObjectTree) (objecttree.ReadableObjectTree, error) {
return objecttree.BuildHistoryTree(objecttree.HistoryTreeParams{
TreeStorage: objTree.Storage(),
AclList: objTree.AclList(),
Storage: objTree.Storage(),
AclList: objTree.AclList(),
})
}
)
@ -133,9 +133,12 @@ func (s *settingsObject) Rebuild(tr objecttree.ObjectTree) error {
}
func (s *settingsObject) Init(ctx context.Context) (err error) {
settingsId := s.store.SpaceSettingsId()
log.Debug("space settings id", zap.String("id", settingsId))
s.SyncTree, err = s.buildFunc(ctx, settingsId, s)
state, err := s.store.StateStorage().GetState(ctx)
if err != nil {
return
}
log.Debug("space settings id", zap.String("id", state.SettingsId))
s.SyncTree, err = s.buildFunc(ctx, state.SettingsId, s)
if err != nil {
return
}
@ -175,9 +178,7 @@ func (s *settingsObject) Close() error {
return nil
}
var isDerivedRoot = objecttree.IsDerivedRoot
func (s *settingsObject) DeleteObject(id string) (err error) {
func (s *settingsObject) DeleteObject(ctx context.Context, id string) (err error) {
s.Lock()
defer s.Unlock()
if s.Id() == id {
@ -186,19 +187,11 @@ func (s *settingsObject) DeleteObject(id string) (err error) {
if s.state.Exists(id) {
return ErrAlreadyDeleted
}
st, err := s.store.TreeStorage(id)
entry, err := s.store.HeadStorage().GetEntry(ctx, id)
if err != nil {
return ErrObjDoesNotExist
return err
}
root, err := st.Root()
if err != nil {
return ErrObjDoesNotExist
}
isDerived, err := isDerivedRoot(root)
if err != nil {
return ErrObjDoesNotExist
}
if isDerived {
if entry.IsDerived {
return ErrCantDeleteDerivedObject
}
isSnapshot := DoSnapshot(s.Len())

29
commonspace/spacestorage/spacestorage.go
View file

@ -6,7 +6,10 @@ import (
"errors"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
"github.com/anyproto/any-sync/commonspace/headsync/statestorage"
"github.com/anyproto/any-sync/commonspace/object/acl/list"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
@ -23,28 +26,14 @@ var (
ErrTreeStorageAlreadyDeleted = errors.New("tree storage already deleted")
)
const (
TreeDeletedStatusQueued = "queued"
TreeDeletedStatusDeleted = "deleted"
)
type SpaceStorage interface {
app.ComponentRunnable
Id() string
SetSpaceDeleted() error
IsSpaceDeleted() (bool, error)
SetTreeDeletedStatus(id, state string) error
TreeDeletedStatus(id string) (string, error)
SpaceSettingsId() string
AclStorage() (liststorage.ListStorage, error)
SpaceHeader() (*spacesyncproto.RawSpaceHeaderWithId, error)
StoredIds() ([]string, error)
TreeRoot(id string) (*treechangeproto.RawTreeChangeWithId, error)
TreeStorage(id string) (treestorage.TreeStorage, error)
HasTree(id string) (bool, error)
CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (treestorage.TreeStorage, error)
WriteSpaceHash(hash string) error
ReadSpaceHash() (hash string, err error)
HeadStorage() headstorage.HeadStorage
StateStorage() statestorage.StateStorage
AclStorage() (list.Storage, error)
TreeStorage(ctx context.Context, id string) (objecttree.Storage, error)
CreateTreeStorage(ctx context.Context, payload treestorage.TreeStorageCreatePayload) (objecttree.Storage, error)
}
type SpaceStorageCreatePayload struct {