From 3a6c0c1cae634ac47137c7e622b8cfaf0e46d19d Mon Sep 17 00:00:00 2001 From: mcrakhman Date: Thu, 16 Jan 2025 00:45:55 +0100 Subject: [PATCH] Not verify during migration --- .../tree/objecttree/objecttreefactory.go | 25 ++++++++ .../tree/objecttree}/treemigrator.go | 57 ++++++++++++------- .../spacestorage/migration/spacemigrator.go | 9 +-- 3 files changed, 68 insertions(+), 23 deletions(-) rename commonspace/{spacestorage/migration => object/tree/objecttree}/treemigrator.go (60%) diff --git a/commonspace/object/tree/objecttree/objecttreefactory.go b/commonspace/object/tree/objecttree/objecttreefactory.go index a7953e0f..aa0726b3 100644 --- a/commonspace/object/tree/objecttree/objecttreefactory.go +++ b/commonspace/object/tree/objecttree/objecttreefactory.go @@ -79,6 +79,22 @@ func verifiableEmptyDataTreeDeps( } } +func nonVerifiableEmptyDataTreeDeps( + rootChange *treechangeproto.RawTreeChangeWithId, + storage Storage, + aclList list.AclList) objectTreeDeps { + changeBuilder := &nonVerifiableChangeBuilder{NewEmptyDataChangeBuilder(crypto.NewKeyStorage(), rootChange)} + treeBuilder := newTreeBuilder(storage, changeBuilder) + return objectTreeDeps{ + changeBuilder: changeBuilder, + treeBuilder: treeBuilder, + storage: storage, + validator: newTreeValidator(false, false), + aclList: aclList, + flusher: &defaultFlusher{}, + } +} + func nonVerifiableTreeDeps( rootChange *treechangeproto.RawTreeChangeWithId, storage Storage, @@ -138,6 +154,15 @@ func BuildEmptyDataTestableTree(storage Storage, aclList list.AclList) (ObjectTr return buildObjectTree(deps) } +func BuildMigratableObjectTree(storage Storage, aclList list.AclList) (ObjectTree, error) { + rootChange, err := storage.Root(context.Background()) + if err != nil { + return nil, err + } + deps := nonVerifiableEmptyDataTreeDeps(rootChange.RawTreeChangeWithId(), storage, aclList) + return buildObjectTree(deps) +} + func BuildKeyFilterableObjectTree(storage Storage, aclList list.AclList) (ObjectTree, error) { rootChange, err := storage.Root(context.Background()) if err != nil { diff --git a/commonspace/spacestorage/migration/treemigrator.go b/commonspace/object/tree/objecttree/treemigrator.go similarity index 60% rename from commonspace/spacestorage/migration/treemigrator.go rename to commonspace/object/tree/objecttree/treemigrator.go index c889649e..00d63cda 100644 --- a/commonspace/spacestorage/migration/treemigrator.go +++ b/commonspace/object/tree/objecttree/treemigrator.go @@ -1,4 +1,4 @@ -package migration +package objecttree import ( "context" @@ -9,46 +9,65 @@ import ( "github.com/anyproto/any-sync/commonspace/headsync/headstorage" "github.com/anyproto/any-sync/commonspace/object/acl/list" - "github.com/anyproto/any-sync/commonspace/object/tree/objecttree" "github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto" "github.com/anyproto/any-sync/commonspace/object/tree/treestorage" - "github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage" "github.com/anyproto/any-sync/util/crypto" "github.com/anyproto/any-sync/util/slice" ) -type treeMigrator struct { +type treeStorage interface { + Id() string + Root() (*treechangeproto.RawTreeChangeWithId, error) + Heads() ([]string, error) + SetHeads(heads []string) error + AddRawChange(change *treechangeproto.RawTreeChangeWithId) error + AddRawChangesSetHeads(changes []*treechangeproto.RawTreeChangeWithId, heads []string) error + GetAllChangeIds() ([]string, error) + + GetRawChange(ctx context.Context, id string) (*treechangeproto.RawTreeChangeWithId, error) + GetAppendRawChange(ctx context.Context, buf []byte, id string) (*treechangeproto.RawTreeChangeWithId, error) + HasChange(ctx context.Context, id string) (bool, error) + Delete() error +} + +type changesIterator interface { + GetAllChanges() ([]*treechangeproto.RawTreeChangeWithId, error) + IterateChanges(proc func(id string, rawChange []byte) error) error +} + +type TreeMigrator struct { idStack []string - cache map[string]*objecttree.Change - storage oldstorage.TreeStorage - builder objecttree.ChangeBuilder + cache map[string]*Change + storage treeStorage + builder ChangeBuilder allChanges []*treechangeproto.RawTreeChangeWithId keyStorage crypto.KeyStorage aclList list.AclList } -func newTreeMigrator(keyStorage crypto.KeyStorage, aclList list.AclList) *treeMigrator { - return &treeMigrator{ +func NewTreeMigrator(keyStorage crypto.KeyStorage, aclList list.AclList) *TreeMigrator { + return &TreeMigrator{ keyStorage: keyStorage, aclList: aclList, } } -func (tm *treeMigrator) migrateTreeStorage(ctx context.Context, storage oldstorage.TreeStorage, headStorage headstorage.HeadStorage, store anystore.DB) error { +func (tm *TreeMigrator) MigrateTreeStorage(ctx context.Context, storage treeStorage, headStorage headstorage.HeadStorage, store anystore.DB) error { rootChange, err := storage.Root() if err != nil { return err } tm.allChanges = []*treechangeproto.RawTreeChangeWithId{rootChange} tm.storage = storage - tm.cache = make(map[string]*objecttree.Change) - tm.builder = objecttree.NewChangeBuilder(tm.keyStorage, rootChange) + tm.cache = make(map[string]*Change) + tm.builder = NewChangeBuilder(tm.keyStorage, rootChange) + tm.builder = &nonVerifiableChangeBuilder{tm.builder} heads, err := storage.Heads() if err != nil { return fmt.Errorf("migration: failed to get heads: %w", err) } - if iterStore, ok := storage.(oldstorage.ChangesIterator); ok { + if iterStore, ok := storage.(changesIterator); ok { tm.allChanges, err = iterStore.GetAllChanges() if err != nil { return fmt.Errorf("migration: failed to get all changes: %w", err) @@ -56,21 +75,21 @@ func (tm *treeMigrator) migrateTreeStorage(ctx context.Context, storage oldstora } else { tm.dfs(ctx, heads, rootChange.Id) } - newStorage, err := objecttree.CreateStorage(ctx, rootChange, headStorage, store) + newStorage, err := CreateStorage(ctx, rootChange, headStorage, store) if err != nil && !errors.Is(err, treestorage.ErrTreeExists) { return fmt.Errorf("migration: failed to create new storage: %w", err) } if errors.Is(err, treestorage.ErrTreeExists) { - newStorage, err = objecttree.NewStorage(ctx, rootChange.Id, headStorage, store) + newStorage, err = NewStorage(ctx, rootChange.Id, headStorage, store) if err != nil { return fmt.Errorf("migration: failed to start old storage: %w", err) } } - objTree, err := objecttree.BuildObjectTree(newStorage, tm.aclList) + objTree, err := BuildMigratableObjectTree(newStorage, tm.aclList) if err != nil { return fmt.Errorf("migration: failed to build object tree: %w", err) } - addPayload := objecttree.RawChangesPayload{ + addPayload := RawChangesPayload{ NewHeads: heads, RawChanges: tm.allChanges, SnapshotPath: []string{rootChange.Id}, @@ -87,7 +106,7 @@ func (tm *treeMigrator) migrateTreeStorage(ctx context.Context, storage oldstora return nil } -func (tm *treeMigrator) dfs(ctx context.Context, heads []string, breakpoint string) { +func (tm *TreeMigrator) dfs(ctx context.Context, heads []string, breakpoint string) { tm.idStack = tm.idStack[:0] uniqMap := map[string]struct{}{breakpoint: {}} tm.idStack = append(tm.idStack, heads...) @@ -114,7 +133,7 @@ func (tm *treeMigrator) dfs(ctx context.Context, heads []string, breakpoint stri } } -func (tm *treeMigrator) loadChange(ctx context.Context, id string) (ch *objecttree.Change, err error) { +func (tm *TreeMigrator) loadChange(ctx context.Context, id string) (ch *Change, err error) { if ch, ok := tm.cache[id]; ok { return ch, nil } diff --git a/commonspace/spacestorage/migration/spacemigrator.go b/commonspace/spacestorage/migration/spacemigrator.go index 24225274..2d585bfb 100644 --- a/commonspace/spacestorage/migration/spacemigrator.go +++ b/commonspace/spacestorage/migration/spacemigrator.go @@ -12,6 +12,7 @@ import ( "github.com/anyproto/any-store/anyenc" "go.uber.org/zap" + "github.com/anyproto/any-sync/commonspace/object/tree/objecttree" "github.com/anyproto/any-sync/commonspace/spacestorage" "github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage" "github.com/anyproto/any-sync/util/crypto" @@ -122,10 +123,10 @@ func (s *spaceMigrator) MigrateId(ctx context.Context, id string, progress Progr if err != nil { return fmt.Errorf("migration: failed to get stored ids: %w", err) } - treeMigrators := make([]*treeMigrator, 0, s.numParallel) - ch := make(chan *treeMigrator, s.numParallel) + treeMigrators := make([]*objecttree.TreeMigrator, 0, s.numParallel) + ch := make(chan *objecttree.TreeMigrator, s.numParallel) for i := 0; i < s.numParallel; i++ { - treeMigrators = append(treeMigrators, newTreeMigrator(crypto.NewKeyStorage(), aclList)) + treeMigrators = append(treeMigrators, objecttree.NewTreeMigrator(crypto.NewKeyStorage(), aclList)) ch <- treeMigrators[i] } var allErrors []error @@ -141,7 +142,7 @@ func (s *spaceMigrator) MigrateId(ctx context.Context, id string, progress Progr log.Warn("migration: failed to get old tree storage", zap.String("id", id), zap.Error(err)) return } - err = tm.migrateTreeStorage(ctx, treeStorage, newStorage.HeadStorage(), newStorage.AnyStore()) + err = tm.MigrateTreeStorage(ctx, treeStorage, newStorage.HeadStorage(), newStorage.AnyStore()) if err != nil { allErrors = append(allErrors, fmt.Errorf("migration: failed to migrate tree storage: %w", err)) return