From af377b369fd98238a01a4f2b5427cbd3873a1a55 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Fri, 4 Oct 2024 17:22:57 +0200 Subject: [PATCH] GO-4192 app start improvements --- core/block/object/objectcreator/installer.go | 7 +- core/domain/bundledObject.go | 38 ++++++++++ space/clientspace/space.go | 75 ++++++++++++------- .../components/dependencies/bundledobjects.go | 3 +- .../internal/objectprovider/objectprovider.go | 4 + 5 files changed, 96 insertions(+), 31 deletions(-) create mode 100644 core/domain/bundledObject.go diff --git a/core/block/object/objectcreator/installer.go b/core/block/object/objectcreator/installer.go index d2de7d2b5..3e3a5874a 100644 --- a/core/block/object/objectcreator/installer.go +++ b/core/block/object/objectcreator/installer.go @@ -26,7 +26,7 @@ func (s *service) BundledObjectsIdsToInstall( ctx context.Context, space clientspace.Space, sourceObjectIds []string, -) (objectIds []string, err error) { +) (ids domain.BundledObjectIds, err error) { marketplaceSpace, err := s.spaceService.Get(ctx, addr.AnytypeMarketplaceWorkspace) if err != nil { return nil, fmt.Errorf("get marketplace space: %w", err) @@ -51,7 +51,10 @@ func (s *service) BundledObjectsIdsToInstall( if err != nil { return err } - objectIds = append(objectIds, objectId) + ids = append(ids, domain.BundledObjectId{ + SourceId: sourceObjectId, + DerivedObjectId: objectId, + }) return nil }) if err != nil { diff --git a/core/domain/bundledObject.go b/core/domain/bundledObject.go new file mode 100644 index 000000000..ced527b60 --- /dev/null +++ b/core/domain/bundledObject.go @@ -0,0 +1,38 @@ +package domain + +type BundledObjectId struct { + SourceId string + DerivedObjectId string +} + +type BundledObjectIds []BundledObjectId + +func (b BundledObjectIds) Len() int { + return len(b) +} + +func (b BundledObjectIds) SourceIds() []string { + var ids = make([]string, 0, len(b)) + for _, bo := range b { + ids = append(ids, bo.SourceId) + } + return ids +} + +func (b BundledObjectIds) DerivedObjectIds() []string { + var ids = make([]string, 0, len(b)) + for _, bo := range b { + ids = append(ids, bo.DerivedObjectId) + } + return ids +} + +func (b BundledObjectIds) Filter(f func(bo BundledObjectId) bool) BundledObjectIds { + var res = make([]BundledObjectId, 0, len(b)) + for _, bo := range b { + if f(bo) { + res = append(res, bo) + } + } + return res +} diff --git a/space/clientspace/space.go b/space/clientspace/space.go index 9d4c99785..277d20805 100644 --- a/space/clientspace/space.go +++ b/space/clientspace/space.go @@ -15,6 +15,7 @@ import ( "github.com/anyproto/any-sync/util/crypto" "github.com/gogo/protobuf/types" "go.uber.org/zap" + "golang.org/x/exp/slices" "github.com/anyproto/anytype-heart/core/block/editor/profilemigration" "github.com/anyproto/anytype-heart/core/block/editor/smartblock" @@ -28,7 +29,6 @@ import ( "github.com/anyproto/anytype-heart/space/spacecore" "github.com/anyproto/anytype-heart/space/spacecore/peermanager" "github.com/anyproto/anytype-heart/space/spacecore/storage" - "github.com/anyproto/anytype-heart/util/slice" ) type Space interface { @@ -67,7 +67,7 @@ type spaceIndexer interface { type bundledObjectsInstaller interface { InstallBundledObjects(ctx context.Context, spc Space, ids []string, isNewSpace bool) ([]string, []*types.Struct, error) - BundledObjectsIdsToInstall(ctx context.Context, spc Space, sourceObjectIds []string) (objectIds []string, err error) + BundledObjectsIdsToInstall(ctx context.Context, spc Space, sourceObjectIds []string) (ids domain.BundledObjectIds, err error) } var log = logger.NewNamed("client.space") @@ -88,6 +88,9 @@ type space struct { loadMandatoryObjectsCh chan struct{} loadMandatoryObjectsErr error + + loadMissingBundledObjectsCtx context.Context + loadMissingBundledObjectsCtxCancel context.CancelFunc } type SpaceDeps struct { @@ -113,6 +116,7 @@ func BuildSpace(ctx context.Context, deps SpaceDeps) (Space, error) { myIdentity: deps.AccountService.Account().SignKey.GetPublic(), loadMandatoryObjectsCh: make(chan struct{}), } + sp.loadMissingBundledObjectsCtx, sp.loadMissingBundledObjectsCtxCancel = context.WithCancel(context.Background()) sp.Cache = objectcache.New(deps.AccountService, deps.ObjectFactory, deps.PersonalSpaceId, sp) sp.ObjectProvider = objectprovider.NewObjectProvider(deps.CommonSpace.Id(), deps.PersonalSpaceId, sp.Cache) var err error @@ -129,7 +133,8 @@ func BuildSpace(ctx context.Context, deps SpaceDeps) (Space, error) { if err != nil { return nil, fmt.Errorf("unmark space created: %w", err) } - if err = sp.InstallBundledObjects(ctx); err != nil { + ids := getBundledObjectsToInstall() + if _, _, err = sp.installer.InstallBundledObjects(ctx, sp, ids, true); err != nil { return nil, fmt.Errorf("install bundled objects: %w", err) } } @@ -137,6 +142,19 @@ func BuildSpace(ctx context.Context, deps SpaceDeps) (Space, error) { return sp, nil } +func (s *space) tryLoadBundledAndInstallIfMissing() { + ctxWithPeerTimeout := context.WithValue(s.loadMissingBundledObjectsCtx, peermanager.ContextPeerFindDeadlineKey, time.Now().Add(BundledObjectsPeerFindTimeout)) + + missingSourceIds, err := s.TryLoadBundledObjects(ctxWithPeerTimeout) + if err != nil { + log.Error("failed to load bundled objects", zap.Error(err)) + } + _, _, err = s.installer.InstallBundledObjects(s.loadMissingBundledObjectsCtx, s, missingSourceIds, true) + if err != nil { + log.Error("failed to install bundled objects", zap.Error(err)) + } +} + func (s *space) mandatoryObjectsLoad(ctx context.Context, disableRemoteLoad bool) { defer close(s.loadMandatoryObjectsCh) s.loadMandatoryObjectsErr = s.indexer.ReindexSpace(s) @@ -151,11 +169,7 @@ func (s *space) mandatoryObjectsLoad(ctx context.Context, disableRemoteLoad bool if s.loadMandatoryObjectsErr != nil { return } - ctxWithPeerTimeout := context.WithValue(loadCtx, peermanager.ContextPeerFindDeadlineKey, time.Now().Add(BundledObjectsPeerFindTimeout)) - if err := s.TryLoadBundledObjects(ctxWithPeerTimeout); err != nil { - log.Error("failed to load bundled objects", zap.Error(err)) - } - s.loadMandatoryObjectsErr = s.InstallBundledObjects(ctx) + go s.tryLoadBundledAndInstallIfMissing() if s.loadMandatoryObjectsErr != nil { return } @@ -258,7 +272,7 @@ func (s *space) Close(ctx context.Context) error { return s.common.Close() } -func (s *space) InstallBundledObjects(ctx context.Context) error { +func getBundledObjectsToInstall() []string { ids := make([]string, 0, len(bundle.SystemTypes)+len(bundle.SystemRelations)) for _, ot := range bundle.SystemTypes { ids = append(ids, ot.BundledURL()) @@ -266,14 +280,10 @@ func (s *space) InstallBundledObjects(ctx context.Context) error { for _, rk := range bundle.SystemRelations { ids = append(ids, rk.BundledURL()) } - _, _, err := s.installer.InstallBundledObjects(ctx, s, ids, true) - if err != nil { - return err - } - return nil + return ids } -func (s *space) TryLoadBundledObjects(ctx context.Context) error { +func (s *space) TryLoadBundledObjects(ctx context.Context) (missingSourceIds []string, err error) { st := time.Now() defer func() { if dur := time.Since(st); dur > time.Millisecond*200 { @@ -281,25 +291,34 @@ func (s *space) TryLoadBundledObjects(ctx context.Context) error { } }() - ids := make([]string, 0, len(bundle.SystemTypes)+len(bundle.SystemRelations)) - for _, ot := range bundle.SystemTypes { - ids = append(ids, ot.BundledURL()) - } - for _, rk := range bundle.SystemRelations { - ids = append(ids, rk.BundledURL()) - } - objectIds, err := s.installer.BundledObjectsIdsToInstall(ctx, s, ids) + ids := getBundledObjectsToInstall() + bundledObjectIds, err := s.installer.BundledObjectsIdsToInstall(ctx, s, ids) if err != nil { - return err + return nil, err } storedIds, err := s.Storage().StoredIds() if err != nil { - return err + return nil, err } + + missingIds := bundledObjectIds.Filter(func(bo domain.BundledObjectId) bool { + return !slices.Contains(storedIds, bo.DerivedObjectId) + }) + // only load objects that are not already stored - objectIds = slice.Difference(objectIds, storedIds) - s.LoadObjectsIgnoreErrs(ctx, objectIds) - return nil + s.LoadObjectsIgnoreErrs(ctx, missingIds.DerivedObjectIds()) + // todo: make LoadObjectsIgnoreErrs return list of loaded ids + + storedIds, err = s.Storage().StoredIds() + if err != nil { + return nil, err + } + + missingIds = bundledObjectIds.Filter(func(bo domain.BundledObjectId) bool { + return !slices.Contains(storedIds, bo.DerivedObjectId) + }) + + return missingIds.SourceIds(), nil } func (s *space) migrationProfileObject(ctx context.Context) error { diff --git a/space/internal/components/dependencies/bundledobjects.go b/space/internal/components/dependencies/bundledobjects.go index ea97c9f49..a9e467f33 100644 --- a/space/internal/components/dependencies/bundledobjects.go +++ b/space/internal/components/dependencies/bundledobjects.go @@ -5,10 +5,11 @@ import ( "github.com/gogo/protobuf/types" + "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/space/clientspace" ) type BundledObjectsInstaller interface { InstallBundledObjects(ctx context.Context, spc clientspace.Space, ids []string, isNewSpace bool) ([]string, []*types.Struct, error) - BundledObjectsIdsToInstall(ctx context.Context, spc clientspace.Space, sourceObjectIds []string) (objectIds []string, err error) + BundledObjectsIdsToInstall(ctx context.Context, spc clientspace.Space, sourceObjectIds []string) (ids domain.BundledObjectIds, err error) } diff --git a/space/internal/objectprovider/objectprovider.go b/space/internal/objectprovider/objectprovider.go index 4ae8421c5..7c50fe3bc 100644 --- a/space/internal/objectprovider/objectprovider.go +++ b/space/internal/objectprovider/objectprovider.go @@ -138,6 +138,8 @@ func (o *objectProvider) loadObjectsAsync(ctx context.Context, objIDs []string) for _, id := range objIDs { select { case <-ctx.Done(): + log.ErrorCtx(ctx, "loadObjectsAsync context done", zap.Error(ctx.Err()), zap.String("spaceId", o.spaceId), zap.String("objectId", id)) + results <- ctx.Err() continue case limiter <- struct{}{}: @@ -148,6 +150,8 @@ func (o *objectProvider) loadObjectsAsync(ctx context.Context, objIDs []string) }() _, err := o.cache.GetObject(ctx, id) if err != nil { + log.ErrorCtx(ctx, "loadObjectsAsync failed", zap.Error(ctx.Err()), zap.String("spaceId", o.spaceId), zap.String("objectId", id)) + // we had a bug that allowed some users to remove their profile // this workaround is to allow these users to load their accounts without errors and export their anytype data if id == o.derivedObjectIds.Profile {