diff --git a/core/anytype/bootstrap.go b/core/anytype/bootstrap.go index 313588ff8..a03752c32 100644 --- a/core/anytype/bootstrap.go +++ b/core/anytype/bootstrap.go @@ -2,6 +2,9 @@ package anytype import ( "context" + "os" + + "github.com/anytypeio/any-sync/app" "github.com/anytypeio/any-sync/commonfile/fileservice" "github.com/anytypeio/any-sync/commonspace" "github.com/anytypeio/any-sync/net/dialer" @@ -9,24 +12,13 @@ import ( "github.com/anytypeio/any-sync/net/secureservice" "github.com/anytypeio/any-sync/net/streampool" "github.com/anytypeio/any-sync/nodeconf" - "github.com/anytypeio/go-anytype-middleware/core/filestorage" - "github.com/anytypeio/go-anytype-middleware/core/filestorage/rpcstore" - "github.com/anytypeio/go-anytype-middleware/space" - "github.com/anytypeio/go-anytype-middleware/space/debug/clientdebugrpc" - "github.com/anytypeio/go-anytype-middleware/space/peermanager" - "github.com/anytypeio/go-anytype-middleware/space/storage" - "github.com/anytypeio/go-anytype-middleware/space/typeprovider" - "github.com/anytypeio/go-anytype-middleware/util/builtinobjects" - "github.com/anytypeio/go-anytype-middleware/util/builtintemplate" - "os" - "github.com/anytypeio/any-sync/app" "github.com/anytypeio/go-anytype-middleware/core/account" "github.com/anytypeio/go-anytype-middleware/core/anytype/config" "github.com/anytypeio/go-anytype-middleware/core/block" "github.com/anytypeio/go-anytype-middleware/core/block/bookmark" decorator "github.com/anytypeio/go-anytype-middleware/core/block/bookmark/bookmarkimporter" - "github.com/anytypeio/go-anytype-middleware/core/block/doc" + "github.com/anytypeio/go-anytype-middleware/core/block/collection" "github.com/anytypeio/go-anytype-middleware/core/block/editor" "github.com/anytypeio/go-anytype-middleware/core/block/export" importer "github.com/anytypeio/go-anytype-middleware/core/block/import" @@ -37,6 +29,8 @@ import ( "github.com/anytypeio/go-anytype-middleware/core/configfetcher" "github.com/anytypeio/go-anytype-middleware/core/debug" "github.com/anytypeio/go-anytype-middleware/core/event" + "github.com/anytypeio/go-anytype-middleware/core/filestorage" + "github.com/anytypeio/go-anytype-middleware/core/filestorage/rpcstore" "github.com/anytypeio/go-anytype-middleware/core/history" "github.com/anytypeio/go-anytype-middleware/core/indexer" "github.com/anytypeio/go-anytype-middleware/core/kanban" @@ -57,6 +51,16 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/profilefinder" walletUtil "github.com/anytypeio/go-anytype-middleware/pkg/lib/wallet" + "github.com/anytypeio/go-anytype-middleware/space" + "github.com/anytypeio/go-anytype-middleware/space/clientserver" + "github.com/anytypeio/go-anytype-middleware/space/debug/clientdebugrpc" + "github.com/anytypeio/go-anytype-middleware/space/localdiscovery" + "github.com/anytypeio/go-anytype-middleware/space/peermanager" + "github.com/anytypeio/go-anytype-middleware/space/peerstore" + "github.com/anytypeio/go-anytype-middleware/space/storage" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" + "github.com/anytypeio/go-anytype-middleware/util/builtinobjects" + "github.com/anytypeio/go-anytype-middleware/util/builtintemplate" "github.com/anytypeio/go-anytype-middleware/util/linkpreview" "github.com/anytypeio/go-anytype-middleware/util/unsplash" ) @@ -116,23 +120,36 @@ func Bootstrap(a *app.App, components ...app.Component) { for _, c := range components { a.Register(c) } + + spaceService := space.New() + sbtProvider := typeprovider.New(spaceService) + objectStore := objectstore.New(sbtProvider) + objectCreator := object.NewCreator(sbtProvider) + blockService := block.New(sbtProvider) + collectionService := collection.New(blockService, objectStore, objectCreator, blockService) + walletService := a.Component(wallet.CName).(wallet.Wallet) + tempDirService := core.NewTempDirService(walletService) + a.Register(clientds.New()). Register(nodeconf.New()). + Register(peerstore.New()). Register(storage.New()). Register(secureservice.New()). Register(dialer.New()). Register(pool.New()). Register(streampool.New()). + Register(clientserver.New()). Register(commonspace.New()). Register(rpcstore.New()). Register(fileservice.New()). Register(filestorage.New()). - Register(space.New()). + Register(localdiscovery.New()). + Register(spaceService). Register(peermanager.New()). - Register(typeprovider.New()). + Register(sbtProvider). Register(relation.New()). Register(ftsearch.New()). - Register(objectstore.New()). + Register(objectStore). Register(filestore.New()). Register(recordsbatcher.New()). Register(files.New()). @@ -144,25 +161,25 @@ func Bootstrap(a *app.App, components ...app.Component) { Register(core.New()). Register(builtintemplate.New()). Register(status.New()). - Register(block.New()). - Register(doc.New()). + Register(blockService). Register(indexer.New()). Register(history.New()). Register(gateway.New()). - Register(export.New()). + Register(export.New(sbtProvider)). Register(linkpreview.New()). - Register(unsplash.New()). - Register(restriction.New()). + Register(unsplash.New(tempDirService)). + Register(restriction.New(sbtProvider)). Register(debug.New()). Register(clientdebugrpc.New()). - Register(subscription.New()). - Register(builtinobjects.New()). - Register(bookmark.New()). + Register(collectionService). + Register(subscription.New(collectionService, sbtProvider)). + Register(builtinobjects.New(sbtProvider)). + Register(bookmark.New(tempDirService)). Register(session.New()). - Register(importer.New()). + Register(importer.New(tempDirService, sbtProvider)). Register(decorator.New()). - Register(object.NewCreator()). + Register(objectCreator). Register(kanban.New()). - Register(editor.NewObjectFactory()) + Register(editor.NewObjectFactory(tempDirService, sbtProvider)) return } diff --git a/core/block/cache.go b/core/block/cache.go index d790ce1f7..6fbcf1cea 100644 --- a/core/block/cache.go +++ b/core/block/cache.go @@ -93,7 +93,8 @@ func (s *Service) cacheLoad(ctx context.Context, id string) (value ocache.Object break } - sbt, _ := coresb.SmartBlockTypeFromID(id) + // TODO Move to source service. IT WILL NOT WORK FOR SETS/COLLECTIONS + sbt, _ := s.sbtProvider.Type(id) switch sbt { case coresb.SmartBlockTypeSubObject: return s.initSubObject(ctx, id) @@ -133,7 +134,7 @@ func (s *Service) GetObject(ctx context.Context, spaceId, id string) (sb smartbl return } return v.(smartblock.SmartBlock), func() { - sbt, _ := coresb.SmartBlockTypeFromID(id) + sbt, _ := s.sbtProvider.Type(id) // TODO: [MR] check if this is correct if sbt == coresb.SmartBlockTypeSubObject { s.cache.Release(s.anytype.PredefinedBlocks().Account) @@ -187,7 +188,8 @@ func (s *Service) DeleteObject(id string) (err error) { return } - sbt, _ := coresb.SmartBlockTypeFromID(id) + // TODO Move to source service ? + sbt, _ := s.sbtProvider.Type(id) switch sbt { case coresb.SmartBlockTypePage: var space commonspace.Space diff --git a/core/block/create.go b/core/block/create.go index ca003b7fa..9be9c8a0d 100644 --- a/core/block/create.go +++ b/core/block/create.go @@ -65,8 +65,12 @@ func (s *Service) TemplateClone(id string) (templateID string, err error) { } func (s *Service) ObjectDuplicate(id string) (objectID string, err error) { - var st *state.State + var ( + st *state.State + sbt coresb.SmartBlockType + ) if err = s.Do(id, func(b smartblock.SmartBlock) error { + sbt = coresb.SmartBlockType(b.Type()) if err = b.Restrictions().Object.Check(model.Restrictions_Duplicate); err != nil { return err } @@ -77,11 +81,6 @@ func (s *Service) ObjectDuplicate(id string) (objectID string, err error) { return } - sbt, err := coresb.SmartBlockTypeFromID(id) - if err != nil { - return - } - objectID, _, err = s.objectCreator.CreateSmartBlockFromState(context.TODO(), sbt, nil, st) if err != nil { return diff --git a/core/block/editor/collection.go b/core/block/editor/collection.go index 5c6c9d873..8e89fcee4 100644 --- a/core/block/editor/collection.go +++ b/core/block/editor/collection.go @@ -8,6 +8,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/core" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" ) type Collection struct { @@ -26,9 +27,10 @@ func NewCollection( objectStore objectstore.ObjectStore, relationService relation.Service, collectionService CollectionService, + sbtProvider typeprovider.SmartBlockTypeProvider, ) *Collection { return &Collection{ - Set: NewSet(anytype, objectStore, relationService), + Set: NewSet(anytype, objectStore, relationService, sbtProvider), collectionService: collectionService, objectStore: objectStore, } diff --git a/core/block/editor/dataview/dataview.go b/core/block/editor/dataview/dataview.go index 2c25a1384..729493ee3 100644 --- a/core/block/editor/dataview/dataview.go +++ b/core/block/editor/dataview/dataview.go @@ -25,6 +25,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" "github.com/anytypeio/go-anytype-middleware/pkg/lib/schema" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" "github.com/anytypeio/go-anytype-middleware/util/slice" ) @@ -73,12 +74,14 @@ func NewDataview( anytype core.Service, objectStore objectstore.ObjectStore, relationService relation2.Service, + sbtProvider typeprovider.SmartBlockTypeProvider, ) Dataview { dv := &sdataview{ SmartBlock: sb, anytype: anytype, objectStore: objectStore, relationService: relationService, + sbtProvider: sbtProvider, } sb.AddHook(dv.checkDVBlocks, smartblock.HookBeforeApply) return dv @@ -89,6 +92,7 @@ type sdataview struct { anytype core.Service objectStore objectstore.ObjectStore relationService relation2.Service + sbtProvider typeprovider.SmartBlockTypeProvider } func (d *sdataview) GetDataviewBlock(s *state.State, blockID string) (dataview.Block, error) { @@ -115,7 +119,7 @@ func (d *sdataview) SetSource(ctx *session.Context, blockId string, source []str return d.Apply(s, smartblock.NoRestrictions) } - dvContent, _, err := DataviewBlockBySource(d.objectStore, source) + dvContent, _, err := DataviewBlockBySource(d.sbtProvider, d.objectStore, source) if err != nil { return } @@ -314,7 +318,7 @@ func (d *sdataview) CreateView(ctx *session.Context, id string, }} } - sbType, err := smartblock2.SmartBlockTypeFromID(d.Id()) + sbType, err := d.sbtProvider.Type(d.Id()) if err != nil { return nil, err } @@ -357,11 +361,11 @@ func (d *sdataview) UpdateViewObjectOrder(ctx *session.Context, blockId string, return d.Apply(st) } -func SchemaBySources(sources []string, store objectstore.ObjectStore, optionalRelations []*model.RelationLink) (schema.Schema, error) { +func SchemaBySources(sbtProvider typeprovider.SmartBlockTypeProvider, sources []string, store objectstore.ObjectStore, optionalRelations []*model.RelationLink) (schema.Schema, error) { var hasRelations, hasType bool for _, source := range sources { - sbt, err := smartblock2.SmartBlockTypeFromID(source) + sbt, err := sbtProvider.Type(source) if err != nil { return nil, err } @@ -433,7 +437,7 @@ func SchemaBySources(sources []string, store objectstore.ObjectStore, optionalRe } func (d *sdataview) getSchema(dvBlock dataview.Block, source []string) (schema.Schema, error) { - return SchemaBySources(source, d.objectStore, dvBlock.Model().GetDataview().RelationLinks) + return SchemaBySources(d.sbtProvider, source, d.objectStore, dvBlock.Model().GetDataview().RelationLinks) } func (d *sdataview) checkDVBlocks(info smartblock.ApplyInfo) (err error) { @@ -565,8 +569,8 @@ func calculateEntriesDiff(a, b []database.Record) (updated []*types.Struct, remo return } -func DataviewBlockBySource(store objectstore.ObjectStore, source []string) (res model.BlockContentOfDataview, schema schema.Schema, err error) { - if schema, err = SchemaBySources(source, store, nil); err != nil { +func DataviewBlockBySource(sbtProvider typeprovider.SmartBlockTypeProvider, store objectstore.ObjectStore, source []string) (res model.BlockContentOfDataview, schema schema.Schema, err error) { + if schema, err = SchemaBySources(sbtProvider, source, store, nil); err != nil { return } diff --git a/core/block/editor/factory.go b/core/block/editor/factory.go index 1ba1d1055..b340548fc 100644 --- a/core/block/editor/factory.go +++ b/core/block/editor/factory.go @@ -2,11 +2,12 @@ package editor import ( "fmt" + "github.com/anytypeio/any-sync/app" "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/go-anytype-middleware/core/block/editor/bookmark" "github.com/anytypeio/go-anytype-middleware/core/block/editor/file" - _import "github.com/anytypeio/go-anytype-middleware/core/block/editor/import" "github.com/anytypeio/go-anytype-middleware/core/block/editor/smartblock" "github.com/anytypeio/go-anytype-middleware/core/block/source" "github.com/anytypeio/go-anytype-middleware/core/event" @@ -15,6 +16,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/core" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" ) type ObjectFactory struct { @@ -23,18 +25,25 @@ type ObjectFactory struct { bookmarkService bookmark.BookmarkService detailsModifier DetailsModifier fileBlockService file.BlockService - importerCtrl _import.Services - objectCreator _import.ObjectCreator objectStore objectstore.ObjectStore relationService relation2.Service sourceService source.Service sendEvent func(e *pb.Event) + tempDirProvider core.TempDirProvider + collectionService CollectionService + sbtProvider typeprovider.SmartBlockTypeProvider app *app.App } -func NewObjectFactory() *ObjectFactory { - return &ObjectFactory{} +func NewObjectFactory( + tempDirProvider core.TempDirProvider, + sbtProvider typeprovider.SmartBlockTypeProvider, +) *ObjectFactory { + return &ObjectFactory{ + tempDirProvider: tempDirProvider, + sbtProvider: sbtProvider, + } } func (f *ObjectFactory) Init(a *app.App) (err error) { @@ -43,12 +52,11 @@ func (f *ObjectFactory) Init(a *app.App) (err error) { f.bookmarkService = app.MustComponent[bookmark.BookmarkService](a) f.detailsModifier = app.MustComponent[DetailsModifier](a) f.fileBlockService = app.MustComponent[file.BlockService](a) - f.importerCtrl = app.MustComponent[_import.Services](a) - f.objectCreator = app.MustComponent[_import.ObjectCreator](a) f.objectStore = app.MustComponent[objectstore.ObjectStore](a) f.relationService = app.MustComponent[relation2.Service](a) f.sourceService = app.MustComponent[source.Service](a) f.sendEvent = app.MustComponent[event.Sender](a).Send + f.collectionService = app.MustComponent[CollectionService](a) f.app = a return nil @@ -103,13 +111,13 @@ func (f *ObjectFactory) New(sbType model.SmartBlockType) smartblock.SmartBlock { case model.SmartBlockType_Page, model.SmartBlockType_Date: return NewPage( f.objectStore, - f.importerCtrl, - f.objectCreator, f.anytype, f.fileBlockService, f.bookmarkBlockService, f.bookmarkService, f.relationService, + f.tempDirProvider, + f.sbtProvider, ) case model.SmartBlockType_Archive: return NewArchive( @@ -120,8 +128,6 @@ func (f *ObjectFactory) New(sbType model.SmartBlockType) smartblock.SmartBlock { return NewDashboard( f.detailsModifier, f.objectStore, - f.importerCtrl, - f.objectCreator, f.anytype, ) case model.SmartBlockType_Set: @@ -129,6 +135,15 @@ func (f *ObjectFactory) New(sbType model.SmartBlockType) smartblock.SmartBlock { f.anytype, f.objectStore, f.relationService, + f.sbtProvider, + ) + case model.SmartBlockType_Collection: + return NewCollection( + f.anytype, + f.objectStore, + f.relationService, + f.collectionService, + f.sbtProvider, ) case model.SmartBlockType_ProfilePage, model.SmartBlockType_AnytypeProfile: return NewProfile( @@ -138,6 +153,7 @@ func (f *ObjectFactory) New(sbType model.SmartBlockType) smartblock.SmartBlock { f.bookmarkBlockService, f.bookmarkService, f.sendEvent, + f.tempDirProvider, ) case model.SmartBlockType_STObjectType, model.SmartBlockType_BundledObjectType: @@ -145,20 +161,17 @@ func (f *ObjectFactory) New(sbType model.SmartBlockType) smartblock.SmartBlock { f.anytype, f.objectStore, f.relationService, + f.sbtProvider, ) case model.SmartBlockType_BundledRelation: return NewSet( f.anytype, f.objectStore, f.relationService, + f.sbtProvider, ) case model.SmartBlockType_SubObject: - return NewSubObject( - f.objectStore, - f.fileBlockService, - f.anytype, - f.relationService, - ) + panic("subobject not supported via factory") case model.SmartBlockType_File: return NewFiles() case model.SmartBlockType_MarketplaceType: @@ -166,40 +179,43 @@ func (f *ObjectFactory) New(sbType model.SmartBlockType) smartblock.SmartBlock { f.anytype, f.objectStore, f.relationService, + f.sbtProvider, ) case model.SmartBlockType_MarketplaceRelation: return NewMarketplaceRelation( f.anytype, f.objectStore, f.relationService, + f.sbtProvider, ) case model.SmartBlockType_MarketplaceTemplate: return NewMarketplaceTemplate( f.anytype, f.objectStore, f.relationService, + f.sbtProvider, ) case model.SmartBlockType_Template: return NewTemplate( f.objectStore, - f.importerCtrl, - f.objectCreator, f.anytype, f.fileBlockService, f.bookmarkBlockService, f.bookmarkService, f.relationService, + f.tempDirProvider, + f.sbtProvider, ) case model.SmartBlockType_BundledTemplate: return NewTemplate( f.objectStore, - f.importerCtrl, - f.objectCreator, f.anytype, f.fileBlockService, f.bookmarkBlockService, f.bookmarkService, f.relationService, + f.tempDirProvider, + f.sbtProvider, ) case model.SmartBlockType_Breadcrumbs: return NewBreadcrumbs() @@ -211,6 +227,8 @@ func (f *ObjectFactory) New(sbType model.SmartBlockType) smartblock.SmartBlock { f.sourceService, f.detailsModifier, f.fileBlockService, + f.tempDirProvider, + f.sbtProvider, ) case model.SmartBlockType_Widget: return NewWidgetObject() diff --git a/core/block/editor/marketplace.go b/core/block/editor/marketplace.go index 183693b6e..9ecd76739 100644 --- a/core/block/editor/marketplace.go +++ b/core/block/editor/marketplace.go @@ -9,6 +9,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" ) @@ -25,9 +26,10 @@ func NewMarketplaceType( anytype core.Service, objectStore objectstore.ObjectStore, relationService relation2.Service, + sbtProvider typeprovider.SmartBlockTypeProvider, ) *MarketplaceType { return &MarketplaceType{ - Set: NewSet(anytype, objectStore, relationService), + Set: NewSet(anytype, objectStore, relationService, sbtProvider), } } @@ -113,9 +115,10 @@ func NewMarketplaceRelation( anytype core.Service, objectStore objectstore.ObjectStore, relationService relation2.Service, + sbtProvider typeprovider.SmartBlockTypeProvider, ) *MarketplaceRelation { return &MarketplaceRelation{ - Set: NewSet(anytype, objectStore, relationService), + Set: NewSet(anytype, objectStore, relationService, sbtProvider), } } @@ -183,9 +186,10 @@ func NewMarketplaceTemplate( anytype core.Service, objectStore objectstore.ObjectStore, relationService relation2.Service, + sbtProvider typeprovider.SmartBlockTypeProvider, ) *MarketplaceTemplate { return &MarketplaceTemplate{ - Set: NewSet(anytype, objectStore, relationService), + Set: NewSet(anytype, objectStore, relationService, sbtProvider), } } diff --git a/core/block/editor/objecttype.go b/core/block/editor/objecttype.go index d5d446166..666764671 100644 --- a/core/block/editor/objecttype.go +++ b/core/block/editor/objecttype.go @@ -18,6 +18,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" "github.com/anytypeio/go-anytype-middleware/util/slice" ) @@ -31,9 +32,10 @@ type ObjectType struct { func NewObjectType(anytype core.Service, objectStore objectstore.ObjectStore, relationService relation2.Service, + sbtProvider typeprovider.SmartBlockTypeProvider, ) *ObjectType { return &ObjectType{ - Set: NewSet(anytype, objectStore, relationService), + Set: NewSet(anytype, objectStore, relationService, sbtProvider), relationService: relationService, } diff --git a/core/block/editor/page.go b/core/block/editor/page.go index 82b921db7..f11f0b76a 100644 --- a/core/block/editor/page.go +++ b/core/block/editor/page.go @@ -16,6 +16,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/core" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" ) @@ -42,6 +43,7 @@ func NewPage( bookmarkService bookmark.BookmarkService, relationService relation2.Service, tempDirProvider core.TempDirProvider, + sbtProvider typeprovider.SmartBlockTypeProvider, ) *Page { sb := smartblock.New() f := file.NewFile( @@ -76,6 +78,7 @@ func NewPage( anytype, objectStore, relationService, + sbtProvider, ), TableEditor: table.NewEditor(sb), diff --git a/core/block/editor/set.go b/core/block/editor/set.go index 0f68ef815..25cd52670 100644 --- a/core/block/editor/set.go +++ b/core/block/editor/set.go @@ -12,6 +12,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/core" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" ) @@ -27,6 +28,7 @@ func NewSet( anytype core.Service, objectStore objectstore.ObjectStore, relationService relation2.Service, + sbtProvider typeprovider.SmartBlockTypeProvider, ) *Set { sb := smartblock.New() return &Set{ @@ -38,6 +40,7 @@ func NewSet( anytype, objectStore, relationService, + sbtProvider, ), Text: stext.NewText( sb, diff --git a/core/block/editor/subobject.go b/core/block/editor/subobject.go index 31403fc13..4e16fd4e9 100644 --- a/core/block/editor/subobject.go +++ b/core/block/editor/subobject.go @@ -21,6 +21,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" ) @@ -43,6 +44,7 @@ func NewSubObject( relationService relation2.Service, forcedObjectType bundle.TypeKey, tempDirProvider core.TempDirProvider, + sbtProvider typeprovider.SmartBlockTypeProvider, ) *SubObject { sb := smartblock.New() return &SubObject{ @@ -69,6 +71,7 @@ func NewSubObject( anytype, objectStore, relationService, + sbtProvider, ), forcedObjectType: forcedObjectType, } diff --git a/core/block/editor/subobjectcollection.go b/core/block/editor/subobjectcollection.go index c000396ce..646bfef8f 100644 --- a/core/block/editor/subobjectcollection.go +++ b/core/block/editor/subobjectcollection.go @@ -22,6 +22,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/internalflag" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" "github.com/anytypeio/go-anytype-middleware/util/slice" @@ -62,6 +63,7 @@ type SubObjectCollection struct { relationService relation2.Service fileBlockService file.BlockService tempDirProvider core.TempDirProvider + sbtProvider typeprovider.SmartBlockTypeProvider } func NewSubObjectCollection( @@ -72,6 +74,7 @@ func NewSubObjectCollection( sourceService source.Service, fileBlockService file.BlockService, tempDirProvider core.TempDirProvider, + sbtProvider typeprovider.SmartBlockTypeProvider, ) *SubObjectCollection { sb := smartblock.New() return &SubObjectCollection{ @@ -87,6 +90,7 @@ func NewSubObjectCollection( anytype, objectStore, relationService, + sbtProvider, ), objectStore: objectStore, @@ -296,13 +300,13 @@ func (c *SubObjectCollection) initSubObject(st *state.State, collection string, switch collection { case collectionKeyObjectTypes: - subObj = NewObjectType(c.anytype, c.objectStore, c.relationService) + subObj = NewObjectType(c.anytype, c.objectStore, c.relationService, c.sbtProvider) default: ot, ok := collectionKeyToObjectType(collection) if !ok { return fmt.Errorf("unknown collection '%s'", collection) } - subObj = NewSubObject(c.objectStore, c.fileBlockService, c.anytype, c.relationService, ot, c.tempDirProvider) + subObj = NewSubObject(c.objectStore, c.fileBlockService, c.anytype, c.relationService, ot, c.tempDirProvider, c.sbtProvider) } var fullId string diff --git a/core/block/editor/template.go b/core/block/editor/template.go index d07bb02f5..caba10d94 100644 --- a/core/block/editor/template.go +++ b/core/block/editor/template.go @@ -13,6 +13,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/core" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" ) @@ -28,6 +29,7 @@ func NewTemplate( bookmarkService bookmark.BookmarkService, relationService relation2.Service, tempDirProvider core.TempDirProvider, + sbtProvider typeprovider.SmartBlockTypeProvider, ) *Template { return &Template{Page: NewPage( objectStore, @@ -37,6 +39,7 @@ func NewTemplate( bookmarkService, relationService, tempDirProvider, + sbtProvider, )} } diff --git a/core/block/editor/workspaces.go b/core/block/editor/workspaces.go index 0dfbb4fa3..acdbe8806 100644 --- a/core/block/editor/workspaces.go +++ b/core/block/editor/workspaces.go @@ -6,13 +6,11 @@ import ( "strings" "time" + "github.com/anytypeio/any-sync/app" "github.com/anytypeio/any-sync/commonspace/object/treegetter" - "github.com/globalsign/mgo/bson" "github.com/gogo/protobuf/types" - "github.com/anytypeio/any-sync/app" - "github.com/anytypeio/go-anytype-middleware/core/block/editor/dataview" "github.com/anytypeio/go-anytype-middleware/core/block/editor/file" "github.com/anytypeio/go-anytype-middleware/core/block/editor/smartblock" @@ -27,6 +25,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" "github.com/anytypeio/go-anytype-middleware/util/slice" ) @@ -70,6 +69,8 @@ func NewWorkspace( sourceService source.Service, modifier DetailsModifier, fileBlockService file.BlockService, + tempDirProvider core.TempDirProvider, + sbtProvider typeprovider.SmartBlockTypeProvider, ) *Workspaces { return &Workspaces{ SubObjectCollection: NewSubObjectCollection( @@ -79,6 +80,8 @@ func NewWorkspace( relationService, sourceService, fileBlockService, + tempDirProvider, + sbtProvider, ), DetailsModifier: modifier, anytype: anytype, diff --git a/core/block/export/export.go b/core/block/export/export.go index 86e7fec3f..13af0d0f1 100644 --- a/core/block/export/export.go +++ b/core/block/export/export.go @@ -31,6 +31,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" "github.com/anytypeio/go-anytype-middleware/util/text" ) @@ -39,10 +40,6 @@ const CName = "export" var log = logging.Logger("anytype-mw-export") -func New() Export { - return new(export) -} - type Export interface { Export(req pb.RpcObjectListExportRequest) (path string, succeed int, err error) app.Component @@ -52,6 +49,13 @@ type export struct { bs *block.Service objectStore objectstore.ObjectStore a core.Service + sbtProvider typeprovider.SmartBlockTypeProvider +} + +func New(sbtProvider typeprovider.SmartBlockTypeProvider) Export { + return &export{ + sbtProvider: sbtProvider, + } } func (e *export) Init(a *app.App) (err error) { @@ -102,14 +106,14 @@ func (e *export) Export(req pb.RpcObjectListExportRequest) (path string, succeed if req.Format == pb.RpcObjectListExport_SVG { format = dot.ExportFormatSVG } - mc := dot.NewMultiConverter(format) + mc := dot.NewMultiConverter(format, e.sbtProvider) mc.SetKnownDocs(docs) var werr error if succeed, werr = e.writeMultiDoc(mc, wr, docs, queue); werr != nil { log.Warnf("can't export docs: %v", werr) } } else if req.Format == pb.RpcObjectListExport_GRAPH_JSON { - mc := graphjson.NewMultiConverter() + mc := graphjson.NewMultiConverter(e.sbtProvider) mc.SetKnownDocs(docs) var werr error if succeed, werr = e.writeMultiDoc(mc, wr, docs, queue); werr != nil { @@ -180,12 +184,12 @@ func (e *export) docsForExport(reqIds []string, includeNested bool) (docs map[st } for _, link := range links { if _, exists := docs[link]; !exists { - sbt, err2 := smartblock.SmartBlockTypeFromID(link) + sbt, err2 := e.sbtProvider.Type(link) if err2 != nil { log.Errorf("failed to get smartblocktype of id %s", link) continue } - if sbt != smartblock.SmartBlockTypePage && sbt != smartblock.SmartBlockTypeSet { + if sbt != smartblock.SmartBlockTypePage && sbt != smartblock.SmartBlockTypeSet && sbt != smartblock.SmartBlockTypeCollection { continue } rec, _ := e.objectStore.QueryById(links) diff --git a/core/block/import/importer.go b/core/block/import/importer.go index 5a144b902..f851e7038 100644 --- a/core/block/import/importer.go +++ b/core/block/import/importer.go @@ -25,6 +25,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/filestore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" ) var log = logging.Logger("import") @@ -37,11 +38,16 @@ type Import struct { oc Creator objectIDGetter IDGetter tempDirProvider core.TempDirProvider + sbtProvider typeprovider.SmartBlockTypeProvider } -func New(tempDirProvider core.TempDirProvider) Importer { +func New( + tempDirProvider core.TempDirProvider, + sbtProvider typeprovider.SmartBlockTypeProvider, +) Importer { return &Import{ tempDirProvider: tempDirProvider, + sbtProvider: sbtProvider, converters: make(map[string]converter.Converter, 0), } } @@ -53,7 +59,7 @@ func (i *Import) Init(a *app.App) (err error) { converters := []converter.Converter{ markdown.New(i.tempDirProvider), notion.New(), - pbc.New(), + pbc.New(i.sbtProvider), web.NewConverter(), newinfra.New(), } diff --git a/core/block/import/pb/converter.go b/core/block/import/pb/converter.go index 433a7504c..4b5b2d96e 100644 --- a/core/block/import/pb/converter.go +++ b/core/block/import/pb/converter.go @@ -11,43 +11,50 @@ import ( "github.com/pkg/errors" "github.com/anytypeio/go-anytype-middleware/core/block/import/converter" + "github.com/anytypeio/go-anytype-middleware/core/block/process" "github.com/anytypeio/go-anytype-middleware/pb" "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/core" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" - "github.com/anytypeio/go-anytype-middleware/util/builtinobjects" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" ) const Name = "PB" type Pb struct { + sbtProvider typeprovider.SmartBlockTypeProvider } -func init() { - converter.RegisterFunc(New) +func New(sbtProvider typeprovider.SmartBlockTypeProvider) converter.Converter { + return &Pb{ + sbtProvider: sbtProvider, + } } -func New(s core.Service) converter.Converter { - return &Pb{} -} - -func (p *Pb) GetSnapshots(req *pb.RpcObjectImportRequest) *converter.Response { +func (p *Pb) GetSnapshots(req *pb.RpcObjectImportRequest, + progress *process.Progress) (*converter.Response, converter.ConvertError) { path, e := p.GetParams(req.Params) allErrors := converter.NewError() if e != nil { allErrors.Add(path, e) - return &converter.Response{Error: allErrors} + return nil, allErrors } pbFiles, err := p.readFile(path, req.Mode.String()) if err != nil && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return &converter.Response{Error: err} + allErrors.Merge(err) + return nil, allErrors } - allSnapshots := make([]*converter.Snapshot, 0) - allErrors.Merge(err) + + progress.SetProgressMessage("Start creating snapshots from files") + progress.SetTotal(int64(len(pbFiles) * 2)) + for name, file := range pbFiles { + if err := progress.TryStep(1); err != nil { + ce := converter.NewFromError(name, err) + return nil, ce + } + id := strings.TrimSuffix(file.Name, filepath.Ext(file.Name)) var ( snapshot *model.SmartBlockSnapshotBase @@ -59,28 +66,16 @@ func (p *Pb) GetSnapshots(req *pb.RpcObjectImportRequest) *converter.Response { if errGS != nil { allErrors.Add(file.Name, errGS) if req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return &converter.Response{Error: allErrors} + return nil, allErrors } else { continue } } - // TODO need to get rid from dependencies from old IDs and extract sb type - sbt, err := smartblock.SmartBlockTypeFromID(id) - if err != nil { - sbt, err = builtinobjects.SmartBlockTypeFromThreadID(id) - if err != nil { - allErrors.Add(path, e) - if req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return &converter.Response{Error: allErrors} - } else { - continue - } - } - } + sbt, err := p.sbtProvider.Type(id) if err != nil { allErrors.Add(path, e) if req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return &converter.Response{Error: allErrors} + return nil, allErrors } else { continue } @@ -96,10 +91,10 @@ func (p *Pb) GetSnapshots(req *pb.RpcObjectImportRequest) *converter.Response { } if allErrors.IsEmpty() { - return &converter.Response{Snapshots: allSnapshots, Error: nil} + return &converter.Response{Snapshots: allSnapshots}, nil } - return &converter.Response{Snapshots: allSnapshots, Error: allErrors} + return &converter.Response{Snapshots: allSnapshots}, allErrors } func (p *Pb) Name() string { @@ -111,8 +106,8 @@ func (p *Pb) GetImage() ([]byte, int64, int64, error) { } func (p *Pb) GetParams(params pb.IsRpcObjectImportRequestParams) (string, error) { - if p, ok := params.(*pb.RpcObjectImportRequestParamsOfNotionParams); ok { - return p.NotionParams.GetPath(), nil + if p, ok := params.(*pb.RpcObjectImportRequestParamsOfMarkdownParams); ok { + return p.MarkdownParams.GetPath(), nil } return "", errors.New("PB: GetParams wrong parameters format") } diff --git a/core/block/object/creator.go b/core/block/object/creator.go index f09b75a39..5218f2f75 100644 --- a/core/block/object/creator.go +++ b/core/block/object/creator.go @@ -25,6 +25,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" "github.com/anytypeio/go-anytype-middleware/pkg/lib/schema" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/internalflag" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" "github.com/anytypeio/go-anytype-middleware/util/uri" @@ -44,6 +45,7 @@ type Creator struct { bookmark bookmark.Service objectFactory *editor.ObjectFactory app *app.App + sbtProvider typeprovider.SmartBlockTypeProvider // TODO: remove it? anytype core.Service @@ -53,8 +55,10 @@ type CollectionService interface { CreateCollection(details *types.Struct, flags []*model.InternalFlag) (coresb.SmartBlockType, *types.Struct, *state.State, error) } -func NewCreator() *Creator { - return &Creator{} +func NewCreator(sbtProvider typeprovider.SmartBlockTypeProvider) *Creator { + return &Creator{ + sbtProvider: sbtProvider, + } } func (c *Creator) Init(a *app.App) (err error) { @@ -201,7 +205,7 @@ func (c *Creator) CreateSet(req *pb.RpcObjectCreateSetRequest) (setID string, ne if len(source) == 0 { source = []string{bundle.TypeKeyPage.URL()} } - if dvContent, dvSchema, err = dataview.DataviewBlockBySource(c.objectStore, source); err != nil { + if dvContent, dvSchema, err = dataview.DataviewBlockBySource(c.sbtProvider, c.objectStore, source); err != nil { return } diff --git a/core/block/restriction/service.go b/core/block/restriction/service.go index ff7ec1d47..8724a8cd9 100644 --- a/core/block/restriction/service.go +++ b/core/block/restriction/service.go @@ -2,53 +2,23 @@ package restriction import ( "errors" + "fmt" "github.com/anytypeio/any-sync/app" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/core" "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" ) const CName = "restriction" -func New() Service { - return new(service) -} - var ErrRestricted = errors.New("restricted") var log = logging.Logger("anytype-mw-restrictions") -type simpleObject struct { - id string - tp model.SmartBlockType -} - -func newSimpleObject(id string) (Object, error) { - tp, err := smartblock.SmartBlockTypeFromID(id) - if err != nil { - return nil, err - } - return &simpleObject{ - id: id, - tp: tp.ToProto(), - }, nil -} - -func (s *simpleObject) Id() string { - return s.id -} - -func (s *simpleObject) Type() model.SmartBlockType { - return s.tp -} - -type Object interface { - Id() string - Type() model.SmartBlockType -} - type Service interface { ObjectRestrictionsByObj(obj Object) (r ObjectRestrictions) RestrictionsByObj(obj Object) (r Restrictions) @@ -58,7 +28,14 @@ type Service interface { } type service struct { - anytype core.Service + anytype core.Service + sbtProvider typeprovider.SmartBlockTypeProvider +} + +func New(sbtProvider typeprovider.SmartBlockTypeProvider) Service { + return &service{ + sbtProvider: sbtProvider, + } } func (s *service) Init(a *app.App) (err error) { @@ -89,13 +66,42 @@ func (s *service) CheckRestrictions(id string, cr ...model.RestrictionsObjectRes } func (s *service) RestrictionsById(id string) (r Restrictions, err error) { - obj, err := newSimpleObject(id) + sbType, err := s.sbtProvider.Type(id) + if err != nil { + return Restrictions{}, fmt.Errorf("get smartblock type: %w", err) + } + obj, err := newSimpleObject(id, sbType) if err != nil { return Restrictions{}, err } return s.RestrictionsByObj(obj), nil } +type simpleObject struct { + id string + tp model.SmartBlockType +} + +func newSimpleObject(id string, sbType smartblock.SmartBlockType) (Object, error) { + return &simpleObject{ + id: id, + tp: sbType.ToProto(), + }, nil +} + +func (s *simpleObject) Id() string { + return s.id +} + +func (s *simpleObject) Type() model.SmartBlockType { + return s.tp +} + +type Object interface { + Id() string + Type() model.SmartBlockType +} + type Restrictions struct { Object ObjectRestrictions Dataview DataviewRestrictions diff --git a/core/block/service.go b/core/block/service.go index 29ae0940a..a9a4939d2 100644 --- a/core/block/service.go +++ b/core/block/service.go @@ -5,21 +5,19 @@ import ( "encoding/base64" "errors" "fmt" - "github.com/anytypeio/any-sync/accountservice" - "github.com/anytypeio/any-sync/commonspace/object/treegetter" - "github.com/anytypeio/go-anytype-middleware/space" - "github.com/hashicorp/go-multierror" "strings" "time" - "github.com/gogo/protobuf/proto" - "github.com/gogo/protobuf/types" - "github.com/ipfs/go-datastore/query" - + "github.com/anytypeio/any-sync/accountservice" "github.com/anytypeio/any-sync/app" "github.com/anytypeio/any-sync/app/ocache" + "github.com/anytypeio/any-sync/commonspace/object/treegetter" + "github.com/gogo/protobuf/proto" + "github.com/gogo/protobuf/types" + "github.com/hashicorp/go-multierror" + "github.com/ipfs/go-datastore/query" + bookmarksvc "github.com/anytypeio/go-anytype-middleware/core/block/bookmark" - "github.com/anytypeio/go-anytype-middleware/core/block/doc" "github.com/anytypeio/go-anytype-middleware/core/block/editor" "github.com/anytypeio/go-anytype-middleware/core/block/editor/basic" "github.com/anytypeio/go-anytype-middleware/core/block/editor/bookmark" @@ -27,7 +25,6 @@ import ( "github.com/anytypeio/go-anytype-middleware/core/block/editor/collection" "github.com/anytypeio/go-anytype-middleware/core/block/editor/dataview" "github.com/anytypeio/go-anytype-middleware/core/block/editor/file" - _import "github.com/anytypeio/go-anytype-middleware/core/block/editor/import" "github.com/anytypeio/go-anytype-middleware/core/block/editor/smartblock" "github.com/anytypeio/go-anytype-middleware/core/block/editor/state" "github.com/anytypeio/go-anytype-middleware/core/block/editor/stext" @@ -45,9 +42,12 @@ import ( coresb "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" "github.com/anytypeio/go-anytype-middleware/pkg/lib/database" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/filestore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/internalflag" "github.com/anytypeio/go-anytype-middleware/util/linkpreview" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" @@ -66,18 +66,12 @@ const ( var ( ErrBlockNotFound = errors.New("block not found") - ErrBlockAlreadyOpen = errors.New("block already open") ErrUnexpectedBlockType = errors.New("unexpected block type") ErrUnknownObjectType = fmt.Errorf("unknown object type") ) var log = logging.Logger("anytype-mw-service") -var ( - blockCacheTTL = time.Minute - blockCleanupTimeout = time.Second * 30 -) - var ( // quick fix for limiting file upload goroutines uploadFilesLimiter = make(chan struct{}, 8) @@ -93,12 +87,14 @@ type SmartblockOpener interface { Open(id string) (sb smartblock.SmartBlock, err error) } -func New() *Service { - return &Service{} +func New(sbtProvider typeprovider.SmartBlockTypeProvider) *Service { + return &Service{ + sbtProvider: sbtProvider, + } } type objectCreator interface { - CreateSmartBlockFromState(ctx context.Context, sbType coresb.SmartBlockType, details *types.Struct, relationIds []string, createState *state.State) (id string, newDetails *types.Struct, err error) + CreateSmartBlockFromState(ctx context.Context, sbType coresb.SmartBlockType, details *types.Struct, createState *state.State) (id string, newDetails *types.Struct, err error) InjectWorkspaceID(details *types.Struct, objectID string) CreateObject(req DetailsGetter, forcedType bundle.TypeKey) (id string, details *types.Struct, err error) @@ -121,7 +117,6 @@ type Service struct { closed bool linkPreview linkpreview.LinkPreview process process.Service - doc doc.Service app *app.App source source.Service objectStore objectstore.ObjectStore @@ -130,10 +125,15 @@ type Service struct { relationService relation.Service cache ocache.OCache - objectCreator objectCreator - objectFactory *editor.ObjectFactory - clientService space.Service - commonAccount accountservice.Service + objectCreator objectCreator + objectFactory *editor.ObjectFactory + clientService space.Service + commonAccount accountservice.Service + fileStore filestore.FileStore + tempDirProvider core.TempDirProvider + sbtProvider typeprovider.SmartBlockTypeProvider + + spaceDashboardID string } func (s *Service) Name() string { @@ -147,7 +147,6 @@ func (s *Service) Init(a *app.App) (err error) { s.process = a.MustComponent(process.CName).(process.Service) s.sendEvent = a.MustComponent(event.CName).(event.Sender).Send s.source = a.MustComponent(source.CName).(source.Service) - s.doc = a.MustComponent(doc.CName).(doc.Service) s.objectStore = a.MustComponent(objectstore.CName).(objectstore.ObjectStore) s.restriction = a.MustComponent(restriction.CName).(restriction.Service) s.bookmark = a.MustComponent("bookmark-importer").(bookmarksvc.Service) @@ -156,6 +155,7 @@ func (s *Service) Init(a *app.App) (err error) { s.clientService = a.MustComponent(space.CName).(space.Service) s.objectFactory = app.MustComponent[*editor.ObjectFactory](a) s.commonAccount = a.MustComponent(accountservice.CName).(accountservice.Service) + s.fileStore = app.MustComponent[filestore.FileStore](a) s.cache = s.createCache() s.app = a return @@ -165,6 +165,22 @@ func (s *Service) Run(ctx context.Context) (err error) { return } +func (s *Service) GetSpaceDashboardID(ctx context.Context) (string, error) { + if s.spaceDashboardID == "" { + obj, release, err := s.CreateTreeObject(ctx, coresb.SmartBlockTypePage, func(id string) *smartblock.InitContext { + return &smartblock.InitContext{ + Ctx: ctx, + } + }) + if err != nil { + return "", err + } + release() + s.spaceDashboardID = obj.Id() + } + return s.spaceDashboardID, nil +} + func (s *Service) Anytype() core.Service { return s.anytype } @@ -201,7 +217,9 @@ func (s *Service) OpenBlock( } afterShowTime := time.Now() // TODO: [MR] add files to status logic - if tp, err := coresb.SmartBlockTypeFromID(id); err == nil && tp == coresb.SmartBlockTypePage { + // TODO: IT WILL NOT WORK for SETS, COLLECTIONS, etc + sbType, err := s.sbtProvider.Type(id) + if err == nil && sbType == coresb.SmartBlockTypePage { s.status.Watch(id, func() []string { return nil }) @@ -210,7 +228,7 @@ func (s *Service) OpenBlock( return nil }, smartblock.HookOnClose) } - //if tid := ob.threadId; tid != thread.Undef && s.status != nil { + // if tid := ob.threadId; tid != thread.Undef && s.status != nil { // var ( // fList = func() []string { // ob.Lock() @@ -226,9 +244,8 @@ func (s *Service) OpenBlock( // return nil // }, smartblock.HookOnClose) // } - //} + // } afterHashesTime := time.Now() - tp, _ := coresb.SmartBlockTypeFromID(id) metrics.SharedClient.RecordEvent(metrics.OpenBlockEvent{ ObjectId: id, GetBlockMs: afterSmartBlockTime.Sub(startTime).Milliseconds(), @@ -236,7 +253,7 @@ func (s *Service) OpenBlock( ApplyMs: afterApplyTime.Sub(afterDataviewTime).Milliseconds(), ShowMs: afterShowTime.Sub(afterApplyTime).Milliseconds(), FileWatcherMs: afterHashesTime.Sub(afterShowTime).Milliseconds(), - SmartblockType: int(tp), + SmartblockType: int(sbType), }) return obj, nil } @@ -287,7 +304,7 @@ func (s *Service) CloseBlock(id string) error { b.ObjectClose() s := b.NewState() isDraft = internalflag.NewFromState(s).Has(model.InternalFlag_editorDeleteEmpty) - //workspaceId = pbtypes.GetString(s.LocalDetails(), bundle.RelationKeyWorkspaceId.String()) + // workspaceId = pbtypes.GetString(s.LocalDetails(), bundle.RelationKeyWorkspaceId.String()) return nil }) if err != nil { @@ -370,13 +387,13 @@ func (s *Service) AddSubObjectsToWorkspace( return } -func (s *Service) RemoveSubObjectsInWorkspace(objectIds []string, workspaceId string) (err error) { +func (s *Service) RemoveSubObjectsInWorkspace(objectIds []string, workspaceId string, orphansGC bool) (err error) { err = s.Do(workspaceId, func(b smartblock.SmartBlock) error { ws, ok := b.(*editor.Workspaces) if !ok { return fmt.Errorf("incorrect workspace id") } - err = ws.RemoveSubObjects(objectIds) + err = ws.RemoveSubObjects(objectIds, orphansGC) return err }) @@ -388,7 +405,7 @@ func (s *Service) SelectWorkspace(req *pb.RpcWorkspaceSelectRequest) error { } func (s *Service) GetCurrentWorkspace(req *pb.RpcWorkspaceGetCurrentRequest) (string, error) { - workspaceId, err := s.anytype.ObjectStore().GetCurrentWorkspaceId() + workspaceId, err := s.objectStore.GetCurrentWorkspaceId() if err != nil && strings.HasSuffix(err.Error(), "key not found") { return "", nil } @@ -401,14 +418,14 @@ func (s *Service) GetAllWorkspaces(req *pb.RpcWorkspaceGetAllRequest) ([]string, func (s *Service) SetIsHighlighted(req *pb.RpcWorkspaceSetIsHighlightedRequest) error { panic("is not implemented") - //workspaceId, _ := s.anytype.GetWorkspaceIdForObject(req.ObjectId) - //return s.Do(workspaceId, func(b smartblock.SmartBlock) error { + // workspaceId, _ := s.anytype.GetWorkspaceIdForObject(req.ObjectId) + // return s.Do(workspaceId, func(b smartblock.SmartBlock) error { // workspace, ok := b.(*editor.Workspaces) // if !ok { // return fmt.Errorf("incorrect object with workspace id") // } // return workspace.SetIsHighlighted(req.ObjectId, req.IsHighlighted) - //}) + // }) } func (s *Service) ObjectAddWithObjectId(req *pb.RpcObjectAddWithObjectIdRequest) error { @@ -427,51 +444,51 @@ func (s *Service) ObjectAddWithObjectId(req *pb.RpcObjectAddWithObjectIdRequest) } // TODO: [MR] check the meaning of method and what should be the result return fmt.Errorf("not implemented") - //return s.Do(s.Anytype().PredefinedBlocks().Account, func(b smartblock.SmartBlock) error { + // return s.Do(s.Anytype().PredefinedBlocks().Account, func(b smartblock.SmartBlock) error { // workspace, ok := b.(*editor.Workspaces) // if !ok { // return fmt.Errorf("incorrect object with workspace id") // } // // return workspace.AddObject(req.ObjectId, protoPayload.Key, protoPayload.Addrs) - //}) + // }) } func (s *Service) ObjectShareByLink(req *pb.RpcObjectShareByLinkRequest) (link string, err error) { return "", fmt.Errorf("not implemented") - //workspaceId, err := s.anytype.GetWorkspaceIdForObject(req.ObjectId) - //if err == core.ErrObjectDoesNotBelongToWorkspace { + // workspaceId, err := s.anytype.GetWorkspaceIdForObject(req.ObjectId) + // if err == core.ErrObjectDoesNotBelongToWorkspace { // workspaceId = s.Anytype().PredefinedBlocks().Account - //} - //var key string - //var addrs []string - //err = s.Do(workspaceId, func(b smartblock.SmartBlock) error { + // } + // var key string + // var addrs []string + // err = s.Do(workspaceId, func(b smartblock.SmartBlock) error { // workspace, ok := b.(*editor.Workspaces) // if !ok { // return fmt.Errorf("incorrect object with workspace id") // } // key, addrs, err = workspace.GetObjectKeyAddrs(req.ObjectId) // return err - //}) - //if err != nil { + // }) + // if err != nil { // return "", err - //} - //payload := &model.ThreadDeeplinkPayload{ + // } + // payload := &model.ThreadDeeplinkPayload{ // Key: key, // Addrs: addrs, - //} - //marshalledPayload, err := proto.Marshal(payload) - //if err != nil { + // } + // marshalledPayload, err := proto.Marshal(payload) + // if err != nil { // return "", fmt.Errorf("failed to marshal deeplink payload: %w", err) - //} - //encodedPayload := base64.RawStdEncoding.EncodeToString(marshalledPayload) + // } + // encodedPayload := base64.RawStdEncoding.EncodeToString(marshalledPayload) // - //params := url.Values{} - //params.Add("id", req.ObjectId) - //params.Add("payload", encodedPayload) - //encoded := params.Encode() + // params := url.Values{} + // params.Add("id", req.ObjectId) + // params.Add("payload", encodedPayload) + // encoded := params.Encode() // - //return fmt.Sprintf("%s%s", linkObjectShare, encoded), nil + // return fmt.Sprintf("%s%s", linkObjectShare, encoded), nil } // SetPagesIsArchived is deprecated @@ -580,6 +597,33 @@ func (s *Service) SetPageIsArchived(req pb.RpcObjectSetIsArchivedRequest) (err e return s.objectLinksCollectionModify(s.anytype.PredefinedBlocks().Archive, req.ContextId, req.IsArchived) } +func (s *Service) SetSource(ctx *session.Context, req pb.RpcObjectSetSourceRequest) (err error) { + return s.Do(req.ContextId, func(b smartblock.SmartBlock) error { + st := b.NewStateCtx(ctx) + st.SetDetailAndBundledRelation(bundle.RelationKeySetOf, pbtypes.StringList(req.Source)) + return b.Apply(st, smartblock.NoRestrictions) + }) +} + +func (s *Service) SetWorkspaceDashboardId(ctx *session.Context, workspaceId string, id string) (setId string, err error) { + s.Do(workspaceId, func(ws smartblock.SmartBlock) error { + if ws.Type() != model.SmartBlockType_Workspace { + return ErrUnexpectedBlockType + } + if err = ws.SetDetails(ctx, []*pb.RpcObjectSetDetailsDetail{ + { + Key: bundle.RelationKeySpaceDashboardId.String(), + Value: pbtypes.String(id), + }, + }, false); err != nil { + return err + } + s.spaceDashboardID = id + return nil + }) + return id, nil +} + func (s *Service) checkArchivedRestriction(isArchived bool, objectId string) error { if !isArchived { return nil @@ -685,16 +729,16 @@ func (s *Service) OnDelete(id string, workspaceRemove func() error) (err error) } for _, fileHash := range fileHashes { - inboundLinks, err := s.Anytype().ObjectStore().GetOutboundLinksById(fileHash) + inboundLinks, err := s.objectStore.GetOutboundLinksById(fileHash) if err != nil { log.Errorf("failed to get inbound links for file %s: %s", fileHash, err.Error()) continue } if len(inboundLinks) == 0 { - if err = s.Anytype().ObjectStore().DeleteObject(fileHash); err != nil { + if err = s.objectStore.DeleteObject(fileHash); err != nil { log.With("file", fileHash).Errorf("failed to delete file from objectstore: %s", err.Error()) } - if err = s.Anytype().FileStore().DeleteByHash(fileHash); err != nil { + if err = s.fileStore.DeleteByHash(fileHash); err != nil { log.With("file", fileHash).Errorf("failed to delete file from filestore: %s", err.Error()) } // space will be reclaimed on the next GC cycle @@ -702,12 +746,16 @@ func (s *Service) OnDelete(id string, workspaceRemove func() error) (err error) log.With("file", fileHash).Errorf("failed to offload file: %s", err.Error()) continue } - if err = s.Anytype().FileStore().DeleteFileKeys(fileHash); err != nil { + if err = s.fileStore.DeleteFileKeys(fileHash); err != nil { log.With("file", fileHash).Errorf("failed to delete file keys: %s", err.Error()) } } } + if err := s.objectStore.DeleteObject(id); err != nil { + return fmt.Errorf("delete object from local store: %w", err) + } + return } @@ -753,7 +801,7 @@ func (s *Service) RemoveListOption(ctx *session.Context, optIds []string, checkI }, }, } - f, err := database.NewFilters(q, nil, nil) + f, err := database.NewFilters(q, nil, s.objectStore, nil) if err != nil { return nil } @@ -774,20 +822,23 @@ func (s *Service) RemoveListOption(ctx *session.Context, optIds []string, checkI return nil } +// TODO: remove proxy func (s *Service) Process() process.Service { return s.process } +// TODO: remove proxy func (s *Service) ProcessAdd(p process.Process) (err error) { return s.process.Add(p) } +// TODO: remove proxy func (s *Service) ProcessCancel(id string) (err error) { return s.process.Cancel(id) } func (s *Service) Close(ctx context.Context) (err error) { - //return s.cache.Close() + // return s.cache.Close() return } @@ -910,21 +961,6 @@ func (s *Service) DoHistory(id string, apply func(b basic.IHistory) error) error return fmt.Errorf("undo operation not available for this block type: %T", sb) } -func (s *Service) DoImport(id string, apply func(b _import.Import) error) error { - sb, release, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "do_import"), id) - if err != nil { - return err - } - defer release() - if bb, ok := sb.(_import.Import); ok { - sb.Lock() - defer sb.Unlock() - return apply(bb) - } - - return fmt.Errorf("import operation not available for this block type: %T", sb) -} - func (s *Service) DoDataview(id string, apply func(b dataview.Dataview) error) error { sb, release, err := s.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "do_dataview"), id) if err != nil { @@ -996,6 +1032,34 @@ func DoState[t any](p Picker, id string, apply func(s *state.State, sb t) error, return DoStateCtx(p, nil, id, apply, flags...) } +// DoState2 picks two blocks and perform an action on them. The order of locks is always the same for two ids. +// It correctly handles the case when two ids are the same. +func DoState2[t1, t2 any](s *Service, firstID, secondID string, f func(*state.State, *state.State, t1, t2) error) error { + if firstID == secondID { + return DoState(s, firstID, func(st *state.State, b t1) error { + // Check that b satisfies t2 + b2, ok := any(b).(t2) + if !ok { + var dummy t2 + return fmt.Errorf("block %s is not of type %T", firstID, dummy) + } + return f(st, st, b, b2) + }) + } + if firstID < secondID { + return DoState(s, firstID, func(firstState *state.State, firstBlock t1) error { + return DoState(s, secondID, func(secondState *state.State, secondBlock t2) error { + return f(firstState, secondState, firstBlock, secondBlock) + }) + }) + } + return DoState(s, secondID, func(secondState *state.State, secondBlock t2) error { + return DoState(s, firstID, func(firstState *state.State, firstBlock t1) error { + return f(firstState, secondState, firstBlock, secondBlock) + }) + }) +} + func DoStateCtx[t any](p Picker, ctx *session.Context, id string, apply func(s *state.State, sb t) error, flags ...smartblock.ApplyFlag) error { sb, release, err := p.PickBlock(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "do"), id) if err != nil { @@ -1076,7 +1140,7 @@ func (s *Service) ResetToState(pageId string, state *state.State) (err error) { } func (s *Service) ObjectBookmarkFetch(req pb.RpcObjectBookmarkFetchRequest) (err error) { - url, err := uri.ProcessURI(req.Url) + url, err := uri.NormalizeURI(req.Url) if err != nil { return fmt.Errorf("process uri: %w", err) } diff --git a/core/block/source/service.go b/core/block/source/service.go index fa23025b2..a6bb67615 100644 --- a/core/block/source/service.go +++ b/core/block/source/service.go @@ -2,19 +2,22 @@ package source import ( "fmt" - "github.com/anytypeio/any-sync/accountservice" - "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" - "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "sync" + "github.com/anytypeio/any-sync/accountservice" "github.com/anytypeio/any-sync/app" + "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/gogo/protobuf/types" + "github.com/anytypeio/go-anytype-middleware/core/block/editor/state" "github.com/anytypeio/go-anytype-middleware/core/status" "github.com/anytypeio/go-anytype-middleware/pkg/lib/core" "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/filestore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" - "github.com/gogo/protobuf/types" + "github.com/anytypeio/go-anytype-middleware/space" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" ) const CName = "source" @@ -25,7 +28,7 @@ func New() Service { type Service interface { NewSource(id string, ot objecttree.ObjectTree) (s Source, err error) - RegisterStaticSource(id string, new func() Source) + RegisterStaticSource(id string, s Source) NewStaticSource(id string, sbType model.SmartBlockType, doc *state.State, pushChange func(p PushChangeParams) (string, error)) SourceWithType RemoveStaticSource(id string) @@ -37,19 +40,22 @@ type Service interface { type service struct { anytype core.Service statusService status.Service - typeProvider typeprovider.ObjectTypeProvider + sbtProvider typeprovider.SmartBlockTypeProvider account accountservice.Service - - staticIds map[string]func() Source - mu sync.Mutex + fileStore filestore.FileStore + spaceService space.Service + staticIds map[string]Source + mu sync.Mutex } func (s *service) Init(a *app.App) (err error) { - s.staticIds = make(map[string]func() Source) + s.staticIds = make(map[string]Source) s.anytype = a.MustComponent(core.CName).(core.Service) s.statusService = a.MustComponent(status.CName).(status.Service) - s.typeProvider = a.MustComponent(typeprovider.CName).(typeprovider.ObjectTypeProvider) + s.sbtProvider = a.MustComponent(typeprovider.CName).(typeprovider.SmartBlockTypeProvider) s.account = a.MustComponent(accountservice.CName).(accountservice.Service) + s.fileStore = app.MustComponent[filestore.FileStore](a) + s.spaceService = app.MustComponent[space.Service](a) return } @@ -61,10 +67,10 @@ func (s *service) NewSource(id string, ot objecttree.ObjectTree) (source Source, if id == addr.AnytypeProfileId { return NewAnytypeProfile(s.anytype, id), nil } - st, err := smartblock.SmartBlockTypeFromID(id) + st, err := s.sbtProvider.Type(id) switch st { case smartblock.SmartBlockTypeFile: - return NewFiles(s.anytype, id), nil + return NewFiles(s.anytype, s.fileStore, id), nil case smartblock.SmartBlockTypeDate: return NewDate(s.anytype, id), nil case smartblock.SmartBlockTypeBundledObjectType: @@ -79,8 +85,8 @@ func (s *service) NewSource(id string, ot objecttree.ObjectTree) (source Source, s.mu.Lock() defer s.mu.Unlock() - if newStatic := s.staticIds[id]; newStatic != nil { - return newStatic(), nil + if src := s.staticIds[id]; src != nil { + return src, nil } if ot == nil { @@ -89,7 +95,7 @@ func (s *service) NewSource(id string, ot objecttree.ObjectTree) (source Source, } // TODO: [MR] get this from objectTree directly - sbt, err := s.typeProvider.Type(id) + sbt, err := s.sbtProvider.Type(id) if err != nil { return nil, err } @@ -99,6 +105,8 @@ func (s *service) NewSource(id string, ot objecttree.ObjectTree) (source Source, accountService: s.account, sbt: sbt, ot: ot, + spaceService: s.spaceService, + sbtProvider: s.sbtProvider, } return newTreeSource(id, deps) } @@ -116,10 +124,10 @@ func (s *service) GetDetailsFromIdBasedSource(id string) (*types.Struct, error) return nil, fmt.Errorf("id unsupported") } -func (s *service) RegisterStaticSource(id string, new func() Source) { +func (s *service) RegisterStaticSource(id string, src Source) { s.mu.Lock() defer s.mu.Unlock() - s.staticIds[id] = new + s.staticIds[id] = src } func (s *service) RemoveStaticSource(id string) { diff --git a/core/block/source/source.go b/core/block/source/source.go index b6c82960a..ea6bbd1e5 100644 --- a/core/block/source/source.go +++ b/core/block/source/source.go @@ -4,17 +4,17 @@ import ( "context" "errors" "fmt" - "github.com/anytypeio/any-sync/accountservice" - "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" - "github.com/anytypeio/any-sync/commonspace/object/tree/synctree" - "github.com/gogo/protobuf/proto" - "go.uber.org/zap" "math/rand" "sync" "time" + "github.com/anytypeio/any-sync/accountservice" + "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/anytypeio/any-sync/commonspace/object/tree/synctree" + "github.com/gogo/protobuf/proto" "github.com/gogo/protobuf/types" "github.com/textileio/go-threads/core/thread" + "go.uber.org/zap" "github.com/anytypeio/go-anytype-middleware/core/block/editor/state" "github.com/anytypeio/go-anytype-middleware/core/status" @@ -23,6 +23,8 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/slice" ) @@ -77,7 +79,7 @@ func (s *service) SourceTypeBySbType(blockType smartblock.SmartBlockType) (Sourc case smartblock.SmartBlockTypeAnytypeProfile: return &anytypeProfile{a: s.anytype}, nil case smartblock.SmartBlockTypeFile: - return &files{a: s.anytype}, nil + return &files{a: s.anytype, fileStore: s.fileStore}, nil case smartblock.SmartBlockTypeBundledObjectType: return &bundledObjectType{a: s.anytype}, nil case smartblock.SmartBlockTypeBundledRelation: @@ -88,7 +90,12 @@ func (s *service) SourceTypeBySbType(blockType smartblock.SmartBlockType) (Sourc if err := blockType.Valid(); err != nil { return nil, err } else { - return &source{a: s.anytype, smartblockType: blockType}, nil + return &source{ + a: s.anytype, + spaceService: s.spaceService, + smartblockType: blockType, + sbtProvider: s.sbtProvider, + }, nil } } } @@ -99,6 +106,8 @@ type sourceDeps struct { accountService accountservice.Service sbt smartblock.SmartBlockType ot objecttree.ObjectTree + spaceService space.Service + sbtProvider typeprovider.SmartBlockTypeProvider } func newTreeSource(id string, deps sourceDeps) (s Source, err error) { @@ -106,11 +115,13 @@ func newTreeSource(id string, deps sourceDeps) (s Source, err error) { ObjectTree: deps.ot, id: id, a: deps.anytype, + spaceService: deps.spaceService, ss: deps.statusService, logId: deps.anytype.Device(), openedAt: time.Now(), smartblockType: deps.sbt, acc: deps.accountService, + sbtProvider: deps.sbtProvider, }, nil } @@ -120,18 +131,22 @@ type ObjectTreeProvider interface { type source struct { objecttree.ObjectTree - id, logId string - tid thread.ID - smartblockType smartblock.SmartBlockType - a core.Service - ss status.Service - lastSnapshotId string - receiver ChangeReceiver - unsubscribe func() - metaOnly bool - closed chan struct{} - openedAt time.Time - acc accountservice.Service + id, logId string + tid thread.ID + smartblockType smartblock.SmartBlockType + lastSnapshotId string + changesSinceSnapshot int + receiver ChangeReceiver + unsubscribe func() + metaOnly bool + closed chan struct{} + openedAt time.Time + + a core.Service + ss status.Service + acc accountservice.Service + spaceService space.Service + sbtProvider typeprovider.SmartBlockTypeProvider } func (s *source) Tree() objecttree.ObjectTree { @@ -141,9 +156,17 @@ func (s *source) Tree() objecttree.ObjectTree { func (s *source) Update(ot objecttree.ObjectTree) { // here it should work, because we always have the most common snapshot of the changes in tree s.lastSnapshotId = ot.Root().Id + prevSnapshot := s.lastSnapshotId err := s.receiver.StateAppend(func(d state.Doc) (st *state.State, changes []*pb.ChangeContent, err error) { - return BuildState(d.(*state.State), ot, s.Anytype().PredefinedBlocks().Profile) + st, changes, sinceSnapshot, err := BuildState(d.(*state.State), ot, s.Anytype().PredefinedBlocks().Profile) + if prevSnapshot != s.lastSnapshotId { + s.changesSinceSnapshot = sinceSnapshot + } else { + s.changesSinceSnapshot += sinceSnapshot + } + return st, changes, err }) + if err != nil { log.With(zap.Error(err)).Debug("failed to append the state and send it to receiver") } @@ -201,7 +224,7 @@ func (s *source) readDoc(ctx context.Context, receiver ChangeReceiver, allowEmpt } func (s *source) buildState() (doc state.Doc, err error) { - st, _, err := BuildState(nil, s.ObjectTree, s.Anytype().PredefinedBlocks().Profile) + st, _, changesAppliedSinceSnapshot, err := BuildState(nil, s.ObjectTree, s.Anytype().PredefinedBlocks().Profile) if err != nil { return } @@ -211,7 +234,7 @@ func (s *source) buildState() (doc state.Doc, err error) { } st.BlocksInit(st) st.InjectDerivedDetails() - + s.changesSinceSnapshot = changesAppliedSinceSnapshot // TODO: check if we can leave only removeDuplicates instead of Normalize if err = st.Normalize(false); err != nil { return @@ -277,15 +300,17 @@ func (s *source) PushChange(params PushChangeParams) (id string, err error) { if c.Snapshot != nil { s.lastSnapshotId = id + s.changesSinceSnapshot = 0 log.Infof("%s: pushed snapshot", s.id) } else { + s.changesSinceSnapshot++ log.Debugf("%s: pushed %d changes", s.id, len(c.Content)) } return } func (s *source) ListIds() (ids []string, err error) { - spc, err := s.a.SpaceService().AccountSpace(context.Background()) + spc, err := s.spaceService.AccountSpace(context.Background()) if err != nil { return } @@ -293,7 +318,7 @@ func (s *source) ListIds() (ids []string, err error) { if s.Anytype().PredefinedBlocks().IsAccount(id) { return false } - t, err := smartblock.SmartBlockTypeFromID(id) + t, err := s.sbtProvider.Type(id) if err != nil { return false } @@ -303,11 +328,30 @@ func (s *source) ListIds() (ids []string, err error) { return ids, nil } +func snapshotChance(changesSinceSnapshot int) bool { + v := 2000 + if changesSinceSnapshot <= 100 { + return false + } + + d := changesSinceSnapshot/50 + 1 + + min := (v / 2) - d + max := (v / 2) + d + + r := rand.Intn(v) + if r >= min && r <= max { + return true + } + + return false +} + func (s *source) needSnapshot() bool { if s.ObjectTree.Heads()[0] == s.ObjectTree.Id() { return true } - return rand.Intn(500) == 42 + return snapshotChance(s.changesSinceSnapshot) } func (s *source) GetFileKeysSnapshot() []*pb.ChangeFileKeys { @@ -391,7 +435,7 @@ func (s *source) Close() (err error) { return s.ObjectTree.Close() } -func BuildState(initState *state.State, ot objecttree.ReadableObjectTree, profileId string) (st *state.State, appliedContent []*pb.ChangeContent, err error) { +func BuildState(initState *state.State, ot objecttree.ReadableObjectTree, profileId string) (st *state.State, appliedContent []*pb.ChangeContent, changesAppliedSinceSnapshot int, err error) { var ( startId string lastChange *objecttree.Change @@ -426,6 +470,7 @@ func BuildState(initState *state.State, ot objecttree.ReadableObjectTree, profil model := change.Model.(*pb.Change) if startId == change.Id { if st == nil { + changesAppliedSinceSnapshot = 0 st = state.NewDocFromSnapshot(ot.Id(), model.Snapshot).(*state.State) st.SetChangeId(startId) return true @@ -434,6 +479,11 @@ func BuildState(initState *state.State, ot objecttree.ReadableObjectTree, profil } return true } + if model.Snapshot != nil { + changesAppliedSinceSnapshot = 0 + } else { + changesAppliedSinceSnapshot++ + } ns := st.NewState() appliedContent = append(appliedContent, model.Content...) ns.ApplyChangeIgnoreErr(model.Content...) diff --git a/core/block/source/static.go b/core/block/source/static.go index 7ff41ad84..00dfd7bcd 100644 --- a/core/block/source/static.go +++ b/core/block/source/static.go @@ -6,7 +6,6 @@ import ( "github.com/anytypeio/go-anytype-middleware/core/block/editor/state" "github.com/anytypeio/go-anytype-middleware/pb" "github.com/anytypeio/go-anytype-middleware/pkg/lib/core" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" ) @@ -68,8 +67,8 @@ func (s *static) PushChange(params PushChangeParams) (id string, err error) { func (s *static) ListIds() (result []string, err error) { s.s.mu.Lock() defer s.s.mu.Unlock() - for id := range s.s.staticIds { - if sbt, _ := smartblock.SmartBlockTypeFromID(id); sbt.ToProto() == s.Type() { + for id, src := range s.s.staticIds { + if src.Type() == s.Type() { result = append(result, id) } } diff --git a/core/converter/dot/dot.go b/core/converter/dot/dot.go index 371639c5c..3175decd9 100644 --- a/core/converter/dot/dot.go +++ b/core/converter/dot/dot.go @@ -6,28 +6,36 @@ package dot import ( "bytes" "fmt" - "github.com/gogo/protobuf/types" "io/ioutil" "github.com/goccy/go-graphviz" "github.com/goccy/go-graphviz/cgraph" + "github.com/gogo/protobuf/types" "github.com/anytypeio/go-anytype-middleware/core/block/editor/state" "github.com/anytypeio/go-anytype-middleware/core/converter" "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" ) -func NewMultiConverter(format graphviz.Format) converter.MultiConverter { +func NewMultiConverter(format graphviz.Format, sbtProvider typeprovider.SmartBlockTypeProvider) converter.MultiConverter { g := graphviz.New() graph, err := g.Graph() if err != nil { return nil } - return &dot{graph: graph, graphviz: g, exportFormat: format, linksByNode: map[string][]linkInfo{}, nodes: map[string]*cgraph.Node{}} + return &dot{ + graph: graph, + graphviz: g, + exportFormat: format, + linksByNode: map[string][]linkInfo{}, + nodes: map[string]*cgraph.Node{}, + sbtProvider: sbtProvider, + } } const ( @@ -58,6 +66,7 @@ type dot struct { exportFormat graphviz.Format nodes map[string]*cgraph.Node linksByNode map[string][]linkInfo + sbtProvider typeprovider.SmartBlockTypeProvider } func (d *dot) SetKnownDocs(docs map[string]*types.Struct) converter.Converter { @@ -84,7 +93,7 @@ func (d *dot) Add(st *state.State) error { image := pbtypes.GetString(st.Details(), bundle.RelationKeyIconImage.String()) if image != "" { n.Set("iconImage", image) - //n.SetImage(image+".jpg") + // n.SetImage(image+".jpg") } iconEmoji := pbtypes.GetString(st.Details(), bundle.RelationKeyIconEmoji.String()) @@ -113,7 +122,7 @@ func (d *dot) Add(st *state.State) error { objIds := pbtypes.GetStringList(st.Details(), rel.Key) for _, objId := range objIds { - t, err := smartblock.SmartBlockTypeFromID(objId) + t, err := d.sbtProvider.Type(objId) if err != nil { continue } @@ -135,7 +144,7 @@ func (d *dot) Add(st *state.State) error { } for _, depID := range st.DepSmartIds(true, true, false, false, false) { - t, err := smartblock.SmartBlockTypeFromID(depID) + t, err := d.sbtProvider.Type(depID) if err != nil { continue } diff --git a/core/converter/graphjson/graphjson.go b/core/converter/graphjson/graphjson.go index ac3c64be1..87c639e7e 100644 --- a/core/converter/graphjson/graphjson.go +++ b/core/converter/graphjson/graphjson.go @@ -2,17 +2,24 @@ package graphjson import ( "encoding/json" + + "github.com/gogo/protobuf/types" + "github.com/anytypeio/go-anytype-middleware/core/block/editor/state" "github.com/anytypeio/go-anytype-middleware/core/converter" "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" - "github.com/gogo/protobuf/types" ) -func NewMultiConverter() converter.MultiConverter { - return &graphjson{linksByNode: map[string][]*Edge{}, nodes: map[string]*Node{}} +func NewMultiConverter(sbtProvider typeprovider.SmartBlockTypeProvider) converter.MultiConverter { + return &graphjson{ + linksByNode: map[string][]*Edge{}, + nodes: map[string]*Node{}, + sbtProvider: sbtProvider, + } } type edgeType int @@ -50,6 +57,7 @@ type graphjson struct { imageHashes []string nodes map[string]*Node linksByNode map[string][]*Edge + sbtProvider typeprovider.SmartBlockTypeProvider } func (g *graphjson) SetKnownDocs(docs map[string]*types.Struct) converter.Converter { @@ -89,7 +97,7 @@ func (g *graphjson) Add(st *state.State) error { objIds := pbtypes.GetStringList(st.Details(), rel.Key) for _, objId := range objIds { - t, err := smartblock.SmartBlockTypeFromID(objId) + t, err := g.sbtProvider.Type(objId) if err != nil { continue } @@ -111,7 +119,7 @@ func (g *graphjson) Add(st *state.State) error { } for _, depID := range st.DepSmartIds(true, true, false, false, false) { - t, err := smartblock.SmartBlockTypeFromID(depID) + t, err := g.sbtProvider.Type(depID) if err != nil { continue } diff --git a/core/history/history.go b/core/history/history.go index 6e43f4133..c0da80080 100644 --- a/core/history/history.go +++ b/core/history/history.go @@ -2,20 +2,17 @@ package history import ( "context" - "fmt" - "github.com/anytypeio/any-sync/commonspace" - "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" - "github.com/anytypeio/go-anytype-middleware/core/block/editor/template" - "github.com/anytypeio/go-anytype-middleware/core/block/source" - "github.com/anytypeio/go-anytype-middleware/space" - "github.com/gogo/protobuf/proto" "time" - "github.com/anytypeio/go-anytype-middleware/core/relation" - "github.com/anytypeio/any-sync/app" + "github.com/anytypeio/any-sync/commonspace" + "github.com/anytypeio/any-sync/commonspace/object/tree/objecttree" + "github.com/gogo/protobuf/proto" + "github.com/anytypeio/go-anytype-middleware/core/block" "github.com/anytypeio/go-anytype-middleware/core/block/editor/state" + "github.com/anytypeio/go-anytype-middleware/core/block/source" + "github.com/anytypeio/go-anytype-middleware/core/relation" "github.com/anytypeio/go-anytype-middleware/pb" "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" "github.com/anytypeio/go-anytype-middleware/pkg/lib/core" @@ -24,6 +21,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" "github.com/anytypeio/go-anytype-middleware/util/slice" ) @@ -71,17 +69,14 @@ func (h *history) Name() (name string) { } func (h *history) Show(pageId, versionId string) (bs *model.ObjectView, ver *pb.RpcHistoryVersion, err error) { - s, ver, err := h.buildState(pageId, versionId) + s, sbType, ver, err := h.buildState(pageId, versionId) if err != nil { return } metaD, _ := h.objectStore.QueryById(s.DepSmartIds(true, true, false, true, false)) details := make([]*model.ObjectViewDetailsSet, 0, len(metaD)) var uniqueObjTypes []string - sbType, err := smartblock.SmartBlockTypeFromID(pageId) - if err != nil { - return nil, nil, fmt.Errorf("incorrect sb type: %w", err) - } + metaD = append(metaD, database.Record{Details: s.CombinedDetails()}) uniqueObjTypes = s.ObjectTypes() for _, m := range metaD { @@ -186,7 +181,7 @@ func (h *history) Versions(pageId, lastVersionId string, limit int) (resp []*pb. } func (h *history) SetVersion(pageId, versionId string) (err error) { - s, _, err := h.buildState(pageId, versionId) + s, _, _, err := h.buildState(pageId, versionId) if err != nil { return } @@ -214,21 +209,17 @@ func (h *history) treeWithId(id, beforeId string, includeBeforeId bool) (ht obje return } -func (h *history) buildState(pageId, versionId string) (st *state.State, ver *pb.RpcHistoryVersion, err error) { +func (h *history) buildState(pageId, versionId string) (st *state.State, sbType smartblock.SmartBlockType, ver *pb.RpcHistoryVersion, err error) { tree, sbType, err := h.treeWithId(pageId, versionId, true) if err != nil { return } - st, _, err = source.BuildState(nil, tree, h.a.PredefinedBlocks().Profile) + st, _, _, err = source.BuildState(nil, tree, h.a.PredefinedBlocks().Profile) if _, _, err = state.ApplyStateFast(st); err != nil { return } - switch sbType { - case smartblock.SmartBlockTypePage, smartblock.SmartBlockTypeProfilePage, smartblock.SmartBlockTypeSet: - // todo: set case not handled - template.InitTemplate(st, template.WithTitle) - } + st.BlocksInit(st) if ch, e := tree.GetChange(versionId); e == nil { profileId, profileName, e := h.getProfileInfo() diff --git a/core/indexer/indexer.go b/core/indexer/indexer.go index 7042f7901..76406873c 100644 --- a/core/indexer/indexer.go +++ b/core/indexer/indexer.go @@ -3,35 +3,37 @@ package indexer import ( "context" "crypto/sha256" + "errors" "fmt" - "github.com/anytypeio/go-anytype-middleware/space/typeprovider" - "golang.org/x/exp/slices" - "math" "strings" "sync" "time" + "github.com/anytypeio/any-sync/app" "github.com/gogo/protobuf/types" ds "github.com/ipfs/go-datastore" "github.com/textileio/go-threads/core/thread" + "golang.org/x/exp/slices" - "github.com/anytypeio/any-sync/app" "github.com/anytypeio/go-anytype-middleware/core/anytype/config" - "github.com/anytypeio/go-anytype-middleware/core/block/doc" - "github.com/anytypeio/go-anytype-middleware/core/block/editor/state" - "github.com/anytypeio/go-anytype-middleware/core/block/editor/template" + "github.com/anytypeio/go-anytype-middleware/core/block" + "github.com/anytypeio/go-anytype-middleware/core/block/editor" + smartblock2 "github.com/anytypeio/go-anytype-middleware/core/block/editor/smartblock" "github.com/anytypeio/go-anytype-middleware/core/block/source" - "github.com/anytypeio/go-anytype-middleware/core/relation" + "github.com/anytypeio/go-anytype-middleware/core/relation/relationutils" "github.com/anytypeio/go-anytype-middleware/metrics" "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" "github.com/anytypeio/go-anytype-middleware/pkg/lib/core" "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" "github.com/anytypeio/go-anytype-middleware/pkg/lib/database" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/filestore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/ftsearch" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" "github.com/anytypeio/go-anytype-middleware/util/slice" ) @@ -49,7 +51,7 @@ const ( ForceBundledObjectsReindexCounter int32 = 5 // reindex objects like anytypeProfile // ForceIdxRebuildCounter erases localstore indexes and reindex all type of objects // (no need to increase ForceThreadsObjectsReindexCounter & ForceFilesReindexCounter) - ForceIdxRebuildCounter int32 = 35 + ForceIdxRebuildCounter int32 = 38 // ForceFulltextIndexCounter performs fulltext indexing for all type of objects (useful when we change fulltext config) ForceFulltextIndexCounter int32 = 4 // ForceFilestoreKeysReindexCounter reindex filestore keys in all objects @@ -80,31 +82,19 @@ type Hasher interface { Hash() string } -type reindexFlags uint64 - -const cacheTimeout = 4 * time.Second - -const ( - reindexBundledTypes reindexFlags = 1 << iota - removeAllIndexedObjects - reindexBundledRelations - eraseIndexes - reindexThreadObjects - reindexFileObjects - reindexFulltext - reindexBundledTemplates - reindexBundledObjects - reindexFileKeys -) +type subObjectCreator interface { + CreateSubObjectsInWorkspace(details []*types.Struct) (ids []string, objects []*types.Struct, err error) +} type indexer struct { - store objectstore.ObjectStore - // todo: move logstore to separate component? - anytype core.Service - source source.Service - relationService relation.Service + store objectstore.ObjectStore + fileStore filestore.FileStore + anytype core.Service + source source.Service + picker block.Picker + ftsearch ftsearch.FTSearch + subObjectCreator subObjectCreator - doc doc.Service quit chan struct{} mu sync.Mutex btHash Hasher @@ -113,25 +103,26 @@ type indexer struct { newAccount bool forceFt chan struct{} - relationBulkMigration relation.BulkMigration - relationMigratorMu sync.Mutex - typeProvider typeprovider.ObjectTypeProvider + typeProvider typeprovider.SmartBlockTypeProvider + spaceService space.Service } func (i *indexer) Init(a *app.App) (err error) { i.newAccount = a.MustComponent(config.CName).(*config.Config).NewAccount i.anytype = a.MustComponent(core.CName).(core.Service) i.store = a.MustComponent(objectstore.CName).(objectstore.ObjectStore) - i.relationService = a.MustComponent(relation.CName).(relation.Service) - i.typeProvider = a.MustComponent(typeprovider.CName).(typeprovider.ObjectTypeProvider) + i.typeProvider = a.MustComponent(typeprovider.CName).(typeprovider.SmartBlockTypeProvider) i.source = a.MustComponent(source.CName).(source.Service) i.btHash = a.MustComponent("builtintemplate").(Hasher) - i.doc = a.MustComponent(doc.CName).(doc.Service) + i.picker = app.MustComponent[block.Picker](a) + i.fileStore = app.MustComponent[filestore.FileStore](a) + i.spaceService = app.MustComponent[space.Service](a) + i.ftsearch = app.MustComponent[ftsearch.FTSearch](a) + i.subObjectCreator = app.MustComponent[subObjectCreator](a) i.quit = make(chan struct{}) i.archivedMap = make(map[string]struct{}, 100) i.favoriteMap = make(map[string]struct{}, 100) i.forceFt = make(chan struct{}) - i.doc.OnWholeChange(i.index) return } @@ -139,6 +130,436 @@ func (i *indexer) Name() (name string) { return CName } +func (i *indexer) Run(context.Context) (err error) { + if ftErr := i.ftInit(); ftErr != nil { + log.Errorf("can't init ft: %v", ftErr) + } + err = i.reindexIfNeeded() + if err != nil { + return err + } + i.migrateRemoveNonindexableObjects() + go i.ftLoop() + return +} + +func (i *indexer) migrateRemoveNonindexableObjects() { + ids, err := i.getIdsForTypes( + smartblock.SmartblockTypeMarketplaceType, smartblock.SmartblockTypeMarketplaceRelation, + smartblock.SmartblockTypeMarketplaceTemplate, smartblock.SmartBlockTypeDate, smartblock.SmartBlockTypeBreadcrumbs, + ) + if err != nil { + log.Errorf("migrateRemoveNonindexableObjects: failed to get ids: %s", err.Error()) + } + + for _, id := range ids { + err = i.store.DeleteDetails(id) + if err != nil { + log.Errorf("migrateRemoveNonindexableObjects: failed to get ids: %s", err.Error()) + } + } +} + +func (i *indexer) Close(ctx context.Context) (err error) { + i.mu.Lock() + quit := i.quit + i.mu.Unlock() + if quit != nil { + close(quit) + i.mu.Lock() + i.quit = nil + i.mu.Unlock() + } + return nil +} + +func (i *indexer) Index(ctx context.Context, info smartblock2.DocInfo) error { + startTime := time.Now() + sbType, err := i.typeProvider.Type(info.Id) + if err != nil { + sbType = smartblock.SmartBlockTypePage + } + saveIndexedHash := func() { + if headsHash := headsHash(info.Heads); headsHash != "" { + err = i.store.SaveLastIndexedHeadsHash(info.Id, headsHash) + if err != nil { + log.With("thread", info.Id).Errorf("failed to save indexed heads hash: %v", err) + } + } + } + + indexDetails, indexLinks := sbType.Indexable() + if !indexDetails && !indexLinks { + saveIndexedHash() + return nil + } + + details := info.State.CombinedDetails() + details.Fields[bundle.RelationKeyLinks.String()] = pbtypes.StringList(info.Links) + setCreator := pbtypes.GetString(info.State.LocalDetails(), bundle.RelationKeyCreator.String()) + if setCreator == "" { + setCreator = i.anytype.ProfileID() + } + indexSetTime := time.Now() + var hasError bool + if indexLinks { + if err = i.store.UpdateObjectLinks(info.Id, info.Links); err != nil { + hasError = true + log.With("thread", info.Id).Errorf("failed to save object links: %v", err) + } + } + + indexLinksTime := time.Now() + if indexDetails { + if err := i.store.UpdateObjectDetails(info.Id, details, false); err != nil { + hasError = true + log.With("thread", info.Id).Errorf("can't update object store: %v", err) + } + if err := i.store.AddToIndexQueue(info.Id); err != nil { + log.With("thread", info.Id).Errorf("can't add id to index queue: %v", err) + } else { + log.With("thread", info.Id).Debugf("to index queue") + } + + go i.indexLinkedFiles(ctx, info.FileHashes) + } else { + _ = i.store.DeleteDetails(info.Id) + } + indexDetailsTime := time.Now() + detailsCount := 0 + if details.GetFields() != nil { + detailsCount = len(details.GetFields()) + } + + if !hasError { + saveIndexedHash() + } + + metrics.SharedClient.RecordEvent(metrics.IndexEvent{ + ObjectId: info.Id, + IndexLinksTimeMs: indexLinksTime.Sub(indexSetTime).Milliseconds(), + IndexDetailsTimeMs: indexDetailsTime.Sub(indexLinksTime).Milliseconds(), + IndexSetRelationsTimeMs: indexSetTime.Sub(startTime).Milliseconds(), + RelationsCount: len(info.State.PickRelationLinks()), + DetailsCount: detailsCount, + }) + + return nil +} + +func (i *indexer) indexLinkedFiles(ctx context.Context, fileHashes []string) { + if len(fileHashes) == 0 { + return + } + existingIDs, err := i.store.HasIDs(fileHashes...) + if err != nil { + log.Errorf("failed to get existing file ids : %s", err.Error()) + } + newIDs := slice.Difference(fileHashes, existingIDs) + for _, id := range newIDs { + // file's hash is id + err = i.reindexDoc(ctx, id) + if err != nil { + log.With("id", id).Errorf("failed to reindex file: %s", err.Error()) + } + + err = i.store.AddToIndexQueue(id) + if err != nil { + log.With("id", id).Error(err.Error()) + } + } +} + +func (i *indexer) reindexIfNeeded() error { + checksums, err := i.store.GetChecksums() + if err != nil && err != ds.ErrNotFound { + return err + } + if checksums == nil { + checksums = &model.ObjectStoreChecksums{ + // do no add bundled relations checksums, because we want to index them for new accounts + ObjectsForceReindexCounter: ForceThreadsObjectsReindexCounter, + FilesForceReindexCounter: ForceFilesReindexCounter, + IdxRebuildCounter: ForceIdxRebuildCounter, + FilestoreKeysForceReindexCounter: ForceFilestoreKeysReindexCounter, + } + } + + var flags reindexFlags + if checksums.BundledRelations != bundle.RelationChecksum { + flags.bundledRelations = true + } + if checksums.BundledObjectTypes != bundle.TypeChecksum { + flags.bundledTypes = true + } + if checksums.ObjectsForceReindexCounter != ForceThreadsObjectsReindexCounter { + flags.threadObjects = true + } + if checksums.FilestoreKeysForceReindexCounter != ForceFilestoreKeysReindexCounter { + flags.fileKeys = true + } + if checksums.FilesForceReindexCounter != ForceFilesReindexCounter { + flags.fileObjects = true + } + if checksums.FulltextRebuild != ForceFulltextIndexCounter { + flags.fulltext = true + } + if checksums.BundledTemplates != i.btHash.Hash() { + flags.bundledTemplates = true + } + if checksums.BundledObjects != ForceBundledObjectsReindexCounter { + flags.bundledObjects = true + } + if checksums.IdxRebuildCounter != ForceIdxRebuildCounter { + flags.enableAll() + } + return i.reindex(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "reindex_forced"), flags) +} + +func (i *indexer) reindex(ctx context.Context, flags reindexFlags) (err error) { + if flags.any() { + log.Infof("start store reindex (%s)", flags.String()) + } + + if flags.fileKeys { + err = i.fileStore.RemoveEmpty() + if err != nil { + log.Errorf("reindex failed to RemoveEmpty filekeys: %v", err.Error()) + } else { + log.Infof("RemoveEmpty filekeys succeed") + } + } + + if flags.removeAllIndexedObjects { + ids, err := i.store.ListIds() + if err != nil { + log.Errorf("reindex failed to get all ids(removeAllIndexedObjects): %v", err.Error()) + } + for _, id := range ids { + err = i.store.DeleteDetails(id) + if err != nil { + log.Errorf("reindex failed to delete details(removeAllIndexedObjects): %v", err.Error()) + } + } + } + var indexesWereRemoved bool + if flags.eraseIndexes { + err = i.store.EraseIndexes() + if err != nil { + log.Errorf("reindex failed to erase indexes: %v", err.Error()) + } else { + log.Infof("all store indexes succesfully erased") + indexesWereRemoved = true + } + } + + // We derive or init predefined blocks here in order to ensure consistency of object store. + // If we call this method before removing objects from store, we will end up with inconsistent state + // because indexing of predefined objects will not run again + err = i.anytype.EnsurePredefinedBlocks(ctx) + if err != nil { + return err + } + + err = i.ensurePreinstalledObjects() + if err != nil { + return fmt.Errorf("ensure preinstalled objects: %w", err) + } + + if flags.any() { + d, err := i.getObjectInfo(ctx, i.anytype.PredefinedBlocks().Archive) + if err != nil { + log.Errorf("reindex failed to open archive: %s", err.Error()) + } else { + for _, target := range d.Links { + i.archivedMap[target] = struct{}{} + } + } + + d, err = i.getObjectInfo(ctx, i.anytype.PredefinedBlocks().Home) + if err != nil { + log.Errorf("reindex failed to open home: %s", err.Error()) + } else { + for _, b := range d.Links { + i.favoriteMap[b] = struct{}{} + } + } + } + + // for all ids except home and archive setting cache timeout for reindexing + // ctx = context.WithValue(ctx, ocache.CacheTimeout, cacheTimeout) + if flags.threadObjects { + ids, err := i.getIdsForTypes( + smartblock.SmartBlockTypePage, + smartblock.SmartBlockTypeSet, + smartblock.SmartBlockTypeObjectType, + smartblock.SmartBlockTypeProfilePage, + smartblock.SmartBlockTypeTemplate, + smartblock.SmartblockTypeMarketplaceType, + smartblock.SmartblockTypeMarketplaceTemplate, + smartblock.SmartblockTypeMarketplaceRelation, + smartblock.SmartBlockTypeArchive, + smartblock.SmartBlockTypeHome, + smartblock.SmartBlockTypeWorkspaceOld, + ) + if err != nil { + return err + } + start := time.Now() + successfullyReindexed := i.reindexIdsIgnoreErr(ctx, ids...) + if metrics.Enabled { + metrics.SharedClient.RecordEvent(metrics.ReindexEvent{ + ReindexType: metrics.ReindexTypeThreads, + Total: len(ids), + Success: successfullyReindexed, + SpentMs: int(time.Since(start).Milliseconds()), + IndexesRemoved: indexesWereRemoved, + }) + } + log.Infof("%d/%d objects have been successfully reindexed", successfullyReindexed, len(ids)) + } else { + go func() { + start := time.Now() + total, success, err := i.reindexOutdatedThreads() + if err != nil { + log.Infof("failed to reindex outdated objects: %s", err.Error()) + } else { + log.Infof("%d/%d outdated objects have been successfully reindexed", success, total) + } + if metrics.Enabled && total > 0 { + metrics.SharedClient.RecordEvent(metrics.ReindexEvent{ + ReindexType: metrics.ReindexTypeOutdatedHeads, + Total: total, + Success: success, + SpentMs: int(time.Since(start).Milliseconds()), + IndexesRemoved: indexesWereRemoved, + }) + } + }() + } + + if flags.fileObjects { + err = i.reindexIDsForSmartblockTypes(ctx, metrics.ReindexTypeFiles, indexesWereRemoved, smartblock.SmartBlockTypeFile) + if err != nil { + return err + } + } + if flags.bundledRelations { + err = i.reindexIDsForSmartblockTypes(ctx, metrics.ReindexTypeBundledRelations, indexesWereRemoved, smartblock.SmartBlockTypeBundledRelation) + if err != nil { + return err + } + } + if flags.bundledTypes { + err = i.reindexIDsForSmartblockTypes(ctx, metrics.ReindexTypeBundledTypes, indexesWereRemoved, smartblock.SmartBlockTypeBundledObjectType, smartblock.SmartBlockTypeAnytypeProfile) + if err != nil { + return err + } + } + if flags.bundledObjects { + // hardcoded for now + ids := []string{addr.AnytypeProfileId} + err = i.reindexIDs(ctx, metrics.ReindexTypeBundledObjects, false, ids) + if err != nil { + return err + } + } + + if flags.bundledTemplates { + existing, _, err := i.store.QueryObjectIds(database.Query{}, []smartblock.SmartBlockType{smartblock.SmartBlockTypeBundledTemplate}) + if err != nil { + return err + } + for _, id := range existing { + i.store.DeleteObject(id) + } + + err = i.reindexIDsForSmartblockTypes(ctx, metrics.ReindexTypeBundledTemplates, indexesWereRemoved, smartblock.SmartBlockTypeBundledTemplate) + if err != nil { + return err + } + } + if flags.fulltext { + ids, err := i.getIdsForTypes(smartblock.SmartBlockTypePage, smartblock.SmartBlockTypeFile, smartblock.SmartBlockTypeBundledRelation, smartblock.SmartBlockTypeBundledObjectType, smartblock.SmartBlockTypeAnytypeProfile) + if err != nil { + return err + } + + var addedToQueue int + for _, id := range ids { + if err := i.store.AddToIndexQueue(id); err != nil { + log.Errorf("failed to add to index queue: %v", err) + } else { + addedToQueue++ + } + } + msg := fmt.Sprintf("%d/%d objects have been successfully added to the fulltext queue", addedToQueue, len(ids)) + if len(ids)-addedToQueue != 0 { + log.Error(msg) + } else { + log.Info(msg) + } + } + + return i.saveLatestChecksums() +} + +func (i *indexer) reindexIDsForSmartblockTypes(ctx context.Context, reindexType metrics.ReindexType, indexesWereRemoved bool, sbTypes ...smartblock.SmartBlockType) error { + ids, err := i.getIdsForTypes(sbTypes...) + if err != nil { + return err + } + return i.reindexIDs(ctx, reindexType, indexesWereRemoved, ids) +} + +func (i *indexer) reindexIDs(ctx context.Context, reindexType metrics.ReindexType, indexesWereRemoved bool, ids []string) error { + start := time.Now() + successfullyReindexed := i.reindexIdsIgnoreErr(ctx, ids...) + if metrics.Enabled && len(ids) > 0 { + metrics.SharedClient.RecordEvent(metrics.ReindexEvent{ + ReindexType: reindexType, + Total: len(ids), + Success: successfullyReindexed, + SpentMs: int(time.Since(start).Milliseconds()), + IndexesRemoved: indexesWereRemoved, + }) + } + msg := fmt.Sprintf("%d/%d %s have been successfully reindexed", successfullyReindexed, len(ids), reindexType) + if len(ids)-successfullyReindexed != 0 { + log.Error(msg) + } else { + log.Info(msg) + } + return nil +} + +func (i *indexer) ensurePreinstalledObjects() error { + var objects []*types.Struct + + for _, ot := range bundle.SystemTypes { + t, err := bundle.GetTypeByUrl(ot.BundledURL()) + if err != nil { + continue + } + objects = append(objects, (&relationutils.ObjectType{ObjectType: t}).ToStruct()) + } + + for _, rk := range bundle.SystemRelations { + rel := bundle.MustGetRelation(rk) + for _, opt := range rel.SelectDict { + opt.RelationKey = rel.Key + objects = append(objects, (&relationutils.Option{RelationOption: opt}).ToStruct()) + } + objects = append(objects, (&relationutils.Relation{Relation: rel}).ToStruct()) + } + + _, _, err := i.subObjectCreator.CreateSubObjectsInWorkspace(objects) + if errors.Is(err, editor.ErrSubObjectAlreadyExists) { + return nil + } + return err +} + func (i *indexer) saveLatestChecksums() error { // todo: add layout indexing when needed checksums := model.ObjectStoreChecksums{ @@ -172,95 +593,8 @@ func (i *indexer) saveLatestCounters() error { return i.store.SaveChecksums(&checksums) } -func (i *indexer) Run(context.Context) (err error) { - if ftErr := i.ftInit(); ftErr != nil { - log.Errorf("can't init ft: %v", ftErr) - } - err = i.reindexIfNeeded() - if err != nil { - return err - } - i.migrateRemoveNonindexableObjects() - go i.ftLoop() - return -} - -func (i *indexer) ForceFTIndex() { - select { - case i.forceFt <- struct{}{}: - default: - } -} - -func (i *indexer) migrateRemoveNonindexableObjects() { - ids, err := i.getIdsForTypes( - smartblock.SmartblockTypeMarketplaceType, smartblock.SmartblockTypeMarketplaceRelation, - smartblock.SmartblockTypeMarketplaceTemplate, smartblock.SmartBlockTypeDate, smartblock.SmartBlockTypeBreadcrumbs, - ) - if err != nil { - log.Errorf("migrateRemoveNonindexableObjects: failed to get ids: %s", err.Error()) - } - - for _, id := range ids { - err = i.store.DeleteDetails(id) - if err != nil { - log.Errorf("migrateRemoveNonindexableObjects: failed to get ids: %s", err.Error()) - } - } -} - -func (i *indexer) reindexIfNeeded() error { - var ( - err error - checksums *model.ObjectStoreChecksums - reindex reindexFlags - ) - checksums, err = i.store.GetChecksums() - if err != nil && err != ds.ErrNotFound { - return err - } - if checksums == nil { - checksums = &model.ObjectStoreChecksums{ - // do no add bundled relations checksums, because we want to index them for new accounts - ObjectsForceReindexCounter: ForceThreadsObjectsReindexCounter, - FilesForceReindexCounter: ForceFilesReindexCounter, - IdxRebuildCounter: ForceIdxRebuildCounter, - FilestoreKeysForceReindexCounter: ForceFilestoreKeysReindexCounter, - } - } - - if checksums.BundledRelations != bundle.RelationChecksum { - reindex = reindex | reindexBundledRelations - } - if checksums.BundledObjectTypes != bundle.TypeChecksum { - reindex = reindex | reindexBundledTypes - } - if checksums.ObjectsForceReindexCounter != ForceThreadsObjectsReindexCounter { - reindex = reindex | reindexThreadObjects - } - if checksums.FilestoreKeysForceReindexCounter != ForceFilestoreKeysReindexCounter { - reindex = reindex | reindexFileKeys - } - if checksums.FilesForceReindexCounter != ForceFilesReindexCounter { - reindex = reindex | reindexFileObjects - } - if checksums.FulltextRebuild != ForceFulltextIndexCounter { - reindex = reindex | reindexFulltext - } - if checksums.BundledTemplates != i.btHash.Hash() { - reindex = reindex | reindexBundledTemplates - } - if checksums.BundledObjects != ForceBundledObjectsReindexCounter { - reindex = reindex | reindexBundledObjects - } - if checksums.IdxRebuildCounter != ForceIdxRebuildCounter { - reindex = math.MaxUint64 - } - return i.Reindex(context.WithValue(context.TODO(), metrics.CtxKeyRequest, "reindex_forced"), reindex) -} - func (i *indexer) reindexOutdatedThreads() (toReindex, success int, err error) { - spc, err := i.anytype.SpaceService().AccountSpace(context.Background()) + spc, err := i.spaceService.AccountSpace(context.Background()) if err != nil { return } @@ -294,28 +628,49 @@ func (i *indexer) reindexOutdatedThreads() (toReindex, success int, err error) { idsToReindex = append(idsToReindex, tid) } } - if len(idsToReindex) > 0 { - for _, id := range idsToReindex { - // TODO: we should reindex it I guess at start - //if i.anytype.PredefinedBlocks().IsAccount(id) { - // continue - //} - ctx := context.WithValue(context.Background(), metrics.CtxKeyRequest, "reindexOutdatedThreads") - d, err := i.doc.GetDocInfo(ctx, id) - if err != nil { - continue - } + ctx := context.WithValue(context.Background(), metrics.CtxKeyRequest, "reindexOutdatedThreads") + success = i.reindexIdsIgnoreErr(ctx, idsToReindex...) + return len(idsToReindex), success, nil +} - err = i.index(ctx, d) - if err == nil { - success++ - } else { - log.With("thread", id).Errorf("reindexOutdatedThreads failed to index doc: %s", err.Error()) - } +func (i *indexer) reindexDoc(ctx context.Context, id string) error { + _, isArchived := i.archivedMap[id] + _, isFavorite := i.favoriteMap[id] + + err := i.store.UpdatePendingLocalDetails(id, func(pending *types.Struct) (*types.Struct, error) { + pending.Fields[bundle.RelationKeyIsArchived.String()] = pbtypes.Bool(isArchived) + pending.Fields[bundle.RelationKeyIsFavorite.String()] = pbtypes.Bool(isFavorite) + return pending, nil + }) + if err != nil { + log.Errorf("failed to update isArchived and isFavorite details for %s: %s", id, err) + } + + // Touch the object to initiate indexing + return block.DoWithContext(ctx, i.picker, id, func(sb smartblock2.SmartBlock) error { + return sb.Apply(sb.NewState(), smartblock2.NoHistory, smartblock2.NoEvent, smartblock2.NoRestrictions) + }) +} + +func (i *indexer) reindexIdsIgnoreErr(ctx context.Context, ids ...string) (successfullyReindexed int) { + for _, id := range ids { + err := i.reindexDoc(ctx, id) + if err != nil { + log.With("thread", id).Errorf("failed to reindex: %v", err) + } else { + successfullyReindexed++ } } - return len(idsToReindex), success, nil + return +} + +func (i *indexer) getObjectInfo(ctx context.Context, id string) (info smartblock2.DocInfo, err error) { + err = block.DoWithContext(ctx, i.picker, id, func(sb smartblock2.SmartBlock) error { + info = sb.GetDocInfo() + return nil + }) + return } func (i *indexer) getIdsForTypes(sbt ...smartblock.SmartBlockType) ([]string, error) { @@ -334,643 +689,6 @@ func (i *indexer) getIdsForTypes(sbt ...smartblock.SmartBlockType) ([]string, er return ids, nil } -func (i *indexer) Reindex(ctx context.Context, reindex reindexFlags) (err error) { - if reindex != 0 { - log.Infof("start store reindex (eraseIndexes=%v, reindexFileObjects=%v, reindexThreadObjects=%v, reindexBundledRelations=%v, reindexBundledTypes=%v, reindexFulltext=%v, reindexBundledTemplates=%v, reindexBundledObjects=%v, reindexFileKeys=%v)", reindex&eraseIndexes != 0, reindex&reindexFileObjects != 0, reindex&reindexThreadObjects != 0, reindex&reindexBundledRelations != 0, reindex&reindexBundledTypes != 0, reindex&reindexFulltext != 0, reindex&reindexBundledTemplates != 0, reindex&reindexBundledObjects != 0, reindex&reindexFileKeys != 0) - } - - if reindex&reindexFileKeys != 0 { - err = i.anytype.FileStore().RemoveEmpty() - if err != nil { - log.Errorf("reindex failed to RemoveEmpty filekeys: %v", err.Error()) - } else { - log.Infof("RemoveEmpty filekeys succeed") - } - } - - if reindex&removeAllIndexedObjects != 0 { - ids, err := i.store.ListIds() - if err != nil { - log.Errorf("reindex failed to get all ids(removeAllIndexedObjects): %v", err.Error()) - } - for _, id := range ids { - err = i.store.DeleteDetails(id) - if err != nil { - log.Errorf("reindex failed to delete details(removeAllIndexedObjects): %v", err.Error()) - } - } - - defer func() { - i.relationMigratorMu.Lock() - defer i.relationMigratorMu.Unlock() - if i.relationBulkMigration == nil { - return - } - err2 := i.relationBulkMigration.Commit() - i.relationBulkMigration = nil - if err2 != nil { - log.Errorf("reindex relation migration error: %s", err2.Error()) - } - }() - i.relationMigratorMu.Lock() - i.relationBulkMigration = i.relationService.CreateBulkMigration() - i.relationMigratorMu.Unlock() - } - var indexesWereRemoved bool - if reindex&eraseIndexes != 0 { - err = i.store.EraseIndexes() - if err != nil { - log.Errorf("reindex failed to erase indexes: %v", err.Error()) - } else { - log.Infof("all store indexes succesfully erased") - // store this flag because underlying localstore needs to now if it needs to amend indexes based on the prev value - indexesWereRemoved = true - } - } - if reindex > 0 { - d, err := i.doc.GetDocInfo(ctx, i.anytype.PredefinedBlocks().Archive) - if err != nil { - log.Errorf("reindex failed to open archive: %s", err.Error()) - } else { - for _, target := range d.Links { - i.archivedMap[target] = struct{}{} - } - } - - d, err = i.doc.GetDocInfo(ctx, i.anytype.PredefinedBlocks().Home) - if err != nil { - log.Errorf("reindex failed to open archive: %s", err.Error()) - } else { - for _, b := range d.Links { - i.favoriteMap[b] = struct{}{} - } - } - } - - // for all ids except home and archive setting cache timeout for reindexing - //ctx = context.WithValue(ctx, ocache.CacheTimeout, cacheTimeout) - if reindex&reindexThreadObjects != 0 { - ids, err := i.getIdsForTypes( - smartblock.SmartBlockTypePage, - smartblock.SmartBlockTypeSet, - smartblock.SmartBlockTypeObjectType, - smartblock.SmartBlockTypeProfilePage, - smartblock.SmartBlockTypeTemplate, - smartblock.SmartblockTypeMarketplaceType, - smartblock.SmartblockTypeMarketplaceTemplate, - smartblock.SmartblockTypeMarketplaceRelation, - smartblock.SmartBlockTypeArchive, - smartblock.SmartBlockTypeHome, - smartblock.SmartBlockTypeWorkspaceOld, - ) - if err != nil { - return err - } - start := time.Now() - successfullyReindexed := i.reindexIdsIgnoreErr(ctx, indexesWereRemoved, ids...) - if metrics.Enabled { - metrics.SharedClient.RecordEvent(metrics.ReindexEvent{ - ReindexType: metrics.ReindexTypeThreads, - Total: len(ids), - Success: successfullyReindexed, - SpentMs: int(time.Since(start).Milliseconds()), - IndexesRemoved: indexesWereRemoved, - }) - } - log.Infof("%d/%d objects have been successfully reindexed", successfullyReindexed, len(ids)) - } else { - go func() { - start := time.Now() - total, success, err := i.reindexOutdatedThreads() - if err != nil { - log.Infof("failed to reindex outdated objects: %s", err.Error()) - } else { - log.Infof("%d/%d outdated objects have been successfully reindexed", success, total) - } - if metrics.Enabled && total > 0 { - metrics.SharedClient.RecordEvent(metrics.ReindexEvent{ - ReindexType: metrics.ReindexTypeOutdatedHeads, - Total: total, - Success: success, - SpentMs: int(time.Since(start).Milliseconds()), - IndexesRemoved: indexesWereRemoved, - }) - } - }() - } - - if reindex&reindexFileObjects != 0 { - ids, err := i.getIdsForTypes(smartblock.SmartBlockTypeFile) - if err != nil { - return err - } - start := time.Now() - successfullyReindexed := i.reindexIdsIgnoreErr(ctx, indexesWereRemoved, ids...) - if metrics.Enabled && len(ids) > 0 { - metrics.SharedClient.RecordEvent(metrics.ReindexEvent{ - ReindexType: metrics.ReindexTypeFiles, - Total: len(ids), - Success: successfullyReindexed, - SpentMs: int(time.Since(start).Milliseconds()), - IndexesRemoved: indexesWereRemoved, - }) - } - msg := fmt.Sprintf("%d/%d files have been successfully reindexed", successfullyReindexed, len(ids)) - if len(ids)-successfullyReindexed != 0 { - log.Error(msg) - } else { - log.Info(msg) - } - } - if reindex&reindexBundledRelations != 0 { - ids, err := i.getIdsForTypes(smartblock.SmartBlockTypeBundledRelation) - if err != nil { - return err - } - start := time.Now() - successfullyReindexed := i.reindexIdsIgnoreErr(ctx, indexesWereRemoved, ids...) - if metrics.Enabled && len(ids) > 0 { - metrics.SharedClient.RecordEvent(metrics.ReindexEvent{ - ReindexType: metrics.ReindexTypeBundledRelations, - Total: len(ids), - Success: successfullyReindexed, - SpentMs: int(time.Since(start).Milliseconds()), - IndexesRemoved: indexesWereRemoved, - }) - } - msg := fmt.Sprintf("%d/%d bundled relations have been successfully reindexed", successfullyReindexed, len(ids)) - if len(ids)-successfullyReindexed != 0 { - log.Error(msg) - } else { - log.Info(msg) - } - } - if reindex&reindexBundledTypes != 0 { - // lets add anytypeProfile here, because it's seems too much to create one more counter especially for it - ids, err := i.getIdsForTypes(smartblock.SmartBlockTypeBundledObjectType, smartblock.SmartBlockTypeAnytypeProfile) - if err != nil { - return err - } - start := time.Now() - successfullyReindexed := i.reindexIdsIgnoreErr(ctx, indexesWereRemoved, ids...) - if metrics.Enabled && len(ids) > 0 { - metrics.SharedClient.RecordEvent(metrics.ReindexEvent{ - ReindexType: metrics.ReindexTypeBundledTypes, - Total: len(ids), - Success: successfullyReindexed, - SpentMs: int(time.Since(start).Milliseconds()), - IndexesRemoved: indexesWereRemoved, - }) - } - msg := fmt.Sprintf("%d/%d bundled types have been successfully reindexed", successfullyReindexed, len(ids)) - if len(ids)-successfullyReindexed != 0 { - log.Error(msg) - } else { - log.Info(msg) - } - - var ots = make([]string, 0, len(bundle.SystemTypes)) - for _, ot := range bundle.SystemTypes { - ots = append(ots, ot.BundledURL()) - } - - for _, ot := range bundle.InternalTypes { - ots = append(ots, ot.BundledURL()) - } - - var rels = make([]*model.Relation, 0, len(bundle.RequiredInternalRelations)) - for _, rel := range bundle.SystemRelations { - rels = append(rels, bundle.MustGetRelation(rel)) - } - i.migrateObjectTypes(ots) - i.migrateRelations(rels) - } - if reindex&reindexBundledObjects != 0 { - // hardcoded for now - ids := []string{addr.AnytypeProfileId} - start := time.Now() - successfullyReindexed := i.reindexIdsIgnoreErr(ctx, indexesWereRemoved, ids...) - if metrics.Enabled && len(ids) > 0 { - metrics.SharedClient.RecordEvent(metrics.ReindexEvent{ - ReindexType: metrics.ReindexTypeBundledTemplates, - Total: len(ids), - Success: successfullyReindexed, - SpentMs: int(time.Since(start).Milliseconds()), - }) - } - msg := fmt.Sprintf("%d/%d bundled objects have been successfully reindexed", successfullyReindexed, len(ids)) - if len(ids)-successfullyReindexed != 0 { - log.Error(msg) - } else { - log.Info(msg) - } - } - - if reindex&reindexBundledTemplates != 0 { - existsRec, _, err := i.store.QueryObjectInfo(database.Query{}, []smartblock.SmartBlockType{smartblock.SmartBlockTypeBundledTemplate}) - if err != nil { - return err - } - existsIds := make([]string, 0, len(existsRec)) - for _, rec := range existsRec { - existsIds = append(existsIds, rec.Id) - } - ids, err := i.getIdsForTypes(smartblock.SmartBlockTypeBundledTemplate) - if err != nil { - return err - } - var removed int - for _, eId := range existsIds { - if slice.FindPos(ids, eId) == -1 { - removed++ - i.store.DeleteObject(eId) - } - } - successfullyReindexed := i.reindexIdsIgnoreErr(ctx, indexesWereRemoved, ids...) - msg := fmt.Sprintf("%d/%d bundled templates have been successfully reindexed; removed: %d", successfullyReindexed, len(ids), removed) - if len(ids)-successfullyReindexed != 0 { - log.Error(msg) - } else { - log.Info(msg) - } - } - if reindex&reindexFulltext != 0 { - var ids []string - ids, err := i.getIdsForTypes(smartblock.SmartBlockTypePage, smartblock.SmartBlockTypeFile, smartblock.SmartBlockTypeBundledRelation, smartblock.SmartBlockTypeBundledObjectType, smartblock.SmartBlockTypeAnytypeProfile) - if err != nil { - return err - } - - var addedToQueue int - for _, id := range ids { - if err := i.store.AddToIndexQueue(id); err != nil { - log.Errorf("failed to add to index queue: %v", err) - } else { - addedToQueue++ - } - } - msg := fmt.Sprintf("%d/%d objects have been successfully added to the fulltext queue", addedToQueue, len(ids)) - if len(ids)-addedToQueue != 0 { - log.Error(msg) - } else { - log.Info(msg) - } - } - - return i.saveLatestChecksums() -} - -func extractOldRelationsFromState(s *state.State) []*model.Relation { - var rels []*model.Relation - if objRels := s.OldExtraRelations(); len(objRels) > 0 { - rels = append(rels, s.OldExtraRelations()...) - } - - if dvBlock := s.Pick(template.DataviewBlockId); dvBlock != nil { - rels = append(rels, dvBlock.Model().GetDataview().GetRelations()...) - } - - return rels -} - -func (i *indexer) migrateRelations(rels []*model.Relation) { - i.relationMigratorMu.Lock() - defer i.relationMigratorMu.Unlock() - - if i.relationBulkMigration != nil { - i.relationBulkMigration.AddRelations(rels) - } else { - err := i.relationService.MigrateRelations(rels) - if err != nil { - log.Errorf("migrateRelations got error: %s", err.Error()) - } - } -} - -func (i *indexer) migrateObjectTypes(ots []string) { - if len(ots) == 0 { - return - } - - var typesModels []*model.ObjectType // do not make - for _, ot := range ots { - t, err := bundle.GetTypeByUrl(ot) - if err != nil { - continue - } - - typesModels = append(typesModels, t) - } - - if len(typesModels) == 0 { - return - } - - i.relationMigratorMu.Lock() - defer i.relationMigratorMu.Unlock() - - if i.relationBulkMigration != nil { - i.relationBulkMigration.AddObjectTypes(typesModels) - } else { - err := i.relationService.MigrateObjectTypes(typesModels) - if err != nil { - log.Errorf("migrateObjectTypes got error: %s", err.Error()) - } - } -} - -func (i *indexer) reindexDoc(ctx context.Context, id string, indexesWereRemoved bool) error { - t, err := i.typeProvider.Type(id) - if err != nil { - return fmt.Errorf("incorrect sb type: %v", err) - } - - d, err := i.doc.GetDocInfo(ctx, id) - if err != nil { - log.Errorf("reindexDoc failed to open %s: %s", id, err.Error()) - return fmt.Errorf("failed to open doc: %s", err.Error()) - } - - indexDetails, indexLinks := t.Indexable() - if indexLinks { - if err := i.store.UpdateObjectLinks(d.Id, d.Links); err != nil { - log.With("thread", d.Id).Errorf("failed to save object links: %v", err) - } - } - - if !indexDetails { - i.store.DeleteDetails(d.Id) - return nil - } - - details := d.State.CombinedDetails() - _, isArchived := i.archivedMap[id] - _, isFavorite := i.favoriteMap[id] - - details.Fields[bundle.RelationKeyIsArchived.String()] = pbtypes.Bool(isArchived) - details.Fields[bundle.RelationKeyIsFavorite.String()] = pbtypes.Bool(isFavorite) - details.Fields[bundle.RelationKeyLinks.String()] = pbtypes.StringList(d.Links) - - var curDetails *types.Struct - curDetailsO, _ := i.store.GetDetails(id) - if curDetailsO.GetDetails().GetFields() != nil { - curDetails = curDetailsO.Details - } - // compare only real object scoped details - detailsObjectScope := pbtypes.StructCutKeys(details, bundle.LocalRelationsKeys) - curDetailsObjectScope := pbtypes.StructCutKeys(curDetails, bundle.LocalRelationsKeys) - if indexesWereRemoved || curDetailsObjectScope == nil || !detailsObjectScope.Equal(curDetailsObjectScope) { - if indexesWereRemoved || curDetails.GetFields() == nil { - if err := i.store.CreateObject(id, details, d.Links, pbtypes.GetString(details, bundle.RelationKeyDescription.String())); err != nil { - return fmt.Errorf("can't create object in the store: %v", err) - } - } else { - if err := i.store.UpdateObjectDetails(id, details, true); err != nil { - return fmt.Errorf("can't update object in the store: %v", err) - } - } - if headsHash := headsHash(d.Heads); headsHash != "" { - err = i.store.SaveLastIndexedHeadsHash(id, headsHash) - if err != nil { - log.With("thread", id).Errorf("failed to save indexed heads hash: %v", err) - } - } - - var skipFulltext bool - if i.store.FTSearch() != nil { - // skip fulltext if we already has the object indexed - if exists, _ := i.store.FTSearch().Has(id); exists { - skipFulltext = true - } - } - - if !skipFulltext { - if err = i.store.AddToIndexQueue(id); err != nil { - log.With("thread", id).Errorf("can't add to index: %v", err) - } - } - } - return nil -} - -func (i *indexer) reindexIdsIgnoreErr(ctx context.Context, indexRemoved bool, ids ...string) (successfullyReindexed int) { - for _, id := range ids { - err := i.reindexDoc(ctx, id, indexRemoved) - if err != nil { - log.With("thread", id).Errorf("failed to reindex: %v", err) - } else { - successfullyReindexed++ - } - } - return -} - -func (i *indexer) index(ctx context.Context, info doc.DocInfo) error { - startTime := time.Now() - sbType, err := i.typeProvider.Type(info.Id) - if err != nil { - sbType = smartblock.SmartBlockTypePage - } - saveIndexedHash := func() { - if headsHash := headsHash(info.Heads); headsHash != "" { - err = i.store.SaveLastIndexedHeadsHash(info.Id, headsHash) - if err != nil { - log.With("thread", info.Id).Errorf("failed to save indexed heads hash: %v", err) - } - } - } - - indexDetails, indexLinks := sbType.Indexable() - if sbType != smartblock.SmartBlockTypeSubObject && sbType != smartblock.SmartBlockTypeWorkspace { - // avoid recursions - log.With("migratedtype", sbType).Warn("migrating types") - if pbtypes.GetString(info.State.CombinedDetails(), bundle.RelationKeyCreator.String()) != addr.AnytypeProfileId { - i.migrateRelations(extractOldRelationsFromState(info.State)) - i.migrateObjectTypes(info.State.ObjectTypesToMigrate()) - } - } - if !indexDetails && !indexLinks { - saveIndexedHash() - return nil - } - - details := info.State.CombinedDetails() - details.Fields[bundle.RelationKeyLinks.String()] = pbtypes.StringList(info.Links) - setCreator := pbtypes.GetString(info.State.LocalDetails(), bundle.RelationKeyCreator.String()) - if setCreator == "" { - setCreator = i.anytype.ProfileID() - } - indexSetTime := time.Now() - var hasError bool - if indexLinks { - if err = i.store.UpdateObjectLinks(info.Id, info.Links); err != nil { - hasError = true - log.With("thread", info.Id).Errorf("failed to save object links: %v", err) - } - } - - indexLinksTime := time.Now() - if indexDetails { - if err := i.store.UpdateObjectDetails(info.Id, details, false); err != nil { - hasError = true - log.With("thread", info.Id).Errorf("can't update object store: %v", err) - } - if err := i.store.AddToIndexQueue(info.Id); err != nil { - log.With("thread", info.Id).Errorf("can't add id to index queue: %v", err) - } else { - log.With("thread", info.Id).Debugf("to index queue") - } - } else { - _ = i.store.DeleteDetails(info.Id) - } - indexDetailsTime := time.Now() - detailsCount := 0 - if details.GetFields() != nil { - detailsCount = len(details.GetFields()) - } - - if !hasError { - saveIndexedHash() - } - - metrics.SharedClient.RecordEvent(metrics.IndexEvent{ - ObjectId: info.Id, - IndexLinksTimeMs: indexLinksTime.Sub(indexSetTime).Milliseconds(), - IndexDetailsTimeMs: indexDetailsTime.Sub(indexLinksTime).Milliseconds(), - IndexSetRelationsTimeMs: indexSetTime.Sub(startTime).Milliseconds(), - RelationsCount: len(info.State.PickRelationLinks()), - DetailsCount: detailsCount, - }) - - return nil -} - -func (i *indexer) ftLoop() { - ticker := time.NewTicker(ftIndexInterval) - i.ftIndex() - var lastForceIndex time.Time - i.mu.Lock() - quit := i.quit - i.mu.Unlock() - for { - select { - case <-quit: - return - case <-ticker.C: - i.ftIndex() - case <-i.forceFt: - if time.Since(lastForceIndex) > ftIndexForceMinInterval { - i.ftIndex() - lastForceIndex = time.Now() - } - } - } -} - -func (i *indexer) ftIndex() { - if err := i.store.IndexForEach(i.ftIndexDoc); err != nil { - log.Errorf("store.IndexForEach error: %v", err) - } -} - -func (i *indexer) ftIndexDoc(id string, _ time.Time) (err error) { - st := time.Now() - //ctx := context.WithValue(context.Background(), ocache.CacheTimeout, cacheTimeout) - ctx := context.WithValue(context.Background(), metrics.CtxKeyRequest, "index_fulltext") - - info, err := i.doc.GetDocInfo(ctx, id) - if err != nil { - return - } - - sbType, err := i.typeProvider.Type(info.Id) - if err != nil { - sbType = smartblock.SmartBlockTypePage - } - indexDetails, _ := sbType.Indexable() - if !indexDetails { - return nil - } - - if err = i.store.UpdateObjectSnippet(id, info.State.Snippet()); err != nil { - return - } - - if len(info.FileHashes) > 0 { - // todo: move file indexing to the main indexer as we have the full state there now - existingIDs, err := i.store.HasIDs(info.FileHashes...) - if err != nil { - log.Errorf("failed to get existing file ids : %s", err.Error()) - } - newIds := slice.Difference(info.FileHashes, existingIDs) - for _, hash := range newIds { - // file's hash is id - err = i.reindexDoc(ctx, hash, false) - if err != nil { - log.With("id", hash).Errorf("failed to reindex file: %s", err.Error()) - } - - err = i.store.AddToIndexQueue(hash) - if err != nil { - log.With("id", hash).Error(err.Error()) - } - } - } - - if fts := i.store.FTSearch(); fts != nil { - title := pbtypes.GetString(info.State.Details(), bundle.RelationKeyName.String()) - if info.State.ObjectType() == bundle.TypeKeyNote.String() || title == "" { - title = info.State.Snippet() - } - ftDoc := ftsearch.SearchDoc{ - Id: id, - Title: title, - Text: info.State.SearchText(), - } - if err := fts.Index(ftDoc); err != nil { - log.Errorf("can't ft index doc: %v", err) - } - log.Debugf("ft search indexed with title: '%s'", ftDoc.Title) - } - - log.With("thread", id).Infof("ft index updated for a %v", time.Since(st)) - return -} - -func (i *indexer) ftInit() error { - if ft := i.store.FTSearch(); ft != nil { - docCount, err := ft.DocCount() - if err != nil { - return err - } - if docCount == 0 { - ids, err := i.store.ListIds() - if err != nil { - return err - } - for _, id := range ids { - if err := i.store.AddToIndexQueue(id); err != nil { - return err - } - } - } - } - return nil -} - -func (i *indexer) Close(ctx context.Context) (err error) { - i.mu.Lock() - quit := i.quit - i.mu.Unlock() - if quit != nil { - close(quit) - i.mu.Lock() - i.quit = nil - i.mu.Unlock() - } - return nil -} - func headsHash(heads []string) string { if len(heads) == 0 { return "" diff --git a/core/object.go b/core/object.go index fcd415545..033f7ad06 100644 --- a/core/object.go +++ b/core/object.go @@ -24,6 +24,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/database/filter" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/internalflag" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" ) @@ -379,6 +380,7 @@ func (mw *Middleware) ObjectSearchUnsubscribe(cctx context.Context, req *pb.RpcO return response(nil) } +// TODO Move logic to block service func (mw *Middleware) ObjectGraph(cctx context.Context, req *pb.RpcObjectGraphRequest) *pb.RpcObjectGraphResponse { response := func(code pb.RpcObjectGraphResponseErrorCode, nodes []*types.Struct, edges []*pb.RpcObjectGraphEdge, err error) *pb.RpcObjectGraphResponse { m := &pb.RpcObjectGraphResponse{Error: &pb.RpcObjectGraphResponseError{Code: code}, Nodes: nodes, Edges: edges} @@ -476,9 +478,11 @@ func (mw *Middleware) ObjectGraph(cctx context.Context, req *pb.RpcObjectGraphRe } } } + + sbtProvider := app.MustComponent[typeprovider.SmartBlockTypeProvider](mw.app) links := pbtypes.GetStringList(rec.Details, bundle.RelationKeyLinks.String()) for _, link := range links { - sbType, _ := smartblock.SmartBlockTypeFromID(link) + sbType, _ := sbtProvider.Type(link) // ignore files because we index all file blocks as outgoing links if sbType == smartblock.SmartBlockTypeFile { continue diff --git a/core/subscription/service.go b/core/subscription/service.go index a9f9b511e..300ce1059 100644 --- a/core/subscription/service.go +++ b/core/subscription/service.go @@ -24,6 +24,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" "github.com/anytypeio/go-anytype-middleware/util/slice" ) @@ -34,9 +35,10 @@ var log = logging.Logger("anytype-mw-subscription") var batchTime = 50 * time.Millisecond -func New(collectionService CollectionService) Service { +func New(collectionService CollectionService, sbtProvider typeprovider.SmartBlockTypeProvider) Service { return &service{ collectionService: collectionService, + sbtProvider: sbtProvider, } } @@ -74,6 +76,7 @@ type service struct { objectStore objectstore.ObjectStore kanban kanban.Service collectionService CollectionService + sbtProvider typeprovider.SmartBlockTypeProvider sendEvent func(e *pb.Event) m sync.Mutex @@ -457,7 +460,7 @@ func (s *service) filtersFromSource(sources []string) (filter.Filter, error) { var objTypeIds, relTypeKeys []string for _, source := range sources { - sbt, err := smartblock.SmartBlockTypeFromID(source) + sbt, err := s.sbtProvider.Type(source) if err != nil { return nil, err } diff --git a/pkg/lib/core/smartblock/smartblock.go b/pkg/lib/core/smartblock/smartblock.go index a76adde86..b3a83d77d 100644 --- a/pkg/lib/core/smartblock/smartblock.go +++ b/pkg/lib/core/smartblock/smartblock.go @@ -4,14 +4,10 @@ import ( "encoding/binary" "errors" "fmt" - "github.com/textileio/go-threads/core/thread" - "strings" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" + "github.com/textileio/go-threads/core/thread" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" - "github.com/globalsign/mgo/bson" - "github.com/ipfs/go-cid" - "github.com/multiformats/go-multihash" ) type SmartBlockType uint64 @@ -45,54 +41,6 @@ const ( var ErrNoSuchSmartblock = errors.New("this id does not relate to any smartblock type") -func SmartBlockTypeFromID(id string) (SmartBlockType, error) { - if strings.HasPrefix(id, addr.BundledRelationURLPrefix) { - return SmartBlockTypeBundledRelation, nil - } - - if strings.HasPrefix(id, addr.BundledObjectTypeURLPrefix) { - return SmartBlockTypeBundledObjectType, nil - } - - if len(strings.Split(id, addr.SubObjectCollectionIdSeparator)) == 2 { - return SmartBlockTypeSubObject, nil - } - - // workaround for options that have no prefix - // todo: remove this after migration to the new records format - if bson.IsObjectIdHex(id) { - return SmartBlockTypeSubObject, nil - } - - if strings.HasPrefix(id, addr.AnytypeProfileId) { - return SmartBlockTypeProfilePage, nil - } - if strings.HasPrefix(id, addr.VirtualPrefix) { - sbt, err := addr.ExtractVirtualSourceType(id) - if err != nil { - return 0, err - } - return SmartBlockType(sbt), nil - } - if strings.HasPrefix(id, addr.DatePrefix) { - return SmartBlockTypeDate, nil - } - - c, err := cid.Decode(id) - if err != nil { - return SmartBlockTypePage, err - } - // TODO: discard this fragile condition as soon as we will move to the multiaddr with prefix - if c.Prefix().Codec == cid.DagProtobuf && c.Prefix().MhType == multihash.SHA2_256 { - return SmartBlockTypeFile, nil - } - if c.Prefix().Codec == cid.DagCBOR { - return SmartBlockTypePage, nil - } - - return SmartBlockTypePage, ErrNoSuchSmartblock -} - func PatchSmartBlockType(id string, sbt SmartBlockType) (string, error) { tid, err := thread.Decode(id) if err != nil { diff --git a/pkg/lib/localstore/objectstore/objects.go b/pkg/lib/localstore/objectstore/objects.go index bc35555cb..cf6d2c2e5 100644 --- a/pkg/lib/localstore/objectstore/objects.go +++ b/pkg/lib/localstore/objectstore/objects.go @@ -31,6 +31,7 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" "github.com/anytypeio/go-anytype-middleware/pkg/lib/schema" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" "github.com/anytypeio/go-anytype-middleware/util/slice" ) @@ -190,8 +191,10 @@ var ( _ ObjectStore = (*dsObjectStore)(nil) ) -func New() ObjectStore { - return &dsObjectStore{} +func New(sbtProvider typeprovider.SmartBlockTypeProvider) ObjectStore { + return &dsObjectStore{ + sbtProvider: sbtProvider, + } } func NewWithLocalstore(ds noctxds.DSTxnBatching) ObjectStore { @@ -292,17 +295,18 @@ type relationObjectType struct { var ErrNotAnObject = fmt.Errorf("not an object") -var filterNotSystemObjects = &filterSmartblockTypes{ - smartBlockTypes: []smartblock.SmartBlockType{ - smartblock.SmartBlockTypeArchive, - smartblock.SmartBlockTypeHome, - }, - not: true, -} - type filterSmartblockTypes struct { smartBlockTypes []smartblock.SmartBlockType not bool + sbtProvider typeprovider.SmartBlockTypeProvider +} + +func newSmartblockTypesFilter(sbtProvider typeprovider.SmartBlockTypeProvider, not bool, smartBlockTypes []smartblock.SmartBlockType) *filterSmartblockTypes { + return &filterSmartblockTypes{ + smartBlockTypes: smartBlockTypes, + not: not, + sbtProvider: sbtProvider, + } } type RelationWithObjectType struct { @@ -314,7 +318,7 @@ func (m *filterSmartblockTypes) Filter(e query.Entry) bool { keyParts := strings.Split(e.Key, "/") id := keyParts[len(keyParts)-1] - t, err := smartblock.SmartBlockTypeFromID(id) + t, err := m.sbtProvider.Type(id) if err != nil { log.Errorf("failed to detect smartblock type for %s: %s", id, err.Error()) return false @@ -343,6 +347,8 @@ type dsObjectStore struct { subscriptions []database.Subscription depSubscriptions []database.Subscription + + sbtProvider typeprovider.SmartBlockTypeProvider } func (m *dsObjectStore) GetCurrentWorkspaceId() (string, error) { @@ -568,19 +574,19 @@ func (m *dsObjectStore) GetAggregatedOptions(relationKey string) (options []*mod } func (m *dsObjectStore) objectTypeFilter(ots ...string) query.Filter { - var filter filterSmartblockTypes + var sbTypes []smartblock.SmartBlockType for _, otUrl := range ots { if ot, err := bundle.GetTypeByUrl(otUrl); err == nil { for _, sbt := range ot.Types { - filter.smartBlockTypes = append(filter.smartBlockTypes, smartblock.SmartBlockType(sbt)) + sbTypes = append(sbTypes, smartblock.SmartBlockType(sbt)) } continue } - if sbt, err := smartblock.SmartBlockTypeFromID(otUrl); err == nil { - filter.smartBlockTypes = append(filter.smartBlockTypes, sbt) + if sbt, err := m.sbtProvider.Type(otUrl); err == nil { + sbTypes = append(sbTypes, sbt) } } - return &filter + return newSmartblockTypesFilter(m.sbtProvider, false, sbTypes) } func (m *dsObjectStore) QueryAndSubscribeForChanges(schema schema.Schema, q database.Query, sub database.Subscription) (records []database.Record, close func(), total int, err error) { @@ -668,6 +674,11 @@ func (m *dsObjectStore) Query(sch schema.Schema, q database.Query) (records []da dsq.Limit = 0 dsq.Prefix = pagesDetailsBase.String() + "/" if !q.WithSystemObjects { + filterNotSystemObjects := newSmartblockTypesFilter(m.sbtProvider, true, []smartblock.SmartBlockType{ + smartblock.SmartBlockTypeArchive, + smartblock.SmartBlockTypeHome, + }) + dsq.Filters = append([]query.Filter{filterNotSystemObjects}, dsq.Filters...) } @@ -790,7 +801,7 @@ func (m *dsObjectStore) QueryObjectInfo(q database.Query, objectTypes []smartblo dsq.Limit = 0 dsq.Prefix = pagesDetailsBase.String() + "/" if len(objectTypes) > 0 { - dsq.Filters = append([]query.Filter{&filterSmartblockTypes{smartBlockTypes: objectTypes}}, dsq.Filters...) + dsq.Filters = append([]query.Filter{newSmartblockTypesFilter(m.sbtProvider, false, objectTypes)}, dsq.Filters...) } if q.FullText != "" { if dsq, err = m.makeFTSQuery(q.FullText, dsq); err != nil { @@ -854,7 +865,7 @@ func (m *dsObjectStore) QueryObjectIds(q database.Query, objectTypes []smartbloc dsq.Limit = 0 dsq.Prefix = pagesDetailsBase.String() + "/" if len(objectTypes) > 0 { - dsq.Filters = append([]query.Filter{&filterSmartblockTypes{smartBlockTypes: objectTypes}}, dsq.Filters...) + dsq.Filters = append([]query.Filter{newSmartblockTypesFilter(m.sbtProvider, false, objectTypes)}, dsq.Filters...) } if q.FullText != "" { if dsq, err = m.makeFTSQuery(q.FullText, dsq); err != nil { @@ -904,7 +915,7 @@ func (m *dsObjectStore) QueryById(ids []string) (records []database.Record, err defer txn.Discard() for _, id := range ids { - if sbt, err := smartblock.SmartBlockTypeFromID(id); err == nil { + if sbt, err := m.sbtProvider.Type(id); err == nil { if indexDetails, _ := sbt.Indexable(); !indexDetails && m.sourceService != nil { details, err := m.sourceService.GetDetailsFromIdBasedSource(id) if err != nil { @@ -1753,7 +1764,7 @@ func (m *dsObjectStore) updateSnippet(txn noctxds.Txn, id string, snippet string } func (m *dsObjectStore) updateDetails(txn noctxds.Txn, id string, oldDetails *model.ObjectDetails, newDetails *model.ObjectDetails) error { - t, err := smartblock.SmartBlockTypeFromID(id) + t, err := m.sbtProvider.Type(id) if err != nil { log.Errorf("updateDetails: failed to detect smartblock type for %s: %s", id, err.Error()) return fmt.Errorf("updateDetails: failed to detect smartblock type for %s: %s", id, err.Error()) @@ -2021,7 +2032,7 @@ func getObjectTypeFromDetails(det *types.Struct) ([]string, error) { } func (m *dsObjectStore) getObjectInfo(txn noctxds.Txn, id string) (*model.ObjectInfo, error) { - sbt, err := smartblock.SmartBlockTypeFromID(id) + sbt, err := m.sbtProvider.Type(id) if err != nil { log.With("thread", id).Errorf("failed to extract smartblock type %s", id) return nil, ErrNotAnObject diff --git a/space/typeprovider/typeprovider.go b/space/typeprovider/typeprovider.go index e2cb5c838..4506debcc 100644 --- a/space/typeprovider/typeprovider.go +++ b/space/typeprovider/typeprovider.go @@ -3,15 +3,22 @@ package typeprovider import ( "context" "errors" + "strings" + "sync" + "github.com/anytypeio/any-sync/app" "github.com/anytypeio/any-sync/commonspace/object/tree/treechangeproto" + "github.com/globalsign/mgo/bson" + "github.com/gogo/protobuf/proto" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + "go.uber.org/zap" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" "github.com/anytypeio/go-anytype-middleware/space" - "github.com/gogo/protobuf/proto" - "go.uber.org/zap" - "sync" ) const CName = "space.typeprovider" @@ -20,49 +27,98 @@ var log = logging.Logger(CName) var ErrUnknownSmartBlockType = errors.New("error unknown smartblock type") -type ObjectTypeProvider interface { +type SmartBlockTypeProvider interface { app.Component Type(id string) (smartblock.SmartBlockType, error) } -func New() ObjectTypeProvider { - return &objectTypeProvider{} -} - -type objectTypeProvider struct { +type provider struct { sync.Mutex spaceService space.Service cache map[string]smartblock.SmartBlockType } -func (o *objectTypeProvider) Init(a *app.App) (err error) { - o.spaceService = a.MustComponent(space.CName).(space.Service) - o.cache = map[string]smartblock.SmartBlockType{} +func New(spaceService space.Service) SmartBlockTypeProvider { + return &provider{ + spaceService: spaceService, + } +} + +func (p *provider) Init(a *app.App) (err error) { + p.cache = map[string]smartblock.SmartBlockType{} return } -func (o *objectTypeProvider) Name() (name string) { +func (p *provider) Name() (name string) { return CName } -func (o *objectTypeProvider) Type(id string) (tp smartblock.SmartBlockType, err error) { - tp, err = smartblock.SmartBlockTypeFromID(id) +func (p *provider) Type(id string) (tp smartblock.SmartBlockType, err error) { + tp, err = smartBlockTypeFromID(id) if err != nil || tp != smartblock.SmartBlockTypePage { return } - return o.objectTypeFromSpace(id) + return p.objectTypeFromSpace(id) } -func (o *objectTypeProvider) objectTypeFromSpace(id string) (tp smartblock.SmartBlockType, err error) { - o.Lock() - tp, exists := o.cache[id] +func smartBlockTypeFromID(id string) (smartblock.SmartBlockType, error) { + if strings.HasPrefix(id, addr.BundledRelationURLPrefix) { + return smartblock.SmartBlockTypeBundledRelation, nil + } + + if strings.HasPrefix(id, addr.BundledObjectTypeURLPrefix) { + return smartblock.SmartBlockTypeBundledObjectType, nil + } + + if len(strings.Split(id, addr.SubObjectCollectionIdSeparator)) == 2 { + return smartblock.SmartBlockTypeSubObject, nil + } + + // workaround for options that have no prefix + // todo: remove this after migration to the new records format + if bson.IsObjectIdHex(id) { + return smartblock.SmartBlockTypeSubObject, nil + } + + if strings.HasPrefix(id, addr.AnytypeProfileId) { + return smartblock.SmartBlockTypeProfilePage, nil + } + if strings.HasPrefix(id, addr.VirtualPrefix) { + sbt, err := addr.ExtractVirtualSourceType(id) + if err != nil { + return 0, err + } + return smartblock.SmartBlockType(sbt), nil + } + if strings.HasPrefix(id, addr.DatePrefix) { + return smartblock.SmartBlockTypeDate, nil + } + + c, err := cid.Decode(id) + if err != nil { + return smartblock.SmartBlockTypePage, err + } + // TODO: discard this fragile condition as soon as we will move to the multiaddr with prefix + if c.Prefix().Codec == cid.DagProtobuf && c.Prefix().MhType == multihash.SHA2_256 { + return smartblock.SmartBlockTypeFile, nil + } + if c.Prefix().Codec == cid.DagCBOR { + return smartblock.SmartBlockTypePage, nil + } + + return smartblock.SmartBlockTypePage, smartblock.ErrNoSuchSmartblock +} + +func (p *provider) objectTypeFromSpace(id string) (tp smartblock.SmartBlockType, err error) { + p.Lock() + tp, exists := p.cache[id] if exists { - o.Unlock() + p.Unlock() return } - o.Unlock() + p.Unlock() - sp, err := o.spaceService.AccountSpace(context.Background()) + sp, err := p.spaceService.AccountSpace(context.Background()) if err != nil { return } @@ -73,22 +129,22 @@ func (o *objectTypeProvider) objectTypeFromSpace(id string) (tp smartblock.Smart return } - root, err := o.unmarshallRoot(rawRoot) + root, err := p.unmarshallRoot(rawRoot) if err != nil { return } - tp, err = o.objectType(root.ChangeType) + tp, err = p.objectType(root.ChangeType) if err != nil { return } - o.Lock() - defer o.Unlock() - o.cache[id] = tp + p.Lock() + defer p.Unlock() + p.cache[id] = tp return } -func (o *objectTypeProvider) objectType(changeType string) (smartblock.SmartBlockType, error) { +func (p *provider) objectType(changeType string) (smartblock.SmartBlockType, error) { log.With(zap.String("changeType", changeType)).Warn("getting change type") if v, exists := model.SmartBlockType_value[changeType]; exists { return smartblock.SmartBlockType(v), nil @@ -97,7 +153,7 @@ func (o *objectTypeProvider) objectType(changeType string) (smartblock.SmartBloc return smartblock.SmartBlockTypePage, nil } -func (o *objectTypeProvider) unmarshallRoot(rawRoot *treechangeproto.RawTreeChangeWithId) (root *treechangeproto.RootChange, err error) { +func (p *provider) unmarshallRoot(rawRoot *treechangeproto.RawTreeChangeWithId) (root *treechangeproto.RootChange, err error) { raw := &treechangeproto.RawTreeChange{} err = proto.Unmarshal(rawRoot.GetRawChange(), raw) if err != nil { diff --git a/util/builtinobjects/builtinobjects.go b/util/builtinobjects/builtinobjects.go index 0c90bf0b8..bf08052d1 100644 --- a/util/builtinobjects/builtinobjects.go +++ b/util/builtinobjects/builtinobjects.go @@ -4,7 +4,6 @@ import ( "archive/zip" "bytes" "context" - _ "embed" "fmt" "io" "io/ioutil" @@ -12,15 +11,13 @@ import ( "strings" "time" - "github.com/textileio/go-threads/core/thread" - - sb "github.com/anytypeio/go-anytype-middleware/core/block/editor/smartblock" - "github.com/anytypeio/any-sync/app" "github.com/gogo/protobuf/types" + "github.com/textileio/go-threads/core/thread" "github.com/anytypeio/go-anytype-middleware/core/anytype/config" "github.com/anytypeio/go-anytype-middleware/core/block" + sb "github.com/anytypeio/go-anytype-middleware/core/block/editor/smartblock" "github.com/anytypeio/go-anytype-middleware/core/block/editor/state" "github.com/anytypeio/go-anytype-middleware/core/block/simple" "github.com/anytypeio/go-anytype-middleware/core/block/simple/bookmark" @@ -37,7 +34,10 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/space/typeprovider" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" + + _ "embed" ) const CName = "builtinobjects" @@ -54,24 +54,27 @@ const ( injectionTimeout = 30 * time.Second ) -func New() BuiltinObjects { - return new(builtinObjects) -} - type BuiltinObjects interface { app.ComponentRunnable } type builtinObjects struct { - cancel func() - source source.Service - service *block.Service - relService relation2.Service + cancel func() + source source.Service + service *block.Service + relService relation2.Service + sbtProvider typeprovider.SmartBlockTypeProvider createBuiltinObjects bool idsMap map[string]string } +func New(sbtProvider typeprovider.SmartBlockTypeProvider) BuiltinObjects { + return &builtinObjects{ + sbtProvider: sbtProvider, + } +} + func (b *builtinObjects) Init(a *app.App) (err error) { b.source = a.MustComponent(source.CName).(source.Service) b.service = a.MustComponent(block.CName).(*block.Service) @@ -117,7 +120,7 @@ func (b *builtinObjects) inject(ctx context.Context) (err error) { isSpaceDashboardIDFound := false for _, zf := range zr.File { id := strings.TrimSuffix(zf.Name, filepath.Ext(zf.Name)) - sbt, err := smartblock.SmartBlockTypeFromID(id) + sbt, err := b.sbtProvider.Type(id) if err != nil { sbt, err = SmartBlockTypeFromThreadID(id) if err != nil { @@ -192,7 +195,7 @@ func (b *builtinObjects) createObject(ctx context.Context, rd io.ReadCloser) (er st.SetRootId(newId) a := st.Get(newId) m := a.Model() - sbt, err := smartblock.SmartBlockTypeFromID(newId) + sbt, err := b.sbtProvider.Type(newId) if sbt == smartblock.SmartBlockTypeSubObject { ot, err := bundle.TypeKeyFromUrl(pbtypes.GetString(st.CombinedDetails(), bundle.RelationKeyType.String())) if err != nil { diff --git a/util/builtintemplate/builtintemplate.go b/util/builtintemplate/builtintemplate.go index dca5742ed..e0de39bb4 100644 --- a/util/builtintemplate/builtintemplate.go +++ b/util/builtintemplate/builtintemplate.go @@ -5,7 +5,6 @@ import ( "bytes" "context" "crypto/md5" - _ "embed" "encoding/binary" "encoding/hex" "fmt" @@ -13,6 +12,7 @@ import ( "io/ioutil" "github.com/anytypeio/any-sync/app" + "github.com/anytypeio/go-anytype-middleware/core/block/editor/state" "github.com/anytypeio/go-anytype-middleware/core/block/simple" "github.com/anytypeio/go-anytype-middleware/core/block/simple/relation" @@ -23,6 +23,8 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" + + _ "embed" ) const CName = "builtintemplate" @@ -119,10 +121,8 @@ func (b *builtinTemplate) registerBuiltin(rd io.ReadCloser) (err error) { if err = b.validate(st.Copy()); err != nil { return } - log.With("id", id).Info("registering template") - b.source.RegisterStaticSource(id, func() source.Source { - return b.source.NewStaticSource(id, model.SmartBlockType_BundledTemplate, st.Copy(), nil) - }) + + b.source.RegisterStaticSource(id, b.source.NewStaticSource(id, model.SmartBlockType_BundledTemplate, st.Copy(), nil)) return }