diff --git a/.mockery.yaml b/.mockery.yaml index fd4e65282..cc5a96085 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -41,6 +41,9 @@ packages: github.com/anyproto/anytype-heart/core/filestorage/filesync: interfaces: FileSync: + github.com/anyproto/anytype-heart/core/subscription: + interfaces: + CollectionService: github.com/anyproto/anytype-heart/core/block/object/objectcache: interfaces: Cache: diff --git a/core/block/collection/service.go b/core/block/collection/service.go index b3c7653ce..0b9f24a86 100644 --- a/core/block/collection/service.go +++ b/core/block/collection/service.go @@ -261,5 +261,5 @@ func setDefaultObjectTypeToViews(st *state.State) { } func isNotCreatableType(key domain.TypeKey) bool { - return lo.Contains(append(bundle.InternalTypes, bundle.TypeKeyObjectType, bundle.TypeKeySet, bundle.TypeKeyCollection), key) + return lo.Contains(append(bundle.InternalTypes, bundle.TypeKeyObjectType), key) } diff --git a/core/block/collection/service_test.go b/core/block/collection/service_test.go index c6c045194..236a1629b 100644 --- a/core/block/collection/service_test.go +++ b/core/block/collection/service_test.go @@ -196,7 +196,7 @@ func TestSetObjectTypeToViews(t *testing.T) { t.Run("object is a set by not creatable type", func(t *testing.T) { // given - st := generateState(bundle.TypeKeySet, bundle.TypeKeyCollection.URL()) + st := generateState(bundle.TypeKeySet, bundle.TypeKeyObjectType.URL()) // when setDefaultObjectTypeToViews(st) diff --git a/core/block/debug.go b/core/block/debug.go new file mode 100644 index 000000000..4ec4b3a45 --- /dev/null +++ b/core/block/debug.go @@ -0,0 +1,65 @@ +package block + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/gogo/protobuf/jsonpb" + + "github.com/anyproto/anytype-heart/core/block/editor/smartblock" + "github.com/anyproto/anytype-heart/tests/blockbuilder" + "github.com/anyproto/anytype-heart/util/debug" +) + +func (s *Service) DebugRouter(r chi.Router) { + r.Get("/objects", debug.JSONHandler(s.debugObjects)) +} + +type debugObject struct { + ID string + Details json.RawMessage + Store *json.RawMessage + Blocks *blockbuilder.Block +} + +func (s *Service) debugObjects(req *http.Request) ([]debugObject, error) { + ids, err := s.objectStore.ListIds() + if err != nil { + return nil, fmt.Errorf("list ids: %w", err) + } + result := make([]debugObject, 0, len(ids)) + marshaller := jsonpb.Marshaler{} + for _, id := range ids { + err = Do(s, id, func(sb smartblock.SmartBlock) error { + st := sb.NewState() + root := blockbuilder.BuildAST(st.Blocks()) + detailsRaw, err := marshaller.MarshalToString(st.CombinedDetails()) + if err != nil { + return fmt.Errorf("marshal details: %w", err) + } + + var storeRaw *json.RawMessage + if store := st.Store(); store != nil { + raw, err := marshaller.MarshalToString(st.Store()) + if err != nil { + return fmt.Errorf("marshal store: %w", err) + } + rawMessage := json.RawMessage(raw) + storeRaw = &rawMessage + } + result = append(result, debugObject{ + ID: id, + Store: storeRaw, + Details: json.RawMessage(detailsRaw), + Blocks: root, + }) + return nil + }) + if err != nil { + return nil, fmt.Errorf("can't get object %s: %w", id, err) + } + } + return result, nil +} diff --git a/core/block/editor/factory.go b/core/block/editor/factory.go index 0476b6604..78164a4fe 100644 --- a/core/block/editor/factory.go +++ b/core/block/editor/factory.go @@ -53,7 +53,6 @@ type ObjectFactory struct { sbtProvider typeprovider.SmartBlockTypeProvider sourceService source.Service tempDirProvider core.TempDirProvider - templateCloner templateCloner fileService files.Service config *config.Config picker getblock.ObjectGetter @@ -77,7 +76,6 @@ func (f *ObjectFactory) Init(a *app.App) (err error) { f.systemObjectService = app.MustComponent[system_object.Service](a) f.restrictionService = app.MustComponent[restriction.Service](a) f.sourceService = app.MustComponent[source.Service](a) - f.templateCloner = app.MustComponent[templateCloner](a) f.fileService = app.MustComponent[files.Service](a) f.config = app.MustComponent[*config.Config](a) f.tempDirProvider = app.MustComponent[core.TempDirProvider](a) @@ -229,7 +227,6 @@ func (f *ObjectFactory) New(sbType coresb.SmartBlockType) (smartblock.SmartBlock f.detailsModifier, f.sbtProvider, f.layoutConverter, - f.templateCloner, f.config, f.eventSender, f.objectDeriver, diff --git a/core/block/editor/page.go b/core/block/editor/page.go index 67364bd09..99d0cd8b3 100644 --- a/core/block/editor/page.go +++ b/core/block/editor/page.go @@ -193,13 +193,7 @@ func (p *Page) CreationStateMigration(ctx *smartblock.InitContext) migration.Mig } func (p *Page) StateMigrations() migration.Migrations { - dataviewMigration := newSubObjectsLinksMigration(p.SpaceID(), p.systemObjectService) - return migration.MakeMigrations([]migration.Migration{ - { - Version: 2, - Proc: dataviewMigration.migrate, - }, - }) + return migration.MakeMigrations(nil) } func GetDefaultViewRelations(rels []*model.Relation) []*model.BlockContentDataviewRelation { diff --git a/core/block/editor/smartblock/smartblock.go b/core/block/editor/smartblock/smartblock.go index b258b0b6f..31e9d1102 100644 --- a/core/block/editor/smartblock/smartblock.go +++ b/core/block/editor/smartblock/smartblock.go @@ -656,7 +656,7 @@ func (sb *smartBlock) Apply(s *state.State, flags ...ApplyFlag) (err error) { lastModified = time.Unix(pbtypes.GetInt64(s.LocalDetails(), bundle.RelationKeyLastModifiedDate.String()), 0) } } - sb.onApply(s) + sb.beforeStateApply(s) // this one will be reverted in case we don't have any actual change being made s.SetLastModified(lastModified.Unix(), sb.coreService.PredefinedObjects(sb.SpaceID()).Profile) @@ -1303,7 +1303,7 @@ func (sb *smartBlock) runIndexer(s *state.State, opts ...IndexOption) { } } -func (sb *smartBlock) onApply(s *state.State) { +func (sb *smartBlock) beforeStateApply(s *state.State) { flags := internalflag.NewFromState(s) // Run empty check only if any of these flags are present @@ -1323,14 +1323,16 @@ func (sb *smartBlock) onApply(s *state.State) { } func (sb *smartBlock) setRestrictionsDetail(s *state.State) { - var ints = make([]int, len(sb.Restrictions().Object)) - for i, v := range sb.Restrictions().Object { - ints[i] = int(v) + rawRestrictions := make([]int, len(sb.Restrictions().Object)) + for i, r := range sb.Restrictions().Object { + rawRestrictions[i] = int(r) } - s.SetLocalDetail(bundle.RelationKeyRestrictions.String(), pbtypes.IntList(ints...)) + s.SetLocalDetail(bundle.RelationKeyRestrictions.String(), pbtypes.IntList(rawRestrictions...)) // todo: verify this logic with clients - if sb.Restrictions().Object.Check(model.Restrictions_Details) != nil && sb.Restrictions().Object.Check(model.Restrictions_Blocks) != nil { + if sb.Restrictions().Object.Check(model.Restrictions_Details) != nil && + sb.Restrictions().Object.Check(model.Restrictions_Blocks) != nil { + s.SetDetailAndBundledRelation(bundle.RelationKeyIsReadonly, pbtypes.Bool(true)) } } diff --git a/core/block/editor/sub_objects_migration.go b/core/block/editor/sub_objects_migration.go index 323df60b3..55f2e0316 100644 --- a/core/block/editor/sub_objects_migration.go +++ b/core/block/editor/sub_objects_migration.go @@ -1 +1,149 @@ package editor + +import ( + "context" + "fmt" + + "github.com/gogo/protobuf/types" + + "github.com/anyproto/anytype-heart/core/block/editor/smartblock" + "github.com/anyproto/anytype-heart/core/block/editor/state" + "github.com/anyproto/anytype-heart/core/block/object/objectcache" + "github.com/anyproto/anytype-heart/core/domain" + "github.com/anyproto/anytype-heart/pkg/lib/bundle" + smartblock2 "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" + "github.com/anyproto/anytype-heart/util/pbtypes" +) + +type objectDeriver interface { + DeriveTreeObject(ctx context.Context, spaceID string, params objectcache.TreeDerivationParams) (sb smartblock.SmartBlock, err error) +} + +// Migrate legacy sub-objects to ordinary objects +type subObjectsMigration struct { + workspace *Workspaces + objectDeriver objectDeriver +} + +func (m *subObjectsMigration) migrateSubObjects(st *state.State) { + m.iterateAllSubObjects( + st, + func(info smartblock.DocInfo) { + uniqueKeyRaw := pbtypes.GetString(info.Details, bundle.RelationKeyUniqueKey.String()) + id, err := m.migrateSubObject(context.Background(), uniqueKeyRaw, info.Details, info.Type) + if err != nil { + log.Errorf("failed to index subobject %s: %s", info.Id, err) + log.With("objectID", id).Errorf("failed to migrate subobject: %v", err) + } else { + log.With("objectId", id, "uniqueKey", uniqueKeyRaw).Warnf("migrated sub-object") + } + }, + ) +} + +func (m *subObjectsMigration) migrateSubObject( + ctx context.Context, + uniqueKeyRaw string, + details *types.Struct, + typeKey domain.TypeKey, +) (id string, err error) { + uniqueKey, err := domain.UnmarshalUniqueKey(uniqueKeyRaw) + if err != nil { + return "", fmt.Errorf("unmarshal unique key: %w", err) + } + sb, err := m.objectDeriver.DeriveTreeObject(ctx, m.workspace.SpaceID(), objectcache.TreeDerivationParams{ + Key: uniqueKey, + InitFunc: func(id string) *smartblock.InitContext { + st := state.NewDocWithUniqueKey(id, nil, uniqueKey).NewState() + st.SetDetails(details) + st.SetObjectTypeKey(typeKey) + return &smartblock.InitContext{ + IsNewObject: true, + State: st, + SpaceID: m.workspace.SpaceID(), + } + }, + }) + if err != nil { + return "", err + } + + return sb.Id(), nil +} + +const ( + collectionKeyRelationOptions = "opt" + collectionKeyRelations = "rel" + collectionKeyObjectTypes = "ot" +) + +var objectTypeToCollection = map[domain.TypeKey]string{ + bundle.TypeKeyObjectType: collectionKeyObjectTypes, + bundle.TypeKeyRelation: collectionKeyRelations, + bundle.TypeKeyRelationOption: collectionKeyRelationOptions, +} + +func collectionKeyToTypeKey(collKey string) (domain.TypeKey, bool) { + for ot, v := range objectTypeToCollection { + if v == collKey { + return ot, true + } + } + return "", false +} + +func (m *subObjectsMigration) iterateAllSubObjects(st *state.State, proc func(smartblock.DocInfo)) { + for typeKey, coll := range objectTypeToCollection { + collection := st.GetSubObjectCollection(coll) + if collection == nil { + continue + } + + for subObjectId, subObjectStruct := range collection.GetFields() { + if v, ok := subObjectStruct.Kind.(*types.Value_StructValue); ok { + uk, err := m.getUniqueKey(coll, subObjectId) + if err != nil { + log.With("collection", coll).Errorf("subobject migration: failed to get uniqueKey: %s", err.Error()) + continue + } + + details := v.StructValue + details.Fields[bundle.RelationKeyUniqueKey.String()] = pbtypes.String(uk.Marshal()) + + proc(smartblock.DocInfo{ + SpaceID: m.workspace.SpaceID(), + Links: nil, + FileHashes: nil, + Heads: nil, + Type: typeKey, + Details: details, + }) + } else { + log.Errorf("got invalid value for %s.%s:%t", coll, subObjectId, subObjectStruct.Kind) + continue + } + } + } + return +} + +func (m *subObjectsMigration) getUniqueKey(collection, key string) (domain.UniqueKey, error) { + typeKey, ok := collectionKeyToTypeKey(collection) + if !ok { + return nil, fmt.Errorf("unknown collection %s", collection) + } + + var sbt smartblock2.SmartBlockType + switch typeKey { + case bundle.TypeKeyRelation: + sbt = smartblock2.SmartBlockTypeRelation + case bundle.TypeKeyObjectType: + sbt = smartblock2.SmartBlockTypeObjectType + case bundle.TypeKeyRelationOption: + sbt = smartblock2.SmartBlockTypeRelationOption + default: + return nil, fmt.Errorf("unknown type key %s", typeKey) + } + + return domain.NewUniqueKey(sbt, key) +} diff --git a/core/block/editor/widget.go b/core/block/editor/widget.go index 0ef3e9b17..8caf59eab 100644 --- a/core/block/editor/widget.go +++ b/core/block/editor/widget.go @@ -99,13 +99,7 @@ func (w *WidgetObject) withDefaultWidgets(st *state.State) { } func (w *WidgetObject) StateMigrations() migration.Migrations { - dataviewMigration := newSubObjectsLinksMigration(w.SpaceID(), w.systemObjectService) - return migration.MakeMigrations([]migration.Migration{ - { - Version: 2, - Proc: dataviewMigration.migrate, - }, - }) + return migration.MakeMigrations(nil) } func (w *WidgetObject) Unlink(ctx session.Context, ids ...string) (err error) { diff --git a/core/block/editor/workspaces.go b/core/block/editor/workspaces.go index 29347ec93..5ba8b9829 100644 --- a/core/block/editor/workspaces.go +++ b/core/block/editor/workspaces.go @@ -1,11 +1,6 @@ package editor import ( - "context" - "fmt" - - "github.com/gogo/protobuf/types" - "github.com/anyproto/anytype-heart/core/anytype/config" "github.com/anyproto/anytype-heart/core/block/editor/basic" "github.com/anyproto/anytype-heart/core/block/editor/converter" @@ -15,7 +10,6 @@ import ( "github.com/anyproto/anytype-heart/core/block/editor/stext" "github.com/anyproto/anytype-heart/core/block/editor/template" "github.com/anyproto/anytype-heart/core/block/migration" - "github.com/anyproto/anytype-heart/core/block/object/objectcache" "github.com/anyproto/anytype-heart/core/block/source" "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/event" @@ -23,7 +17,6 @@ import ( "github.com/anyproto/anytype-heart/metrics" "github.com/anyproto/anytype-heart/pkg/lib/bundle" "github.com/anyproto/anytype-heart/pkg/lib/core" - smartblock2 "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" "github.com/anyproto/anytype-heart/space/spacecore/typeprovider" @@ -38,7 +31,6 @@ type Workspaces struct { stext.Text DetailsModifier DetailsModifier - templateCloner templateCloner sourceService source.Service anytype core.Service objectStore objectstore.ObjectStore @@ -55,7 +47,6 @@ func NewWorkspace( modifier DetailsModifier, sbtProvider typeprovider.SmartBlockTypeProvider, layoutConverter converter.LayoutConverter, - templateCloner templateCloner, config *config.Config, eventSender event.Sender, objectDeriver objectDeriver, @@ -80,16 +71,11 @@ func NewWorkspace( anytype: anytype, objectStore: objectStore, sourceService: sourceService, - templateCloner: templateCloner, config: config, objectDeriver: objectDeriver, } } -type objectDeriver interface { - DeriveTreeObject(ctx context.Context, spaceID string, params objectcache.TreeDerivationParams) (sb smartblock.SmartBlock, err error) -} - func (w *Workspaces) Init(ctx *smartblock.InitContext) (err error) { err = w.SmartBlock.Init(ctx) if err != nil { @@ -97,6 +83,12 @@ func (w *Workspaces) Init(ctx *smartblock.InitContext) (err error) { } w.initTemplate(ctx) + subObjectMigration := subObjectsMigration{ + workspace: w, + objectDeriver: w.objectDeriver, + } + subObjectMigration.migrateSubObjects(ctx.State) + return nil } @@ -122,19 +114,6 @@ func (w *Workspaces) initTemplate(ctx *smartblock.InitContext) { ) } -type templateCloner interface { - TemplateClone(spaceID string, id string) (templateID string, err error) -} - -type WorkspaceParameters struct { - IsHighlighted bool - WorkspaceId string -} - -func (wp *WorkspaceParameters) Equal(other *WorkspaceParameters) bool { - return wp.IsHighlighted == other.IsHighlighted -} - func (w *Workspaces) CreationStateMigration(ctx *smartblock.InitContext) migration.Migration { // TODO Maybe move init logic here? return migration.Migration{ @@ -146,141 +125,5 @@ func (w *Workspaces) CreationStateMigration(ctx *smartblock.InitContext) migrati } func (w *Workspaces) StateMigrations() migration.Migrations { - return migration.MakeMigrations([]migration.Migration{ - { - Version: 1, - Proc: w.migrateSubObjects, - }, - }) -} - -func (w *Workspaces) migrateSubObjects(_ *state.State) { - w.iterateAllSubObjects( - func(info smartblock.DocInfo) bool { - uniqueKeyRaw := pbtypes.GetString(info.Details, bundle.RelationKeyUniqueKey.String()) - id, err := w.migrateSubObject(context.Background(), uniqueKeyRaw, info.Details, info.Type) - if err != nil { - log.Errorf("failed to index subobject %s: %s", info.Id, err) - log.With("objectID", id).Errorf("failed to migrate subobject: %v", err) - return true - } - log.With("objectId", id, "uniqueKey", uniqueKeyRaw).Warnf("migrated sub-object") - return true - }, - ) -} - -func (w *Workspaces) migrateSubObject( - ctx context.Context, - uniqueKeyRaw string, - details *types.Struct, - typeKey domain.TypeKey, -) (id string, err error) { - uniqueKey, err := domain.UnmarshalUniqueKey(uniqueKeyRaw) - if err != nil { - return "", fmt.Errorf("unmarshal unique key: %w", err) - } - sb, err := w.objectDeriver.DeriveTreeObject(ctx, w.SpaceID(), objectcache.TreeDerivationParams{ - Key: uniqueKey, - InitFunc: func(id string) *smartblock.InitContext { - st := state.NewDocWithUniqueKey(id, nil, uniqueKey).NewState() - st.SetDetails(details) - st.SetObjectTypeKey(typeKey) - return &smartblock.InitContext{ - IsNewObject: true, - State: st, - SpaceID: w.SpaceID(), - } - }, - }) - if err != nil { - return "", err - } - - return sb.Id(), nil -} - -const ( - collectionKeyRelationOptions = "opt" - collectionKeyRelations = "rel" - collectionKeyObjectTypes = "ot" -) - -var objectTypeToCollection = map[domain.TypeKey]string{ - bundle.TypeKeyObjectType: collectionKeyObjectTypes, - bundle.TypeKeyRelation: collectionKeyRelations, - bundle.TypeKeyRelationOption: collectionKeyRelationOptions, -} - -func collectionKeyToTypeKey(collKey string) (domain.TypeKey, bool) { - for ot, v := range objectTypeToCollection { - if v == collKey { - return ot, true - } - } - return "", false -} - -func (w *Workspaces) iterateAllSubObjects(proc func(smartblock.DocInfo) bool) { - st := w.NewState() - for _, coll := range objectTypeToCollection { - data := st.GetSubObjectCollection(coll) - if data == nil { - continue - } - tk, ok := collectionKeyToTypeKey(coll) - if !ok { - log.With("collection", coll).Errorf("subobject migration: collection is invalid") - continue - } - - for subId, d := range data.GetFields() { - if st, ok := d.Kind.(*types.Value_StructValue); !ok { - log.Errorf("got invalid value for %s.%s:%t", coll, subId, d.Kind) - continue - } else { - uk, err := w.getUniqueKey(coll, subId) - if err != nil { - log.With("collection", coll).Errorf("subobject migration: failed to get uniqueKey: %s", err.Error()) - continue - } - - d := st.StructValue - d.Fields[bundle.RelationKeyUniqueKey.String()] = pbtypes.String(uk.Marshal()) - - if !proc(smartblock.DocInfo{ - SpaceID: w.SpaceID(), - Links: nil, - FileHashes: nil, - Heads: nil, - Type: tk, - Details: d, - }) { - return - } - } - } - } - return -} - -func (w *Workspaces) getUniqueKey(collection, key string) (domain.UniqueKey, error) { - typeKey, ok := collectionKeyToTypeKey(collection) - if !ok { - return nil, fmt.Errorf("unknown collection %s", collection) - } - - var sbt smartblock2.SmartBlockType - switch typeKey { - case bundle.TypeKeyRelation: - sbt = smartblock2.SmartBlockTypeRelation - case bundle.TypeKeyObjectType: - sbt = smartblock2.SmartBlockTypeObjectType - case bundle.TypeKeyRelationOption: - sbt = smartblock2.SmartBlockTypeRelationOption - default: - return nil, fmt.Errorf("unknown collection %s", collection) - } - - return domain.NewUniqueKey(sbt, key) + return migration.MakeMigrations(nil) } diff --git a/core/block/object/treesyncer/treesyncer.go b/core/block/object/treesyncer/treesyncer.go index 5dad55369..22a9019c8 100644 --- a/core/block/object/treesyncer/treesyncer.go +++ b/core/block/object/treesyncer/treesyncer.go @@ -12,7 +12,6 @@ import ( "github.com/anyproto/any-sync/commonspace/object/treesyncer" "github.com/anyproto/any-sync/net/peer" "github.com/anyproto/any-sync/net/streampool" - "go.uber.org/zap" ) diff --git a/core/block/simple/bookmark/bookmark.go b/core/block/simple/bookmark/bookmark.go index cf099e410..503b9ab44 100644 --- a/core/block/simple/bookmark/bookmark.go +++ b/core/block/simple/bookmark/bookmark.go @@ -176,15 +176,10 @@ func (b *Bookmark) FillFileHashes(hashes []string) []string { return hashes } -func (l *Bookmark) ReplaceSmartIds(f func(id string) (newId string, replaced bool)) (anyReplaced bool) { +func (l *Bookmark) ReplaceLinkIds(replacer func(oldId string) (newId string)) { if l.content.TargetObjectId != "" { - newId, replaced := f(l.content.TargetObjectId) - if replaced { - l.content.TargetObjectId = newId - anyReplaced = true - } + l.content.TargetObjectId = replacer(l.content.TargetObjectId) } - return } diff --git a/core/block/simple/dataview/dataview.go b/core/block/simple/dataview/dataview.go index efdfefad7..e19cc1faf 100644 --- a/core/block/simple/dataview/dataview.go +++ b/core/block/simple/dataview/dataview.go @@ -226,15 +226,10 @@ func (l *Dataview) FillSmartIds(ids []string) []string { return ids } -func (l *Dataview) ReplaceSmartIds(f func(id string) (newId string, replaced bool)) (anyReplaced bool) { +func (l *Dataview) ReplaceLinkIds(replacer func(oldId string) (newId string)) { if l.content.TargetObjectId != "" { - newId, replaced := f(l.content.TargetObjectId) - if replaced { - l.content.TargetObjectId = newId - return true - } + l.content.TargetObjectId = replacer(l.content.TargetObjectId) } - return } diff --git a/core/block/simple/link/link.go b/core/block/simple/link/link.go index 70238cbd7..2899ea0bc 100644 --- a/core/block/simple/link/link.go +++ b/core/block/simple/link/link.go @@ -114,15 +114,10 @@ func (l *Link) Diff(b simple.Block) (msgs []simple.EventMessage, err error) { return } -func (l *Link) ReplaceSmartIds(f func(id string) (newId string, replaced bool)) (anyReplaced bool) { +func (l *Link) ReplaceLinkIds(replacer func(oldId string) (newId string)) { if l.content.TargetBlockId != "" { - newId, replaced := f(l.content.TargetBlockId) - if replaced { - l.content.TargetBlockId = newId - return true - } + l.content.TargetBlockId = replacer(l.content.TargetBlockId) } - return } diff --git a/core/block/simple/simple.go b/core/block/simple/simple.go index 1179e742f..e7fec80d2 100644 --- a/core/block/simple/simple.go +++ b/core/block/simple/simple.go @@ -49,7 +49,7 @@ type DetailsHandler interface { } type ObjectLinkReplacer interface { - ReplaceSmartIds(f func(id string) (newId string, replaced bool)) (anyReplaced bool) + ReplaceLinkIds(replacer func(oldId string) (newId string)) } type EventMessage struct { diff --git a/core/block/simple/text/text.go b/core/block/simple/text/text.go index 2777e83bf..fa6db5c83 100644 --- a/core/block/simple/text/text.go +++ b/core/block/simple/text/text.go @@ -651,16 +651,13 @@ func (t *Text) FillSmartIds(ids []string) []string { return ids } -func (t *Text) ReplaceSmartIds(f func(id string) (newId string, replaced bool)) (anyReplaced bool) { +func (t *Text) ReplaceLinkIds(replacer func(oldId string) (newId string)) { if t.content.Marks != nil { for _, m := range t.content.Marks.Marks { if (m.Type == model.BlockContentTextMark_Mention || m.Type == model.BlockContentTextMark_Object) && m.Param != "" { - newId, replaced := f(m.Param) - if replaced { - m.Param = newId - anyReplaced = true - } + + m.Param = replacer(m.Param) } } } diff --git a/core/block/source/source.go b/core/block/source/source.go index 3b8e41c90..35bde9a92 100644 --- a/core/block/source/source.go +++ b/core/block/source/source.go @@ -214,6 +214,15 @@ func (s *source) buildState() (doc state.Doc, err error) { } st.BlocksInit(st) + // This is temporary migration. We will move it to persistent migration later after several releases. + // The reason is to minimize the number of glitches for users of both old and new versions of Anytype. + // For example, if we persist this migration for Dataview block now, user will see "No query selected" + // error in the old version of Anytype. We want to avoid this as much as possible by making this migration + // temporary, though the applying change to this Dataview block will persist this migration, breaking backward + // compatibility. But in many cases we expect that users update object not so often as they just view them. + migration := newSubObjectsLinksMigration(s.spaceID, s.systemObjectService) + migration.migrate(st) + s.changesSinceSnapshot = changesAppliedSinceSnapshot // TODO: check if we can leave only removeDuplicates instead of Normalize if err = st.Normalize(false); err != nil { diff --git a/core/block/editor/sub_object_links_migration.go b/core/block/source/sub_object_links_migration.go similarity index 70% rename from core/block/editor/sub_object_links_migration.go rename to core/block/source/sub_object_links_migration.go index 8135a9934..8cf3ee323 100644 --- a/core/block/editor/sub_object_links_migration.go +++ b/core/block/source/sub_object_links_migration.go @@ -1,17 +1,20 @@ -package editor +package source import ( "context" "fmt" + "github.com/globalsign/mgo/bson" + "github.com/anyproto/anytype-heart/core/block/editor/state" "github.com/anyproto/anytype-heart/core/block/simple" dataview2 "github.com/anyproto/anytype-heart/core/block/simple/dataview" + "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/core/system_object" "github.com/anyproto/anytype-heart/pkg/lib/bundle" + "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" "github.com/anyproto/anytype-heart/util/pbtypes" - "github.com/anyproto/anytype-heart/core/domain" ) // Migrate old relation (rel-name, etc.) and object type (ot-page, etc.) IDs to new ones (just ordinary object IDs) @@ -28,22 +31,20 @@ func newSubObjectsLinksMigration(spaceID string, systemObjectService system_obje } } -// TODO Refactor func (m *subObjectsLinksMigration) replaceSubObjectLinksInDetails(s *state.State) { for _, rel := range s.GetRelationLinks() { - if rel.Format == model.RelationFormat_object || rel.Format == model.RelationFormat_tag || rel.Format == model.RelationFormat_status { - vals := pbtypes.GetStringList(s.Details(), rel.Key) + if m.canRelationContainObjectValues(rel.Format) { + ids := pbtypes.GetStringList(s.Details(), rel.Key) changed := false - for i := range vals { - newId, replaced := m.migrateSubObjectId(vals[i]) - if !replaced { - continue + for i, oldId := range ids { + newId := m.migrateSubObjectId(oldId) + if oldId != newId { + ids[i] = newId + changed = true } - vals[i] = newId - changed = true } if changed { - s.SetDetail(rel.Key, pbtypes.StringList(vals)) + s.SetDetail(rel.Key, pbtypes.StringList(ids)) } } } @@ -60,30 +61,45 @@ func (m *subObjectsLinksMigration) migrate(s *state.State) { m.migrateFilters(dv) } - if v, ok := block.(simple.ObjectLinkReplacer); ok { - // TODO Analyze this method (ReplaceSmartIds) - // TODO Looks like we should just map here: oldId OR newId - v.ReplaceSmartIds(m.migrateSubObjectId) + if _, ok := block.(simple.ObjectLinkReplacer); ok { + // Mark block as mutable + b := s.Get(block.Model().Id) + replacer := b.(simple.ObjectLinkReplacer) + replacer.ReplaceLinkIds(m.migrateSubObjectId) } return true }) } -func (m *subObjectsLinksMigration) migrateSubObjectId(id string) (newID string, migrated bool) { - // this should be replaced by the persisted state migration - // TODO Smells like SubObjectIdToUniqueKey should be here in-place, not in domain package! - uk, valid := domain.SubObjectIdToUniqueKey(id) +func (m *subObjectsLinksMigration) migrateSubObjectId(oldId string) (newId string) { + uniqueKey, valid := subObjectIdToUniqueKey(oldId) if !valid { - return "", false + return oldId } - newID, err := m.systemObjectService.GetObjectIdByUniqueKey(context.Background(), m.spaceID, uk) + newId, err := m.systemObjectService.GetObjectIdByUniqueKey(context.Background(), m.spaceID, uniqueKey) if err != nil { - log.With("uk", uk.Marshal()).Errorf("failed to derive id: %s", err.Error()) - return "", false + log.With("uniqueKey", uniqueKey.Marshal()).Errorf("failed to derive id: %s", err) + return oldId } - return newID, true + return newId +} + +// subObjectIdToUniqueKey converts legacy sub-object id to uniqueKey +// if id is not supported subObjectId, it will return nil, false +// suppose to be used only for migration and almost free to use +func subObjectIdToUniqueKey(id string) (uniqueKey domain.UniqueKey, valid bool) { + // historically, we don't have the prefix for the options, + // so we need to handled it this ugly way + if bson.IsObjectIdHex(id) { + return domain.MustUniqueKey(smartblock.SmartBlockTypeRelationOption, id), true + } + uniqueKey, err := domain.UnmarshalUniqueKey(id) + if err != nil { + return nil, false + } + return uniqueKey, true } func (m *subObjectsLinksMigration) migrateFilters(dv dataview2.Block) { @@ -103,7 +119,7 @@ func (m *subObjectsLinksMigration) migrateFilter(filter *model.BlockContentDatav return fmt.Errorf("failed to get relation by key %s: %w", filter.RelationKey, err) } - if m.canRelationContainObjectValues(relation) { + if m.canRelationContainObjectValues(relation.Format) { if oldID := filter.Value.GetStringValue(); oldID != "" { newID, err := m.migrateID(oldID) if err != nil { @@ -163,13 +179,12 @@ func (m *subObjectsLinksMigration) migrateID(id string) (string, error) { return id, nil } -func (m *subObjectsLinksMigration) canRelationContainObjectValues(relation *model.Relation) bool { - switch relation.Format { +func (m *subObjectsLinksMigration) canRelationContainObjectValues(format model.RelationFormat) bool { + switch format { case model.RelationFormat_status, model.RelationFormat_tag, - model.RelationFormat_object, - model.RelationFormat_relations: + model.RelationFormat_object: return true default: return false diff --git a/core/domain/uniquekey_test.go b/core/block/source/sub_object_links_migration_test.go similarity index 87% rename from core/domain/uniquekey_test.go rename to core/block/source/sub_object_links_migration_test.go index 27059a964..baa4fbc12 100644 --- a/core/domain/uniquekey_test.go +++ b/core/block/source/sub_object_links_migration_test.go @@ -1,7 +1,9 @@ -package domain +package source import ( "testing" + + "github.com/anyproto/anytype-heart/core/domain" ) func TestSubObjectIdToUniqueKey(t *testing.T) { @@ -25,7 +27,7 @@ func TestSubObjectIdToUniqueKey(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotUk, gotValid := SubObjectIdToUniqueKey(tt.args.id) + gotUk, gotValid := subObjectIdToUniqueKey(tt.args.id) if gotValid != tt.wantValid { t.Errorf("SubObjectIdToUniqueKey() gotValid = %v, want %v", gotValid, tt.wantValid) t.Fail() @@ -35,7 +37,7 @@ func TestSubObjectIdToUniqueKey(t *testing.T) { return } - wantUk, err := UnmarshalUniqueKey(tt.wantUk) + wantUk, err := domain.UnmarshalUniqueKey(tt.wantUk) if err != nil { t.Errorf("SubObjectIdToUniqueKey() error = %v", err) t.Fail() diff --git a/core/debug/handler.go b/core/debug/handler.go new file mode 100644 index 000000000..1a46e0e58 --- /dev/null +++ b/core/debug/handler.go @@ -0,0 +1,9 @@ +//go:build !anydebug + +package debug + +import "github.com/anyproto/any-sync/app" + +func (d *debug) initHandlers(a *app.App) { + // no-op +} diff --git a/core/debug/handler_anydebug.go b/core/debug/handler_anydebug.go new file mode 100644 index 000000000..45e8fc1c6 --- /dev/null +++ b/core/debug/handler_anydebug.go @@ -0,0 +1,35 @@ +//go:build anydebug + +package debug + +import ( + "fmt" + "net/http" + "os" + + "github.com/anyproto/any-sync/app" + "github.com/go-chi/chi/v5" +) + +func (d *debug) initHandlers(a *app.App) { + if addr, ok := os.LookupEnv("ANYDEBUG"); ok && addr != "" { + r := chi.NewRouter() + a.IterateComponents(func(c app.Component) { + if d, ok := c.(Debuggable); ok { + fmt.Println("debug router registered for component: ", c.Name()) + r.Route("/debug/"+c.Name(), d.DebugRouter) + } + }) + routes := r.Routes() + r.Get("/debug", func(w http.ResponseWriter, req *http.Request) { + err := renderLinksList(w, "/", routes) + if err != nil { + logger.Error("failed to render links list", err) + } + }) + d.server = &http.Server{ + Addr: addr, + Handler: r, + } + } +} diff --git a/core/debug/service.go b/core/debug/service.go index 91ccae303..d1e81ab85 100644 --- a/core/debug/service.go +++ b/core/debug/service.go @@ -55,26 +55,7 @@ func (d *debug) Init(a *app.App) (err error) { d.block = a.MustComponent(block.CName).(*block.Service) d.spaceService = app.MustComponent[spacecore.SpaceCoreService](a) - if addr, ok := os.LookupEnv("ANYDEBUG"); ok && addr != "" { - r := chi.NewRouter() - a.IterateComponents(func(c app.Component) { - if d, ok := c.(Debuggable); ok { - fmt.Println("debug router registered for component: ", c.Name()) - r.Route("/debug/"+c.Name(), d.DebugRouter) - } - }) - routes := r.Routes() - r.Get("/debug", func(w http.ResponseWriter, req *http.Request) { - err := renderLinksList(w, "/", routes) - if err != nil { - logger.Error("failed to render links list", err) - } - }) - d.server = &http.Server{ - Addr: addr, - Handler: r, - } - } + d.initHandlers(a) return nil } diff --git a/core/domain/uniquekey.go b/core/domain/uniquekey.go index 783c07748..72ede6080 100644 --- a/core/domain/uniquekey.go +++ b/core/domain/uniquekey.go @@ -5,8 +5,6 @@ import ( "fmt" "strings" - "github.com/globalsign/mgo/bson" - "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" ) @@ -108,20 +106,3 @@ func GetTypeKeyFromRawUniqueKey(raw string) (TypeKey, error) { } return TypeKey(uk.InternalKey()), nil } - -// SubObjectIdToUniqueKey converts legacy subobject id to uniqueKey -// if id is not supported subObjectId, it will return nil, false -// suppose to be used only for migration and almost free to use -func SubObjectIdToUniqueKey(id string) (uk UniqueKey, valid bool) { - if bson.IsObjectIdHex(id) { - // historically, we don't have the prefix for the options - // so we need to handled it this ugly way - id = smartBlockTypeToKey[smartblock.SmartBlockTypeRelationOption] + uniqueKeySeparator + id - } - uk, err := UnmarshalUniqueKey(id) - if err != nil { - return nil, false - } - - return uk, true -} diff --git a/core/files/files.go b/core/files/files.go index c69beb91c..31e151aa5 100644 --- a/core/files/files.go +++ b/core/files/files.go @@ -142,9 +142,19 @@ func (s *service) fileAdd(ctx context.Context, spaceID string, opts AddOptions) return "", nil, err } + err = s.storeFileSize(spaceID, nodeHash) + if err != nil { + return "", nil, fmt.Errorf("store file size: %w", err) + } + return nodeHash, fileInfo, nil } +func (s *service) storeFileSize(spaceId string, hash string) error { + _, err := s.fileSync.CalculateFileSize(context.Background(), spaceId, hash) + return err +} + // fileRestoreKeys restores file path=>key map from the IPFS DAG using the keys in the localStore func (s *service) fileRestoreKeys(ctx context.Context, id domain.FullID) (map[string]string, error) { dagService := s.dagServiceForSpace(id.SpaceID) diff --git a/core/files/images.go b/core/files/images.go index ee52797aa..5b62efe86 100644 --- a/core/files/images.go +++ b/core/files/images.go @@ -24,6 +24,9 @@ func (s *service) ImageByHash(ctx context.Context, id domain.FullID) (Image, err return nil, err } + // TODO Can we use FileByHash here? FileByHash contains important syncing logic. Yes, we use FileByHash before ImageByHash + // but it doesn't seem to be clear why we repeat file indexing process here + // check the image files count explicitly because we have a bug when the info can be cached not fully(only for some files) if len(files) < 4 || files[0].MetaHash == "" { // index image files info from ipfs @@ -114,5 +117,11 @@ func (s *service) imageAdd(ctx context.Context, spaceID string, opts AddOptions) variantsByWidth[int(v.GetNumberValue())] = f } } + + err = s.storeFileSize(spaceID, nodeHash) + if err != nil { + return "", nil, fmt.Errorf("store file size: %w", err) + } + return nodeHash, variantsByWidth, nil } diff --git a/core/filestorage/filesync/filestore_mock.go b/core/filestorage/filesync/filestore_mock.go index 33138afba..831d9dff1 100644 --- a/core/filestorage/filesync/filestore_mock.go +++ b/core/filestorage/filesync/filestore_mock.go @@ -211,6 +211,21 @@ func (mr *MockFileStoreMockRecorder) GetFileKeys(arg0 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFileKeys", reflect.TypeOf((*MockFileStore)(nil).GetFileKeys), arg0) } +// GetFileSize mocks base method. +func (m *MockFileStore) GetFileSize(arg0 string) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetFileSize", arg0) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetFileSize indicates an expected call of GetFileSize. +func (mr *MockFileStoreMockRecorder) GetFileSize(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFileSize", reflect.TypeOf((*MockFileStore)(nil).GetFileSize), arg0) +} + // GetSyncStatus mocks base method. func (m *MockFileStore) GetSyncStatus(arg0 string) (int, error) { m.ctrl.T.Helper() @@ -370,6 +385,20 @@ func (mr *MockFileStoreMockRecorder) SetChunksCount(arg0, arg1 any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetChunksCount", reflect.TypeOf((*MockFileStore)(nil).SetChunksCount), arg0, arg1) } +// SetFileSize mocks base method. +func (m *MockFileStore) SetFileSize(arg0 string, arg1 int) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetFileSize", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetFileSize indicates an expected call of SetFileSize. +func (mr *MockFileStoreMockRecorder) SetFileSize(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetFileSize", reflect.TypeOf((*MockFileStore)(nil).SetFileSize), arg0, arg1) +} + // SetIsFileImported mocks base method. func (m *MockFileStore) SetIsFileImported(arg0 string, arg1 bool) error { m.ctrl.T.Helper() diff --git a/core/filestorage/filesync/filesync.go b/core/filestorage/filesync/filesync.go index 31de2454c..3f650f803 100644 --- a/core/filestorage/filesync/filesync.go +++ b/core/filestorage/filesync/filesync.go @@ -43,6 +43,7 @@ type FileSync interface { DebugQueue(*http.Request) (*QueueInfo, error) SendImportEvents() ClearImportEvents() + CalculateFileSize(ctx context.Context, spaceId string, fileID string) (int, error) app.ComponentRunnable } diff --git a/core/filestorage/filesync/filesync_test.go b/core/filestorage/filesync/filesync_test.go index e3e58632d..7eccef50c 100644 --- a/core/filestorage/filesync/filesync_test.go +++ b/core/filestorage/filesync/filesync_test.go @@ -49,7 +49,8 @@ func TestFileSync_AddFile(t *testing.T) { spaceId := "space1" fx.fileStoreMock.EXPECT().GetSyncStatus(fileId).Return(int(syncstatus.StatusNotSynced), nil) - + fx.fileStoreMock.EXPECT().GetFileSize(fileId).Return(0, fmt.Errorf("not found")) + fx.fileStoreMock.EXPECT().SetFileSize(fileId, gomock.Any()).Return(nil) fx.fileStoreMock.EXPECT().ListByTarget(fileId).Return([]*storage.FileInfo{ {}, // We can use just empty struct here, because we don't use any fields }, nil).AnyTimes() diff --git a/core/filestorage/filesync/mock_filesync/mock_FileSync.go b/core/filestorage/filesync/mock_filesync/mock_FileSync.go index e4d08fca3..2257a6aee 100644 --- a/core/filestorage/filesync/mock_filesync/mock_FileSync.go +++ b/core/filestorage/filesync/mock_filesync/mock_FileSync.go @@ -70,6 +70,60 @@ func (_c *MockFileSync_AddFile_Call) RunAndReturn(run func(string, string, bool, return _c } +// CalculateFileSize provides a mock function with given fields: ctx, spaceId, fileID +func (_m *MockFileSync) CalculateFileSize(ctx context.Context, spaceId string, fileID string) (int, error) { + ret := _m.Called(ctx, spaceId, fileID) + + var r0 int + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) (int, error)); ok { + return rf(ctx, spaceId, fileID) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) int); ok { + r0 = rf(ctx, spaceId, fileID) + } else { + r0 = ret.Get(0).(int) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, spaceId, fileID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockFileSync_CalculateFileSize_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CalculateFileSize' +type MockFileSync_CalculateFileSize_Call struct { + *mock.Call +} + +// CalculateFileSize is a helper method to define mock.On call +// - ctx context.Context +// - spaceId string +// - fileID string +func (_e *MockFileSync_Expecter) CalculateFileSize(ctx interface{}, spaceId interface{}, fileID interface{}) *MockFileSync_CalculateFileSize_Call { + return &MockFileSync_CalculateFileSize_Call{Call: _e.mock.On("CalculateFileSize", ctx, spaceId, fileID)} +} + +func (_c *MockFileSync_CalculateFileSize_Call) Run(run func(ctx context.Context, spaceId string, fileID string)) *MockFileSync_CalculateFileSize_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *MockFileSync_CalculateFileSize_Call) Return(_a0 int, _a1 error) *MockFileSync_CalculateFileSize_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockFileSync_CalculateFileSize_Call) RunAndReturn(run func(context.Context, string, string) (int, error)) *MockFileSync_CalculateFileSize_Call { + _c.Call.Return(run) + return _c +} + // ClearImportEvents provides a mock function with given fields: func (_m *MockFileSync) ClearImportEvents() { _m.Called() diff --git a/core/filestorage/filesync/upload.go b/core/filestorage/filesync/upload.go index 8ec5304f2..97076c3a0 100644 --- a/core/filestorage/filesync/upload.go +++ b/core/filestorage/filesync/upload.go @@ -4,15 +4,12 @@ import ( "context" "errors" "fmt" - "math" "strings" "time" "github.com/anyproto/any-sync/commonfile/fileproto" "github.com/anyproto/any-sync/commonfile/fileproto/fileprotoerr" - "github.com/anyproto/any-sync/commonfile/fileservice" "github.com/anyproto/any-sync/commonspace/syncstatus" - "github.com/cheggaaa/mb/v3" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" @@ -132,7 +129,6 @@ func (f *fileSync) tryToUpload() (string, error) { } return fileId, err } - log.Warn("done upload", zap.String("fileID", fileId)) if f.onUpload != nil { err := f.onUpload(spaceId, fileId) if err != nil { @@ -155,142 +151,42 @@ func isLimitReachedErr(err error) bool { return errors.Is(err, errReachedLimit) || strings.Contains(err.Error(), fileprotoerr.ErrSpaceLimitExceeded.Error()) } -func (f *fileSync) uploadFile(ctx context.Context, spaceId, fileId string) (err error) { - log.Debug("uploading file", zap.String("fileId", fileId)) +func (f *fileSync) uploadFile(ctx context.Context, spaceID string, fileID string) error { + log.Debug("uploading file", zap.String("fileID", fileID)) - var ( - batcher = mb.New[blocks.Block](10) - dagErr = make(chan error, 1) - bs []blocks.Block - ) - defer func() { - _ = batcher.Close() - }() - - blocksToUpload, err := f.prepareToUpload(ctx, spaceId, fileId) + fileSize, err := f.CalculateFileSize(ctx, spaceID, fileID) if err != nil { - return err + return fmt.Errorf("calculate file size: %w", err) + } + stat, err := f.getAndUpdateSpaceStat(ctx, spaceID) + if err != nil { + return fmt.Errorf("get space stat: %w", err) } - if len(blocksToUpload) == 0 { - return nil + bytesLeft := stat.BytesLimit - stat.BytesUsage + if fileSize > bytesLeft { + return errReachedLimit } - log.Info("start uploading file", zap.String("fileID", fileId), zap.Int("blocksCount", len(blocksToUpload))) - - go func() { - defer func() { - _ = batcher.Close() - }() - proc := func() error { - for _, b := range blocksToUpload { - if addErr := batcher.Add(ctx, b); addErr != nil { - return addErr - } - } - return nil + var totalBytesUploaded int + err = f.walkFileBlocks(ctx, spaceID, fileID, func(fileBlocks []blocks.Block) error { + bytesToUpload, blocksToUpload, err := f.selectBlocksToUploadAndBindExisting(ctx, spaceID, fileID, fileBlocks) + if err != nil { + return fmt.Errorf("select blocks to upload: %w", err) } - dagErr <- proc() - }() - - for { - if bs, err = batcher.Wait(ctx); err != nil { - if err == mb.ErrClosed { - err = nil - break - } else { - return err - } - } - - if err = f.rpcStore.AddToFile(ctx, spaceId, fileId, bs); err != nil { + if err = f.rpcStore.AddToFile(ctx, spaceID, fileID, blocksToUpload); err != nil { return err } - } - return <-dagErr -} - -func (f *fileSync) prepareToUpload(ctx context.Context, spaceId string, fileId string) ([]blocks.Block, error) { - estimatedSize, err := f.estimateFileSize(fileId) + totalBytesUploaded += bytesToUpload + return nil + }) if err != nil { - return nil, fmt.Errorf("estimate file size: %w", err) - } - stat, err := f.getAndUpdateSpaceStat(ctx, spaceId) - if err != nil { - return nil, fmt.Errorf("get space stat: %w", err) - } - vacantSpace := stat.BytesLimit - stat.BytesUsage - - if estimatedSize > vacantSpace { - return nil, errReachedLimit + return fmt.Errorf("walk file blocks: %w", err) } - fileBlocks, err := f.collectFileBlocks(ctx, spaceId, fileId) - if err != nil { - return nil, fmt.Errorf("collect file blocks: %w", err) - } + log.Warn("done upload", zap.String("fileID", fileID), zap.Int("estimatedSize", fileSize), zap.Int("bytesUploaded", totalBytesUploaded)) - bytesToUpload, blocksToUpload, err := f.selectBlocksToUploadAndBindExisting(ctx, spaceId, fileId, fileBlocks) - if err != nil { - return nil, fmt.Errorf("select blocks to upload: %w", err) - } - - if len(blocksToUpload) > 0 { - log.Warn("collecting blocks to upload", - zap.String("fileID", fileId), - zap.Int("blocksToUpload", len(blocksToUpload)), - zap.Int("totalBlocks", len(fileBlocks)), - ) - } - - if len(blocksToUpload) > 0 && bytesToUpload > vacantSpace { - return nil, errReachedLimit - } - - return blocksToUpload, nil -} - -// estimateFileSize use heuristic to estimate file size. It's pretty accurate for ordinary files, -// But getting worse for images because they have a different structure. -// Samples of estimation errors in bytes (the bigger the file, the bigger the error): -/* - Estimation error: 968 — 10Mb jpeg file - Estimation error: 1401 — 30Mb png file - Estimation error: 810 — 50Mb ordinary file - Estimation error: 3576 — 250Mb ordinary file -*/ -// Ordinary file structure: -/* -- dir (root) - - dir (content and meta pair) - - meta - - content (just content if file < 1Mb, and chunks list otherwise) - - chunk1 - - chunk2 - ... -*/ -func (f *fileSync) estimateFileSize(fileID string) (int, error) { - const ( - linkSize = 50 // Roughly minimal size of a link - metaNodeSize = 300 // Roughly minimal size of a meta node - - chunkSize = fileservice.ChunkSize - ) - fileInfos, err := f.fileStore.ListByTarget(fileID) - if err != nil { - return 0, fmt.Errorf("list file info: %w", err) - } - var totalSize int - for _, info := range fileInfos { - // Content is divided by chunks of 1Mb, and chunk is linked to the directory node - chunksCount := math.Ceil(float64(info.Size_) / float64(chunkSize)) - totalSize += int(info.Size_) + int(chunksCount)*linkSize - - totalSize += metaNodeSize + linkSize - } - totalSize += linkSize // for root node - - return totalSize, nil + return nil } func (f *fileSync) hasFileInStore(fileID string) (bool, error) { @@ -376,30 +272,86 @@ func (f *fileSync) selectBlocksToUploadAndBindExisting(ctx context.Context, spac return bytesToUpload, blocksToUpload, nil } -func (f *fileSync) collectFileBlocks(ctx context.Context, spaceID string, fileID string) (result []blocks.Block, err error) { +func (f *fileSync) walkDAG(ctx context.Context, spaceId string, fileID string, visit func(node ipld.Node) error) error { fileCid, err := cid.Parse(fileID) if err != nil { - return + return fmt.Errorf("parse CID %s: %w", fileID, err) } - dagService := f.dagServiceForSpace(spaceID) - node, err := dagService.Get(ctx, fileCid) + dagService := f.dagServiceForSpace(spaceId) + rootNode, err := dagService.Get(ctx, fileCid) if err != nil { - return + return fmt.Errorf("get root node: %w", err) } - walker := ipld.NewWalker(ctx, ipld.NewNavigableIPLDNode(node, dagService)) - err = walker.Iterate(func(node ipld.NavigableNode) error { - b, err := blocks.NewBlockWithCid(node.GetIPLDNode().RawData(), node.GetIPLDNode().Cid()) + visited := map[cid.Cid]struct{}{} + walker := ipld.NewWalker(ctx, ipld.NewNavigableIPLDNode(rootNode, dagService)) + err = walker.Iterate(func(navNode ipld.NavigableNode) error { + node := navNode.GetIPLDNode() + if _, ok := visited[node.Cid()]; !ok { + visited[node.Cid()] = struct{}{} + return visit(node) + } + return nil + }) + if errors.Is(err, ipld.EndOfDag) { + err = nil + } + return err +} + +// CalculateFileSize calculates or gets already calculated file size +func (f *fileSync) CalculateFileSize(ctx context.Context, spaceId string, fileID string) (int, error) { + size, err := f.fileStore.GetFileSize(fileID) + if err == nil { + return size, nil + } + + size = 0 + err = f.walkDAG(ctx, spaceId, fileID, func(node ipld.Node) error { + size += len(node.RawData()) + return nil + }) + if err != nil { + return 0, fmt.Errorf("walk DAG: %w", err) + } + err = f.fileStore.SetFileSize(fileID, size) + if err != nil { + log.Error("can't store file size", zap.String("fileID", fileID), zap.Error(err)) + } + return size, nil +} + +const batchSize = 10 + +func (f *fileSync) walkFileBlocks(ctx context.Context, spaceId string, fileID string, proc func(fileBlocks []blocks.Block) error) error { + blocksBuf := make([]blocks.Block, 0, batchSize) + + err := f.walkDAG(ctx, spaceId, fileID, func(node ipld.Node) error { + b, err := blocks.NewBlockWithCid(node.RawData(), node.Cid()) if err != nil { return err } - result = append(result, b) + blocksBuf = append(blocksBuf, b) + if len(blocksBuf) == batchSize { + err = proc(blocksBuf) + if err != nil { + return fmt.Errorf("process batch: %w", err) + } + blocksBuf = blocksBuf[:0] + } return nil }) - if err == ipld.EndOfDag { - err = nil + if err != nil { + return fmt.Errorf("walk DAG: %w", err) } - return + + if len(blocksBuf) > 0 { + err = proc(blocksBuf) + if err != nil { + return fmt.Errorf("process batch: %w", err) + } + } + return nil } func (f *fileSync) HasUpload(spaceId, fileId string) (ok bool, err error) { diff --git a/core/subscription/fixture_test.go b/core/subscription/fixture_test.go index 4eec721de..f29ea1be4 100644 --- a/core/subscription/fixture_test.go +++ b/core/subscription/fixture_test.go @@ -15,6 +15,7 @@ import ( "github.com/anyproto/anytype-heart/core/event" "github.com/anyproto/anytype-heart/core/event/mock_event" + "github.com/anyproto/anytype-heart/core/subscription/mock_subscription" "github.com/anyproto/anytype-heart/core/system_object/mock_system_object" "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" @@ -24,15 +25,14 @@ import ( ) type collectionServiceMock struct { - updateCh chan []string + *mock_subscription.MockCollectionService } -func (c *collectionServiceMock) SubscribeForCollection(collectionID string, subscriptionID string) ([]string, <-chan []string, error) { - return nil, c.updateCh, nil +func (c *collectionServiceMock) Name() string { + return "collectionService" } -func (c *collectionServiceMock) UnsubscribeFromCollection(collectionID string, subscriptionID string) { -} +func (c *collectionServiceMock) Init(a *app.App) error { return nil } type fixture struct { Service @@ -42,6 +42,7 @@ type fixture struct { systemObjectService *mock_system_object.MockService sender *mock_event.MockSender events []*pb.Event + collectionService *collectionServiceMock } func newFixture(t *testing.T) *fixture { @@ -49,7 +50,6 @@ func newFixture(t *testing.T) *fixture { a := new(app.App) testMock.RegisterMockObjectStore(ctrl, a) testMock.RegisterMockKanban(ctrl, a) - a.Register(&collectionServiceMock{}) sbtProvider := mock_typeprovider.NewMockSmartBlockTypeProvider(t) sbtProvider.EXPECT().Name().Return("smartBlockTypeProvider") sbtProvider.EXPECT().Init(mock.Anything).Return(nil) @@ -57,12 +57,17 @@ func newFixture(t *testing.T) *fixture { systemObjectService := mock_system_object.NewMockService(t) a.Register(testutil.PrepareMock(a, systemObjectService)) + + collectionService := &collectionServiceMock{MockCollectionService: mock_subscription.NewMockCollectionService(t)} + a.Register(collectionService) + fx := &fixture{ Service: New(), a: a, ctrl: ctrl, store: a.MustComponent(objectstore.CName).(*testMock.MockObjectStore), systemObjectService: systemObjectService, + collectionService: collectionService, } sender := mock_event.NewMockSender(t) sender.EXPECT().Init(mock.Anything).Return(nil) diff --git a/core/subscription/mock_subscription/mock_CollectionService.go b/core/subscription/mock_subscription/mock_CollectionService.go new file mode 100644 index 000000000..196c8c73e --- /dev/null +++ b/core/subscription/mock_subscription/mock_CollectionService.go @@ -0,0 +1,131 @@ +// Code generated by mockery v2.26.1. DO NOT EDIT. + +package mock_subscription + +import mock "github.com/stretchr/testify/mock" + +// MockCollectionService is an autogenerated mock type for the CollectionService type +type MockCollectionService struct { + mock.Mock +} + +type MockCollectionService_Expecter struct { + mock *mock.Mock +} + +func (_m *MockCollectionService) EXPECT() *MockCollectionService_Expecter { + return &MockCollectionService_Expecter{mock: &_m.Mock} +} + +// SubscribeForCollection provides a mock function with given fields: collectionID, subscriptionID +func (_m *MockCollectionService) SubscribeForCollection(collectionID string, subscriptionID string) ([]string, <-chan []string, error) { + ret := _m.Called(collectionID, subscriptionID) + + var r0 []string + var r1 <-chan []string + var r2 error + if rf, ok := ret.Get(0).(func(string, string) ([]string, <-chan []string, error)); ok { + return rf(collectionID, subscriptionID) + } + if rf, ok := ret.Get(0).(func(string, string) []string); ok { + r0 = rf(collectionID, subscriptionID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + if rf, ok := ret.Get(1).(func(string, string) <-chan []string); ok { + r1 = rf(collectionID, subscriptionID) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(<-chan []string) + } + } + + if rf, ok := ret.Get(2).(func(string, string) error); ok { + r2 = rf(collectionID, subscriptionID) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// MockCollectionService_SubscribeForCollection_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SubscribeForCollection' +type MockCollectionService_SubscribeForCollection_Call struct { + *mock.Call +} + +// SubscribeForCollection is a helper method to define mock.On call +// - collectionID string +// - subscriptionID string +func (_e *MockCollectionService_Expecter) SubscribeForCollection(collectionID interface{}, subscriptionID interface{}) *MockCollectionService_SubscribeForCollection_Call { + return &MockCollectionService_SubscribeForCollection_Call{Call: _e.mock.On("SubscribeForCollection", collectionID, subscriptionID)} +} + +func (_c *MockCollectionService_SubscribeForCollection_Call) Run(run func(collectionID string, subscriptionID string)) *MockCollectionService_SubscribeForCollection_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string)) + }) + return _c +} + +func (_c *MockCollectionService_SubscribeForCollection_Call) Return(_a0 []string, _a1 <-chan []string, _a2 error) *MockCollectionService_SubscribeForCollection_Call { + _c.Call.Return(_a0, _a1, _a2) + return _c +} + +func (_c *MockCollectionService_SubscribeForCollection_Call) RunAndReturn(run func(string, string) ([]string, <-chan []string, error)) *MockCollectionService_SubscribeForCollection_Call { + _c.Call.Return(run) + return _c +} + +// UnsubscribeFromCollection provides a mock function with given fields: collectionID, subscriptionID +func (_m *MockCollectionService) UnsubscribeFromCollection(collectionID string, subscriptionID string) { + _m.Called(collectionID, subscriptionID) +} + +// MockCollectionService_UnsubscribeFromCollection_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UnsubscribeFromCollection' +type MockCollectionService_UnsubscribeFromCollection_Call struct { + *mock.Call +} + +// UnsubscribeFromCollection is a helper method to define mock.On call +// - collectionID string +// - subscriptionID string +func (_e *MockCollectionService_Expecter) UnsubscribeFromCollection(collectionID interface{}, subscriptionID interface{}) *MockCollectionService_UnsubscribeFromCollection_Call { + return &MockCollectionService_UnsubscribeFromCollection_Call{Call: _e.mock.On("UnsubscribeFromCollection", collectionID, subscriptionID)} +} + +func (_c *MockCollectionService_UnsubscribeFromCollection_Call) Run(run func(collectionID string, subscriptionID string)) *MockCollectionService_UnsubscribeFromCollection_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string)) + }) + return _c +} + +func (_c *MockCollectionService_UnsubscribeFromCollection_Call) Return() *MockCollectionService_UnsubscribeFromCollection_Call { + _c.Call.Return() + return _c +} + +func (_c *MockCollectionService_UnsubscribeFromCollection_Call) RunAndReturn(run func(string, string)) *MockCollectionService_UnsubscribeFromCollection_Call { + _c.Call.Return(run) + return _c +} + +type mockConstructorTestingTNewMockCollectionService interface { + mock.TestingT + Cleanup(func()) +} + +// NewMockCollectionService creates a new instance of MockCollectionService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewMockCollectionService(t mockConstructorTestingTNewMockCollectionService) *MockCollectionService { + mock := &MockCollectionService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/subscription/service.go b/core/subscription/service.go index d2ffed8ac..cef886dce 100644 --- a/core/subscription/service.go +++ b/core/subscription/service.go @@ -274,7 +274,7 @@ func (s *service) subscribeForCollection(req pb.RpcObjectSearchSubscribeRequest, var depRecords, subRecords []*types.Struct subRecords = sub.getActiveRecords() - if sub.sortedSub.depSub != nil { + if sub.sortedSub.depSub != nil && !sub.sortedSub.disableDep { depRecords = sub.sortedSub.depSub.getActiveRecords() } diff --git a/core/subscription/service_test.go b/core/subscription/service_test.go index 3186bbf35..949a0a7c9 100644 --- a/core/subscription/service_test.go +++ b/core/subscription/service_test.go @@ -2,9 +2,9 @@ package subscription import ( "context" + "fmt" "testing" - "github.com/anyproto/any-sync/app" "github.com/gogo/protobuf/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -299,6 +299,354 @@ func TestService_Search(t *testing.T) { assert.NotEmpty(t, fx.events[0].Messages[3].GetSubscriptionCounters()) assert.NotEmpty(t, fx.events[0].Messages[0].GetObjectDetailsSet().Details) }) + + t.Run("collection: error getting collections entries - no records in response", func(t *testing.T) { + fx := newFixture(t) + defer fx.a.Close(context.Background()) + defer fx.ctrl.Finish() + + collectionID := "id" + subscriptionID := "subId" + fx.collectionService.EXPECT().SubscribeForCollection(collectionID, subscriptionID).Return(nil, nil, fmt.Errorf("error")) + var resp, err = fx.Search(pb.RpcObjectSearchSubscribeRequest{ + SubId: "subId", + CollectionId: collectionID, + }) + require.Error(t, err) + assert.Nil(t, resp) + }) + + t.Run("collection: collection is empty - no records in response", func(t *testing.T) { + fx := newFixture(t) + defer fx.a.Close(context.Background()) + defer fx.ctrl.Finish() + + collectionID := "id" + subscriptionID := "subId" + fx.collectionService.EXPECT().SubscribeForCollection(collectionID, subscriptionID).Return(nil, nil, nil) + fx.collectionService.EXPECT().UnsubscribeFromCollection(collectionID, subscriptionID).Return() + var resp, err = fx.Search(pb.RpcObjectSearchSubscribeRequest{ + SubId: subscriptionID, + CollectionId: collectionID, + }) + + require.NoError(t, err) + assert.NotNil(t, resp) + assert.Len(t, resp.Records, 0) + assert.Len(t, resp.Dependencies, 0) + }) + + t.Run("collection: collection has 2 objects - return 2 objects in response", func(t *testing.T) { + fx := newFixture(t) + defer fx.a.Close(context.Background()) + defer fx.ctrl.Finish() + + collectionID := "id" + subscriptionID := "subId" + + fx.collectionService.EXPECT().SubscribeForCollection(collectionID, subscriptionID).Return([]string{"1", "2"}, nil, nil) + fx.collectionService.EXPECT().UnsubscribeFromCollection(collectionID, subscriptionID).Return() + + fx.store.EXPECT().QueryByID([]string{"1", "2"}).Return([]database.Record{ + {Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("1"), + "name": pbtypes.String("1"), + }}}, + {Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("2"), + "name": pbtypes.String("2"), + }}}, + }, nil) + + fx.systemObjectService.EXPECT().GetRelationByKey(bundle.RelationKeyName.String()).Return(&model.Relation{ + Key: bundle.RelationKeyName.String(), + Format: model.RelationFormat_shorttext, + }, nil).Maybe() + fx.systemObjectService.EXPECT().GetRelationByKey(bundle.RelationKeyId.String()).Return(&model.Relation{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_shorttext, + }, nil).Maybe() + + var resp, err = fx.Search(pb.RpcObjectSearchSubscribeRequest{ + SubId: subscriptionID, + Keys: []string{bundle.RelationKeyName.String(), bundle.RelationKeyId.String()}, + CollectionId: collectionID, + }) + + require.NoError(t, err) + assert.NotNil(t, resp) + assert.Len(t, resp.Records, 2) + assert.Equal(t, "1", pbtypes.GetString(resp.Records[0], bundle.RelationKeyName.String())) + assert.Equal(t, "1", pbtypes.GetString(resp.Records[0], bundle.RelationKeyId.String())) + assert.Equal(t, "2", pbtypes.GetString(resp.Records[1], bundle.RelationKeyName.String())) + assert.Equal(t, "2", pbtypes.GetString(resp.Records[1], bundle.RelationKeyId.String())) + }) + + t.Run("collection: collection has 3 objects, 1 is filtered - return 2 objects in response", func(t *testing.T) { + fx := newFixture(t) + defer fx.a.Close(context.Background()) + defer fx.ctrl.Finish() + + collectionID := "id" + subscriptionID := "subId" + + fx.collectionService.EXPECT().SubscribeForCollection(collectionID, subscriptionID).Return([]string{"1", "2", "3"}, nil, nil) + fx.collectionService.EXPECT().UnsubscribeFromCollection(collectionID, subscriptionID).Return() + + fx.store.EXPECT().QueryByID([]string{"1", "2", "3"}).Return([]database.Record{ + {Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("1"), + "name": pbtypes.String("1"), + }}}, + {Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("2"), + "name": pbtypes.String("2"), + }}}, + {Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("3"), + "name": pbtypes.String("3"), + }}}, + }, nil) + + fx.systemObjectService.EXPECT().GetRelationByKey(bundle.RelationKeyName.String()).Return(&model.Relation{ + Key: bundle.RelationKeyName.String(), + Format: model.RelationFormat_shorttext, + }, nil).Maybe() + fx.systemObjectService.EXPECT().GetRelationByKey(bundle.RelationKeyId.String()).Return(&model.Relation{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_shorttext, + }, nil).Maybe() + + var resp, err = fx.Search(pb.RpcObjectSearchSubscribeRequest{ + SubId: subscriptionID, + Keys: []string{bundle.RelationKeyName.String(), bundle.RelationKeyId.String()}, + CollectionId: collectionID, + Filters: []*model.BlockContentDataviewFilter{ + { + Id: "1", + RelationKey: bundle.RelationKeyName.String(), + Condition: model.BlockContentDataviewFilter_NotEqual, + Value: pbtypes.String("3"), + }, + }, + }) + + require.NoError(t, err) + assert.NotNil(t, resp) + assert.Len(t, resp.Records, 2) + assert.Equal(t, "1", pbtypes.GetString(resp.Records[0], bundle.RelationKeyName.String())) + assert.Equal(t, "1", pbtypes.GetString(resp.Records[0], bundle.RelationKeyId.String())) + assert.Equal(t, "2", pbtypes.GetString(resp.Records[1], bundle.RelationKeyName.String())) + assert.Equal(t, "2", pbtypes.GetString(resp.Records[1], bundle.RelationKeyId.String())) + }) + t.Run("collection: collection has 3 objects, offset = 2 - return 1 object after offset", func(t *testing.T) { + fx := newFixture(t) + defer fx.a.Close(context.Background()) + defer fx.ctrl.Finish() + + collectionID := "id" + subscriptionID := "subId" + + fx.collectionService.EXPECT().SubscribeForCollection(collectionID, subscriptionID).Return([]string{"1", "2", "3"}, nil, nil) + fx.collectionService.EXPECT().UnsubscribeFromCollection(collectionID, subscriptionID).Return() + + fx.store.EXPECT().QueryByID([]string{"1", "2", "3"}).Return([]database.Record{ + {Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("1"), + "name": pbtypes.String("1"), + }}}, + {Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("2"), + "name": pbtypes.String("2"), + }}}, + {Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("3"), + "name": pbtypes.String("3"), + }}}, + }, nil) + + fx.systemObjectService.EXPECT().GetRelationByKey(bundle.RelationKeyName.String()).Return(&model.Relation{ + Key: bundle.RelationKeyName.String(), + Format: model.RelationFormat_shorttext, + }, nil).Maybe() + fx.systemObjectService.EXPECT().GetRelationByKey(bundle.RelationKeyId.String()).Return(&model.Relation{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_shorttext, + }, nil).Maybe() + + var resp, err = fx.Search(pb.RpcObjectSearchSubscribeRequest{ + SubId: subscriptionID, + Keys: []string{bundle.RelationKeyName.String(), bundle.RelationKeyId.String()}, + CollectionId: collectionID, + Offset: 2, + }) + + require.NoError(t, err) + assert.NotNil(t, resp) + assert.Len(t, resp.Records, 1) + assert.Equal(t, "3", pbtypes.GetString(resp.Records[0], bundle.RelationKeyName.String())) + assert.Equal(t, "3", pbtypes.GetString(resp.Records[0], bundle.RelationKeyId.String())) + }) + t.Run("collection: collection has object with dependency - return objects without dependency", func(t *testing.T) { + fx := newFixture(t) + defer fx.a.Close(context.Background()) + defer fx.ctrl.Finish() + + collectionID := "id" + subscriptionID := "subId" + testRelationKey := "link_to_object" + + fx.collectionService.EXPECT().SubscribeForCollection(collectionID, subscriptionID).Return([]string{"1"}, nil, nil) + fx.collectionService.EXPECT().UnsubscribeFromCollection(collectionID, subscriptionID).Return() + + fx.store.EXPECT().QueryByID([]string{"1"}).Return([]database.Record{ + {Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("1"), + "name": pbtypes.String("1"), + testRelationKey: pbtypes.String("2"), + }}}, + }, nil) + + // dependency + fx.store.EXPECT().QueryByID([]string{"2"}).Return([]database.Record{ + {Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("2"), + "name": pbtypes.String("2"), + }}}, + }, nil) + + fx.systemObjectService.EXPECT().GetRelationByKey(bundle.RelationKeyName.String()).Return(&model.Relation{ + Key: bundle.RelationKeyName.String(), + Format: model.RelationFormat_shorttext, + }, nil).Maybe() + + fx.systemObjectService.EXPECT().GetRelationByKey(testRelationKey).Return(&model.Relation{ + Key: testRelationKey, + Format: model.RelationFormat_object, + }, nil).Maybe() + + s := fx.Service.(*service) + s.ds = newDependencyService(s) + + var resp, err = fx.Search(pb.RpcObjectSearchSubscribeRequest{ + SubId: subscriptionID, + Keys: []string{bundle.RelationKeyName.String(), bundle.RelationKeyId.String(), testRelationKey}, + CollectionId: collectionID, + NoDepSubscription: true, + }) + + require.NoError(t, err) + assert.NotNil(t, resp) + assert.Len(t, resp.Records, 1) + assert.Len(t, resp.Dependencies, 0) + assert.Equal(t, "1", pbtypes.GetString(resp.Records[0], bundle.RelationKeyName.String())) + assert.Equal(t, "1", pbtypes.GetString(resp.Records[0], bundle.RelationKeyId.String())) + }) + t.Run("collection: collection has object with dependency - return objects with dependency", func(t *testing.T) { + fx := newFixture(t) + defer fx.a.Close(context.Background()) + defer fx.ctrl.Finish() + + collectionID := "id" + subscriptionID := "subId" + testRelationKey := "link_to_object" + + fx.collectionService.EXPECT().SubscribeForCollection(collectionID, subscriptionID).Return([]string{"1"}, nil, nil) + fx.collectionService.EXPECT().UnsubscribeFromCollection(collectionID, subscriptionID).Return() + + fx.store.EXPECT().QueryByID([]string{"1"}).Return([]database.Record{ + {Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("1"), + "name": pbtypes.String("1"), + testRelationKey: pbtypes.String("2"), + }}}, + }, nil) + + // dependency + fx.store.EXPECT().QueryByID([]string{"2"}).Return([]database.Record{ + {Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("2"), + "name": pbtypes.String("2"), + }}}, + }, nil) + + fx.systemObjectService.EXPECT().GetRelationByKey(bundle.RelationKeyName.String()).Return(&model.Relation{ + Key: bundle.RelationKeyName.String(), + Format: model.RelationFormat_shorttext, + }, nil).Maybe() + + fx.systemObjectService.EXPECT().GetRelationByKey(testRelationKey).Return(&model.Relation{ + Key: testRelationKey, + Format: model.RelationFormat_object, + }, nil).Maybe() + + s := fx.Service.(*service) + s.ds = newDependencyService(s) + + var resp, err = fx.Search(pb.RpcObjectSearchSubscribeRequest{ + SubId: subscriptionID, + Keys: []string{bundle.RelationKeyName.String(), bundle.RelationKeyId.String(), testRelationKey}, + CollectionId: collectionID, + NoDepSubscription: false, + }) + + require.NoError(t, err) + assert.NotNil(t, resp) + assert.Len(t, resp.Records, 1) + assert.Len(t, resp.Dependencies, 1) + assert.Equal(t, "1", pbtypes.GetString(resp.Records[0], bundle.RelationKeyName.String())) + assert.Equal(t, "1", pbtypes.GetString(resp.Records[0], bundle.RelationKeyId.String())) + assert.Equal(t, "2", pbtypes.GetString(resp.Dependencies[0], bundle.RelationKeyName.String())) + assert.Equal(t, "2", pbtypes.GetString(resp.Dependencies[0], bundle.RelationKeyId.String())) + }) + t.Run("collection: collection has 3 objects, but limit = 2 - return 2 objects in response", func(t *testing.T) { + fx := newFixture(t) + defer fx.a.Close(context.Background()) + defer fx.ctrl.Finish() + + collectionID := "id" + subscriptionID := "subId" + + fx.collectionService.EXPECT().SubscribeForCollection(collectionID, subscriptionID).Return([]string{"1", "2", "3"}, nil, nil) + fx.collectionService.EXPECT().UnsubscribeFromCollection(collectionID, subscriptionID).Return() + + fx.store.EXPECT().QueryByID([]string{"1", "2", "3"}).Return([]database.Record{ + {Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("1"), + "name": pbtypes.String("1"), + }}}, + {Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("2"), + "name": pbtypes.String("2"), + }}}, + {Details: &types.Struct{Fields: map[string]*types.Value{ + "id": pbtypes.String("3"), + "name": pbtypes.String("3"), + }}}, + }, nil) + + fx.systemObjectService.EXPECT().GetRelationByKey(bundle.RelationKeyName.String()).Return(&model.Relation{ + Key: bundle.RelationKeyName.String(), + Format: model.RelationFormat_shorttext, + }, nil).Maybe() + fx.systemObjectService.EXPECT().GetRelationByKey(bundle.RelationKeyId.String()).Return(&model.Relation{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_shorttext, + }, nil).Maybe() + + var resp, err = fx.Search(pb.RpcObjectSearchSubscribeRequest{ + SubId: subscriptionID, + Keys: []string{bundle.RelationKeyName.String(), bundle.RelationKeyId.String()}, + CollectionId: collectionID, + Limit: 1, + }) + + require.NoError(t, err) + assert.NotNil(t, resp) + assert.Len(t, resp.Records, 1) + assert.Equal(t, "1", pbtypes.GetString(resp.Records[0], bundle.RelationKeyName.String())) + assert.Equal(t, "1", pbtypes.GetString(resp.Records[0], bundle.RelationKeyId.String())) + }) } func TestNestedSubscription(t *testing.T) { @@ -353,12 +701,6 @@ func TestNestedSubscription(t *testing.T) { }) } -func (c *collectionServiceMock) Name() string { - return "collectionService" -} - -func (c *collectionServiceMock) Init(a *app.App) error { return nil } - func testCreateSubscriptionWithNestedFilter(t *testing.T) *fixtureRealStore { fx := newFixtureWithRealObjectStore(t) fx.systemObjectServiceMock.EXPECT().GetRelationByKey(mock.Anything).Return(&model.Relation{}, nil) diff --git a/docs/Debug.md b/docs/Debug.md index 783f9ff5e..afc80f78a 100644 --- a/docs/Debug.md +++ b/docs/Debug.md @@ -14,6 +14,9 @@ If you want to change the default port(9999): ## Useful tools for debug ### Debug server + +Firstly, build anytype-heart with build tag `-anydebug` + Use env var ANYDEBUG=address to enable debugging HTTP server. For example: `ANYDEBUG=:6061` will start debug server on port 6061 You can find all endpoints in `/debug` page. For example: http://localhost:6061/debug @@ -86,4 +89,4 @@ echo '{"details": {"name": "hello there", "type": "ot-page"}}' | grpcurl -import - use `ANYTYPE_PROM=0.0.0.0:9094` when running middleware to enable metrics collection. Client commands metrics available only in gRPC mode - open http://127.0.0.1:3000 to view collected metrics in Grafana. You can find several dashboards there: - **MW** internal middleware metrics such as changes, added and created threads histograms - - **MW commands server** metrics for clients commands. Works only in grpc-server mode \ No newline at end of file + - **MW commands server** metrics for clients commands. Works only in grpc-server mode diff --git a/pkg/lib/localstore/filestore/files.go b/pkg/lib/localstore/filestore/files.go index 656c23302..6e31a7ca9 100644 --- a/pkg/lib/localstore/filestore/files.go +++ b/pkg/lib/localstore/filestore/files.go @@ -26,6 +26,7 @@ var ( filesKeysBase = dsCtx.NewKey("/" + filesPrefix + "/keys") chunksCountBase = dsCtx.NewKey("/" + filesPrefix + "/chunks_count") syncStatusBase = dsCtx.NewKey("/" + filesPrefix + "/sync_status") + fileSizeBase = dsCtx.NewKey("/" + filesPrefix + "/file_size") isImportedBase = dsCtx.NewKey("/" + filesPrefix + "/is_imported") indexMillSourceOpts = localstore.Index{ @@ -102,6 +103,8 @@ type FileStore interface { SetSyncStatus(hash string, syncStatus int) error IsFileImported(hash string) (bool, error) SetIsFileImported(hash string, isImported bool) error + SetFileSize(hash string, size int) error + GetFileSize(hash string) (int, error) } func New() FileStore { @@ -525,6 +528,16 @@ func (m *dsFileStore) SetIsFileImported(hash string, isImported bool) error { return m.setInt(key, raw) } +func (m *dsFileStore) GetFileSize(hash string) (int, error) { + key := fileSizeBase.ChildString(hash) + return m.getInt(key) +} + +func (m *dsFileStore) SetFileSize(hash string, status int) error { + key := fileSizeBase.ChildString(hash) + return m.setInt(key, status) +} + func (ls *dsFileStore) Close(ctx context.Context) (err error) { return nil } diff --git a/pkg/lib/pb/model/models.pb.go b/pkg/lib/pb/model/models.pb.go index eb93fceec..ec50beebd 100644 --- a/pkg/lib/pb/model/models.pb.go +++ b/pkg/lib/pb/model/models.pb.go @@ -5,11 +5,12 @@ package model import ( fmt "fmt" - proto "github.com/gogo/protobuf/proto" - types "github.com/gogo/protobuf/types" io "io" math "math" math_bits "math/bits" + + proto "github.com/gogo/protobuf/proto" + types "github.com/gogo/protobuf/types" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/space/service.go b/space/service.go index aa2a84e5e..2d45e3d56 100644 --- a/space/service.go +++ b/space/service.go @@ -2,6 +2,9 @@ package space import ( "context" + "errors" + "strconv" + "strings" "sync" "github.com/anyproto/any-sync/app" @@ -21,6 +24,8 @@ const CName = "client.space" var log = logger.NewNamed(CName) +var ErrIncorrectSpaceID = errors.New("incorrect space id") + func New() SpaceService { return &service{} } @@ -95,6 +100,12 @@ func (s *service) Run(_ context.Context) (err error) { return } + // TODO: move this logic to any-sync + s.repKey, err = getRepKey(s.personalSpaceID) + if err != nil { + return + } + err = s.indexer.ReindexCommonObjects() if err != nil { return @@ -172,3 +183,11 @@ func (s *service) Close(ctx context.Context) (err error) { } return nil } + +func getRepKey(spaceID string) (uint64, error) { + sepIdx := strings.Index(spaceID, ".") + if sepIdx == -1 { + return 0, ErrIncorrectSpaceID + } + return strconv.ParseUint(spaceID[sepIdx+1:], 36, 64) +} diff --git a/space/spacecore/streamhandler.go b/space/spacecore/streamhandler.go index 69677ecdf..a99d1e9c1 100644 --- a/space/spacecore/streamhandler.go +++ b/space/spacecore/streamhandler.go @@ -2,13 +2,13 @@ package spacecore import ( "errors" - "go.uber.org/zap" "sync/atomic" "time" "github.com/anyproto/any-sync/commonspace/objectsync" "github.com/anyproto/any-sync/commonspace/spacesyncproto" "github.com/anyproto/any-sync/net/peer" + "go.uber.org/zap" "golang.org/x/net/context" "storj.io/drpc" ) diff --git a/space/techspace/techspace.go b/space/techspace/techspace.go index 731cc7668..bf3bf21ae 100644 --- a/space/techspace/techspace.go +++ b/space/techspace/techspace.go @@ -80,6 +80,7 @@ func (s *techSpace) wakeUpViews() (err error) { return nil }) } + s.techCore.TreeSyncer().StartSync() return } @@ -88,10 +89,10 @@ func (s *techSpace) CreateSpaceView(ctx context.Context, spaceID string) (spaceV if err != nil { return } - obj, err := s.objectCache.DeriveTreeObject(ctx, spaceID, objectcache.TreeDerivationParams{ + obj, err := s.objectCache.DeriveTreeObject(ctx, s.techCore.Id(), objectcache.TreeDerivationParams{ Key: uniqueKey, InitFunc: func(id string) *editorsb.InitContext { - return &editorsb.InitContext{Ctx: ctx, SpaceID: spaceID, State: state.NewDoc(id, nil).(*state.State)} + return &editorsb.InitContext{Ctx: ctx, SpaceID: s.techCore.Id(), State: state.NewDoc(id, nil).(*state.State)} }, TargetSpaceID: spaceID, }) diff --git a/tests/blockbuilder/block_render.go b/tests/blockbuilder/block_render.go new file mode 100644 index 000000000..8f8b8d7ac --- /dev/null +++ b/tests/blockbuilder/block_render.go @@ -0,0 +1,67 @@ +package blockbuilder + +import ( + "encoding/json" + "fmt" + + "github.com/gogo/protobuf/jsonpb" + "github.com/gogo/protobuf/proto" + + "github.com/anyproto/anytype-heart/pkg/lib/pb/model" +) + +type blockView struct { + Id string + Fields *json.RawMessage `json:"Fields,omitempty"` + Children []*Block `json:"Children,omitempty"` + Restrictions *model.BlockRestrictions `json:"Restrictions,omitempty"` + BackgroundColor string `json:"BackgroundColor,omitempty"` + Align model.BlockAlign `json:"Align,omitempty"` + VerticalAlign model.BlockVerticalAlign `json:"VerticalAlign,omitempty"` + Content *json.RawMessage `json:"Content"` +} + +func marshalProtoMessage(pbMessage proto.Message) (*json.RawMessage, error) { + marshaller := &jsonpb.Marshaler{} + raw, err := marshaller.MarshalToString(pbMessage) + if err != nil { + return nil, fmt.Errorf("marshal content: %w", err) + } + rawMessage := json.RawMessage(raw) + return &rawMessage, nil +} + +func (b *Block) MarshalJSON() ([]byte, error) { + var ( + err error + rawContent *json.RawMessage + ) + if content := b.block.Content; content != nil { + contentWrapper := &model.Block{ + Content: content, + } + rawContent, err = marshalProtoMessage(contentWrapper) + if err != nil { + return nil, fmt.Errorf("marshal content: %w", err) + } + } + var rawFields *json.RawMessage + if fields := b.block.Fields; fields != nil { + rawFields, err = marshalProtoMessage(b.block.Fields) + if err != nil { + return nil, fmt.Errorf("marshal fields: %w", err) + } + } + + v := blockView{ + Id: b.block.Id, + Fields: rawFields, + Children: b.children, + Restrictions: b.block.Restrictions, + BackgroundColor: b.block.BackgroundColor, + Align: b.block.Align, + VerticalAlign: b.block.VerticalAlign, + Content: rawContent, + } + return json.Marshal(v) +} diff --git a/util/builtinobjects/data/notes_diary.zip b/util/builtinobjects/data/notes_diary.zip index 8169447f0..ad948ed5d 100644 Binary files a/util/builtinobjects/data/notes_diary.zip and b/util/builtinobjects/data/notes_diary.zip differ diff --git a/util/testMock/mockBuiltinTemplate/builtintemplate_mock.go b/util/testMock/mockBuiltinTemplate/builtintemplate_mock.go index 1f39f3d37..b82119fe2 100644 --- a/util/testMock/mockBuiltinTemplate/builtintemplate_mock.go +++ b/util/testMock/mockBuiltinTemplate/builtintemplate_mock.go @@ -1,6 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/anyproto/anytype-heart/util/builtintemplate (interfaces: BuiltinTemplate) - +// +// Generated by this command: +// +// mockgen -package mockBuiltinTemplate -destination builtintemplate_mock.go github.com/anyproto/anytype-heart/util/builtintemplate BuiltinTemplate +// // Package mockBuiltinTemplate is a generated GoMock package. package mockBuiltinTemplate @@ -44,7 +48,7 @@ func (m *MockBuiltinTemplate) Close(arg0 context.Context) error { } // Close indicates an expected call of Close. -func (mr *MockBuiltinTemplateMockRecorder) Close(arg0 interface{}) *gomock.Call { +func (mr *MockBuiltinTemplateMockRecorder) Close(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockBuiltinTemplate)(nil).Close), arg0) } @@ -72,7 +76,7 @@ func (m *MockBuiltinTemplate) Init(arg0 *app.App) error { } // Init indicates an expected call of Init. -func (mr *MockBuiltinTemplateMockRecorder) Init(arg0 interface{}) *gomock.Call { +func (mr *MockBuiltinTemplateMockRecorder) Init(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockBuiltinTemplate)(nil).Init), arg0) } @@ -100,7 +104,7 @@ func (m *MockBuiltinTemplate) Run(arg0 context.Context) error { } // Run indicates an expected call of Run. -func (mr *MockBuiltinTemplateMockRecorder) Run(arg0 interface{}) *gomock.Call { +func (mr *MockBuiltinTemplateMockRecorder) Run(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockBuiltinTemplate)(nil).Run), arg0) } diff --git a/util/testMock/mockCreator/creator_mock.go b/util/testMock/mockCreator/creator_mock.go new file mode 100644 index 000000000..e7a89b09a --- /dev/null +++ b/util/testMock/mockCreator/creator_mock.go @@ -0,0 +1,153 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/anyproto/anytype-heart/core/block/object/objectcreator (interfaces: Service) +// +// Generated by this command: +// +// mockgen -package mockCreator -destination creator_mock.go github.com/anyproto/anytype-heart/core/block/object/objectcreator Service +// +// Package mockCreator is a generated GoMock package. +package mockCreator + +import ( + context "context" + reflect "reflect" + + app "github.com/anyproto/any-sync/app" + types "github.com/gogo/protobuf/types" + gomock "go.uber.org/mock/gomock" + + state "github.com/anyproto/anytype-heart/core/block/editor/state" + pb "github.com/anyproto/anytype-heart/pb" + smartblock "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" +) + +// MockService is a mock of Service interface. +type MockService struct { + ctrl *gomock.Controller + recorder *MockServiceMockRecorder +} + +// MockServiceMockRecorder is the mock recorder for MockService. +type MockServiceMockRecorder struct { + mock *MockService +} + +// NewMockService creates a new mock instance. +func NewMockService(ctrl *gomock.Controller) *MockService { + mock := &MockService{ctrl: ctrl} + mock.recorder = &MockServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockService) EXPECT() *MockServiceMockRecorder { + return m.recorder +} + +// CreateSet mocks base method. +func (m *MockService) CreateSet(arg0 *pb.RpcObjectCreateSetRequest) (string, *types.Struct, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateSet", arg0) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(*types.Struct) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// CreateSet indicates an expected call of CreateSet. +func (mr *MockServiceMockRecorder) CreateSet(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSet", reflect.TypeOf((*MockService)(nil).CreateSet), arg0) +} + +// CreateSmartBlockFromState mocks base method. +func (m *MockService) CreateSmartBlockFromState(arg0 context.Context, arg1 smartblock.SmartBlockType, arg2 *types.Struct, arg3 *state.State) (string, *types.Struct, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateSmartBlockFromState", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(*types.Struct) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// CreateSmartBlockFromState indicates an expected call of CreateSmartBlockFromState. +func (mr *MockServiceMockRecorder) CreateSmartBlockFromState(arg0, arg1, arg2, arg3 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSmartBlockFromState", reflect.TypeOf((*MockService)(nil).CreateSmartBlockFromState), arg0, arg1, arg2, arg3) +} + +// CreateSmartBlockFromTemplate mocks base method. +func (m *MockService) CreateSmartBlockFromTemplate(arg0 context.Context, arg1 smartblock.SmartBlockType, arg2 *types.Struct, arg3 string) (string, *types.Struct, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateSmartBlockFromTemplate", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(*types.Struct) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// CreateSmartBlockFromTemplate indicates an expected call of CreateSmartBlockFromTemplate. +func (mr *MockServiceMockRecorder) CreateSmartBlockFromTemplate(arg0, arg1, arg2, arg3 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSmartBlockFromTemplate", reflect.TypeOf((*MockService)(nil).CreateSmartBlockFromTemplate), arg0, arg1, arg2, arg3) +} + +// CreateSubObjectInWorkspace mocks base method. +func (m *MockService) CreateSubObjectInWorkspace(arg0 *types.Struct, arg1 string) (string, *types.Struct, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateSubObjectInWorkspace", arg0, arg1) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(*types.Struct) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// CreateSubObjectInWorkspace indicates an expected call of CreateSubObjectInWorkspace. +func (mr *MockServiceMockRecorder) CreateSubObjectInWorkspace(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSubObjectInWorkspace", reflect.TypeOf((*MockService)(nil).CreateSubObjectInWorkspace), arg0, arg1) +} + +// CreateSubObjectsInWorkspace mocks base method. +func (m *MockService) CreateSubObjectsInWorkspace(arg0 []*types.Struct) ([]string, []*types.Struct, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateSubObjectsInWorkspace", arg0) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].([]*types.Struct) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// CreateSubObjectsInWorkspace indicates an expected call of CreateSubObjectsInWorkspace. +func (mr *MockServiceMockRecorder) CreateSubObjectsInWorkspace(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSubObjectsInWorkspace", reflect.TypeOf((*MockService)(nil).CreateSubObjectsInWorkspace), arg0) +} + +// Init mocks base method. +func (m *MockService) Init(arg0 *app.App) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Init", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Init indicates an expected call of Init. +func (mr *MockServiceMockRecorder) Init(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockService)(nil).Init), arg0) +} + +// Name mocks base method. +func (m *MockService) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockServiceMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockService)(nil).Name)) +} diff --git a/util/testMock/mockSpace/space_mock.go b/util/testMock/mockSpace/space_mock.go index c1330017c..8c7650319 100644 --- a/util/testMock/mockSpace/space_mock.go +++ b/util/testMock/mockSpace/space_mock.go @@ -1,6 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/anyproto/anytype-heart/space (interfaces: Service) - +// +// Generated by this command: +// +// mockgen -package mockSpace -destination space_mock.go github.com/anyproto/anytype-heart/space Service +// // Package mockSpace is a generated GoMock package. package mockSpace @@ -63,7 +67,7 @@ func (m *MockService) AccountSpace(arg0 context.Context) (commonspace.Space, err } // AccountSpace indicates an expected call of AccountSpace. -func (mr *MockServiceMockRecorder) AccountSpace(arg0 interface{}) *gomock.Call { +func (mr *MockServiceMockRecorder) AccountSpace(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AccountSpace", reflect.TypeOf((*MockService)(nil).AccountSpace), arg0) } @@ -77,7 +81,7 @@ func (m *MockService) Close(arg0 context.Context) error { } // Close indicates an expected call of Close. -func (mr *MockServiceMockRecorder) Close(arg0 interface{}) *gomock.Call { +func (mr *MockServiceMockRecorder) Close(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockService)(nil).Close), arg0) } @@ -107,7 +111,7 @@ func (m *MockService) DeleteAccount(arg0 context.Context, arg1 bool) (space.Netw } // DeleteAccount indicates an expected call of DeleteAccount. -func (mr *MockServiceMockRecorder) DeleteAccount(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockServiceMockRecorder) DeleteAccount(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAccount", reflect.TypeOf((*MockService)(nil).DeleteAccount), arg0, arg1) } @@ -122,7 +126,7 @@ func (m *MockService) DeleteSpace(arg0 context.Context, arg1 string, arg2 bool) } // DeleteSpace indicates an expected call of DeleteSpace. -func (mr *MockServiceMockRecorder) DeleteSpace(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockServiceMockRecorder) DeleteSpace(arg0, arg1, arg2 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSpace", reflect.TypeOf((*MockService)(nil).DeleteSpace), arg0, arg1, arg2) } @@ -137,7 +141,7 @@ func (m *MockService) DeriveSpace(arg0 context.Context, arg1 commonspace.SpaceDe } // DeriveSpace indicates an expected call of DeriveSpace. -func (mr *MockServiceMockRecorder) DeriveSpace(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockServiceMockRecorder) DeriveSpace(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeriveSpace", reflect.TypeOf((*MockService)(nil).DeriveSpace), arg0, arg1) } @@ -152,7 +156,7 @@ func (m *MockService) GetSpace(arg0 context.Context, arg1 string) (commonspace.S } // GetSpace indicates an expected call of GetSpace. -func (mr *MockServiceMockRecorder) GetSpace(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockServiceMockRecorder) GetSpace(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSpace", reflect.TypeOf((*MockService)(nil).GetSpace), arg0, arg1) } @@ -166,7 +170,7 @@ func (m *MockService) Init(arg0 *app.App) error { } // Init indicates an expected call of Init. -func (mr *MockServiceMockRecorder) Init(arg0 interface{}) *gomock.Call { +func (mr *MockServiceMockRecorder) Init(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockService)(nil).Init), arg0) } @@ -194,7 +198,7 @@ func (m *MockService) Run(arg0 context.Context) error { } // Run indicates an expected call of Run. -func (mr *MockServiceMockRecorder) Run(arg0 interface{}) *gomock.Call { +func (mr *MockServiceMockRecorder) Run(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockService)(nil).Run), arg0) } diff --git a/util/testMock/mockStatus/status_mock.go b/util/testMock/mockStatus/status_mock.go index 9227989bd..55f722303 100644 --- a/util/testMock/mockStatus/status_mock.go +++ b/util/testMock/mockStatus/status_mock.go @@ -1,6 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/anyproto/anytype-heart/core/syncstatus (interfaces: Service) - +// +// Generated by this command: +// +// mockgen -package mockStatus -destination status_mock.go github.com/anyproto/anytype-heart/core/syncstatus Service +// // Package mockStatus is a generated GoMock package. package mockStatus @@ -44,7 +48,7 @@ func (m *MockService) Close(arg0 context.Context) error { } // Close indicates an expected call of Close. -func (mr *MockServiceMockRecorder) Close(arg0 interface{}) *gomock.Call { +func (mr *MockServiceMockRecorder) Close(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockService)(nil).Close), arg0) } @@ -58,7 +62,7 @@ func (m *MockService) Init(arg0 *app.App) error { } // Init indicates an expected call of Init. -func (mr *MockServiceMockRecorder) Init(arg0 interface{}) *gomock.Call { +func (mr *MockServiceMockRecorder) Init(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockService)(nil).Init), arg0) } @@ -86,7 +90,7 @@ func (m *MockService) OnFileUpload(arg0, arg1 string) error { } // OnFileUpload indicates an expected call of OnFileUpload. -func (mr *MockServiceMockRecorder) OnFileUpload(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockServiceMockRecorder) OnFileUpload(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnFileUpload", reflect.TypeOf((*MockService)(nil).OnFileUpload), arg0, arg1) } @@ -100,34 +104,34 @@ func (m *MockService) Run(arg0 context.Context) error { } // Run indicates an expected call of Run. -func (mr *MockServiceMockRecorder) Run(arg0 interface{}) *gomock.Call { +func (mr *MockServiceMockRecorder) Run(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockService)(nil).Run), arg0) } // Unwatch mocks base method. -func (m *MockService) Unwatch(arg0, arg1 string) { +func (m *MockService) Unwatch(arg0 string) { m.ctrl.T.Helper() - m.ctrl.Call(m, "Unwatch", arg0, arg1) + m.ctrl.Call(m, "Unwatch", arg0) } // Unwatch indicates an expected call of Unwatch. -func (mr *MockServiceMockRecorder) Unwatch(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockServiceMockRecorder) Unwatch(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unwatch", reflect.TypeOf((*MockService)(nil).Unwatch), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unwatch", reflect.TypeOf((*MockService)(nil).Unwatch), arg0) } // Watch mocks base method. -func (m *MockService) Watch(arg0, arg1 string, arg2 func() []string) (bool, error) { +func (m *MockService) Watch(arg0 string, arg1 func() []string) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Watch", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "Watch", arg0, arg1) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } // Watch indicates an expected call of Watch. -func (mr *MockServiceMockRecorder) Watch(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockServiceMockRecorder) Watch(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Watch", reflect.TypeOf((*MockService)(nil).Watch), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Watch", reflect.TypeOf((*MockService)(nil).Watch), arg0, arg1) }