diff --git a/.github/workflows/perftests.yml b/.github/workflows/perftests.yml index 4a05ca6ba..94d63ba8e 100644 --- a/.github/workflows/perftests.yml +++ b/.github/workflows/perftests.yml @@ -26,6 +26,7 @@ permissions: name: Perf tests jobs: build: + timeout-minutes: 60 runs-on: 'ARM64' steps: - name: Setup GO diff --git a/cmd/perfstand/internal/util.go b/cmd/perfstand/internal/util.go index 424fb92e9..451c472ac 100644 --- a/cmd/perfstand/internal/util.go +++ b/cmd/perfstand/internal/util.go @@ -68,7 +68,7 @@ func SendResultsToHttp(apiKey string, events []Event) error { } func KillServer() error { - return ExecuteCommand("kill $(lsof -i :31007 -t) ; echo \"Server killed\"") + return ExecuteCommand("kill -9 $(lsof -i :31007 -t) ; echo \"Server killed\"") } func ExecuteCommand(command string) error { diff --git a/core/block/export/export.go b/core/block/export/export.go index 4e004780d..4982029f1 100644 --- a/core/block/export/export.go +++ b/core/block/export/export.go @@ -260,97 +260,494 @@ func (e *export) docsForExport(spaceID string, req pb.RpcObjectListExportRequest return } -func (e *export) getObjectsByIDs(spaceId string, reqIds []string, includeNested bool, includeFiles bool, isProtobuf bool) (map[string]*types.Struct, error) { +func (e *export) getObjectsByIDs(spaceId string, reqIds []string, includeNested, includeFiles, isProtobuf bool) (map[string]*types.Struct, error) { + res, err := e.queryAndFilterObjectsByRelation(spaceId, reqIds, bundle.RelationKeyId.String()) + if err != nil { + return nil, err + } docs := make(map[string]*types.Struct) - res, err := e.objectStore.SpaceIndex(spaceId).Query(database.Query{ + for _, object := range res { + id := pbtypes.GetString(object.Details, bundle.RelationKeyId.String()) + docs[id] = object.Details + } + if isProtobuf { + return e.processProtobuf(spaceId, docs, includeNested, includeFiles) + } + return e.processNotProtobuf(spaceId, docs, includeNested, includeFiles) +} + +func (e *export) queryAndFilterObjectsByRelation(spaceId string, reqIds []string, relationFilter string) ([]database.Record, error) { + var allObjects []database.Record + const singleBatchCount = 50 + for j := 0; j < len(reqIds); { + if j+singleBatchCount < len(reqIds) { + records, err := e.queryObjectsByIds(spaceId, reqIds[j:j+singleBatchCount], relationFilter) + if err != nil { + return nil, err + } + allObjects = append(allObjects, records...) + } else { + records, err := e.queryObjectsByIds(spaceId, reqIds[j:], relationFilter) + if err != nil { + return nil, err + } + allObjects = append(allObjects, records...) + } + j += singleBatchCount + } + return allObjects, nil +} + +func (e *export) queryObjectsByIds(spaceId string, reqIds []string, relationFilter string) ([]database.Record, error) { + return e.objectStore.Query(database.Query{ Filters: []*model.BlockContentDataviewFilter{ { - RelationKey: bundle.RelationKeyId.String(), + RelationKey: relationFilter, Condition: model.BlockContentDataviewFilter_In, Value: pbtypes.StringList(reqIds), }, { - RelationKey: bundle.RelationKeyIsArchived.String(), + RelationKey: bundle.RelationKeySpaceId.String(), Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.Bool(false), + Value: pbtypes.String(spaceId), + }, + }, + }) +} + +func (e *export) processNotProtobuf(spaceId string, docs map[string]*types.Struct, includeNested, includeFiles bool) (map[string]*types.Struct, error) { + ids := e.fillObjectsIds(docs) + if includeFiles { + fileObjectsIds, err := e.processFiles(spaceId, ids, docs) + if err != nil { + return nil, err + } + ids = append(ids, fileObjectsIds...) + } + if includeNested { + for _, id := range ids { + e.addNestedObject(spaceId, id, docs, map[string]*types.Struct{}) + } + } + return docs, nil +} + +func (e *export) processProtobuf(spaceId string, docs map[string]*types.Struct, includeNested, includeFiles bool) (map[string]*types.Struct, error) { + ids := e.fillObjectsIds(docs) + if includeFiles { + err := e.addFileObjects(spaceId, docs, ids, includeNested) + if err != nil { + return nil, err + } + } + err := e.addDerivedObjects(spaceId, docs) + if err != nil { + return nil, err + } + ids = e.fillTemplateIds(docs, ids) + if includeNested { + err = e.addNestedObjects(spaceId, docs, ids, includeFiles) + if err != nil { + return nil, err + } + } + return docs, nil +} + +func (e *export) fillObjectsIds(docs map[string]*types.Struct) []string { + ids := make([]string, 0, len(docs)) + for id := range docs { + ids = append(ids, id) + } + return ids +} + +func (e *export) addFileObjects(spaceId string, docs map[string]*types.Struct, ids []string, includeNested bool) error { + fileObjectsIds, err := e.processFiles(spaceId, ids, docs) + if err != nil { + return err + } + if includeNested { + err = e.addNestedObjects(spaceId, docs, fileObjectsIds, true) + if err != nil { + return err + } + } + return nil +} + +func (e *export) processFiles(spaceId string, ids []string, docs map[string]*types.Struct) ([]string, error) { + spc, err := e.spaceService.Get(context.Background(), spaceId) + if err != nil { + return nil, fmt.Errorf("get space: %w", err) + } + var fileObjectsIds []string + for _, id := range ids { + objectFiles, err := e.fillLinkedFiles(spc, id, docs) + if err != nil { + return nil, err + } + fileObjectsIds = lo.Union(fileObjectsIds, objectFiles) + } + return fileObjectsIds, nil +} + +func (e *export) addDerivedObjects(spaceId string, docs map[string]*types.Struct) error { + processedObjects := make(map[string]struct{}, 0) + allRelations, allTypes, allSetOfList, err := e.getRelationsAndTypes(spaceId, docs, processedObjects) + if err != nil { + return err + } + templateRelations, templateTypes, templateSetOfList, err := e.getTemplatesRelationsAndTypes(spaceId, docs, lo.Union(allTypes, allSetOfList), processedObjects) + if err != nil { + return err + } + allRelations = lo.Union(allRelations, templateRelations) + allTypes = lo.Union(allTypes, templateTypes) + allSetOfList = lo.Union(allSetOfList, templateSetOfList) + err = e.addRelationsAndTypes(spaceId, docs, allTypes, allRelations, allSetOfList) + if err != nil { + return err + } + return nil +} + +func (e *export) getRelationsAndTypes( + spaceId string, + objects map[string]*types.Struct, + processedObjects map[string]struct{}, +) ([]string, []string, []string, error) { + allRelations, allTypes, allSetOfList, err := e.collectDerivedObjects(objects) + if err != nil { + return nil, nil, nil, err + } + // get derived objects only from types, + // because relations currently have only system relations and object type + if len(allTypes) > 0 || len(allSetOfList) > 0 { + relations, objectTypes, setOfList, err := e.getDerivedObjectsForTypes(spaceId, lo.Union(allTypes, allSetOfList), processedObjects) + if err != nil { + return nil, nil, nil, err + } + allRelations = lo.Union(allRelations, relations) + allTypes = lo.Union(allTypes, objectTypes) + allSetOfList = lo.Union(allSetOfList, setOfList) + } + return allRelations, allTypes, allSetOfList, nil +} + +func (e *export) collectDerivedObjects(objects map[string]*types.Struct) ([]string, []string, []string, error) { + var relations, objectsTypes, setOf []string + for id := range objects { + err := cache.Do(e.picker, id, func(b sb.SmartBlock) error { + state := b.NewState() + relations = lo.Union(relations, e.getObjectRelations(state)) + details := state.CombinedDetails() + if e.isObjectWithDataview(details) { + dataviewRelations, err := e.getDataviewRelations(state) + if err != nil { + return err + } + relations = lo.Union(relations, dataviewRelations) + } + objectTypeId := pbtypes.GetString(details, bundle.RelationKeyType.String()) + objectsTypes = lo.Union(objectsTypes, []string{objectTypeId}) + setOfList := pbtypes.GetStringList(details, bundle.RelationKeySetOf.String()) + setOf = lo.Union(setOf, setOfList) + return nil + }) + if err != nil { + return nil, nil, nil, err + } + } + return relations, objectsTypes, setOf, nil +} + +func (e *export) getObjectRelations(state *state.State) []string { + relationLinks := state.GetRelationLinks() + relations := make([]string, 0, len(relationLinks)) + for _, link := range relationLinks { + relations = append(relations, link.Key) + } + return relations +} + +func (e *export) isObjectWithDataview(details *types.Struct) bool { + return pbtypes.GetFloat64(details, bundle.RelationKeyLayout.String()) == float64(model.ObjectType_collection) || + pbtypes.GetFloat64(details, bundle.RelationKeyLayout.String()) == float64(model.ObjectType_set) +} + +func (e *export) getDataviewRelations(state *state.State) ([]string, error) { + var relations []string + err := state.Iterate(func(b simple.Block) (isContinue bool) { + if dataview := b.Model().GetDataview(); dataview != nil { + for _, view := range dataview.Views { + for _, relation := range view.Relations { + relations = append(relations, relation.Key) + } + } + } + return true + }) + return relations, err +} + +func (e *export) getDerivedObjectsForTypes( + spaceId string, + allTypes []string, + processedObjects map[string]struct{}, +) ([]string, []string, []string, error) { + notProceedTypes := make(map[string]*types.Struct, 0) + var relations, objectTypes []string + for _, object := range allTypes { + if _, ok := processedObjects[object]; ok { + continue + } + notProceedTypes[object] = nil + processedObjects[object] = struct{}{} + } + if len(notProceedTypes) == 0 { + return relations, objectTypes, nil, nil + } + relations, objectTypes, setOfList, err := e.getRelationsAndTypes(spaceId, notProceedTypes, processedObjects) + if err != nil { + return nil, nil, nil, err + } + return relations, objectTypes, setOfList, nil +} + +func (e *export) getTemplatesRelationsAndTypes( + spaceId string, + allObjects map[string]*types.Struct, + allTypes []string, + processedObjects map[string]struct{}, +) ([]string, []string, []string, error) { + + templates, err := e.queryAndFilterObjectsByRelation(spaceId, allTypes, bundle.RelationKeyTargetObjectType.String()) + if err != nil { + return nil, nil, nil, err + } + if len(templates) == 0 { + return nil, nil, nil, nil + } + templatesToProcess := make(map[string]*types.Struct, len(templates)) + for _, template := range templates { + id := pbtypes.GetString(template.Details, bundle.RelationKeyId.String()) + if _, ok := allObjects[id]; !ok { + allObjects[id] = template.Details + templatesToProcess[id] = template.Details + } + } + templateRelations, templateType, templateSetOfList, err := e.getRelationsAndTypes(spaceId, templatesToProcess, processedObjects) + if err != nil { + return nil, nil, nil, err + } + return templateRelations, templateType, templateSetOfList, nil +} + +func (e *export) addRelationsAndTypes( + spaceId string, + allObjects map[string]*types.Struct, + types []string, + relations []string, + setOfList []string, +) error { + err := e.addRelations(spaceId, allObjects, relations) + if err != nil { + return err + } + err = e.processObjectTypesAndSetOfList(spaceId, types, allObjects, setOfList) + if err != nil { + return err + } + return nil +} + +func (e *export) addRelations(spaceId string, allObjects map[string]*types.Struct, relations []string) error { + storeRelations, err := e.getRelationsFromStore(spaceId, relations) + if err != nil { + return err + } + for _, storeRelation := range storeRelations { + e.addRelation(storeRelation, allObjects) + err := e.addOptionIfTag(spaceId, storeRelation, allObjects) + if err != nil { + return err + } + } + return nil +} + +func (e *export) getRelationsFromStore(spaceId string, relations []string) ([]database.Record, error) { + uniqueKeys := make([]string, 0, len(relations)) + for _, relation := range relations { + uniqueKey, err := domain.NewUniqueKey(smartblock.SmartBlockTypeRelation, relation) + if err != nil { + return nil, err + } + uniqueKeys = append(uniqueKeys, uniqueKey.Marshal()) + } + storeRelations, err := e.queryAndFilterObjectsByRelation(spaceId, uniqueKeys, bundle.RelationKeyUniqueKey.String()) + if err != nil { + return nil, err + } + return storeRelations, nil +} + +func (e *export) addRelation(relation database.Record, derivedObjects map[string]*types.Struct) { + if relationKey := pbtypes.GetString(relation.Details, bundle.RelationKeyRelationKey.String()); relationKey != "" { + if !bundle.HasRelation(relationKey) { + id := pbtypes.GetString(relation.Details, bundle.RelationKeyId.String()) + derivedObjects[id] = relation.Details + } + } +} + +func (e *export) addOptionIfTag(spaceId string, relation database.Record, derivedObjects map[string]*types.Struct) error { + format := pbtypes.GetInt64(relation.Details, bundle.RelationKeyRelationFormat.String()) + relationKey := pbtypes.GetString(relation.Details, bundle.RelationKeyRelationKey.String()) + if format == int64(model.RelationFormat_tag) || format == int64(model.RelationFormat_status) { + err := e.addRelationOptions(spaceId, relationKey, derivedObjects) + if err != nil { + return err + } + } + return nil +} + +func (e *export) addRelationOptions(spaceId string, relationKey string, derivedObjects map[string]*types.Struct) error { + relationOptions, err := e.getRelationOptions(spaceId, relationKey) + if err != nil { + return err + } + for _, option := range relationOptions { + id := pbtypes.GetString(option.Details, bundle.RelationKeyId.String()) + derivedObjects[id] = option.Details + } + return nil +} + +func (e *export) getRelationOptions(spaceId, relationKey string) ([]database.Record, error) { + relationOptionsDetails, err := e.objectStore.Query(database.Query{ + Filters: []*model.BlockContentDataviewFilter{ + { + RelationKey: bundle.RelationKeyLayout.String(), + Condition: model.BlockContentDataviewFilter_Equal, + Value: pbtypes.Int64(int64(model.ObjectType_relationOption)), }, { - RelationKey: bundle.RelationKeyIsDeleted.String(), + RelationKey: bundle.RelationKeyRelationKey.String(), Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.Bool(false), + Value: pbtypes.String(relationKey), + }, + { + RelationKey: bundle.RelationKeySpaceId.String(), + Condition: model.BlockContentDataviewFilter_Equal, + Value: pbtypes.String(spaceId), }, }, }) if err != nil { return nil, err } - ids := make([]string, 0, len(res)) - for _, r := range res { - id := pbtypes.GetString(r.Details, bundle.RelationKeyId.String()) - docs[id] = r.Details - ids = append(ids, id) - } - var nestedDocsIds []string - if includeNested { - for _, id := range ids { - nestedDocsIds = e.getNested(spaceId, id, docs) - } - } - ids = append(ids, nestedDocsIds...) - if includeFiles { - spc, err := e.spaceService.Get(context.Background(), spaceId) - if err != nil { - return nil, fmt.Errorf("get space: %w", err) - } - for _, id := range ids { - err = e.fillLinkedFiles(spc, id, docs) - if err != nil { - return nil, err - } - } - } - - if !isProtobuf { - return docs, nil - } - - err = e.addDerivedObjects(spaceId, docs, includeNested) - if err != nil { - return nil, err - } - return docs, nil + return relationOptionsDetails, nil } -func (e *export) addDerivedObjects(spaceId string, docs map[string]*types.Struct, includeNested bool) error { - derivedObjects, err := e.getRelatedDerivedObjects(spaceId, docs) +func (e *export) processObjectTypesAndSetOfList(spaceId string, objectTypes []string, allObjects map[string]*types.Struct, setOfList []string) error { + objectDetails, err := e.queryAndFilterObjectsByRelation(spaceId, lo.Union(objectTypes, setOfList), bundle.RelationKeyId.String()) if err != nil { return err } - derivedObjectsMap := make(map[string]*types.Struct) - for _, object := range derivedObjects { - id := pbtypes.GetString(object.Details, bundle.RelationKeyId.String()) - derivedObjectsMap[id] = object.Details + if len(objectDetails) == 0 { + return nil } - if includeNested { - for _, object := range derivedObjects { - id := pbtypes.GetString(object.Details, bundle.RelationKeyId.String()) - e.getNested(spaceId, id, derivedObjectsMap) - } + recommendedRelations, err := e.addObjectsAndCollectRecommendedRelations(objectDetails, allObjects) + if err != nil { + return err } - for id, details := range derivedObjectsMap { - docs[id] = details + err = e.addRecommendedRelations(spaceId, recommendedRelations, allObjects) + if err != nil { + return err } return nil } -func (e *export) getNested(spaceID string, id string, docs map[string]*types.Struct) []string { - store := e.objectStore.SpaceIndex(spaceID) - links, err := store.GetOutboundLinksByID(id) +func (e *export) addObjectsAndCollectRecommendedRelations( + objectTypes []database.Record, + allObjects map[string]*types.Struct, +) ([]string, error) { + recommendedRelations := make([]string, 0, len(objectTypes)) + for i := 0; i < len(objectTypes); i++ { + rawUniqueKey := pbtypes.GetString(objectTypes[i].Details, bundle.RelationKeyUniqueKey.String()) + uniqueKey, err := domain.UnmarshalUniqueKey(rawUniqueKey) + if err != nil { + return nil, err + } + id := pbtypes.GetString(objectTypes[i].Details, bundle.RelationKeyId.String()) + allObjects[id] = objectTypes[i].Details + if uniqueKey.SmartblockType() == smartblock.SmartBlockTypeObjectType { + key, err := domain.GetTypeKeyFromRawUniqueKey(rawUniqueKey) + if err != nil { + return nil, err + } + if bundle.IsInternalType(key) { + continue + } + recommendedRelations = append(recommendedRelations, pbtypes.GetStringList(objectTypes[i].Details, bundle.RelationKeyRecommendedRelations.String())...) + } + } + return recommendedRelations, nil +} + +func (e *export) addRecommendedRelations(spaceId string, recommendedRelations []string, allObjects map[string]*types.Struct) error { + relations, err := e.queryAndFilterObjectsByRelation(spaceId, recommendedRelations, bundle.RelationKeyId.String()) if err != nil { - log.Errorf("export failed to get outbound links for id: %s", err) + return err + } + for _, relation := range relations { + id := pbtypes.GetString(relation.Details, bundle.RelationKeyId.String()) + if id == addr.MissingObject { + continue + } + + relationKey := pbtypes.GetString(relation.Details, bundle.RelationKeyUniqueKey.String()) + uniqueKey, err := domain.UnmarshalUniqueKey(relationKey) + if err != nil { + return err + } + if bundle.IsSystemRelation(domain.RelationKey(uniqueKey.InternalKey())) { + continue + } + allObjects[id] = relation.Details + } + return nil +} + +func (e *export) addNestedObjects(spaceId string, docs map[string]*types.Struct, ids []string, includeFiles bool) error { + nestedDocs := make(map[string]*types.Struct, 0) + for _, id := range ids { + e.addNestedObject(spaceId, id, docs, nestedDocs) + } + if len(nestedDocs) == 0 { return nil } - var nestedDocsIds []string + nestedDocs, err := e.processProtobuf(spaceId, nestedDocs, false, includeFiles) + if err != nil { + return err + } + for id, object := range nestedDocs { + if _, ok := docs[id]; !ok { + docs[id] = object + } + } + return nil +} + +func (e *export) addNestedObject(spaceID string, id string, docs map[string]*types.Struct, nestedDocs map[string]*types.Struct) { + links, err := e.objectStore.GetOutboundLinksByID(id) + if err != nil { + log.Errorf("export failed to get outbound links for id: %s", err) + return + } for _, link := range links { if _, exists := docs[link]; !exists { sbt, sbtErr := e.sbtProvider.Type(spaceID, link) @@ -361,45 +758,58 @@ func (e *export) getNested(spaceID string, id string, docs map[string]*types.Str if !validType(sbt) { continue } - rec, qErr := store.QueryByID([]string{link}) + rec, qErr := e.objectStore.QueryByID([]string{link}) if qErr != nil { log.Errorf("failed to query id %s, err: %s", qErr, err) continue } if isLinkedObjectExist(rec) { + nestedDocs[link] = rec[0].Details docs[link] = rec[0].Details - nestedDocsIds = append(nestedDocsIds, link) - nestedDocsIds = append(nestedDocsIds, e.getNested(spaceID, link, docs)...) + e.addNestedObject(spaceID, link, docs, nestedDocs) } } } - return nestedDocsIds } -func (e *export) fillLinkedFiles(space clientspace.Space, id string, docs map[string]*types.Struct) error { - return space.Do(id, func(b sb.SmartBlock) error { - store := e.objectStore.SpaceIndex(space.Id()) +func (e *export) fillLinkedFiles(space clientspace.Space, id string, docs map[string]*types.Struct) ([]string, error) { + var fileObjectsIds []string + err := space.Do(id, func(b sb.SmartBlock) error { b.NewState().IterateLinkedFiles(func(fileObjectId string) { - details, err := store.GetDetails(fileObjectId) + res, err := e.objectStore.Query(database.Query{ + Filters: []*model.BlockContentDataviewFilter{ + { + RelationKey: bundle.RelationKeyId.String(), + Condition: model.BlockContentDataviewFilter_Equal, + Value: pbtypes.String(fileObjectId), + }, + }, + }) if err != nil { log.Errorf("failed to get details for file object id %s: %v", fileObjectId, err) return } - docs[fileObjectId] = details.GetDetails() - + if len(res) == 0 { + return + } + docs[fileObjectId] = res[0].Details + fileObjectsIds = append(fileObjectsIds, fileObjectId) }) return nil }) + if err != nil { + return nil, err + } + return fileObjectsIds, nil } func (e *export) getExistedObjects(spaceID string, includeArchived bool, isProtobuf bool) (map[string]*types.Struct, error) { - store := e.objectStore.SpaceIndex(spaceID) - res, err := store.List(false) + res, err := e.objectStore.List(spaceID, false) if err != nil { return nil, err } if includeArchived { - archivedObjects, err := store.List(true) + archivedObjects, err := e.objectStore.List(spaceID, true) if err != nil { return nil, err } @@ -717,306 +1127,13 @@ func (e *export) cleanupFile(wr writer) { os.Remove(wr.Path()) } -func (e *export) getRelatedDerivedObjects(spaceId string, objects map[string]*types.Struct) ([]database.Record, error) { - derivedObjects, typesAndTemplates, err := e.iterateObjects(spaceId, objects) - if err != nil { - return nil, err - } - // get derived objects only from types and templates, - // because relations currently have only system relations and object type - if len(typesAndTemplates) > 0 { - derivedObjectsMap := make(map[string]*types.Struct, 0) - for _, object := range typesAndTemplates { - id := pbtypes.GetString(object.Details, bundle.RelationKeyId.String()) - derivedObjectsMap[id] = object.Details - } - iteratedObjects, typesAndTemplates, err := e.iterateObjects(spaceId, derivedObjectsMap) - if err != nil { - return nil, err - } - derivedObjects = append(derivedObjects, iteratedObjects...) - derivedObjects = append(derivedObjects, typesAndTemplates...) - } - return derivedObjects, nil -} - -func (e *export) iterateObjects(spaceId string, objects map[string]*types.Struct, -) (allObjects []database.Record, typesAndTemplates []database.Record, err error) { - var relations []string - for id, object := range objects { - err = cache.Do(e.picker, id, func(b sb.SmartBlock) error { - state := b.NewState() - relations = e.getObjectRelations(state, relations) - details := state.Details() - if e.isObjectWithDataview(details) { - dataviewRelations, err := e.getDataviewRelations(state) - if err != nil { - return err - } - relations = lo.Union(relations, dataviewRelations) - } - return nil - }) - if err != nil { - return nil, nil, err - } - allObjects, typesAndTemplates, err = e.processObject(spaceId, object, allObjects, typesAndTemplates, relations) - if err != nil { - return nil, nil, err +func (e *export) fillTemplateIds(docs map[string]*types.Struct, ids []string) []string { + for id, object := range docs { + if pbtypes.Get(object, bundle.RelationKeyTargetObjectType.String()) != nil { + ids = append(ids, id) } } - return allObjects, typesAndTemplates, nil -} - -func (e *export) getDataviewRelations(state *state.State) ([]string, error) { - var relations []string - err := state.Iterate(func(b simple.Block) (isContinue bool) { - if dataview := b.Model().GetDataview(); dataview != nil { - for _, view := range dataview.Views { - for _, relation := range view.Relations { - relations = append(relations, relation.Key) - } - } - } - return true - }) - return relations, err -} - -func (e *export) getObjectRelations(state *state.State, relations []string) []string { - relationLinks := state.GetRelationLinks() - for _, link := range relationLinks { - relations = append(relations, link.Key) - } - return relations -} - -func (e *export) isObjectWithDataview(details *types.Struct) bool { - return pbtypes.GetFloat64(details, bundle.RelationKeyLayout.String()) == float64(model.ObjectType_collection) || - pbtypes.GetFloat64(details, bundle.RelationKeyLayout.String()) == float64(model.ObjectType_set) -} - -func (e *export) processObject(spaceId string, object *types.Struct, - derivedObjects []database.Record, - typesAndTemplates []database.Record, - relations []string, -) ([]database.Record, []database.Record, error) { - for _, relation := range relations { - storeRelation, err := e.getRelation(spaceId, relation) - if err != nil { - return nil, nil, err - } - if storeRelation != nil { - derivedObjects, err = e.addRelationAndOptions(spaceId, storeRelation, derivedObjects, relation) - if err != nil { - return nil, nil, err - } - } - } - objectTypeId := pbtypes.GetString(object, bundle.RelationKeyType.String()) - - var err error - derivedObjects, typesAndTemplates, err = e.addObjectType(spaceId, objectTypeId, derivedObjects, typesAndTemplates) - if err != nil { - return nil, nil, err - } - - derivedObjects, typesAndTemplates, err = e.addTemplates(spaceId, objectTypeId, derivedObjects, typesAndTemplates) - if err != nil { - return nil, nil, err - } - derivedObjects, err = e.handleSetOfRelation(spaceId, object, derivedObjects) - if err != nil { - return nil, nil, err - } - return derivedObjects, typesAndTemplates, nil -} - -func (e *export) addObjectType(spaceId string, objectTypeId string, derivedObjects []database.Record, typesAndTemplates []database.Record) ([]database.Record, []database.Record, error) { - store := e.objectStore.SpaceIndex(spaceId) - objectTypeDetails, err := store.GetDetails(objectTypeId) - if err != nil { - return nil, nil, err - } - if objectTypeDetails == nil || objectTypeDetails.Details == nil || len(objectTypeDetails.Details.Fields) == 0 { - return derivedObjects, typesAndTemplates, nil - } - uniqueKey := pbtypes.GetString(objectTypeDetails.Details, bundle.RelationKeyUniqueKey.String()) - key, err := domain.GetTypeKeyFromRawUniqueKey(uniqueKey) - if err != nil { - return nil, nil, err - } - if bundle.IsInternalType(key) { - return derivedObjects, typesAndTemplates, nil - } - recommendedRelations := pbtypes.GetStringList(objectTypeDetails.Details, bundle.RelationKeyRecommendedRelations.String()) - for _, relation := range recommendedRelations { - if relation == addr.MissingObject { - continue - } - details, err := store.GetDetails(relation) - if err != nil { - return nil, nil, err - } - relationKey := pbtypes.GetString(details.Details, bundle.RelationKeyUniqueKey.String()) - uniqueKey, err := domain.UnmarshalUniqueKey(relationKey) - if err != nil { - return nil, nil, err - } - if bundle.IsSystemRelation(domain.RelationKey(uniqueKey.InternalKey())) { - continue - } - derivedObjects = append(derivedObjects, database.Record{Details: details.Details}) - } - derivedObjects = append(derivedObjects, database.Record{Details: objectTypeDetails.Details}) - typesAndTemplates = append(typesAndTemplates, database.Record{Details: objectTypeDetails.Details}) - return derivedObjects, typesAndTemplates, nil -} - -func (e *export) getRelation(spaceId string, key string) (*database.Record, error) { - store := e.objectStore.SpaceIndex(spaceId) - uniqueKey, err := domain.NewUniqueKey(smartblock.SmartBlockTypeRelation, key) - if err != nil { - return nil, err - } - relation, err := store.Query(database.Query{ - Filters: []*model.BlockContentDataviewFilter{ - { - RelationKey: bundle.RelationKeyUniqueKey.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.String(uniqueKey.Marshal()), - }, - { - RelationKey: bundle.RelationKeyIsArchived.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.Bool(false), - }, - { - RelationKey: bundle.RelationKeyIsDeleted.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.Bool(false), - }, - }, - }) - if err != nil { - return nil, err - } - if len(relation) == 0 { - return nil, nil - } - return &relation[0], nil -} - -func (e *export) addRelationAndOptions(spaceId string, relation *database.Record, derivedObjects []database.Record, relationKey string) ([]database.Record, error) { - derivedObjects = e.addRelation(*relation, derivedObjects) - format := pbtypes.GetInt64(relation.Details, bundle.RelationKeyRelationFormat.String()) - if format == int64(model.RelationFormat_tag) || format == int64(model.RelationFormat_status) { - relationOptions, err := e.getRelationOptions(spaceId, relationKey) - if err != nil { - return nil, err - } - derivedObjects = append(derivedObjects, relationOptions...) - } - - return derivedObjects, nil -} - -func (e *export) addRelation(relation database.Record, derivedObjects []database.Record) []database.Record { - if relationKey := pbtypes.GetString(relation.Details, bundle.RelationKeyRelationKey.String()); relationKey != "" { - if !bundle.HasRelation(relationKey) { - derivedObjects = append(derivedObjects, relation) - } - } - return derivedObjects -} - -func (e *export) getRelationOptions(spaceId string, relationKey string) ([]database.Record, error) { - relationOptionsDetails, err := e.objectStore.SpaceIndex(spaceId).Query(database.Query{ - Filters: []*model.BlockContentDataviewFilter{ - { - RelationKey: bundle.RelationKeyLayout.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.Int64(int64(model.ObjectType_relationOption)), - }, - { - RelationKey: bundle.RelationKeyRelationKey.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.String(relationKey), - }, - { - RelationKey: bundle.RelationKeyIsArchived.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.Bool(false), - }, - { - RelationKey: bundle.RelationKeyIsDeleted.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.Bool(false), - }, - }, - }) - if err != nil { - return nil, err - } - return relationOptionsDetails, nil -} - -func (e *export) addTemplates(spaceId string, id string, derivedObjects []database.Record, typesAndTemplates []database.Record) ([]database.Record, []database.Record, error) { - templates, err := e.objectStore.SpaceIndex(spaceId).Query(database.Query{ - Filters: []*model.BlockContentDataviewFilter{ - { - RelationKey: bundle.RelationKeyTargetObjectType.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.String(id), - }, - { - RelationKey: bundle.RelationKeyIsArchived.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.Bool(false), - }, - { - RelationKey: bundle.RelationKeyIsDeleted.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.Bool(false), - }, - }, - }) - if err != nil { - return nil, nil, err - } - derivedObjects = append(derivedObjects, templates...) - typesAndTemplates = append(typesAndTemplates, templates...) - return derivedObjects, typesAndTemplates, nil -} - -func (e *export) handleSetOfRelation(spaceId string, object *types.Struct, derivedObjects []database.Record) ([]database.Record, error) { - setOfList := pbtypes.GetStringList(object, bundle.RelationKeySetOf.String()) - if len(setOfList) > 0 { - types, err := e.objectStore.SpaceIndex(spaceId).Query(database.Query{ - Filters: []*model.BlockContentDataviewFilter{ - { - RelationKey: bundle.RelationKeyId.String(), - Condition: model.BlockContentDataviewFilter_In, - Value: pbtypes.StringList(setOfList), - }, - { - RelationKey: bundle.RelationKeyIsArchived.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.Bool(false), - }, - { - RelationKey: bundle.RelationKeyIsDeleted.String(), - Condition: model.BlockContentDataviewFilter_Equal, - Value: pbtypes.Bool(false), - }, - }, - }) - if err != nil { - return nil, err - } - derivedObjects = append(derivedObjects, types...) - } - return derivedObjects, nil + return ids } func isLinkedObjectExist(rec []database.Record) bool { diff --git a/core/block/export/export_test.go b/core/block/export/export_test.go index f4be82d85..b34e1b5b4 100644 --- a/core/block/export/export_test.go +++ b/core/block/export/export_test.go @@ -8,10 +8,12 @@ import ( "github.com/gogo/protobuf/types" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/anyproto/anytype-heart/core/block/cache/mock_cache" "github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest" "github.com/anyproto/anytype-heart/core/block/editor/state" + "github.com/anyproto/anytype-heart/core/block/simple" "github.com/anyproto/anytype-heart/core/converter/pbjson" "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/pb" @@ -20,6 +22,8 @@ import ( "github.com/anyproto/anytype-heart/pkg/lib/localstore/addr" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" + "github.com/anyproto/anytype-heart/space/clientspace/mock_clientspace" + "github.com/anyproto/anytype-heart/space/mock_space" "github.com/anyproto/anytype-heart/space/spacecore/typeprovider/mock_typeprovider" "github.com/anyproto/anytype-heart/util/pbtypes" ) @@ -55,12 +59,14 @@ func Test_docsForExport(t *testing.T) { storeFixture := objectstore.NewStoreFixture(t) storeFixture.AddObjects(t, spaceId, []objectstore.TestObject{ { - bundle.RelationKeyId: pbtypes.String("id"), - bundle.RelationKeyName: pbtypes.String("name1"), + bundle.RelationKeyId: pbtypes.String("id"), + bundle.RelationKeyName: pbtypes.String("name1"), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, { - bundle.RelationKeyId: pbtypes.String("id1"), - bundle.RelationKeyName: pbtypes.String("name2"), + bundle.RelationKeyId: pbtypes.String("id1"), + bundle.RelationKeyName: pbtypes.String("name2"), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, }) err := storeFixture.SpaceIndex(spaceId).UpdateObjectLinks(context.Background(), "id", []string{"id1"}) @@ -89,12 +95,14 @@ func Test_docsForExport(t *testing.T) { storeFixture := objectstore.NewStoreFixture(t) storeFixture.AddObjects(t, spaceId, []objectstore.TestObject{ { - bundle.RelationKeyId: pbtypes.String("id"), - bundle.RelationKeyName: pbtypes.String("name"), + bundle.RelationKeyId: pbtypes.String("id"), + bundle.RelationKeyName: pbtypes.String("name"), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, { bundle.RelationKeyId: pbtypes.String("id1"), bundle.RelationKeyIsDeleted: pbtypes.Bool(true), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, }) err := storeFixture.SpaceIndex(spaceId).UpdateObjectLinks(context.Background(), "id", []string{"id1"}) @@ -127,6 +135,7 @@ func Test_docsForExport(t *testing.T) { bundle.RelationKeyId: pbtypes.String("id"), domain.RelationKey(relationKey): pbtypes.String("value"), bundle.RelationKeyType: pbtypes.String("objectType"), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, }) err := storeFixture.SpaceIndex(spaceId).UpdateObjectLinks(context.Background(), "id", []string{"id1"}) @@ -148,7 +157,24 @@ func Test_docsForExport(t *testing.T) { Format: model.RelationFormat_longtext, }) smartBlockTest.Doc = doc + + objectType := smarttest.New("objectType") + objectTypeDoc := objectType.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String("objectType"), + bundle.RelationKeyType.String(): pbtypes.String("objectType"), + }}) + objectTypeDoc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + objectType.Doc = objectTypeDoc + objectGetter.EXPECT().GetObject(context.Background(), "id").Return(smartBlockTest, nil) + objectGetter.EXPECT().GetObject(context.Background(), "objectType").Return(objectType, nil) e := &export{ objectStore: storeFixture, @@ -178,11 +204,13 @@ func Test_docsForExport(t *testing.T) { bundle.RelationKeyId: pbtypes.String("id"), domain.RelationKey(relationKey): pbtypes.String("value"), bundle.RelationKeyType: pbtypes.String("objectType"), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, { bundle.RelationKeyId: pbtypes.String(relationKey), bundle.RelationKeyRelationKey: pbtypes.String(relationKey), bundle.RelationKeyUniqueKey: pbtypes.String(uniqueKey.Marshal()), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, }) @@ -207,7 +235,23 @@ func Test_docsForExport(t *testing.T) { }) smartBlockTest.Doc = doc + objectType := smarttest.New("objectType") + objectTypeDoc := objectType.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String("objectType"), + bundle.RelationKeyType.String(): pbtypes.String("objectType"), + }}) + objectTypeDoc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + objectType.Doc = objectTypeDoc + objectGetter.EXPECT().GetObject(context.Background(), "id").Return(smartBlockTest, nil) + objectGetter.EXPECT().GetObject(context.Background(), "objectType").Return(objectType, nil) e := &export{ objectStore: storeFixture, @@ -238,12 +282,14 @@ func Test_docsForExport(t *testing.T) { bundle.RelationKeyId: pbtypes.String("id"), domain.RelationKey(relationKey): pbtypes.String("value"), bundle.RelationKeyType: pbtypes.String("objectType"), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, { bundle.RelationKeyId: pbtypes.String(relationKey), bundle.RelationKeyRelationKey: pbtypes.String(relationKey), bundle.RelationKeyUniqueKey: pbtypes.String(uniqueKey.Marshal()), bundle.RelationKeyRelationFormat: pbtypes.Int64(int64(model.RelationFormat_status)), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, }) @@ -265,7 +311,23 @@ func Test_docsForExport(t *testing.T) { }) smartBlockTest.Doc = doc + objectType := smarttest.New("objectType") + objectTypeDoc := objectType.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String("objectType"), + bundle.RelationKeyType.String(): pbtypes.String("objectType"), + }}) + objectTypeDoc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + objectType.Doc = objectTypeDoc + objectGetter.EXPECT().GetObject(context.Background(), "id").Return(smartBlockTest, nil) + objectGetter.EXPECT().GetObject(context.Background(), "objectType").Return(objectType, nil) e := &export{ objectStore: storeFixture, @@ -298,6 +360,7 @@ func Test_docsForExport(t *testing.T) { bundle.RelationKeyId: pbtypes.String("id"), domain.RelationKey(relationKey): pbtypes.String(optionId), bundle.RelationKeyType: pbtypes.String("objectType"), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, { bundle.RelationKeyId: pbtypes.String(relationKey), @@ -305,12 +368,14 @@ func Test_docsForExport(t *testing.T) { bundle.RelationKeyUniqueKey: pbtypes.String(uniqueKey.Marshal()), bundle.RelationKeyRelationFormat: pbtypes.Int64(int64(model.RelationFormat_tag)), bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_relation)), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, { bundle.RelationKeyId: pbtypes.String(optionId), bundle.RelationKeyRelationKey: pbtypes.String(relationKey), bundle.RelationKeyUniqueKey: pbtypes.String(optionUniqueKey.Marshal()), bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_relationOption)), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, }) @@ -332,7 +397,23 @@ func Test_docsForExport(t *testing.T) { }) smartBlockTest.Doc = doc + objectType := smarttest.New("objectType") + objectTypeDoc := objectType.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String("objectType"), + bundle.RelationKeyType.String(): pbtypes.String("objectType"), + }}) + objectTypeDoc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + objectType.Doc = objectTypeDoc + objectGetter.EXPECT().GetObject(context.Background(), "id").Return(smartBlockTest, nil) + objectGetter.EXPECT().GetObject(context.Background(), "objectType").Return(objectType, nil) e := &export{ objectStore: storeFixture, @@ -370,6 +451,7 @@ func Test_docsForExport(t *testing.T) { assert.Nil(t, err) templateId := "templateId" + templateObjectTypeId := "templateObjectTypeId" linkedObjectId := "linkedObjectId" storeFixture.AddObjects(t, spaceId, []objectstore.TestObject{ @@ -377,32 +459,40 @@ func Test_docsForExport(t *testing.T) { bundle.RelationKeyId: pbtypes.String("id"), domain.RelationKey(relationKey): pbtypes.String("test"), bundle.RelationKeyType: pbtypes.String(objectTypeKey), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, { bundle.RelationKeyId: pbtypes.String(relationKey), bundle.RelationKeyRelationKey: pbtypes.String(relationKey), bundle.RelationKeyUniqueKey: pbtypes.String(uniqueKey.Marshal()), bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_relation)), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, { bundle.RelationKeyId: pbtypes.String(objectTypeKey), bundle.RelationKeyUniqueKey: pbtypes.String(objectTypeUniqueKey.Marshal()), bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_objectType)), bundle.RelationKeyRecommendedRelations: pbtypes.StringList([]string{recommendedRelationKey}), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + bundle.RelationKeyType: pbtypes.String(objectTypeKey), }, { bundle.RelationKeyId: pbtypes.String(recommendedRelationKey), bundle.RelationKeyRelationKey: pbtypes.String(recommendedRelationKey), bundle.RelationKeyUniqueKey: pbtypes.String(recommendedRelationUniqueKey.Marshal()), bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_relation)), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, { bundle.RelationKeyId: pbtypes.String(templateId), bundle.RelationKeyTargetObjectType: pbtypes.String(objectTypeKey), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + bundle.RelationKeyType: pbtypes.String(templateObjectTypeId), }, { - bundle.RelationKeyId: pbtypes.String(linkedObjectId), - bundle.RelationKeyType: pbtypes.String(objectTypeKey), + bundle.RelationKeyId: pbtypes.String(linkedObjectId), + bundle.RelationKeyType: pbtypes.String(objectTypeKey), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, }) @@ -411,14 +501,27 @@ func Test_docsForExport(t *testing.T) { objectGetter := mock_cache.NewMockObjectGetter(t) + template := smarttest.New(templateId) + templateDoc := template.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String(templateId), + bundle.RelationKeyType.String(): pbtypes.String(templateObjectTypeId), + }}) + templateDoc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + template.Doc = templateDoc + smartBlockTest := smarttest.New("id") - smartBlockTemplate := smarttest.New(templateId) - smartBlockObjectType := smarttest.New(objectTypeKey) doc := smartBlockTest.NewState().SetDetails(&types.Struct{ Fields: map[string]*types.Value{ bundle.RelationKeyId.String(): pbtypes.String("id"), relationKey: pbtypes.String("value"), - bundle.RelationKeyType.String(): pbtypes.String("objectType"), + bundle.RelationKeyType.String(): pbtypes.String(objectTypeKey), }}) doc.AddRelationLinks(&model.RelationLink{ Key: bundle.RelationKeyId.String(), @@ -429,9 +532,56 @@ func Test_docsForExport(t *testing.T) { }) smartBlockTest.Doc = doc + objectType := smarttest.New(objectTypeKey) + objectTypeDoc := objectType.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String(objectTypeKey), + bundle.RelationKeyType.String(): pbtypes.String(objectTypeKey), + }}) + objectTypeDoc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + objectType.Doc = objectTypeDoc + + templateObjectType := smarttest.New(objectTypeKey) + templateObjectTypeDoc := templateObjectType.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String(templateId), + bundle.RelationKeyType.String(): pbtypes.String(templateObjectTypeId), + }}) + templateObjectTypeDoc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + templateObjectType.Doc = templateObjectTypeDoc + + linkedObject := smarttest.New(objectTypeKey) + linkedObjectDoc := smartBlockTest.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String(linkedObjectId), + bundle.RelationKeyType.String(): pbtypes.String(objectTypeKey), + }}) + linkedObjectDoc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + linkedObject.Doc = linkedObjectDoc + objectGetter.EXPECT().GetObject(context.Background(), "id").Return(smartBlockTest, nil) - objectGetter.EXPECT().GetObject(context.Background(), templateId).Return(smartBlockTemplate, nil) - objectGetter.EXPECT().GetObject(context.Background(), objectTypeKey).Return(smartBlockObjectType, nil) + objectGetter.EXPECT().GetObject(context.Background(), templateId).Return(template, nil) + objectGetter.EXPECT().GetObject(context.Background(), objectTypeKey).Return(objectType, nil) + objectGetter.EXPECT().GetObject(context.Background(), templateObjectTypeId).Return(templateObjectType, nil) + objectGetter.EXPECT().GetObject(context.Background(), linkedObjectId).Return(linkedObject, nil) provider := mock_typeprovider.NewMockSmartBlockTypeProvider(t) provider.EXPECT().Type(spaceId, linkedObjectId).Return(smartblock.SmartBlockTypePage, nil) @@ -457,30 +607,59 @@ func Test_docsForExport(t *testing.T) { t.Run("get derived objects, object type have missing relations - return only object and its type", func(t *testing.T) { // given storeFixture := objectstore.NewStoreFixture(t) - objectTypeKey := "customObjectType" - objectTypeUniqueKey, err := domain.NewUniqueKey(smartblock.SmartBlockTypeObjectType, objectTypeKey) + objectTypeId := "customObjectType" + objectTypeUniqueKey, err := domain.NewUniqueKey(smartblock.SmartBlockTypeObjectType, objectTypeId) assert.Nil(t, err) storeFixture.AddObjects(t, spaceId, []objectstore.TestObject{ { - bundle.RelationKeyId: pbtypes.String("id"), - bundle.RelationKeyType: pbtypes.String(objectTypeKey), + bundle.RelationKeyId: pbtypes.String("id"), + bundle.RelationKeyType: pbtypes.String(objectTypeId), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, { - bundle.RelationKeyId: pbtypes.String(objectTypeKey), + bundle.RelationKeyId: pbtypes.String(objectTypeId), bundle.RelationKeyUniqueKey: pbtypes.String(objectTypeUniqueKey.Marshal()), bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_objectType)), bundle.RelationKeyRecommendedRelations: pbtypes.StringList([]string{addr.MissingObject}), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), }, }) objectGetter := mock_cache.NewMockObjectGetter(t) smartBlockTest := smarttest.New("id") - smartBlockObjectType := smarttest.New(objectTypeKey) + doc := smartBlockTest.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String("id"), + bundle.RelationKeyType.String(): pbtypes.String(objectTypeId), + }}) + doc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + smartBlockTest.Doc = doc + + objectType := smarttest.New(objectTypeId) + objectTypeDoc := objectType.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String(objectTypeId), + bundle.RelationKeyType.String(): pbtypes.String(objectTypeId), + }}) + objectTypeDoc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + objectType.Doc = objectTypeDoc objectGetter.EXPECT().GetObject(context.Background(), "id").Return(smartBlockTest, nil) - objectGetter.EXPECT().GetObject(context.Background(), objectTypeKey).Return(smartBlockObjectType, nil) + objectGetter.EXPECT().GetObject(context.Background(), objectTypeId).Return(objectType, nil) e := &export{ objectStore: storeFixture, @@ -498,6 +677,480 @@ func Test_docsForExport(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 2, len(docsForExport)) }) + t.Run("objects without links", func(t *testing.T) { + // given + storeFixture := objectstore.NewStoreFixture(t) + objectTypeId := "objectTypeId" + objectTypeUniqueKey, err := domain.NewUniqueKey(smartblock.SmartBlockTypeObjectType, objectTypeId) + assert.Nil(t, err) + + storeFixture.AddObjects(t, []objectstore.TestObject{ + { + bundle.RelationKeyId: pbtypes.String("id"), + bundle.RelationKeyName: pbtypes.String("name1"), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + bundle.RelationKeyType: pbtypes.String(objectTypeId), + }, + { + bundle.RelationKeyId: pbtypes.String("id1"), + bundle.RelationKeyName: pbtypes.String("name2"), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + bundle.RelationKeyType: pbtypes.String(objectTypeId), + }, + { + bundle.RelationKeyId: pbtypes.String(objectTypeId), + bundle.RelationKeyUniqueKey: pbtypes.String(objectTypeUniqueKey.Marshal()), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_objectType)), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + bundle.RelationKeyType: pbtypes.String(objectTypeId), + }, + }) + + smartBlockTest := smarttest.New("id") + doc := smartBlockTest.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String("id"), + bundle.RelationKeyType.String(): pbtypes.String(objectTypeId), + }}) + doc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + smartBlockTest.Doc = doc + + objectType := smarttest.New(objectTypeId) + objectTypeDoc := objectType.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String(objectTypeId), + bundle.RelationKeyType.String(): pbtypes.String(objectTypeId), + }}) + objectTypeDoc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + objectType.Doc = objectTypeDoc + + objectGetter := mock_cache.NewMockObjectGetter(t) + objectGetter.EXPECT().GetObject(context.Background(), "id").Return(smartBlockTest, nil) + objectGetter.EXPECT().GetObject(context.Background(), objectTypeId).Return(objectType, nil) + + e := &export{ + objectStore: storeFixture, + picker: objectGetter, + } + + // when + docsForExport, err := e.docsForExport("spaceId", pb.RpcObjectListExportRequest{ + SpaceId: "spaceId", + ObjectIds: []string{"id"}, + IncludeNested: true, + Format: model.Export_Protobuf, + }) + + // then + assert.Nil(t, err) + assert.Equal(t, 2, len(docsForExport)) + }) + t.Run("objects with dataview", func(t *testing.T) { + // given + storeFixture := objectstore.NewStoreFixture(t) + objectTypeId := "objectTypeId" + objectTypeUniqueKey, err := domain.NewUniqueKey(smartblock.SmartBlockTypeObjectType, objectTypeId) + assert.Nil(t, err) + + relationKey := "key" + relationKeyUniqueKey, err := domain.NewUniqueKey(smartblock.SmartBlockTypeRelation, relationKey) + assert.Nil(t, err) + + storeFixture.AddObjects(t, []objectstore.TestObject{ + { + bundle.RelationKeyId: pbtypes.String("id"), + bundle.RelationKeyName: pbtypes.String("name1"), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + bundle.RelationKeyType: pbtypes.String(objectTypeId), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_set)), + }, + { + bundle.RelationKeyId: pbtypes.String(objectTypeId), + bundle.RelationKeyUniqueKey: pbtypes.String(objectTypeUniqueKey.Marshal()), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_objectType)), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + bundle.RelationKeyType: pbtypes.String(objectTypeId), + }, + { + bundle.RelationKeyId: pbtypes.String(bundle.RelationKeyTag.String()), + bundle.RelationKeyName: pbtypes.String(bundle.RelationKeyTag.String()), + bundle.RelationKeyRelationKey: pbtypes.String(bundle.RelationKeyTag.String()), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_relation)), + bundle.RelationKeyUniqueKey: pbtypes.String(bundle.RelationKeyTag.URL()), + }, + { + bundle.RelationKeyId: pbtypes.String(relationKey), + bundle.RelationKeyName: pbtypes.String(relationKey), + bundle.RelationKeyRelationKey: pbtypes.String(relationKey), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_relation)), + bundle.RelationKeyUniqueKey: pbtypes.String(relationKeyUniqueKey.Marshal()), + }, + }) + + smartBlockTest := smarttest.New("id") + doc := smartBlockTest.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String("id"), + bundle.RelationKeyType.String(): pbtypes.String(objectTypeId), + bundle.RelationKeyLayout.String(): pbtypes.Int64(int64(model.ObjectType_set)), + }}) + doc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyLayout.String(), + Format: model.RelationFormat_number, + }) + doc.Set(simple.New(&model.Block{ + Id: "id", + ChildrenIds: []string{"blockId"}, + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}, + })) + + doc.Set(simple.New(&model.Block{ + Id: "blockId", + Content: &model.BlockContentOfDataview{Dataview: &model.BlockContentDataview{ + Views: []*model.BlockContentDataviewView{ + { + Relations: []*model.BlockContentDataviewRelation{ + { + Key: bundle.RelationKeyTag.String(), + }, + { + Key: relationKey, + }, + }, + }, + }, + }}, + })) + smartBlockTest.Doc = doc + + objectType := smarttest.New(objectTypeId) + objectTypeDoc := objectType.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String(objectTypeId), + bundle.RelationKeyType.String(): pbtypes.String(objectTypeId), + }}) + objectTypeDoc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + objectType.Doc = objectTypeDoc + + objectGetter := mock_cache.NewMockObjectGetter(t) + objectGetter.EXPECT().GetObject(context.Background(), "id").Return(smartBlockTest, nil) + objectGetter.EXPECT().GetObject(context.Background(), objectTypeId).Return(objectType, nil) + + e := &export{ + objectStore: storeFixture, + picker: objectGetter, + } + + // when + docsForExport, err := e.docsForExport("spaceId", pb.RpcObjectListExportRequest{ + SpaceId: "spaceId", + ObjectIds: []string{"id"}, + IncludeNested: true, + Format: model.Export_Protobuf, + }) + + // then + assert.Nil(t, err) + assert.Equal(t, 3, len(docsForExport)) + }) + t.Run("objects without file", func(t *testing.T) { + // given + storeFixture := objectstore.NewStoreFixture(t) + objectTypeId := "objectTypeId" + objectTypeUniqueKey, err := domain.NewUniqueKey(smartblock.SmartBlockTypeObjectType, objectTypeId) + assert.Nil(t, err) + + storeFixture.AddObjects(t, []objectstore.TestObject{ + { + bundle.RelationKeyId: pbtypes.String("id"), + bundle.RelationKeyName: pbtypes.String("name1"), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + bundle.RelationKeyType: pbtypes.String(objectTypeId), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_set)), + }, + { + bundle.RelationKeyId: pbtypes.String(objectTypeId), + bundle.RelationKeyUniqueKey: pbtypes.String(objectTypeUniqueKey.Marshal()), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_objectType)), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + bundle.RelationKeyType: pbtypes.String(objectTypeId), + }, + }) + + smartBlockTest := smarttest.New("id") + doc := smartBlockTest.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String("id"), + bundle.RelationKeyType.String(): pbtypes.String(objectTypeId), + bundle.RelationKeyLayout.String(): pbtypes.Int64(int64(model.ObjectType_set)), + }}) + doc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyLayout.String(), + Format: model.RelationFormat_number, + }) + smartBlockTest.Doc = doc + + objectType := smarttest.New(objectTypeId) + objectTypeDoc := objectType.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String(objectTypeId), + bundle.RelationKeyType.String(): pbtypes.String(objectTypeId), + }}) + objectTypeDoc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + objectType.Doc = objectTypeDoc + + objectGetter := mock_cache.NewMockObjectGetter(t) + objectGetter.EXPECT().GetObject(context.Background(), "id").Return(smartBlockTest, nil) + objectGetter.EXPECT().GetObject(context.Background(), objectTypeId).Return(objectType, nil) + + service := mock_space.NewMockService(t) + space := mock_clientspace.NewMockSpace(t) + space.EXPECT().Do("id", mock.Anything).Return(nil) + + service.EXPECT().Get(context.Background(), "spaceId").Return(space, nil) + e := &export{ + objectStore: storeFixture, + picker: objectGetter, + spaceService: service, + } + + // when + docsForExport, err := e.docsForExport("spaceId", pb.RpcObjectListExportRequest{ + SpaceId: "spaceId", + ObjectIds: []string{"id"}, + IncludeFiles: true, + IncludeNested: true, + Format: model.Export_Protobuf, + }) + + // then + assert.Nil(t, err) + assert.Equal(t, 2, len(docsForExport)) + }) + t.Run("objects without file, not protobuf export", func(t *testing.T) { + // given + storeFixture := objectstore.NewStoreFixture(t) + objectTypeId := "objectTypeId" + objectTypeUniqueKey, err := domain.NewUniqueKey(smartblock.SmartBlockTypeObjectType, objectTypeId) + assert.Nil(t, err) + + storeFixture.AddObjects(t, []objectstore.TestObject{ + { + bundle.RelationKeyId: pbtypes.String("id"), + bundle.RelationKeyName: pbtypes.String("name1"), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + bundle.RelationKeyType: pbtypes.String(objectTypeId), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_set)), + }, + { + bundle.RelationKeyId: pbtypes.String(objectTypeId), + bundle.RelationKeyUniqueKey: pbtypes.String(objectTypeUniqueKey.Marshal()), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_objectType)), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + bundle.RelationKeyType: pbtypes.String(objectTypeId), + }, + }) + + service := mock_space.NewMockService(t) + space := mock_clientspace.NewMockSpace(t) + space.EXPECT().Do("id", mock.Anything).Return(nil) + service.EXPECT().Get(context.Background(), "spaceId").Return(space, nil) + e := &export{ + objectStore: storeFixture, + spaceService: service, + } + + // when + docsForExport, err := e.docsForExport("spaceId", pb.RpcObjectListExportRequest{ + SpaceId: "spaceId", + ObjectIds: []string{"id"}, + IncludeFiles: true, + IncludeNested: true, + Format: model.Export_Markdown, + }) + + // then + assert.Nil(t, err) + assert.Equal(t, 1, len(docsForExport)) + }) + + t.Run("get derived objects - relation, object type with recommended relations, template with link", func(t *testing.T) { + // given + storeFixture := objectstore.NewStoreFixture(t) + objectTypeKey := "customObjectType" + objectTypeUniqueKey, err := domain.NewUniqueKey(smartblock.SmartBlockTypeObjectType, objectTypeKey) + assert.Nil(t, err) + + relationKey := "key" + uniqueKey, err := domain.NewUniqueKey(smartblock.SmartBlockTypeRelation, relationKey) + assert.Nil(t, err) + + recommendedRelationKey := "recommendedRelationKey" + recommendedRelationUniqueKey, err := domain.NewUniqueKey(smartblock.SmartBlockTypeRelation, recommendedRelationKey) + assert.Nil(t, err) + + relationObjectTypeKey := "relation" + relationObjectTypeUK, err := domain.NewUniqueKey(smartblock.SmartBlockTypeObjectType, relationObjectTypeKey) + assert.Nil(t, err) + + storeFixture.AddObjects(t, []objectstore.TestObject{ + { + bundle.RelationKeyId: pbtypes.String("id"), + bundle.RelationKeySetOf: pbtypes.StringList([]string{relationKey}), + bundle.RelationKeyType: pbtypes.String(objectTypeKey), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + }, + { + bundle.RelationKeyId: pbtypes.String(relationKey), + bundle.RelationKeyRelationKey: pbtypes.String(relationKey), + bundle.RelationKeyUniqueKey: pbtypes.String(uniqueKey.Marshal()), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_relation)), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + bundle.RelationKeyType: pbtypes.String(relationObjectTypeKey), + }, + { + bundle.RelationKeyId: pbtypes.String(objectTypeKey), + bundle.RelationKeyUniqueKey: pbtypes.String(objectTypeUniqueKey.Marshal()), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_objectType)), + bundle.RelationKeyRecommendedRelations: pbtypes.StringList([]string{recommendedRelationKey}), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + bundle.RelationKeyType: pbtypes.String(objectTypeKey), + }, + { + bundle.RelationKeyId: pbtypes.String(relationObjectTypeKey), + bundle.RelationKeyUniqueKey: pbtypes.String(relationObjectTypeUK.Marshal()), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_objectType)), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + bundle.RelationKeyType: pbtypes.String(objectTypeKey), + }, + { + bundle.RelationKeyId: pbtypes.String(recommendedRelationKey), + bundle.RelationKeyRelationKey: pbtypes.String(recommendedRelationKey), + bundle.RelationKeyUniqueKey: pbtypes.String(recommendedRelationUniqueKey.Marshal()), + bundle.RelationKeyLayout: pbtypes.Int64(int64(model.ObjectType_relation)), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + }, + }) + + objectGetter := mock_cache.NewMockObjectGetter(t) + + smartBlockTest := smarttest.New("id") + doc := smartBlockTest.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String("id"), + bundle.RelationKeySetOf.String(): pbtypes.StringList([]string{relationKey}), + bundle.RelationKeyType.String(): pbtypes.String(objectTypeKey), + }}) + doc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }) + smartBlockTest.Doc = doc + + objectType := smarttest.New(objectTypeKey) + objectTypeDoc := objectType.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String(objectTypeKey), + bundle.RelationKeyType.String(): pbtypes.String(objectTypeKey), + }}) + objectTypeDoc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + objectType.Doc = objectTypeDoc + + relationObject := smarttest.New(relationKey) + relationObjectDoc := relationObject.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String(relationKey), + bundle.RelationKeyType.String(): pbtypes.String(relationObjectTypeKey), + }}) + relationObjectDoc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + relationObject.Doc = relationObjectDoc + + relationObjectType := smarttest.New(relationObjectTypeKey) + relationObjectTypeDoc := relationObjectType.NewState().SetDetails(&types.Struct{ + Fields: map[string]*types.Value{ + bundle.RelationKeyId.String(): pbtypes.String(relationObjectTypeKey), + bundle.RelationKeyType.String(): pbtypes.String(objectTypeKey), + }}) + relationObjectTypeDoc.AddRelationLinks(&model.RelationLink{ + Key: bundle.RelationKeyId.String(), + Format: model.RelationFormat_longtext, + }, &model.RelationLink{ + Key: bundle.RelationKeyType.String(), + Format: model.RelationFormat_longtext, + }) + relationObjectType.Doc = relationObjectTypeDoc + + objectGetter.EXPECT().GetObject(context.Background(), "id").Return(smartBlockTest, nil) + objectGetter.EXPECT().GetObject(context.Background(), objectTypeKey).Return(objectType, nil) + objectGetter.EXPECT().GetObject(context.Background(), relationKey).Return(relationObject, nil) + objectGetter.EXPECT().GetObject(context.Background(), relationObjectTypeKey).Return(relationObjectType, nil) + + e := &export{ + objectStore: storeFixture, + picker: objectGetter, + } + + // when + docsForExport, err := e.docsForExport("spaceId", pb.RpcObjectListExportRequest{ + SpaceId: "spaceId", + ObjectIds: []string{"id"}, + Format: model.Export_Protobuf, + }) + + // then + assert.Nil(t, err) + assert.Equal(t, 5, len(docsForExport)) + }) } func Test_provideFileName(t *testing.T) { @@ -564,3 +1217,52 @@ func Test_provideFileName(t *testing.T) { assert.Equal(t, spaceDirectory+string(filepath.Separator)+spaceId+string(filepath.Separator)+filesObjects+string(filepath.Separator)+"docId.pb.json", fileName) }) } + +func Test_queryObjectsFromStoreByIds(t *testing.T) { + t.Run("query 10 objects", func(t *testing.T) { + // given + fixture := objectstore.NewStoreFixture(t) + ids := make([]string, 0, 10) + for i := 0; i < 10; i++ { + id := fmt.Sprintf("%d", i) + fixture.AddObjects(t, []objectstore.TestObject{ + { + bundle.RelationKeyId: pbtypes.String(id), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + }, + }) + ids = append(ids, id) + } + e := &export{objectStore: fixture} + + // when + records, err := e.queryAndFilterObjectsByRelation("spaceId", ids, bundle.RelationKeyId.String()) + + // then + assert.Nil(t, err) + assert.Len(t, records, 10) + }) + t.Run("query 2000 objects", func(t *testing.T) { + // given + fixture := objectstore.NewStoreFixture(t) + ids := make([]string, 0, 2000) + for i := 0; i < 2000; i++ { + id := fmt.Sprintf("%d", i) + fixture.AddObjects(t, []objectstore.TestObject{ + { + bundle.RelationKeyId: pbtypes.String(id), + bundle.RelationKeySpaceId: pbtypes.String("spaceId"), + }, + }) + ids = append(ids, id) + } + e := &export{objectStore: fixture} + + // when + records, err := e.queryAndFilterObjectsByRelation("spaceId", ids, bundle.RelationKeyId.String()) + + // then + assert.Nil(t, err) + assert.Len(t, records, 2000) + }) +} diff --git a/go.sum b/go.sum index 4b4f0b3b5..d21e6c411 100644 --- a/go.sum +++ b/go.sum @@ -113,8 +113,8 @@ github.com/anyproto/protobuf v1.3.3-0.20240814124528-72b8c7e0e0f5 h1:aY7tBzQ+z8H github.com/anyproto/protobuf v1.3.3-0.20240814124528-72b8c7e0e0f5/go.mod h1:5+PHE01DgsDPkralb8MYmGg2sPQahsqEJ9ue7ciDHKg= github.com/anyproto/ristretto v0.1.2-0.20240221153107-2b23839cc50c h1:GicoaTUyB2mtCIl3YMrO0OzysqRT5GA4vuvDsqEkhSM= github.com/anyproto/ristretto v0.1.2-0.20240221153107-2b23839cc50c/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= -github.com/anyproto/tantivy-go v0.1.0 h1:5SuItuJnYAEK9U6cfxPnwFUXGboURIwF4JAp7s46rII= -github.com/anyproto/tantivy-go v0.1.0/go.mod h1:Pl0zaeEX7dkVDTDKROzlLlGCVjbZHjyPrZloFuVEASc= +github.com/anyproto/tantivy-go v0.1.1 h1:IdeZca63wV7/C+wMpkntImdh/mkkEddQJTShGEVxGe0= +github.com/anyproto/tantivy-go v0.1.1/go.mod h1:Pl0zaeEX7dkVDTDKROzlLlGCVjbZHjyPrZloFuVEASc= github.com/anyproto/tantivy-go v0.1.1 h1:IdeZca63wV7/C+wMpkntImdh/mkkEddQJTShGEVxGe0= github.com/anyproto/tantivy-go v0.1.1/go.mod h1:Pl0zaeEX7dkVDTDKROzlLlGCVjbZHjyPrZloFuVEASc= github.com/anyproto/zeroconf/v2 v2.2.1-0.20240228113933-f90a5cc4439d h1:5bj7nX/AS8sxGpTIrapE7PC4oPlhkHMwMqXlJbUHBlg=