diff --git a/core/block/editor/widget.go b/core/block/editor/widget.go index 1aa41ad94..53a24550b 100644 --- a/core/block/editor/widget.go +++ b/core/block/editor/widget.go @@ -28,6 +28,7 @@ type WidgetObject struct { basic.Unlinkable basic.Updatable widget.Widget + basic.DetailsSettable } func NewWidgetObject( @@ -37,11 +38,12 @@ func NewWidgetObject( ) *WidgetObject { bs := basic.NewBasic(sb, objectStore, layoutConverter, nil) return &WidgetObject{ - SmartBlock: sb, - Movable: bs, - Updatable: bs, - IHistory: basic.NewHistory(sb), - Widget: widget.NewWidget(sb), + SmartBlock: sb, + Movable: bs, + Updatable: bs, + DetailsSettable: bs, + IHistory: basic.NewHistory(sb), + Widget: widget.NewWidget(sb), } } diff --git a/core/block/import/common/error.go b/core/block/import/common/error.go index a099c7fd9..69276bd1e 100644 --- a/core/block/import/common/error.go +++ b/core/block/import/common/error.go @@ -67,6 +67,13 @@ func (ce *ConvertError) IsEmpty() bool { return ce == nil || len(ce.errors) == 0 } +func (ce *ConvertError) ErrorOrNil() *ConvertError { + if ce.IsEmpty() { + return nil + } + return ce +} + func (ce *ConvertError) Error() error { var pattern = "error: %s" + "\n" var errorString bytes.Buffer diff --git a/core/block/import/common/types.go b/core/block/import/common/types.go index d1ee5f169..3049f8aca 100644 --- a/core/block/import/common/types.go +++ b/core/block/import/common/types.go @@ -141,3 +141,67 @@ type Response struct { Snapshots []*Snapshot RootCollectionID string } + +type SnapshotContext struct { + snapshots []*Snapshot + widget, workspace *Snapshot +} + +func NewSnapshotContext() *SnapshotContext { + return &SnapshotContext{ + snapshots: []*Snapshot{}, + } +} + +func (sl *SnapshotContext) List() []*Snapshot { + if sl == nil { + return nil + } + return sl.snapshots +} + +func (sl *SnapshotContext) Len() int { + if sl == nil { + return 0 + } + return len(sl.snapshots) +} + +func (sl *SnapshotContext) Add(snapshots ...*Snapshot) *SnapshotContext { + sl.snapshots = append(sl.snapshots, snapshots...) + return sl +} + +func (sl *SnapshotContext) GetWorkspace() *Snapshot { + if sl == nil { + return nil + } + return sl.workspace +} + +func (sl *SnapshotContext) SetWorkspace(w *Snapshot) *SnapshotContext { + sl.workspace = w + return sl +} + +func (sl *SnapshotContext) GetWidget() *Snapshot { + if sl == nil { + return nil + } + return sl.widget +} + +func (sl *SnapshotContext) SetWidget(w *Snapshot) *SnapshotContext { + sl.widget = w + return sl +} + +func (sl *SnapshotContext) Merge(sl2 *SnapshotContext) { + sl.snapshots = append(sl.snapshots, sl2.snapshots...) + if sl2.widget != nil { + sl.widget = sl2.widget + } + if sl2.workspace != nil { + sl.workspace = sl2.workspace + } +} diff --git a/core/block/import/pb/collectionprovider.go b/core/block/import/pb/collectionprovider.go index 29085b9f0..0c3555f03 100644 --- a/core/block/import/pb/collectionprovider.go +++ b/core/block/import/pb/collectionprovider.go @@ -7,7 +7,12 @@ import ( ) type CollectionProvider interface { - ProvideCollection(snapshots []*common.Snapshot, widget *common.Snapshot, oldToNewID map[string]string, params *pb.RpcObjectImportRequestPbParams, workspaceSnapshot *common.Snapshot, isNewSpace bool) ([]*common.Snapshot, error) + ProvideCollection( + snapshots *common.SnapshotContext, + oldToNewID map[string]string, + params *pb.RpcObjectImportRequestPbParams, + isNewSpace bool, + ) ([]*common.Snapshot, error) } func GetProvider(importType pb.RpcObjectImportRequestPbParamsType, service *collection.Service) CollectionProvider { diff --git a/core/block/import/pb/converter.go b/core/block/import/pb/converter.go index e6544d974..86e73a36d 100644 --- a/core/block/import/pb/converter.go +++ b/core/block/import/pb/converter.go @@ -43,7 +43,13 @@ type Pb struct { service *collection.Service accountService account.Service tempDirProvider core.TempDirProvider - iconOption int64 + + progress process.Progress + errors *common.ConvertError + params *pb.RpcObjectImportRequestPbParams + pathCount int + + isMigration, isNewSpace, importWidgets bool } func New(service *collection.Service, accountService account.Service, tempDirProvider core.TempDirProvider) common.Converter { @@ -54,42 +60,56 @@ func New(service *collection.Service, accountService account.Service, tempDirPro } } -func (p *Pb) GetSnapshots(ctx context.Context, req *pb.RpcObjectImportRequest, progress process.Progress) (*common.Response, *common.ConvertError) { - params, e := p.getParams(req.Params) - if e != nil || params == nil { - return nil, common.NewFromError(fmt.Errorf("wrong parameters"), req.Mode) +func (p *Pb) GetSnapshots(_ context.Context, req *pb.RpcObjectImportRequest, progress process.Progress) (*common.Response, *common.ConvertError) { + if err := p.init(req, progress); err != nil { + return nil, common.NewFromError(err, req.Mode) } - allErrors := common.NewError(req.Mode) - allSnapshots, widgetSnapshot, workspaceSnapshot := p.getSnapshots(progress, params, req.IsMigration, allErrors) - oldToNewID := p.updateLinksToObjects(allSnapshots, allErrors, len(params.GetPath())) - p.updateDetails(allSnapshots) - if allErrors.ShouldAbortImport(len(params.GetPath()), req.Type) { - return nil, allErrors + snapshots := p.getSnapshots() + if snapshots == nil { + if p.errors.IsEmpty() { + p.errors.Add(fmt.Errorf("PB: no snapshots are gathered")) + } + return nil, p.errors } - collectionProvider := GetProvider(params.GetImportType(), p.service) + oldToNewID := p.updateLinksToObjects(snapshots.List()) + p.updateDetails(snapshots.List()) + if p.errors.ShouldAbortImport(len(p.params.GetPath()), req.Type) { + return nil, p.errors + } + collectionProvider := GetProvider(p.params.GetImportType(), p.service) var rootCollectionID string - rootCollections, colErr := collectionProvider.ProvideCollection(allSnapshots, widgetSnapshot, oldToNewID, params, workspaceSnapshot, req.IsNewSpace) + rootCollections, colErr := collectionProvider.ProvideCollection(snapshots, oldToNewID, p.params, req.IsNewSpace) if colErr != nil { - allErrors.Add(colErr) - if allErrors.ShouldAbortImport(len(params.GetPath()), req.Type) { - return nil, allErrors + p.errors.Add(colErr) + if p.errors.ShouldAbortImport(len(p.params.GetPath()), req.Type) { + return nil, p.errors } } if len(rootCollections) > 0 { - allSnapshots = append(allSnapshots, rootCollections...) + snapshots.Add(rootCollections...) rootCollectionID = rootCollections[0].Id } - progress.SetTotalPreservingRatio(int64(len(allSnapshots))) - if allErrors.IsEmpty() { - return &common.Response{Snapshots: allSnapshots, RootCollectionID: rootCollectionID}, nil - } - return &common.Response{Snapshots: allSnapshots, RootCollectionID: rootCollectionID}, allErrors + progress.SetTotalPreservingRatio(int64(snapshots.Len())) + return &common.Response{Snapshots: snapshots.List(), RootCollectionID: rootCollectionID}, p.errors.ErrorOrNil() } func (p *Pb) Name() string { return Name } +func (p *Pb) init(req *pb.RpcObjectImportRequest, progress process.Progress) (err error) { + p.params, err = p.getParams(req.Params) + if err != nil || p.params == nil { + return err + } + p.progress = progress + p.errors = common.NewError(req.Mode) + p.isMigration = req.IsMigration + p.isNewSpace = req.IsNewSpace + p.pathCount = len(p.params.GetPath()) + return nil +} + func (p *Pb) getParams(params pb.IsRpcObjectImportRequestParams) (*pb.RpcObjectImportRequestPbParams, error) { if p, ok := params.(*pb.RpcObjectImportRequestParamsOfPbParams); ok { return p.PbParams, nil @@ -97,71 +117,54 @@ func (p *Pb) getParams(params pb.IsRpcObjectImportRequestParams) (*pb.RpcObjectI return nil, fmt.Errorf("PB: getParams wrong parameters format") } -func (p *Pb) getSnapshots( - progress process.Progress, - params *pb.RpcObjectImportRequestPbParams, - isMigration bool, - allErrors *common.ConvertError, -) ( - allSnapshots []*common.Snapshot, - widgetSnapshot *common.Snapshot, - workspaceSnapshot *common.Snapshot, -) { - for _, path := range params.GetPath() { - if err := progress.TryStep(1); err != nil { - allErrors.Add(common.ErrCancel) - return nil, nil, nil +func (p *Pb) getSnapshots() (allSnapshots *common.SnapshotContext) { + allSnapshots = common.NewSnapshotContext() + for _, path := range p.params.GetPath() { + if err := p.progress.TryStep(1); err != nil { + p.errors.Add(common.ErrCancel) + return nil } - snapshots, widget, workspace := p.handleImportPath(len(path), path, allErrors, isMigration, params.GetImportType()) - if allErrors.ShouldAbortImport(len(params.GetPath()), model.Import_Pb) { - return nil, nil, nil + snapshots := p.handleImportPath(path) + if p.errors.ShouldAbortImport(len(p.params.GetPath()), model.Import_Pb) { + return nil } - allSnapshots = append(allSnapshots, snapshots...) - widgetSnapshot = widget - workspaceSnapshot = workspace + allSnapshots.Merge(snapshots) } - return allSnapshots, widgetSnapshot, workspaceSnapshot + return allSnapshots } -func (p *Pb) handleImportPath( - pathCount int, - path string, - allErrors *common.ConvertError, - isMigration bool, - importType pb.RpcObjectImportRequestPbParamsType, -) ([]*common.Snapshot, *common.Snapshot, *common.Snapshot) { +func (p *Pb) handleImportPath(path string) *common.SnapshotContext { importSource := source.GetSource(path) defer importSource.Close() err := p.extractFiles(path, importSource) if err != nil { - allErrors.Add(err) - if allErrors.ShouldAbortImport(pathCount, model.Import_Pb) { - return nil, nil, nil + p.errors.Add(err) + if p.errors.ShouldAbortImport(p.pathCount, model.Import_Pb) { + return nil } } var ( - profileID string - needToImportWidgets bool + profileID string ) profile, err := p.getProfileFromFiles(importSource) if err != nil { - allErrors.Add(err) - if allErrors.ShouldAbortImport(pathCount, model.Import_Pb) { - return nil, nil, nil + p.errors.Add(err) + if p.errors.ShouldAbortImport(p.pathCount, model.Import_Pb) { + return nil } } if profile != nil { pr, e := p.accountService.ProfileInfo() if e != nil { - allErrors.Add(e) - if allErrors.ShouldAbortImport(pathCount, model.Import_Pb) { - return nil, nil, nil + p.errors.Add(e) + if p.errors.ShouldAbortImport(p.pathCount, model.Import_Pb) { + return nil } } - needToImportWidgets = p.needToImportWidgets(profile.Address, pr.AccountId) + p.importWidgets = p.needToImportWidgets(profile.Address, pr.AccountId) profileID = profile.ProfileId } - return p.getSnapshotsFromProvidedFiles(pathCount, importSource, allErrors, path, profileID, needToImportWidgets, isMigration, importType) + return p.getSnapshotsFromProvidedFiles(importSource, path, profileID) } func (p *Pb) extractFiles(importPath string, importSource source.Source) error { @@ -210,51 +213,41 @@ func (p *Pb) needToImportWidgets(address, accountID string) bool { return address == accountID } -func (p *Pb) getSnapshotsFromProvidedFiles( - pathCount int, - pbFiles source.Source, - allErrors *common.ConvertError, - path, profileID string, - needToImportWidgets, isMigration bool, - importType pb.RpcObjectImportRequestPbParamsType, -) ( - allSnapshots []*common.Snapshot, - widgetSnapshot *common.Snapshot, - workspaceSnapshot *common.Snapshot, -) { +func (p *Pb) getSnapshotsFromProvidedFiles(pbFiles source.Source, path, profileID string) (snapshots *common.SnapshotContext) { + snapshots = common.NewSnapshotContext() if iterateErr := pbFiles.Iterate(func(fileName string, fileReader io.ReadCloser) (isContinue bool) { // skip files from "files" directory if filepath.Dir(fileName) == fileDir { return true } - snapshot, err := p.makeSnapshot(fileName, profileID, path, fileReader, isMigration, pbFiles) + snapshot, err := p.makeSnapshot(fileName, profileID, path, fileReader, pbFiles) if err != nil { - allErrors.Add(err) - if allErrors.ShouldAbortImport(pathCount, model.Import_Pb) { + p.errors.Add(err) + if p.errors.ShouldAbortImport(p.pathCount, model.Import_Pb) { return false } } if snapshot != nil { - if p.shouldImportSnapshot(snapshot, needToImportWidgets, importType) { - allSnapshots = append(allSnapshots, snapshot) + if p.shouldImportSnapshot(snapshot) { + snapshots.Add(snapshot) } - if snapshot.Snapshot.SbType == smartblock.SmartBlockTypeWidget { - widgetSnapshot = snapshot - } - if snapshot.Snapshot.SbType == smartblock.SmartBlockTypeWorkspace { - workspaceSnapshot = snapshot + switch snapshot.Snapshot.SbType { + case smartblock.SmartBlockTypeWidget: + snapshots.SetWidget(snapshot) + case smartblock.SmartBlockTypeWorkspace: + snapshots.SetWorkspace(snapshot) } } return true }); iterateErr != nil { - allErrors.Add(iterateErr) + p.errors.Add(iterateErr) } - return allSnapshots, widgetSnapshot, workspaceSnapshot + return snapshots } -func (p *Pb) makeSnapshot(name, profileID, path string, +func (p *Pb) makeSnapshot( + name, profileID, path string, file io.ReadCloser, - isMigration bool, pbFiles source.Source, ) (*common.Snapshot, error) { if name == constant.ProfileFile || name == configFile { @@ -269,7 +262,7 @@ func (p *Pb) makeSnapshot(name, profileID, path string, return nil, fmt.Errorf("%w: %s", common.ErrPbNotAnyBlockFormat, errGS.Error()) } id := uuid.New().String() - id, err := p.normalizeSnapshot(snapshot, id, profileID, path, isMigration, pbFiles) + id, err := p.normalizeSnapshot(snapshot, id, profileID, path, pbFiles) if err != nil { return nil, fmt.Errorf("normalize snapshot: %w", err) } @@ -305,10 +298,11 @@ func (p *Pb) getSnapshotFromFile(rd io.ReadCloser, name string) (*common.Snapsho return nil, ErrNotAnyBlockExtension } -func (p *Pb) normalizeSnapshot(snapshot *common.SnapshotModel, +func (p *Pb) normalizeSnapshot( + snapshot *common.SnapshotModel, id, profileID, path string, - isMigration bool, - pbFiles source.Source) (string, error) { + pbFiles source.Source, +) (string, error) { if _, ok := model.SmartBlockType_name[int32(snapshot.SbType)]; !ok { newSbType := model.SmartBlockType_Page if int32(snapshot.SbType) == 96 { // fallback for objectType smartblocktype @@ -349,7 +343,7 @@ func (p *Pb) normalizeSnapshot(snapshot *common.SnapshotModel, if snapshot.SbType == coresb.SmartBlockTypeProfilePage { var err error - id, err = p.getIDForUserProfile(snapshot, profileID, id, isMigration) + id, err = p.getIDForUserProfile(snapshot, profileID, id) if err != nil { return "", fmt.Errorf("get user profile id: %w", err) } @@ -386,9 +380,9 @@ func (p *Pb) normalizeFilePath(snapshot *common.SnapshotModel, pbFiles source.So return nil } -func (p *Pb) getIDForUserProfile(snapshot *common.SnapshotModel, profileID string, id string, isMigration bool) (string, error) { +func (p *Pb) getIDForUserProfile(snapshot *common.SnapshotModel, profileID string, id string) (string, error) { objectID := snapshot.Data.Details.GetString(bundle.RelationKeyId) - if objectID == profileID && isMigration { + if objectID == profileID && p.isMigration { return p.accountService.ProfileObjectId() } return id, nil @@ -403,10 +397,7 @@ func (p *Pb) setProfileIconOption(snapshot *common.SnapshotModel, profileID stri } func (p *Pb) getIconOption() int64 { - if p.iconOption == 0 { - p.iconOption = int64(rand.Intn(16) + 1) - } - return p.iconOption + return int64(rand.Intn(16) + 1) } // cleanupEmptyBlockMigration is fixing existing pages, imported from Notion @@ -448,13 +439,13 @@ func (p *Pb) setSourceFilePath(snapshot *common.SnapshotModel) { snapshot.Data.Details.SetString(bundle.RelationKeySourceFilePath, sourceFilePath) } -func (p *Pb) shouldImportSnapshot(snapshot *common.Snapshot, needToImportWidgets bool, importType pb.RpcObjectImportRequestPbParamsType) bool { - return (snapshot.Snapshot.SbType == smartblock.SmartBlockTypeWorkspace && importType == pb.RpcObjectImportRequestPbParams_SPACE) || +func (p *Pb) shouldImportSnapshot(snapshot *common.Snapshot) bool { + return (snapshot.Snapshot.SbType == smartblock.SmartBlockTypeWorkspace && p.isNewSpace) || (snapshot.Snapshot.SbType != smartblock.SmartBlockTypeWidget && snapshot.Snapshot.SbType != smartblock.SmartBlockTypeWorkspace) || - (snapshot.Snapshot.SbType == smartblock.SmartBlockTypeWidget && (needToImportWidgets || importType == pb.RpcObjectImportRequestPbParams_EXPERIENCE)) // we import widget in case of experience import + (snapshot.Snapshot.SbType == smartblock.SmartBlockTypeWidget && (p.importWidgets || p.params.GetImportType() == pb.RpcObjectImportRequestPbParams_EXPERIENCE)) // we import widget in case of experience import } -func (p *Pb) updateLinksToObjects(snapshots []*common.Snapshot, allErrors *common.ConvertError, pathCount int) map[string]string { +func (p *Pb) updateLinksToObjects(snapshots []*common.Snapshot) map[string]string { oldToNewID := make(map[string]string, len(snapshots)) for _, snapshot := range snapshots { id := snapshot.Snapshot.Data.Details.GetString(bundle.RelationKeyId) @@ -464,8 +455,8 @@ func (p *Pb) updateLinksToObjects(snapshots []*common.Snapshot, allErrors *commo st := state.NewDocFromSnapshot("", snapshot.Snapshot.ToProto()) err := common.UpdateLinksToObjects(st.(*state.State), oldToNewID) if err != nil { - allErrors.Add(err) - if allErrors.ShouldAbortImport(pathCount, model.Import_Pb) { + p.errors.Add(err) + if p.errors.ShouldAbortImport(p.pathCount, model.Import_Pb) { return nil } continue diff --git a/core/block/import/pb/gallery.go b/core/block/import/pb/gallery.go index e82fe3bf2..ae7059a71 100644 --- a/core/block/import/pb/gallery.go +++ b/core/block/import/pb/gallery.go @@ -25,23 +25,22 @@ func NewGalleryImport(service *collection.Service) *GalleryImport { return &GalleryImport{service: service} } -func (g *GalleryImport) ProvideCollection(snapshots []*common.Snapshot, - widget *common.Snapshot, +func (g *GalleryImport) ProvideCollection( + snapshots *common.SnapshotContext, _ map[string]string, params *pb.RpcObjectImportRequestPbParams, - workspaceSnapshot *common.Snapshot, isNewSpace bool, ) (collectionsSnapshots []*common.Snapshot, err error) { if isNewSpace { return nil, nil } var widgetObjects []string - if widget != nil { + if widget := snapshots.GetWidget(); widget != nil { widgetObjects = g.getObjectsFromWidgets(widget) } var icon string - if workspaceSnapshot != nil { // we use space icon for import collection - icon = workspaceSnapshot.Snapshot.Data.Details.GetString(bundle.RelationKeyIconImage) + if workspace := snapshots.GetWorkspace(); workspace != nil { // we use space icon for import collection + icon = workspace.Snapshot.Data.Details.GetString(bundle.RelationKeyIconImage) } collectionName := params.GetCollectionTitle() // collection name should be the name of experience if collectionName == "" { @@ -49,12 +48,12 @@ func (g *GalleryImport) ProvideCollection(snapshots []*common.Snapshot, } rootCollection := common.NewImportCollection(g.service) if len(widgetObjects) > 0 { - collectionsSnapshots, err = g.getWidgetsCollection(collectionName, rootCollection, widgetObjects, icon, widget, collectionsSnapshots) + collectionsSnapshots, err = g.getWidgetsCollection(collectionName, rootCollection, widgetObjects, icon, snapshots.GetWidget(), collectionsSnapshots) if err != nil { return nil, err } } - objectsIDs := g.getObjectsIDs(snapshots) + objectsIDs := g.getObjectsIDs(snapshots.List()) settings := common.NewImportCollectionSetting( common.WithCollectionName(collectionName), common.WithTargetObjects(objectsIDs), diff --git a/core/block/import/pb/gallery_test.go b/core/block/import/pb/gallery_test.go index 140390ca6..a80d9b5c9 100644 --- a/core/block/import/pb/gallery_test.go +++ b/core/block/import/pb/gallery_test.go @@ -24,7 +24,7 @@ func TestGalleryImport_ProvideCollection(t *testing.T) { params := &pb.RpcObjectImportRequestPbParams{} // when - collection, err := collectionProvider.ProvideCollection(nil, nil, nil, params, nil, false) + collection, err := collectionProvider.ProvideCollection(nil, nil, params, false) // then assert.Nil(t, err) @@ -37,7 +37,7 @@ func TestGalleryImport_ProvideCollection(t *testing.T) { params := &pb.RpcObjectImportRequestPbParams{} // when - collection, err := collectionProvider.ProvideCollection(nil, nil, nil, params, nil, false) + collection, err := collectionProvider.ProvideCollection(nil, nil, params, false) // then assert.Nil(t, err) @@ -50,7 +50,7 @@ func TestGalleryImport_ProvideCollection(t *testing.T) { params := &pb.RpcObjectImportRequestPbParams{CollectionTitle: "test"} // when - collection, err := collectionProvider.ProvideCollection(nil, nil, nil, params, nil, false) + collection, err := collectionProvider.ProvideCollection(nil, nil, params, false) // then assert.Nil(t, err) @@ -129,9 +129,10 @@ func TestGalleryImport_ProvideCollection(t *testing.T) { }, }, } + snapshotList := common.NewSnapshotContext().Add(allSnapshot...).SetWidget(widgetSnapshot) // when - collection, err := p.ProvideCollection(allSnapshot, widgetSnapshot, nil, params, nil, false) + collection, err := p.ProvideCollection(snapshotList, nil, params, false) // then assert.Nil(t, err) @@ -200,9 +201,10 @@ func TestGalleryImport_ProvideCollection(t *testing.T) { }, }, } + snapshotList := common.NewSnapshotContext().Add(allSnapshot...).SetWidget(widgetSnapshot) // when - collection, err := p.ProvideCollection(allSnapshot, widgetSnapshot, nil, params, nil, false) + collection, err := p.ProvideCollection(snapshotList, nil, params, false) // then assert.Nil(t, err) @@ -235,9 +237,10 @@ func TestGalleryImport_ProvideCollection(t *testing.T) { }, }, } + snapshotList := common.NewSnapshotContext().SetWidget(widgetSnapshot) // when - collection, err := p.ProvideCollection(nil, widgetSnapshot, nil, params, nil, false) + collection, err := p.ProvideCollection(snapshotList, nil, params, false) // then assert.Nil(t, err) @@ -251,7 +254,7 @@ func TestGalleryImport_ProvideCollection(t *testing.T) { params := &pb.RpcObjectImportRequestPbParams{NoCollection: false} // when - collection, err := p.ProvideCollection(nil, nil, nil, params, nil, false) + collection, err := p.ProvideCollection(nil, nil, params, false) // then assert.Nil(t, err) @@ -273,8 +276,10 @@ func TestGalleryImport_ProvideCollection(t *testing.T) { }, }, } + snapshotList := common.NewSnapshotContext().SetWorkspace(workspace) + // when - collection, err := p.ProvideCollection(nil, nil, nil, params, workspace, false) + collection, err := p.ProvideCollection(snapshotList, nil, params, false) // then assert.Nil(t, err) @@ -298,8 +303,10 @@ func TestGalleryImport_ProvideCollection(t *testing.T) { }, }, } + snapshotList := common.NewSnapshotContext().SetWorkspace(workspace) + // when - collection, err := p.ProvideCollection(nil, nil, nil, params, workspace, false) + collection, err := p.ProvideCollection(snapshotList, nil, params, false) // then assert.Nil(t, err) @@ -312,7 +319,7 @@ func TestGalleryImport_ProvideCollection(t *testing.T) { params := &pb.RpcObjectImportRequestPbParams{NoCollection: false} // when - collection, err := p.ProvideCollection(nil, nil, nil, params, nil, true) + collection, err := p.ProvideCollection(nil, nil, params, true) // then assert.Nil(t, err) @@ -343,9 +350,10 @@ func TestGalleryImport_ProvideCollection(t *testing.T) { }, }, } + snapshotList := common.NewSnapshotContext().SetWidget(widgetSnapshot) // when - collection, err := p.ProvideCollection(nil, widgetSnapshot, nil, params, nil, false) + collection, err := p.ProvideCollection(snapshotList, nil, params, false) // then assert.Nil(t, err) diff --git a/core/block/import/pb/space.go b/core/block/import/pb/space.go index 6a967e62a..2cea4afce 100644 --- a/core/block/import/pb/space.go +++ b/core/block/import/pb/space.go @@ -21,11 +21,10 @@ func NewSpaceImport(service *collection.Service) *SpaceImport { return &SpaceImport{service: service} } -func (s *SpaceImport) ProvideCollection(snapshots []*common.Snapshot, - widgetSnapshot *common.Snapshot, +func (s *SpaceImport) ProvideCollection( + snapshots *common.SnapshotContext, oldToNewID map[string]string, params *pb.RpcObjectImportRequestPbParams, - _ *common.Snapshot, _ bool, ) ([]*common.Snapshot, error) { if params.GetNoCollection() { @@ -37,9 +36,9 @@ func (s *SpaceImport) ProvideCollection(snapshots []*common.Snapshot, objectsNotInWidget []*common.Snapshot ) - if widgetSnapshot != nil { + if widgetSnapshot := snapshots.GetWidget(); widgetSnapshot != nil { widgetFlags, rootObjects = s.getObjectsFromWidget(widgetSnapshot, oldToNewID) - objectsNotInWidget = lo.Filter(snapshots, func(item *common.Snapshot, index int) bool { + objectsNotInWidget = lo.Filter(snapshots.List(), func(item *common.Snapshot, index int) bool { return !lo.Contains(rootObjects, item.Id) }) } @@ -48,7 +47,7 @@ func (s *SpaceImport) ProvideCollection(snapshots []*common.Snapshot, rootObjects = append(rootObjects, s.filterObjects(widgetFlags, objectsNotInWidget)...) } else { // if we don't have any widget, we add everything (except sub objects and templates) to root collection - rootObjects = lo.FilterMap(snapshots, func(item *common.Snapshot, index int) (string, bool) { + rootObjects = lo.FilterMap(snapshots.List(), func(item *common.Snapshot, index int) (string, bool) { if !s.objectShouldBeSkipped(item) { return item.Id, true } diff --git a/core/block/import/pb/space_test.go b/core/block/import/pb/space_test.go index 119fa6cbe..f80c14f17 100644 --- a/core/block/import/pb/space_test.go +++ b/core/block/import/pb/space_test.go @@ -23,7 +23,7 @@ func TestSpaceImport_ProvideCollection(t *testing.T) { params := &pb.RpcObjectImportRequestPbParams{NoCollection: false} // when - collection, err := collectionProvider.ProvideCollection(nil, nil, nil, params, nil, false) + collection, err := collectionProvider.ProvideCollection(nil, nil, params, false) // then assert.Nil(t, err) @@ -38,7 +38,7 @@ func TestSpaceImport_ProvideCollection(t *testing.T) { params := &pb.RpcObjectImportRequestPbParams{NoCollection: true} // when - collection, err := collectionProvider.ProvideCollection(nil, nil, nil, params, nil, false) + collection, err := collectionProvider.ProvideCollection(nil, nil, params, false) // then assert.Nil(t, err) @@ -75,9 +75,10 @@ func TestSpaceImport_ProvideCollection(t *testing.T) { }, }, } + snapshotList := common.NewSnapshotContext().Add(allSnapshot...) // when - collection, err := p.ProvideCollection(allSnapshot, nil, nil, params, nil, false) + collection, err := p.ProvideCollection(snapshotList, nil, params, false) // then assert.Nil(t, err) @@ -165,9 +166,10 @@ func TestSpaceImport_ProvideCollection(t *testing.T) { }, }, } + snapshotList := common.NewSnapshotContext().Add(allSnapshot...).SetWidget(widgetSnapshot) // when - collection, err := p.ProvideCollection(allSnapshot, widgetSnapshot, nil, params, nil, false) + collection, err := p.ProvideCollection(snapshotList, nil, params, false) // then assert.Nil(t, err) @@ -254,9 +256,10 @@ func TestSpaceImport_ProvideCollection(t *testing.T) { }, }, } + snapshotList := common.NewSnapshotContext().Add(allSnapshot...).SetWidget(widgetSnapshot) // when - collection, err := p.ProvideCollection(allSnapshot, widgetSnapshot, nil, params, nil, false) + collection, err := p.ProvideCollection(snapshotList, nil, params, false) // then assert.Nil(t, err) @@ -358,9 +361,10 @@ func TestSpaceImport_ProvideCollection(t *testing.T) { }, }, } + snapshotList := common.NewSnapshotContext().Add(allSnapshot...).SetWidget(widgetSnapshot) // when - collection, err := p.ProvideCollection(allSnapshot, widgetSnapshot, map[string]string{"oldObjectInWidget": "newObjectInWidget"}, params, nil, false) + collection, err := p.ProvideCollection(snapshotList, map[string]string{"oldObjectInWidget": "newObjectInWidget"}, params, false) // then assert.Nil(t, err) diff --git a/core/block/process/service.go b/core/block/process/service.go index 4c3830bbf..ebcdf9477 100644 --- a/core/block/process/service.go +++ b/core/block/process/service.go @@ -152,6 +152,9 @@ func (s *service) Close(ctx context.Context) (err error) { var errs []error for _, id := range ids { if err := s.Cancel(id); err != nil { + if errors.Is(err, ErrNotFound) { + continue + } errs = append(errs, err) } } diff --git a/go.mod b/go.mod index 2169c180d..7f3e05849 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,8 @@ require ( github.com/PuerkitoBio/goquery v1.10.2 github.com/VividCortex/ewma v1.2.0 github.com/adrium/goheif v0.0.0-20230113233934-ca402e77a786 - github.com/anyproto/any-store v0.1.12 - github.com/anyproto/any-sync v0.6.10 + github.com/anyproto/any-store v0.1.13 + github.com/anyproto/any-sync v0.6.13 github.com/anyproto/anytype-publish-server/publishclient v0.0.0-20250131145601-de288583ff2a github.com/anyproto/go-chash v0.1.0 github.com/anyproto/go-naturaldate/v2 v2.0.2-0.20230524105841-9829cfd13438 @@ -291,10 +291,10 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect lukechampine.com/blake3 v1.4.0 // indirect - modernc.org/libc v1.61.13 // indirect + modernc.org/libc v1.62.1 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.9.1 // indirect - modernc.org/sqlite v1.36.2 // indirect + modernc.org/sqlite v1.37.0 // indirect nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index 803ed7530..0b76c2fd9 100644 --- a/go.sum +++ b/go.sum @@ -78,10 +78,10 @@ github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= -github.com/anyproto/any-store v0.1.12 h1:VgTyfxM4BvFnYMbwWxV9FQmXWNlwp9xriWFh6VOjcm0= -github.com/anyproto/any-store v0.1.12/go.mod h1:T6WNuCHcuXIRiaZ+QAcBHdxIbPbgNCMIf1u3P9jvAyU= -github.com/anyproto/any-sync v0.6.10 h1:TV3yLkp5NK7FkddUvVtfxT01t/Xi+DlKbvisGEQwmC0= -github.com/anyproto/any-sync v0.6.10/go.mod h1:TSKgCoTV40Bt8AfCh3RxPUUAfYGrhc8Mzh8/AiVlvX4= +github.com/anyproto/any-store v0.1.13 h1:1wmm0qQIRShaycBLKwcgkQbRKy3WrNPAShTE5fwzfCY= +github.com/anyproto/any-store v0.1.13/go.mod h1:2M0Xf4rmijoKGd+nqqeKG8I1yIokCLEIxrAXEoHjXn4= +github.com/anyproto/any-sync v0.6.13 h1:p1uc6uc+69X/WaaWVKs83sH8zDAbBo0COKYTcvHFXX8= +github.com/anyproto/any-sync v0.6.13/go.mod h1:TSKgCoTV40Bt8AfCh3RxPUUAfYGrhc8Mzh8/AiVlvX4= github.com/anyproto/anytype-publish-server/publishclient v0.0.0-20250131145601-de288583ff2a h1:ZZM+0OUCQMWSLSflpkf0ZMVo3V76qEDDIXPpQOClNs0= github.com/anyproto/anytype-publish-server/publishclient v0.0.0-20250131145601-de288583ff2a/go.mod h1:4fkueCZcGniSMXkrwESO8zzERrh/L7WHimRNWecfGM0= github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580 h1:Ba80IlCCxkZ9H1GF+7vFu/TSpPvbpDCxXJ5ogc4euYc= @@ -1684,16 +1684,16 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= lukechampine.com/blake3 v1.4.0 h1:xDbKOZCVbnZsfzM6mHSYcGRHZ3YrLDzqz8XnV4uaD5w= lukechampine.com/blake3 v1.4.0/go.mod h1:MQJNQCTnR+kwOP/JEZSxj3MaQjp80FOFSNMMHXcSeX0= -modernc.org/cc/v4 v4.24.4 h1:TFkx1s6dCkQpd6dKurBNmpo+G8Zl4Sq/ztJ+2+DEsh0= -modernc.org/cc/v4 v4.24.4/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= -modernc.org/ccgo/v4 v4.23.16 h1:Z2N+kk38b7SfySC1ZkpGLN2vthNJP1+ZzGZIlH7uBxo= -modernc.org/ccgo/v4 v4.23.16/go.mod h1:nNma8goMTY7aQZQNTyN9AIoJfxav4nvTnvKThAeMDdo= +modernc.org/cc/v4 v4.25.2 h1:T2oH7sZdGvTaie0BRNFbIYsabzCxUQg8nLqCdQ2i0ic= +modernc.org/cc/v4 v4.25.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.25.1 h1:TFSzPrAGmDsdnhT9X2UrcPMI3N/mJ9/X9ykKXwLhDsU= +modernc.org/ccgo/v4 v4.25.1/go.mod h1:njjuAYiPflywOOrm3B7kCB444ONP5pAVr8PIEoE0uDw= modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.6.3 h1:aJVhcqAte49LF+mGveZ5KPlsp4tdGdAOT4sipJXADjw= -modernc.org/gc/v2 v2.6.3/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/libc v1.61.13 h1:3LRd6ZO1ezsFiX1y+bHd1ipyEHIJKvuprv0sLTBwLW8= -modernc.org/libc v1.61.13/go.mod h1:8F/uJWL/3nNil0Lgt1Dpz+GgkApWh04N3el3hxJcA6E= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/libc v1.62.1 h1:s0+fv5E3FymN8eJVmnk0llBe6rOxCu/DEU+XygRbS8s= +modernc.org/libc v1.62.1/go.mod h1:iXhATfJQLjG3NWy56a6WVU73lWOcdYVxsvwCgoPljuo= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.9.1 h1:V/Z1solwAVmMW1yttq3nDdZPJqV1rM05Ccq6KMSZ34g= @@ -1702,8 +1702,8 @@ modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.36.2 h1:vjcSazuoFve9Wm0IVNHgmJECoOXLZM1KfMXbcX2axHA= -modernc.org/sqlite v1.36.2/go.mod h1:ADySlx7K4FdY5MaJcEv86hTJ0PjedAloTUuif0YS3ws= +modernc.org/sqlite v1.37.0 h1:s1TMe7T3Q3ovQiK2Ouz4Jwh7dw4ZDqbebSDTlSJdfjI= +modernc.org/sqlite v1.37.0/go.mod h1:5YiWv+YviqGMuGw4V+PNplcyaJ5v+vQd7TQOgkACoJM= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/pkg/lib/bundle/types.gen.go b/pkg/lib/bundle/types.gen.go index df36c8b8e..e5bf897e8 100644 --- a/pkg/lib/bundle/types.gen.go +++ b/pkg/lib/bundle/types.gen.go @@ -9,7 +9,7 @@ import ( "github.com/anyproto/anytype-heart/pkg/lib/pb/model" ) -const TypeChecksum = "91c804ca7dd4df74fafbdaff99e50fa2ffcc6cb0fb073887a90bdf1b4a3d98d5" +const TypeChecksum = "f2672b9f52915f228e335b78ed94ce1b6c7b195ceb1cba9a44b25a870a3b2ac0" const ( TypePrefix = "_ot" ) @@ -426,7 +426,7 @@ var ( Name: "Task", PluralName: "Tasks", Readonly: true, - RelationLinks: []*model.RelationLink{MustGetRelationLink(RelationKeyTag), MustGetRelationLink(RelationKeyAssignee), MustGetRelationLink(RelationKeyDone), MustGetRelationLink(RelationKeyDueDate), MustGetRelationLink(RelationKeyLinkedProjects), MustGetRelationLink(RelationKeyPriority), MustGetRelationLink(RelationKeyStatus), MustGetRelationLink(RelationKeyTasks)}, + RelationLinks: []*model.RelationLink{MustGetRelationLink(RelationKeyTag), MustGetRelationLink(RelationKeyAssignee), MustGetRelationLink(RelationKeyDone), MustGetRelationLink(RelationKeyDueDate), MustGetRelationLink(RelationKeyLinkedProjects), MustGetRelationLink(RelationKeyStatus)}, Revision: 2, Types: []model.SmartBlockType{model.SmartBlockType_Page}, Url: TypePrefix + "task", diff --git a/pkg/lib/bundle/types.json b/pkg/lib/bundle/types.json index 96cabd17d..9d9bccac6 100644 --- a/pkg/lib/bundle/types.json +++ b/pkg/lib/bundle/types.json @@ -107,9 +107,7 @@ "done", "dueDate", "linkedProjects", - "priority", - "status", - "tasks" + "status" ], "description": "A piece of work to be done or undertaken", "revision": 2 diff --git a/pkg/lib/core/tmp_dir.go b/pkg/lib/core/tmp_dir.go index fa7e22a24..5ea43f085 100644 --- a/pkg/lib/core/tmp_dir.go +++ b/pkg/lib/core/tmp_dir.go @@ -77,6 +77,7 @@ func recursiveCleanup(path string, cutoff time.Time) { info, err := entry.Info() if err != nil { log.Warnf("tmp cleanup entry: %v", err) + continue } if entry.IsDir() { recursiveCleanup(fullEntryPath, cutoff) diff --git a/pkg/lib/localstore/objectstore/service.go b/pkg/lib/localstore/objectstore/service.go index 13be262e2..b371cc7bb 100644 --- a/pkg/lib/localstore/objectstore/service.go +++ b/pkg/lib/localstore/objectstore/service.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "sync" + "time" anystore "github.com/anyproto/any-store" "github.com/anyproto/any-store/anyenc" @@ -379,10 +380,12 @@ func (s *dsObjectStore) getOrInitSpaceIndex(spaceId string) spaceindex.Store { func (s *dsObjectStore) getAnyStoreConfig() *anystore.Config { return &anystore.Config{ - Namespace: s.anyStoreConfig.Namespace, - ReadConnections: s.anyStoreConfig.ReadConnections, - SQLiteConnectionOptions: maps.Clone(s.anyStoreConfig.SQLiteConnectionOptions), - SyncPoolElementMaxSize: s.anyStoreConfig.SyncPoolElementMaxSize, + Namespace: s.anyStoreConfig.Namespace, + ReadConnections: s.anyStoreConfig.ReadConnections, + SQLiteConnectionOptions: maps.Clone(s.anyStoreConfig.SQLiteConnectionOptions), + SyncPoolElementMaxSize: s.anyStoreConfig.SyncPoolElementMaxSize, + StalledConnectionsDetectorEnabled: true, + StalledConnectionsPanicOnClose: time.Second * 30, } } @@ -430,6 +433,10 @@ func collectCrossSpace[T any](s *dsObjectStore, proc func(store spaceindex.Store var result []T for _, store := range stores { + err := store.Init() + if err != nil { + return nil, fmt.Errorf("init store: %w", err) + } items, err := proc(store) if err != nil { return nil, err diff --git a/space/spacecore/storage/anystorage/storageservice.go b/space/spacecore/storage/anystorage/storageservice.go index fe2892235..06247537b 100644 --- a/space/spacecore/storage/anystorage/storageservice.go +++ b/space/spacecore/storage/anystorage/storageservice.go @@ -8,6 +8,7 @@ import ( "path" "strings" "sync" + "time" anystore "github.com/anyproto/any-store" "github.com/anyproto/any-sync/app" @@ -141,5 +142,8 @@ func (s *storageService) anyStoreConfig() *anystore.Config { ReadConnections: 4, SQLiteConnectionOptions: opts, SQLiteGlobalPageCachePreallocateSizeBytes: 1 << 26, + + StalledConnectionsPanicOnClose: time.Second * 45, + StalledConnectionsDetectorEnabled: true, } } diff --git a/util/internalflag/flag.go b/util/internalflag/flag.go index 4129b8370..4fdb5f5e4 100644 --- a/util/internalflag/flag.go +++ b/util/internalflag/flag.go @@ -49,7 +49,7 @@ func (s *Set) Remove(flag model.InternalFlagValue) { func (s *Set) AddToState(st *state.State) { if len(s.flags) == 0 { - st.RemoveDetail(relationKey) + st.SetDetailAndBundledRelation(relationKey, domain.Float64List([]float64{})) return } st.SetDetailAndBundledRelation(relationKey, domain.Float64List(s.flags))