anyproto/anytype-heart
1
0
Fork 0
mirror of https://github.com/anyproto/anytype-heart.git synced 2025-06-09 17:44:59 +09:00
Code Activity

Integrate: fix import

Browse source
This commit is contained in:
Sergey 2024-08-01 16:24:46 +02:00
parent 7d17c9e41b
commit b46cf6aa05
No known key found for this signature in database
GPG key ID: 3B6BEF79160221C6
37 changed files with 739 additions and 460 deletions
Download patch file Download diff file Expand all files Collapse all files

7
core/block/bookmark/bookmarkimporter/bookmark_importer_decorator.go
View file

@ -4,14 +4,13 @@ import (
"context"
"github.com/anyproto/any-sync/app"
"github.com/gogo/protobuf/types"
bookmarksvc "github.com/anyproto/anytype-heart/core/block/bookmark"
"github.com/anyproto/anytype-heart/core/block/import"
"github.com/anyproto/anytype-heart/core/domain"
"github.com/anyproto/anytype-heart/pb"
"github.com/anyproto/anytype-heart/pkg/lib/bundle"
"github.com/anyproto/anytype-heart/pkg/lib/logging"
"github.com/anyproto/anytype-heart/util/pbtypes"
)
const CName = "bookmark-importer"
@ -19,7 +18,7 @@ const CName = "bookmark-importer"
var log = logging.Logger("bookmark-importer")
type Importer interface {
ImportWeb(ctx context.Context, req *pb.RpcObjectImportRequest) (string, *types.Struct, error)
ImportWeb(ctx context.Context, req *pb.RpcObjectImportRequest) (string, *domain.Details, error)
}
type BookmarkImporterDecorator struct {
@ -38,7 +37,7 @@ func (bd *BookmarkImporterDecorator) Init(a *app.App) (err error) {
return nil
}
func (bd *BookmarkImporterDecorator) CreateBookmarkObject(ctx context.Context, spaceID string, details *types.Struct, getContent bookmarksvc.ContentFuture) (objectId string, newDetails *types.Struct, err error) {
func (bd *BookmarkImporterDecorator) CreateBookmarkObject(ctx context.Context, spaceID string, details *domain.Details, getContent bookmarksvc.ContentFuture) (objectId string, newDetails *domain.Details, err error) {
url := details.GetStringOrDefault(bundle.RelationKeySource, "")
if objectId, newDetails, err = bd.Importer.ImportWeb(nil, &pb.RpcObjectImportRequest{
Params: &pb.RpcObjectImportRequestParamsOfBookmarksParams{BookmarksParams: &pb.RpcObjectImportRequestBookmarksParams{Url: url}},

6
core/block/create.go
View file

@ -5,8 +5,6 @@ import (
"errors"
"fmt"
"github.com/gogo/protobuf/types"
"github.com/anyproto/anytype-heart/core/block/cache"
"github.com/anyproto/anytype-heart/core/block/editor/basic"
"github.com/anyproto/anytype-heart/core/block/editor/smartblock"
@ -81,7 +79,7 @@ func (s *Service) CreateLinkToTheNewObject(
ctx context.Context,
sctx session.Context,
req *pb.RpcBlockLinkCreateWithObjectRequest,
) (linkID string, objectId string, objectDetails *types.Struct, err error) {
) (linkID string, objectId string, objectDetails *domain.Details, err error) {
if req.ContextId == req.TemplateId && req.ContextId != "" {
err = fmt.Errorf("unable to create link to template from this template")
return
@ -93,7 +91,7 @@ func (s *Service) CreateLinkToTheNewObject(
}
createReq := objectcreator.CreateObjectRequest{
Details: req.Details,
Details: domain.NewDetailsFromProto(req.Details),
InternalFlags: req.InternalFlags,
ObjectTypeKey: objectTypeKey,
TemplateId: req.TemplateId,

2
core/block/debug.go
View file

@ -137,7 +137,7 @@ func (s *Service) getDebugObject(id string) (debugObject, error) {
st := sb.NewState()
root := blockbuilder.BuildAST(st.Blocks())
marshaller := jsonpb.Marshaler{}
detailsRaw, err := marshaller.MarshalToString(st.CombinedDetails())
detailsRaw, err := marshaller.MarshalToString(st.CombinedDetails().ToProto())
if err != nil {
return fmt.Errorf("marshal details: %w", err)
}

13
core/block/editor.go
View file

@ -5,8 +5,6 @@ import (
"errors"
"fmt"
"github.com/gogo/protobuf/types"
"github.com/anyproto/anytype-heart/core/block/cache"
"github.com/anyproto/anytype-heart/core/block/editor/basic"
"github.com/anyproto/anytype-heart/core/block/editor/bookmark"
@ -30,6 +28,7 @@ import (
"github.com/anyproto/anytype-heart/pkg/lib/bundle"
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
"github.com/anyproto/anytype-heart/util/pbtypes"
"github.com/anyproto/anytype-heart/util/slice"
)
var ErrOptionUsedByOtherObjects = fmt.Errorf("option is used by other objects")
@ -523,7 +522,7 @@ func (s *Service) CreateAndUploadFile(
return
}
func (s *Service) UploadFile(ctx context.Context, spaceId string, req FileUploadRequest) (objectId string, details *types.Struct, err error) {
func (s *Service) UploadFile(ctx context.Context, spaceId string, req FileUploadRequest) (objectId string, details *domain.Details, err error) {
upl := s.fileUploaderService.NewUploader(spaceId, req.ObjectOrigin)
if req.DisableEncryption {
log.Errorf("DisableEncryption is deprecated and has no effect")
@ -533,7 +532,7 @@ func (s *Service) UploadFile(ctx context.Context, spaceId string, req FileUpload
upl.SetCustomEncryptionKeys(req.CustomEncryptionKeys)
}
upl.SetStyle(req.Style)
upl.SetAdditionalDetails(req.Details)
upl.SetAdditionalDetails(domain.NewDetailsFromProto(req.Details))
if req.Type != model.BlockContentFile_None {
upl.SetType(req.Type)
}
@ -645,7 +644,7 @@ func (s *Service) GetRelations(ctx session.Context, objectId string) (relations
}
// ModifyDetails performs details get and update under the sb lock to make sure no modifications are done in the middle
func (s *Service) ModifyDetails(objectId string, modifier func(current *types.Struct) (*types.Struct, error)) (err error) {
func (s *Service) ModifyDetails(objectId string, modifier func(current *domain.Details) (*domain.Details, error)) (err error) {
return cache.Do(s, objectId, func(du basic.DetailsUpdatable) error {
return du.UpdateDetails(modifier)
})
@ -656,7 +655,7 @@ func (s *Service) AddExtraRelations(ctx session.Context, objectId string, relati
return nil
}
return cache.Do(s, objectId, func(b smartblock.SmartBlock) error { // TODO RQ: check if empty
return b.AddRelationLinks(ctx, relationIds...)
return b.AddRelationLinks(ctx, slice.StringsInto[domain.RelationKey](relationIds)...)
})
}
@ -676,7 +675,7 @@ func (s *Service) SetObjectTypes(ctx session.Context, objectId string, objectTyp
func (s *Service) RemoveExtraRelations(ctx session.Context, objectTypeId string, relationKeys []string) (err error) {
return cache.Do(s, objectTypeId, func(b smartblock.SmartBlock) error {
return b.RemoveExtraRelations(ctx, relationKeys)
return b.RemoveExtraRelations(ctx, slice.StringsInto[domain.RelationKey](relationKeys))
})
}

16
core/block/editor/basic/extract_objects.go
View file

@ -5,7 +5,6 @@ import (
"fmt"
"github.com/globalsign/mgo/bson"
"github.com/gogo/protobuf/types"
"github.com/anyproto/anytype-heart/core/block/editor/state"
"github.com/anyproto/anytype-heart/core/block/simple"
@ -18,11 +17,11 @@ import (
)
type ObjectCreator interface {
CreateSmartBlockFromState(ctx context.Context, spaceID string, objectTypeKeys []domain.TypeKey, createState *state.State) (id string, newDetails *types.Struct, err error)
CreateSmartBlockFromState(ctx context.Context, spaceID string, objectTypeKeys []domain.TypeKey, createState *state.State) (id string, newDetails *domain.Details, err error)
}
type TemplateStateCreator interface {
CreateTemplateStateWithDetails(templateId string, details *types.Struct) (*state.State, error)
CreateTemplateStateWithDetails(templateId string, details *domain.Details) (*state.State, error)
}
// ExtractBlocksToObjects extracts child blocks from the object to separate objects and
@ -87,7 +86,7 @@ func (bs *basic) prepareTargetObjectDetails(
spaceID string,
typeUniqueKey domain.UniqueKey,
rootBlock simple.Block,
) (*types.Struct, error) {
) (*domain.Details, error) {
objType, err := bs.objectStore.GetObjectByUniqueKey(spaceID, typeUniqueKey)
if err != nil {
return nil, err
@ -162,15 +161,12 @@ func removeBlocks(state *state.State, descendants []simple.Block) {
}
}
func createTargetObjectDetails(nameText string, layout model.ObjectTypeLayout) *types.Struct {
fields := map[string]*types.Value{}
func createTargetObjectDetails(nameText string, layout model.ObjectTypeLayout) *domain.Details {
details := domain.NewDetails()
// Without this check title will be duplicated in template.WithNameToFirstBlock
if layout != model.ObjectType_note {
fields[bundle.RelationKeyName.String()] = pbtypes.String(nameText)
details.Set(bundle.RelationKeyName, nameText)
}
details := &types.Struct{Fields: fields}
return details
}

4
core/block/import/common/collection.go
View file

@ -69,9 +69,9 @@ func (r *RootCollection) getRootCollectionSnapshot(
return &Snapshot{
Id: uuid.New().String(),
FileName: collectionName,
SbType: sb.SmartBlockTypePage,
Snapshot: &SnapshotModel{
Data: &SnapshotModelData{
SbType: sb.SmartBlockTypePage,
Data: &StateSnapshot{
Blocks: st.Blocks(),
Details: detailsStruct,
ObjectTypes: []string{bundle.TypeKeyCollection.String()},

48
core/block/import/common/objectcreator/objectcreator.go
View file

@ -6,7 +6,6 @@ import (
"fmt"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/gogo/protobuf/types"
"github.com/samber/lo"
"go.uber.org/zap"
@ -40,7 +39,7 @@ var log = logging.Logger("import")
// Service incapsulate logic with creation of given smartblocks
type Service interface {
//nolint:lll
Create(dataObject *DataObject, sn *common.Snapshot) (*types.Struct, string, error)
Create(dataObject *DataObject, sn *common.Snapshot) (*domain.Details, string, error)
}
type BlockService interface {
@ -78,7 +77,7 @@ func New(service BlockService,
}
// Create creates smart blocks from given snapshots
func (oc *ObjectCreator) Create(dataObject *DataObject, sn *common.Snapshot) (*types.Struct, string, error) {
func (oc *ObjectCreator) Create(dataObject *DataObject, sn *common.Snapshot) (*domain.Details, string, error) {
snapshot := sn.Snapshot.Data
oldIDtoNew := dataObject.oldIDtoNew
fileIDs := dataObject.fileIDs
@ -89,14 +88,14 @@ func (oc *ObjectCreator) Create(dataObject *DataObject, sn *common.Snapshot) (*t
var err error
newID := oldIDtoNew[sn.Id]
if sn.SbType == coresb.SmartBlockTypeFile {
if sn.Snapshot.SbType == coresb.SmartBlockTypeFile {
return nil, newID, nil
}
oc.setRootBlock(snapshot, newID)
oc.injectImportDetails(sn, origin)
st := state.NewDocFromSnapshot(newID, sn.Snapshot, state.WithUniqueKeyMigration(sn.SbType)).(*state.State)
st := state.NewDocFromSnapshot(newID, sn.Snapshot.ToProto(), state.WithUniqueKeyMigration(sn.Snapshot.SbType)).(*state.State)
st.SetLocalDetail(bundle.RelationKeyLastModifiedDate, pbtypes.Int64(snapshot.Details.GetInt64OrDefault(bundle.RelationKeyLastModifiedDate, 0)))
var filesToDelete []string
@ -112,12 +111,12 @@ func (oc *ObjectCreator) Create(dataObject *DataObject, sn *common.Snapshot) (*t
}
oc.updateKeys(st, oldIDtoNew)
if sn.SbType == coresb.SmartBlockTypeWorkspace {
if sn.Snapshot.SbType == coresb.SmartBlockTypeWorkspace {
oc.setSpaceDashboardID(spaceID, st)
return nil, newID, nil
}
if sn.SbType == coresb.SmartBlockTypeWidget {
if sn.Snapshot.SbType == coresb.SmartBlockTypeWidget {
return oc.updateWidgetObject(st)
}
@ -130,7 +129,7 @@ func (oc *ObjectCreator) Create(dataObject *DataObject, sn *common.Snapshot) (*t
})
typeKeys := st.ObjectTypeKeys()
if sn.SbType == coresb.SmartBlockTypeObjectType {
if sn.Snapshot.SbType == coresb.SmartBlockTypeObjectType {
// we widen typeKeys here to install bundled templates for imported object type
typeKeys = append(typeKeys, domain.TypeKey(st.UniqueKeyInternal()))
}
@ -138,7 +137,7 @@ func (oc *ObjectCreator) Create(dataObject *DataObject, sn *common.Snapshot) (*t
if err != nil {
log.With("objectID", newID).Errorf("failed to install bundled relations and types: %s", err)
}
var respDetails *types.Struct
var respDetails *domain.Details
if payload := dataObject.createPayloads[newID]; payload.RootRawChange != nil {
respDetails, err = oc.createNewObject(ctx, spaceID, payload, st, newID, oldIDtoNew)
if err != nil {
@ -146,7 +145,7 @@ func (oc *ObjectCreator) Create(dataObject *DataObject, sn *common.Snapshot) (*t
return nil, "", err
}
} else {
if canUpdateObject(sn.SbType) {
if canUpdateObject(sn.Snapshot.SbType) {
respDetails = oc.updateExistingObject(st, oldIDtoNew, newID)
}
}
@ -194,7 +193,7 @@ func (oc *ObjectCreator) injectImportDetails(sn *common.Snapshot, origin objecto
// we don't need to inject relatonLinks, they will be automatically injected for bundled relations
}
func (oc *ObjectCreator) updateExistingObject(st *state.State, oldIDtoNew map[string]string, newID string) *types.Struct {
func (oc *ObjectCreator) updateExistingObject(st *state.State, oldIDtoNew map[string]string, newID string) *domain.Details {
if st.Store() != nil {
oc.updateLinksInCollections(st, oldIDtoNew, false)
}
@ -212,7 +211,7 @@ func (oc *ObjectCreator) installBundledRelationsAndTypes(
idsToCheck := make([]string, 0, len(links)+len(objectTypeKeys))
for _, link := range links {
// TODO: check if we have them in oldIDtoNew
if !bundle.HasRelation(link.Key) {
if !bundle.HasRelation(domain.RelationKey(link.Key)) {
continue
}
@ -241,8 +240,8 @@ func (oc *ObjectCreator) createNewObject(
payload treestorage.TreeStorageCreatePayload,
st *state.State,
newID string,
oldIDtoNew map[string]string) (*types.Struct, error) {
var respDetails *types.Struct
oldIDtoNew map[string]string) (*domain.Details, error) {
var respDetails *domain.Details
spc, err := oc.spaceService.Get(ctx, spaceID)
if err != nil {
return nil, fmt.Errorf("get space %s: %w", spaceID, err)
@ -280,7 +279,7 @@ func (oc *ObjectCreator) createNewObject(
return respDetails, nil
}
func (oc *ObjectCreator) setRootBlock(snapshot *model.SmartBlockSnapshotBase, newID string) {
func (oc *ObjectCreator) setRootBlock(snapshot *common.StateSnapshot, newID string) {
var found bool
for _, b := range snapshot.Blocks {
if b.Id == newID {
@ -363,8 +362,8 @@ func (oc *ObjectCreator) setSpaceDashboardID(spaceID string, st *state.State) {
}
}
func (oc *ObjectCreator) resetState(newID string, st *state.State) *types.Struct {
var respDetails *types.Struct
func (oc *ObjectCreator) resetState(newID string, st *state.State) *domain.Details {
var respDetails *domain.Details
err := cache.Do(oc.service, newID, func(b smartblock.SmartBlock) error {
err := history.ResetToVersion(b, st)
if err != nil {
@ -387,7 +386,7 @@ func (oc *ObjectCreator) resetState(newID string, st *state.State) *types.Struct
return respDetails
}
func (oc *ObjectCreator) setFavorite(snapshot *model.SmartBlockSnapshotBase, newID string) {
func (oc *ObjectCreator) setFavorite(snapshot *common.StateSnapshot, newID string) {
isFavorite := snapshot.Details.GetBoolOrDefault(bundle.RelationKeyIsFavorite, false)
if isFavorite {
err := oc.service.SetPageIsFavorite(pb.RpcObjectSetIsFavoriteRequest{ContextId: newID, IsFavorite: true})
@ -397,7 +396,7 @@ func (oc *ObjectCreator) setFavorite(snapshot *model.SmartBlockSnapshotBase, new
}
}
func (oc *ObjectCreator) setArchived(snapshot *model.SmartBlockSnapshotBase, newID string) {
func (oc *ObjectCreator) setArchived(snapshot *common.StateSnapshot, newID string) {
isArchive := snapshot.Details.GetBoolOrDefault(bundle.RelationKeyIsArchived, false)
if isArchive {
err := oc.service.SetPageIsArchived(pb.RpcObjectSetIsArchivedRequest{ContextId: newID, IsArchived: true})
@ -470,7 +469,7 @@ func (oc *ObjectCreator) mergeCollections(existedObjects []string, st *state.Sta
st.UpdateStoreSlice(template.CollectionStoreKey, result)
}
func (oc *ObjectCreator) updateWidgetObject(st *state.State) (*types.Struct, string, error) {
func (oc *ObjectCreator) updateWidgetObject(st *state.State) (*domain.Details, string, error) {
err := cache.DoState(oc.service, st.RootId(), func(oldState *state.State, sb smartblock.SmartBlock) error {
blocks := st.Blocks()
blocksMap := make(map[string]*model.Block, len(blocks))
@ -540,12 +539,13 @@ func (oc *ObjectCreator) getExistingWidgetsTargetIDs(oldState *state.State) (map
}
func (oc *ObjectCreator) updateKeys(st *state.State, oldIDtoNew map[string]string) {
for key, value := range st.Details().GetFields() {
if newKey, ok := oldIDtoNew[key]; ok {
st.SetDetail(newKey, value)
st.Details().Iterate(func(key domain.RelationKey, value any) bool {
if newKey, ok := oldIDtoNew[string(key)]; ok {
st.SetDetail(domain.RelationKey(newKey), value)
st.RemoveRelation(key)
}
}
return true
})
if newKey, ok := oldIDtoNew[st.ObjectTypeKey().String()]; ok {
st.SetObjectTypeKey(domain.TypeKey(newKey))

4
core/block/import/common/objectcreator/task.go
View file

@ -4,9 +4,9 @@ import (
"context"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/gogo/protobuf/types"
"github.com/anyproto/anytype-heart/core/block/import/common"
"github.com/anyproto/anytype-heart/core/domain"
"github.com/anyproto/anytype-heart/core/domain/objectorigin"
)
@ -22,7 +22,7 @@ type DataObject struct {
}
type Result struct {
Details *types.Struct
Details *domain.Details
NewID string
Err error
}

8
core/block/import/common/objectid/derivedobject.go
View file

@ -43,18 +43,18 @@ func (d *derivedObject) GetIDAndPayload(ctx context.Context, spaceID string, sn
rawUniqueKey := sn.Snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeyUniqueKey, "")
uniqueKey, err := domain.UnmarshalUniqueKey(rawUniqueKey)
if err != nil {
uniqueKey, err = domain.NewUniqueKey(sn.SbType, sn.Snapshot.Data.Key)
uniqueKey, err = domain.NewUniqueKey(sn.Snapshot.SbType, sn.Snapshot.Data.Key)
if err != nil {
return "", treestorage.TreeStorageCreatePayload{}, fmt.Errorf("create unique key from %s and %q: %w", sn.SbType, sn.Snapshot.Data.Key, err)
return "", treestorage.TreeStorageCreatePayload{}, fmt.Errorf("create unique key from %s and %q: %w", sn.Snapshot.SbType, sn.Snapshot.Data.Key, err)
}
}
var key string
if d.isDeletedObject(spaceID, uniqueKey.Marshal()) {
key = bson.NewObjectId().Hex()
uniqueKey, err = domain.NewUniqueKey(sn.SbType, key)
uniqueKey, err = domain.NewUniqueKey(sn.Snapshot.SbType, key)
if err != nil {
return "", treestorage.TreeStorageCreatePayload{}, fmt.Errorf("create unique key from %s: %w", sn.SbType, err)
return "", treestorage.TreeStorageCreatePayload{}, fmt.Errorf("create unique key from %s: %w", sn.Snapshot.SbType, err)
}
}
d.internalKey = key

1
core/block/import/common/objectid/fileobject.go
View file

@ -11,7 +11,6 @@ import (
"github.com/anyproto/anytype-heart/core/block/import/common"
"github.com/anyproto/anytype-heart/core/domain/objectorigin"
"github.com/anyproto/anytype-heart/pkg/lib/bundle"
"github.com/anyproto/anytype-heart/util/pbtypes"
)
type fileObject struct {

1
core/block/import/common/objectid/participant.go
View file

@ -11,7 +11,6 @@ import (
"github.com/anyproto/anytype-heart/core/domain"
"github.com/anyproto/anytype-heart/core/domain/objectorigin"
"github.com/anyproto/anytype-heart/pkg/lib/bundle"
"github.com/anyproto/anytype-heart/util/pbtypes"
)
type participant struct{}

2
core/block/import/common/objectid/provider.go
View file

@ -81,7 +81,7 @@ func (p *Provider) GetIDAndPayload(
getExisting bool,
origin objectorigin.ObjectOrigin,
) (string, treestorage.TreeStorageCreatePayload, error) {
if idProvider, ok := p.idProviderBySmartBlockType[sn.SbType]; ok {
if idProvider, ok := p.idProviderBySmartBlockType[sn.Snapshot.SbType]; ok {
return idProvider.GetIDAndPayload(ctx, spaceID, sn, createdTime, getExisting, origin)
}
return "", treestorage.TreeStorageCreatePayload{}, fmt.Errorf("unsupported smartblock to import")

2
core/block/import/common/objectid/treeobject.go
View file

@ -36,7 +36,7 @@ func (t *treeObject) GetIDAndPayload(ctx context.Context, spaceID string, sn *co
}
payload, err = spc.CreateTreePayload(ctx, payloadcreator.PayloadCreationParams{
Time: time.Now(),
SmartblockType: sn.SbType,
SmartblockType: sn.Snapshot.SbType,
})
if err != nil {
return "", treestorage.TreeStorageCreatePayload{}, fmt.Errorf("create tree payload: %w", err)

4
core/block/import/common/syncer/types.go
View file

@ -3,8 +3,6 @@ package syncer
import (
"context"
"github.com/gogo/protobuf/types"
"github.com/anyproto/anytype-heart/core/block"
"github.com/anyproto/anytype-heart/core/block/editor/smartblock"
"github.com/anyproto/anytype-heart/core/block/simple"
@ -15,7 +13,7 @@ import (
type BlockService interface {
GetObject(ctx context.Context, objectID string) (sb smartblock.SmartBlock, err error)
GetObjectByFullID(ctx context.Context, id domain.FullID) (sb smartblock.SmartBlock, err error)
UploadFile(ctx context.Context, spaceId string, req block.FileUploadRequest) (objectId string, details *types.Struct, err error)
UploadFile(ctx context.Context, spaceId string, req block.FileUploadRequest) (objectId string, details *domain.Details, err error)
UploadFileBlock(contextID string, req block.UploadRequest) (fileObjectId string, err error)
}

53
core/block/import/common/types.go
View file

@ -45,11 +45,19 @@ type Snapshot struct {
type SnapshotModel struct {
SbType coresb.SmartBlockType
LogHeads map[string]string
Data *SnapshotModelData
Data *StateSnapshot
FileKeys []*pb.ChangeFileKeys
}
type SnapshotModelData struct {
func (sn *SnapshotModel) ToProto() *pb.ChangeSnapshot {
return &pb.ChangeSnapshot{
Data: sn.Data.ToProto(),
LogHeads: sn.LogHeads,
FileKeys: sn.FileKeys,
}
}
type StateSnapshot struct {
Blocks []*model.Block
Details *domain.Details
FileKeys *types.Struct
@ -63,6 +71,47 @@ type SnapshotModelData struct {
FileInfo *model.FileInfo
}
func (sn *StateSnapshot) ToProto() *model.SmartBlockSnapshotBase {
return &model.SmartBlockSnapshotBase{
Blocks: sn.Blocks,
Details: sn.Details.ToProto(),
FileKeys: sn.FileKeys,
ExtraRelations: sn.ExtraRelations,
ObjectTypes: sn.ObjectTypes,
Collections: sn.Collections,
RemovedCollectionKeys: sn.RemovedCollectionKeys,
RelationLinks: sn.RelationLinks,
Key: sn.Key,
OriginalCreatedTimestamp: sn.OriginalCreatedTimestamp,
FileInfo: sn.FileInfo,
}
}
func NewStateSnapshotFromProto(sn *model.SmartBlockSnapshotBase) *StateSnapshot {
return &StateSnapshot{
Blocks: sn.Blocks,
Details: domain.NewDetailsFromProto(sn.Details),
FileKeys: sn.FileKeys,
ExtraRelations: sn.ExtraRelations,
ObjectTypes: sn.ObjectTypes,
Collections: sn.Collections,
RemovedCollectionKeys: sn.RemovedCollectionKeys,
RelationLinks: sn.RelationLinks,
Key: sn.Key,
OriginalCreatedTimestamp: sn.OriginalCreatedTimestamp,
FileInfo: sn.FileInfo,
}
}
func NewSnapshotModelFromProto(sn *pb.SnapshotWithType) *SnapshotModel {
return &SnapshotModel{
SbType: coresb.SmartBlockType(sn.SbType),
LogHeads: sn.Snapshot.LogHeads,
Data: NewStateSnapshotFromProto(sn.Snapshot.Data),
FileKeys: sn.Snapshot.FileKeys,
}
}
// Response expected response of each converter, incapsulate blocks snapshots and converting errors
type Response struct {
Snapshots []*Snapshot

52
core/block/import/csv/collectionstrategy.go
View file

@ -7,7 +7,6 @@ import (
"strings"
"github.com/globalsign/mgo/bson"
"github.com/gogo/protobuf/types"
"github.com/google/uuid"
"github.com/samber/lo"
@ -73,7 +72,7 @@ func (c *CollectionStrategy) CreateObjects(path string, csvTable [][]string, par
return snapshot.Id, snapshots, nil
}
func updateDetailsForTransposeCollection(details *types.Struct, transpose bool) {
func updateDetailsForTransposeCollection(details *domain.Details, transpose bool) {
if transpose {
source := details.GetStringOrDefault(bundle.RelationKeySourceFilePath, "")
source = source + string(filepath.Separator) + transposeSource
@ -118,13 +117,14 @@ func getDetailsFromCSVTable(csvTable [][]string, useFirstRowForRelations bool) (
Key: id,
})
relationsSnapshots = append(relationsSnapshots, &common.Snapshot{
Id: id,
SbType: smartblock.SmartBlockTypeRelation,
Snapshot: &pb.ChangeSnapshot{Data: &model.SmartBlockSnapshotBase{
Details: getRelationDetails(relationName, id, float64(model.RelationFormat_longtext)),
ObjectTypes: []string{bundle.TypeKeyRelation.String()},
Key: id,
}},
Id: id,
Snapshot: &common.SnapshotModel{
SbType: smartblock.SmartBlockTypeRelation,
Data: &common.StateSnapshot{
Details: getRelationDetails(relationName, id, float64(model.RelationFormat_longtext)),
ObjectTypes: []string{bundle.TypeKeyRelation.String()},
Key: id,
}},
})
}
return relations, relationsSnapshots, err
@ -171,8 +171,8 @@ func getDefaultRelationName(i int) string {
return defaultRelationName + " " + strconv.FormatInt(int64(i), 10)
}
func getRelationDetails(name, key string, format float64) *types.Struct {
details := &types.Struct{Fields: map[string]*types.Value{}}
func getRelationDetails(name, key string, format float64) *domain.Details {
details := domain.NewDetails()
details.Set(bundle.RelationKeyRelationFormat, pbtypes.Float64(format))
details.Set(bundle.RelationKeyName, pbtypes.String(name))
details.Set(bundle.RelationKeyRelationKey, pbtypes.String(key))
@ -233,15 +233,15 @@ func buildSourcePath(path string, i int, transpose bool) string {
transposePart
}
func getDetailsForObject(relationsValues []string, relations []*model.Relation, path string, objectOrderIndex int, transpose bool) (*types.Struct, []*model.RelationLink) {
details := &types.Struct{Fields: map[string]*types.Value{}}
func getDetailsForObject(relationsValues []string, relations []*model.Relation, path string, objectOrderIndex int, transpose bool) (*domain.Details, []*model.RelationLink) {
details := domain.NewDetails()
relationLinks := make([]*model.RelationLink, 0)
for j, value := range relationsValues {
if len(relations) <= j {
break
}
relation := relations[j]
details.Set(relation.Key, pbtypes.String(value))
details.Set(domain.RelationKey(relation.Key), pbtypes.String(value))
relationLinks = append(relationLinks, &model.RelationLink{
Key: relation.Key,
Format: relation.Format,
@ -252,12 +252,12 @@ func getDetailsForObject(relationsValues []string, relations []*model.Relation,
return details, relationLinks
}
func provideObjectSnapshot(st *state.State, details *types.Struct) *common.Snapshot {
func provideObjectSnapshot(st *state.State, details *domain.Details) *common.Snapshot {
snapshot := &common.Snapshot{
Id: uuid.New().String(),
SbType: smartblock.SmartBlockTypePage,
Snapshot: &pb.ChangeSnapshot{
Data: &model.SmartBlockSnapshotBase{
Id: uuid.New().String(),
Snapshot: &common.SnapshotModel{
SbType: smartblock.SmartBlockTypePage,
Data: &common.StateSnapshot{
Blocks: st.Blocks(),
Details: details,
RelationLinks: st.GetRelationLinks(),
@ -268,8 +268,8 @@ func provideObjectSnapshot(st *state.State, details *types.Struct) *common.Snaps
return snapshot
}
func (c *CollectionStrategy) getCollectionSnapshot(details *types.Struct, st *state.State, p string, relations []*model.Relation) *common.Snapshot {
details = pbtypes.StructMerge(st.CombinedDetails(), details, false)
func (c *CollectionStrategy) getCollectionSnapshot(details *domain.Details, st *state.State, p string, relations []*model.Relation) *common.Snapshot {
details = st.CombinedDetails().Merge(details)
details.Set(bundle.RelationKeyLayout, pbtypes.Float64(float64(model.ObjectType_collection)))
for _, relation := range relations {
@ -284,8 +284,8 @@ func (c *CollectionStrategy) getCollectionSnapshot(details *types.Struct, st *st
return c.provideCollectionSnapshots(details, st, p)
}
func (c *CollectionStrategy) provideCollectionSnapshots(details *types.Struct, st *state.State, p string) *common.Snapshot {
sn := &model.SmartBlockSnapshotBase{
func (c *CollectionStrategy) provideCollectionSnapshots(details *domain.Details, st *state.State, p string) *common.Snapshot {
sn := &common.StateSnapshot{
Blocks: st.Blocks(),
Details: details,
ObjectTypes: []string{bundle.TypeKeyCollection.String()},
@ -296,8 +296,10 @@ func (c *CollectionStrategy) provideCollectionSnapshots(details *types.Struct, s
snapshot := &common.Snapshot{
Id: uuid.New().String(),
FileName: p,
Snapshot: &pb.ChangeSnapshot{Data: sn},
SbType: smartblock.SmartBlockTypePage,
Snapshot: &common.SnapshotModel{
SbType: smartblock.SmartBlockTypePage,
Data: sn,
},
}
return snapshot
}

8
core/block/import/csv/tablestrategy.go
View file

@ -40,7 +40,7 @@ func (c *TableStrategy) CreateObjects(path string, csvTable [][]string, params *
}
details := common.GetCommonDetails(path, "", "", model.ObjectType_basic)
sn := &model.SmartBlockSnapshotBase{
sn := &common.StateSnapshot{
Blocks: st.Blocks(),
Details: details,
ObjectTypes: []string{bundle.TypeKeyPage.String()},
@ -50,9 +50,11 @@ func (c *TableStrategy) CreateObjects(path string, csvTable [][]string, params *
snapshot := &common.Snapshot{
Id: uuid.New().String(),
SbType: smartblock.SmartBlockTypePage,
FileName: path,
Snapshot: &pb.ChangeSnapshot{Data: sn},
Snapshot: &common.SnapshotModel{
SbType: smartblock.SmartBlockTypePage,
Data: sn,
},
}
progress.AddDone(1)
return snapshot.Id, []*common.Snapshot{snapshot}, nil

8
core/block/import/html/converter.go
View file

@ -199,7 +199,7 @@ func (h *HTML) updateFilesInLinks(block *model.Block, filesSource source.Source,
}
func (h *HTML) getSnapshot(blocks []*model.Block, p string) (*common.Snapshot, string) {
sn := &common.SnapshotModelData{
sn := &common.StateSnapshot{
Blocks: blocks,
Details: common.GetCommonDetails(p, "", "", model.ObjectType_basic),
ObjectTypes: []string{bundle.TypeKeyPage.String()},
@ -208,8 +208,10 @@ func (h *HTML) getSnapshot(blocks []*model.Block, p string) (*common.Snapshot, s
snapshot := &common.Snapshot{
Id: uuid.New().String(),
FileName: p,
Snapshot: &common.SnapshotModel{Data: sn},
SbType: smartblock.SmartBlockTypePage,
Snapshot: &common.SnapshotModel{
SbType: smartblock.SmartBlockTypePage,
Data: sn,
},
}
return snapshot, snapshot.Id
}

21
core/block/import/importer.go
View file

@ -10,7 +10,6 @@ import (
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/gogo/protobuf/types"
"github.com/google/uuid"
"github.com/samber/lo"
"go.uber.org/zap"
@ -211,8 +210,10 @@ func (i *Import) importFromExternalSource(ctx context.Context,
sn := make([]*common.Snapshot, len(req.Snapshots))
for i, s := range req.Snapshots {
sn[i] = &common.Snapshot{
Id: s.GetId(),
Snapshot: &pb.ChangeSnapshot{Data: s.Snapshot},
Id: s.GetId(),
Snapshot: &common.SnapshotModel{
Data: common.NewStateSnapshotFromProto(s.Snapshot),
},
}
}
res := &common.Response{
@ -277,7 +278,7 @@ func (i *Import) ValidateNotionToken(
return tv.Validate(ctx, req.GetToken())
}
func (i *Import) ImportWeb(ctx context.Context, req *pb.RpcObjectImportRequest) (string, *types.Struct, error) {
func (i *Import) ImportWeb(ctx context.Context, req *pb.RpcObjectImportRequest) (string, *domain.Details, error) {
progress := process.NewProgress(pb.ModelProcess_Import)
defer progress.Finish(nil)
allErrors := common.NewError(0)
@ -329,7 +330,7 @@ func (i *Import) createObjects(ctx context.Context,
func (i *Import) getFilesIDs(res *common.Response) []string {
fileIDs := make([]string, 0)
for _, snapshot := range res.Snapshots {
fileIDs = append(fileIDs, lo.Map(snapshot.Snapshot.GetFileKeys(), func(item *pb.ChangeFileKeys, index int) string {
fileIDs = append(fileIDs, lo.Map(snapshot.Snapshot.FileKeys, func(item *pb.ChangeFileKeys, index int) string {
return item.Hash
})...)
}
@ -347,7 +348,7 @@ func (i *Import) getIDForAllObjects(ctx context.Context,
createPayloads := make(map[string]treestorage.TreeStorageCreatePayload, len(res.Snapshots))
for _, snapshot := range res.Snapshots {
// we will get id of relation options after we figure out according relations keys
if lo.Contains(snapshot.Snapshot.GetData().GetObjectTypes(), bundle.TypeKeyRelationOption.String()) {
if lo.Contains(snapshot.Snapshot.Data.ObjectTypes, bundle.TypeKeyRelationOption.String()) {
relationOptions = append(relationOptions, snapshot)
continue
}
@ -375,7 +376,7 @@ func (i *Import) getIDForAllObjects(ctx context.Context,
}
func (i *Import) replaceRelationKeyWithNew(option *common.Snapshot, oldIDToNew map[string]string) {
if option.Snapshot.Data.Details == nil || len(option.Snapshot.Data.Details.Fields) == 0 {
if option.Snapshot.Data.Details == nil || option.Snapshot.Data.Details.Len() == 0 {
return
}
key := option.Snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeyRelationKey, "")
@ -396,7 +397,7 @@ func (i *Import) getObjectID(
) error {
// Preload file keys
for _, fileKeys := range snapshot.Snapshot.GetFileKeys() {
for _, fileKeys := range snapshot.Snapshot.FileKeys {
err := i.fileStore.AddFileKeys(domain.FileEncryptionKeys{
FileId: domain.FileId(fileKeys.Hash),
EncryptionKeys: fileKeys.Keys,
@ -405,7 +406,7 @@ func (i *Import) getObjectID(
return fmt.Errorf("add file keys: %w", err)
}
}
if fileInfo := snapshot.Snapshot.GetData().GetFileInfo(); fileInfo != nil {
if fileInfo := snapshot.Snapshot.Data.FileInfo; fileInfo != nil {
keys := make(map[string]string, len(fileInfo.EncryptionKeys))
for _, key := range fileInfo.EncryptionKeys {
keys[key.Path] = key.Key
@ -435,7 +436,7 @@ func (i *Import) getObjectID(
}
func (i *Import) extractInternalKey(snapshot *common.Snapshot, oldIDToNew map[string]string) error {
newUniqueKey := i.idProvider.GetInternalKey(snapshot.SbType)
newUniqueKey := i.idProvider.GetInternalKey(snapshot.Snapshot.SbType)
if newUniqueKey != "" {
oldUniqueKey := snapshot.Snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeyUniqueKey, "")
if oldUniqueKey == "" {

13
core/block/import/markdown/import.go
View file

@ -387,12 +387,13 @@ func (m *Markdown) createSnapshots(files map[string]*FileInfo,
snapshots = append(snapshots, &common.Snapshot{
Id: file.PageID,
FileName: name,
SbType: smartblock.SmartBlockTypePage,
Snapshot: &common.SnapshotModel{Data: &common.SnapshotModelData{
Blocks: file.ParsedBlocks,
Details: details[name],
ObjectTypes: []string{bundle.TypeKeyPage.String()},
}},
Snapshot: &common.SnapshotModel{
SbType: smartblock.SmartBlockTypePage,
Data: &common.StateSnapshot{
Blocks: file.ParsedBlocks,
Details: details[name],
ObjectTypes: []string{bundle.TypeKeyPage.String()},
}},
})
}

9
core/block/import/notion/api/commonobjects.go
View file

@ -6,11 +6,10 @@ import (
"time"
"github.com/globalsign/mgo/bson"
"github.com/gogo/protobuf/types"
"github.com/anyproto/anytype-heart/core/domain"
"github.com/anyproto/anytype-heart/pkg/lib/bundle"
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
"github.com/anyproto/anytype-heart/util/pbtypes"
)
type RichTextType string
@ -285,9 +284,9 @@ type Icon struct {
External *FileProperty `json:"external,omitempty"`
}
func SetIcon(details map[string]*types.Value, icon *Icon) *model.RelationLink {
func SetIcon(details *domain.Details, icon *Icon) *model.RelationLink {
if icon.Emoji != nil {
details[bundle.RelationKeyIconEmoji.String()] = pbtypes.String(*icon.Emoji)
details.Set(bundle.RelationKeyIconEmoji, *icon.Emoji)
}
var linkToIconImage string
if icon.Type == External && icon.External != nil {
@ -297,7 +296,7 @@ func SetIcon(details map[string]*types.Value, icon *Icon) *model.RelationLink {
linkToIconImage = icon.File.URL
}
if linkToIconImage != "" {
details[bundle.RelationKeyIconImage.String()] = pbtypes.String(linkToIconImage)
details.Set(bundle.RelationKeyIconImage, linkToIconImage)
return &model.RelationLink{
Key: bundle.RelationKeyIconImage.String(),
Format: model.RelationFormat_file,

81
core/block/import/notion/api/database/database.go
View file

@ -78,8 +78,8 @@ func (ds *Service) GetDatabase(_ context.Context,
)
progress.SetProgressMessage("Start creating pages from notion databases")
relations := &property.PropertiesStore{
PropertyIdsToSnapshots: make(map[string]*model.SmartBlockSnapshotBase, 0),
RelationsIdsToOptions: make(map[string][]*model.SmartBlockSnapshotBase, 0),
PropertyIdsToSnapshots: make(map[string]*common.StateSnapshot, 0),
RelationsIdsToOptions: make(map[string][]*common.StateSnapshot, 0),
}
for _, d := range databases {
if err := progress.TryStep(1); err != nil {
@ -106,14 +106,13 @@ func (ds *Service) makeDatabaseSnapshot(d Database,
importContext *api.NotionImportContext,
relations *property.PropertiesStore) ([]*common.Snapshot, error) {
details := ds.getCollectionDetails(d)
detailsStruct := &types.Struct{Fields: details}
_, _, st, err := ds.collectionService.CreateCollection(detailsStruct, nil)
_, _, st, err := ds.collectionService.CreateCollection(details, nil)
if err != nil {
return nil, err
}
detailsStruct = pbtypes.StructMerge(st.CombinedDetails(), detailsStruct, false)
details = st.CombinedDetails().Merge(details)
snapshots := ds.makeRelationsSnapshots(d, st, relations)
id, databaseSnapshot := ds.provideDatabaseSnapshot(d, st, detailsStruct)
id, databaseSnapshot := ds.provideDatabaseSnapshot(d, st, details)
ds.fillImportContext(d, importContext, id, databaseSnapshot)
snapshots = append(snapshots, databaseSnapshot)
return snapshots, nil
@ -121,7 +120,7 @@ func (ds *Service) makeDatabaseSnapshot(d Database,
func (ds *Service) fillImportContext(d Database, req *api.NotionImportContext, id string, databaseSnapshot *common.Snapshot) {
req.NotionDatabaseIdsToAnytype[d.ID] = id
req.DatabaseNameToID[d.ID] = databaseSnapshot.Snapshot.GetData().GetDetails().GetStringOrDefault(bundle.RelationKeyName, "")
req.DatabaseNameToID[d.ID] = databaseSnapshot.Snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeyName, "")
if d.Parent.DatabaseID != "" {
req.PageTree.ParentPageToChildIDs[d.Parent.DatabaseID] = append(req.PageTree.ParentPageToChildIDs[d.Parent.DatabaseID], d.ID)
}
@ -172,7 +171,7 @@ func (ds *Service) getNameAndRelationKeyForTagProperty(databaseProperty property
}
func (ds *Service) handleNameProperty(databaseProperty property.DatabasePropertyHandler, st *state.State) *common.Snapshot {
databaseProperty.SetDetail(bundle.RelationKeyName, st.Details().GetFields())
databaseProperty.SetDetail(bundle.RelationKeyName.String(), st.Details())
relationLinks := &model.RelationLink{
Key: bundle.RelationKeyName.String(),
Format: model.RelationFormat_shorttext,
@ -189,15 +188,15 @@ func (ds *Service) makeRelationSnapshotFromDatabaseProperty(relations *property.
name, relationKey string,
st *state.State) *common.Snapshot {
var (
rel *model.SmartBlockSnapshotBase
rel *common.StateSnapshot
sn *common.Snapshot
)
if rel = relations.ReadRelationsMap(databaseProperty.GetID()); rel == nil {
rel, sn = ds.getRelationSnapshot(relationKey, databaseProperty, name)
relations.WriteToRelationsMap(databaseProperty.GetID(), rel)
}
relKey := rel.GetDetails().GetStringOrDefault(bundle.RelationKeyRelationKey, "")
databaseProperty.SetDetail(relKey, st.Details().GetFields())
relKey := rel.Details.GetStringOrDefault(bundle.RelationKeyRelationKey, "")
databaseProperty.SetDetail(relKey, st.Details())
relationLinks := &model.RelationLink{
Key: relKey,
Format: databaseProperty.GetFormat(),
@ -217,28 +216,28 @@ func (ds *Service) makeRelationSnapshotFromDatabaseProperty(relations *property.
return sn
}
func (ds *Service) getRelationSnapshot(relationKey string, databaseProperty property.DatabasePropertyHandler, name string) (*model.SmartBlockSnapshotBase, *common.Snapshot) {
func (ds *Service) getRelationSnapshot(relationKey string, databaseProperty property.DatabasePropertyHandler, name string) (*common.StateSnapshot, *common.Snapshot) {
relationDetails := ds.getRelationDetails(databaseProperty, name, relationKey)
relationSnapshot := &model.SmartBlockSnapshotBase{
relationSnapshot := &common.StateSnapshot{
Details: relationDetails,
ObjectTypes: []string{bundle.TypeKeyRelation.String()},
Key: relationKey,
}
snapshot := &common.Snapshot{
Id: relationDetails.GetStringOrDefault(bundle.RelationKeyId, ""),
Snapshot: &pb.ChangeSnapshot{
Data: relationSnapshot,
Snapshot: &common.SnapshotModel{
SbType: sb.SmartBlockTypeRelation,
Data: relationSnapshot,
},
SbType: sb.SmartBlockTypeRelation,
}
return relationSnapshot, snapshot
}
func (ds *Service) getRelationDetails(databaseProperty property.DatabasePropertyHandler, name, key string) *types.Struct {
func (ds *Service) getRelationDetails(databaseProperty property.DatabasePropertyHandler, name, key string) *domain.Details {
if name == "" {
name = property.UntitledProperty
}
details := &types.Struct{Fields: map[string]*types.Value{}}
details := domain.NewDetails()
details.Set(bundle.RelationKeyRelationFormat, pbtypes.Float64(float64(databaseProperty.GetFormat())))
details.Set(bundle.RelationKeyName, pbtypes.String(name))
details.Set(bundle.RelationKeyRelationKey, pbtypes.String(key))
@ -254,46 +253,46 @@ func (ds *Service) getRelationDetails(databaseProperty property.DatabaseProperty
return details
}
func (ds *Service) getCollectionDetails(d Database) map[string]*types.Value {
details := make(map[string]*types.Value, 0)
details[bundle.RelationKeySourceFilePath.String()] = pbtypes.String(d.ID)
func (ds *Service) getCollectionDetails(d Database) *domain.Details {
details := domain.NewDetails()
details.Set(bundle.RelationKeySourceFilePath, d.ID)
if len(d.Title) > 0 {
details[bundle.RelationKeyName.String()] = pbtypes.String(d.Title[0].PlainText)
details.Set(bundle.RelationKeyName, d.Title[0].PlainText)
}
if d.Icon != nil && d.Icon.Emoji != nil {
details[bundle.RelationKeyIconEmoji.String()] = pbtypes.String(*d.Icon.Emoji)
details.Set(bundle.RelationKeyIconEmoji, *d.Icon.Emoji)
}
if d.Cover != nil {
if d.Cover.Type == api.External {
details[bundle.RelationKeyCoverId.String()] = pbtypes.String(d.Cover.External.URL)
details[bundle.RelationKeyCoverType.String()] = pbtypes.Float64(1)
details.Set(bundle.RelationKeyCoverId, d.Cover.External.URL)
details.Set(bundle.RelationKeyCoverType, 1)
}
if d.Cover.Type == api.File {
details[bundle.RelationKeyCoverId.String()] = pbtypes.String(d.Cover.File.URL)
details[bundle.RelationKeyCoverType.String()] = pbtypes.Float64(1)
details.Set(bundle.RelationKeyCoverId, d.Cover.File.URL)
details.Set(bundle.RelationKeyCoverType, 1)
}
}
if d.Icon != nil {
api.SetIcon(details, d.Icon)
}
details[bundle.RelationKeyCreator.String()] = pbtypes.String(d.CreatedBy.Name)
details[bundle.RelationKeyIsArchived.String()] = pbtypes.Bool(d.Archived)
details[bundle.RelationKeyLastModifiedBy.String()] = pbtypes.String(d.LastEditedBy.Name)
details[bundle.RelationKeyDescription.String()] = pbtypes.String(api.RichTextToDescription(d.Description))
details[bundle.RelationKeyIsFavorite.String()] = pbtypes.Bool(false)
details[bundle.RelationKeyLayout.String()] = pbtypes.Float64(float64(model.ObjectType_collection))
details.Set(bundle.RelationKeyCreator, d.CreatedBy.Name)
details.Set(bundle.RelationKeyIsArchived, d.Archived)
details.Set(bundle.RelationKeyLastModifiedBy, d.LastEditedBy.Name)
details.Set(bundle.RelationKeyDescription, api.RichTextToDescription(d.Description))
details.Set(bundle.RelationKeyIsFavorite, false)
details.Set(bundle.RelationKeyLayout, float64(model.ObjectType_collection))
details[bundle.RelationKeyLastModifiedDate.String()] = pbtypes.Float64(float64(d.LastEditedTime.Unix()))
details[bundle.RelationKeyCreatedDate.String()] = pbtypes.Float64(float64(d.CreatedTime.Unix()))
details.Set(bundle.RelationKeyLastModifiedDate, float64(d.LastEditedTime.Unix()))
details.Set(bundle.RelationKeyCreatedDate, float64(d.CreatedTime.Unix()))
return details
}
func (ds *Service) provideDatabaseSnapshot(d Database, st *state.State, detailsStruct *types.Struct) (string, *common.Snapshot) {
snapshot := &model.SmartBlockSnapshotBase{
func (ds *Service) provideDatabaseSnapshot(d Database, st *state.State, details *domain.Details) (string, *common.Snapshot) {
snapshot := &common.StateSnapshot{
Blocks: st.Blocks(),
Details: detailsStruct,
Details: details,
ObjectTypes: []string{bundle.TypeKeyCollection.String()},
Collections: st.Store(),
RelationLinks: st.GetRelationLinks(),
@ -303,8 +302,10 @@ func (ds *Service) provideDatabaseSnapshot(d Database, st *state.State, detailsS
databaseSnapshot := &common.Snapshot{
Id: id,
FileName: d.URL,
Snapshot: &pb.ChangeSnapshot{Data: snapshot},
SbType: sb.SmartBlockTypePage,
Snapshot: &common.SnapshotModel{
SbType: sb.SmartBlockTypePage,
Data: snapshot,
},
}
return id, databaseSnapshot
}

107
core/block/import/notion/api/page/task.go
View file

@ -8,7 +8,6 @@ import (
"time"
"github.com/globalsign/mgo/bson"
"github.com/gogo/protobuf/types"
"github.com/samber/lo"
"github.com/anyproto/anytype-heart/core/block/import/common"
@ -64,22 +63,26 @@ func (pt *Task) Execute(data interface{}) interface{} {
sn := &common.Snapshot{
Id: pageId,
FileName: pt.p.URL,
Snapshot: &pb.ChangeSnapshot{Data: snapshot},
SbType: smartblock.SmartBlockTypePage,
Snapshot: &common.SnapshotModel{
SbType: smartblock.SmartBlockTypePage,
Data: snapshot,
},
}
resultSnapshots = append(resultSnapshots, sn)
for _, objectsSnapshot := range subObjectsSnapshots {
sbType := pt.getSmartBlockTypeAndID(objectsSnapshot)
sbType := pt.getRelationOrOptionType(objectsSnapshot)
resultSnapshots = append(resultSnapshots, &common.Snapshot{
Id: objectsSnapshot.Details.GetStringOrDefault(bundle.RelationKeyId, ""),
SbType: sbType,
Snapshot: &pb.ChangeSnapshot{Data: objectsSnapshot},
Id: objectsSnapshot.Details.GetStringOrDefault(bundle.RelationKeyId, ""),
Snapshot: &common.SnapshotModel{
SbType: sbType,
Data: objectsSnapshot,
},
})
}
return &Result{snapshot: resultSnapshots, ce: allErrors}
}
func (pt *Task) makeSnapshotFromPages(object *DataObject, allErrors *common.ConvertError) (*model.SmartBlockSnapshotBase, []*model.SmartBlockSnapshotBase) {
func (pt *Task) makeSnapshotFromPages(object *DataObject, allErrors *common.ConvertError) (*common.StateSnapshot, []*common.StateSnapshot) {
details, subObjectsSnapshots, relationLinks := pt.provideDetails(object)
notionBlocks, blocksAndChildrenErr := pt.blockService.GetBlocksAndChildren(object.ctx, pt.p.ID, object.apiKey, pageSize, object.mode)
if blocksAndChildrenErr != nil {
@ -93,7 +96,7 @@ func (pt *Task) makeSnapshotFromPages(object *DataObject, allErrors *common.Conv
return snapshot, subObjectsSnapshots
}
func (pt *Task) provideDetails(object *DataObject) (map[string]*types.Value, []*model.SmartBlockSnapshotBase, []*model.RelationLink) {
func (pt *Task) provideDetails(object *DataObject) (*domain.Details, []*common.StateSnapshot, []*model.RelationLink) {
details, relationLinks := pt.prepareDetails()
relationsSnapshots, notionRelationLinks := pt.handlePageProperties(object, details)
relationLinks = append(relationLinks, notionRelationLinks...)
@ -101,39 +104,39 @@ func (pt *Task) provideDetails(object *DataObject) (map[string]*types.Value, []*
return details, relationsSnapshots, relationLinks
}
func (pt *Task) provideSnapshot(notionBlocks []*model.Block, details map[string]*types.Value, relationLinks []*model.RelationLink) *model.SmartBlockSnapshotBase {
snapshot := &model.SmartBlockSnapshotBase{
func (pt *Task) provideSnapshot(notionBlocks []*model.Block, details *domain.Details, relationLinks []*model.RelationLink) *common.StateSnapshot {
snapshot := &common.StateSnapshot{
Blocks: notionBlocks,
Details: &types.Struct{Fields: details},
Details: details,
ObjectTypes: []string{bundle.TypeKeyPage.String()},
RelationLinks: relationLinks,
}
return snapshot
}
func (pt *Task) prepareDetails() (map[string]*types.Value, []*model.RelationLink) {
details := make(map[string]*types.Value, 0)
func (pt *Task) prepareDetails() (*domain.Details, []*model.RelationLink) {
details := domain.NewDetails()
var relationLinks []*model.RelationLink
details[bundle.RelationKeySourceFilePath.String()] = pbtypes.String(pt.p.ID)
details.Set(bundle.RelationKeySourceFilePath, pt.p.ID)
if pt.p.Icon != nil {
if iconRelationLink := api.SetIcon(details, pt.p.Icon); iconRelationLink != nil {
relationLinks = append(relationLinks, iconRelationLink)
}
}
details[bundle.RelationKeyIsArchived.String()] = pbtypes.Bool(pt.p.Archived)
details[bundle.RelationKeyIsFavorite.String()] = pbtypes.Bool(false)
details.Set(bundle.RelationKeyIsArchived, pt.p.Archived)
details.Set(bundle.RelationKeyIsFavorite, false)
createdTime := common.ConvertStringToTime(pt.p.CreatedTime)
lastEditedTime := common.ConvertStringToTime(pt.p.LastEditedTime)
details[bundle.RelationKeyLastModifiedDate.String()] = pbtypes.Float64(float64(lastEditedTime))
details[bundle.RelationKeyCreatedDate.String()] = pbtypes.Float64(float64(createdTime))
details[bundle.RelationKeyLayout.String()] = pbtypes.Float64(float64(model.ObjectType_basic))
details.Set(bundle.RelationKeyLastModifiedDate, float64(lastEditedTime))
details.Set(bundle.RelationKeyCreatedDate, float64(createdTime))
details.Set(bundle.RelationKeyLayout, float64(model.ObjectType_basic))
return details, relationLinks
}
// handlePageProperties gets properties values by their ids from notion api
// and transforms them to Details and RelationLinks
func (pt *Task) handlePageProperties(object *DataObject, details map[string]*types.Value) ([]*model.SmartBlockSnapshotBase, []*model.RelationLink) {
relationsSnapshots := make([]*model.SmartBlockSnapshotBase, 0)
func (pt *Task) handlePageProperties(object *DataObject, details *domain.Details) ([]*common.StateSnapshot, []*model.RelationLink) {
relationsSnapshots := make([]*common.StateSnapshot, 0)
relationsLinks := make([]*model.RelationLink, 0)
hasTag := isPageContainsTagProperty(pt.p.Properties)
var tagExist bool
@ -152,7 +155,7 @@ func (pt *Task) handlePageProperties(object *DataObject, details map[string]*typ
return relationsSnapshots, relationsLinks
}
func (pt *Task) retrieveRelation(object *DataObject, key string, propObject property.Object, details map[string]*types.Value, hasTag bool, tagExist bool) ([]*model.SmartBlockSnapshotBase, *model.RelationLink, error) {
func (pt *Task) retrieveRelation(object *DataObject, key string, propObject property.Object, details *domain.Details, hasTag bool, tagExist bool) ([]*common.StateSnapshot, *model.RelationLink, error) {
if err := pt.handlePagination(object.ctx, object.apiKey, propObject); err != nil {
return nil, nil, err
}
@ -162,15 +165,15 @@ func (pt *Task) retrieveRelation(object *DataObject, key string, propObject prop
func (pt *Task) makeRelationFromProperty(relation *property.PropertiesStore,
propObject property.Object,
details map[string]*types.Value,
details *domain.Details,
name string,
hasTag, tagExist bool) ([]*model.SmartBlockSnapshotBase, *model.RelationLink, error) {
hasTag, tagExist bool) ([]*common.StateSnapshot, *model.RelationLink, error) {
pt.relationCreateMutex.Lock()
defer pt.relationCreateMutex.Unlock()
var (
snapshot *model.SmartBlockSnapshotBase
snapshot *common.StateSnapshot
key string
subObjectsSnapshots []*model.SmartBlockSnapshotBase
subObjectsSnapshots []*common.StateSnapshot
)
if snapshot = relation.ReadRelationsMap(propObject.GetID()); snapshot == nil {
snapshot, key = pt.getRelationSnapshot(name, propObject, hasTag, tagExist)
@ -180,7 +183,7 @@ func (pt *Task) makeRelationFromProperty(relation *property.PropertiesStore,
}
}
if key == "" {
key = snapshot.GetDetails().GetStringOrDefault(bundle.RelationKeyRelationKey, "")
key = snapshot.Details.GetStringOrDefault(bundle.RelationKeyRelationKey, "")
}
subObjectsSnapshots = append(subObjectsSnapshots, pt.provideRelationOptionsSnapshots(key, propObject, relation)...)
if err := pt.setDetails(propObject, key, details); err != nil {
@ -193,7 +196,7 @@ func (pt *Task) makeRelationFromProperty(relation *property.PropertiesStore,
return subObjectsSnapshots, relationLink, nil
}
func (pt *Task) getRelationSnapshot(name string, propObject property.Object, hasTag bool, tagExist bool) (*model.SmartBlockSnapshotBase, string) {
func (pt *Task) getRelationSnapshot(name string, propObject property.Object, hasTag bool, tagExist bool) (*common.StateSnapshot, string) {
key := bson.NewObjectId().Hex()
if propObject.GetPropertyType() == property.PropertyConfigTypeTitle {
return nil, bundle.RelationKeyName.String()
@ -202,7 +205,7 @@ func (pt *Task) getRelationSnapshot(name string, propObject property.Object, has
key = bundle.RelationKeyTag.String()
}
details := pt.getRelationDetails(key, name, propObject)
rel := &model.SmartBlockSnapshotBase{
rel := &common.StateSnapshot{
Details: details,
ObjectTypes: []string{bundle.TypeKeyRelation.String()},
Key: key,
@ -210,10 +213,10 @@ func (pt *Task) getRelationSnapshot(name string, propObject property.Object, has
return rel, key
}
func (pt *Task) provideRelationOptionsSnapshots(id string, propObject property.Object, relation *property.PropertiesStore) []*model.SmartBlockSnapshotBase {
func (pt *Task) provideRelationOptionsSnapshots(id string, propObject property.Object, relation *property.PropertiesStore) []*common.StateSnapshot {
pt.relationOptCreateMutex.Lock()
defer pt.relationOptCreateMutex.Unlock()
subObjectsSnapshots := make([]*model.SmartBlockSnapshotBase, 0)
subObjectsSnapshots := make([]*common.StateSnapshot, 0)
if isPropertyTag(propObject) {
subObjectsSnapshots = append(subObjectsSnapshots, getRelationOptions(propObject, id, relation)...)
}
@ -287,7 +290,7 @@ func (pt *Task) handlePaginatedProperties(propObject property.Object, properties
}
}
func (pt *Task) setDetails(propObject property.Object, key string, details map[string]*types.Value) error {
func (pt *Task) setDetails(propObject property.Object, key string, details *domain.Details) error {
var (
ds property.DetailSetter
ok bool
@ -299,7 +302,7 @@ func (pt *Task) setDetails(propObject property.Object, key string, details map[s
return nil
}
func (pt *Task) getSmartBlockTypeAndID(objectSnapshot *model.SmartBlockSnapshotBase) smartblock.SmartBlockType {
func (pt *Task) getRelationOrOptionType(objectSnapshot *common.StateSnapshot) smartblock.SmartBlockType {
if lo.Contains(objectSnapshot.ObjectTypes, bundle.TypeKeyRelationOption.String()) {
return smartblock.SmartBlockTypeRelationOption
}
@ -330,16 +333,16 @@ func handleRelationItem(properties []interface{}, pr *property.RelationItem) {
pr.Relation = relationItems
}
func addCoverDetail(p Page, details map[string]*types.Value) {
func addCoverDetail(p Page, details *domain.Details) {
if p.Cover != nil {
if p.Cover.Type == api.External {
details[bundle.RelationKeyCoverId.String()] = pbtypes.String(p.Cover.External.URL)
details[bundle.RelationKeyCoverType.String()] = pbtypes.Float64(1)
details.Set(bundle.RelationKeyCoverId, p.Cover.External.URL)
details.Set(bundle.RelationKeyCoverType, 1)
}
if p.Cover.Type == api.File {
details[bundle.RelationKeyCoverId.String()] = pbtypes.String(p.Cover.File.URL)
details[bundle.RelationKeyCoverType.String()] = pbtypes.Float64(1)
details.Set(bundle.RelationKeyCoverId, p.Cover.File.URL)
details.Set(bundle.RelationKeyCoverType, 1)
}
}
}
@ -359,8 +362,8 @@ func isPropertyTag(pr property.Object) bool {
pr.GetPropertyType() == property.PropertyConfigTypePeople
}
func getRelationOptions(pr property.Object, rel string, relation *property.PropertiesStore) []*model.SmartBlockSnapshotBase {
var opts []*model.SmartBlockSnapshotBase
func getRelationOptions(pr property.Object, rel string, relation *property.PropertiesStore) []*common.StateSnapshot {
var opts []*common.StateSnapshot
switch property := pr.(type) {
case *property.StatusItem:
options := statusItemOptions(property, rel, relation)
@ -380,8 +383,8 @@ func getRelationOptions(pr property.Object, rel string, relation *property.Prope
return opts
}
func peopleItemOptions(property *property.PeopleItem, rel string, relation *property.PropertiesStore) []*model.SmartBlockSnapshotBase {
peopleOptions := make([]*model.SmartBlockSnapshotBase, 0, len(property.People))
func peopleItemOptions(property *property.PeopleItem, rel string, relation *property.PropertiesStore) []*common.StateSnapshot {
peopleOptions := make([]*common.StateSnapshot, 0, len(property.People))
for _, po := range property.People {
if po.Name == "" {
continue
@ -400,8 +403,8 @@ func peopleItemOptions(property *property.PeopleItem, rel string, relation *prop
return peopleOptions
}
func multiselectItemOptions(property *property.MultiSelectItem, rel string, relation *property.PropertiesStore) []*model.SmartBlockSnapshotBase {
multiSelectOptions := make([]*model.SmartBlockSnapshotBase, 0, len(property.MultiSelect))
func multiselectItemOptions(property *property.MultiSelectItem, rel string, relation *property.PropertiesStore) []*common.StateSnapshot {
multiSelectOptions := make([]*common.StateSnapshot, 0, len(property.MultiSelect))
for _, so := range property.MultiSelect {
if so.Name == "" {
continue
@ -420,7 +423,7 @@ func multiselectItemOptions(property *property.MultiSelectItem, rel string, rela
return multiSelectOptions
}
func selectItemOptions(property *property.SelectItem, rel string, relation *property.PropertiesStore) *model.SmartBlockSnapshotBase {
func selectItemOptions(property *property.SelectItem, rel string, relation *property.PropertiesStore) *common.StateSnapshot {
if property.Select.Name == "" {
return nil
}
@ -432,11 +435,11 @@ func selectItemOptions(property *property.SelectItem, rel string, relation *prop
details, optSnapshot := provideRelationOptionSnapshot(property.Select.Name, property.Select.Color, rel)
optionID = details.GetStringOrDefault(bundle.RelationKeyId, "")
property.Select.ID = optionID
relation.WriteToRelationsOptionsMap(rel, []*model.SmartBlockSnapshotBase{optSnapshot})
relation.WriteToRelationsOptionsMap(rel, []*common.StateSnapshot{optSnapshot})
return optSnapshot
}
func statusItemOptions(property *property.StatusItem, rel string, relation *property.PropertiesStore) *model.SmartBlockSnapshotBase {
func statusItemOptions(property *property.StatusItem, rel string, relation *property.PropertiesStore) *common.StateSnapshot {
if property.Status == nil || property.Status.Name == "" {
return nil
}
@ -448,7 +451,7 @@ func statusItemOptions(property *property.StatusItem, rel string, relation *prop
details, optSnapshot := provideRelationOptionSnapshot(property.Status.Name, property.Status.Color, rel)
optionID = details.GetStringOrDefault(bundle.RelationKeyId, "")
property.Status.ID = optionID
relation.WriteToRelationsOptionsMap(rel, []*model.SmartBlockSnapshotBase{optSnapshot})
relation.WriteToRelationsOptionsMap(rel, []*common.StateSnapshot{optSnapshot})
return optSnapshot
}
@ -464,11 +467,11 @@ func isOptionAlreadyExist(optName, rel string, relation *property.PropertiesStor
return false, ""
}
func provideRelationOptionSnapshot(name, color, rel string) (*domain.Details, *model.SmartBlockSnapshotBase) {
func provideRelationOptionSnapshot(name, color, rel string) (*domain.Details, *common.StateSnapshot) {
id, details := getDetailsForRelationOption(name, rel)
details.Set(bundle.RelationKeyRelationOptionColor, pbtypes.String(api.NotionColorToAnytype[color]))
optSnapshot := &model.SmartBlockSnapshotBase{
Details: details.ToProto(),
optSnapshot := &common.StateSnapshot{
Details: details,
ObjectTypes: []string{bundle.TypeKeyRelationOption.String()},
Key: id,
}

80
core/block/import/notion/api/property/databaseproperty.go
View file

@ -3,10 +3,8 @@ package property
import (
"encoding/json"
"github.com/gogo/protobuf/types"
"github.com/anyproto/anytype-heart/core/domain"
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
"github.com/anyproto/anytype-heart/util/pbtypes"
)
// DatabaseProperties represent database properties (their structure is different from pages properties)
@ -129,8 +127,8 @@ func (t *DatabaseTitle) GetID() string {
return t.ID
}
func (t *DatabaseTitle) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.String("")
func (t *DatabaseTitle) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), "")
}
func (t *DatabaseTitle) GetFormat() model.RelationFormat {
@ -145,8 +143,8 @@ func (rt *DatabaseRichText) GetID() string {
return rt.ID
}
func (rt *DatabaseRichText) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.String("")
func (rt *DatabaseRichText) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), "")
}
func (rt *DatabaseRichText) GetFormat() model.RelationFormat {
@ -161,8 +159,8 @@ func (np *DatabaseNumber) GetID() string {
return np.ID
}
func (np *DatabaseNumber) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.Float64(0)
func (np *DatabaseNumber) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), 0)
}
func (np *DatabaseNumber) GetFormat() model.RelationFormat {
@ -177,8 +175,8 @@ func (sp *DatabaseSelect) GetID() string {
return sp.ID
}
func (sp *DatabaseSelect) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.StringList(nil)
func (sp *DatabaseSelect) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), []string{})
}
func (sp *DatabaseSelect) GetFormat() model.RelationFormat {
@ -193,8 +191,8 @@ func (ms *DatabaseMultiSelect) GetID() string {
return ms.ID
}
func (ms *DatabaseMultiSelect) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.StringList(nil)
func (ms *DatabaseMultiSelect) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), []string{})
}
func (ms *DatabaseMultiSelect) GetFormat() model.RelationFormat {
@ -209,8 +207,8 @@ func (dp *DatabaseDate) GetID() string {
return dp.ID
}
func (dp *DatabaseDate) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.Int64(0)
func (dp *DatabaseDate) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), 0)
}
func (dp *DatabaseDate) GetFormat() model.RelationFormat {
@ -225,8 +223,8 @@ func (rp *DatabaseRelation) GetID() string {
return rp.ID
}
func (rp *DatabaseRelation) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.String("")
func (rp *DatabaseRelation) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), "")
}
func (rp *DatabaseRelation) GetFormat() model.RelationFormat {
@ -241,8 +239,8 @@ func (p *DatabasePeople) GetID() string {
return p.ID
}
func (p *DatabasePeople) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.StringList(nil)
func (p *DatabasePeople) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), []string{})
}
func (p *DatabasePeople) GetFormat() model.RelationFormat {
@ -257,8 +255,8 @@ func (f *DatabaseFile) GetID() string {
return f.ID
}
func (f *DatabaseFile) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.String("")
func (f *DatabaseFile) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), "")
}
func (f *DatabaseFile) GetFormat() model.RelationFormat {
@ -273,8 +271,8 @@ func (c *DatabaseCheckbox) GetID() string {
return c.ID
}
func (c *DatabaseCheckbox) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.Bool(false)
func (c *DatabaseCheckbox) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), false)
}
func (c *DatabaseCheckbox) GetFormat() model.RelationFormat {
@ -289,8 +287,8 @@ func (u *DatabaseURL) GetID() string {
return u.ID
}
func (u *DatabaseURL) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.String("")
func (u *DatabaseURL) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), "")
}
func (u *DatabaseURL) GetFormat() model.RelationFormat {
@ -305,8 +303,8 @@ func (e *DatabaseEmail) GetID() string {
return e.ID
}
func (e *DatabaseEmail) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.String("")
func (e *DatabaseEmail) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), "")
}
func (e *DatabaseEmail) GetFormat() model.RelationFormat {
@ -329,8 +327,8 @@ func (ct *DatabaseCreatedTime) GetID() string {
return ct.ID
}
func (ct *DatabaseCreatedTime) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.Int64(0)
func (ct *DatabaseCreatedTime) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), 0)
}
func (ct *DatabaseCreatedTime) GetFormat() model.RelationFormat {
@ -345,8 +343,8 @@ func (cb *DatabaseCreatedBy) GetID() string {
return cb.ID
}
func (cb *DatabaseCreatedBy) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.String("")
func (cb *DatabaseCreatedBy) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), "")
}
func (cb *DatabaseCreatedBy) GetFormat() model.RelationFormat {
@ -361,8 +359,8 @@ func (le *DatabaseLastEditedTime) GetID() string {
return le.ID
}
func (le *DatabaseLastEditedTime) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.Int64(0)
func (le *DatabaseLastEditedTime) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), 0)
}
func (le *DatabaseLastEditedTime) GetFormat() model.RelationFormat {
@ -377,8 +375,8 @@ func (lb *DatabaseLastEditedBy) GetID() string {
return lb.ID
}
func (lb *DatabaseLastEditedBy) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.String("")
func (lb *DatabaseLastEditedBy) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), "")
}
func (lb *DatabaseLastEditedBy) GetFormat() model.RelationFormat {
@ -393,8 +391,8 @@ func (sp *DatabaseStatus) GetID() string {
return sp.ID
}
func (sp *DatabaseStatus) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.StringList(nil)
func (sp *DatabaseStatus) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), []string{})
}
func (sp *DatabaseStatus) GetFormat() model.RelationFormat {
@ -421,8 +419,8 @@ func (v *DatabaseVerification) GetID() string {
return v.ID
}
func (v *DatabaseVerification) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.StringList(nil)
func (v *DatabaseVerification) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), []string{})
}
type DatabaseUnique struct {
@ -437,6 +435,6 @@ func (u *DatabaseUnique) GetID() string {
return u.ID
}
func (u *DatabaseUnique) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.StringList(nil)
func (u *DatabaseUnique) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), []string{})
}

126
core/block/import/notion/api/property/propertyitem.go
View file

@ -7,14 +7,13 @@ import (
"strings"
"time"
"github.com/gogo/protobuf/types"
"go.uber.org/zap"
"github.com/anyproto/anytype-heart/core/block/import/common"
"github.com/anyproto/anytype-heart/core/block/import/notion/api"
"github.com/anyproto/anytype-heart/core/domain"
"github.com/anyproto/anytype-heart/pkg/lib/bundle"
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
"github.com/anyproto/anytype-heart/util/pbtypes"
)
type ConfigType string
@ -59,7 +58,7 @@ const (
)
type DetailSetter interface {
SetDetail(key string, details map[string]*types.Value)
SetDetail(key string, details *domain.Details)
}
type TitleItem struct {
@ -80,8 +79,8 @@ func (t *TitleItem) GetTitle() string {
return richText.String()
}
func (t *TitleItem) SetDetail(key string, details map[string]*types.Value) {
details[bundle.RelationKeyName.String()] = pbtypes.String(t.GetTitle())
func (t *TitleItem) SetDetail(key string, details *domain.Details) {
details.Set(bundle.RelationKeyName, t.GetTitle())
}
func (t *TitleItem) GetPropertyType() ConfigType {
@ -103,7 +102,7 @@ type RichTextItem struct {
RichText []*api.RichText `json:"rich_text"`
}
func (rt *RichTextItem) SetDetail(key string, details map[string]*types.Value) {
func (rt *RichTextItem) SetDetail(key string, details *domain.Details) {
var richText strings.Builder
for i, r := range rt.RichText {
richText.WriteString(r.PlainText)
@ -111,7 +110,7 @@ func (rt *RichTextItem) SetDetail(key string, details map[string]*types.Value) {
richText.WriteString("\n")
}
}
details[key] = pbtypes.String(richText.String())
details.Set(domain.RelationKey(key), richText.String())
}
func (rt *RichTextItem) GetPropertyType() ConfigType {
@ -133,9 +132,9 @@ type NumberItem struct {
Number *float64 `json:"number"`
}
func (np *NumberItem) SetDetail(key string, details map[string]*types.Value) {
func (np *NumberItem) SetDetail(key string, details *domain.Details) {
if np.Number != nil {
details[key] = pbtypes.Float64(*np.Number)
details.Set(domain.RelationKey(key), *np.Number)
}
}
@ -164,8 +163,8 @@ type SelectOption struct {
Color string `json:"color"`
}
func (sp *SelectItem) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.StringList([]string{sp.Select.ID})
func (sp *SelectItem) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), []string{sp.Select.ID})
}
func (sp *SelectItem) GetPropertyType() ConfigType {
@ -187,12 +186,12 @@ type MultiSelectItem struct {
MultiSelect []*SelectOption `json:"multi_select"`
}
func (ms *MultiSelectItem) SetDetail(key string, details map[string]*types.Value) {
func (ms *MultiSelectItem) SetDetail(key string, details *domain.Details) {
msList := make([]string, 0)
for _, so := range ms.MultiSelect {
msList = append(msList, so.ID)
}
details[key] = pbtypes.StringList(msList)
details.Set(domain.RelationKey(key), msList)
}
func (ms *MultiSelectItem) GetPropertyType() ConfigType {
@ -214,10 +213,10 @@ type DateItem struct {
Date *api.DateObject `json:"date"`
}
func (dp *DateItem) SetDetail(key string, details map[string]*types.Value) {
func (dp *DateItem) SetDetail(key string, details *domain.Details) {
if dp.Date != nil {
date := common.ConvertStringToTime(dp.Date.Start)
details[key] = pbtypes.Int64(date)
details.Set(domain.RelationKey(key), date)
}
}
@ -247,24 +246,24 @@ type FormulaItem struct {
Formula map[string]interface{} `json:"formula"`
}
func (f *FormulaItem) SetDetail(key string, details map[string]*types.Value) {
func (f *FormulaItem) SetDetail(key string, details *domain.Details) {
if f.Formula == nil {
return
}
switch f.Formula["type"].(string) {
case StringFormula:
if f.Formula["string"] != nil {
details[key] = pbtypes.String(f.Formula["string"].(string))
details.Set(domain.RelationKey(key), f.Formula["string"].(string))
}
case NumberFormula:
if f.Formula["number"] != nil {
stringNumber := strconv.FormatFloat(f.Formula["number"].(float64), 'f', 6, 64)
details[key] = pbtypes.String(stringNumber)
details.Set(domain.RelationKey(key), stringNumber)
}
case BooleanFormula:
if f.Formula["boolean"] != nil {
stringBool := strconv.FormatBool(f.Formula["boolean"].(bool))
details[key] = pbtypes.String(stringBool)
details.Set(domain.RelationKey(key), stringBool)
}
default:
return
@ -295,12 +294,12 @@ type Relation struct {
ID string `json:"id"`
}
func (rp *RelationItem) SetDetail(key string, details map[string]*types.Value) {
func (rp *RelationItem) SetDetail(key string, details *domain.Details) {
relation := make([]string, 0, len(rp.Relation))
for _, rel := range rp.Relation {
relation = append(relation, rel.ID)
}
details[key] = pbtypes.StringList(relation)
details.Set(domain.RelationKey(key), relation)
}
func (rp *RelationItem) GetPropertyType() ConfigType {
@ -322,12 +321,12 @@ type PeopleItem struct {
People []*api.User `json:"people"`
}
func (p *PeopleItem) SetDetail(key string, details map[string]*types.Value) {
func (p *PeopleItem) SetDetail(key string, details *domain.Details) {
peopleList := make([]string, 0, len(p.People))
for _, people := range p.People {
peopleList = append(peopleList, people.ID)
}
details[key] = pbtypes.StringList(peopleList)
details.Set(domain.RelationKey(key), peopleList)
}
func (p *PeopleItem) GetPropertyType() ConfigType {
@ -361,7 +360,7 @@ func (f *FileItem) GetFormat() model.RelationFormat {
return model.RelationFormat_file
}
func (f *FileItem) SetDetail(key string, details map[string]*types.Value) {
func (f *FileItem) SetDetail(key string, details *domain.Details) {
fileList := make([]string, len(f.File))
for i, fo := range f.File {
if fo.External.URL != "" {
@ -370,7 +369,7 @@ func (f *FileItem) SetDetail(key string, details map[string]*types.Value) {
fileList[i] = fo.File.URL
}
}
details[key] = pbtypes.StringList(fileList)
details.Set(domain.RelationKey(key), fileList)
}
type CheckboxItem struct {
@ -380,8 +379,8 @@ type CheckboxItem struct {
Checkbox bool `json:"checkbox"`
}
func (c *CheckboxItem) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.Bool(c.Checkbox)
func (c *CheckboxItem) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), c.Checkbox)
}
func (c *CheckboxItem) GetPropertyType() ConfigType {
@ -403,9 +402,9 @@ type URLItem struct {
URL *string `json:"url"`
}
func (u *URLItem) SetDetail(key string, details map[string]*types.Value) {
func (u *URLItem) SetDetail(key string, details *domain.Details) {
if u.URL != nil {
details[key] = pbtypes.String(*u.URL)
details.Set(domain.RelationKey(key), *u.URL)
}
}
@ -428,9 +427,9 @@ type EmailItem struct {
Email *string `json:"email"`
}
func (e *EmailItem) SetDetail(key string, details map[string]*types.Value) {
func (e *EmailItem) SetDetail(key string, details *domain.Details) {
if e.Email != nil {
details[key] = pbtypes.String(*e.Email)
details.Set(domain.RelationKey(key), *e.Email)
}
}
@ -453,9 +452,9 @@ type PhoneItem struct {
Phone *string `json:"phone_number"`
}
func (p *PhoneItem) SetDetail(key string, details map[string]*types.Value) {
func (p *PhoneItem) SetDetail(key string, details *domain.Details) {
if p.Phone != nil {
details[key] = pbtypes.String(*p.Phone)
details.Set(domain.RelationKey(key), *p.Phone)
}
}
@ -478,13 +477,13 @@ type CreatedTimeItem struct {
CreatedTime string `json:"created_time"`
}
func (ct *CreatedTimeItem) SetDetail(key string, details map[string]*types.Value) {
func (ct *CreatedTimeItem) SetDetail(key string, details *domain.Details) {
t, err := time.Parse(time.RFC3339, ct.CreatedTime)
if err != nil {
log.With(zap.String("method", "SetDetail")).Errorf("failed to parse time %v", err)
return
}
details[key] = pbtypes.Int64(t.Unix())
details.Set(domain.RelationKey(key), t.Unix())
}
func (ct *CreatedTimeItem) GetPropertyType() ConfigType {
@ -506,8 +505,8 @@ type CreatedByItem struct {
CreatedBy api.User `json:"created_by"`
}
func (cb *CreatedByItem) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.String(cb.CreatedBy.Name)
func (cb *CreatedByItem) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), cb.CreatedBy.Name)
}
func (cb *CreatedByItem) GetPropertyType() ConfigType {
@ -529,13 +528,13 @@ type LastEditedTimeItem struct {
LastEditedTime string `json:"last_edited_time"`
}
func (le *LastEditedTimeItem) SetDetail(key string, details map[string]*types.Value) {
func (le *LastEditedTimeItem) SetDetail(key string, details *domain.Details) {
t, err := time.Parse(time.RFC3339, le.LastEditedTime)
if err != nil {
log.With(zap.String("method", "SetDetail")).Errorf("failed to parse time %v", err)
return
}
details[key] = pbtypes.Int64(t.Unix())
details.Set(domain.RelationKey(key), t.Unix())
}
func (le *LastEditedTimeItem) GetPropertyType() ConfigType {
@ -557,8 +556,8 @@ type LastEditedByItem struct {
LastEditedBy api.User `json:"last_edited_by"`
}
func (lb *LastEditedByItem) SetDetail(key string, details map[string]*types.Value) {
details[key] = pbtypes.String(lb.LastEditedBy.Name)
func (lb *LastEditedByItem) SetDetail(key string, details *domain.Details) {
details.Set(domain.RelationKey(key), lb.LastEditedBy.Name)
}
func (lb *LastEditedByItem) GetPropertyType() ConfigType {
@ -585,11 +584,11 @@ type Status struct {
Color string `json:"color,omitempty"`
}
func (sp *StatusItem) SetDetail(key string, details map[string]*types.Value) {
func (sp *StatusItem) SetDetail(key string, details *domain.Details) {
if sp.Status != nil {
details[key] = pbtypes.StringList([]string{sp.Status.ID})
details.Set(domain.RelationKey(key), []string{sp.Status.ID})
} else {
details[key] = pbtypes.StringList([]string{})
details.Set(domain.RelationKey(key), []string{})
}
}
@ -627,10 +626,10 @@ type RollupObject struct {
Array propertyObjects `json:"array"`
}
func (r *RollupItem) SetDetail(key string, details map[string]*types.Value) {
func (r *RollupItem) SetDetail(key string, details *domain.Details) {
switch r.Rollup.Type {
case rollupNumber:
details[key] = pbtypes.Float64(r.Rollup.Number)
details.Set(domain.RelationKey(key), r.Rollup.Number)
case rollupDate:
di := DateItem{Date: r.Rollup.Date}
di.SetDetail(key, details)
@ -639,10 +638,10 @@ func (r *RollupItem) SetDetail(key string, details map[string]*types.Value) {
}
}
func (r *RollupItem) handleArrayType(key string, details map[string]*types.Value) {
func (r *RollupItem) handleArrayType(key string, details *domain.Details) {
result := make([]string, 0)
for _, pr := range r.Rollup.Array {
tempDetails := make(map[string]*types.Value, 0)
tempDetails := domain.NewDetails()
object := getPropertyObject(pr)
if object == nil {
continue
@ -651,24 +650,19 @@ func (r *RollupItem) handleArrayType(key string, details map[string]*types.Value
ds.SetDetail(key, tempDetails)
}
if _, ok := object.(*TitleItem); ok {
name := tempDetails[bundle.RelationKeyName.String()]
result = append(result, name.GetStringValue())
name := tempDetails.GetStringOrDefault(bundle.RelationKeyName, "")
result = append(result, name)
}
if value, ok := tempDetails[key]; ok && value != nil {
switch value.GetKind().(type) {
case *types.Value_StringValue:
res := value.GetStringValue()
result = append(result, res)
case *types.Value_BoolValue:
res := value.GetBoolValue()
result = append(result, strconv.FormatBool(res))
case *types.Value_NumberValue:
res := value.GetNumberValue()
result = append(result, strconv.FormatFloat(res, 'f', 0, 64))
}
if v, ok := tempDetails.GetString(domain.RelationKey(key)); ok {
result = append(result, v)
} else if v, ok := tempDetails.GetBool(domain.RelationKey(key)); ok {
result = append(result, strconv.FormatBool(v))
} else if v, ok := tempDetails.GetFloat(domain.RelationKey(key)); ok {
result = append(result, strconv.FormatFloat(v, 'f', 0, 64))
}
}
details[key] = pbtypes.StringList(result)
details.Set(domain.RelationKey(key), result)
}
func (r *RollupItem) GetPropertyType() ConfigType {
@ -697,7 +691,7 @@ type VerificationItem struct {
Verification struct{} `json:"verification"`
}
func (v VerificationItem) SetDetail(_ string, _ map[string]*types.Value) {}
func (v VerificationItem) SetDetail(_ string, _ *domain.Details) {}
func (v VerificationItem) GetPropertyType() ConfigType {
return PropertyConfigVerification
@ -722,12 +716,12 @@ type UniqueID struct {
Prefix string `json:"prefix"`
}
func (u UniqueIDItem) SetDetail(key string, details map[string]*types.Value) {
func (u UniqueIDItem) SetDetail(key string, details *domain.Details) {
id := strconv.FormatInt(u.UniqueID.Number, 10)
if u.UniqueID.Prefix != "" {
id = u.UniqueID.Prefix + "-" + id
}
details[key] = pbtypes.String(id)
details.Set(domain.RelationKey(key), id)
}
func (u UniqueIDItem) GetPropertyType() ConfigType {

16
core/block/import/notion/api/property/relationsmaps.go
View file

@ -1,30 +1,32 @@
package property
import "github.com/anyproto/anytype-heart/pkg/lib/pb/model"
import (
"github.com/anyproto/anytype-heart/core/block/import/common"
)
type PropertiesStore struct {
PropertyIdsToSnapshots map[string]*model.SmartBlockSnapshotBase
RelationsIdsToOptions map[string][]*model.SmartBlockSnapshotBase
PropertyIdsToSnapshots map[string]*common.StateSnapshot
RelationsIdsToOptions map[string][]*common.StateSnapshot
}
func (m *PropertiesStore) ReadRelationsMap(key string) *model.SmartBlockSnapshotBase {
func (m *PropertiesStore) ReadRelationsMap(key string) *common.StateSnapshot {
if snapshot, ok := m.PropertyIdsToSnapshots[key]; ok {
return snapshot
}
return nil
}
func (m *PropertiesStore) WriteToRelationsMap(key string, relation *model.SmartBlockSnapshotBase) {
func (m *PropertiesStore) WriteToRelationsMap(key string, relation *common.StateSnapshot) {
m.PropertyIdsToSnapshots[key] = relation
}
func (m *PropertiesStore) ReadRelationsOptionsMap(key string) []*model.SmartBlockSnapshotBase {
func (m *PropertiesStore) ReadRelationsOptionsMap(key string) []*common.StateSnapshot {
if snapshot, ok := m.RelationsIdsToOptions[key]; ok {
return snapshot
}
return nil
}
func (m *PropertiesStore) WriteToRelationsOptionsMap(key string, relationOptions []*model.SmartBlockSnapshotBase) {
func (m *PropertiesStore) WriteToRelationsOptionsMap(key string, relationOptions []*common.StateSnapshot) {
m.RelationsIdsToOptions[key] = append(m.RelationsIdsToOptions[key], relationOptions...)
}

125
core/block/import/pb/converter.go
View file

@ -10,7 +10,6 @@ import (
"time"
"github.com/gogo/protobuf/jsonpb"
"github.com/gogo/protobuf/types"
"github.com/google/uuid"
"github.com/samber/lo"
@ -26,6 +25,7 @@ import (
"github.com/anyproto/anytype-heart/pkg/lib/bundle"
"github.com/anyproto/anytype-heart/pkg/lib/core"
"github.com/anyproto/anytype-heart/pkg/lib/core/smartblock"
coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock"
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
"github.com/anyproto/anytype-heart/util/constant"
"github.com/anyproto/anytype-heart/util/pbtypes"
@ -241,10 +241,10 @@ func (p *Pb) getSnapshotsFromProvidedFiles(
if p.shouldImportSnapshot(snapshot, needToImportWidgets, importType) {
allSnapshots = append(allSnapshots, snapshot)
}
if snapshot.SbType == smartblock.SmartBlockTypeWidget {
if snapshot.Snapshot.SbType == smartblock.SmartBlockTypeWidget {
widgetSnapshot = snapshot
}
if snapshot.SbType == smartblock.SmartBlockTypeWorkspace {
if snapshot.Snapshot.SbType == smartblock.SmartBlockTypeWorkspace {
workspaceSnapshot = snapshot
}
}
@ -282,13 +282,12 @@ func (p *Pb) makeSnapshot(name, profileID, path string,
p.injectImportDetails(snapshot)
return &common.Snapshot{
Id: id,
SbType: smartblock.SmartBlockType(snapshot.SbType),
FileName: name,
Snapshot: snapshot.Snapshot,
Snapshot: snapshot,
}, nil
}
func (p *Pb) getSnapshotFromFile(rd io.ReadCloser, name string) (*pb.SnapshotWithType, error) {
func (p *Pb) getSnapshotFromFile(rd io.ReadCloser, name string) (*common.SnapshotModel, error) {
defer rd.Close()
if filepath.Ext(name) == ".json" {
snapshot := &pb.SnapshotWithType{}
@ -296,7 +295,7 @@ func (p *Pb) getSnapshotFromFile(rd io.ReadCloser, name string) (*pb.SnapshotWit
if uErr := um.Unmarshal(rd, snapshot); uErr != nil {
return nil, ErrWrongFormat
}
return snapshot, nil
return common.NewSnapshotModelFromProto(snapshot), nil
}
if filepath.Ext(name) == ".pb" {
snapshot := &pb.SnapshotWithType{}
@ -307,12 +306,12 @@ func (p *Pb) getSnapshotFromFile(rd io.ReadCloser, name string) (*pb.SnapshotWit
if err = snapshot.Unmarshal(data); err != nil {
return nil, ErrWrongFormat
}
return snapshot, nil
return common.NewSnapshotModelFromProto(snapshot), nil
}
return nil, ErrNotAnyBlockExtension
}
func (p *Pb) normalizeSnapshot(snapshot *common.Snapshot,
func (p *Pb) normalizeSnapshot(snapshot *common.SnapshotModel,
id, profileID, path string,
isMigration bool,
pbFiles source.Source) (string, error) {
@ -321,30 +320,30 @@ func (p *Pb) normalizeSnapshot(snapshot *common.Snapshot,
if int32(snapshot.SbType) == 96 { // fallback for objectType smartblocktype
newSbType = model.SmartBlockType_SubObject
}
snapshot.SbType = newSbType
snapshot.SbType = coresb.SmartBlockType(newSbType)
}
if snapshot.SbType == model.SmartBlockType_SubObject {
details := snapshot.Snapshot.Data.Details
originalId := snapshot.Snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeyId, "")
if snapshot.SbType == coresb.SmartBlockTypeSubObject {
details := snapshot.Data.Details
originalId := snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeyId, "")
var sourceObjectId string
// migrate old sub objects into real objects
if snapshot.Snapshot.Data.ObjectTypes[0] == bundle.TypeKeyObjectType.URL() {
snapshot.SbType = model.SmartBlockType_STType
if snapshot.Data.ObjectTypes[0] == bundle.TypeKeyObjectType.URL() {
snapshot.SbType = coresb.SmartBlockTypeObjectType
typeKey, err := bundle.TypeKeyFromUrl(originalId)
if err == nil {
sourceObjectId = typeKey.BundledURL()
}
} else if snapshot.Snapshot.Data.ObjectTypes[0] == bundle.TypeKeyRelation.URL() {
snapshot.SbType = model.SmartBlockType_STRelation
} else if snapshot.Data.ObjectTypes[0] == bundle.TypeKeyRelation.URL() {
snapshot.SbType = coresb.SmartBlockTypeRelation
relationKey, err := bundle.RelationKeyFromID(originalId)
if err == nil {
sourceObjectId = relationKey.BundledURL()
}
} else if snapshot.Snapshot.Data.ObjectTypes[0] == bundle.TypeKeyRelationOption.URL() {
snapshot.SbType = model.SmartBlockType_STRelationOption
} else if snapshot.Data.ObjectTypes[0] == bundle.TypeKeyRelationOption.URL() {
snapshot.SbType = coresb.SmartBlockTypeRelationOption
} else {
return "", fmt.Errorf("unknown sub object type %s", snapshot.Snapshot.Data.ObjectTypes[0])
return "", fmt.Errorf("unknown sub object type %s", snapshot.Data.ObjectTypes[0])
}
if sourceObjectId != "" {
if details.GetStringOrDefault(bundle.RelationKeySourceObject, "") == "" {
@ -354,7 +353,7 @@ func (p *Pb) normalizeSnapshot(snapshot *common.Snapshot,
id = originalId
}
if snapshot.SbType == model.SmartBlockType_ProfilePage {
if snapshot.SbType == coresb.SmartBlockTypeProfilePage {
var err error
id, err = p.getIDForUserProfile(snapshot, profileID, id, isMigration)
if err != nil {
@ -362,16 +361,16 @@ func (p *Pb) normalizeSnapshot(snapshot *common.Snapshot,
}
p.setProfileIconOption(snapshot, profileID)
}
if snapshot.SbType == model.SmartBlockType_Page {
if snapshot.SbType == coresb.SmartBlockTypePage {
p.cleanupEmptyBlock(snapshot)
}
if snapshot.SbType == model.SmartBlockType_File {
if snapshot.SbType == coresb.SmartBlockTypeFile {
err := p.normalizeFilePath(snapshot, pbFiles, path)
if err != nil {
return "", fmt.Errorf("failed to update file path in file snapshot %w", err)
}
}
if snapshot.SbType == model.SmartBlockType_FileObject {
if snapshot.SbType == coresb.SmartBlockTypeFileObject {
err := p.normalizeFilePath(snapshot, pbFiles, path)
if err != nil {
return "", fmt.Errorf("failed to update file path in file snapshot %w", err)
@ -380,33 +379,33 @@ func (p *Pb) normalizeSnapshot(snapshot *common.Snapshot,
return id, nil
}
func (p *Pb) normalizeFilePath(snapshot *pb.SnapshotWithType, pbFiles source.Source, path string) error {
filePath := snapshot.Snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeySource, "")
func (p *Pb) normalizeFilePath(snapshot *common.SnapshotModel, pbFiles source.Source, path string) error {
filePath := snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeySource, "")
fileName, _, err := common.ProvideFileName(filePath, pbFiles, path, p.tempDirProvider)
if err != nil {
return err
}
if snapshot.Snapshot.Data.Details == nil || snapshot.Snapshot.Data.Details.Fields == nil {
snapshot.Snapshot.Data.Details.Fields = map[string]*types.Value{}
if snapshot.Data.Details == nil {
snapshot.Data.Details = domain.NewDetails()
}
snapshot.Snapshot.Data.Details.Set(bundle.RelationKeySource, pbtypes.String(fileName))
snapshot.Data.Details.Set(bundle.RelationKeySource, pbtypes.String(fileName))
return nil
}
func (p *Pb) getIDForUserProfile(mo *pb.SnapshotWithType, profileID string, id string, isMigration bool) (string, error) {
objectID := mo.Snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeyId, "")
func (p *Pb) getIDForUserProfile(snapshot *common.SnapshotModel, profileID string, id string, isMigration bool) (string, error) {
objectID := snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeyId, "")
if objectID == profileID && isMigration {
return p.accountService.ProfileObjectId()
}
return id, nil
}
func (p *Pb) setProfileIconOption(mo *pb.SnapshotWithType, profileID string) {
objectID := mo.Snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeyId, "")
func (p *Pb) setProfileIconOption(snapshot *common.SnapshotModel, profileID string) {
objectID := snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeyId, "")
if objectID != profileID {
return
}
mo.Snapshot.Data.Details.Set(bundle.RelationKeyIconOption, pbtypes.Int64(p.getIconOption()))
snapshot.Data.Details.Set(bundle.RelationKeyIconOption, pbtypes.Int64(p.getIconOption()))
}
func (p *Pb) getIconOption() int64 {
@ -417,12 +416,12 @@ func (p *Pb) getIconOption() int64 {
}
// cleanupEmptyBlockMigration is fixing existing pages, imported from Notion
func (p *Pb) cleanupEmptyBlock(snapshot *pb.SnapshotWithType) {
func (p *Pb) cleanupEmptyBlock(snapshot *common.SnapshotModel) {
var (
emptyBlock *model.Block
)
for _, block := range snapshot.Snapshot.Data.Blocks {
for _, block := range snapshot.Data.Blocks {
if block.Content == nil {
emptyBlock = block
} else if block.GetSmartblock() != nil {
@ -434,31 +433,31 @@ func (p *Pb) cleanupEmptyBlock(snapshot *pb.SnapshotWithType) {
}
}
func (p *Pb) injectImportDetails(sn *pb.SnapshotWithType) {
if sn.Snapshot.Data.Details == nil || sn.Snapshot.Data.Details.Fields == nil {
sn.Snapshot.Data.Details = &types.Struct{Fields: map[string]*types.Value{}}
func (p *Pb) injectImportDetails(snapshot *common.SnapshotModel) {
if snapshot.Data.Details == nil {
snapshot.Data.Details = domain.NewDetails()
}
if id := sn.Snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeyId, ""); id != "" {
sn.Snapshot.Data.Details.Set(bundle.RelationKeyOldAnytypeID, pbtypes.String(id))
if id := snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeyId, ""); id != "" {
snapshot.Data.Details.Set(bundle.RelationKeyOldAnytypeID, pbtypes.String(id))
}
p.setSourceFilePath(sn)
createdDate := sn.Snapshot.Data.Details.GetInt64OrDefault(bundle.RelationKeyCreatedDate, 0)
p.setSourceFilePath(snapshot)
createdDate := snapshot.Data.Details.GetInt64OrDefault(bundle.RelationKeyCreatedDate, 0)
if createdDate == 0 {
sn.Snapshot.Data.Details.Set(bundle.RelationKeyCreatedDate, pbtypes.Int64(time.Now().Unix()))
snapshot.Data.Details.Set(bundle.RelationKeyCreatedDate, pbtypes.Int64(time.Now().Unix()))
}
}
func (p *Pb) setSourceFilePath(sn *pb.SnapshotWithType) {
spaceId := sn.Snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeySpaceId, "")
id := sn.Snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeyId, "")
func (p *Pb) setSourceFilePath(snapshot *common.SnapshotModel) {
spaceId := snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeySpaceId, "")
id := snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeyId, "")
sourceFilePath := filepath.Join(spaceId, id)
sn.Snapshot.Data.Details.Set(bundle.RelationKeySourceFilePath, pbtypes.String(sourceFilePath))
snapshot.Data.Details.Set(bundle.RelationKeySourceFilePath, pbtypes.String(sourceFilePath))
}
func (p *Pb) shouldImportSnapshot(snapshot *common.Snapshot, needToImportWidgets bool, importType pb.RpcObjectImportRequestPbParamsType) bool {
return (snapshot.SbType == smartblock.SmartBlockTypeWorkspace && importType == pb.RpcObjectImportRequestPbParams_SPACE) ||
(snapshot.SbType != smartblock.SmartBlockTypeWidget && snapshot.SbType != smartblock.SmartBlockTypeWorkspace) ||
(snapshot.SbType == smartblock.SmartBlockTypeWidget && (needToImportWidgets || importType == pb.RpcObjectImportRequestPbParams_EXPERIENCE)) // we import widget in case of experience import
return (snapshot.Snapshot.SbType == smartblock.SmartBlockTypeWorkspace && importType == pb.RpcObjectImportRequestPbParams_SPACE) ||
(snapshot.Snapshot.SbType != smartblock.SmartBlockTypeWidget && snapshot.Snapshot.SbType != smartblock.SmartBlockTypeWorkspace) ||
(snapshot.Snapshot.SbType == smartblock.SmartBlockTypeWidget && (needToImportWidgets || importType == pb.RpcObjectImportRequestPbParams_EXPERIENCE)) // we import widget in case of experience import
}
func (p *Pb) updateLinksToObjects(snapshots []*common.Snapshot, allErrors *common.ConvertError, pathCount int) map[string]string {
@ -467,12 +466,12 @@ func (p *Pb) updateLinksToObjects(snapshots []*common.Snapshot, allErrors *commo
for _, snapshot := range snapshots {
id := snapshot.Snapshot.Data.Details.GetStringOrDefault(bundle.RelationKeyId, "")
oldToNewID[id] = snapshot.Id
fileIDs = append(fileIDs, lo.Map(snapshot.Snapshot.GetFileKeys(), func(item *pb.ChangeFileKeys, index int) string {
fileIDs = append(fileIDs, lo.Map(snapshot.Snapshot.FileKeys, func(item *pb.ChangeFileKeys, index int) string {
return item.Hash
})...)
}
for _, snapshot := range snapshots {
st := state.NewDocFromSnapshot("", snapshot.Snapshot, state.WithUniqueKeyMigration(snapshot.SbType))
st := state.NewDocFromSnapshot("", snapshot.Snapshot.ToProto(), state.WithUniqueKeyMigration(snapshot.Snapshot.SbType))
err := common.UpdateLinksToObjects(st.(*state.State), oldToNewID, fileIDs)
if err != nil {
allErrors.Add(err)
@ -491,24 +490,24 @@ func (p *Pb) updateLinksToObjects(snapshots []*common.Snapshot, allErrors *commo
}
func (p *Pb) updateSnapshot(snapshot *common.Snapshot, st *state.State) {
snapshot.Snapshot.Data.Details = pbtypes.StructMerge(snapshot.Snapshot.Data.Details, st.CombinedDetails(), false)
snapshot.Snapshot.Data.Details = snapshot.Snapshot.Data.Details.Merge(st.CombinedDetails())
snapshot.Snapshot.Data.Blocks = st.Blocks()
snapshot.Snapshot.Data.ObjectTypes = domain.MarshalTypeKeys(st.ObjectTypeKeys())
snapshot.Snapshot.Data.Collections = st.Store()
}
func (p *Pb) updateDetails(snapshots []*common.Snapshot) {
removeKeys := slice.Filter(bundle.LocalAndDerivedRelationKeys, func(key string) bool {
removeKeys := slice.Filter(bundle.LocalAndDerivedRelationKeys, func(key domain.RelationKey) bool {
// preserve some keys we have special cases for
return key != bundle.RelationKeyIsFavorite.String() &&
key != bundle.RelationKeyIsArchived.String() &&
key != bundle.RelationKeyCreatedDate.String() &&
key != bundle.RelationKeyLastModifiedDate.String() &&
key != bundle.RelationKeyId.String()
return key != bundle.RelationKeyIsFavorite &&
key != bundle.RelationKeyIsArchived &&
key != bundle.RelationKeyCreatedDate &&
key != bundle.RelationKeyLastModifiedDate &&
key != bundle.RelationKeyId
})
for _, snapshot := range snapshots {
details := pbtypes.StructCutKeys(snapshot.Snapshot.Data.Details, removeKeys)
details := snapshot.Snapshot.Data.Details.CopyWithoutKeys(removeKeys...)
snapshot.Snapshot.Data.Details = details
}
}
@ -525,6 +524,6 @@ func (p *Pb) updateObjectsIDsInCollection(st *state.State, newToOldIDs map[strin
}
}
func (p *Pb) isSnapshotValid(snapshot *pb.SnapshotWithType) bool {
return !(snapshot == nil || snapshot.Snapshot == nil || snapshot.Snapshot.Data == nil)
func (p *Pb) isSnapshotValid(snapshot *common.SnapshotModel) bool {
return !(snapshot == nil || snapshot.Data == nil)
}

5
core/block/import/pb/gallery.go
View file

@ -14,7 +14,6 @@ import (
"github.com/anyproto/anytype-heart/pkg/lib/core/smartblock"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/addr"
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
"github.com/anyproto/anytype-heart/util/pbtypes"
)
const widgetCollectionPattern = "'s Widgets"
@ -92,7 +91,7 @@ func (g *GalleryImport) getWidgetsCollection(collectionName string,
}
func (g *GalleryImport) getObjectsFromWidgets(widgetSnapshot *common.Snapshot) []string {
widgetState := state.NewDocFromSnapshot("", widgetSnapshot.Snapshot).(*state.State)
widgetState := state.NewDocFromSnapshot("", widgetSnapshot.Snapshot.ToProto()).(*state.State)
var objectsInWidget []string
err := widgetState.Iterate(func(b simple.Block) (isContinue bool) {
if link := b.Model().GetLink(); link != nil && link.TargetBlockId != "" {
@ -140,7 +139,7 @@ func (g *GalleryImport) addCollectionWidget(widgetSnapshot *common.Snapshot, col
func (g *GalleryImport) getObjectsIDs(snapshots []*common.Snapshot) []string {
var resultIDs []string
for _, snapshot := range snapshots {
if snapshot.SbType == smartblock.SmartBlockTypePage {
if snapshot.Snapshot.SbType == smartblock.SmartBlockTypePage {
resultIDs = append(resultIDs, snapshot.Id)
}
}

8
core/block/import/pb/space.go
View file

@ -64,13 +64,13 @@ func (s *SpaceImport) ProvideCollection(snapshots []*common.Snapshot,
}
func (s *SpaceImport) objectShouldBeSkipped(item *common.Snapshot) bool {
return item.SbType == smartblock.SmartBlockTypeSubObject || item.SbType == smartblock.SmartBlockTypeTemplate ||
item.SbType == smartblock.SmartBlockTypeRelation || item.SbType == smartblock.SmartBlockTypeObjectType ||
item.SbType == smartblock.SmartBlockTypeRelationOption
return item.Snapshot.SbType == smartblock.SmartBlockTypeSubObject || item.Snapshot.SbType == smartblock.SmartBlockTypeTemplate ||
item.Snapshot.SbType == smartblock.SmartBlockTypeRelation || item.Snapshot.SbType == smartblock.SmartBlockTypeObjectType ||
item.Snapshot.SbType == smartblock.SmartBlockTypeRelationOption
}
func (s *SpaceImport) getObjectsFromWidget(widgetSnapshot *common.Snapshot, oldToNewID map[string]string) (widget.ImportWidgetFlags, []string) {
widgetState := state.NewDocFromSnapshot("", widgetSnapshot.Snapshot).(*state.State)
widgetState := state.NewDocFromSnapshot("", widgetSnapshot.Snapshot.ToProto()).(*state.State)
var (
objectsInWidget []string
objectTypesToImport widget.ImportWidgetFlags

8
core/block/import/txt/converter.go
View file

@ -154,7 +154,7 @@ func (t *TXT) getBlocksForSnapshot(rc io.ReadCloser) ([]*model.Block, error) {
}
func (t *TXT) getSnapshot(blocks []*model.Block, p string) (*common.Snapshot, string) {
sn := &common.SnapshotModelData{
sn := &common.StateSnapshot{
Blocks: blocks,
Details: common.GetCommonDetails(p, "", "", model.ObjectType_basic),
ObjectTypes: []string{bundle.TypeKeyPage.String()},
@ -163,8 +163,10 @@ func (t *TXT) getSnapshot(blocks []*model.Block, p string) (*common.Snapshot, st
snapshot := &common.Snapshot{
Id: uuid.New().String(),
FileName: p,
Snapshot: &common.SnapshotModel{Data: sn},
SbType: smartblock.SmartBlockTypePage,
Snapshot: &common.SnapshotModel{
SbType: smartblock.SmartBlockTypePage,
Data: sn,
},
}
return snapshot, snapshot.Id
}

4
core/block/import/types.go
View file

@ -4,12 +4,12 @@ import (
"context"
"github.com/anyproto/any-sync/app"
"github.com/gogo/protobuf/types"
_ "github.com/anyproto/anytype-heart/core/block/import/markdown"
_ "github.com/anyproto/anytype-heart/core/block/import/pb"
_ "github.com/anyproto/anytype-heart/core/block/import/web"
"github.com/anyproto/anytype-heart/core/block/process"
"github.com/anyproto/anytype-heart/core/domain"
"github.com/anyproto/anytype-heart/core/domain/objectorigin"
"github.com/anyproto/anytype-heart/pb"
)
@ -31,7 +31,7 @@ type Importer interface {
) *ImportResponse
ListImports(req *pb.RpcObjectImportListRequest) ([]*pb.RpcObjectImportListImportResponse, error)
ImportWeb(ctx context.Context, req *pb.RpcObjectImportRequest) (string, *types.Struct, error)
ImportWeb(ctx context.Context, req *pb.RpcObjectImportRequest) (string, *domain.Details, error)
// nolint: lll
ValidateNotionToken(ctx context.Context, req *pb.RpcObjectImportNotionValidateTokenRequest) (pb.RpcObjectImportNotionValidateTokenResponseErrorCode, error)
}

2
core/block/import/web/parsers/types.go
View file

@ -13,6 +13,6 @@ func RegisterFunc(p RegisterParser) {
}
type Parser interface {
ParseUrl(url string) (*common.SnapshotModelData, error)
ParseUrl(url string) (*common.StateSnapshot, error)
MatchUrl(url string) bool
}

28
core/block/import/web/parsers/wiki.go
View file

@ -7,12 +7,11 @@ import (
"path/filepath"
"regexp"
"github.com/gogo/protobuf/types"
"github.com/anyproto/anytype-heart/core/block/import/common"
"github.com/anyproto/anytype-heart/core/block/import/markdown/anymark"
"github.com/anyproto/anytype-heart/core/domain"
"github.com/anyproto/anytype-heart/pkg/lib/bundle"
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
"github.com/anyproto/anytype-heart/util/pbtypes"
)
const wikiRegexp = "\\/wiki\\/([\\w%]+)"
@ -23,7 +22,7 @@ func New() Parser {
return new(DumbWikiParser)
}
func (w *DumbWikiParser) ParseUrl(url string) (*model.SmartBlockSnapshotBase, error) {
func (w *DumbWikiParser) ParseUrl(url string) (*common.StateSnapshot, error) {
resp, err := http.Get(url)
if err != nil {
return nil, fmt.Errorf("WikiParser: ParseUrl: %w", err)
@ -39,8 +38,8 @@ func (w *DumbWikiParser) ParseUrl(url string) (*model.SmartBlockSnapshotBase, er
return nil, fmt.Errorf("WikiParser: ParseUrl: %w", err)
}
snapshots := &model.SmartBlockSnapshotBase{}
snapshots.Blocks = blocks
snapshot := &common.StateSnapshot{}
snapshot.Blocks = blocks
var name string
for _, b := range blocks {
if text := b.GetText(); text != nil && text.Style == model.BlockContentText_Header1 {
@ -50,16 +49,13 @@ func (w *DumbWikiParser) ParseUrl(url string) (*model.SmartBlockSnapshotBase, er
if name == "" {
name = filepath.Base(url)
}
details := &types.Struct{
Fields: map[string]*types.Value{
bundle.RelationKeyName.String(): pbtypes.String(name),
bundle.RelationKeySource.String(): pbtypes.String(url),
bundle.RelationKeyType.String(): pbtypes.String(bundle.TypeKeyBookmark.String()),
bundle.RelationKeySource.String(): pbtypes.String(url),
},
}
snapshots.Details = details
return snapshots, nil
details := domain.NewDetailsFromMap(map[domain.RelationKey]any{
bundle.RelationKeyName: name,
bundle.RelationKeySource: url,
bundle.RelationKeyType: bundle.TypeKeyBookmark.String(),
})
snapshot.Details = details
return snapshot, nil
}
func (w *DumbWikiParser) MatchUrl(url string) bool {

46
core/block/service.go
View file

@ -10,7 +10,6 @@ import (
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/globalsign/mgo/bson"
"github.com/gogo/protobuf/types"
"github.com/hashicorp/go-multierror"
"github.com/samber/lo"
"go.uber.org/zap"
@ -123,7 +122,7 @@ type builtinObjects interface {
}
type templateService interface {
CreateTemplateStateWithDetails(templateId string, details *types.Struct) (*state.State, error)
CreateTemplateStateWithDetails(templateId string, details *domain.Details) (*state.State, error)
}
type openedObjects struct {
@ -295,7 +294,7 @@ func (s *Service) SpaceInstallBundledObject(
ctx context.Context,
spaceId string,
sourceObjectId string,
) (id string, object *types.Struct, err error) {
) (id string, object *domain.Details, err error) {
spc, err := s.spaceService.Get(ctx, spaceId)
if err != nil {
return "", nil, fmt.Errorf("get space: %w", err)
@ -315,7 +314,7 @@ func (s *Service) SpaceInstallBundledObjects(
ctx context.Context,
spaceId string,
sourceObjectIds []string,
) (ids []string, objects []*types.Struct, err error) {
) (ids []string, objects []*domain.Details, err error) {
spc, err := s.spaceService.Get(ctx, spaceId)
if err != nil {
return nil, nil, fmt.Errorf("get space: %w", err)
@ -729,13 +728,11 @@ func (s *Service) ObjectToBookmark(ctx context.Context, id string, url string) (
if err != nil {
return "", fmt.Errorf("resolve spaceID: %w", err)
}
details := domain.NewDetails()
details.Set(bundle.RelationKeySource, url)
req := objectcreator.CreateObjectRequest{
ObjectTypeKey: bundle.TypeKeyBookmark,
Details: &types.Struct{
Fields: map[string]*types.Value{
bundle.RelationKeySource.String(): pbtypes.String(url),
},
},
Details: details,
}
objectId, _, err = s.objectCreator.CreateObject(ctx, spaceID, req)
if err != nil {
@ -763,7 +760,7 @@ func (s *Service) ObjectToBookmark(ctx context.Context, id string, url string) (
}
func (s *Service) CreateObjectFromUrl(ctx context.Context, req *pb.RpcObjectCreateFromUrlRequest,
) (id string, objectDetails *types.Struct, err error) {
) (id string, objectDetails *domain.Details, err error) {
url, err := uri.NormalizeURI(req.Url)
if err != nil {
return "", nil, err
@ -772,10 +769,11 @@ func (s *Service) CreateObjectFromUrl(ctx context.Context, req *pb.RpcObjectCrea
if err != nil {
return "", nil, err
}
s.enrichDetailsWithOrigin(req.Details, model.ObjectOrigin_webclipper)
details := domain.NewDetailsFromProto(req.Details)
s.enrichDetailsWithOrigin(details, model.ObjectOrigin_webclipper)
createReq := objectcreator.CreateObjectRequest{
ObjectTypeKey: objectTypeKey,
Details: req.Details,
Details: details,
}
id, objectDetails, err = s.objectCreator.CreateObject(ctx, req.SpaceId, createReq)
if err != nil {
@ -784,7 +782,7 @@ func (s *Service) CreateObjectFromUrl(ctx context.Context, req *pb.RpcObjectCrea
res := s.bookmark.FetchBookmarkContent(req.SpaceId, url, req.AddPageContent)
content := res()
shouldUpdateDetails := s.updateBookmarkContentWithUserDetails(req.Details, objectDetails, content)
shouldUpdateDetails := s.updateBookmarkContentWithUserDetails(details, objectDetails, content)
if shouldUpdateDetails {
err = s.bookmark.UpdateObject(id, content)
if err != nil {
@ -820,19 +818,19 @@ func (s *Service) pasteBlocks(id string, content *bookmark.ObjectContent) error
return nil
}
func (s *Service) updateBookmarkContentWithUserDetails(userDetails, objectDetails *types.Struct, content *bookmark.ObjectContent) bool {
func (s *Service) updateBookmarkContentWithUserDetails(userDetails, objectDetails *domain.Details, content *bookmark.ObjectContent) bool {
shouldUpdate := false
bookmarkRelationToValue := map[string]*string{
bundle.RelationKeyName.String(): &content.BookmarkContent.Title,
bundle.RelationKeyDescription.String(): &content.BookmarkContent.Description,
bundle.RelationKeySource.String(): &content.BookmarkContent.Url,
bundle.RelationKeyPicture.String(): &content.BookmarkContent.ImageHash,
bundle.RelationKeyIconImage.String(): &content.BookmarkContent.FaviconHash,
bookmarkRelationToValue := map[domain.RelationKey]*string{
bundle.RelationKeyName: &content.BookmarkContent.Title,
bundle.RelationKeyDescription: &content.BookmarkContent.Description,
bundle.RelationKeySource: &content.BookmarkContent.Url,
bundle.RelationKeyPicture: &content.BookmarkContent.ImageHash,
bundle.RelationKeyIconImage: &content.BookmarkContent.FaviconHash,
}
for relation, valueFromBookmark := range bookmarkRelationToValue {
// Don't change details of the object, if they are provided by client in request
if userValue := pbtypes.GetString(userDetails, relation); userValue != "" {
if userValue := userDetails.GetStringOrDefault(relation, ""); userValue != "" {
*valueFromBookmark = userValue
} else {
// if detail wasn't provided in request, we get it from bookmark and set it later in bookmark.UpdateObject
@ -858,9 +856,9 @@ func (s *Service) GetLogFields() []zap.Field {
return fields
}
func (s *Service) enrichDetailsWithOrigin(details *types.Struct, origin model.ObjectOrigin) {
if details == nil || details.Fields == nil {
details = &types.Struct{Fields: map[string]*types.Value{}}
func (s *Service) enrichDetailsWithOrigin(details *domain.Details, origin model.ObjectOrigin) {
if details == nil {
details = domain.NewDetails()
}
details.Set(bundle.RelationKeyOrigin, pbtypes.Int64(int64(origin)))
}

17
docs/proto.md
View file

@ -1643,6 +1643,7 @@
- [pb/protos/snapshot.proto](#pb_protos_snapshot-proto)
- [Profile](#anytype-Profile)
- [SnapshotWithType](#anytype-SnapshotWithType)
- [pkg/lib/pb/model/protos/localstore.proto](#pkg_lib_pb_model_protos_localstore-proto)
- [ObjectDetails](#anytype-model-ObjectDetails)
@ -25898,6 +25899,22 @@ Precondition: user A and user B opened the same block
<a name="anytype-SnapshotWithType"></a>
### SnapshotWithType
| Field | Type | Label | Description |
| ----- | ---- | ----- | ----------- |
| sbType | [model.SmartBlockType](#anytype-model-SmartBlockType) | | |
| snapshot | [Change.Snapshot](#anytype-Change-Snapshot) | | |

8
pb/protos/snapshot.proto
View file

@ -2,6 +2,14 @@ syntax = "proto3";
package anytype;
option go_package = "pb";
import "pkg/lib/pb/model/protos/models.proto";
import "pb/protos/changes.proto";
message SnapshotWithType {
anytype.model.SmartBlockType sbType = 1;
anytype.Change.Snapshot snapshot = 2;
}
message Profile {
string name = 1;
string avatar = 2;

256
pb/snapshot.pb.go
View file

@ -5,7 +5,7 @@ package pb
import (
fmt "fmt"
_ "github.com/anyproto/anytype-heart/pkg/lib/pb/model"
model "github.com/anyproto/anytype-heart/pkg/lib/pb/model"
proto "github.com/gogo/protobuf/proto"
io "io"
math "math"
@ -23,6 +23,58 @@ var _ = math.Inf
// proto package needs to be updated.
const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
type SnapshotWithType struct {
SbType model.SmartBlockType `protobuf:"varint,1,opt,name=sbType,proto3,enum=anytype.model.SmartBlockType" json:"sbType,omitempty"`
Snapshot *ChangeSnapshot `protobuf:"bytes,2,opt,name=snapshot,proto3" json:"snapshot,omitempty"`
}
func (m *SnapshotWithType) Reset() { *m = SnapshotWithType{} }
func (m *SnapshotWithType) String() string { return proto.CompactTextString(m) }
func (*SnapshotWithType) ProtoMessage() {}
func (*SnapshotWithType) Descriptor() ([]byte, []int) {
return fileDescriptor_022f1596c727bff6, []int{0}
}
func (m *SnapshotWithType) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *SnapshotWithType) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_SnapshotWithType.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *SnapshotWithType) XXX_Merge(src proto.Message) {
xxx_messageInfo_SnapshotWithType.Merge(m, src)
}
func (m *SnapshotWithType) XXX_Size() int {
return m.Size()
}
func (m *SnapshotWithType) XXX_DiscardUnknown() {
xxx_messageInfo_SnapshotWithType.DiscardUnknown(m)
}
var xxx_messageInfo_SnapshotWithType proto.InternalMessageInfo
func (m *SnapshotWithType) GetSbType() model.SmartBlockType {
if m != nil {
return m.SbType
}
return model.SmartBlockType_AccountOld
}
func (m *SnapshotWithType) GetSnapshot() *ChangeSnapshot {
if m != nil {
return m.Snapshot
}
return nil
}
type Profile struct {
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
Avatar string `protobuf:"bytes,2,opt,name=avatar,proto3" json:"avatar,omitempty"`
@ -36,7 +88,7 @@ func (m *Profile) Reset() { *m = Profile{} }
func (m *Profile) String() string { return proto.CompactTextString(m) }
func (*Profile) ProtoMessage() {}
func (*Profile) Descriptor() ([]byte, []int) {
return fileDescriptor_022f1596c727bff6, []int{0}
return fileDescriptor_022f1596c727bff6, []int{1}
}
func (m *Profile) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
@ -108,29 +160,74 @@ func (m *Profile) GetAnalyticsId() string {
}
func init() {
proto.RegisterType((*SnapshotWithType)(nil), "anytype.SnapshotWithType")
proto.RegisterType((*Profile)(nil), "anytype.Profile")
}
func init() { proto.RegisterFile("pb/protos/snapshot.proto", fileDescriptor_022f1596c727bff6) }
var fileDescriptor_022f1596c727bff6 = []byte{
// 250 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x8f, 0xb1, 0x4a, 0x34, 0x31,
0x10, 0xc7, 0x37, 0xc7, 0x7d, 0xbb, 0x5c, 0xbe, 0x46, 0x52, 0x68, 0x38, 0x8e, 0x70, 0x88, 0x85,
0x58, 0x98, 0xc2, 0x37, 0x10, 0x9b, 0xed, 0xc4, 0xd2, 0x6e, 0xb2, 0x89, 0xb7, 0x8b, 0x7b, 0x49,
0xc8, 0x04, 0x61, 0xdf, 0xc2, 0xf7, 0xf1, 0x05, 0x2c, 0xaf, 0xb4, 0x94, 0xdd, 0x17, 0x11, 0xe6,
0xee, 0x38, 0xc1, 0x2e, 0xbf, 0xdf, 0xff, 0x9f, 0x19, 0x86, 0xcb, 0x68, 0x74, 0x4c, 0x21, 0x07,
0xd4, 0xe8, 0x21, 0x62, 0x1b, 0xf2, 0x2d, 0xb1, 0xa8, 0xc0, 0x0f, 0x79, 0x88, 0x6e, 0x79, 0x15,
0x5f, 0x37, 0xba, 0xef, 0x8c, 0x8e, 0x46, 0x6f, 0x83, 0x75, 0xfd, 0xf1, 0x03, 0x01, 0xee, 0xeb,
0xcb, 0x8b, 0xd3, 0xa0, 0xa6, 0x05, 0xbf, 0x71, 0x87, 0xe0, 0xf2, 0x83, 0xf1, 0xea, 0x31, 0x85,
0x97, 0xae, 0x77, 0x42, 0xf0, 0xb9, 0x87, 0xad, 0x93, 0x6c, 0xcd, 0xae, 0x17, 0x4f, 0xf4, 0x16,
0xe7, 0xbc, 0x84, 0x37, 0xc8, 0x90, 0xe4, 0x8c, 0xec, 0x81, 0x84, 0xe4, 0x15, 0x58, 0x9b, 0x1c,
0xa2, 0x9c, 0x53, 0x70, 0x44, 0x71, 0xc3, 0xcf, 0x30, 0x42, 0xe3, 0x1e, 0x00, 0x5b, 0x13, 0x20,
0xd9, 0xda, 0xca, 0x7f, 0x54, 0xf9, 0xe3, 0xc5, 0x8a, 0x2f, 0xe2, 0x7e, 0x79, 0x6d, 0x65, 0x49,
0xa5, 0x93, 0x10, 0x6b, 0xfe, 0x1f, 0x3c, 0xf4, 0x43, 0xee, 0x1a, 0xac, 0xad, 0xac, 0x28, 0xff,
0xad, 0xee, 0x57, 0x9f, 0xa3, 0x62, 0xbb, 0x51, 0xb1, 0xef, 0x51, 0xb1, 0xf7, 0x49, 0x15, 0xbb,
0x49, 0x15, 0x5f, 0x93, 0x2a, 0x9e, 0x67, 0xd1, 0x98, 0x92, 0x4e, 0xbc, 0xfb, 0x09, 0x00, 0x00,
0xff, 0xff, 0xaa, 0x08, 0xb0, 0xc4, 0x46, 0x01, 0x00, 0x00,
// 318 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x90, 0xc1, 0x4a, 0xc3, 0x30,
0x18, 0xc7, 0x97, 0x31, 0x5b, 0x97, 0x81, 0x8c, 0x1c, 0x34, 0x8c, 0x59, 0xc6, 0xf0, 0x30, 0x3c,
0xb4, 0x30, 0xf5, 0x05, 0xa6, 0x97, 0xdd, 0xa4, 0x13, 0x04, 0x6f, 0x5f, 0x9a, 0xb8, 0x96, 0x75,
0x4d, 0x48, 0x82, 0xd0, 0x93, 0xaf, 0xe0, 0xfb, 0xf8, 0x02, 0x1e, 0x77, 0xf4, 0x28, 0xdb, 0x8b,
0xc8, 0xd2, 0x76, 0x13, 0xbc, 0x7d, 0xff, 0xef, 0xff, 0xfb, 0xfe, 0xff, 0x10, 0x4c, 0x15, 0x8b,
0x94, 0x96, 0x56, 0x9a, 0xc8, 0x14, 0xa0, 0x4c, 0x2a, 0x6d, 0xe8, 0x34, 0xf1, 0xa1, 0x28, 0x6d,
0xa9, 0xc4, 0xe0, 0x4a, 0xad, 0x96, 0x51, 0x9e, 0xb1, 0x48, 0xb1, 0x68, 0x2d, 0xb9, 0xc8, 0x9b,
0x03, 0x27, 0x4c, 0x85, 0x0f, 0x2e, 0x8e, 0x41, 0x49, 0x0a, 0xc5, 0x52, 0xd4, 0xc6, 0xf8, 0x1d,
0xf7, 0x17, 0x75, 0xf2, 0x73, 0x66, 0xd3, 0xa7, 0x52, 0x09, 0x72, 0x87, 0x3d, 0xc3, 0xf6, 0x13,
0x45, 0x23, 0x34, 0x39, 0x9b, 0x5e, 0x86, 0x75, 0x59, 0xe8, 0x32, 0xc3, 0xc5, 0x1a, 0xb4, 0x9d,
0xe5, 0x32, 0x59, 0xed, 0xa1, 0xb8, 0x86, 0xc9, 0x2d, 0x3e, 0x6d, 0x1e, 0x49, 0xdb, 0x23, 0x34,
0xe9, 0x4d, 0xe9, 0xe1, 0xf0, 0xde, 0x95, 0x86, 0x4d, 0x55, 0x7c, 0x20, 0xc7, 0x9f, 0x08, 0xfb,
0x8f, 0x5a, 0xbe, 0x66, 0xb9, 0x20, 0x04, 0x77, 0x0a, 0x58, 0x57, 0xb5, 0xdd, 0xd8, 0xcd, 0xe4,
0x1c, 0x7b, 0xf0, 0x06, 0x16, 0xb4, 0xcb, 0xec, 0xc6, 0xb5, 0x22, 0x14, 0xfb, 0xc0, 0xb9, 0x16,
0xc6, 0xd0, 0x8e, 0x33, 0x1a, 0x49, 0xae, 0x71, 0xdf, 0x28, 0x48, 0xc4, 0x03, 0x98, 0x94, 0x49,
0xd0, 0x7c, 0xce, 0xe9, 0x89, 0x43, 0xfe, 0xed, 0xc9, 0x10, 0x77, 0x55, 0x55, 0x3e, 0xe7, 0xd4,
0x73, 0xd0, 0x71, 0x41, 0x46, 0xb8, 0x07, 0x05, 0xe4, 0xa5, 0xcd, 0x12, 0x33, 0xe7, 0xd4, 0x77,
0xfe, 0xdf, 0xd5, 0x6c, 0xf8, 0xb5, 0x0d, 0xd0, 0x66, 0x1b, 0xa0, 0x9f, 0x6d, 0x80, 0x3e, 0x76,
0x41, 0x6b, 0xb3, 0x0b, 0x5a, 0xdf, 0xbb, 0xa0, 0xf5, 0xd2, 0x56, 0x8c, 0x79, 0xee, 0x8f, 0x6f,
0x7e, 0x03, 0x00, 0x00, 0xff, 0xff, 0xab, 0x51, 0x3f, 0xdf, 0xc7, 0x01, 0x00, 0x00,
}
func (m *SnapshotWithType) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *SnapshotWithType) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *SnapshotWithType) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if m.Snapshot != nil {
{
size, err := m.Snapshot.MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
i -= size
i = encodeVarintSnapshot(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0x12
}
if m.SbType != 0 {
i = encodeVarintSnapshot(dAtA, i, uint64(m.SbType))
i--
dAtA[i] = 0x8
}
return len(dAtA) - i, nil
}
func (m *Profile) Marshal() (dAtA []byte, err error) {
@ -209,6 +306,22 @@ func encodeVarintSnapshot(dAtA []byte, offset int, v uint64) int {
dAtA[offset] = uint8(v)
return base
}
func (m *SnapshotWithType) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
if m.SbType != 0 {
n += 1 + sovSnapshot(uint64(m.SbType))
}
if m.Snapshot != nil {
l = m.Snapshot.Size()
n += 1 + l + sovSnapshot(uint64(l))
}
return n
}
func (m *Profile) Size() (n int) {
if m == nil {
return 0
@ -248,6 +361,111 @@ func sovSnapshot(x uint64) (n int) {
func sozSnapshot(x uint64) (n int) {
return sovSnapshot(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *SnapshotWithType) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowSnapshot
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: SnapshotWithType: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: SnapshotWithType: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field SbType", wireType)
}
m.SbType = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowSnapshot
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.SbType |= model.SmartBlockType(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Snapshot", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowSnapshot
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthSnapshot
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthSnapshot
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.Snapshot == nil {
m.Snapshot = &ChangeSnapshot{}
}
if err := m.Snapshot.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipSnapshot(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthSnapshot
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *Profile) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0