diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 4932557bd..e8911178b 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -19,4 +19,4 @@ jobs: with: version: latest only-new-issues: true - args: --timeout 15m \ No newline at end of file + args: --timeout 15m --skip-files ".*_test.go" \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index a6fc1f043..e8085aaac 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -10,6 +10,11 @@ un: go: '1.18' linters-settings: + lll: + line-length: 150 + funlen: + lines: 100 + statements: 50 errcheck: check-blank: true errchkjson: @@ -44,14 +49,12 @@ linters: - nestif - prealloc - revive - - wsl - unused - errcheck - funlen - gosimple - govet - typecheck - - stylecheck - unconvert max-issues-per-linter: 0 diff --git a/Makefile b/Makefile index 8e7f88ebb..790686e90 100644 --- a/Makefile +++ b/Makefile @@ -248,14 +248,14 @@ install-linter: run-linter: ifdef GOLANGCI_LINT_BRANCH - @golangci-lint run -v ./... --new-from-rev=$(GOLANGCI_LINT_BRANCH) --timeout 15m + @golangci-lint run -v ./... --new-from-rev=$(GOLANGCI_LINT_BRANCH) --timeout 15m --skip-files ".*_test.go" else - @golangci-lint run -v ./... --new-from-rev=master --timeout 15m + @golangci-lint run -v ./... --new-from-rev=master --timeout 15m --skip-files ".*_test.go" endif run-linter-fix: ifdef GOLANGCI_LINT_BRANCH - @golangci-lint run -v ./... --new-from-rev=$(GOLANGCI_LINT_BRANCH) --timeout 15m --fix + @golangci-lint run -v ./... --new-from-rev=$(GOLANGCI_LINT_BRANCH) --timeout 15m --skip-files ".*_test.go" --fix else - @golangci-lint run -v ./... --new-from-rev=master --timeout 15m --fix + @golangci-lint run -v ./... --new-from-rev=master --timeout 15m --skip-files ".*_test.go" --fix endif \ No newline at end of file diff --git a/core/block/import/converter/types.go b/core/block/import/converter/types.go index 2e3986fb5..076774c49 100644 --- a/core/block/import/converter/types.go +++ b/core/block/import/converter/types.go @@ -41,9 +41,18 @@ type Snapshot struct { Snapshot *model.SmartBlockSnapshotBase } +// Relation incapsulate name and relations format. We need this structure, so we don't create relations in Anytype +// during GetSnapshots step in converter and create them in RelationCreator +type Relation struct { + BlockID string // if relations is used as a block + Name string + Format model.RelationFormat +} + // Response expected response of each converter, incapsulate blocks snapshots and converting errors type Response struct { Snapshots []*Snapshot + Relations map[string][]*Relation // object id to its relations Error ConvertError } diff --git a/core/block/import/importer.go b/core/block/import/importer.go index 199a66cd8..9c0190687 100644 --- a/core/block/import/importer.go +++ b/core/block/import/importer.go @@ -177,7 +177,11 @@ func (i *Import) createObjects(ctx *session.Context, res *converter.Response, pr default: } progress.AddDone(1) - detail, err := i.oc.Create(ctx, snapshot.Snapshot, snapshot.Id, sbType, req.UpdateExistingObjects) + var relations []*converter.Relation + if res.Relations != nil { + relations = res.Relations[snapshot.Id] + } + detail, err := i.oc.Create(ctx, snapshot.Snapshot, relations, snapshot.Id, sbType, req.UpdateExistingObjects) if err != nil { allErrors[getFileName(snapshot)] = err if req.Mode != pb.RpcObjectImportRequest_IGNORE_ERRORS { @@ -192,4 +196,4 @@ func (i *Import) createObjects(ctx *session.Context, res *converter.Response, pr func convertType(cType string) pb.RpcObjectImportListImportResponseType { return pb.RpcObjectImportListImportResponseType(pb.RpcObjectImportListImportResponseType_value[cType]) -} \ No newline at end of file +} diff --git a/core/block/import/importer_test.go b/core/block/import/importer_test.go index fd395eddd..2e5825450 100644 --- a/core/block/import/importer_test.go +++ b/core/block/import/importer_test.go @@ -38,7 +38,7 @@ func Test_ImportSuccess(t *testing.T) { i.converters = make(map[string]cv.Converter, 0) i.converters["Notion"] = converter creator := NewMockCreator(ctrl) - creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1) + creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1) i.oc = creator err := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{ @@ -99,7 +99,7 @@ func Test_ImportErrorFromObjectCreator(t *testing.T) { i.converters = make(map[string]cv.Converter, 0) i.converters["Notion"] = converter creator := NewMockCreator(ctrl) - creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("creator error")).Times(1) + creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("creator error")).Times(1) i.oc = creator res := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{ @@ -137,7 +137,7 @@ func Test_ImportIgnoreErrorMode(t *testing.T) { i.converters = make(map[string]cv.Converter, 0) i.converters["Notion"] = converter creator := NewMockCreator(ctrl) - creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1) + creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1) i.oc = creator res := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{ @@ -175,7 +175,7 @@ func Test_ImportIgnoreErrorModeWithTwoErrorsPerFile(t *testing.T) { i.converters = make(map[string]cv.Converter, 0) i.converters["Notion"] = converter creator := NewMockCreator(ctrl) - creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("creator error")).Times(1) + creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("creator error")).Times(1) i.oc = creator res := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{ @@ -198,7 +198,7 @@ func Test_ImportExternalPlugin(t *testing.T) { i.converters = make(map[string]cv.Converter, 0) creator := NewMockCreator(ctrl) - creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1) + creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1) i.oc = creator snapshots := make([]*pb.RpcObjectImportRequestSnapshot, 0) @@ -327,7 +327,7 @@ func Test_ImportWebSuccess(t *testing.T) { i.converters = make(map[string]cv.Converter, 0) creator := NewMockCreator(ctrl) - creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1) + creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1) i.oc = creator parser := parsers.NewMockParser(ctrl) @@ -365,7 +365,7 @@ func Test_ImportWebFailedToCreateObject(t *testing.T) { i.converters = make(map[string]cv.Converter, 0) creator := NewMockCreator(ctrl) - creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("error")).Times(1) + creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("error")).Times(1) i.oc = creator parser := parsers.NewMockParser(ctrl) diff --git a/core/block/import/mock.go b/core/block/import/mock.go index 07403e2ce..42e49c7f8 100644 --- a/core/block/import/mock.go +++ b/core/block/import/mock.go @@ -188,7 +188,7 @@ func (m *MockCreator) EXPECT() *MockCreatorMockRecorder { } // Create mocks base method. -func (m *MockCreator) Create(ctx *session.Context, cs *model.SmartBlockSnapshotBase, pageID string, sbType smartblock.SmartBlockType, updateExisting bool) (*types.Struct, error) { +func (m *MockCreator) Create(ctx *session.Context, cs *model.SmartBlockSnapshotBase, relations []*converter.Relation, pageID string, sbType smartblock.SmartBlockType, updateExisting bool) (*types.Struct, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Create", ctx, cs, pageID, sbType, updateExisting) ret0, _ := ret[0].(*types.Struct) @@ -197,7 +197,7 @@ func (m *MockCreator) Create(ctx *session.Context, cs *model.SmartBlockSnapshotB } // Create indicates an expected call of Create. -func (mr *MockCreatorMockRecorder) Create(ctx, cs, pageID, sbType, updateExisting interface{}) *gomock.Call { +func (mr *MockCreatorMockRecorder) Create(ctx, cs, relations, pageID, sbType, updateExisting interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockCreator)(nil).Create), ctx, cs, pageID, sbType, updateExisting) } diff --git a/core/block/import/notion/api/block/block.go b/core/block/import/notion/api/block/block.go index e9867d1ac..ed18547d4 100644 --- a/core/block/import/notion/api/block/block.go +++ b/core/block/import/notion/api/block/block.go @@ -1,7 +1,10 @@ package block import ( + "github.com/globalsign/mgo/bson" + "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" ) type BlockType string @@ -52,3 +55,37 @@ type Block struct { HasChildren bool `json:"has_children"` Type BlockType `json:"type"` } + +type Identifiable interface { + GetID() string +} + +type ChildSetter interface { + Identifiable + HasChild() bool + SetChildren(children []interface{}) +} + +type Getter interface { + GetBlocks(req *MapRequest) *MapResponse +} + +const unsupportedBlockMessage = "Unsupported block" + +type UnsupportedBlock struct{} + +func (*UnsupportedBlock) GetBlocks(req *MapRequest) *MapResponse { + id := bson.NewObjectId().Hex() + bl := &model.Block{ + Id: id, + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{ + Text: unsupportedBlockMessage, + }, + }, + } + return &MapResponse{ + Blocks: []*model.Block{bl}, + BlockIDs: []string{id}, + } +} diff --git a/core/block/import/notion/api/block/div.go b/core/block/import/notion/api/block/div.go index 263aa3d18..bfd7e4aa4 100644 --- a/core/block/import/notion/api/block/div.go +++ b/core/block/import/notion/api/block/div.go @@ -1,8 +1,9 @@ package block import ( - "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" "github.com/globalsign/mgo/bson" + + "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" ) type DividerBlock struct { @@ -10,14 +11,18 @@ type DividerBlock struct { Divider struct{} `json:"divider"` } -func (*DividerBlock) GetDivBlock() (*model.Block, string) { +func (*DividerBlock) GetBlocks(*MapRequest) *MapResponse { id := bson.NewObjectId().Hex() - return &model.Block{ + block := &model.Block{ Id: id, Content: &model.BlockContentOfDiv{ Div: &model.BlockContentDiv{ Style: 0, }, }, - }, id + } + return &MapResponse{ + Blocks: []*model.Block{block}, + BlockIDs: []string{id}, + } } diff --git a/core/block/import/notion/api/block/file.go b/core/block/import/notion/api/block/file.go index 20056c56c..b3ffb486c 100644 --- a/core/block/import/notion/api/block/file.go +++ b/core/block/import/notion/api/block/file.go @@ -1,6 +1,9 @@ package block -import "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api" +import ( + "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" +) type FileBlock struct { Block @@ -8,17 +11,49 @@ type FileBlock struct { Caption []api.RichText `json:"caption"` } +func (f *FileBlock) GetBlocks(*MapRequest) *MapResponse { + block, id := f.File.GetFileBlock(model.BlockContentFile_File) + return &MapResponse{ + Blocks: []*model.Block{block}, + BlockIDs: []string{id}, + } +} + type ImageBlock struct { Block File api.FileObject `json:"image"` } +func (i *ImageBlock) GetBlocks(*MapRequest) *MapResponse { + block, id := i.File.GetFileBlock(model.BlockContentFile_Image) + return &MapResponse{ + Blocks: []*model.Block{block}, + BlockIDs: []string{id}, + } +} + type PdfBlock struct { Block File api.FileObject `json:"pdf"` } +func (p *PdfBlock) GetBlocks(*MapRequest) *MapResponse { + block, id := p.File.GetFileBlock(model.BlockContentFile_PDF) + return &MapResponse{ + Blocks: []*model.Block{block}, + BlockIDs: []string{id}, + } +} + type VideoBlock struct { Block File api.FileObject `json:"video"` } + +func (p *VideoBlock) GetBlocks(*MapRequest) *MapResponse { + block, id := p.File.GetFileBlock(model.BlockContentFile_Video) + return &MapResponse{ + Blocks: []*model.Block{block}, + BlockIDs: []string{id}, + } +} diff --git a/core/block/import/notion/api/block/link.go b/core/block/import/notion/api/block/link.go new file mode 100644 index 000000000..21b9e1569 --- /dev/null +++ b/core/block/import/notion/api/block/link.go @@ -0,0 +1,157 @@ +package block + +import ( + "strings" + + "github.com/globalsign/mgo/bson" + + "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + textUtil "github.com/anytypeio/go-anytype-middleware/util/text" +) + +const notFoundPageMessage = "Can't access object in Notion, please provide access in API" + +type EmbedBlock struct { + Block + Embed LinkToWeb `json:"embed"` +} + +type LinkToWeb struct { + URL string `json:"url"` +} + +type LinkPreviewBlock struct { + Block + LinkPreview LinkToWeb `json:"link_preview"` +} + +func (b *LinkToWeb) GetBlocks(*MapRequest) *MapResponse { + id := bson.NewObjectId().Hex() + + to := textUtil.UTF16RuneCountString(b.URL) + + bl := &model.Block{ + Id: id, + ChildrenIds: nil, + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{ + Text: b.URL, + Marks: &model.BlockContentTextMarks{ + Marks: []*model.BlockContentTextMark{ + { + Range: &model.Range{ + From: int32(0), + To: int32(to), + }, + Type: model.BlockContentTextMark_Link, + Param: b.URL, + }, + }, + }, + }, + }, + } + return &MapResponse{ + Blocks: []*model.Block{bl}, + BlockIDs: []string{id}, + } +} + +type ChildPageBlock struct { + Block + ChildPage Child `json:"child_page"` +} + +func (b *ChildPageBlock) GetBlocks(req *MapRequest) *MapResponse { + bl, id := b.ChildPage.GetLinkToObjectBlock(req.NotionPageIdsToAnytype, req.PageNameToID) + return &MapResponse{ + Blocks: []*model.Block{bl}, + BlockIDs: []string{id}, + } +} + +type ChildDatabaseBlock struct { + Block + ChildDatabase Child `json:"child_database"` +} + +func (b *ChildDatabaseBlock) GetBlocks(req *MapRequest) *MapResponse { + bl, id := b.ChildDatabase.GetLinkToObjectBlock(req.NotionDatabaseIdsToAnytype, req.DatabaseNameToID) + return &MapResponse{ + Blocks: []*model.Block{bl}, + BlockIDs: []string{id}, + } +} + +type Child struct { + Title string `json:"title"` +} + +func (c *Child) GetLinkToObjectBlock(notionIdsToAnytype, idToName map[string]string) (*model.Block, string) { + var ( + targetBlockID string + ok bool + ) + for id, name := range idToName { + if strings.EqualFold(name, c.Title) { + if len(notionIdsToAnytype) > 0 { + targetBlockID, ok = notionIdsToAnytype[id] + } + break + } + } + + id := bson.NewObjectId().Hex() + if !ok { + return &model.Block{ + Id: id, + ChildrenIds: nil, + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{ + Text: notFoundPageMessage, + Marks: &model.BlockContentTextMarks{ + Marks: []*model.BlockContentTextMark{}, + }, + }, + }, + }, id + } + + return &model.Block{ + Id: id, + ChildrenIds: nil, + Content: &model.BlockContentOfLink{ + Link: &model.BlockContentLink{ + TargetBlockId: targetBlockID, + }, + }}, id +} + +type LinkToPageBlock struct { + Block + LinkToPage api.Parent `json:"link_to_page"` +} + +func (l *LinkToPageBlock) GetBlocks(req *MapRequest) *MapResponse { + var anytypeID string + if l.LinkToPage.PageID != "" { + anytypeID = req.NotionPageIdsToAnytype[l.LinkToPage.PageID] + } + if l.LinkToPage.DatabaseID != "" { + anytypeID = req.NotionDatabaseIdsToAnytype[l.LinkToPage.DatabaseID] + } + id := bson.NewObjectId().Hex() + bl := &model.Block{ + Id: id, + ChildrenIds: nil, + Content: &model.BlockContentOfLink{ + Link: &model.BlockContentLink{ + TargetBlockId: anytypeID, + }, + }} + return &MapResponse{ + Blocks: []*model.Block{bl}, + BlockIDs: []string{id}, + } +} diff --git a/core/block/import/notion/api/block/link_test.go b/core/block/import/notion/api/block/link_test.go new file mode 100644 index 000000000..ff12cca77 --- /dev/null +++ b/core/block/import/notion/api/block/link_test.go @@ -0,0 +1,36 @@ +package block + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" +) + +func Test_GetLinkToObjectBlockSuccess(t *testing.T) { + c := &Child{Title: "title"} + nameToID := map[string]string{"id": "title"} + notionIdsToAnytype := map[string]string{"id": "anytypeId"} + bl, _ := c.GetLinkToObjectBlock(notionIdsToAnytype, nameToID) + assert.NotNil(t, bl) + content, ok := bl.Content.(*model.BlockContentOfLink) + assert.True(t, ok) + assert.Equal(t, content.Link.TargetBlockId, "anytypeId") +} + +func Test_GetLinkToObjectBlockFail(t *testing.T) { + c := &Child{Title: "title"} + bl, _ := c.GetLinkToObjectBlock(nil, nil) + assert.NotNil(t, bl) + content, ok := bl.Content.(*model.BlockContentOfText) + assert.True(t, ok) + assert.Equal(t, content.Text.Text, notFoundPageMessage) + + nameToID := map[string]string{"id": "title"} + bl, _ = c.GetLinkToObjectBlock(nameToID, nil) + assert.NotNil(t, bl) + content, ok = bl.Content.(*model.BlockContentOfText) + assert.True(t, ok) + assert.Equal(t, content.Text.Text, notFoundPageMessage) +} diff --git a/core/block/import/notion/api/block/mapper.go b/core/block/import/notion/api/block/mapper.go index 775d04b39..9430639cb 100644 --- a/core/block/import/notion/api/block/mapper.go +++ b/core/block/import/notion/api/block/mapper.go @@ -1,131 +1,56 @@ package block import ( + "github.com/gogo/protobuf/types" + + "github.com/anytypeio/go-anytype-middleware/core/block/import/converter" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" ) -type Mapper struct{} +// MapRequest is a data object with all necessary structures for blocks +type MapRequest struct { + Blocks []interface{} + // Need all these maps for correct mapping of pages and databases from notion to anytype + // for such blocks as mentions or links to objects + NotionPageIdsToAnytype map[string]string + NotionDatabaseIdsToAnytype map[string]string + PageNameToID map[string]string + DatabaseNameToID map[string]string +} -func (m *Mapper) MapBlocks(blocks []interface{}, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID map[string]string) ([]*model.Block, []string) { - var ( - anytypeBlocks = make([]*model.Block, 0) - ids = make([]string, 0) - ) - for _, bl := range blocks { - switch block := bl.(type) { - case *ParagraphBlock: - childIds := make([]string, 0) - var childBlocks []*model.Block - if block.HasChildren { - childBlocks, childIds = m.MapBlocks(block.Paragraph.Children, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - } - allBlocks, blockIDs := block.Paragraph.GetTextBlocks(model.BlockContentText_Paragraph, childIds, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - anytypeBlocks = append(anytypeBlocks, allBlocks...) - anytypeBlocks = append(anytypeBlocks, childBlocks...) - ids = append(ids, blockIDs...) - case *Heading1Block: - allBlocks, blockIDs := block.Heading1.GetTextBlocks(model.BlockContentText_Header1, []string{}, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - anytypeBlocks = append(anytypeBlocks, allBlocks...) - ids = append(ids, blockIDs...) - case *Heading2Block: - allBlocks, blockIDs := block.Heading2.GetTextBlocks(model.BlockContentText_Header2, []string{}, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - anytypeBlocks = append(anytypeBlocks, allBlocks...) - ids = append(ids, blockIDs...) - case *Heading3Block: - allBlocks, blockIDs := block.Heading3.GetTextBlocks(model.BlockContentText_Header3, []string{}, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - anytypeBlocks = append(anytypeBlocks, allBlocks...) - ids = append(ids, blockIDs...) - case *CalloutBlock: - childIds := make([]string, 0) - var childBlocks []*model.Block - if block.HasChildren { - childBlocks, childIds = m.MapBlocks(block.Callout.Children, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - } - calloutBlocks, blockIDs := block.Callout.GetCalloutBlocks(childIds) - anytypeBlocks = append(anytypeBlocks, calloutBlocks...) - anytypeBlocks = append(anytypeBlocks, childBlocks...) - ids = append(ids, blockIDs...) - case *QuoteBlock: - childIds := make([]string, 0) - var childBlocks []*model.Block - if block.HasChildren { - childBlocks, childIds = m.MapBlocks(block.Quote.Children, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - } - allBlocks, blockIDs := block.Quote.GetTextBlocks(model.BlockContentText_Quote, childIds, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - anytypeBlocks = append(anytypeBlocks, allBlocks...) - anytypeBlocks = append(anytypeBlocks, childBlocks...) - ids = append(ids, blockIDs...) - case *BulletedListBlock: - childIds := make([]string, 0) - var childBlocks []*model.Block - if block.HasChildren { - childBlocks, childIds = m.MapBlocks(block.BulletedList.Children, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - } - allBlocks, blockIDs := block.BulletedList.GetTextBlocks(model.BlockContentText_Marked, childIds, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - anytypeBlocks = append(anytypeBlocks, allBlocks...) - anytypeBlocks = append(anytypeBlocks, childBlocks...) - ids = append(ids, blockIDs...) - case *NumberedListBlock: - childIds := make([]string, 0) - var childBlocks []*model.Block - if block.HasChildren { - childBlocks, childIds = m.MapBlocks(block.NumberedList.Children, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - } - allBlocks, blockIDs := block.NumberedList.GetTextBlocks(model.BlockContentText_Numbered, childIds, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - anytypeBlocks = append(anytypeBlocks, allBlocks...) - anytypeBlocks = append(anytypeBlocks, childBlocks...) - ids = append(ids, blockIDs...) - case *ToggleBlock: - childIds := make([]string, 0) - var childBlocks []*model.Block - if block.HasChildren { - childBlocks, childIds = m.MapBlocks(block.Toggle.Children, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - } - allBlocks, blockIDs := block.Toggle.GetTextBlocks(model.BlockContentText_Toggle, childIds, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - anytypeBlocks = append(anytypeBlocks, allBlocks...) - anytypeBlocks = append(anytypeBlocks, childBlocks...) - ids = append(ids, blockIDs...) - case *CodeBlock: - c := bl.(*CodeBlock) - anytypeBlocks = append(anytypeBlocks, c.Code.GetCodeBlock()) - case *EquationBlock: - e := bl.(*EquationBlock) - anytypeBlocks = append(anytypeBlocks, e.Equation.HandleEquation()) - case *ToDoBlock: - childIds := make([]string, 0) - var childBlocks []*model.Block - if block.HasChildren { - childBlocks, childIds = m.MapBlocks(block.ToDo.Children, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - } - allBlocks, blockIDs := block.ToDo.GetTextBlocks(model.BlockContentText_Checkbox, childIds, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - anytypeBlocks = append(anytypeBlocks, allBlocks...) - anytypeBlocks = append(anytypeBlocks, childBlocks...) - ids = append(ids, blockIDs...) - case *FileBlock: - fileBlock, id := block.File.GetFileBlock(model.BlockContentFile_File) - anytypeBlocks = append(anytypeBlocks, fileBlock) - ids = append(ids, id) - case *ImageBlock: - fileBlock, id := block.File.GetFileBlock(model.BlockContentFile_Image) - anytypeBlocks = append(anytypeBlocks, fileBlock) - ids = append(ids, id) - case *VideoBlock: - fileBlock, id := block.File.GetFileBlock(model.BlockContentFile_Video) - anytypeBlocks = append(anytypeBlocks, fileBlock) - ids = append(ids, id) - case *PdfBlock: - fileBlock, id := block.File.GetFileBlock(model.BlockContentFile_PDF) - anytypeBlocks = append(anytypeBlocks, fileBlock) - ids = append(ids, id) - case *DividerBlock: - db, id := block.GetDivBlock() - anytypeBlocks = append(anytypeBlocks, db) - ids = append(ids, id) - case *TableOfContentsBlock: - db, id := block.GetTableOfContentsBlock() - anytypeBlocks = append(anytypeBlocks, db) - ids = append(ids, id) +type MapResponse struct { + Blocks []*model.Block + Relations []*converter.Relation + Details map[string]*types.Value + BlockIDs []string +} + +func (m *MapResponse) Merge(mergedResp *MapResponse) { + if mergedResp != nil { + m.BlockIDs = append(m.BlockIDs, mergedResp.BlockIDs...) + m.Relations = append(m.Relations, mergedResp.Relations...) + m.Blocks = append(m.Blocks, mergedResp.Blocks...) + m.MergeDetails(mergedResp.Details) + } +} + +func (m *MapResponse) MergeDetails(mergeDetails map[string]*types.Value) { + if m.Details == nil { + m.Details = make(map[string]*types.Value, 0) + } + for k, v := range mergeDetails { + m.Details[k] = v + } +} + +func MapBlocks(req *MapRequest) *MapResponse { + resp := &MapResponse{} + for _, bl := range req.Blocks { + if ba, ok := bl.(Getter); ok { + textResp := ba.GetBlocks(req) + resp.Merge(textResp) + continue } } - return anytypeBlocks, ids + return resp } diff --git a/core/block/import/notion/api/block/retrieve.go b/core/block/import/notion/api/block/retrieve.go index 82842ef56..dde37c05b 100644 --- a/core/block/import/notion/api/block/retrieve.go +++ b/core/block/import/notion/api/block/retrieve.go @@ -15,7 +15,6 @@ import ( "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/client" "github.com/anytypeio/go-anytype-middleware/pb" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" ) var logger = logging.Logger("notion-get-blocks") @@ -29,13 +28,11 @@ const ( type Service struct { client *client.Client - mapper *Mapper } func New(client *client.Client) *Service { return &Service{ client: client, - mapper: &Mapper{}, } } @@ -56,103 +53,29 @@ func (s *Service) GetBlocksAndChildren(ctx context.Context, pageID, apiKey strin return nil, ce } } - for _, b := range blocks { - switch bl := b.(type) { - case *Heading1Block, *Heading2Block, *Heading3Block, *CodeBlock, *EquationBlock, *FileBlock, *ImageBlock, *VideoBlock, *PdfBlock, *DividerBlock, *TableOfContentsBlock: - allBlocks = append(allBlocks, bl) - case *ParagraphBlock: - if bl.HasChildren { - children, err := s.GetBlocksAndChildren(ctx, bl.ID, apiKey, pageSize, mode) - if err != nil { - ce.Merge(*err) - if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return nil, ce - } - } - bl.Paragraph.Children = children - } - allBlocks = append(allBlocks, bl) - case *CalloutBlock: - if bl.HasChildren { - children, err := s.GetBlocksAndChildren(ctx, bl.ID, apiKey, pageSize, mode) - if err != nil { - ce.Merge(*err) - if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return nil, ce - } - } - bl.Callout.Children = children - } - allBlocks = append(allBlocks, bl) - case *QuoteBlock: - if bl.HasChildren { - children, err := s.GetBlocksAndChildren(ctx, bl.ID, apiKey, pageSize, mode) - if err != nil { - ce.Merge(*err) - if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return nil, ce - } - } - bl.Quote.Children = children - } - allBlocks = append(allBlocks, bl) - case *BulletedListBlock: - if bl.HasChildren { - children, err := s.GetBlocksAndChildren(ctx, bl.ID, apiKey, pageSize, mode) - if err != nil { - ce.Merge(*err) - if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return nil, ce - } - } - bl.BulletedList.Children = children - } - allBlocks = append(allBlocks, bl) - case *NumberedListBlock: - if bl.HasChildren { - children, err := s.GetBlocksAndChildren(ctx, bl.ID, apiKey, pageSize, mode) - if err != nil { - ce.Merge(*err) - if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return nil, ce - } - } - bl.NumberedList.Children = children - } - allBlocks = append(allBlocks, bl) - case *ToggleBlock: - if bl.HasChildren { - children, err := s.GetBlocksAndChildren(ctx, bl.ID, apiKey, pageSize, mode) - if err != nil { - ce.Merge(*err) - if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return nil, ce - } - } - bl.Toggle.Children = children - } - allBlocks = append(allBlocks, bl) - case *ToDoBlock: - if bl.HasChildren { - children, err := s.GetBlocksAndChildren(ctx, bl.ID, apiKey, pageSize, mode) - if err != nil { - ce.Merge(*err) - if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return nil, ce - } - } - bl.ToDo.Children = children - } - allBlocks = append(allBlocks, bl) + cs, ok := b.(ChildSetter) + if !ok { + allBlocks = append(allBlocks, b) + continue } + if cs.HasChild() { + children, err := s.GetBlocksAndChildren(ctx, cs.GetID(), apiKey, pageSize, mode) + if err != nil { + ce.Merge(*err) + } + if err != nil && mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { + return nil, ce + } + cs.SetChildren(children) + } + allBlocks = append(allBlocks, b) } return allBlocks, nil } -func (s *Service) MapNotionBlocksToAnytype(blocks []interface{}, notionPagesIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID map[string]string) []*model.Block { - allBlocks, _ := s.mapper.MapBlocks(blocks, notionPagesIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) - return allBlocks +func (s *Service) MapNotionBlocksToAnytype(req *MapRequest) *MapResponse { + return MapBlocks(req) } func (s *Service) getBlocks(ctx context.Context, pageID, apiKey string, pagination int64) ([]interface{}, error) { @@ -360,6 +283,54 @@ func (s *Service) getBlocks(ctx context.Context, pageID, apiKey string, paginati continue } blocks = append(blocks, &t) + case Embed: + var e EmbedBlock + err = json.Unmarshal(buffer, &e) + if err != nil { + logger.With(zap.String("method", "getBlocks")).Error(err) + continue + } + blocks = append(blocks, &e) + case LinkPreview: + var lp LinkPreviewBlock + err = json.Unmarshal(buffer, &lp) + if err != nil { + logger.With(zap.String("method", "getBlocks")).Error(err) + continue + } + blocks = append(blocks, &lp) + case ChildDatabase: + var c ChildDatabaseBlock + err = json.Unmarshal(buffer, &c) + if err != nil { + logger.With(zap.String("method", "getBlocks")).Error(err) + continue + } + blocks = append(blocks, &c) + case ChildPage: + var c ChildPageBlock + err = json.Unmarshal(buffer, &c) + if err != nil { + logger.With(zap.String("method", "getBlocks")).Error(err) + continue + } + blocks = append(blocks, &c) + case LinkToPage: + var l LinkToPageBlock + err = json.Unmarshal(buffer, &l) + if err != nil { + logger.With(zap.String("method", "getBlocks")).Error(err) + continue + } + blocks = append(blocks, &l) + case Unsupported: + var u UnsupportedBlock + err = json.Unmarshal(buffer, &u) + if err != nil { + logger.With(zap.String("method", "getBlocks")).Error(err) + continue + } + blocks = append(blocks, &u) } } diff --git a/core/block/import/notion/api/block/tableofcontent.go b/core/block/import/notion/api/block/tableofcontent.go index 3b175f9b0..84c39f893 100644 --- a/core/block/import/notion/api/block/tableofcontent.go +++ b/core/block/import/notion/api/block/tableofcontent.go @@ -18,18 +18,23 @@ type TableOfContentsObject struct { Color string `json:"color"` } -func (t *TableOfContentsBlock) GetTableOfContentsBlock() (*model.Block, string) { +func (t *TableOfContentsBlock) GetBlocks(req *MapRequest) *MapResponse { id := bson.NewObjectId().Hex() var color string // Anytype Table Of Content doesn't support different colors of text, only background if strings.HasSuffix(t.TableOfContent.Color, api.NotionBackgroundColorSuffix) { color = api.NotionColorToAnytype[t.TableOfContent.Color] } - return &model.Block{ + + block := &model.Block{ Id: id, BackgroundColor: color, Content: &model.BlockContentOfTableOfContents{ TableOfContents: &model.BlockContentTableOfContents{}, }, - }, id + } + return &MapResponse{ + Blocks: []*model.Block{block}, + BlockIDs: []string{id}, + } } diff --git a/core/block/import/notion/api/block/text.go b/core/block/import/notion/api/block/text.go index a47404b5f..07c697672 100644 --- a/core/block/import/notion/api/block/text.go +++ b/core/block/import/notion/api/block/text.go @@ -1,8 +1,6 @@ package block import ( - "strings" - "github.com/globalsign/mgo/bson" "github.com/gogo/protobuf/types" @@ -17,21 +15,56 @@ type ParagraphBlock struct { Paragraph TextObjectWithChildren `json:"paragraph"` } +func (p *ParagraphBlock) GetBlocks(req *MapRequest) *MapResponse { + childResp := &MapResponse{} + if p.HasChildren { + mapper := ChildrenMapper(&p.Paragraph) + childResp = mapper.MapChildren(req) + } + resp := p.Paragraph.GetTextBlocks(model.BlockContentText_Paragraph, childResp.BlockIDs, req) + resp.Merge(childResp) + return resp +} + +func (p *ParagraphBlock) HasChild() bool { + return p.HasChildren +} + +func (p *ParagraphBlock) SetChildren(children []interface{}) { + p.Paragraph.Children = children +} + +func (p *ParagraphBlock) GetID() string { + return p.ID +} + type Heading1Block struct { Block Heading1 HeadingObject `json:"heading_1"` } +func (h *Heading1Block) GetBlocks(req *MapRequest) *MapResponse { + return h.Heading1.GetTextBlocks(model.BlockContentText_Header1, nil, req) +} + type Heading2Block struct { Block Heading2 HeadingObject `json:"heading_2"` } +func (h *Heading2Block) GetBlocks(req *MapRequest) *MapResponse { + return h.Heading2.GetTextBlocks(model.BlockContentText_Header2, nil, req) +} + type Heading3Block struct { Block Heading3 HeadingObject `json:"heading_3"` } +func (p *Heading3Block) GetBlocks(req *MapRequest) *MapResponse { + return p.Heading3.GetTextBlocks(model.BlockContentText_Header3, nil, req) +} + type HeadingObject struct { TextObject IsToggleable bool `json:"is_toggleable"` @@ -47,21 +80,129 @@ type CalloutObject struct { Icon *api.Icon `json:"icon"` } +func (c *CalloutBlock) GetBlocks(req *MapRequest) *MapResponse { + calloutResp := c.Callout.GetTextBlocks(model.BlockContentText_Callout, nil, req) + extendedBlocks := make([]*model.Block, 0, len(calloutResp.Blocks)) + for _, cb := range calloutResp.Blocks { + text, ok := cb.Content.(*model.BlockContentOfText) + if !ok { + extendedBlocks = append(extendedBlocks, cb) + continue + } + if c.Callout.Icon != nil { + if c.Callout.Icon.Emoji != nil { + text.Text.IconEmoji = *c.Callout.Icon.Emoji + } + if c.Callout.Icon.Type == api.External && c.Callout.Icon.External != nil { + text.Text.IconImage = c.Callout.Icon.External.URL + } + if c.Callout.Icon.Type == api.File && c.Callout.Icon.File != nil { + text.Text.IconImage = c.Callout.Icon.File.URL + } + } + cb.Content = text + extendedBlocks = append(extendedBlocks, cb) + } + calloutResp.Blocks = extendedBlocks + return calloutResp +} + +func (c *CalloutBlock) HasChild() bool { + return c.HasChildren +} + +func (c *CalloutBlock) SetChildren(children []interface{}) { + c.Callout.Children = children +} + +func (c *CalloutBlock) GetID() string { + return c.ID +} + type QuoteBlock struct { Block Quote TextObjectWithChildren `json:"quote"` } +func (q *QuoteBlock) GetBlocks(req *MapRequest) *MapResponse { + childResp := &MapResponse{} + if q.HasChildren { + mapper := ChildrenMapper(&q.Quote) + childResp = mapper.MapChildren(req) + } + resp := q.Quote.GetTextBlocks(model.BlockContentText_Quote, childResp.BlockIDs, req) + resp.Merge(childResp) + return resp +} + +func (q *QuoteBlock) HasChild() bool { + return q.HasChildren +} + +func (q *QuoteBlock) SetChildren(children []interface{}) { + q.Quote.Children = children +} + +func (q *QuoteBlock) GetID() string { + return q.ID +} + type NumberedListBlock struct { Block NumberedList TextObjectWithChildren `json:"bulleted_list_item"` } +func (n *NumberedListBlock) GetBlocks(req *MapRequest) *MapResponse { + childResp := &MapResponse{} + if n.HasChildren { + mapper := ChildrenMapper(&n.NumberedList) + childResp = mapper.MapChildren(req) + } + resp := n.NumberedList.GetTextBlocks(model.BlockContentText_Numbered, childResp.BlockIDs, req) + resp.Merge(childResp) + return resp +} + +func (n *NumberedListBlock) HasChild() bool { + return n.HasChildren +} + +func (n *NumberedListBlock) SetChildren(children []interface{}) { + n.NumberedList.Children = children +} + +func (n *NumberedListBlock) GetID() string { + return n.ID +} + type ToDoBlock struct { Block ToDo ToDoObject `json:"to_do"` } +func (t *ToDoBlock) GetBlocks(req *MapRequest) *MapResponse { + childResp := &MapResponse{} + if t.HasChildren { + mapper := ChildrenMapper(&t.ToDo) + childResp = mapper.MapChildren(req) + } + resp := t.ToDo.GetTextBlocks(model.BlockContentText_Checkbox, childResp.BlockIDs, req) + resp.Merge(childResp) + return resp +} + +func (t *ToDoBlock) HasChild() bool { + return t.HasChildren +} + +func (t *ToDoBlock) SetChildren(children []interface{}) { + t.ToDo.Children = children +} + +func (t *ToDoBlock) GetID() string { + return t.ID +} + type ToDoObject struct { TextObjectWithChildren Checked bool `json:"checked"` @@ -72,212 +213,55 @@ type BulletedListBlock struct { BulletedList TextObjectWithChildren `json:"bulleted_list_item"` } +func (b *BulletedListBlock) GetBlocks(req *MapRequest) *MapResponse { + childResp := &MapResponse{} + if b.HasChildren { + mapper := ChildrenMapper(&b.BulletedList) + childResp = mapper.MapChildren(req) + } + resp := b.BulletedList.GetTextBlocks(model.BlockContentText_Marked, childResp.BlockIDs, req) + resp.Merge(childResp) + return resp +} + +func (b *BulletedListBlock) HasChild() bool { + return b.HasChildren +} + +func (b *BulletedListBlock) SetChildren(children []interface{}) { + b.BulletedList.Children = children +} + +func (b *BulletedListBlock) GetID() string { + return b.ID +} + type ToggleBlock struct { Block Toggle TextObjectWithChildren `json:"toggle"` } -type TextObjectWithChildren struct { - TextObject - Children []interface{} `json:"children"` +func (t *ToggleBlock) GetBlocks(req *MapRequest) *MapResponse { + childResp := &MapResponse{} + if t.HasChildren { + mapper := ChildrenMapper(&t.Toggle) + childResp = mapper.MapChildren(req) + } + resp := t.Toggle.GetTextBlocks(model.BlockContentText_Toggle, childResp.BlockIDs, req) + resp.Merge(childResp) + return resp } -type TextObject struct { - RichText []api.RichText `json:"rich_text"` - Color string `json:"color"` +func (t *ToggleBlock) HasChild() bool { + return t.HasChildren } -func (c *CalloutObject) GetCalloutBlocks(childIds []string) ([]*model.Block, []string) { - calloutBlocks, blockIDs := c.GetTextBlocks(model.BlockContentText_Callout, childIds, nil, nil, nil, nil) - for _, cb := range calloutBlocks { - text, ok := cb.Content.(*model.BlockContentOfText) - if ok { - if c.Icon != nil { - if c.Icon.Emoji != nil { - text.Text.IconEmoji = *c.Icon.Emoji - } - if c.Icon.Type == api.External && c.Icon.External != nil { - text.Text.IconImage = c.Icon.External.URL - } - if c.Icon.Type == api.File && c.Icon.File != nil { - text.Text.IconImage = c.Icon.File.URL - } - } - cb.Content = text - } - } - return calloutBlocks, blockIDs +func (t *ToggleBlock) SetChildren(children []interface{}) { + t.Toggle.Children = children } -func (t *TextObject) GetTextBlocks(style model.BlockContentTextStyle, childIds []string, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID map[string]string) ([]*model.Block, []string) { - marks := []*model.BlockContentTextMark{} - id := bson.NewObjectId().Hex() - allBlocks := make([]*model.Block, 0) - allIds := make([]string, 0) - var text strings.Builder - for _, rt := range t.RichText { - if rt.Type == api.Text { - marks = append(marks, t.handleTextType(rt, &text, notionPageIdsToAnytype, notionDatabaseIdsToAnytype)...) - } - if rt.Type == api.Mention { - marks = append(marks, t.handleMentionType(rt, &text, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID)...) - } - if rt.Type == api.Equation { - eqBlock := rt.Equation.HandleEquation() - allBlocks = append(allBlocks, eqBlock) - allIds = append(allIds, eqBlock.Id) - } - } - var backgroundColor string - if strings.HasSuffix(t.Color, api.NotionBackgroundColorSuffix) { - backgroundColor = api.NotionColorToAnytype[t.Color] - } - - if len(t.RichText) == 1 && t.RichText[0].Type == api.Equation { - return allBlocks, allIds - } - allBlocks = append(allBlocks, &model.Block{ - Id: id, - ChildrenIds: childIds, - BackgroundColor: backgroundColor, - Content: &model.BlockContentOfText{ - Text: &model.BlockContentText{ - Text: text.String(), - Style: style, - Marks: &model.BlockContentTextMarks{Marks: marks}, - Checked: false, - Color: api.NotionColorToAnytype[t.Color], - }, - }, - }) - for _, b := range allBlocks { - allIds = append(allIds, b.Id) - } - return allBlocks, allIds -} - -func (t *TextObject) handleTextType(rt api.RichText, text *strings.Builder, notionPageIdsToAnytype, notionDatabaseIdsToAnytype map[string]string) []*model.BlockContentTextMark { - marks := []*model.BlockContentTextMark{} - from := textUtil.UTF16RuneCountString(text.String()) - if rt.Text != nil && rt.Text.Link != nil && rt.Text.Link.Url != "" { - text.WriteString(rt.Text.Content) - } else { - text.WriteString(rt.PlainText) - } - to := textUtil.UTF16RuneCountString(text.String()) - if rt.Text != nil && rt.Text.Link != nil && rt.Text.Link.Url != "" { - url := strings.Trim(rt.Text.Link.Url, "/") - if databaseID, ok := notionDatabaseIdsToAnytype[url]; ok { - url = databaseID - } - if pageID, ok := notionPageIdsToAnytype[url]; ok { - url = pageID - } - marks = append(marks, &model.BlockContentTextMark{ - Range: &model.Range{ - From: int32(from), - To: int32(to), - }, - Type: model.BlockContentTextMark_Link, - Param: url, - }) - } - marks = append(marks, rt.BuildMarkdownFromAnnotations(int32(from), int32(to))...) - return marks -} - -func (t *TextObject) handleMentionType(rt api.RichText, text *strings.Builder, notionPageIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID map[string]string) []*model.BlockContentTextMark { - if rt.Mention.Type == api.UserMention { - return t.handleUserMention(rt, text) - } - if rt.Mention.Type == api.Database { - return t.handleDatabaseMention(rt, text, notionDatabaseIdsToAnytype, databaseNameToID) - } - if rt.Mention.Type == api.Page { - return t.handlePageMention(rt, text, notionPageIdsToAnytype, pageNameToID) - } - if rt.Mention.Type == api.Date { - return t.handleDateMention(rt, text) - } - if rt.Mention.Type == api.LinkPreview { - return t.handleLinkPreviewMention(rt, text) - } - return nil -} - -func (t *TextObject) handleUserMention(rt api.RichText, text *strings.Builder) []*model.BlockContentTextMark { - from := textUtil.UTF16RuneCountString(text.String()) - text.WriteString(rt.Mention.User.Name) - to := textUtil.UTF16RuneCountString(text.String()) - return rt.BuildMarkdownFromAnnotations(int32(from), int32(to)) -} - -func (t *TextObject) handleDatabaseMention(rt api.RichText, text *strings.Builder, notionDatabaseIdsToAnytype, databaseNameToID map[string]string) []*model.BlockContentTextMark { - if notionDatabaseIdsToAnytype == nil { - return nil - } - from := textUtil.UTF16RuneCountString(text.String()) - text.WriteString(databaseNameToID[rt.Mention.Database.ID]) - to := textUtil.UTF16RuneCountString(text.String()) - marks := rt.BuildMarkdownFromAnnotations(int32(from), int32(to)) - marks = append(marks, &model.BlockContentTextMark{ - Range: &model.Range{ - From: int32(from), - To: int32(to), - }, - Type: model.BlockContentTextMark_Mention, - Param: notionDatabaseIdsToAnytype[rt.Mention.Database.ID], - }) - return marks -} - -func (t *TextObject) handlePageMention(rt api.RichText, text *strings.Builder, notionPageIdsToAnytype, pageNameToID map[string]string) []*model.BlockContentTextMark { - if notionPageIdsToAnytype == nil { - return nil - } - from := textUtil.UTF16RuneCountString(text.String()) - text.WriteString(pageNameToID[rt.Mention.Page.ID]) - to := textUtil.UTF16RuneCountString(text.String()) - marks := rt.BuildMarkdownFromAnnotations(int32(from), int32(to)) - marks = append(marks, &model.BlockContentTextMark{ - Range: &model.Range{ - From: int32(from), - To: int32(to), - }, - Type: model.BlockContentTextMark_Mention, - Param: notionPageIdsToAnytype[rt.Mention.Page.ID], - }) - return marks -} - -func (t *TextObject) handleDateMention(rt api.RichText, text *strings.Builder) []*model.BlockContentTextMark { - var textDate string - if rt.Mention.Date.Start != "" { - textDate = rt.Mention.Date.Start - } - if rt.Mention.Date.End != "" { - textDate += " " + rt.Mention.Date.End - } - from := textUtil.UTF16RuneCountString(text.String()) - text.WriteString(textDate) - to := textUtil.UTF16RuneCountString(text.String()) - marks := rt.BuildMarkdownFromAnnotations(int32(from), int32(to)) - return marks -} - -func (t *TextObject) handleLinkPreviewMention(rt api.RichText, text *strings.Builder) []*model.BlockContentTextMark { - from := textUtil.UTF16RuneCountString(text.String()) - text.WriteString(rt.Mention.LinkPreview.Url) - to := textUtil.UTF16RuneCountString(text.String()) - marks := rt.BuildMarkdownFromAnnotations(int32(from), int32(to)) - marks = append(marks, &model.BlockContentTextMark{ - Range: &model.Range{ - From: int32(from), - To: int32(to), - }, - Type: model.BlockContentTextMark_Link, - }) - return marks +func (t *ToggleBlock) GetID() string { + return t.ID } type CodeBlock struct { @@ -291,17 +275,17 @@ type CodeObject struct { Language string `json:"language"` } -func (c *CodeObject) GetCodeBlock() *model.Block { +func (c *CodeBlock) GetBlocks(req *MapRequest) *MapResponse { id := bson.NewObjectId().Hex() bl := &model.Block{ Id: id, Fields: &types.Struct{ - Fields: map[string]*types.Value{"lang": pbtypes.String(c.Language)}, + Fields: map[string]*types.Value{"lang": pbtypes.String(c.Code.Language)}, }, } marks := []*model.BlockContentTextMark{} var code string - for _, rt := range c.RichText { + for _, rt := range c.Code.RichText { from := textUtil.UTF16RuneCountString(code) code += rt.PlainText to := textUtil.UTF16RuneCountString(code) @@ -316,7 +300,10 @@ func (c *CodeObject) GetCodeBlock() *model.Block { }, }, } - return bl + return &MapResponse{ + Blocks: []*model.Block{bl}, + BlockIDs: []string{id}, + } } type EquationBlock struct { diff --git a/core/block/import/notion/api/block/text_test.go b/core/block/import/notion/api/block/text_test.go index eaa3f285b..43d5fba60 100644 --- a/core/block/import/notion/api/block/text_test.go +++ b/core/block/import/notion/api/block/text_test.go @@ -24,11 +24,11 @@ func Test_GetTextBlocksTextSuccess(t *testing.T) { Color: api.RedBackGround, } - bl, _ := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, nil, nil, nil, nil) - assert.Len(t, bl, 1) - assert.Equal(t, bl[0].GetText().Style, model.BlockContentText_Paragraph) - assert.Equal(t, bl[0].BackgroundColor, api.AnytypeRed) - assert.Equal(t, bl[0].GetText().Text, "testtest2") + bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{}) + assert.Len(t, bl.Blocks, 1) + assert.Equal(t, bl.Blocks[0].GetText().Style, model.BlockContentText_Paragraph) + assert.Equal(t, bl.Blocks[0].BackgroundColor, api.AnytypeRed) + assert.Equal(t, bl.Blocks[0].GetText().Text, "testtest2") } func Test_GetTextBlocksTextUserMention(t *testing.T) { @@ -47,11 +47,11 @@ func Test_GetTextBlocksTextUserMention(t *testing.T) { }, } - bl, _ := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, nil, nil, nil, nil) - assert.Len(t, bl, 1) - assert.Equal(t, bl[0].GetText().Style, model.BlockContentText_Paragraph) - assert.Len(t, bl[0].GetText().Marks.Marks, 0) - assert.Equal(t, bl[0].GetText().Text, "Nastya") + bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{}) + assert.Len(t, bl.Blocks, 1) + assert.Equal(t, bl.Blocks[0].GetText().Style, model.BlockContentText_Paragraph) + assert.Len(t, bl.Blocks[0].GetText().Marks.Marks, 0) + assert.Equal(t, bl.Blocks[0].GetText().Text, "Nastya") } func Test_GetTextBlocksTextPageMention(t *testing.T) { @@ -69,12 +69,15 @@ func Test_GetTextBlocksTextPageMention(t *testing.T) { }, } - bl, _ := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, map[string]string{"notionID": "anytypeID"}, nil, map[string]string{"notionID": "Page"}, nil) - assert.Len(t, bl, 1) - assert.Equal(t, bl[0].GetText().Style, model.BlockContentText_Paragraph) - assert.Len(t, bl[0].GetText().Marks.Marks, 1) - assert.Equal(t, bl[0].GetText().Marks.Marks[0].Type, model.BlockContentTextMark_Mention) - assert.Equal(t, bl[0].GetText().Text, "Page") + bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{ + NotionPageIdsToAnytype: map[string]string{"notionID": "anytypeID"}, + PageNameToID: map[string]string{"notionID": "Page"}, + }) + assert.Len(t, bl.Blocks, 1) + assert.Equal(t, bl.Blocks[0].GetText().Style, model.BlockContentText_Paragraph) + assert.Len(t, bl.Blocks[0].GetText().Marks.Marks, 1) + assert.Equal(t, bl.Blocks[0].GetText().Marks.Marks[0].Type, model.BlockContentTextMark_Mention) + assert.Equal(t, bl.Blocks[0].GetText().Text, "Page") } func Test_GetTextBlocksDatabaseMention(t *testing.T) { @@ -92,12 +95,15 @@ func Test_GetTextBlocksDatabaseMention(t *testing.T) { }, } - bl, _ := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, nil, map[string]string{"notionID": "anytypeID"}, nil, map[string]string{"notionID": "Database"}) - assert.Len(t, bl, 1) - assert.Equal(t, bl[0].GetText().Style, model.BlockContentText_Paragraph) - assert.Len(t, bl[0].GetText().Marks.Marks, 1) - assert.Equal(t, bl[0].GetText().Marks.Marks[0].Type, model.BlockContentTextMark_Mention) - assert.Equal(t, bl[0].GetText().Text, "Database") + bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{ + NotionDatabaseIdsToAnytype: map[string]string{"notionID": "anytypeID"}, + DatabaseNameToID: map[string]string{"notionID": "Database"}, + }) + assert.Len(t, bl.Blocks, 1) + assert.Equal(t, bl.Blocks[0].GetText().Style, model.BlockContentText_Paragraph) + assert.Len(t, bl.Blocks[0].GetText().Marks.Marks, 1) + assert.Equal(t, bl.Blocks[0].GetText().Marks.Marks[0].Type, model.BlockContentTextMark_Mention) + assert.Equal(t, bl.Blocks[0].GetText().Text, "Database") } func Test_GetTextBlocksDateMention(t *testing.T) { @@ -115,11 +121,10 @@ func Test_GetTextBlocksDateMention(t *testing.T) { }, } - bl, _ := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, nil, nil, nil, nil) - assert.Len(t, bl, 1) - assert.Equal(t, bl[0].GetText().Style, model.BlockContentText_Paragraph) - assert.Len(t, bl[0].GetText().Marks.Marks, 0) - assert.Equal(t, bl[0].GetText().Text, "2022-11-14") + bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{}) + assert.Len(t, bl.Blocks, 1) + assert.NotNil(t, bl.Blocks[0].GetRelation()) + assert.Len(t, bl.Relations, 1) } func Test_GetTextBlocksLinkPreview(t *testing.T) { @@ -137,13 +142,13 @@ func Test_GetTextBlocksLinkPreview(t *testing.T) { }, } - bl, _ := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, nil, nil, nil, nil) - assert.Len(t, bl, 1) - assert.Equal(t, bl[0].GetText().Style, model.BlockContentText_Paragraph) - assert.NotNil(t, bl[0].GetText().Marks) - assert.Len(t, bl[0].GetText().Marks.Marks, 1) - assert.Equal(t, bl[0].GetText().Marks.Marks[0].Type, model.BlockContentTextMark_Link) - assert.Equal(t, bl[0].GetText().Text, "ref") + bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{}) + assert.Len(t, bl.Blocks, 1) + assert.Equal(t, bl.Blocks[0].GetText().Style, model.BlockContentText_Paragraph) + assert.NotNil(t, bl.Blocks[0].GetText().Marks) + assert.Len(t, bl.Blocks[0].GetText().Marks.Marks, 1) + assert.Equal(t, bl.Blocks[0].GetText().Marks.Marks[0].Type, model.BlockContentTextMark_Link) + assert.Equal(t, bl.Blocks[0].GetText().Text, "ref") } func Test_GetTextBlocksEquation(t *testing.T) { @@ -158,10 +163,10 @@ func Test_GetTextBlocksEquation(t *testing.T) { }, } - bl, _ := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, nil, nil, nil, nil) - assert.Len(t, bl, 1) - assert.NotNil(t, bl[0].GetLatex()) - assert.Equal(t, bl[0].GetLatex().Text, "Equation") + bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{}) + assert.Len(t, bl.Blocks, 1) + assert.NotNil(t, bl.Blocks[0].GetLatex()) + assert.Equal(t, bl.Blocks[0].GetLatex().Text, "Equation") } func Test_GetCodeBlocksSuccess(t *testing.T) { @@ -176,7 +181,8 @@ func Test_GetCodeBlocksSuccess(t *testing.T) { Language: "Go", }, } - bl := co.Code.GetCodeBlock() + bl := co.GetBlocks(&MapRequest{}) assert.NotNil(t, bl) - assert.Equal(t, bl.GetText().Text, "Code") + assert.Len(t, bl.Blocks, 1) + assert.Equal(t, bl.Blocks[0].GetText().Text, "Code") } \ No newline at end of file diff --git a/core/block/import/notion/api/block/textobject.go b/core/block/import/notion/api/block/textobject.go new file mode 100644 index 000000000..8c9f1f9e2 --- /dev/null +++ b/core/block/import/notion/api/block/textobject.go @@ -0,0 +1,259 @@ +package block + +import ( + "strings" + "time" + + "github.com/globalsign/mgo/bson" + "github.com/gogo/protobuf/types" + + "github.com/anytypeio/go-anytype-middleware/core/block/import/converter" + "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" + "github.com/anytypeio/go-anytype-middleware/util/pbtypes" + textUtil "github.com/anytypeio/go-anytype-middleware/util/text" +) + +const DateMentionTimeFormat = "2006-01-02" + +type ChildrenMapper interface { + MapChildren(req *MapRequest) *MapResponse +} + +type TextObject struct { + RichText []api.RichText `json:"rich_text"` + Color string `json:"color"` +} + +func (t *TextObject) GetTextBlocks(style model.BlockContentTextStyle, childIds []string, req *MapRequest) *MapResponse { + marks := []*model.BlockContentTextMark{} + id := bson.NewObjectId().Hex() + allBlocks := make([]*model.Block, 0) + allIds := make([]string, 0) + var ( + text strings.Builder + relations []*converter.Relation + details map[string]*types.Value + ) + for _, rt := range t.RichText { + if rt.Type == api.Text { + marks = append(marks, t.handleTextType(rt, &text, req.NotionPageIdsToAnytype, req.NotionDatabaseIdsToAnytype)...) + } + if rt.Type == api.Mention { + // Return Relation block for Date mention + if rt.Mention.Type == api.Date { + var ( + relationBlock *model.Block + relationBlockID string + relation *converter.Relation + ) + relationBlock, relation, details, relationBlockID = t.handleDateMention(rt, &text) + allBlocks = append(allBlocks, relationBlock) + allIds = append(allIds, relationBlockID) + relations = append(relations, relation) + continue + } + marks = append(marks, t.handleMentionType(rt, &text, req)...) + } + if rt.Type == api.Equation { + eqBlock := rt.Equation.HandleEquation() + allBlocks = append(allBlocks, eqBlock) + allIds = append(allIds, eqBlock.Id) + } + } + var backgroundColor string + if strings.Contains(t.Color, api.NotionBackgroundColorSuffix) { + backgroundColor = api.NotionColorToAnytype[t.Color] + } + + if t.isNotTextBlocks() { + return &MapResponse{ + Blocks: allBlocks, + Relations: relations, + Details: details, + BlockIDs: allIds, + } + } + allBlocks = append(allBlocks, &model.Block{ + Id: id, + ChildrenIds: childIds, + BackgroundColor: backgroundColor, + Content: &model.BlockContentOfText{ + Text: &model.BlockContentText{ + Text: text.String(), + Style: style, + Marks: &model.BlockContentTextMarks{Marks: marks}, + Checked: false, + Color: api.NotionColorToAnytype[t.Color], + }, + }, + }) + for _, b := range allBlocks { + allIds = append(allIds, b.Id) + } + return &MapResponse{ + Blocks: allBlocks, + Relations: relations, + Details: details, + BlockIDs: allIds, + } +} + +func (t *TextObject) handleTextType(rt api.RichText, + text *strings.Builder, + notionPageIdsToAnytype, + notionDatabaseIdsToAnytype map[string]string) []*model.BlockContentTextMark { + marks := []*model.BlockContentTextMark{} + from := textUtil.UTF16RuneCountString(text.String()) + if rt.Text != nil && rt.Text.Link != nil && rt.Text.Link.Url != "" { + text.WriteString(rt.Text.Content) + } else { + text.WriteString(rt.PlainText) + } + to := textUtil.UTF16RuneCountString(text.String()) + if rt.Text != nil && rt.Text.Link != nil && rt.Text.Link.Url != "" { + url := strings.Trim(rt.Text.Link.Url, "/") + if databaseID, ok := notionDatabaseIdsToAnytype[url]; ok { + url = databaseID + } + if pageID, ok := notionPageIdsToAnytype[url]; ok { + url = pageID + } + marks = append(marks, &model.BlockContentTextMark{ + Range: &model.Range{ + From: int32(from), + To: int32(to), + }, + Type: model.BlockContentTextMark_Link, + Param: url, + }) + } + marks = append(marks, rt.BuildMarkdownFromAnnotations(int32(from), int32(to))...) + return marks +} + +func (t *TextObject) handleMentionType(rt api.RichText, text *strings.Builder, req *MapRequest) []*model.BlockContentTextMark { + if rt.Mention.Type == api.UserMention { + return t.handleUserMention(rt, text) + } + if rt.Mention.Type == api.Database { + return t.handleDatabaseMention(rt, text, req.NotionDatabaseIdsToAnytype, req.DatabaseNameToID) + } + if rt.Mention.Type == api.Page { + return t.handlePageMention(rt, text, req.NotionPageIdsToAnytype, req.PageNameToID) + } + if rt.Mention.Type == api.LinkPreview { + return t.handleLinkPreviewMention(rt, text) + } + return nil +} + +func (t *TextObject) handleUserMention(rt api.RichText, text *strings.Builder) []*model.BlockContentTextMark { + from := textUtil.UTF16RuneCountString(text.String()) + text.WriteString(rt.Mention.User.Name) + to := textUtil.UTF16RuneCountString(text.String()) + return rt.BuildMarkdownFromAnnotations(int32(from), int32(to)) +} + +func (t *TextObject) handleDatabaseMention(rt api.RichText, + text *strings.Builder, + notionDatabaseIdsToAnytype, databaseNameToID map[string]string) []*model.BlockContentTextMark { + if notionDatabaseIdsToAnytype == nil { + return nil + } + from := textUtil.UTF16RuneCountString(text.String()) + text.WriteString(databaseNameToID[rt.Mention.Database.ID]) + to := textUtil.UTF16RuneCountString(text.String()) + marks := rt.BuildMarkdownFromAnnotations(int32(from), int32(to)) + marks = append(marks, &model.BlockContentTextMark{ + Range: &model.Range{ + From: int32(from), + To: int32(to), + }, + Type: model.BlockContentTextMark_Mention, + Param: notionDatabaseIdsToAnytype[rt.Mention.Database.ID], + }) + return marks +} + +func (t *TextObject) handlePageMention(rt api.RichText, + text *strings.Builder, + notionPageIdsToAnytype, pageNameToID map[string]string) []*model.BlockContentTextMark { + if notionPageIdsToAnytype == nil { + return nil + } + from := textUtil.UTF16RuneCountString(text.String()) + text.WriteString(pageNameToID[rt.Mention.Page.ID]) + to := textUtil.UTF16RuneCountString(text.String()) + marks := rt.BuildMarkdownFromAnnotations(int32(from), int32(to)) + marks = append(marks, &model.BlockContentTextMark{ + Range: &model.Range{ + From: int32(from), + To: int32(to), + }, + Type: model.BlockContentTextMark_Mention, + Param: notionPageIdsToAnytype[rt.Mention.Page.ID], + }) + return marks +} + +func (t *TextObject) handleDateMention(rt api.RichText, text *strings.Builder) (*model.Block, *converter.Relation, map[string]*types.Value, string) { + var textDate string + if rt.Mention.Date.Start != "" { + textDate = rt.Mention.Date.Start + } + if rt.Mention.Date.End != "" { + textDate += " " + rt.Mention.Date.End + } + date, err := time.Parse(DateMentionTimeFormat, textDate) + if err != nil { + return nil, nil, nil, "" + } + rel := converter.Relation{ + BlockID: bson.NewObjectId().Hex(), + Name: "Date", + Format: model.RelationFormat_date, + } + id := bson.NewObjectId().Hex() + details := map[string]*types.Value{rel.Name: pbtypes.Int64(date.Unix())} + return &model.Block{ + Id: id, + Content: &model.BlockContentOfRelation{ + Relation: &model.BlockContentRelation{ + Key: rel.BlockID, + }, + }, + }, &rel, details, id +} + +func (t *TextObject) handleLinkPreviewMention(rt api.RichText, text *strings.Builder) []*model.BlockContentTextMark { + from := textUtil.UTF16RuneCountString(text.String()) + text.WriteString(rt.Mention.LinkPreview.Url) + to := textUtil.UTF16RuneCountString(text.String()) + marks := rt.BuildMarkdownFromAnnotations(int32(from), int32(to)) + marks = append(marks, &model.BlockContentTextMark{ + Range: &model.Range{ + From: int32(from), + To: int32(to), + }, + Type: model.BlockContentTextMark_Link, + }) + return marks +} + +func (t *TextObject) isNotTextBlocks() bool { + return (len(t.RichText) == 1 && t.RichText[0].Type == api.Mention && t.RichText[0].Mention.Type == api.Date) || + len(t.RichText) == 1 && t.RichText[0].Type == api.Equation +} + +type TextObjectWithChildren struct { + TextObject + Children []interface{} `json:"children"` +} + +func (t *TextObjectWithChildren) MapChildren(req *MapRequest) *MapResponse { + childReq := *req + childReq.Blocks = t.Children + resp := MapBlocks(&childReq) + return resp +} diff --git a/core/block/import/notion/api/commonobjects.go b/core/block/import/notion/api/commonobjects.go index 6fb1fc2ea..8607fe788 100644 --- a/core/block/import/notion/api/commonobjects.go +++ b/core/block/import/notion/api/commonobjects.go @@ -1,7 +1,6 @@ package api import ( - "bytes" "encoding/json" "strings" "time" @@ -304,12 +303,13 @@ type Parent struct { DatabaseID string `json:"database_id"` } -func RichTextToDescription(rt []RichText) string { - var description bytes.Buffer - - for _, text := range rt { - description.WriteString(text.PlainText) - description.WriteRune('\n') +func RichTextToDescription(rt []*RichText) string { + var richText strings.Builder + for i, title := range rt { + richText.WriteString(title.PlainText) + if i != len(rt)-1 { + richText.WriteString("\n") + } } - return description.String() + return richText.String() } diff --git a/core/block/import/notion/api/database/database.go b/core/block/import/notion/api/database/database.go index 36b994d07..f8e6d5393 100644 --- a/core/block/import/notion/api/database/database.go +++ b/core/block/import/notion/api/database/database.go @@ -38,7 +38,7 @@ type Database struct { Parent api.Parent `json:"parent"` URL string `json:"url"` Properties interface{} `json:"properties"` // can't support it for databases yet - Description []api.RichText `json:"description"` + Description []*api.RichText `json:"description"` IsInline bool `json:"is_inline"` Archived bool `json:"archived"` Icon *api.Icon `json:"icon,omitempty"` diff --git a/core/block/import/notion/api/page/page.go b/core/block/import/notion/api/page/page.go index 07c0573d5..7010836dd 100644 --- a/core/block/import/notion/api/page/page.go +++ b/core/block/import/notion/api/page/page.go @@ -14,30 +14,29 @@ import ( "github.com/anytypeio/go-anytype-middleware/pb" "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" "github.com/anytypeio/go-anytype-middleware/pkg/lib/threads" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" ) +var logger = logging.Logger("notion-page") + const ( ObjectType = "page" pageSize = 100 ) type Service struct { - propertyService *property.Service - detailSetter *property.DetailValueSetter - blockService *block.Service - client *client.Client + blockService *block.Service + client *client.Client } // New is a constructor for Service func New(client *client.Client) *Service { return &Service{ - propertyService: property.New(client), - detailSetter: property.NewDetailSetter(), - blockService: block.New(client), - client: client, + blockService: block.New(client), + client: client, } } @@ -62,12 +61,21 @@ func (p *Page) GetObjectType() string { } // GetPages transform Page objects from Notion to snaphots -func (ds *Service) GetPages(ctx context.Context, apiKey string, mode pb.RpcObjectImportRequestMode, pages []Page, notionDatabaseIdsToAnytype, databaseNameToID map[string]string) *converter.Response { +func (ds *Service) GetPages(ctx context.Context, + apiKey string, + mode pb.RpcObjectImportRequestMode, + pages []Page, + request *block.MapRequest) (*converter.Response, map[string]string) { convereterError := converter.ConvertError{} - return ds.mapPagesToSnaphots(ctx, apiKey, mode, pages, convereterError, notionDatabaseIdsToAnytype, databaseNameToID) + return ds.mapPagesToSnaphots(ctx, apiKey, mode, pages, convereterError, request) } -func (ds *Service) mapPagesToSnaphots(ctx context.Context, apiKey string, mode pb.RpcObjectImportRequestMode, pages []Page, convereterError converter.ConvertError, notionDatabaseIdsToAnytype, databaseNameToID map[string]string) *converter.Response { +func (ds *Service) mapPagesToSnaphots(ctx context.Context, + apiKey string, + mode pb.RpcObjectImportRequestMode, + pages []Page, + convereterError converter.ConvertError, + request *block.MapRequest) (*converter.Response, map[string]string) { var allSnapshots = make([]*converter.Snapshot, 0) var notionPagesIdsToAnytype = make(map[string]string, 0) for _, p := range pages { @@ -75,104 +83,128 @@ func (ds *Service) mapPagesToSnaphots(ctx context.Context, apiKey string, mode p if err != nil { convereterError.Add(p.ID, err) if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return &converter.Response{Error: convereterError} + return &converter.Response{Error: convereterError}, nil } else { continue } } notionPagesIdsToAnytype[p.ID] = tid.String() } + relationsToPageID := make(map[string][]*converter.Relation) + // Need to collect pages title and notion ids mapping for such blocks as ChildPage and ChildDatabase, + // because we only get title in those blocks from API + pageNameToID := make(map[string]string, 0) for _, p := range pages { - snapshot, ce := ds.transformPages(ctx, apiKey, p, mode, notionPagesIdsToAnytype, notionDatabaseIdsToAnytype, databaseNameToID) + for _, v := range p.Properties { + if t, ok := v.(*property.TitleItem); ok { + title := api.RichTextToDescription(t.Title) + pageNameToID[p.ID] = title + break + } + } + } + request.NotionPageIdsToAnytype = notionPagesIdsToAnytype + request.PageNameToID = pageNameToID + for _, p := range pages { + snapshot, relations, ce := ds.transformPages(ctx, apiKey, p, mode, request) if ce != nil { convereterError.Merge(*ce) if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return &converter.Response{Error: convereterError} + return &converter.Response{Error: convereterError}, nil } else { continue } } - + pageID := notionPagesIdsToAnytype[p.ID] allSnapshots = append(allSnapshots, &converter.Snapshot{ - Id: notionPagesIdsToAnytype[p.ID], + Id: pageID, FileName: p.URL, Snapshot: snapshot, }) + relationsToPageID[pageID] = relations } if convereterError.IsEmpty() { - return &converter.Response{Snapshots: allSnapshots, Error: nil} + return &converter.Response{Snapshots: allSnapshots, Relations: relationsToPageID, Error: nil}, notionPagesIdsToAnytype } - return &converter.Response{Snapshots: allSnapshots, Error: convereterError} + return &converter.Response{Snapshots: allSnapshots, Relations: relationsToPageID, Error: convereterError}, notionPagesIdsToAnytype } -func (ds *Service) transformPages(ctx context.Context, apiKey string, d Page, mode pb.RpcObjectImportRequestMode, notionPagesIdsToAnytype, notionDatabaseIdsToAnytype, databaseNameToID map[string]string) (*model.SmartBlockSnapshotBase, *converter.ConvertError) { +func (ds *Service) transformPages(ctx context.Context, + apiKey string, + p Page, + mode pb.RpcObjectImportRequestMode, + request *block.MapRequest) (*model.SmartBlockSnapshotBase, []*converter.Relation, *converter.ConvertError) { details := make(map[string]*types.Value, 0) - details[bundle.RelationKeySource.String()] = pbtypes.String(d.URL) - if d.Icon != nil && d.Icon.Emoji != nil { - details[bundle.RelationKeyIconEmoji.String()] = pbtypes.String(*d.Icon.Emoji) + details[bundle.RelationKeySource.String()] = pbtypes.String(p.URL) + if p.Icon != nil && p.Icon.Emoji != nil { + details[bundle.RelationKeyIconEmoji.String()] = pbtypes.String(*p.Icon.Emoji) } - details[bundle.RelationKeyIsArchived.String()] = pbtypes.Bool(d.Archived) + details[bundle.RelationKeyIsArchived.String()] = pbtypes.Bool(p.Archived) + details[bundle.RelationKeyIsFavorite.String()] = pbtypes.Bool(true) var ( allErrors = &converter.ConvertError{} - relations []*model.RelationLink + relations []*converter.Relation ) - relations, pageNameToID, ce := ds.handlePageProperties(apiKey, d.ID, d.Properties, details, mode) - if ce != nil { - allErrors.Merge(*ce) - if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return nil, allErrors - } - } + relations = ds.handlePageProperties(apiKey, p.ID, p.Properties, details, request.NotionPageIdsToAnytype, request.NotionDatabaseIdsToAnytype) - notionBlocks, blocksAndChildrenErr := ds.blockService.GetBlocksAndChildren(ctx, d.ID, apiKey, pageSize, mode) + notionBlocks, blocksAndChildrenErr := ds.blockService.GetBlocksAndChildren(ctx, p.ID, apiKey, pageSize, mode) if blocksAndChildrenErr != nil { allErrors.Merge(*blocksAndChildrenErr) if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return nil, allErrors + return nil, nil, allErrors } } - anytypeBlocks := ds.blockService.MapNotionBlocksToAnytype(notionBlocks, notionPagesIdsToAnytype, notionDatabaseIdsToAnytype, pageNameToID, databaseNameToID) + request.Blocks = notionBlocks + resp := ds.blockService.MapNotionBlocksToAnytype(request) + resp.MergeDetails(details) + relations = append(relations, resp.Relations...) snapshot := &model.SmartBlockSnapshotBase{ - Blocks: anytypeBlocks, - Details: &types.Struct{Fields: details}, - ObjectTypes: []string{bundle.TypeKeyPage.URL()}, - RelationLinks: relations, + Blocks: resp.Blocks, + Details: &types.Struct{Fields: resp.Details}, + ObjectTypes: []string{bundle.TypeKeyPage.URL()}, } - return snapshot, nil + return snapshot, relations, nil } // handlePageProperties gets properties values by their ids from notion api and transforms them to Details and RelationLinks -func (ds *Service) handlePageProperties(apiKey, pageID string, p property.Properties, d map[string]*types.Value, mode pb.RpcObjectImportRequestMode) ([]*model.RelationLink, map[string]string, *converter.ConvertError) { - ce := converter.ConvertError{} - relations := make([]*model.RelationLink, 0) - pageNameToID := make(map[string]string, 0) +func (ds *Service) handlePageProperties(apiKey, pageID string, + p property.Properties, + d map[string]*types.Value, + notionPagesIdsToAnytype, notionDatabaseIdsToAnytype map[string]string) []*converter.Relation { + relations := make([]*converter.Relation, 0) for k, v := range p { - object, err := ds.propertyService.GetPropertyObject(context.TODO(), pageID, v.GetID(), apiKey, v.GetPropertyType()) - if err != nil { - ce.Add("property: "+v.GetID(), err) - if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return relations, pageNameToID, &ce - } + if rel, ok := v.(*property.RelationItem); ok { + linkRelationsIDWithAnytypeID(rel, notionPagesIdsToAnytype, notionDatabaseIdsToAnytype) } - err = ds.detailSetter.SetDetailValue(k, v.GetPropertyType(), object, d) - if err != nil && mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - ce.Add("property: "+v.GetID(), err) - if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { - return relations, pageNameToID, &ce - } + var ( + ds property.DetailSetter + ok bool + ) + if ds, ok = v.(property.DetailSetter); !ok { + logger.With("method", "handlePageProperties").Errorf("failed to convert to interface DetailSetter, %s", v.GetPropertyType()) + continue } - relations = append(relations, &model.RelationLink{ - Key: k, + ds.SetDetail(k, d) + relations = append(relations, &converter.Relation{ + Name: k, Format: v.GetFormat(), }) - if v.GetPropertyType() == property.PropertyConfigTypeTitle { - if name, ok := d[bundle.RelationKeyName.String()]; ok { - pageNameToID[pageID] = name.GetStringValue() - } + } + return relations +} + +// linkRelationsIDWithAnytypeID take anytype ID based on page/database ID from Notin. In property we get id from Notion, so we somehow need to +// map this ID with anytype for correct Relation. We use two maps notionPagesIdsToAnytype, notionDatabaseIdsToAnytype for this +func linkRelationsIDWithAnytypeID(rel *property.RelationItem, notionPagesIdsToAnytype, notionDatabaseIdsToAnytype map[string]string) { + for _, r := range rel.Relation { + if anytypeID, ok := notionPagesIdsToAnytype[r.ID]; ok { + r.ID = anytypeID + } + if anytypeID, ok := notionDatabaseIdsToAnytype[r.ID]; ok { + r.ID = anytypeID } } - return relations, pageNameToID, nil } diff --git a/core/block/import/notion/api/page/page_test.go b/core/block/import/notion/api/page/page_test.go index f144b4246..826858b2c 100644 --- a/core/block/import/notion/api/page/page_test.go +++ b/core/block/import/notion/api/page/page_test.go @@ -5,48 +5,50 @@ import ( "net/http/httptest" "testing" - "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/client" - "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/property" - "github.com/anytypeio/go-anytype-middleware/pb" "github.com/gogo/protobuf/types" "github.com/stretchr/testify/assert" + + "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api" + "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/client" + "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/property" + "github.com/anytypeio/go-anytype-middleware/util/pbtypes" ) func Test_handlePagePropertiesSelect(t *testing.T) { - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"object":"property_item","type":"select","id":"C%5E%7DO","select":{"id":"f56c757b-eb58-4a15-b528-055a0e3e85b4","name":"dddd","color":"red"}}`)) - })) - + details := make(map[string]*types.Value, 0) c := client.NewClient() - c.BasePath = s.URL ps := New(c) - details := make(map[string]*types.Value, 0) - - p := property.SelectProperty{ID:"id", Type: string(property.PropertyConfigTypeSelect)} + p := property.SelectItem{ + Object: "", + ID: "id", + Type: string(property.PropertyConfigTypeSelect), + Select: property.SelectOption{ + ID: "id", + Name: "Name", + Color: api.Blue, + }, + } pr := property.Properties{"Select": &p} - _, _, ce := ps.handlePageProperties("key", "id", pr, details, pb.RpcObjectImportRequest_ALL_OR_NOTHING) + _ = ps.handlePageProperties("key", "id", pr, details, nil, nil) - assert.Nil(t, ce) assert.NotEmpty(t, details["Select"]) } func Test_handlePagePropertiesLastEditedTime(t *testing.T) { - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"object":"property_item","type":"last_edited_time","id":"MeQJ","last_edited_time":"2022-11-04T13:02:00.000Z"}`)) - })) - c := client.NewClient() - c.BasePath = s.URL ps := New(c) details := make(map[string]*types.Value, 0) - p := property.LastEditedTime{ID: "id", Type: string(property.PropertyConfigLastEditedTime)} + p := property.LastEditedTimeItem{ + ID: "id", + Type: string(property.PropertyConfigLastEditedTime), + LastEditedTime: "2022-12-07", + } pr := property.Properties{"LastEditedTime": &p} - _, _, ce := ps.handlePageProperties("key", "id", pr, details, pb.RpcObjectImportRequest_ALL_OR_NOTHING) + _ = ps.handlePageProperties("key", "id", pr, details, nil, nil) - assert.Nil(t, ce) assert.NotEmpty(t, details["LastEditedTime"]) } @@ -61,30 +63,31 @@ func Test_handlePagePropertiesRichText(t *testing.T) { details := make(map[string]*types.Value, 0) - p := property.RichText{ID: "id", Type: string(property.PropertyConfigLastEditedTime)} + p := property.RichTextItem{ID: "id", Type: string(property.PropertyConfigLastEditedTime)} pr := property.Properties{"RichText": &p} - _, _, ce := ps.handlePageProperties("key", "id", pr, details, pb.RpcObjectImportRequest_ALL_OR_NOTHING) + _ = ps.handlePageProperties("key", "id", pr, details, nil, nil) - assert.Nil(t, ce) assert.NotEmpty(t, details["RichText"]) } func Test_handlePagePropertiesStatus(t *testing.T) { - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"object":"property_item","type":"status","id":"VwSP","status":{"id":"e4927bd2-4580-4e37-9095-eb0af45923bc","name":"In progress","color":"blue"}}`)) - })) - c := client.NewClient() - c.BasePath = s.URL ps := New(c) details := make(map[string]*types.Value, 0) - p := property.StatusProperty{ID: "id", Type: property.PropertyConfigStatus} + p := property.StatusItem{ + ID: "id", + Type: property.PropertyConfigStatus, + Status: &property.Status{ + Name: "Done", + ID: "id", + Color: api.Pink, + }, + } pr := property.Properties{"Status": &p} - _, _, ce := ps.handlePageProperties("key", "id", pr, details, pb.RpcObjectImportRequest_ALL_OR_NOTHING) + _ = ps.handlePageProperties("key", "id", pr, details, nil, nil) - assert.Nil(t, ce) assert.NotEmpty(t, details["Status"]) } @@ -99,11 +102,15 @@ func Test_handlePagePropertiesNumber(t *testing.T) { details := make(map[string]*types.Value, 0) - p := property.NumberProperty{ID: "id", Type: string(property.PropertyConfigTypeNumber)} + num := int64(12) + p := property.NumberItem{ + ID: "id", + Type: string(property.PropertyConfigTypeNumber), + Number: &num, + } pr := property.Properties{"Number": &p} - _, _, ce := ps.handlePageProperties("key", "id", pr, details, pb.RpcObjectImportRequest_ALL_OR_NOTHING) + _ = ps.handlePageProperties("key", "id", pr, details, nil, nil) - assert.Nil(t, ce) assert.NotEmpty(t, details["Number"]) } @@ -118,11 +125,20 @@ func Test_handlePagePropertiesMultiSelect(t *testing.T) { details := make(map[string]*types.Value, 0) - p := property.NumberProperty{ID: "id", Type: string(property.PropertyConfigTypeMultiSelect)} + p := property.MultiSelectItem{ + ID: "id", + Type: string(property.PropertyConfigTypeMultiSelect), + MultiSelect: []*property.SelectOption{ + { + ID: "id", + Name: "Name", + Color: api.Blue, + }, + }, + } pr := property.Properties{"MultiSelect": &p} - _, _, ce := ps.handlePageProperties("key", "id", pr, details, pb.RpcObjectImportRequest_ALL_OR_NOTHING) + _ = ps.handlePageProperties("key", "id", pr, details, nil, nil) - assert.Nil(t, ce) assert.NotEmpty(t, details["MultiSelect"]) } @@ -137,36 +153,38 @@ func Test_handlePagePropertiesCheckbox(t *testing.T) { details := make(map[string]*types.Value, 0) - p := property.Checkbox{ID: "id", Type: string(property.PropertyConfigTypeCheckbox)} + p := property.CheckboxItem{ + ID: "id", + Type: string(property.PropertyConfigTypeCheckbox), + Checkbox: true, + } pr := property.Properties{"Checkbox": &p} - _, _, ce := ps.handlePageProperties("key", "id", pr, details, pb.RpcObjectImportRequest_ALL_OR_NOTHING) + _ = ps.handlePageProperties("key", "id", pr, details, nil, nil) - assert.Nil(t, ce) assert.NotEmpty(t, details["Checkbox"]) } func Test_handlePagePropertiesEmail(t *testing.T) { - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"object":"property_item","type":"email","id":"bQRa","email":null}`)) - })) - c := client.NewClient() - c.BasePath = s.URL ps := New(c) details := make(map[string]*types.Value, 0) - p := property.Email{ID: "id", Type: string(property.PropertyConfigTypeEmail)} + email := "a@mail.com" + p := property.EmailItem{ + ID: "id", + Type: string(property.PropertyConfigTypeEmail), + Email: &email, + } pr := property.Properties{"Email": &p} - _, _, ce := ps.handlePageProperties("key", "id", pr, details, pb.RpcObjectImportRequest_ALL_OR_NOTHING) + _ = ps.handlePageProperties("key", "id", pr, details, nil, nil) - assert.Nil(t, ce) assert.NotEmpty(t, details["Email"]) } func Test_handlePagePropertiesRelation(t *testing.T) { s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"object":"list","results":[{"object":"property_item","type":"relation","id":"cm~~","relation":{"id":"18e660df-d7f4-4d4b-b30c-eeb88ffee645"}}],"next_cursor":null,"has_more":false,"type":"property_item","property_item":{"id":"cm~~","next_url":null,"type":"relation","relation":{}}}`)) + w.Write([]byte(`{"object":"list","results":[{"object":"property_item","type":"relation","id":"cm~~","relation":{"id":"id"}}],"next_cursor":null,"has_more":false,"type":"property_item","property_item":{"id":"cm~~","next_url":null,"type":"relation","relation":{}}}`)) })) c := client.NewClient() @@ -175,67 +193,64 @@ func Test_handlePagePropertiesRelation(t *testing.T) { details := make(map[string]*types.Value, 0) - p := property.RelationProperty{ID: "id", Type: string(property.PropertyConfigTypeRelation)} + p := property.RelationItem{ID: "id", Type: string(property.PropertyConfigTypeRelation), Relation: []*property.Relation{{ID: "id"}}} pr := property.Properties{"Relation": &p} - _, _, ce := ps.handlePageProperties("key", "id", pr, details, pb.RpcObjectImportRequest_ALL_OR_NOTHING) + notionPageIdsToAnytype := map[string]string{"id": "anytypeID"} + notionDatabaseIdsToAnytype := map[string]string{"id": "anytypeID"} + _ = ps.handlePageProperties("key", "id", pr, details, notionPageIdsToAnytype, notionDatabaseIdsToAnytype) - assert.Nil(t, ce) - assert.NotEmpty(t, details["Relation"]) + assert.NotNil(t, details["Relation"].GetListValue()) + assert.Len(t, details["Relation"].GetListValue().Values, 1) + assert.Equal(t, pbtypes.GetStringListValue(details["Relation"])[0], "anytypeID") } func Test_handlePagePropertiesPeople(t *testing.T) { - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"object":"list","results":[{"object":"property_item","type":"people","id":"nWZg","people":{"object":"user","id":"60faafc6-0c5c-4479-a3f7-67d77cd8a56d"}}],"next_cursor":null,"has_more":false,"type":"property_item","property_item":{"id":"nWZg","next_url":null,"type":"people","people":{}}}`)) - })) - c := client.NewClient() - c.BasePath = s.URL ps := New(c) details := make(map[string]*types.Value, 0) - p := property.People{ID: "id", Type: string(property.PropertyConfigTypePeople)} + p := property.PeopleItem{ + Object: "", + ID: "id", + Type: string(property.PropertyConfigTypePeople), + } pr := property.Properties{"People": &p} - _, _, ce := ps.handlePageProperties("key", "id", pr, details, pb.RpcObjectImportRequest_ALL_OR_NOTHING) + _ = ps.handlePageProperties("key", "id", pr, details, nil, nil) - assert.Nil(t, ce) assert.NotEmpty(t, details["People"]) } func Test_handlePagePropertiesFormula(t *testing.T) { - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"object":"property_item","type":"formula","id":"%7Do%40%7B","formula":{"type":"number","number":11.745674324002}}`)) - })) - c := client.NewClient() - c.BasePath = s.URL ps := New(c) details := make(map[string]*types.Value, 0) - p := property.Formula{ID: "id", Type: string(property.PropertyConfigTypeFormula)} + p := property.FormulaItem{ + ID: "id", + Type: string(property.PropertyConfigTypeFormula), + Formula: map[string]interface{}{"type": property.NumberFormula, "number": float64(1)}, + } pr := property.Properties{"Formula": &p} - _, _, ce := ps.handlePageProperties("key", "id", pr, details, pb.RpcObjectImportRequest_ALL_OR_NOTHING) + _ = ps.handlePageProperties("key", "id", pr, details, nil, nil) - assert.Nil(t, ce) assert.NotEmpty(t, details["Formula"]) } func Test_handlePagePropertiesTitle(t *testing.T) { - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"object":"list","results":[{"object":"property_item","type":"title","id":"title","title":{"type":"text","text":{"content":"Daily Entry","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Daily Entry","href":null}}],"next_cursor":null,"has_more":false,"type":"property_item","property_item":{"id":"title","next_url":null,"type":"title","title":{}}}`)) - })) - c := client.NewClient() - c.BasePath = s.URL ps := New(c) details := make(map[string]*types.Value, 0) - p := property.Title{ID: "id", Type: string(property.PropertyConfigTypeTitle)} + p := property.TitleItem{ + ID: "id", + Type: string(property.PropertyConfigTypeTitle), + Title: []*api.RichText{{PlainText: "Title"}}, + } pr := property.Properties{"Title": &p} - _, _, ce := ps.handlePageProperties("key", "id", pr, details, pb.RpcObjectImportRequest_ALL_OR_NOTHING) + _ = ps.handlePageProperties("key", "id", pr, details, nil, nil) - assert.Nil(t, ce) assert.NotEmpty(t, details["name"]) } \ No newline at end of file diff --git a/core/block/import/notion/api/page/pageslink.go b/core/block/import/notion/api/page/pageslink.go new file mode 100644 index 000000000..6e4bbbac7 --- /dev/null +++ b/core/block/import/notion/api/page/pageslink.go @@ -0,0 +1,53 @@ +package page + +import ( + "github.com/globalsign/mgo/bson" + + "github.com/anytypeio/go-anytype-middleware/core/block/import/converter" + "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/database" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" +) + +// TODO tests +func SetPageLinksInDatabase(databaseSnaphots *converter.Response, + pages []Page, + databases []database.Database, + notionPageIdsToAnytype, notionDatabaseIdsToAnytype map[string]string) { + snapshots := makeSnapshotMapFromArray(databaseSnaphots.Snapshots) + + for _, p := range pages { + if p.Parent.DatabaseID != "" { + if parentID, ok := notionDatabaseIdsToAnytype[p.Parent.DatabaseID]; ok { + addLinkBlockToDatabase(snapshots[parentID], notionPageIdsToAnytype[p.ID]) + } + } + } + for _, d := range databases { + if d.Parent.DatabaseID != "" { + if parentID, ok := notionDatabaseIdsToAnytype[d.Parent.DatabaseID]; ok { + addLinkBlockToDatabase(snapshots[parentID], notionDatabaseIdsToAnytype[d.ID]) + } + } + } +} + +func addLinkBlockToDatabase(snapshots *converter.Snapshot, targetID string) { + id := bson.NewObjectId().Hex() + link := &model.Block{ + Id: id, + ChildrenIds: []string{}, + Content: &model.BlockContentOfLink{ + Link: &model.BlockContentLink{ + TargetBlockId: targetID, + }, + }} + snapshots.Snapshot.Blocks = append(snapshots.Snapshot.Blocks, link) +} + +func makeSnapshotMapFromArray(snapshots []*converter.Snapshot) map[string]*converter.Snapshot { + snapshotsMap := make(map[string]*converter.Snapshot, len(snapshots)) + for _, s := range snapshots { + snapshotsMap[s.Id] = s + } + return snapshotsMap +} diff --git a/core/block/import/notion/api/property/detailsetter.go b/core/block/import/notion/api/property/detailsetter.go deleted file mode 100644 index 71af30933..000000000 --- a/core/block/import/notion/api/property/detailsetter.go +++ /dev/null @@ -1,27 +0,0 @@ -package property - -import ( - "github.com/gogo/protobuf/types" -) - -type DetailValueSetter struct{} - -// New is a constructor for DetailValueSetter -func NewDetailSetter() *DetailValueSetter { - return &DetailValueSetter{} -} - -// SetDetailValue creates Detail based on property type and value -func (*DetailValueSetter) SetDetailValue(key string, propertyType PropertyConfigType, property []DetailSetter, details map[string]*types.Value) error { - if len(property) == 0 { - return nil - } - if IsVector(propertyType) { - for _, pr := range property { - pr.SetDetail(key, details) - } - } else { - property[0].SetDetail(key, details) - } - return nil -} \ No newline at end of file diff --git a/core/block/import/notion/api/property/propertyitem.go b/core/block/import/notion/api/property/propertyitem.go index ccf35e2de..52c93eac8 100644 --- a/core/block/import/notion/api/property/propertyitem.go +++ b/core/block/import/notion/api/property/propertyitem.go @@ -4,6 +4,7 @@ package property import ( "strconv" + "strings" "time" "github.com/gogo/protobuf/types" @@ -14,53 +15,124 @@ import ( "github.com/anytypeio/go-anytype-middleware/util/pbtypes" ) +type ConfigType string + +type Object interface { + GetPropertyType() ConfigType + GetID() string + GetFormat() model.RelationFormat +} + +const ( + PropertyConfigTypeTitle ConfigType = "title" + PropertyConfigTypeRichText ConfigType = "rich_text" + PropertyConfigTypeNumber ConfigType = "number" + PropertyConfigTypeSelect ConfigType = "select" + PropertyConfigTypeMultiSelect ConfigType = "multi_select" + PropertyConfigTypeDate ConfigType = "date" + PropertyConfigTypePeople ConfigType = "people" + PropertyConfigTypeFiles ConfigType = "files" + PropertyConfigTypeCheckbox ConfigType = "checkbox" + PropertyConfigTypeURL ConfigType = "url" + PropertyConfigTypeEmail ConfigType = "email" + PropertyConfigTypePhoneNumber ConfigType = "phone_number" + PropertyConfigTypeFormula ConfigType = "formula" + PropertyConfigTypeRelation ConfigType = "relation" + PropertyConfigTypeRollup ConfigType = "rollup" + PropertyConfigCreatedTime ConfigType = "created_time" + PropertyConfigCreatedBy ConfigType = "created_by" + PropertyConfigLastEditedTime ConfigType = "last_edited_time" + PropertyConfigLastEditedBy ConfigType = "last_edited_by" + PropertyConfigStatus ConfigType = "status" +) + type DetailSetter interface { - SetDetail(key string, details map[string]*types.Value) + SetDetail(key string, details map[string]*types.Value) } type TitleItem struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - Title api.RichText `json:"title"` + Object string `json:"object"` + ID string `json:"id"` + Type string `json:"type"` + Title []*api.RichText `json:"title"` } func (t *TitleItem) SetDetail(key string, details map[string]*types.Value) { - var title string - if existingTitle, ok := details[bundle.RelationKeyName.String()]; ok { - title = existingTitle.GetStringValue() + var richText strings.Builder + for i, title := range t.Title { + richText.WriteString(title.PlainText) + if i != len(title.PlainText)-1 { + richText.WriteString("\n") + } } - title += t.Title.PlainText - title += "\n" - details[bundle.RelationKeyName.String()] = pbtypes.String(title) + details[bundle.RelationKeyName.String()] = pbtypes.String(richText.String()) +} + +func (t *TitleItem) GetPropertyType() ConfigType { + return PropertyConfigTypeTitle +} + +func (t *TitleItem) GetID() string { + return t.ID +} + +func (t *TitleItem) GetFormat() model.RelationFormat { + return model.RelationFormat_shorttext } type RichTextItem struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - RichText api.RichText `json:"rich_text"` + Object string `json:"object"` + ID string `json:"id"` + Type string `json:"type"` + RichText []*api.RichText `json:"rich_text"` } func (rt *RichTextItem) SetDetail(key string, details map[string]*types.Value) { - var richText string - if existingText, ok := details[key]; ok { - richText = existingText.GetStringValue() + var richText strings.Builder + for i, r := range rt.RichText { + richText.WriteString(r.PlainText) + if i != len(rt.RichText)-1 { + richText.WriteString("\n") + } } - richText += rt.RichText.PlainText - richText += "\n" - details[key] = pbtypes.String(richText) + details[key] = pbtypes.String(richText.String()) +} + +func (rt *RichTextItem) GetPropertyType() ConfigType { + return PropertyConfigTypeRichText +} + +func (rt *RichTextItem) GetID() string { + return rt.ID +} + +func (rt *RichTextItem) GetFormat() model.RelationFormat { + return model.RelationFormat_longtext } type NumberItem struct { Object string `json:"object"` ID string `json:"id"` Type string `json:"type"` - Number int64 `json:"number"` + Number *int64 `json:"number"` } func (np *NumberItem) SetDetail(key string, details map[string]*types.Value) { - details[key] = pbtypes.Int64(np.Number) + if np.Number != nil { + details[key] = pbtypes.Int64(*np.Number) + } +} + +func (np *NumberItem) GetPropertyType() ConfigType { + return PropertyConfigTypeNumber +} + +func (np *NumberItem) GetID() string { + return np.ID +} + +func (np *NumberItem) GetFormat() model.RelationFormat { + return model.RelationFormat_number } type SelectItem struct { @@ -71,8 +143,8 @@ type SelectItem struct { } type SelectOption struct { - ID string `json:"id,omitempty"` - Name string `json:"name"` + ID string `json:"id,omitempty"` + Name string `json:"name"` Color string `json:"color"` } @@ -80,11 +152,23 @@ func (sp *SelectItem) SetDetail(key string, details map[string]*types.Value) { details[key] = pbtypes.StringList([]string{sp.Select.Name}) } +func (sp *SelectItem) GetPropertyType() ConfigType { + return PropertyConfigTypeSelect +} + +func (sp *SelectItem) GetID() string { + return sp.ID +} + +func (sp *SelectItem) GetFormat() model.RelationFormat { + return model.RelationFormat_tag +} + type MultiSelectItem struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - MultiSelect []SelectOption `json:"multi_select"` + Object string `json:"object"` + ID string `json:"id"` + Type string `json:"type"` + MultiSelect []*SelectOption `json:"multi_select"` } func (ms *MultiSelectItem) SetDetail(key string, details map[string]*types.Value) { @@ -95,15 +179,52 @@ func (ms *MultiSelectItem) SetDetail(key string, details map[string]*types.Value details[key] = pbtypes.StringList(msList) } -//can't support it yet -type DateItem struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - Date api.DateObject `json:"date"` +func (ms *MultiSelectItem) GetPropertyType() ConfigType { + return PropertyConfigTypeMultiSelect } -func (dp *DateItem) SetDetail(key string, details map[string]*types.Value) {} +func (ms *MultiSelectItem) GetID() string { + return ms.ID +} + +func (ms *MultiSelectItem) GetFormat() model.RelationFormat { + return model.RelationFormat_tag +} + +//can't support it yet +type DateItem struct { + Object string `json:"object"` + ID string `json:"id"` + Type string `json:"type"` + Date *api.DateObject `json:"date"` +} + +func (dp *DateItem) SetDetail(key string, details map[string]*types.Value) { + if dp.Date != nil { + var date strings.Builder + if dp.Date.Start != "" { + date.WriteString(dp.Date.Start) + } + if dp.Date.End != "" { + if dp.Date.Start != "" { + date.WriteString(" ") + } + date.WriteString(dp.Date.End) + } + } +} + +func (dp *DateItem) GetPropertyType() ConfigType { + return PropertyConfigTypeDate +} + +func (dp *DateItem) GetID() string { + return dp.ID +} + +func (dp *DateItem) GetFormat() model.RelationFormat { + return model.RelationFormat_date +} const ( NumberFormula string = "number" @@ -120,6 +241,9 @@ type FormulaItem struct { } func (f *FormulaItem) SetDetail(key string, details map[string]*types.Value) { + if f.Formula == nil { + return + } switch f.Formula["type"].(string) { case StringFormula: if f.Formula["string"] != nil { @@ -140,51 +264,77 @@ func (f *FormulaItem) SetDetail(key string, details map[string]*types.Value) { } } +func (f *FormulaItem) GetPropertyType() ConfigType { + return PropertyConfigTypeFormula +} + +func (f *FormulaItem) GetID() string { + return f.ID +} + +func (f *FormulaItem) GetFormat() model.RelationFormat { + return model.RelationFormat_shorttext +} + type RelationItem struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - Relation Relation `json:"relation"` - HasMore bool `json:"has_more"` + Object string `json:"object"` + ID string `json:"id"` + Type string `json:"type"` + Relation []*Relation `json:"relation"` + HasMore bool `json:"has_more"` } type Relation struct { ID string `json:"id"` } -func (r *RelationItem) SetDetail(key string, details map[string]*types.Value) { - var ( - relation = make([]string, 0) - ) - if rel, ok := details[key]; ok { - existingRelation := rel.GetListValue() - for _, v := range existingRelation.Values { - relation = append(relation, v.GetStringValue()) - } +func (rp *RelationItem) SetDetail(key string, details map[string]*types.Value) { + relation := make([]string, 0, len(rp.Relation)) + for _, rel := range rp.Relation { + relation = append(relation, rel.ID) } - relation = append(relation, r.Relation.ID) details[key] = pbtypes.StringList(relation) } +func (rp *RelationItem) GetPropertyType() ConfigType { + return PropertyConfigTypeRelation +} + +func (rp *RelationItem) GetID() string { + return rp.ID +} + +func (rp *RelationItem) GetFormat() model.RelationFormat { + return model.RelationFormat_object +} + type PeopleItem struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - People api.User `json:"people"` + Object string `json:"object"` + ID string `json:"id"` + Type string `json:"type"` + People []*api.User `json:"people"` } func (p *PeopleItem) SetDetail(key string, details map[string]*types.Value) { - var peopleList = make([]string, 0) - if existingPeople, ok := details[key]; ok { - list := existingPeople.GetListValue() - for _, v := range list.Values { - peopleList = append(peopleList, v.GetStringValue()) - } + peopleList := make([]string, 0, len(p.People)) + for _, people := range p.People { + peopleList = append(peopleList, people.Name) } - peopleList = append(peopleList, p.People.Name) details[key] = pbtypes.StringList(peopleList) } +func (p *PeopleItem) GetPropertyType() ConfigType { + return PropertyConfigTypePeople +} + +func (p *PeopleItem) GetID() string { + return p.ID +} + +func (p *PeopleItem) GetFormat() model.RelationFormat { + return model.RelationFormat_tag +} + type FileItem struct { Object string `json:"object"` ID string `json:"id"` @@ -192,8 +342,20 @@ type FileItem struct { File []api.FileObject `json:"files"` } +func (f *FileItem) GetPropertyType() ConfigType { + return PropertyConfigTypeFiles +} + +func (f *FileItem) GetID() string { + return f.ID +} + +func (f *FileItem) GetFormat() model.RelationFormat { + return model.RelationFormat_file +} + func (f *FileItem) SetDetail(key string, details map[string]*types.Value) { - var fileList = make([]string, len(f.File)) + fileList := make([]string, len(f.File)) for i, fo := range f.File { if fo.External.URL != "" { fileList[i] = fo.External.URL @@ -204,10 +366,6 @@ func (f *FileItem) SetDetail(key string, details map[string]*types.Value) { details[key] = pbtypes.StringList(fileList) } -func (f *FileItem) GetFormat() model.RelationFormat { - return model.RelationFormat_file -} - type CheckboxItem struct { Object string `json:"object"` ID string `json:"id"` @@ -219,37 +377,91 @@ func (c *CheckboxItem) SetDetail(key string, details map[string]*types.Value) { details[key] = pbtypes.Bool(c.Checkbox) } +func (c *CheckboxItem) GetPropertyType() ConfigType { + return PropertyConfigTypeCheckbox +} + +func (c *CheckboxItem) GetID() string { + return c.ID +} + +func (c *CheckboxItem) GetFormat() model.RelationFormat { + return model.RelationFormat_checkbox +} + type UrlItem struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - URL string `json:"url"` + Object string `json:"object"` + ID string `json:"id"` + Type string `json:"type"` + URL *string `json:"url"` } func (u *UrlItem) SetDetail(key string, details map[string]*types.Value) { - details[key] = pbtypes.String(u.URL) + if u.URL != nil { + details[key] = pbtypes.String(*u.URL) + } +} + +func (u *UrlItem) GetPropertyType() ConfigType { + return PropertyConfigTypeURL +} + +func (u *UrlItem) GetID() string { + return u.ID +} + +func (u *UrlItem) GetFormat() model.RelationFormat { + return model.RelationFormat_url } type EmailItem struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - Email string `json:"email"` + Object string `json:"object"` + ID string `json:"id"` + Type string `json:"type"` + Email *string `json:"email"` } func (e *EmailItem) SetDetail(key string, details map[string]*types.Value) { - details[key] = pbtypes.String(e.Email) + if e.Email != nil { + details[key] = pbtypes.String(*e.Email) + } +} + +func (e *EmailItem) GetPropertyType() ConfigType { + return PropertyConfigTypeURL +} + +func (e *EmailItem) GetID() string { + return e.ID +} + +func (e *EmailItem) GetFormat() model.RelationFormat { + return model.RelationFormat_email } type PhoneItem struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - Phone string `json:"phone_number"` + Object string `json:"object"` + ID string `json:"id"` + Type string `json:"type"` + Phone *string `json:"phone_number"` } func (p *PhoneItem) SetDetail(key string, details map[string]*types.Value) { - details[key] = pbtypes.String(p.Phone) + if p.Phone != nil { + details[key] = pbtypes.String(*p.Phone) + } +} + +func (p *PhoneItem) GetPropertyType() ConfigType { + return PropertyConfigTypePhoneNumber +} + +func (p *PhoneItem) GetID() string { + return p.ID +} + +func (p *PhoneItem) GetFormat() model.RelationFormat { + return model.RelationFormat_phone } type CreatedTimeItem struct { @@ -264,6 +476,18 @@ func (ct *CreatedTimeItem) SetDetail(key string, details map[string]*types.Value details[key] = pbtypes.Int64(t.Unix()) } +func (ct *CreatedTimeItem) GetPropertyType() ConfigType { + return PropertyConfigCreatedTime +} + +func (ct *CreatedTimeItem) GetID() string { + return ct.ID +} + +func (ct *CreatedTimeItem) GetFormat() model.RelationFormat { + return model.RelationFormat_date +} + type CreatedByItem struct { Object string `json:"object"` ID string `json:"id"` @@ -275,6 +499,18 @@ func (cb *CreatedByItem) SetDetail(key string, details map[string]*types.Value) details[key] = pbtypes.String(cb.CreatedBy.Name) } +func (cb *CreatedByItem) GetPropertyType() ConfigType { + return PropertyConfigCreatedBy +} + +func (cb *CreatedByItem) GetID() string { + return cb.ID +} + +func (cb *CreatedByItem) GetFormat() model.RelationFormat { + return model.RelationFormat_shorttext +} + type LastEditedTimeItem struct { Object string `json:"object"` ID string `json:"id"` @@ -287,6 +523,18 @@ func (le *LastEditedTimeItem) SetDetail(key string, details map[string]*types.Va details[key] = pbtypes.Int64(t.Unix()) } +func (le *LastEditedTimeItem) GetPropertyType() ConfigType { + return PropertyConfigLastEditedTime +} + +func (le *LastEditedTimeItem) GetID() string { + return le.ID +} + +func (le *LastEditedTimeItem) GetFormat() model.RelationFormat { + return model.RelationFormat_date +} + type LastEditedByItem struct { Object string `json:"object"` ID string `json:"id"` @@ -298,10 +546,22 @@ func (lb *LastEditedByItem) SetDetail(key string, details map[string]*types.Valu details[key] = pbtypes.String(lb.LastEditedBy.Name) } +func (lb *LastEditedByItem) GetPropertyType() ConfigType { + return PropertyConfigLastEditedBy +} + +func (lb *LastEditedByItem) GetID() string { + return lb.ID +} + +func (lb *LastEditedByItem) GetFormat() model.RelationFormat { + return model.RelationFormat_shorttext +} + type StatusItem struct { - ID string `json:"id"` - Type PropertyConfigType `json:"type"` - Status Status `json:"status"` + ID string `json:"id"` + Type ConfigType `json:"type"` + Status *Status `json:"status"` } type Status struct { @@ -314,6 +574,30 @@ func (sp *StatusItem) SetDetail(key string, details map[string]*types.Value) { details[key] = pbtypes.StringList([]string{sp.Status.Name}) } -type RollupItem struct {} +func (sp *StatusItem) GetPropertyType() ConfigType { + return PropertyConfigStatus +} -func (sp *RollupItem) SetDetail(key string, details map[string]*types.Value) {} \ No newline at end of file +func (sp *StatusItem) GetID() string { + return sp.ID +} + +func (sp *StatusItem) GetFormat() model.RelationFormat { + return model.RelationFormat_status +} + +type RollupItem struct{} + +func (r *RollupItem) SetDetail(key string, details map[string]*types.Value) {} + +func (r *RollupItem) GetPropertyType() ConfigType { + return PropertyConfigTypeRollup +} + +func (r *RollupItem) GetFormat() model.RelationFormat { + return model.RelationFormat_longtext +} + +func (r *RollupItem) GetID() string { + return "" +} diff --git a/core/block/import/notion/api/property/propertyobject.go b/core/block/import/notion/api/property/propertyobject.go index 7a411291f..f6e0fada8 100644 --- a/core/block/import/notion/api/property/propertyobject.go +++ b/core/block/import/notion/api/property/propertyobject.go @@ -1,31 +1,11 @@ package property import ( - "bytes" - "context" "encoding/json" "fmt" - "io/ioutil" - "net/http" - - "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/client" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" ) -var logger = logging.Logger("notion-property-retriever") - -const endpoint = "/pages/%s/properties/%s" -type Service struct { - client *client.Client -} - -func New(client *client.Client) *Service { - return &Service{ - client: client, - } -} - -type Properties map[string]PropertyObject +type Properties map[string]Object func (p *Properties) UnmarshalJSON(data []byte) error { var raw map[string]interface{} @@ -44,50 +24,50 @@ func (p *Properties) UnmarshalJSON(data []byte) error { func parsePropertyConfigs(raw map[string]interface{}) (Properties, error) { result := make(Properties) for k, v := range raw { - var p PropertyObject + var p Object switch rawProperty := v.(type) { case map[string]interface{}: - switch PropertyConfigType(rawProperty["type"].(string)) { + switch ConfigType(rawProperty["type"].(string)) { case PropertyConfigTypeTitle: - p = &Title{} + p = &TitleItem{} case PropertyConfigTypeRichText: - p = &RichText{} + p = &RichTextItem{} case PropertyConfigTypeNumber: - p = &NumberProperty{} + p = &NumberItem{} case PropertyConfigTypeSelect: - p = &SelectProperty{} + p = &SelectItem{} case PropertyConfigTypeMultiSelect: - p = &MultiSelect{} + p = &MultiSelectItem{} case PropertyConfigTypeDate: - p = &DateProperty{} + p = &DateItem{} case PropertyConfigTypePeople: - p = &People{} + p = &PeopleItem{} case PropertyConfigTypeFiles: - p = &File{} + p = &FileItem{} case PropertyConfigTypeCheckbox: - p = &Checkbox{} + p = &CheckboxItem{} case PropertyConfigTypeURL: - p = &Url{} + p = &UrlItem{} case PropertyConfigTypeEmail: - p = &Email{} + p = &EmailItem{} case PropertyConfigTypePhoneNumber: - p = &Phone{} + p = &PhoneItem{} case PropertyConfigTypeFormula: - p = &Formula{} + p = &FormulaItem{} case PropertyConfigTypeRelation: - p = &RelationProperty{} + p = &RelationItem{} case PropertyConfigTypeRollup: - p = &Rollup{} + p = &RollupItem{} case PropertyConfigCreatedTime: - p = &CreatedTime{} + p = &CreatedTimeItem{} case PropertyConfigCreatedBy: - p = &CreatedBy{} + p = &CreatedByItem{} case PropertyConfigLastEditedTime: - p = &LastEditedTime{} + p = &LastEditedTimeItem{} case PropertyConfigLastEditedBy: - p = &LastEditedBy{} + p = &LastEditedByItem{} case PropertyConfigStatus: - p = &StatusProperty{} + p = &StatusItem{} default: return nil, fmt.Errorf("unsupported property type: %s", rawProperty["type"].(string)) } @@ -108,251 +88,3 @@ func parsePropertyConfigs(raw map[string]interface{}) (Properties, error) { return result, nil } - -type PropertyPaginatedRespone struct{ - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - Results []interface{} `json:"results"` - Item interface{} `json:"property_item"` - HasMore bool `json:"has_more"` - NextCursor string `json:"next_cursor"` -} - -// GetPropertyObject get from Notion properties values based on type and id and marshal it to according structure from propertyitem.go -func (s *Service) GetPropertyObject(ctx context.Context, pageID, propertyID, apiKey string, propertyType PropertyConfigType) ([]DetailSetter, error) { - var ( - hasMore = true - body = &bytes.Buffer{} - startCursor string - response PropertyPaginatedRespone - paginatedResponse = make([]DetailSetter, 0) - ) - - type Request struct { - StartCursor string `json:"start_cursor,omitempty"` - } - - for hasMore { - err := json.NewEncoder(body).Encode(&Request{StartCursor: startCursor}) - - if err != nil { - return nil, fmt.Errorf("GetPropertyObject: %s", err) - } - - request := fmt.Sprintf(endpoint, pageID, propertyID) - req, err := s.client.PrepareRequest(ctx, apiKey, http.MethodGet, request, body) - - if err != nil { - return nil, fmt.Errorf("GetPropertyObject: %s", err) - } - res, err := s.client.HttpClient.Do(req) - - if err != nil { - return nil, fmt.Errorf("GetPropertyObject: %s", err) - } - defer res.Body.Close() - - b, err := ioutil.ReadAll(res.Body) - - if err != nil { - return nil, err - } - - if res.StatusCode != http.StatusOK { - notionErr := client.TransformHttpCodeToError(b) - if notionErr == nil { - return nil, fmt.Errorf("GetPropertyObject: failed http request, %d code", res.StatusCode) - } - return nil, notionErr - } - - switch propertyType { - case PropertyConfigTypeTitle, PropertyConfigTypeRichText, PropertyConfigTypeRelation, PropertyConfigTypePeople: - err = json.Unmarshal(b, &response) - if err != nil { - continue - } - res := response.Results - for _, v := range res { - buffer, err := json.Marshal(v) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal: %s", err) - continue - } - if propertyType == PropertyConfigTypeTitle { - p := TitleItem{} - err = json.Unmarshal(buffer, &p) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal TitleItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &p) - } - if propertyType == PropertyConfigTypeRichText { - p := RichTextItem{} - err = json.Unmarshal(buffer, &p) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal RichTextItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &p) - } - if propertyType == PropertyConfigTypeRelation { - p := RelationItem{} - err = json.Unmarshal(buffer, &p) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal RelationItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &p) - } - if propertyType == PropertyConfigTypePeople { - p := PeopleItem{} - err = json.Unmarshal(buffer, &p) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal PeopleItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &p) - } - } - if response.HasMore { - startCursor = response.NextCursor - continue - } - case PropertyConfigTypeNumber: - p := NumberItem{} - err = json.Unmarshal(b, &p) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal NumberItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &p) - case PropertyConfigTypeSelect: - p := SelectItem{} - err = json.Unmarshal(b, &p) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal SelectItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &p) - case PropertyConfigTypeMultiSelect: - p := MultiSelectItem{} - err = json.Unmarshal(b, &p) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal MultiSelectItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &p) - case PropertyConfigTypeDate: - date := DateItem{} - err = json.Unmarshal(b, &date) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal DateItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &date) - case PropertyConfigTypeFiles: - file := FileItem{} - err = json.Unmarshal(b, &file) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal FileItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &file) - case PropertyConfigTypeCheckbox: - checkbox := CheckboxItem{} - err = json.Unmarshal(b, &checkbox) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal CheckboxItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &checkbox) - case PropertyConfigTypeURL: - url := UrlItem{} - err = json.Unmarshal(b, &url) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal UrlItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &url) - case PropertyConfigTypeEmail: - email := EmailItem{} - err = json.Unmarshal(b, &email) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal EmailItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &email) - case PropertyConfigTypePhoneNumber: - phone := PhoneItem{} - err = json.Unmarshal(b, &phone) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal PhoneItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &phone) - case PropertyConfigTypeFormula: - formula := FormulaItem{} - err = json.Unmarshal(b, &formula) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal FormulaItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &formula) - case PropertyConfigTypeRollup: - rollup := RollupItem{} - err = json.Unmarshal(b, &rollup) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal Rollup: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &rollup) - case PropertyConfigCreatedTime: - ct := CreatedTimeItem{} - err = json.Unmarshal(b, &ct) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal CreatedTimeItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &ct) - case PropertyConfigCreatedBy: - cb := CreatedByItem{} - err = json.Unmarshal(b, &cb) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal CreatedByItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &cb) - case PropertyConfigLastEditedTime: - lt := LastEditedTimeItem{} - err = json.Unmarshal(b, <) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal LastEditedTimeItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, <) - case PropertyConfigLastEditedBy: - le := LastEditedByItem{} - err = json.Unmarshal(b, &le) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal LastEditedByItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &le) - case PropertyConfigStatus: - sp := StatusItem{} - err = json.Unmarshal(b, &sp) - if err != nil { - logger.Errorf("GetPropertyObject: failed to marshal StatusItem: %s", err) - continue - } - paginatedResponse = append(paginatedResponse, &sp) - default: - return nil, fmt.Errorf("GetPropertyObject: unsupported property type: %s", propertyType) - } - hasMore = false - } - return paginatedResponse, nil -} diff --git a/core/block/import/notion/api/property/propertyvalue.go b/core/block/import/notion/api/property/propertyvalue.go deleted file mode 100644 index cb9e0080e..000000000 --- a/core/block/import/notion/api/property/propertyvalue.go +++ /dev/null @@ -1,418 +0,0 @@ -package property - -// This file represent property configuration from Notion https://developers.notion.com/reference/property-object -import ( - "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" -) - -type PropertyConfigType string - -func IsVector (p PropertyConfigType) bool { - return p == PropertyConfigTypeTitle || p == PropertyConfigTypeRichText || p == PropertyConfigTypePeople || p == PropertyConfigTypeRelation -} - -const ( - PropertyConfigTypeTitle PropertyConfigType = "title" - PropertyConfigTypeRichText PropertyConfigType = "rich_text" - PropertyConfigTypeNumber PropertyConfigType = "number" - PropertyConfigTypeSelect PropertyConfigType = "select" - PropertyConfigTypeMultiSelect PropertyConfigType = "multi_select" - PropertyConfigTypeDate PropertyConfigType = "date" - PropertyConfigTypePeople PropertyConfigType = "people" - PropertyConfigTypeFiles PropertyConfigType = "files" - PropertyConfigTypeCheckbox PropertyConfigType = "checkbox" - PropertyConfigTypeURL PropertyConfigType = "url" - PropertyConfigTypeEmail PropertyConfigType = "email" - PropertyConfigTypePhoneNumber PropertyConfigType = "phone_number" - PropertyConfigTypeFormula PropertyConfigType = "formula" - PropertyConfigTypeRelation PropertyConfigType = "relation" - PropertyConfigTypeRollup PropertyConfigType = "rollup" - PropertyConfigCreatedTime PropertyConfigType = "created_time" - PropertyConfigCreatedBy PropertyConfigType = "created_by" - PropertyConfigLastEditedTime PropertyConfigType = "last_edited_time" - PropertyConfigLastEditedBy PropertyConfigType = "last_edited_by" - PropertyConfigStatus PropertyConfigType = "status" -) -type PropertyObject interface { - GetPropertyType() PropertyConfigType - GetID() string - GetFormat() model.RelationFormat -} - -type Title struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - Title interface{} `json:"title"` -} - -func (t *Title) GetPropertyType() PropertyConfigType { - return PropertyConfigTypeTitle -} - -func (t *Title) GetID() string { - return t.ID -} - -func (t *Title) GetFormat() model.RelationFormat { - return model.RelationFormat_shorttext -} - -type RichText struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - RichText interface{} `json:"rich_text"` -} - -func (rt *RichText) GetPropertyType() PropertyConfigType { - return PropertyConfigTypeRichText -} - -func (rt *RichText) GetID() string { - return rt.ID -} - -func (rt *RichText) GetFormat() model.RelationFormat { - return model.RelationFormat_longtext -} - -type NumberProperty struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - Number interface{} `json:"number"` -} - -func (np *NumberProperty) GetPropertyType() PropertyConfigType { - return PropertyConfigTypeNumber -} - -func (np *NumberProperty) GetID() string { - return np.ID -} - -func (np *NumberProperty) GetFormat() model.RelationFormat { - return model.RelationFormat_number -} - -type SelectProperty struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - Select interface{} `json:"select"` -} - -func (sp *SelectProperty) GetPropertyType() PropertyConfigType { - return PropertyConfigTypeSelect -} - -func (sp *SelectProperty) GetID() string { - return sp.ID -} - -func (sp *SelectProperty) GetFormat() model.RelationFormat { - return model.RelationFormat_tag -} - -type MultiSelect struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - MultiSelect interface{} `json:"multi_select"` -} - -func (ms *MultiSelect) GetPropertyType() PropertyConfigType { - return PropertyConfigTypeMultiSelect -} - -func (ms *MultiSelect) GetID() string { - return ms.ID -} - -func (ms *MultiSelect) GetFormat() model.RelationFormat { - return model.RelationFormat_tag -} - -type DateProperty struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - Date interface{} `json:"date"` -} - -func (dp *DateProperty) GetPropertyType() PropertyConfigType { - return PropertyConfigTypeDate -} - -func (dp *DateProperty) GetID() string { - return dp.ID -} - -func (dp *DateProperty) GetFormat() model.RelationFormat { - return model.RelationFormat_date -} - -type Formula struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - Formula interface{} `json:"formula"` -} - -func (f *Formula) GetPropertyType() PropertyConfigType { - return PropertyConfigTypeFormula -} - -func (f *Formula) GetID() string { - return f.ID -} - -func (f *Formula) GetFormat() model.RelationFormat { - return model.RelationFormat_shorttext -} - -type RelationProperty struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - Relation interface{} `json:"relation"` -} - -func (rp *RelationProperty) GetPropertyType() PropertyConfigType { - return PropertyConfigTypeRelation -} - -func (rp *RelationProperty) GetID() string { - return rp.ID -} - -func (r *RelationProperty) GetFormat() model.RelationFormat { - return model.RelationFormat_object -} - -//can't support it yet -type Rollup struct { - Object string `json:"object"` - ID string `json:"id"` -} - -func (r *Rollup) GetPropertyType() PropertyConfigType { - return PropertyConfigTypeRollup -} - -func (p *Rollup) GetFormat() model.RelationFormat { - return model.RelationFormat_longtext -} - -func (r *Rollup) GetID() string { - return r.ID -} - -type People struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - People interface{} `json:"people"` -} - -func (p *People) GetPropertyType() PropertyConfigType { - return PropertyConfigTypePeople -} - -func (p *People) GetID() string { - return p.ID -} - -func (p *People) GetFormat() model.RelationFormat { - return model.RelationFormat_tag -} - -type File struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - File interface{} `json:"files"` -} - -func (f *File) GetPropertyType() PropertyConfigType { - return PropertyConfigTypeFiles -} - -func (f *File) GetID() string { - return f.ID -} - -func (f *File) GetFormat() model.RelationFormat { - return model.RelationFormat_file -} - -type Checkbox struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - Checkbox interface{} `json:"checkbox"` -} - -func (c *Checkbox) GetPropertyType() PropertyConfigType { - return PropertyConfigTypeCheckbox -} - -func (c *Checkbox) GetID() string { - return c.ID -} - -func (c *Checkbox) GetFormat() model.RelationFormat { - return model.RelationFormat_checkbox -} - -type Url struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - URL interface{} `json:"url"` -} - -func (u *Url) GetPropertyType() PropertyConfigType { - return PropertyConfigTypeURL -} - -func (u *Url) GetID() string { - return u.ID -} - -func (u *Url) GetFormat() model.RelationFormat { - return model.RelationFormat_url -} - -type Email struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - Email interface{} `json:"email"` -} - -func (e *Email) GetPropertyType() PropertyConfigType { - return PropertyConfigTypeURL -} - -func (e *Email) GetID() string { - return e.ID -} - -func (e *Email) GetFormat() model.RelationFormat { - return model.RelationFormat_email -} - -type Phone struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - Phone interface{} `json:"phone_number"` -} - -func (p *Phone) GetPropertyType() PropertyConfigType { - return PropertyConfigTypePhoneNumber -} - -func (p *Phone) GetID() string { - return p.ID -} - -func (p *Phone) GetFormat() model.RelationFormat { - return model.RelationFormat_phone -} - -type CreatedTime struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - CreatedTime interface{} `json:"created_time"` -} - -func (ct *CreatedTime) GetPropertyType() PropertyConfigType { - return PropertyConfigCreatedTime -} - -func (ct *CreatedTime) GetID() string { - return ct.ID -} - -func (ct *CreatedTime) GetFormat() model.RelationFormat { - return model.RelationFormat_date -} - -type CreatedBy struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - CreatedBy interface{} `json:"created_by"` -} - -func (cb *CreatedBy) GetPropertyType() PropertyConfigType { - return PropertyConfigCreatedBy -} - -func (cb *CreatedBy) GetID() string { - return cb.ID -} - -func (cb *CreatedBy) GetFormat() model.RelationFormat { - return model.RelationFormat_shorttext -} - -type LastEditedTime struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - LastEditedTime interface{} `json:"last_edited_time"` -} - -func (le *LastEditedTime) GetPropertyType() PropertyConfigType { - return PropertyConfigLastEditedTime -} - -func (le *LastEditedTime) GetID() string { - return le.ID -} - -func (le *LastEditedTime) GetFormat() model.RelationFormat { - return model.RelationFormat_date -} - -type LastEditedBy struct { - Object string `json:"object"` - ID string `json:"id"` - Type string `json:"type"` - LastEditedBy interface{} `json:"last_edited_by"` -} - -func (lb *LastEditedBy) GetPropertyType() PropertyConfigType { - return PropertyConfigLastEditedBy -} - -func (lb *LastEditedBy) GetID() string { - return lb.ID -} - -func (lb *LastEditedBy) GetFormat() model.RelationFormat { - return model.RelationFormat_shorttext -} - -type StatusProperty struct { - ID string `json:"id"` - Type PropertyConfigType `json:"type"` - Status interface{} `json:"status"` -} - -func (sp *StatusProperty) GetPropertyType() PropertyConfigType { - return PropertyConfigStatus -} - -func (sp *StatusProperty) GetID() string { - return sp.ID -} - -func (sp *StatusProperty) GetFormat() model.RelationFormat { - return model.RelationFormat_status -} diff --git a/core/block/import/notion/api/search/search_test.go b/core/block/import/notion/api/search/search_test.go index 0fed43b33..85bc07713 100644 --- a/core/block/import/notion/api/search/search_test.go +++ b/core/block/import/notion/api/search/search_test.go @@ -6,11 +6,12 @@ import ( "net/http/httptest" "testing" + "github.com/stretchr/testify/assert" + "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/client" "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/database" - "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/page" + // "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/page" "github.com/anytypeio/go-anytype-middleware/pb" - "github.com/stretchr/testify/assert" ) func Test_GetDatabaseSuccess(t *testing.T) { @@ -41,52 +42,273 @@ func Test_GetPagesSuccess(t *testing.T) { s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(` { - "object": "list", - "results": [ - { - "object": "page", - "id": "43b4db4f-23b8-46f9-9909-c783b033fb7d", - "created_time": "2022-10-25T11:44:00.000Z", - "last_edited_time": "2022-11-04T12:00:00.000Z", - "created_by": { - "object": "user", - "id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d" - }, - "last_edited_by": { - "object": "user", - "id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d" - }, - "cover": { - "type": "external", - "external": { - "url": "https://www.notion.so/images/page-cover/nasa_eagle_in_lunar_orbit.jpg" - } - }, - "icon": { - "type": "emoji", - "emoji": "📣" - }, - "parent": { - "type": "database_id", - "database_id": "072a11cb-684f-4f2b-9490-79592700c67e" - }, - "archived": false, - "properties": { - "✔️ Task List": { - "id": "_OI%5E", - "type": "relation", - "relation": [], - "has_more": true - } - }, - "url": "https://www.notion.so/dd-43b4db4f23b846f99909c783b033fb7d" - } - ], - "next_cursor": null, - "has_more": false, - "type": "page_or_database", - "page_or_database": {} - } + "object": "list", + "results": [ + { + "object": "page", + "id": "48cfec01-2e79-4af1-aaec-c1a3a8a95855", + "created_time": "2022-12-06T11:19:00.000Z", + "last_edited_time": "2022-12-07T08:34:00.000Z", + "created_by": { + "object": "user", + "id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d" + }, + "last_edited_by": { + "object": "user", + "id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d" + }, + "cover": null, + "icon": null, + "parent": { + "type": "database_id", + "database_id": "48f51ca6-f1e3-40ee-97a5-953c2e5d8dda" + }, + "archived": false, + "properties": { + "Tags": { + "id": "!'(w", + "type": "multi_select", + "multi_select": [ + { + "id": "00a58cba-c800-40cd-a8f1-6e42527b0a29", + "name": "Special Event", + "color": "yellow" + }, + { + "id": "4322f3ac-635f-4d2f-808f-22d639bc393b", + "name": "Daily", + "color": "purple" + } + ] + }, + "Rollup": { + "id": "%3Df%3E%7B", + "type": "rollup", + "rollup": { + "type": "number", + "number": 2, + "function": "count" + } + }, + "Related Journal 1": { + "id": "%3D%7CO%7B", + "type": "relation", + "relation": [ + { + "id": "088b08d5-b692-4805-8338-1b147a3bff4a" + } + ], + "has_more": false + }, + "Files & media": { + "id": "%3FmtK", + "type": "files", + "files": [ + { + "name": "2022-11-28 11.54.58.jpg", + "type": "file", + "file": { + "url": "", + "expiry_time": "2022-12-07T09:35:05.952Z" + } + } + ] + }, + "Last edited time": { + "id": "%40x%3DJ", + "type": "last_edited_time", + "last_edited_time": "2022-12-07T08:34:00.000Z" + }, + "Number": { + "id": "I%60O%7D", + "type": "number", + "number": null + }, + "Multi-select": { + "id": "M%5Btn", + "type": "multi_select", + "multi_select": [ + { + "id": "49d921f8-44b4-4175-8ae9-c0dc7dd70d76", + "name": "q", + "color": "blue" + }, + { + "id": "55b166d6-7713-4628-b412-560013f6e0ad", + "name": "w", + "color": "brown" + }, + { + "id": "fd5b1266-7c51-4208-83d4-ff3c01efc3b8", + "name": "r", + "color": "pink" + } + ] + }, + "Checkbox": { + "id": "O%5DNd", + "type": "checkbox", + "checkbox": false + }, + "Status": { + "id": "OdD%3A", + "type": "status", + "status": { + "id": "01648775-b1d6-4c21-b093-dab131155840", + "name": "In progress", + "color": "blue" + } + }, + "Created by": { + "id": "WCk%3B", + "type": "created_by", + "created_by": { + "object": "user", + "id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d", + "name": "Anastasia Shemyakinskaya", + "avatar_url": "", + "type": "person", + "person": {} + } + }, + "https://developers.notion.com/": { + "id": "%5BaNB", + "type": "url", + "url": "https://developers.notion.com/" + }, + "Created time": { + "id": "%5C%3B_p", + "type": "created_time", + "created_time": "2022-12-06T11:19:00.000Z" + }, + "Date": { + "id": "%5DIZz", + "type": "date", + "date": { + "start": "2022-12-16", + "end": "2022-12-16", + "time_zone": null + } + }, + "Text": { + "id": "%5DS%3AW", + "type": "rich_text", + "rich_text": [ + { + "type": "text", + "text": { + "content": "sdfsdfsdf", + "link": null + }, + "annotations": { + "bold": false, + "italic": false, + "strikethrough": false, + "underline": false, + "code": false, + "color": "default" + }, + "plain_text": "sdfsdfsdf", + "href": null + } + ] + }, + "Related Journal": { + "id": "d%5DpH", + "type": "relation", + "relation": [ + { + "id": "f90772d0-0155-4ba1-8086-5a9daa750308" + }, + { + "id": "088b08d5-b692-4805-8338-1b147a3bff4a" + } + ], + "has_more": false + }, + "email": { + "id": "ijvk", + "type": "email", + "email": null + }, + "👜 Page": { + "id": "kZi%3D", + "type": "relation", + "relation": [], + "has_more": true + }, + "Checkbox 1": { + "id": "n_gn", + "type": "checkbox", + "checkbox": true + }, + "Last edited by": { + "id": "n%7Biq", + "type": "last_edited_by", + "last_edited_by": { + "object": "user", + "id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d", + "name": "Anastasia Shemyakinskaya", + "avatar_url": "", + "type": "person", + "person": {} + } + }, + "URL": { + "id": "vj%5Dv", + "type": "url", + "url": null + }, + "Phone": { + "id": "wtAo", + "type": "phone_number", + "phone_number": "phone_number" + }, + "Created": { + "id": "%7D%25j%7B", + "type": "created_time", + "created_time": "2022-12-06T11:19:00.000Z" + }, + "Formula": { + "id": "%7DdGa", + "type": "formula", + "formula": { + "type": "string", + "string": "Page" + } + }, + "Name": { + "id": "title", + "type": "title", + "title": [ + { + "type": "text", + "text": { + "content": "Test", + "link": null + }, + "annotations": { + "bold": false, + "italic": false, + "strikethrough": false, + "underline": false, + "code": false, + "color": "default" + }, + "plain_text": "Test", + "href": null + } + ] + } + }, + "url": "https://www.notion.so/" + } + ], + "next_cursor": null, + "has_more": false, + "type": "page_or_database", + "page_or_database": {} +} `)) })) @@ -100,19 +322,6 @@ func Test_GetPagesSuccess(t *testing.T) { assert.NotNil(t, p) assert.Len(t, p, 1) assert.Nil(t, err) - - s = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"object":"list","results":[{"object":"property_item","type":"relation","id":"cm~~","relation":{"id":"18e660df-d7f4-4d4b-b30c-eeb88ffee645"}}],"next_cursor":null,"has_more":false,"type":"property_item","property_item":{"id":"cm~~","next_url":null,"type":"relation","relation":{}}}`)) - })) - - c = client.NewClient() - c.BasePath = s.URL - ps := page.New(c) - pages := ps.GetPages(context.Background(), "key", pb.RpcObjectImportRequest_ALL_OR_NOTHING, p, map[string]string{}, map[string]string{}) - - assert.NotNil(t, pages) - assert.Len(t, pages.Snapshots, 1) - assert.Nil(t, pages.Error) } diff --git a/core/block/import/notion/converter.go b/core/block/import/notion/converter.go index 05e53e519..d7888524c 100644 --- a/core/block/import/notion/converter.go +++ b/core/block/import/notion/converter.go @@ -6,6 +6,7 @@ import ( "time" "github.com/anytypeio/go-anytype-middleware/core/block/import/converter" + "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/block" "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/client" "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/database" "github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/page" @@ -65,7 +66,11 @@ func (n *Notion) GetSnapshots(req *pb.RpcObjectImportRequest) *converter.Respons } } - pagesSnapshots := n.pageService.GetPages(context.TODO(), apiKey, req.Mode, pages, notionIdsToAnytype, databaseNameToID) + request := &block.MapRequest{ + NotionDatabaseIdsToAnytype: notionIdsToAnytype, + DatabaseNameToID: databaseNameToID, + } + pagesSnapshots, notionPageIdsToAnytype := n.pageService.GetPages(context.TODO(), apiKey, req.Mode, pages, request) if pagesSnapshots.Error != nil && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING { ce.Merge(pagesSnapshots.Error) return &converter.Response{ @@ -73,9 +78,12 @@ func (n *Notion) GetSnapshots(req *pb.RpcObjectImportRequest) *converter.Respons } } + page.SetPageLinksInDatabase(databasesSnapshots, pages, databases, notionPageIdsToAnytype, notionIdsToAnytype) + allSnaphots := make([]*converter.Snapshot, 0, len(pagesSnapshots.Snapshots)+len(databasesSnapshots.Snapshots)) allSnaphots = append(allSnaphots, pagesSnapshots.Snapshots...) allSnaphots = append(allSnaphots, databasesSnapshots.Snapshots...) + relations := mergeMaps(databasesSnapshots.Relations, pagesSnapshots.Relations) if pagesSnapshots.Error != nil { ce.Merge(pagesSnapshots.Error) } @@ -85,11 +93,13 @@ func (n *Notion) GetSnapshots(req *pb.RpcObjectImportRequest) *converter.Respons if !ce.IsEmpty() { return &converter.Response{ Snapshots: allSnaphots, + Relations: relations, Error: ce, } } return &converter.Response{ Snapshots: allSnaphots, + Relations: relations, Error: nil, } } @@ -104,3 +114,17 @@ func (n *Notion) getParams(param *pb.RpcObjectImportRequest) string { func (n *Notion) Name() string { return name } + +func mergeMaps(first, second map[string][]*converter.Relation) map[string][]*converter.Relation { + res := make(map[string][]*converter.Relation, 0) + + for pageID, rel := range first { + res[pageID] = rel + } + + for pageID, rel := range second { + res[pageID] = rel + } + + return res +} diff --git a/core/block/import/objectcreator.go b/core/block/import/objectcreator.go index 4ac8c92a8..fd2c812ef 100644 --- a/core/block/import/objectcreator.go +++ b/core/block/import/objectcreator.go @@ -4,13 +4,16 @@ import ( "context" "fmt" + "github.com/gogo/protobuf/types" + "github.com/textileio/go-threads/core/thread" "go.uber.org/zap" "github.com/anytypeio/go-anytype-middleware/core/block" + editor "github.com/anytypeio/go-anytype-middleware/core/block/editor/smartblock" "github.com/anytypeio/go-anytype-middleware/core/block/editor/state" + "github.com/anytypeio/go-anytype-middleware/core/block/import/converter" "github.com/anytypeio/go-anytype-middleware/core/block/import/syncer" "github.com/anytypeio/go-anytype-middleware/core/block/simple" - "github.com/anytypeio/go-anytype-middleware/core/block/simple/relation" "github.com/anytypeio/go-anytype-middleware/core/session" "github.com/anytypeio/go-anytype-middleware/pb" "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" @@ -19,8 +22,6 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" - "github.com/gogo/protobuf/types" - "github.com/textileio/go-threads/core/thread" ) type ObjectCreator struct { @@ -36,7 +37,12 @@ func NewCreator(service block.Service, core core.Service, updater Updater, syncF } // Create creates smart blocks from given snapshots -func (oc *ObjectCreator) Create(ctx *session.Context, snapshot *model.SmartBlockSnapshotBase, pageID string, sbType smartblock.SmartBlockType, updateExisting bool) (*types.Struct, error) { +func (oc *ObjectCreator) Create(ctx *session.Context, + snapshot *model.SmartBlockSnapshotBase, + relations []*converter.Relation, + pageID string, + sbType smartblock.SmartBlockType, + updateExisting bool) (*types.Struct, error) { isFavorite := pbtypes.GetBool(snapshot.Details, bundle.RelationKeyIsFavorite.String()) var err error @@ -68,10 +74,6 @@ func (oc *ObjectCreator) Create(ctx *session.Context, snapshot *model.SmartBlock st.SetLocalDetail(bundle.RelationKeyLastModifiedBy.String(), pbtypes.String(addr.AnytypeProfileId)) st.InjectDerivedDetails() - if err = oc.validate(st); err != nil { - return nil, fmt.Errorf("new id not found for '%s'", st.RootId()) - } - var filesToDelete []string defer func() { // delete file in ipfs if there is error after creation @@ -92,7 +94,9 @@ func (oc *ObjectCreator) Create(ctx *session.Context, snapshot *model.SmartBlock return nil, fmt.Errorf("create object '%s'", st.RootId()) } - filesToDelete, err = oc.relationCreator.Create(ctx, snapshot, newId) + var oldRelationBlocksToNew map[string]*model.Block + filesToDelete, oldRelationBlocksToNew, err = oc.relationCreator.Create(ctx, snapshot, relations, pageID) + if err != nil { return nil, fmt.Errorf("relation create '%s'", err) } @@ -105,6 +109,8 @@ func (oc *ObjectCreator) Create(ctx *session.Context, snapshot *model.SmartBlock } } + oc.replaceRelationBlock(ctx, st, oldRelationBlocksToNew, pageID) + st.Iterate(func(bl simple.Block) (isContinue bool) { s := oc.syncFactory.GetSyncer(bl) if s != nil { @@ -116,27 +122,6 @@ func (oc *ObjectCreator) Create(ctx *session.Context, snapshot *model.SmartBlock return details, nil } -func (oc *ObjectCreator) validate(st *state.State) (err error) { - var relKeys []string - for _, rel := range st.OldExtraRelations() { - if !bundle.HasRelation(rel.Key) { - log.Errorf("builtin objects should not contain custom relations, got %s in %s(%s)", rel.Name, st.RootId(), pbtypes.GetString(st.Details(), bundle.RelationKeyName.String())) - } - } - st.Iterate(func(b simple.Block) (isContinue bool) { - if rb, ok := b.(relation.Block); ok { - relKeys = append(relKeys, rb.Model().GetRelation().Key) - } - return true - }) - for _, rk := range relKeys { - if !st.HasRelation(rk) { - return fmt.Errorf("bundled template validation: relation '%v' exists in block but not in extra relations", rk) - } - } - return nil -} - func (oc *ObjectCreator) createSmartBlock(sbType smartblock.SmartBlockType, st *state.State) (string, *types.Struct, error) { newId, details, err := oc.service.CreateSmartBlockFromState(context.TODO(), sbType, nil, nil, st) if err != nil { @@ -203,3 +188,36 @@ func (oc *ObjectCreator) deleteFile(hash string) { } } } + +func (oc *ObjectCreator) replaceRelationBlock(ctx *session.Context, + st *state.State, + oldRelationBlocksToNew map[string]*model.Block, + pageID string) { + if err := st.Iterate(func(b simple.Block) (isContinue bool) { + if b.Model().GetRelation() == nil { + return true + } + bl, ok := oldRelationBlocksToNew[b.Model().GetId()] + if !ok { + return true + } + if sbErr := oc.service.Do(pageID, func(sb editor.SmartBlock) error { + s := sb.NewStateCtx(ctx) + simpleBlock := simple.New(bl) + s.Add(simpleBlock) + if err := s.InsertTo(b.Model().GetId(), model.Block_Replace, simpleBlock.Model().GetId()); err != nil { + return err + } + if err := sb.Apply(s); err != nil { + return err + } + return nil + }); sbErr != nil { + log.With(zap.String("object id", pageID)).Errorf("failed to replace relation block: %w", sbErr) + } + + return true + }); err != nil { + log.With(zap.String("object id", pageID)).Errorf("failed to replace relation block: %w", err) + } +} diff --git a/core/block/import/relationcreator.go b/core/block/import/relationcreator.go index 0bac505d2..b4749bc5b 100644 --- a/core/block/import/relationcreator.go +++ b/core/block/import/relationcreator.go @@ -3,14 +3,17 @@ package importer import ( "strings" + "github.com/globalsign/mgo/bson" + "github.com/gogo/protobuf/types" + "github.com/anytypeio/go-anytype-middleware/core/block" "github.com/anytypeio/go-anytype-middleware/core/block/editor" + "github.com/anytypeio/go-anytype-middleware/core/block/import/converter" "github.com/anytypeio/go-anytype-middleware/core/session" "github.com/anytypeio/go-anytype-middleware/pb" "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" - "github.com/gogo/protobuf/types" ) type RelationService struct { @@ -26,17 +29,22 @@ func NewRelationCreator(service block.Service) RelationCreator { // Create read relations link from snaphot and create according relations in anytype, also set details for according relation in object // for files it loads them in ipfs -func (rc *RelationService) Create(ctx *session.Context, snapshot *model.SmartBlockSnapshotBase, pageID string) ([]string, error) { +func (rc *RelationService) Create(ctx *session.Context, + snapshot *model.SmartBlockSnapshotBase, + relations []*converter.Relation, + pageID string) ([]string, map[string]*model.Block, error) { var ( - object *types.Struct - relationID string - err error - filesToDelete = make([]string, 0) + object *types.Struct + relationID string + err error + filesToDelete = make([]string, 0) + oldRelationBlockToNew = make(map[string]*model.Block, 0) ) - for _, r := range snapshot.RelationLinks { + + for _, r := range relations { detail := &types.Struct{ Fields: map[string]*types.Value{ - bundle.RelationKeyName.String(): pbtypes.String(r.Key), + bundle.RelationKeyName.String(): pbtypes.String(r.Name), bundle.RelationKeyRelationFormat.String(): pbtypes.Int64(int64(r.Format)), }, } @@ -45,7 +53,7 @@ func (rc *RelationService) Create(ctx *session.Context, snapshot *model.SmartBlo continue } - if object != nil && object.Fields != nil &&object.Fields[bundle.RelationKeyRelationKey.String()] != nil { + if object != nil && object.Fields != nil && object.Fields[bundle.RelationKeyRelationKey.String()] != nil { relationID = object.Fields[bundle.RelationKeyRelationKey.String()].GetStringValue() } else { continue @@ -57,7 +65,7 @@ func (rc *RelationService) Create(ctx *session.Context, snapshot *model.SmartBlo } if snapshot.Details != nil && snapshot.Details.Fields != nil && object != nil { - if snapshot.Details.Fields[r.Key].GetListValue() != nil { + if snapshot.Details.Fields[r.Name].GetListValue() != nil { rc.handleListValue(ctx, snapshot, r, relationID) } @@ -67,36 +75,42 @@ func (rc *RelationService) Create(ctx *session.Context, snapshot *model.SmartBlo details := make([]*pb.RpcObjectSetDetailsDetail, 0) details = append(details, &pb.RpcObjectSetDetailsDetail{ Key: relationID, - Value: snapshot.Details.Fields[r.Key], + Value: snapshot.Details.Fields[r.Name], }) err = rc.service.SetDetails(ctx, pb.RpcObjectSetDetailsRequest{ ContextId: pageID, - Details: details, + Details: details, }) if err != nil { log.Errorf("set details %s", err) continue } } + if r.BlockID != "" { + original, new := rc.linkRelationsBlocks(snapshot, r.BlockID, relationID) + if original != nil && new != nil { + oldRelationBlockToNew[original.GetId()] = new + } + } } - return filesToDelete, nil + return filesToDelete, oldRelationBlockToNew, nil } -func (rc *RelationService) handleListValue(ctx *session.Context, snapshot *model.SmartBlockSnapshotBase, r *model.RelationLink, relationID string) { - var( +func (rc *RelationService) handleListValue(ctx *session.Context, snapshot *model.SmartBlockSnapshotBase, r *converter.Relation, relationID string) { + var ( optionsIds = make([]string, 0) - id string - err error + id string + err error ) - for _, v := range snapshot.Details.Fields[r.Key].GetListValue().Values { + for _, v := range snapshot.Details.Fields[r.Name].GetListValue().Values { if r.Format == model.RelationFormat_tag || r.Format == model.RelationFormat_status { if id, _, err = rc.service.CreateRelationOption(&types.Struct{ - Fields: map[string]*types.Value{ - bundle.RelationKeyName.String(): pbtypes.String(v.GetStringValue()), - bundle.RelationKeyRelationKey.String(): pbtypes.String(relationID), - bundle.RelationKeyType.String(): pbtypes.String(bundle.TypeKeyRelationOption.URL()), - bundle.RelationKeyLayout.String(): pbtypes.Float64(float64(model.ObjectType_relationOption)), - }, + Fields: map[string]*types.Value{ + bundle.RelationKeyName.String(): pbtypes.String(v.GetStringValue()), + bundle.RelationKeyRelationKey.String(): pbtypes.String(relationID), + bundle.RelationKeyType.String(): pbtypes.String(bundle.TypeKeyRelationOption.URL()), + bundle.RelationKeyLayout.String(): pbtypes.Float64(float64(model.ObjectType_relationOption)), + }, }); err != nil { log.Errorf("add extra relation %s", err) } @@ -105,30 +119,51 @@ func (rc *RelationService) handleListValue(ctx *session.Context, snapshot *model } optionsIds = append(optionsIds, id) } - snapshot.Details.Fields[r.Key] = pbtypes.StringList(optionsIds) + snapshot.Details.Fields[r.Name] = pbtypes.StringList(optionsIds) } -func (rc *RelationService) handleFileRelation(ctx *session.Context, snapshot *model.SmartBlockSnapshotBase, r *model.RelationLink, filesToDelete []string) { - if files := snapshot.Details.Fields[r.Key].GetListValue(); files != nil { - allFilesHashes := make([]string, 0) - for _, f := range files.Values { - file := f.GetStringValue() - if file != "" { - req := pb.RpcFileUploadRequest{LocalPath: file} - if strings.HasPrefix(file, "http://") || strings.HasPrefix(file, "https://") { - req.Url = file - req.LocalPath = "" - } - hash, err := rc.service.UploadFile(req) - if err != nil { - log.Errorf("file uploading %s", err) - } else { - file = hash - } - filesToDelete = append(filesToDelete, file) - allFilesHashes = append(allFilesHashes, file) - } - } - snapshot.Details.Fields[r.Key] = pbtypes.StringList(allFilesHashes) +func (rc *RelationService) handleFileRelation(ctx *session.Context, + snapshot *model.SmartBlockSnapshotBase, + r *converter.Relation, + filesToDelete []string) { + files := snapshot.Details.Fields[r.Name].GetListValue() + + if files == nil { + return } -} \ No newline at end of file + allFilesHashes := make([]string, 0) + for _, f := range files.Values { + file := f.GetStringValue() + if file != "" { + req := pb.RpcFileUploadRequest{LocalPath: file} + if strings.HasPrefix(file, "http://") || strings.HasPrefix(file, "https://") { + req.Url = file + req.LocalPath = "" + } + hash, err := rc.service.UploadFile(req) + if err != nil { + log.Errorf("file uploading %s", err) + } else { + file = hash + } + filesToDelete = append(filesToDelete, file) + allFilesHashes = append(allFilesHashes, file) + } + } + snapshot.Details.Fields[r.Name] = pbtypes.StringList(allFilesHashes) +} + +func (rc *RelationService) linkRelationsBlocks(snapshot *model.SmartBlockSnapshotBase, oldID, newID string) (*model.Block, *model.Block) { + for _, b := range snapshot.Blocks { + if rel, ok := b.Content.(*model.BlockContentOfRelation); ok && rel.Relation.GetKey() == oldID { + return b, &model.Block{ + Id: bson.NewObjectId().Hex(), + Content: &model.BlockContentOfRelation{ + Relation: &model.BlockContentRelation{ + Key: newID, + }, + }} + } + } + return nil, nil +} diff --git a/core/block/import/types.go b/core/block/import/types.go index 782456867..f5617803a 100644 --- a/core/block/import/types.go +++ b/core/block/import/types.go @@ -2,6 +2,7 @@ package importer import ( "github.com/anytypeio/go-anytype-middleware/app" + "github.com/anytypeio/go-anytype-middleware/core/block/import/converter" _ "github.com/anytypeio/go-anytype-middleware/core/block/import/markdown" _ "github.com/anytypeio/go-anytype-middleware/core/block/import/notion" _ "github.com/anytypeio/go-anytype-middleware/core/block/import/pb" @@ -22,7 +23,8 @@ type Importer interface { // Creator incapsulate logic with creation of given smartblocks type Creator interface { - Create(ctx *session.Context, cs *model.SmartBlockSnapshotBase, pageID string, sbType smartblock.SmartBlockType, updateExisting bool) (*types.Struct, error) + //nolint: lll + Create(ctx *session.Context, cs *model.SmartBlockSnapshotBase, relations []*converter.Relation, pageID string, sbType smartblock.SmartBlockType, updateExisting bool) (*types.Struct, error) } // Updater is interface for updating existing objects @@ -32,5 +34,6 @@ type Updater interface { // RelationCreator incapsulates logic for creation of relations type RelationCreator interface { - Create(ctx *session.Context, snapshot *model.SmartBlockSnapshotBase, pageID string) ([]string, error) + //nolint: lll + Create(ctx *session.Context, snapshot *model.SmartBlockSnapshotBase, relations []*converter.Relation, pageID string) ([]string, map[string]*model.Block, error) }