Merge branch 'master' of github.com:anytypeio/go-anytype-middleware into go-551-remove-unused-code

# Conflicts:
#	core/block/editor/import/import.go
#	pb/commands.pb.go
#	pkg/lib/core/core.go
#	util/uri/uri.go

.github/workflows/build.yml

@@ -38,7 +38,10 @@ jobs:
           curl https://raw.githubusercontent.com/Homebrew/homebrew-core/31b24d65a7210ea0a5689d5ad00dd8d1bf5211db/Formula/protobuf.rb --output protobuf.rb
           HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 brew install ./protobuf.rb
           HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 brew install --ignore-dependencies swift-protobuf
-          HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 brew install mingw-w64 jq FiloSottile/musl-cross/musl-cross
+          HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 brew install mingw-w64
+          brew tap filosottile/musl-cross
+          brew install filosottile/musl-cross/musl-cross -h
+          brew install filosottile/musl-cross/musl-cross --with-aarch64
           npm i -g node-gyp
       - name: Checkout
         uses: actions/checkout@v2
@@ -72,9 +75,11 @@ jobs:
         run: |
           echo $FLAGS
           mkdir -p .release
-          gox -ldflags="$FLAGS" -osarch="darwin/amd64 darwin/arm64 linux/amd64" --tags="nographviz nowatchdog nosigar nomutexdeadlockdetector" -output="{{.OS}}-{{.Arch}}" github.com/anytypeio/go-anytype-middleware/cmd/grpcserver
+          gox -cgo -ldflags="$FLAGS" -osarch="darwin/amd64 darwin/arm64" --tags="nographviz nowatchdog nosigar nomutexdeadlockdetector" -output="{{.OS}}-{{.Arch}}" github.com/anytypeio/go-anytype-middleware/cmd/grpcserver
+          CC="x86_64-linux-musl-gcc" CXX="x86_64-linux-musl-g++" gox -cgo -osarch="linux/amd64" --tags="nographviz nowatchdog nosigar nomutexdeadlockdetector" -output="{{.OS}}-{{.Arch}}" github.com/anytypeio/go-anytype-middleware/cmd/grpcserver
+          CC="aarch64-linux-musl-gcc" CXX="aarch64-linux-musl-g++" gox -cgo -osarch="linux/arm64" --tags="nographviz nowatchdog nosigar nomutexdeadlockdetector" -output="{{.OS}}-{{.Arch}}" github.com/anytypeio/go-anytype-middleware/cmd/grpcserver
           make protos-server
-          CC="x86_64-w64-mingw32-gcc" CXX="x86_64-w64-mingw32-g++" gox -ldflags="$FLAGS" -osarch="windows/amd64" --tags="nographviz nowatchdog nosigar nomutexdeadlockdetector" -output="{{.OS}}-{{.Arch}}" github.com/anytypeio/go-anytype-middleware/cmd/grpcserver
+          CC="x86_64-w64-mingw32-gcc" CXX="x86_64-w64-mingw32-g++" gox -cgo -ldflags="$FLAGS" -osarch="windows/amd64" --tags="nographviz nowatchdog nosigar nomutexdeadlockdetector" -output="{{.OS}}-{{.Arch}}" github.com/anytypeio/go-anytype-middleware/cmd/grpcserver
           ls -lha .
       - name: Make JS protos
         run: |

.github/workflows/golangci-lint.yml

@@ -19,4 +19,4 @@ jobs:
         with:
           version: latest
           only-new-issues: true
-          args: --timeout 15m
+          args: --timeout 15m --skip-files ".*_test.go"

.golangci.yml

@@ -1,11 +1,10 @@
-un:
+run:
   deadline: 15m
   timeout:  15m
   # didn't run linter on tests
   tests: false
   # don't check generated protobuf files
   skip-dirs:
-    - pb/external_libs
     - pkg/lib/pb
   go: '1.18'
 

Makefile

@@ -248,18 +248,19 @@ build-js: setup-go build-server protos-js
 	@echo "Run 'make install-dev-js' instead if you want to build&install into ../js-anytype"
 
 install-linter:
+	@go install github.com/daixiang0/gci@latest
 	@go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
 
 run-linter:
 ifdef GOLANGCI_LINT_BRANCH
-	@golangci-lint run -v ./... --new-from-rev=$(GOLANGCI_LINT_BRANCH) --timeout 15m
+	@golangci-lint run -v ./... --new-from-rev=$(GOLANGCI_LINT_BRANCH) --skip-files ".*_test.go" --timeout 15m
 else 
-	@golangci-lint run -v ./... --new-from-rev=master --timeout 15m
+	@golangci-lint run -v ./... --new-from-rev=master --skip-files ".*_test.go" --timeout 15m
 endif
 
 run-linter-fix:
 ifdef GOLANGCI_LINT_BRANCH
-	@golangci-lint run -v ./... --new-from-rev=$(GOLANGCI_LINT_BRANCH) --timeout 15m --fix
+	@golangci-lint run -v ./... --new-from-rev=$(GOLANGCI_LINT_BRANCH) --skip-files ".*_test.go" --timeout 15m --fix
 else 
-	@golangci-lint run -v ./... --new-from-rev=master --timeout 15m --fix
+	@golangci-lint run -v ./... --new-from-rev=master --skip-files ".*_test.go" --timeout 15m --fix
 endif

change/change_test.go

@@ -88,7 +88,7 @@ func Test_Issue605Tree(t *testing.T) {
 	}
 	doc := state.NewDocFromSnapshot("bafyba6akblqzapgdmlu3swkrsb62mgrvgdv2g72eqvtufuis4vxhgcgl", root.GetSnapshot()).(*state.State)
 	doc.SetChangeId(root.Id)
-	st, err := BuildStateSimpleCRDT(doc, tree)
+	st, _, err := BuildStateSimpleCRDT(doc, tree)
 	if err != nil {
 		return
 	}
@@ -132,7 +132,7 @@ func Test_Home_ecz5pu(t *testing.T) {
 	doc := state.NewDocFromSnapshot("", root.GetSnapshot()).(*state.State)
 	doc.SetChangeId(root.Id)
 	doc.RootId()
-	st, err := BuildStateSimpleCRDT(doc, tree)
+	st, _, err := BuildStateSimpleCRDT(doc, tree)
 	if err != nil {
 		return
 	}

change/state.go

@@ -41,13 +41,12 @@ func (sc *stateCache) Get(id string) *state.State {
 }
 
 // Simple implementation hopes for CRDT and ignores errors. No merge
-func BuildStateSimpleCRDT(root *state.State, t *Tree) (s *state.State, err error) {
+func BuildStateSimpleCRDT(root *state.State, t *Tree) (s *state.State, changesApplied int, err error) {
 	var (
 		startId    string
 		applyRoot  bool
 		st         = time.Now()
 		lastChange *Change
-		count      int
 	)
 	if startId = root.ChangeId(); startId == "" {
 		startId = t.RootId()
@@ -55,7 +54,7 @@ func BuildStateSimpleCRDT(root *state.State, t *Tree) (s *state.State, err error
 	}
 
 	t.Iterate(startId, func(c *Change) (isContinue bool) {
-		count++
+		changesApplied++
 		lastChange = c
 		if startId == c.Id {
 			s = root.NewState()
@@ -77,12 +76,12 @@ func BuildStateSimpleCRDT(root *state.State, t *Tree) (s *state.State, err error
 		return true
 	})
 	if err != nil {
-		return nil, err
+		return nil, changesApplied, err
 	}
 	if lastChange != nil {
 		s.SetLastModified(lastChange.Timestamp, lastChange.Account)
 	}
 
-	log.Infof("build state (crdt): changes: %d; dur: %v;", count, time.Since(st))
-	return s, err
+	log.Infof("build state (crdt): changes: %d; dur: %v;", changesApplied, time.Since(st))
+	return s, changesApplied, err
 }

change/state_test.go

@@ -43,7 +43,7 @@ func BenchmarkOpenDoc(b *testing.B) {
 	root := tree.Root()
 	doc := state.NewDocFromSnapshot("bafybapt3aap3tmkbs7mkj5jao3vhjblijkiwqq37wxlylx5nn7cqokgk", root.GetSnapshot()).(*state.State)
 	doc.SetChangeId(root.Id)
-	_, err = BuildStateSimpleCRDT(doc, tree)
+	_, _, err = BuildStateSimpleCRDT(doc, tree)
 	require.NoError(b, err)
 	b.Log("build state:", time.Since(st))
 
@@ -57,7 +57,7 @@ func BenchmarkOpenDoc(b *testing.B) {
 		for i := 0; i < b.N; i++ {
 			doc := state.NewDocFromSnapshot("bafybapt3aap3tmkbs7mkj5jao3vhjblijkiwqq37wxlylx5nn7cqokgk", root.GetSnapshot()).(*state.State)
 			doc.SetChangeId(root.Id)
-			_, err := BuildStateSimpleCRDT(doc, tree)
+			_, _, err := BuildStateSimpleCRDT(doc, tree)
 			require.NoError(b, err)
 		}
 	})

change/tree.go

@@ -6,6 +6,7 @@ import (
 	"crypto/md5"
 	"fmt"
 	"sort"
+	"time"
 
 	"github.com/anytypeio/go-anytype-middleware/util/slice"
 )
@@ -64,7 +65,7 @@ func (t *Tree) AddFast(changes ...*Change) {
 		}
 		t.add(c)
 	}
-	t.updateHeads()
+	t.updateHeads(changes)
 }
 
 func (t *Tree) Add(changes ...*Change) (mode Mode) {
@@ -85,7 +86,7 @@ func (t *Tree) Add(changes ...*Change) (mode Mode) {
 	if !attached {
 		return Nothing
 	}
-	t.updateHeads()
+	t.updateHeads(changes)
 	if empty {
 		return Rebuild
 	}
@@ -184,24 +185,54 @@ func (t *Tree) after(id1, id2 string) (found bool) {
 	return
 }
 
-func (t *Tree) updateHeads() {
-	var newHeadIds, newMetaHeadIds []string
+func (t *Tree) recalculateHeads() (heads []string, metaHeads []string) {
+	start := time.Now()
+	total := 0
 	t.iterate(t.root, func(c *Change) (isContinue bool) {
+		total++
 		if len(c.Next) == 0 {
-			newHeadIds = append(newHeadIds, c.Id)
+			heads = append(heads, c.Id)
 		}
 		if c.HasMeta() {
 			for _, prevDetId := range c.PreviousMetaIds {
-				newMetaHeadIds = slice.Remove(newMetaHeadIds, prevDetId)
+				metaHeads = slice.Remove(metaHeads, prevDetId)
 			}
-			newMetaHeadIds = append(newMetaHeadIds, c.Id)
+			metaHeads = append(metaHeads, c.Id)
 		}
 		return true
 	})
-	t.headIds = newHeadIds
-	t.metaHeadIds = newMetaHeadIds
-	sort.Strings(t.headIds)
-	sort.Strings(t.metaHeadIds)
+	if time.Since(start) > time.Millisecond*100 {
+		log.Errorf("recalculateHeads took %s for %d changes", time.Since(start), total)
+	}
+
+	return
+}
+
+func (t *Tree) updateHeads(chs []*Change) {
+	var newHeadIds, newMetaHeadIds []string
+	if len(chs) == 1 && slice.UnsortedEquals(chs[0].PreviousIds, t.headIds) {
+		// shortcut when adding to the top of the tree
+		// only cover edge case when adding one change, otherwise it's not worth it
+		newHeadIds = []string{chs[0].Id}
+	}
+	if len(chs) == 1 && chs[0].HasMeta() && slice.UnsortedEquals(chs[0].PreviousMetaIds, t.metaHeadIds) {
+		// shortcut when adding to the top of the tree
+		// only cover edge case when adding one change, otherwise it's not worth it
+		newMetaHeadIds = []string{chs[0].Id}
+	}
+
+	if newHeadIds == nil {
+		newHeadIds, newMetaHeadIds = t.recalculateHeads()
+	}
+	if newHeadIds != nil {
+		t.headIds = newHeadIds
+		sort.Strings(t.headIds)
+	}
+
+	if newMetaHeadIds != nil {
+		t.metaHeadIds = newMetaHeadIds
+		sort.Strings(t.metaHeadIds)
+	}
 }
 
 func (t *Tree) iterate(start *Change, f func(c *Change) (isContinue bool)) {

core/block/editor.go

@@ -485,7 +485,12 @@ func (s *Service) UploadFile(req pb.RpcFileUploadRequest) (hash string, err erro
 	} else {
 		upl.AutoType(true)
 	}
-	res := upl.SetFile(req.LocalPath).Upload(context.TODO())
+	if req.LocalPath != "" {
+		upl.SetFile(req.LocalPath)
+	} else if req.Url != "" {
+		upl.SetUrl(req.Url)
+	}
+	res := upl.Upload(context.TODO())
 	if res.Err != nil {
 		return "", res.Err
 	}

core/block/editor/bookmark/bookmark.go

@@ -74,7 +74,7 @@ func (b *sbookmark) fetch(s *state.State, id, url string, isSync bool) (err erro
 	if b == nil {
 		return smartblock.ErrSimpleBlockNotFound
 	}
-	url, err = uri.ProcessURI(url)
+	url, err = uri.NormalizeURI(url)
 	if err != nil {
 		// Do nothing
 	}

core/block/editor/file/uploader.go

@@ -14,13 +14,14 @@ import (
 	"strings"
 	"time"
 
+	"github.com/h2non/filetype"
+
 	"github.com/anytypeio/go-anytype-middleware/core/block/simple"
 	"github.com/anytypeio/go-anytype-middleware/core/block/simple/file"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/core"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/files"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
 	"github.com/anytypeio/go-anytype-middleware/util/uri"
-	"github.com/h2non/filetype"
 )
 
 var (
@@ -175,7 +176,10 @@ func (u *uploader) AddOptions(options ...files.AddOption) Uploader {
 }
 
 func (u *uploader) SetUrl(url string) Uploader {
-	url, _ = uri.ProcessURI(url)
+	url, err := uri.NormalizeURI(url)
+	if err != nil {
+		// do nothing
+	}
 	u.name = strings.Split(filepath.Base(url), "?")[0]
 	u.getReader = func(ctx context.Context) (*fileReader, error) {
 		req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
@@ -183,7 +187,11 @@ func (u *uploader) SetUrl(url string) Uploader {
 			return nil, err
 		}
 
-		resp, err := http.DefaultClient.Do(req)
+		// setting timeout to avoid locking for a long time
+		cl := http.DefaultClient
+		cl.Timeout = time.Second * 20
+
+		resp, err := cl.Do(req)
 		if err != nil {
 			return nil, err
 		}

core/block/editor/subobjectcollection.go

@@ -150,8 +150,7 @@ func (c *SubObjectCollection) removeObject(st *state.State, objectId string) (er
 		return err
 	}
 	if len(links) > 0 {
-		// todo: return the error to user?
-		log.Errorf("workspace removeObject: found inbound links: %v", links)
+		log.With("id", objectId).With("total", len(links)).Debugf("workspace removeObject: found inbound links: %v", links)
 	}
 	st.RemoveFromStore([]string{collection, key})
 	if v, exists := c.collections[collection]; exists {

core/block/editor/table/block.go

@@ -129,7 +129,7 @@ func (b *block) Duplicate(s *state.State) (newID string, visitedIds []string, bl
 			visitedIds = append(visitedIds, cellID)
 			cell := s.Pick(cellID)
 			cell = cell.Copy()
-			cell.Model().Id = makeCellID(row.Model().Id, newColID)
+			cell.Model().Id = MakeCellID(row.Model().Id, newColID)
 			blocks = append(blocks, cell)
 
 			row.Model().ChildrenIds[j] = cell.Model().Id

core/block/editor/table/table.go

@@ -304,7 +304,7 @@ func (t *Editor) RowDuplicate(s *state.State, req pb.RpcBlockTableRowDuplicateRe
 		}
 
 		newCell := cell.Copy()
-		newCell.Model().Id = makeCellID(newRow.Model().Id, colID)
+		newCell.Model().Id = MakeCellID(newRow.Model().Id, colID)
 		if !s.Add(newCell) {
 			return "", fmt.Errorf("add new cell %s", newCell.Model().Id)
 		}
@@ -332,10 +332,10 @@ func (t *Editor) RowListFill(s *state.State, req pb.RpcBlockTableRowListFillRequ
 			return fmt.Errorf("get row %s: %w", rowID, err)
 		}
 
-		newIDs := make([]string, 0, len(columns))
+		newIds := make([]string, 0, len(columns))
 		for _, colID := range columns {
-			id := makeCellID(rowID, colID)
-			newIDs = append(newIDs, id)
+			id := MakeCellID(rowID, colID)
+			newIds = append(newIds, id)
 
 			if !s.Exists(id) {
 				_, err := addCell(s, rowID, colID)
@@ -344,7 +344,7 @@ func (t *Editor) RowListFill(s *state.State, req pb.RpcBlockTableRowListFillRequ
 				}
 			}
 		}
-		row.Model().ChildrenIds = newIDs
+		row.Model().ChildrenIds = newIds
 	}
 	return nil
 }
@@ -407,7 +407,7 @@ func (t *Editor) ColumnListFill(s *state.State, req pb.RpcBlockTableColumnListFi
 
 	for _, colID := range req.BlockIds {
 		for _, rowID := range rows {
-			id := makeCellID(rowID, colID)
+			id := MakeCellID(rowID, colID)
 			if s.Exists(id) {
 				continue
 			}
@@ -531,7 +531,7 @@ func (t *Editor) cloneColumnStyles(s *state.State, srcColID, targetColID string)
 		}
 
 		if protoBlock != nil && protoBlock.Model().BackgroundColor != "" {
-			targetCellID := makeCellID(rowID, targetColID)
+			targetCellID := MakeCellID(rowID, targetColID)
 
 			if !s.Exists(targetCellID) {
 				_, err := addCell(s, rowID, targetColID)
@@ -613,7 +613,7 @@ func (t *Editor) ColumnDuplicate(s *state.State, req pb.RpcBlockTableColumnDupli
 			return "", fmt.Errorf("cell %s is not found", cellID)
 		}
 		cell = cell.Copy()
-		cell.Model().Id = makeCellID(rowID, newCol.Model().Id)
+		cell.Model().Id = MakeCellID(rowID, newCol.Model().Id)
 
 		if !s.Add(cell) {
 			return "", fmt.Errorf("add cell block")
@@ -826,7 +826,7 @@ func pickColumn(s *state.State, id string) (simple.Block, error) {
 	return b, nil
 }
 
-func makeCellID(rowID, colID string) string {
+func MakeCellID(rowID, colID string) string {
 	return fmt.Sprintf("%s-%s", rowID, colID)
 }
 
@@ -840,7 +840,7 @@ func ParseCellID(id string) (rowID string, colID string, err error) {
 
 func addCell(s *state.State, rowID, colID string) (string, error) {
 	c := simple.New(&model.Block{
-		Id: makeCellID(rowID, colID),
+		Id: MakeCellID(rowID, colID),
 		Content: &model.BlockContentOfText{
 			Text: &model.BlockContentText{},
 		},

core/block/editor/workspaces.go

@@ -841,14 +841,159 @@ func (w *Workspaces) CreateSubObjects(details []*types.Struct) (ids []string, ob
 	return
 }
 
-func (w *Workspaces) RemoveSubObjects(objectIds []string) (err error) {
+// objectTypeRelationsForGC returns the list of relation IDs that are safe to remove alongside with the provided object type
+// - they were installed from the marketplace(not custom by the user)
+// - they are not used as recommended in other installed/custom object types
+// - they are not used directly in some object
+func (w *Workspaces) objectTypeRelationsForGC(objectTypeID string) (ids []string, err error) {
+	obj, err := w.objectStore.GetDetails(objectTypeID)
+	if err != nil {
+		return nil, err
+	}
+
+	source := pbtypes.GetString(obj.Details, bundle.RelationKeySourceObject.String())
+	if source == "" {
+		// type was not installed from marketplace
+		return nil, nil
+	}
+
+	var skipIDs = map[string]struct{}{}
+	for _, rel := range bundle.SystemRelations {
+		skipIDs[addr.RelationKeyToIdPrefix+rel.String()] = struct{}{}
+	}
+
+	relIds := pbtypes.GetStringList(obj.Details, bundle.RelationKeyRecommendedRelations.String())
+
+	// find relations that are custom(was not installed from somewhere)
+	records, _, err := w.objectStore.Query(nil, database2.Query{
+		Filters: []*model.BlockContentDataviewFilter{
+			{
+				RelationKey: bundle.RelationKeyId.String(),
+				Condition:   model.BlockContentDataviewFilter_In,
+				Value:       pbtypes.StringList(relIds),
+			},
+			{
+				RelationKey: bundle.RelationKeySourceObject.String(),
+				Condition:   model.BlockContentDataviewFilter_Empty,
+			},
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, rec := range records {
+		skipIDs[pbtypes.GetString(rec.Details, bundle.RelationKeyId.String())] = struct{}{}
+	}
+
+	// check if this relation is used in some other installed object types
+	records, _, err = w.objectStore.Query(nil, database2.Query{
+		Filters: []*model.BlockContentDataviewFilter{
+			{
+				RelationKey: bundle.RelationKeyType.String(),
+				Condition:   model.BlockContentDataviewFilter_Equal,
+				Value:       pbtypes.String(bundle.TypeKeyObjectType.URL()),
+			},
+			{
+				RelationKey: bundle.RelationKeyRecommendedRelations.String(),
+				Condition:   model.BlockContentDataviewFilter_In,
+				Value:       pbtypes.StringList(relIds),
+			},
+			{
+				RelationKey: bundle.RelationKeyWorkspaceId.String(),
+				Condition:   model.BlockContentDataviewFilter_Equal,
+				Value:       pbtypes.String(w.Id()),
+			},
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, rec := range records {
+		recId := pbtypes.GetString(rec.Details, bundle.RelationKeyId.String())
+		if recId == objectTypeID {
+			continue
+		}
+		rels := pbtypes.GetStringList(rec.Details, bundle.RelationKeyRecommendedRelations.String())
+		for _, rel := range rels {
+			if slice.FindPos(relIds, rel) > -1 {
+				skipIDs[rel] = struct{}{}
+			}
+		}
+	}
+
+	for _, relId := range relIds {
+		if _, exists := skipIDs[relId]; exists {
+			continue
+		}
+		relKey, err := pbtypes.RelationIdToKey(relId)
+		if err != nil {
+			log.Errorf("failed to get relation key from id %s: %s", relId, err.Error())
+			continue
+		}
+		records, _, err := w.objectStore.Query(nil, database2.Query{
+			Limit: 1,
+			Filters: []*model.BlockContentDataviewFilter{
+				{
+					// exclude installed templates that we don't remove yet and they may depend on the relation
+					RelationKey: bundle.RelationKeyTargetObjectType.String(),
+					Condition:   model.BlockContentDataviewFilter_NotEqual,
+					Value:       pbtypes.String(objectTypeID),
+				},
+				{
+					RelationKey: bundle.RelationKeyWorkspaceId.String(),
+					Condition:   model.BlockContentDataviewFilter_Equal,
+					Value:       pbtypes.String(w.Id()),
+				},
+				{
+					RelationKey: relKey,
+					Condition:   model.BlockContentDataviewFilter_NotEmpty,
+				},
+			},
+		})
+		if len(records) > 0 {
+			skipIDs[relId] = struct{}{}
+		}
+	}
+	return slice.Filter(relIds, func(s string) bool {
+		_, exists := skipIDs[s]
+		return !exists
+	}), nil
+}
+
+// RemoveSubObjects removes sub objects from the workspace collection
+// if orphansGC is true, then relations that are not used by any object in the workspace will be removed as well
+func (w *Workspaces) RemoveSubObjects(objectIds []string, orphansGC bool) (err error) {
 	st := w.NewState()
 	for _, id := range objectIds {
+		// special case for object types
+		var idsToRemove []string
+		if strings.HasPrefix(id, addr.ObjectTypeKeyToIdPrefix) && orphansGC {
+			idsToRemove, err = w.objectTypeRelationsForGC(id)
+			if err != nil {
+				log.Errorf("objectTypeRelationsForGC failed: %s", err.Error())
+				continue
+			}
+			if len(idsToRemove) > 0 {
+				log.Debugf("objectTypeRelationsForGC, relations to remove: %v", idsToRemove)
+			}
+		}
+
 		err = w.removeObject(st, id)
 		if err != nil {
 			log.Errorf("failed to remove sub object: %s", err.Error())
 			continue
 		}
+		if orphansGC && len(idsToRemove) > 0 {
+			for _, relId := range idsToRemove {
+				err = w.removeObject(st, relId)
+				if err != nil {
+					log.Errorf("failed to remove dependent sub object: %s", err.Error())
+					continue
+				}
+			}
+		}
 	}
 
 	// reset error in case we have at least 1 object created

core/block/import/converter/error.go

@@ -11,6 +11,14 @@ func NewError() ConvertError {
 	return ConvertError{}
 }
 
+func NewFromError(name string, initialError error) ConvertError {
+	ce := ConvertError{}
+
+	ce.Add(name, initialError)
+
+	return ce
+}
+
 func (ce ConvertError) Add(objectName string, err error) {
 	ce[objectName] = err
 }
@@ -26,17 +34,17 @@ func (ce ConvertError) IsEmpty() bool {
 }
 
 func (ce ConvertError) Error() error {
-	var pattern = "file: %s, error: %s" + "\n"
+	var pattern = "source: %s, error: %s" + "\n"
 	var errorString bytes.Buffer
 	if ce.IsEmpty() {
 		return nil
 	}
 	for name, err := range ce {
-		errorString.WriteString(fmt.Sprintf(pattern, name, err))
+		errorString.WriteString(fmt.Sprintf(pattern, name, err.Error()))
 	}
 	return fmt.Errorf(errorString.String())
 }
 
 func (ce ConvertError) Get(objectName string) error {
 	return ce[objectName]
-}
+}

core/block/import/converter/types.go

@@ -3,6 +3,7 @@ package converter
 import (
 	"io"
 
+	"github.com/anytypeio/go-anytype-middleware/core/block/process"
 	"github.com/anytypeio/go-anytype-middleware/pb"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/core"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
@@ -21,7 +22,7 @@ func RegisterFunc(c ConverterCreator) {
 
 // Converter incapsulate logic with transforming some data to smart blocks
 type Converter interface {
-	GetSnapshots(req *pb.RpcObjectImportRequest) *Response
+	GetSnapshots(req *pb.RpcObjectImportRequest, progress *process.Progress) (*Response, ConvertError)
 	Name() string
 }
 
@@ -41,9 +42,16 @@ type Snapshot struct {
 	Snapshot *model.SmartBlockSnapshotBase
 }
 
+// during GetSnapshots step in converter and create them in RelationCreator
+type Relation struct {
+	BlockID string // if relations is used as a block
+	*model.Relation
+}
+
 // Response expected response of each converter, incapsulate blocks snapshots and converting errors
 type Response struct {
 	Snapshots []*Snapshot
+	Relations map[string][]*Relation // object id to its relations
 	Error     ConvertError
 }
 

core/block/import/importer.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	"github.com/gogo/protobuf/types"
-	"github.com/pkg/errors"
 	"go.uber.org/zap"
 
 	"github.com/anytypeio/go-anytype-middleware/app"
@@ -18,6 +17,7 @@ import (
 	"github.com/anytypeio/go-anytype-middleware/pb"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/core"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/filestore"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/logging"
 )
 
@@ -44,9 +44,12 @@ func (i *Import) Init(a *app.App) (err error) {
 		converter := f(core)
 		i.converters[converter.Name()] = converter
 	}
-	factory := syncer.New(syncer.NewFileSyncer(i.s), syncer.NewBookmarkSyncer(i.s))
-	ou := NewObjectUpdater(i.s, core, factory)
-	i.oc = NewCreator(i.s, a.MustComponent(object.CName).(objectCreator), core, ou, factory)
+	factory := syncer.New(syncer.NewFileSyncer(i.s), syncer.NewBookmarkSyncer(i.s), syncer.NewIconSyncer(i.s))
+	fs := a.MustComponent(filestore.CName).(filestore.FileStore)
+	objCreator := a.MustComponent(object.CName).(objectCreator)
+	relationCreator := NewRelationCreator(i.s, objCreator, fs, core)
+	ou := NewObjectUpdater(i.s, core, factory, relationCreator)
+	i.oc = NewCreator(i.s, objCreator, ou, factory, relationCreator)
 	return nil
 }
 
@@ -59,13 +62,13 @@ func (i *Import) Import(ctx *session.Context, req *pb.RpcObjectImportRequest) er
 	}
 	allErrors := converter.NewError()
 	if c, ok := i.converters[req.Type.String()]; ok {
-		progress.SetProgressMessage("import snapshots")
-		res := i.importObjects(c, req)
+		res, err := c.GetSnapshots(req, progress)
 		if res == nil {
-			return fmt.Errorf("empty response from converter")
+			return fmt.Errorf("no files to import")
 		}
-		if res.Error != nil {
-			allErrors.Merge(res.Error)
+
+		if len(err) != 0 {
+			allErrors.Merge(err)
 			if req.Mode != pb.RpcObjectImportRequest_IGNORE_ERRORS {
 				return allErrors.Error()
 			}
@@ -73,7 +76,8 @@ func (i *Import) Import(ctx *session.Context, req *pb.RpcObjectImportRequest) er
 		if len(res.Snapshots) == 0 {
 			return fmt.Errorf("no files to import")
 		}
-		progress.SetProgressMessage("create blocks")
+
+		progress.SetProgressMessage("Create objects")
 		i.createObjects(ctx, res, progress, req, allErrors)
 		return allErrors.Error()
 	}
@@ -88,7 +92,6 @@ func (i *Import) Import(ctx *session.Context, req *pb.RpcObjectImportRequest) er
 			}
 			res := &converter.Response{
 				Snapshots: sn,
-				Error:     nil,
 			}
 			i.createObjects(ctx, res, progress, req, allErrors)
 			return allErrors.Error()
@@ -117,16 +120,19 @@ func (i *Import) ImportWeb(ctx *session.Context, req *pb.RpcObjectImportRequest)
 	progress := process.NewProgress(pb.ModelProcess_Import)
 	defer progress.Finish()
 	allErrors := make(map[string]error, 0)
-	progress.SetProgressMessage("parse url")
+
+	progress.SetProgressMessage("Parse url")
 	w := web.NewConverter()
-	res := w.GetSnapshots(req)
-	if res.Error != nil {
-		return "", nil, res.Error.Error()
+	res, err := w.GetSnapshots(req, progress)
+
+	if err != nil {
+		return "", nil, err.Error()
 	}
 	if res.Snapshots == nil || len(res.Snapshots) == 0 {
 		return "", nil, fmt.Errorf("snpashots are empty")
 	}
-	progress.SetProgressMessage("create blocks")
+
+	progress.SetProgressMessage("Create objects")
 	details := i.createObjects(ctx, res, progress, req, allErrors)
 	if len(allErrors) != 0 {
 		return "", nil, fmt.Errorf("couldn't create objects")
@@ -135,10 +141,6 @@ func (i *Import) ImportWeb(ctx *session.Context, req *pb.RpcObjectImportRequest)
 	return res.Snapshots[0].Id, details[res.Snapshots[0].Id], nil
 }
 
-func (i *Import) importObjects(c converter.Converter, req *pb.RpcObjectImportRequest) *converter.Response {
-	return c.GetSnapshots(req)
-}
-
 func (i *Import) createObjects(ctx *session.Context, res *converter.Response, progress *process.Progress, req *pb.RpcObjectImportRequest, allErrors map[string]error) map[string]*types.Struct {
 	var (
 		sbType smartblock.SmartBlockType
@@ -155,6 +157,7 @@ func (i *Import) createObjects(ctx *session.Context, res *converter.Response, pr
 	}
 
 	details := make(map[string]*types.Struct, 0)
+
 	for _, snapshot := range res.Snapshots {
 		switch {
 		case snapshot.Id != "":
@@ -169,15 +172,16 @@ func (i *Import) createObjects(ctx *session.Context, res *converter.Response, pr
 		default:
 			sbType = smartblock.SmartBlockTypePage
 		}
-		progress.SetTotal(int64(len(res.Snapshots)))
-		select {
-		case <-progress.Canceled():
-			allErrors[getFileName(snapshot)] = errors.New("canceled")
+
+		if err := progress.TryStep(1); err != nil {
+			allErrors[getFileName(snapshot)] = err
 			return nil
-		default:
 		}
-		progress.AddDone(1)
-		detail, err := i.oc.Create(ctx, snapshot.Snapshot, snapshot.Id, sbType, req.UpdateExistingObjects)
+		var relations []*converter.Relation
+		if res.Relations != nil {
+			relations = res.Relations[snapshot.Id]
+		}
+		detail, err := i.oc.Create(ctx, snapshot.Snapshot, relations, snapshot.Id, sbType, req.UpdateExistingObjects)
 		if err != nil {
 			allErrors[getFileName(snapshot)] = err
 			if req.Mode != pb.RpcObjectImportRequest_IGNORE_ERRORS {

core/block/import/importer_test.go

@@ -21,7 +21,7 @@ func Test_ImportSuccess(t *testing.T) {
 
 	ctrl := gomock.NewController(t)
 	converter := NewMockConverter(ctrl)
-	converter.EXPECT().GetSnapshots(gomock.Any()).Return(&cv.Response{Snapshots: []*cv.Snapshot{{
+	converter.EXPECT().GetSnapshots(gomock.Any(), gomock.Any()).Return(&cv.Response{Snapshots: []*cv.Snapshot{{
 		Snapshot: &model.SmartBlockSnapshotBase{
 			Blocks: []*model.Block{&model.Block{
 				Id: "1",
@@ -34,15 +34,15 @@ func Test_ImportSuccess(t *testing.T) {
 			},
 			},
 		},
-		Id: "bafybbbbruo3kqubijrbhr24zonagbz3ksxbrutwjjoczf37axdsusu4a"}}, Error: nil}).Times(1)
+		Id: "bafybbbbruo3kqubijrbhr24zonagbz3ksxbrutwjjoczf37axdsusu4a"}}}, cv.ConvertError{}).Times(1)
 	i.converters = make(map[string]cv.Converter, 0)
 	i.converters["Notion"] = converter
 	creator := NewMockCreator(ctrl)
-	creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1)
+	creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1)
 	i.oc = creator
 
 	err := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{
-		Params: &pb.RpcObjectImportRequestParamsOfNotionParams{NotionParams: &pb.RpcObjectImportRequestNotionParams{Path: "bafybbbbruo3kqubijrbhr24zonagbz3ksxbrutwjjoczf37axdsusu4a.pb"}},
+		Params:                &pb.RpcObjectImportRequestParamsOfMarkdownParams{MarkdownParams: &pb.RpcObjectImportRequestMarkdownParams{Path: "bafybbbbruo3kqubijrbhr24zonagbz3ksxbrutwjjoczf37axdsusu4a.pb"}},
 		UpdateExistingObjects: false,
 		Type:                  0,
 		Mode:                  0,
@@ -58,23 +58,21 @@ func Test_ImportErrorFromConverter(t *testing.T) {
 	converter := NewMockConverter(ctrl)
 	e := cv.NewError()
 	e.Add("error", fmt.Errorf("converter error"))
-	converter.EXPECT().GetSnapshots(gomock.Any()).Return(&cv.Response{
-		Error: e,
-	}).Times(1)
+	converter.EXPECT().GetSnapshots(gomock.Any(), gomock.Any()).Return(nil, e).Times(1)
 	i.converters = make(map[string]cv.Converter, 0)
 	i.converters["Notion"] = converter
 	creator := NewMockCreator(ctrl)
 	i.oc = creator
 
 	err := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{
-		Params: &pb.RpcObjectImportRequestParamsOfNotionParams{NotionParams: &pb.RpcObjectImportRequestNotionParams{Path: "test"}},
+		Params:                &pb.RpcObjectImportRequestParamsOfMarkdownParams{MarkdownParams: &pb.RpcObjectImportRequestMarkdownParams{Path: "test"}},
 		UpdateExistingObjects: false,
 		Type:                  0,
 		Mode:                  0,
 	})
 
 	assert.NotNil(t, err)
-	assert.Contains(t, err.Error(), "converter error")
+	assert.Contains(t, err.Error(), "no files to import")
 }
 
 func Test_ImportErrorFromObjectCreator(t *testing.T) {
@@ -82,7 +80,7 @@ func Test_ImportErrorFromObjectCreator(t *testing.T) {
 
 	ctrl := gomock.NewController(t)
 	converter := NewMockConverter(ctrl)
-	converter.EXPECT().GetSnapshots(gomock.Any()).Return(&cv.Response{Snapshots: []*cv.Snapshot{{
+	converter.EXPECT().GetSnapshots(gomock.Any(), gomock.Any()).Return(&cv.Response{Snapshots: []*cv.Snapshot{{
 		Snapshot: &model.SmartBlockSnapshotBase{
 			Blocks: []*model.Block{&model.Block{
 				Id: "1",
@@ -95,15 +93,15 @@ func Test_ImportErrorFromObjectCreator(t *testing.T) {
 			},
 			},
 		},
-		Id: "bafybbbbruo3kqubijrbhr24zonagbz3ksxbrutwjjoczf37axdsusu4a"}}, Error: nil}).Times(1)
+		Id: "bafybbbbruo3kqubijrbhr24zonagbz3ksxbrutwjjoczf37axdsusu4a"}}}, cv.ConvertError{}).Times(1)
 	i.converters = make(map[string]cv.Converter, 0)
 	i.converters["Notion"] = converter
 	creator := NewMockCreator(ctrl)
-	creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("creator error")).Times(1)
+	creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("creator error")).Times(1)
 	i.oc = creator
 
 	res := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{
-		Params: &pb.RpcObjectImportRequestParamsOfNotionParams{NotionParams: &pb.RpcObjectImportRequestNotionParams{Path: "test"}},
+		Params:                &pb.RpcObjectImportRequestParamsOfMarkdownParams{MarkdownParams: &pb.RpcObjectImportRequestMarkdownParams{Path: "test"}},
 		UpdateExistingObjects: false,
 		Type:                  0,
 		Mode:                  0,
@@ -120,7 +118,7 @@ func Test_ImportIgnoreErrorMode(t *testing.T) {
 	converter := NewMockConverter(ctrl)
 	e := cv.NewError()
 	e.Add("error", fmt.Errorf("converter error"))
-	converter.EXPECT().GetSnapshots(gomock.Any()).Return(&cv.Response{Snapshots: []*cv.Snapshot{{
+	converter.EXPECT().GetSnapshots(gomock.Any(), gomock.Any()).Return(&cv.Response{Snapshots: []*cv.Snapshot{{
 		Snapshot: &model.SmartBlockSnapshotBase{
 			Blocks: []*model.Block{&model.Block{
 				Id: "1",
@@ -133,15 +131,15 @@ func Test_ImportIgnoreErrorMode(t *testing.T) {
 			},
 			},
 		},
-		Id: "bafybbbbruo3kqubijrbhr24zonagbz3ksxbrutwjjoczf37axdsusu4a"}}, Error: e}).Times(1)
+		Id: "bafybbbbruo3kqubijrbhr24zonagbz3ksxbrutwjjoczf37axdsusu4a"}}}, e).Times(1)
 	i.converters = make(map[string]cv.Converter, 0)
 	i.converters["Notion"] = converter
 	creator := NewMockCreator(ctrl)
-	creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1)
+	creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1)
 	i.oc = creator
 
 	res := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{
-		Params: &pb.RpcObjectImportRequestParamsOfNotionParams{NotionParams: &pb.RpcObjectImportRequestNotionParams{Path: "test"}},
+		Params:                &pb.RpcObjectImportRequestParamsOfMarkdownParams{MarkdownParams: &pb.RpcObjectImportRequestMarkdownParams{Path: "test"}},
 		UpdateExistingObjects: false,
 		Type:                  0,
 		Mode:                  1,
@@ -158,7 +156,7 @@ func Test_ImportIgnoreErrorModeWithTwoErrorsPerFile(t *testing.T) {
 	converter := NewMockConverter(ctrl)
 	e := cv.NewError()
 	e.Add("error", fmt.Errorf("converter error"))
-	converter.EXPECT().GetSnapshots(gomock.Any()).Return(&cv.Response{Snapshots: []*cv.Snapshot{{
+	converter.EXPECT().GetSnapshots(gomock.Any(), gomock.Any()).Return(&cv.Response{Snapshots: []*cv.Snapshot{{
 		Snapshot: &model.SmartBlockSnapshotBase{
 			Blocks: []*model.Block{&model.Block{
 				Id: "1",
@@ -171,15 +169,15 @@ func Test_ImportIgnoreErrorModeWithTwoErrorsPerFile(t *testing.T) {
 			},
 			},
 		},
-		Id: "bafybbbbruo3kqubijrbhr24zonagbz3ksxbrutwjjoczf37axdsusu4a"}}, Error: e}).Times(1)
+		Id: "bafybbbbruo3kqubijrbhr24zonagbz3ksxbrutwjjoczf37axdsusu4a"}}}, e).Times(1)
 	i.converters = make(map[string]cv.Converter, 0)
 	i.converters["Notion"] = converter
 	creator := NewMockCreator(ctrl)
-	creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("creator error")).Times(1)
+	creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("creator error")).Times(1)
 	i.oc = creator
 
 	res := i.Import(session.NewContext(), &pb.RpcObjectImportRequest{
-		Params: &pb.RpcObjectImportRequestParamsOfNotionParams{NotionParams: &pb.RpcObjectImportRequestNotionParams{Path: "test"}},
+		Params:                &pb.RpcObjectImportRequestParamsOfMarkdownParams{MarkdownParams: &pb.RpcObjectImportRequestMarkdownParams{Path: "test"}},
 		UpdateExistingObjects: false,
 		Type:                  0,
 		Mode:                  1,
@@ -198,7 +196,7 @@ func Test_ImportExternalPlugin(t *testing.T) {
 	i.converters = make(map[string]cv.Converter, 0)
 
 	creator := NewMockCreator(ctrl)
-	creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1)
+	creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1)
 	i.oc = creator
 
 	snapshots := make([]*pb.RpcObjectImportRequestSnapshot, 0)
@@ -226,7 +224,7 @@ func Test_ImportExternalPlugin(t *testing.T) {
 		Params:                nil,
 		Snapshots:             snapshots,
 		UpdateExistingObjects: false,
-		Type:                  1,
+		Type:                  2,
 		Mode:                  2,
 	})
 	assert.Nil(t, res)
@@ -246,7 +244,7 @@ func Test_ImportExternalPluginError(t *testing.T) {
 		Params:                nil,
 		Snapshots:             nil,
 		UpdateExistingObjects: false,
-		Type:                  1,
+		Type:                  2,
 		Mode:                  2,
 	})
 	assert.NotNil(t, res)
@@ -268,10 +266,9 @@ func Test_ListImports(t *testing.T) {
 	assert.Nil(t, err)
 	assert.NotNil(t, res)
 	assert.Len(t, res, 1)
-	assert.True(t,  res[0].Type == pb.RpcObjectImportListImportResponseType(0) || res[1].Type == pb.RpcObjectImportListImportResponseType(0))
+	assert.True(t, res[0].Type == pb.RpcObjectImportListImportResponseType(0) || res[1].Type == pb.RpcObjectImportListImportResponseType(0))
 }
 
-
 func Test_ImportWebNoParser(t *testing.T) {
 	i := Import{}
 
@@ -283,7 +280,7 @@ func Test_ImportWebNoParser(t *testing.T) {
 	i.oc = creator
 
 	_, _, err := i.ImportWeb(session.NewContext(), &pb.RpcObjectImportRequest{
-		Params:  &pb.RpcObjectImportRequestParamsOfBookmarksParams{BookmarksParams: &pb.RpcObjectImportRequestBookmarksParams{Url: "http://example.com"}},
+		Params:                &pb.RpcObjectImportRequestParamsOfBookmarksParams{BookmarksParams: &pb.RpcObjectImportRequestBookmarksParams{Url: "http://example.com"}},
 		UpdateExistingObjects: true,
 	})
 
@@ -311,7 +308,7 @@ func Test_ImportWebFailedToParse(t *testing.T) {
 	parsers.RegisterFunc(new)
 
 	_, _, err := i.ImportWeb(session.NewContext(), &pb.RpcObjectImportRequest{
-		Params:  &pb.RpcObjectImportRequestParamsOfBookmarksParams{BookmarksParams: &pb.RpcObjectImportRequestBookmarksParams{Url: "http://example.com"}},
+		Params:                &pb.RpcObjectImportRequestParamsOfBookmarksParams{BookmarksParams: &pb.RpcObjectImportRequestBookmarksParams{Url: "http://example.com"}},
 		UpdateExistingObjects: true,
 	})
 
@@ -327,7 +324,7 @@ func Test_ImportWebSuccess(t *testing.T) {
 	i.converters = make(map[string]cv.Converter, 0)
 
 	creator := NewMockCreator(ctrl)
-	creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1)
+	creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil).Times(1)
 	i.oc = creator
 
 	parser := parsers.NewMockParser(ctrl)
@@ -347,8 +344,8 @@ func Test_ImportWebSuccess(t *testing.T) {
 	}
 	parsers.RegisterFunc(new)
 
-	id,_, err := i.ImportWeb(session.NewContext(), &pb.RpcObjectImportRequest{
-		Params:  &pb.RpcObjectImportRequestParamsOfBookmarksParams{BookmarksParams: &pb.RpcObjectImportRequestBookmarksParams{Url: "http://example.com"}},
+	id, _, err := i.ImportWeb(session.NewContext(), &pb.RpcObjectImportRequest{
+		Params:                &pb.RpcObjectImportRequestParamsOfBookmarksParams{BookmarksParams: &pb.RpcObjectImportRequestBookmarksParams{Url: "http://example.com"}},
 		UpdateExistingObjects: true,
 	})
 
@@ -365,7 +362,7 @@ func Test_ImportWebFailedToCreateObject(t *testing.T) {
 	i.converters = make(map[string]cv.Converter, 0)
 
 	creator := NewMockCreator(ctrl)
-	creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("error")).Times(1)
+	creator.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("error")).Times(1)
 	i.oc = creator
 
 	parser := parsers.NewMockParser(ctrl)
@@ -386,10 +383,10 @@ func Test_ImportWebFailedToCreateObject(t *testing.T) {
 	parsers.RegisterFunc(new)
 
 	_, _, err := i.ImportWeb(session.NewContext(), &pb.RpcObjectImportRequest{
-		Params:  &pb.RpcObjectImportRequestParamsOfBookmarksParams{BookmarksParams: &pb.RpcObjectImportRequestBookmarksParams{Url: "http://example.com"}},
+		Params:                &pb.RpcObjectImportRequestParamsOfBookmarksParams{BookmarksParams: &pb.RpcObjectImportRequestBookmarksParams{Url: "http://example.com"}},
 		UpdateExistingObjects: true,
 	})
 
 	assert.NotNil(t, err)
 	assert.Equal(t, "couldn't create objects", err.Error())
-}
+}

core/block/import/markdown/anymark/blocks_renderer.go

@@ -306,7 +306,8 @@ func (r *blocksRenderer) CloseTextBlock(content model.BlockContentTextStyle) {
 	}
 
 	if parentBlock != nil {
-		if parentText := parentBlock.GetText(); parentText != nil && parentText.Text == "" && !isBlockCanHaveChild(closingBlock.Block) && t.Text != "" {
+		if parentText := parentBlock.GetText(); parentText != nil && parentText.Text == "" &&
+			!isBlockCanHaveChild(closingBlock.Block) && t.Text != "" {
 			parentText.Marks = t.Marks
 			parentText.Checked = t.Checked
 			parentText.Color = t.Color

core/block/import/markdown/anymark/markdown.go

@@ -29,7 +29,9 @@ func convertBlocks(source []byte, r renderer.NodeRenderer) error {
 	return gm.Convert(source, &bytes.Buffer{})
 }
 
-func MarkdownToBlocks(markdownSource []byte, baseFilepath string, allFileShortPaths []string) (blocks []*model.Block, rootBlockIDs []string, err error) {
+func MarkdownToBlocks(markdownSource []byte,
+	baseFilepath string,
+	allFileShortPaths []string) (blocks []*model.Block, rootBlockIDs []string, err error) {
 	r := NewRenderer(baseFilepath, allFileShortPaths)
 
 	// allFileShortPaths,

core/block/import/markdown/anymark/renderer.go

@@ -64,12 +64,18 @@ func (r *Renderer) writeLines(source []byte, n ast.Node) {
 	}
 }
 
-func (r *Renderer) renderDocument(_ util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderDocument(_ util.BufWriter,
+	source []byte,
+	node ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	// nothing to do
 	return ast.WalkContinue, nil
 }
 
-func (r *Renderer) renderHeading(_ util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderHeading(_ util.BufWriter,
+	source []byte,
+	node ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	n := node.(*ast.Heading)
 
 	var style model.BlockContentTextStyle
@@ -97,7 +103,10 @@ func (r *Renderer) renderHeading(_ util.BufWriter, source []byte, node ast.Node,
 	return ast.WalkContinue, nil
 }
 
-func (r *Renderer) renderBlockquote(_ util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderBlockquote(_ util.BufWriter,
+	source []byte,
+	n ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	if entering {
 		r.OpenNewTextBlock(model.BlockContentText_Quote)
 	} else {
@@ -106,7 +115,10 @@ func (r *Renderer) renderBlockquote(_ util.BufWriter, source []byte, n ast.Node,
 	return ast.WalkContinue, nil
 }
 
-func (r *Renderer) renderCodeBlock(_ util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderCodeBlock(_ util.BufWriter,
+	source []byte,
+	n ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	if entering {
 		r.OpenNewTextBlock(model.BlockContentText_Code)
 	} else {
@@ -115,7 +127,10 @@ func (r *Renderer) renderCodeBlock(_ util.BufWriter, source []byte, n ast.Node,
 	return ast.WalkContinue, nil
 }
 
-func (r *Renderer) renderFencedCodeBlock(_ util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderFencedCodeBlock(_ util.BufWriter,
+	source []byte,
+	node ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	n := node.(*ast.FencedCodeBlock)
 	if entering {
 		r.OpenNewTextBlock(model.BlockContentText_Code)
@@ -126,12 +141,18 @@ func (r *Renderer) renderFencedCodeBlock(_ util.BufWriter, source []byte, node a
 	return ast.WalkContinue, nil
 }
 
-func (r *Renderer) renderHTMLBlock(_ util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderHTMLBlock(_ util.BufWriter,
+	source []byte,
+	node ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	// Do not render
 	return ast.WalkContinue, nil
 }
 
-func (r *Renderer) renderList(_ util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderList(_ util.BufWriter,
+	source []byte,
+	node ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	n := node.(*ast.List)
 
 	r.SetListState(entering, n.IsOrdered())
@@ -139,7 +160,10 @@ func (r *Renderer) renderList(_ util.BufWriter, source []byte, node ast.Node, en
 	return ast.WalkContinue, nil
 }
 
-func (r *Renderer) renderListItem(_ util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderListItem(_ util.BufWriter,
+	source []byte,
+	n ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	tag := model.BlockContentText_Marked
 
 	if r.GetIsNumberedList() {
@@ -154,7 +178,10 @@ func (r *Renderer) renderListItem(_ util.BufWriter, source []byte, n ast.Node, e
 	return ast.WalkContinue, nil
 }
 
-func (r *Renderer) renderParagraph(_ util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderParagraph(_ util.BufWriter,
+	source []byte,
+	n ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	if entering {
 		r.OpenNewTextBlock(model.BlockContentText_Paragraph)
 	} else {
@@ -163,7 +190,10 @@ func (r *Renderer) renderParagraph(_ util.BufWriter, source []byte, n ast.Node,
 	return ast.WalkContinue, nil
 }
 
-func (r *Renderer) renderTextBlock(_ util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderTextBlock(_ util.BufWriter,
+	source []byte,
+	n ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	if !entering {
 		// TODO: check it
 		// r.CloseTextBlock(model.BlockContentText_Paragraph)
@@ -171,7 +201,10 @@ func (r *Renderer) renderTextBlock(_ util.BufWriter, source []byte, n ast.Node,
 	return ast.WalkContinue, nil
 }
 
-func (r *Renderer) renderThematicBreak(_ util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderThematicBreak(_ util.BufWriter,
+	source []byte,
+	n ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	if entering {
 		r.ForceCloseTextBlock()
 	} else {
@@ -181,7 +214,10 @@ func (r *Renderer) renderThematicBreak(_ util.BufWriter, source []byte, n ast.No
 	return ast.WalkContinue, nil
 }
 
-func (r *Renderer) renderAutoLink(_ util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderAutoLink(_ util.BufWriter,
+	source []byte,
+	node ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	n := node.(*ast.AutoLink)
 	if !entering {
 		return ast.WalkContinue, nil
@@ -201,7 +237,8 @@ func (r *Renderer) renderAutoLink(_ util.BufWriter, source []byte, node ast.Node
 	}
 
 	// add basefilepath
-	if !strings.HasPrefix(strings.ToLower(linkPath), "http://") && !strings.HasPrefix(strings.ToLower(linkPath), "https://") {
+	if !strings.HasPrefix(strings.ToLower(linkPath), "http://") &&
+		!strings.HasPrefix(strings.ToLower(linkPath), "https://") {
 		linkPath = filepath.Join(r.GetBaseFilepath(), linkPath)
 	}
 
@@ -214,7 +251,10 @@ func (r *Renderer) renderAutoLink(_ util.BufWriter, source []byte, node ast.Node
 	return ast.WalkContinue, nil
 }
 
-func (r *Renderer) renderCodeSpan(_ util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderCodeSpan(_ util.BufWriter,
+	source []byte,
+	n ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	if entering {
 		r.SetMarkStart()
 
@@ -242,7 +282,10 @@ func (r *Renderer) renderCodeSpan(_ util.BufWriter, source []byte, n ast.Node, e
 	return ast.WalkContinue, nil
 }
 
-func (r *Renderer) renderEmphasis(_ util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderEmphasis(_ util.BufWriter,
+	source []byte,
+	node ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	n := node.(*ast.Emphasis)
 	tag := model.BlockContentTextMark_Italic
 	if n.Level == 2 {
@@ -262,7 +305,10 @@ func (r *Renderer) renderEmphasis(_ util.BufWriter, source []byte, node ast.Node
 	return ast.WalkContinue, nil
 }
 
-func (r *Renderer) renderLink(_ util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderLink(_ util.BufWriter,
+	source []byte,
+	node ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	n := node.(*ast.Link)
 
 	destination := n.Destination
@@ -277,7 +323,8 @@ func (r *Renderer) renderLink(_ util.BufWriter, source []byte, node ast.Node, en
 			linkPath = string(destination)
 		}
 
-		if !strings.HasPrefix(strings.ToLower(linkPath), "http://") && !strings.HasPrefix(strings.ToLower(linkPath), "https://") {
+		if !strings.HasPrefix(strings.ToLower(linkPath), "http://") &&
+			!strings.HasPrefix(strings.ToLower(linkPath), "https://") {
 			linkPath = filepath.Join(r.GetBaseFilepath(), linkPath)
 		}
 
@@ -292,7 +339,10 @@ func (r *Renderer) renderLink(_ util.BufWriter, source []byte, node ast.Node, en
 	return ast.WalkContinue, nil
 }
 
-func (r *Renderer) renderImage(_ util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderImage(_ util.BufWriter,
+	source []byte,
+	node ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	if !entering {
 		return ast.WalkContinue, nil
 	}
@@ -303,7 +353,10 @@ func (r *Renderer) renderImage(_ util.BufWriter, source []byte, node ast.Node, e
 	return ast.WalkSkipChildren, nil
 }
 
-func (r *Renderer) renderRawHTML(_ util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderRawHTML(_ util.BufWriter,
+	source []byte,
+	node ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 	n, ok := node.(*ast.RawHTML)
 	if !ok {
 		return ast.WalkSkipChildren, nil
@@ -332,7 +385,10 @@ func (r *Renderer) renderRawHTML(_ util.BufWriter, source []byte, node ast.Node,
 	return ast.WalkContinue, nil
 }
 
-func (r *Renderer) renderText(_ util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+func (r *Renderer) renderText(_ util.BufWriter,
+	source []byte,
+	node ast.Node,
+	entering bool) (ast.WalkStatus, error) {
 
 	if !entering {
 		return ast.WalkContinue, nil

core/block/import/markdown/blockconverter.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -18,6 +17,7 @@ import (
 	"github.com/anytypeio/go-anytype-middleware/core/block/import/markdown/anymark"
 	"github.com/anytypeio/go-anytype-middleware/pb"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+	"github.com/anytypeio/go-anytype-middleware/util/uri"
 )
 
 type Service interface {
@@ -258,7 +258,7 @@ func (m *mdConverter) convertTextToPageLink(block *model.Block) {
 }
 
 func (m *mdConverter) convertTextToBookmark(block *model.Block) {
-	if _, err := url.Parse(block.GetText().Marks.Marks[0].Param); err != nil {
+	if err := uri.ValidateURI(block.GetText().Marks.Marks[0].Param); err != nil {
 		return
 	}
 

core/block/import/markdown/import.go

@@ -9,10 +9,10 @@ import (
 
 	"github.com/globalsign/mgo/bson"
 	"github.com/gogo/protobuf/types"
-	"github.com/pkg/errors"
 	"github.com/textileio/go-threads/core/thread"
 
 	"github.com/anytypeio/go-anytype-middleware/core/block/import/converter"
+	"github.com/anytypeio/go-anytype-middleware/core/block/process"
 	"github.com/anytypeio/go-anytype-middleware/pb"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/core"
@@ -31,6 +31,8 @@ var (
 	articleIcons = []string{"📓", "📕", "📗", "📘", "📙", "📖", "📔", "📒", "📝", "📄", "📑"}
 )
 
+const numberOfStages = 9 // 8 cycles to get snaphots and 1 cycle to create objects
+
 func init() {
 	converter.RegisterFunc(New)
 }
@@ -39,7 +41,7 @@ type Markdown struct {
 	blockConverter *mdConverter
 }
 
-const Name = "Notion"
+const Name = "Markdown"
 
 func New(s core.Service) converter.Converter {
 	return &Markdown{blockConverter: newMDConverter(s)}
@@ -49,255 +51,102 @@ func (m *Markdown) Name() string {
 	return Name
 }
 
-func (m *Markdown) GetParams(params pb.IsRpcObjectImportRequestParams) (string, error) {
-	if p, ok := params.(*pb.RpcObjectImportRequestParamsOfNotionParams); ok {
-		return p.NotionParams.GetPath(), nil
+func (m *Markdown) GetParams(req *pb.RpcObjectImportRequest) string {
+	if p := req.GetMarkdownParams(); p != nil {
+		return p.Path
 	}
-	return "", errors.Wrap(errors.New("wrong parameters format"), "Markdown: GetParams")
+
+	return ""
 }
 
 func (m *Markdown) GetImage() ([]byte, int64, int64, error) {
 	return nil, 0, 0, nil
 }
 
-func (m *Markdown) GetSnapshots(req *pb.RpcObjectImportRequest) *converter.Response {
-	path, err := m.GetParams(req.Params)
-	allErrors := converter.NewError()
-	if err != nil {
-		allErrors.Add(path, err)
-		return &converter.Response{Error: allErrors}
-	}
+func (m *Markdown) GetSnapshots(req *pb.RpcObjectImportRequest,
+	progress *process.Progress) (*converter.Response, converter.ConvertError) {
+	path := m.GetParams(req)
+
 	files, allErrors := m.blockConverter.markdownToBlocks(path, req.GetMode().String())
 	if !allErrors.IsEmpty() && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
 		if req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
-			return &converter.Response{Error: allErrors}
+			return nil, allErrors
 		}
 	}
 
+	progress.SetTotal(int64(numberOfStages * len(files)))
+
 	if len(files) == 0 {
 		allErrors.Add(path, fmt.Errorf("couldn't found md files"))
-		return &converter.Response{Error: allErrors}
+		return nil, allErrors
 	}
 
-	for name, file := range files {
-		// index links in the root csv file
-		if !file.IsRootFile || !strings.EqualFold(filepath.Ext(name), ".csv") {
-			continue
-		}
+	progress.SetProgressMessage("Start linking database file with pages")
 
-		ext := filepath.Ext(name)
-		csvDir := strings.TrimSuffix(name, ext)
-
-		for targetName, targetFile := range files {
-			fileExt := filepath.Ext(targetName)
-			if filepath.Dir(targetName) == csvDir && strings.EqualFold(fileExt, ".md") {
-				targetFile.HasInboundLinks = true
-			}
-		}
+	if cancellErr := m.setInboundLinks(files, progress); cancellErr != nil {
+		return nil, cancellErr
 	}
 
 	var (
-		emoji, title string
-		details      = make(map[string]*types.Struct, 0)
+		details = make(map[string]*types.Struct, 0)
 	)
-	for name, file := range files {
-		if strings.EqualFold(filepath.Ext(name), ".md") || strings.EqualFold(filepath.Ext(name), ".csv") {
-			tid, err := threads.ThreadCreateID(thread.AccessControlled, smartblock.SmartBlockTypePage)
-			if err != nil {
-				allErrors.Add(name, err)
-				if req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
-					return &converter.Response{Error: allErrors}
-				}
-			}
-			file.PageID = tid.String()
 
-			if len(file.ParsedBlocks) > 0 {
-				if text := file.ParsedBlocks[0].GetText(); text != nil && text.Style == model.BlockContentText_Header1 {
-					title = text.Text
-					titleParts := strings.SplitN(title, " ", 2)
+	progress.SetProgressMessage("Start creating blocks")
 
-					// only select the first rune to see if it looks like emoji
-					if len(titleParts) == 2 && emojiAproxRegexp.MatchString(string([]rune(titleParts[0])[0:1])) {
-						// first symbol is emoji - just use it all before the space
-						emoji = titleParts[0]
-						title = titleParts[1]
-					}
-					// remove title block
-					file.ParsedBlocks = file.ParsedBlocks[1:]
-				}
-			}
-
-			if emoji == "" {
-				emoji = slice.GetRandomString(articleIcons, name)
-			}
-
-			if title == "" {
-				title = strings.TrimSuffix(filepath.Base(name), filepath.Ext(name))
-				titleParts := strings.Split(title, " ")
-				title = strings.Join(titleParts[:len(titleParts)-1], " ")
-			}
-
-			file.Title = title
-			// FIELD-BLOCK
-			fields := map[string]*types.Value{
-				bundle.RelationKeyName.String():      pbtypes.String(title),
-				bundle.RelationKeyIconEmoji.String(): pbtypes.String(emoji),
-				bundle.RelationKeySource.String():    pbtypes.String(file.Source),
-			}
-
-			details[name] = &types.Struct{Fields: fields}
-			emoji = ""
-			title = ""
-		}
+	if cancellErr := m.createThreadObject(files, progress, details, allErrors, req.Mode); cancellErr != nil {
+		return nil, cancellErr
 	}
 
-	for name, file := range files {
+	progress.SetProgressMessage("Start linking blocks")
 
-		if file.PageID == "" {
-			// file is not a page
-			continue
-		}
-
-		file.ParsedBlocks = m.processFieldBlockIfItIs(file.ParsedBlocks, files)
-
-		for _, block := range file.ParsedBlocks {
-			if link := block.GetLink(); link != nil {
-				target, err := url.PathUnescape(link.TargetBlockId)
-				if err != nil {
-					allErrors.Add(name, err)
-					if req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
-						return &converter.Response{Error: allErrors}
-					}
-					log.Warnf("err while url.PathUnescape: %s \n \t\t\t url: %s", err, link.TargetBlockId)
-					target = link.TargetBlockId
-				}
-
-				if files[target] != nil {
-					link.TargetBlockId = files[target].PageID
-					files[target].HasInboundLinks = true
-				}
-
-			} else if text := block.GetText(); text != nil && text.Marks != nil && len(text.Marks.Marks) > 0 {
-				for _, mark := range text.Marks.Marks {
-					if mark.Type != model.BlockContentTextMark_Mention && mark.Type != model.BlockContentTextMark_Object {
-						continue
-					}
-
-					if targetFile, exists := files[mark.Param]; exists {
-						mark.Param = targetFile.PageID
-					}
-				}
-			}
-		}
+	if cancelErr := m.createMarkdownForLink(files, progress, allErrors, req.Mode); cancelErr != nil {
+		return nil, cancelErr
 	}
 
-	for name, file := range files {
-		if file.IsRootFile && strings.EqualFold(filepath.Ext(name), ".csv") {
-			details[name].Fields[bundle.RelationKeyIsFavorite.String()] = pbtypes.Bool(true)
-			file.ParsedBlocks = m.convertCsvToLinks(name, files)
-		}
+	progress.SetProgressMessage("Start linking database with pages")
 
-		if file.PageID == "" {
-			// file is not a page
-			continue
-		}
-
-		var blocks = make([]*model.Block, 0, len(file.ParsedBlocks))
-		for i, b := range file.ParsedBlocks {
-			if f := b.GetFile(); f != nil && strings.EqualFold(filepath.Ext(f.Name), ".csv") {
-				if csvFile, exists := files[f.Name]; exists {
-					csvFile.HasInboundLinks = true
-				} else {
-					continue
-				}
-
-				csvInlineBlocks := m.convertCsvToLinks(f.Name, files)
-				blocks = append(blocks, csvInlineBlocks...)
-			} else {
-				blocks = append(blocks, file.ParsedBlocks[i])
-			}
-		}
-
-		file.ParsedBlocks = blocks
+	if cancellErr := m.linkPagesWithRootFile(files, progress, details); cancellErr != nil {
+		return nil, cancellErr
 	}
 
-	// process file blocks
-	for _, file := range files {
-		if file.PageID == "" {
-			// not a page
-			continue
-		}
+	progress.SetProgressMessage("Start creating file blocks")
 
-		for _, b := range file.ParsedBlocks {
-			if b.Id == "" {
-				b.Id = bson.NewObjectId().Hex()
-			}
-		}
+	if cancelErr := m.fillEmptyBlocks(files, progress); cancelErr != nil {
+		return nil, cancelErr
 	}
 
-	for _, file := range files {
-		if file.PageID == "" {
-			// not a page
-			continue
-		}
+	progress.SetProgressMessage("Start creating link blocks")
 
-		if file.HasInboundLinks {
-			continue
-		}
-
-		file.ParsedBlocks = append(file.ParsedBlocks, &model.Block{
-			Content: &model.BlockContentOfLink{
-				Link: &model.BlockContentLink{
-					TargetBlockId: file.PageID,
-					Style:         model.BlockContentLink_Page,
-					Fields:        nil,
-				},
-			},
-		})
+	if cancellErr := m.addLinkBlocks(files, progress); cancellErr != nil {
+		return nil, cancellErr
 	}
 
-	for _, file := range files {
-		if file.PageID == "" {
-			// not a page
-			continue
-		}
+	progress.SetProgressMessage("Start creating root blocks")
 
-		var childrenIds = make([]string, len(file.ParsedBlocks))
-		for _, b := range file.ParsedBlocks {
-			childrenIds = append(childrenIds, b.Id)
-		}
-
-		file.ParsedBlocks = append(file.ParsedBlocks, &model.Block{
-			Id:          file.PageID,
-			ChildrenIds: childrenIds,
-			Content:     &model.BlockContentOfSmartblock{},
-		})
+	if cancellErr := m.addChildBlocks(files, progress); cancellErr != nil {
+		return nil, cancellErr
 	}
 
-	snapshots := make([]*converter.Snapshot, 0)
-	for name, file := range files {
-		if file.PageID == "" {
-			// file is not a page
-			continue
-		}
-		snapshots = append(snapshots, &converter.Snapshot{
-			Id:       file.PageID,
-			FileName: name,
-			Snapshot: &model.SmartBlockSnapshotBase{
-				Blocks:      file.ParsedBlocks,
-				Details:     details[name],
-				ObjectTypes: pbtypes.GetStringList(details[name], bundle.RelationKeyType.String()),
-			},
-		})
+	progress.SetProgressMessage("Start creating snaphots")
+
+	var (
+		snapshots  []*converter.Snapshot
+		cancellErr converter.ConvertError
+	)
+
+	if snapshots, cancellErr = m.createSnapshots(files, progress, details); cancellErr != nil {
+		return nil, cancellErr
 	}
+
 	if len(snapshots) == 0 {
 		allErrors.Add(path, fmt.Errorf("failed to get snaphots from path, no md files"))
 	}
 
 	if allErrors.IsEmpty() {
-		return &converter.Response{Snapshots: snapshots}
+		return &converter.Response{Snapshots: snapshots}, nil
 	}
-	return &converter.Response{Snapshots: snapshots, Error: allErrors}
+
+	return &converter.Response{Snapshots: snapshots}, allErrors
 }
 
 func (m *Markdown) convertCsvToLinks(csvFileName string, files map[string]*FileInfo) (blocks []*model.Block) {
@@ -421,3 +270,309 @@ func (m *Markdown) getIdFromPath(path string) (id string) {
 	}
 	return b[:len(b)-3]
 }
+
+func (m *Markdown) setInboundLinks(files map[string]*FileInfo, progress *process.Progress) converter.ConvertError {
+	for name, file := range files {
+		if err := progress.TryStep(1); err != nil {
+			cancellError := converter.NewFromError(name, err)
+			return cancellError
+		}
+
+		if !file.IsRootFile || !strings.EqualFold(filepath.Ext(name), ".csv") {
+			continue
+		}
+
+		ext := filepath.Ext(name)
+		csvDir := strings.TrimSuffix(name, ext)
+
+		for targetName, targetFile := range files {
+			fileExt := filepath.Ext(targetName)
+			if filepath.Dir(targetName) == csvDir && strings.EqualFold(fileExt, ".md") {
+				targetFile.HasInboundLinks = true
+			}
+		}
+	}
+
+	return nil
+}
+
+func (m *Markdown) linkPagesWithRootFile(files map[string]*FileInfo,
+	progress *process.Progress,
+	details map[string]*types.Struct) converter.ConvertError {
+	for name, file := range files {
+		if err := progress.TryStep(1); err != nil {
+			cancellError := converter.NewFromError(name, err)
+			return cancellError
+		}
+
+		if file.IsRootFile && strings.EqualFold(filepath.Ext(name), ".csv") {
+			details[name].Fields[bundle.RelationKeyIsFavorite.String()] = pbtypes.Bool(true)
+			file.ParsedBlocks = m.convertCsvToLinks(name, files)
+		}
+
+		if file.PageID == "" {
+			// file is not a page
+			continue
+		}
+
+		var blocks = make([]*model.Block, 0, len(file.ParsedBlocks))
+
+		for i, b := range file.ParsedBlocks {
+			if f := b.GetFile(); f != nil && strings.EqualFold(filepath.Ext(f.Name), ".csv") {
+				if csvFile, exists := files[f.Name]; exists {
+					csvFile.HasInboundLinks = true
+				} else {
+					continue
+				}
+
+				csvInlineBlocks := m.convertCsvToLinks(f.Name, files)
+				blocks = append(blocks, csvInlineBlocks...)
+			} else {
+				blocks = append(blocks, file.ParsedBlocks[i])
+			}
+		}
+
+		file.ParsedBlocks = blocks
+	}
+
+	return nil
+}
+func (m *Markdown) addLinkBlocks(files map[string]*FileInfo, progress *process.Progress) converter.ConvertError {
+	for name, file := range files {
+		if err := progress.TryStep(1); err != nil {
+			cancellError := converter.NewFromError(name, err)
+			return cancellError
+		}
+
+		if file.PageID == "" {
+			// not a page
+			continue
+		}
+
+		if file.HasInboundLinks {
+			continue
+		}
+
+		file.ParsedBlocks = append(file.ParsedBlocks, &model.Block{
+			Content: &model.BlockContentOfLink{
+				Link: &model.BlockContentLink{
+					TargetBlockId: file.PageID,
+					Style:         model.BlockContentLink_Page,
+					Fields:        nil,
+				},
+			},
+		})
+	}
+
+	return nil
+}
+
+func (m *Markdown) createSnapshots(files map[string]*FileInfo,
+	progress *process.Progress,
+	details map[string]*types.Struct) ([]*converter.Snapshot, converter.ConvertError) {
+	snapshots := make([]*converter.Snapshot, 0)
+
+	for name, file := range files {
+		if err := progress.TryStep(1); err != nil {
+			cancellError := converter.NewFromError(name, err)
+			return nil, cancellError
+		}
+
+		if file.PageID == "" {
+			// file is not a page
+			continue
+		}
+
+		snapshots = append(snapshots, &converter.Snapshot{
+			Id:       file.PageID,
+			FileName: name,
+			Snapshot: &model.SmartBlockSnapshotBase{
+				Blocks:      file.ParsedBlocks,
+				Details:     details[name],
+				ObjectTypes: pbtypes.GetStringList(details[name], bundle.RelationKeyType.String()),
+			},
+		})
+	}
+
+	return snapshots, nil
+}
+
+func (m *Markdown) addChildBlocks(files map[string]*FileInfo, progress *process.Progress) converter.ConvertError {
+	for name, file := range files {
+		if err := progress.TryStep(1); err != nil {
+			cancellError := converter.NewFromError(name, err)
+			return cancellError
+		}
+
+		if file.PageID == "" {
+			// not a page
+			continue
+		}
+
+		var childrenIds = make([]string, len(file.ParsedBlocks))
+		for _, b := range file.ParsedBlocks {
+			childrenIds = append(childrenIds, b.Id)
+		}
+
+		file.ParsedBlocks = append(file.ParsedBlocks, &model.Block{
+			Id:          file.PageID,
+			ChildrenIds: childrenIds,
+			Content:     &model.BlockContentOfSmartblock{},
+		})
+	}
+
+	return nil
+}
+
+func (m *Markdown) createMarkdownForLink(files map[string]*FileInfo,
+	progress *process.Progress,
+	allErrors converter.ConvertError,
+	mode pb.RpcObjectImportRequestMode) converter.ConvertError {
+	for name, file := range files {
+		if err := progress.TryStep(1); err != nil {
+			cancellError := converter.NewFromError(name, err)
+			return cancellError
+		}
+
+		if file.PageID == "" {
+			// file is not a page
+			continue
+		}
+
+		file.ParsedBlocks = m.processFieldBlockIfItIs(file.ParsedBlocks, files)
+
+		for _, block := range file.ParsedBlocks {
+			if link := block.GetLink(); link != nil {
+				target, err := url.PathUnescape(link.TargetBlockId)
+				if err != nil && mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
+					allErrors.Add(name, err)
+					return allErrors
+				}
+
+				if err != nil {
+					allErrors.Add(name, err)
+					log.Warnf("err while url.PathUnescape: %s \n \t\t\t url: %s", err, link.TargetBlockId)
+					target = link.TargetBlockId
+				}
+
+				if files[target] != nil {
+					link.TargetBlockId = files[target].PageID
+					files[target].HasInboundLinks = true
+				}
+
+				continue
+			}
+
+			if text := block.GetText(); text != nil && text.Marks != nil && len(text.Marks.Marks) > 0 {
+				for _, mark := range text.Marks.Marks {
+					if mark.Type != model.BlockContentTextMark_Mention && mark.Type != model.BlockContentTextMark_Object {
+						continue
+					}
+
+					if targetFile, exists := files[mark.Param]; exists {
+						mark.Param = targetFile.PageID
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (m *Markdown) fillEmptyBlocks(files map[string]*FileInfo, progress *process.Progress) converter.ConvertError {
+	for name, file := range files {
+		if err := progress.TryStep(1); err != nil {
+			cancellError := converter.NewFromError(name, err)
+			return cancellError
+		}
+
+		if file.PageID == "" {
+			// not a page
+			continue
+		}
+
+		for _, b := range file.ParsedBlocks {
+			if b.Id == "" {
+				b.Id = bson.NewObjectId().Hex()
+			}
+		}
+	}
+
+	return nil
+}
+
+func (m *Markdown) createThreadObject(files map[string]*FileInfo,
+	progress *process.Progress,
+	details map[string]*types.Struct,
+	allErrors converter.ConvertError,
+	mode pb.RpcObjectImportRequestMode) converter.ConvertError {
+	for name, file := range files {
+		if err := progress.TryStep(1); err != nil {
+			cancellError := converter.NewFromError(name, err)
+			return cancellError
+		}
+
+		if strings.EqualFold(filepath.Ext(name), ".md") || strings.EqualFold(filepath.Ext(name), ".csv") {
+			tid, err := threads.ThreadCreateID(thread.AccessControlled, smartblock.SmartBlockTypePage)
+			if err != nil {
+				allErrors.Add(name, err)
+			}
+
+			if err != nil && mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
+				return allErrors
+			}
+
+			file.PageID = tid.String()
+
+			m.setTitleAndEmoji(file, name, details)
+		}
+	}
+
+	return nil
+}
+
+func (m *Markdown) setTitleAndEmoji(file *FileInfo, name string, details map[string]*types.Struct) {
+	var title, emoji string
+	if len(file.ParsedBlocks) > 0 {
+		title, emoji = m.extractTitleAndEmojiFromBlock(file)
+	}
+
+	if emoji == "" {
+		emoji = slice.GetRandomString(articleIcons, name)
+	}
+
+	if title == "" {
+		title = strings.TrimSuffix(filepath.Base(name), filepath.Ext(name))
+		titleParts := strings.Split(title, " ")
+		title = strings.Join(titleParts[:len(titleParts)-1], " ")
+	}
+
+	file.Title = title
+	// FIELD-BLOCK
+	fields := map[string]*types.Value{
+		bundle.RelationKeyName.String():      pbtypes.String(title),
+		bundle.RelationKeyIconEmoji.String(): pbtypes.String(emoji),
+		bundle.RelationKeySource.String():    pbtypes.String(file.Source),
+	}
+	details[name] = &types.Struct{Fields: fields}
+}
+
+func (m *Markdown) extractTitleAndEmojiFromBlock(file *FileInfo) (string, string) {
+	var title, emoji string
+	if text := file.ParsedBlocks[0].GetText(); text != nil && text.Style == model.BlockContentText_Header1 {
+		title = text.Text
+		titleParts := strings.SplitN(title, " ", 2)
+
+		// only select the first rune to see if it looks like emoji
+		if len(titleParts) == 2 && emojiAproxRegexp.MatchString(string([]rune(titleParts[0])[0:1])) {
+			// first symbol is emoji - just use it all before the space
+			emoji = titleParts[0]
+			title = titleParts[1]
+		}
+		// remove title block
+		file.ParsedBlocks = file.ParsedBlocks[1:]
+	}
+
+	return title, emoji
+}

core/block/import/mock.go

@@ -7,14 +7,16 @@ package importer
 import (
 	reflect "reflect"
 
+	types "github.com/gogo/protobuf/types"
+	gomock "github.com/golang/mock/gomock"
+
 	app "github.com/anytypeio/go-anytype-middleware/app"
 	"github.com/anytypeio/go-anytype-middleware/core/block/import/converter"
+	"github.com/anytypeio/go-anytype-middleware/core/block/process"
 	session "github.com/anytypeio/go-anytype-middleware/core/session"
 	pb "github.com/anytypeio/go-anytype-middleware/pb"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock"
 	model "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
-	types "github.com/gogo/protobuf/types"
-	gomock "github.com/golang/mock/gomock"
 )
 
 // MockImporter is a mock of Importer interface.
@@ -120,17 +122,18 @@ func (m *MockConverter) EXPECT() *MockConverterMockRecorder {
 }
 
 // GetSnapshots mocks base method.
-func (m *MockConverter) GetSnapshots(req *pb.RpcObjectImportRequest) *converter.Response {
+func (m *MockConverter) GetSnapshots(req *pb.RpcObjectImportRequest, progress *process.Progress) (*converter.Response, converter.ConvertError) {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "GetSnapshots", req)
+	ret := m.ctrl.Call(m, "GetSnapshots", req, progress)
 	ret0, _ := ret[0].(*converter.Response)
-	return ret0
+	ret1, _ := ret[1].(converter.ConvertError)
+	return ret0, ret1
 }
 
 // GetSnapshots indicates an expected call of GetSnapshots.
-func (mr *MockConverterMockRecorder) GetSnapshots(req interface{}) *gomock.Call {
+func (mr *MockConverterMockRecorder) GetSnapshots(req interface{}, progress interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSnapshots", reflect.TypeOf((*MockConverter)(nil).GetSnapshots), req)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSnapshots", reflect.TypeOf((*MockConverter)(nil).GetSnapshots), req, progress)
 }
 
 // Name mocks base method.
@@ -188,7 +191,7 @@ func (m *MockCreator) EXPECT() *MockCreatorMockRecorder {
 }
 
 // Create mocks base method.
-func (m *MockCreator) Create(ctx *session.Context, cs *model.SmartBlockSnapshotBase, pageID string, sbType smartblock.SmartBlockType, updateExisting bool) (*types.Struct, error) {
+func (m *MockCreator) Create(ctx *session.Context, cs *model.SmartBlockSnapshotBase, relations []*converter.Relation, pageID string, sbType smartblock.SmartBlockType, updateExisting bool) (*types.Struct, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "Create", ctx, cs, pageID, sbType, updateExisting)
 	ret0, _ := ret[0].(*types.Struct)
@@ -197,7 +200,7 @@ func (m *MockCreator) Create(ctx *session.Context, cs *model.SmartBlockSnapshotB
 }
 
 // Create indicates an expected call of Create.
-func (mr *MockCreatorMockRecorder) Create(ctx, cs, pageID, sbType, updateExisting interface{}) *gomock.Call {
+func (mr *MockCreatorMockRecorder) Create(ctx, cs, relations, pageID, sbType, updateExisting interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockCreator)(nil).Create), ctx, cs, pageID, sbType, updateExisting)
 }

core/block/import/notion/api/block/block.go

@@ -0,0 +1,91 @@
+package block
+
+import (
+	"github.com/globalsign/mgo/bson"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+)
+
+type Type string
+
+const (
+	Paragraph       Type = "paragraph"
+	BulletList      Type = "bulleted_list_item"
+	NumberList      Type = "numbered_list_item"
+	Toggle          Type = "toggle"
+	SyncedBlock     Type = "synced_block"
+	Template        Type = "template"
+	Column          Type = "column"
+	ChildPage       Type = "child_page"
+	ChildDatabase   Type = "child_database"
+	Table           Type = "table"
+	Heading1        Type = "heading_1"
+	Heading2        Type = "heading_2"
+	Heading3        Type = "heading_3"
+	ToDo            Type = "to_do"
+	Embed           Type = "embed"
+	Image           Type = "image"
+	Video           Type = "video"
+	File            Type = "file"
+	Pdf             Type = "pdf"
+	Bookmark        Type = "bookmark"
+	Callout         Type = "callout"
+	Quote           Type = "quote"
+	Equation        Type = "equation"
+	Divider         Type = "divider"
+	TableOfContents Type = "table_of_contents"
+	ColumnList      Type = "column_list"
+	LinkPreview     Type = "link_preview"
+	LinkToPage      Type = "link_to_page"
+	TableRow        Type = "table_row"
+	Code            Type = "code"
+	Unsupported     Type = "unsupported"
+)
+
+type Block struct {
+	Object         string     `json:"object"`
+	ID             string     `json:"id"`
+	CreatedTime    string     `json:"created_time"`
+	LastEditedTime string     `json:"last_edited_time"`
+	CreatedBy      api.User   `json:"created_by,omitempty"`
+	LastEditedBy   api.User   `json:"last_edited_by,omitempty"`
+	Parent         api.Parent `json:"parent"`
+	Archived       bool       `json:"archived"`
+	HasChildren    bool       `json:"has_children"`
+	Type           Type       `json:"type"`
+}
+
+type Identifiable interface {
+	GetID() string
+}
+
+type ChildSetter interface {
+	Identifiable
+	HasChild() bool
+	SetChildren(children []interface{})
+}
+
+type Getter interface {
+	GetBlocks(req *MapRequest) *MapResponse
+}
+
+const unsupportedBlockMessage = "Unsupported block"
+
+type UnsupportedBlock struct{}
+
+func (*UnsupportedBlock) GetBlocks(req *MapRequest) *MapResponse {
+	id := bson.NewObjectId().Hex()
+	bl := &model.Block{
+		Id: id,
+		Content: &model.BlockContentOfText{
+			Text: &model.BlockContentText{
+				Text: unsupportedBlockMessage,
+			},
+		},
+	}
+	return &MapResponse{
+		Blocks:   []*model.Block{bl},
+		BlockIDs: []string{id},
+	}
+}

core/block/import/notion/api/block/columns.go

@@ -0,0 +1,51 @@
+package block
+
+type ColumnListBlock struct {
+	Block
+	ColumnList interface{} `json:"column_list"`
+}
+
+func (c *ColumnListBlock) GetID() string {
+	return c.ID
+}
+
+func (c *ColumnListBlock) HasChild() bool {
+	return c.HasChildren
+}
+
+func (c *ColumnListBlock) SetChildren(children []interface{}) {
+	c.ColumnList = children
+}
+
+func (c *ColumnListBlock) GetBlocks(req *MapRequest) *MapResponse {
+	req.Blocks = c.ColumnList.([]interface{})
+	resp := MapBlocks(req)
+	return resp
+}
+
+type ColumnBlock struct {
+	Block
+	Column *ColumnObject `json:"column"`
+}
+
+type ColumnObject struct {
+	Children []interface{} `json:"children"`
+}
+
+func (c *ColumnBlock) GetBlocks(req *MapRequest) *MapResponse {
+	req.Blocks = c.Column.Children
+	resp := MapBlocks(req)
+	return resp
+}
+
+func (c *ColumnBlock) GetID() string {
+	return c.ID
+}
+
+func (c *ColumnBlock) HasChild() bool {
+	return c.HasChildren
+}
+
+func (c *ColumnBlock) SetChildren(children []interface{}) {
+	c.Column.Children = children
+}

core/block/import/notion/api/block/div.go

@@ -0,0 +1,28 @@
+package block
+
+import (
+	"github.com/globalsign/mgo/bson"
+
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+)
+
+type DividerBlock struct {
+	Block
+	Divider struct{} `json:"divider"`
+}
+
+func (*DividerBlock) GetBlocks(*MapRequest) *MapResponse {
+	id := bson.NewObjectId().Hex()
+	block := &model.Block{
+		Id: id,
+		Content: &model.BlockContentOfDiv{
+			Div: &model.BlockContentDiv{
+				Style: 0,
+			},
+		},
+	}
+	return &MapResponse{
+		Blocks:   []*model.Block{block},
+		BlockIDs: []string{id},
+	}
+}

core/block/import/notion/api/block/file.go

@@ -0,0 +1,59 @@
+package block
+
+import (
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+)
+
+type FileBlock struct {
+	Block
+	File    api.FileObject `json:"file"`
+	Caption []api.RichText `json:"caption"`
+}
+
+func (f *FileBlock) GetBlocks(*MapRequest) *MapResponse {
+	block, id := f.File.GetFileBlock(model.BlockContentFile_File)
+	return &MapResponse{
+		Blocks:   []*model.Block{block},
+		BlockIDs: []string{id},
+	}
+}
+
+type ImageBlock struct {
+	Block
+	File api.FileObject `json:"image"`
+}
+
+func (i *ImageBlock) GetBlocks(*MapRequest) *MapResponse {
+	block, id := i.File.GetFileBlock(model.BlockContentFile_Image)
+	return &MapResponse{
+		Blocks:   []*model.Block{block},
+		BlockIDs: []string{id},
+	}
+}
+
+type PdfBlock struct {
+	Block
+	File api.FileObject `json:"pdf"`
+}
+
+func (p *PdfBlock) GetBlocks(*MapRequest) *MapResponse {
+	block, id := p.File.GetFileBlock(model.BlockContentFile_PDF)
+	return &MapResponse{
+		Blocks:   []*model.Block{block},
+		BlockIDs: []string{id},
+	}
+}
+
+type VideoBlock struct {
+	Block
+	File api.FileObject `json:"video"`
+}
+
+func (p *VideoBlock) GetBlocks(*MapRequest) *MapResponse {
+	block, id := p.File.GetFileBlock(model.BlockContentFile_Video)
+	return &MapResponse{
+		Blocks:   []*model.Block{block},
+		BlockIDs: []string{id},
+	}
+}

core/block/import/notion/api/block/link.go

@@ -0,0 +1,198 @@
+package block
+
+import (
+	"strings"
+
+	"github.com/globalsign/mgo/bson"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+	textUtil "github.com/anytypeio/go-anytype-middleware/util/text"
+)
+
+const notFoundPageMessage = "Can't access object in Notion, please provide access in API"
+
+type EmbedBlock struct {
+	Block
+	Embed LinkToWeb `json:"embed"`
+}
+
+func (b *EmbedBlock) GetBlocks(req *MapRequest) *MapResponse {
+	return b.Embed.GetBlocks(req)
+}
+
+type LinkToWeb struct {
+	URL string `json:"url"`
+}
+
+type LinkPreviewBlock struct {
+	Block
+	LinkPreview LinkToWeb `json:"link_preview"`
+}
+
+func (b *LinkPreviewBlock) GetBlocks(req *MapRequest) *MapResponse {
+	return b.LinkPreview.GetBlocks(req)
+}
+
+func (b *LinkToWeb) GetBlocks(*MapRequest) *MapResponse {
+	id := bson.NewObjectId().Hex()
+
+	to := textUtil.UTF16RuneCountString(b.URL)
+
+	bl := &model.Block{
+		Id:          id,
+		ChildrenIds: nil,
+		Content: &model.BlockContentOfText{
+			Text: &model.BlockContentText{
+				Text: b.URL,
+				Marks: &model.BlockContentTextMarks{
+					Marks: []*model.BlockContentTextMark{
+						{
+							Range: &model.Range{
+								From: int32(0),
+								To:   int32(to),
+							},
+							Type:  model.BlockContentTextMark_Link,
+							Param: b.URL,
+						},
+					},
+				},
+			},
+		},
+	}
+	return &MapResponse{
+		Blocks:   []*model.Block{bl},
+		BlockIDs: []string{id},
+	}
+}
+
+type ChildPageBlock struct {
+	Block
+	ChildPage Child `json:"child_page"`
+}
+
+func (b *ChildPageBlock) GetBlocks(req *MapRequest) *MapResponse {
+	bl, id := b.ChildPage.GetLinkToObjectBlock(req.NotionPageIdsToAnytype, req.PageNameToID)
+	return &MapResponse{
+		Blocks:   []*model.Block{bl},
+		BlockIDs: []string{id},
+	}
+}
+
+type ChildDatabaseBlock struct {
+	Block
+	ChildDatabase Child `json:"child_database"`
+}
+
+func (b *ChildDatabaseBlock) GetBlocks(req *MapRequest) *MapResponse {
+	bl, id := b.ChildDatabase.GetLinkToObjectBlock(req.NotionDatabaseIdsToAnytype, req.DatabaseNameToID)
+	return &MapResponse{
+		Blocks:   []*model.Block{bl},
+		BlockIDs: []string{id},
+	}
+}
+
+type Child struct {
+	Title string `json:"title"`
+}
+
+func (c *Child) GetLinkToObjectBlock(notionIdsToAnytype, idToName map[string]string) (*model.Block, string) {
+	var (
+		targetBlockID string
+		ok            bool
+	)
+	for id, name := range idToName {
+		if strings.EqualFold(name, c.Title) {
+			if len(notionIdsToAnytype) > 0 {
+				targetBlockID, ok = notionIdsToAnytype[id]
+			}
+			break
+		}
+	}
+
+	id := bson.NewObjectId().Hex()
+	if !ok {
+		return &model.Block{
+			Id:          id,
+			ChildrenIds: nil,
+			Content: &model.BlockContentOfText{
+				Text: &model.BlockContentText{
+					Text: notFoundPageMessage,
+					Marks: &model.BlockContentTextMarks{
+						Marks: []*model.BlockContentTextMark{},
+					},
+				},
+			},
+		}, id
+	}
+
+	return &model.Block{
+		Id:          id,
+		ChildrenIds: nil,
+		Content: &model.BlockContentOfLink{
+			Link: &model.BlockContentLink{
+				TargetBlockId: targetBlockID,
+			},
+		}}, id
+}
+
+type LinkToPageBlock struct {
+	Block
+	LinkToPage api.Parent `json:"link_to_page"`
+}
+
+func (l *LinkToPageBlock) GetBlocks(req *MapRequest) *MapResponse {
+	var anytypeID string
+	if l.LinkToPage.PageID != "" {
+		anytypeID = req.NotionPageIdsToAnytype[l.LinkToPage.PageID]
+	}
+	if l.LinkToPage.DatabaseID != "" {
+		anytypeID = req.NotionDatabaseIdsToAnytype[l.LinkToPage.DatabaseID]
+	}
+	id := bson.NewObjectId().Hex()
+	bl := &model.Block{
+		Id:          id,
+		ChildrenIds: nil,
+		Content: &model.BlockContentOfLink{
+			Link: &model.BlockContentLink{
+				TargetBlockId: anytypeID,
+			},
+		}}
+	return &MapResponse{
+		Blocks:   []*model.Block{bl},
+		BlockIDs: []string{id},
+	}
+}
+
+type BookmarkBlock struct {
+	Block
+	Bookmark BookmarkObject `json:"bookmark"`
+}
+
+func (b *BookmarkBlock) GetBlocks(*MapRequest) *MapResponse {
+	bl, id := b.Bookmark.GetBookmarkBlock()
+	return &MapResponse{
+		Blocks:   []*model.Block{bl},
+		BlockIDs: []string{id},
+	}
+}
+
+type BookmarkObject struct {
+	URL     string          `json:"url"`
+	Caption []*api.RichText `json:"caption"`
+}
+
+func (b BookmarkObject) GetBookmarkBlock() (*model.Block, string) {
+	id := bson.NewObjectId().Hex()
+	title := api.RichTextToDescription(b.Caption)
+
+	return &model.Block{
+		Id:          id,
+		ChildrenIds: []string{},
+		Content: &model.BlockContentOfBookmark{
+			Bookmark: &model.BlockContentBookmark{
+				Url:   b.URL,
+				Title: title,
+			},
+		}}, id
+}

core/block/import/notion/api/block/link_test.go

@@ -0,0 +1,69 @@
+package block
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+)
+
+func Test_GetBookmarkBlock(t *testing.T) {
+	bo := &BookmarkObject{
+		URL:     "",
+		Caption: []*api.RichText{},
+	}
+	bb, _ := bo.GetBookmarkBlock()
+	assert.NotNil(t, bb)
+	assert.Equal(t, bb.GetBookmark().GetUrl(), bo.URL)
+	assert.Equal(t, bb.GetBookmark().GetTitle(), "")
+
+	bo = &BookmarkObject{
+		URL:     "http://example.com",
+		Caption: []*api.RichText{},
+	}
+	bb, _ = bo.GetBookmarkBlock()
+	assert.NotNil(t, bb)
+	assert.Equal(t, bb.GetBookmark().GetUrl(), bo.URL)
+	assert.Equal(t, bb.GetBookmark().GetTitle(), "")
+
+	bo = &BookmarkObject{
+		URL: "",
+		Caption: []*api.RichText{{
+			Type:      api.Text,
+			PlainText: "Text",
+		}},
+	}
+	bb, _ = bo.GetBookmarkBlock()
+	assert.NotNil(t, bb)
+	assert.Equal(t, bb.GetBookmark().GetUrl(), bo.URL)
+	assert.Equal(t, bb.GetBookmark().GetTitle(), "Text")
+}
+
+func Test_GetLinkToObjectBlockSuccess(t *testing.T) {
+	c := &Child{Title: "title"}
+	nameToID := map[string]string{"id": "title"}
+	notionIdsToAnytype := map[string]string{"id": "anytypeId"}
+	bl, _ := c.GetLinkToObjectBlock(notionIdsToAnytype, nameToID)
+	assert.NotNil(t, bl)
+	content, ok := bl.Content.(*model.BlockContentOfLink)
+	assert.True(t, ok)
+	assert.Equal(t, content.Link.TargetBlockId, "anytypeId")
+}
+
+func Test_GetLinkToObjectBlockFail(t *testing.T) {
+	c := &Child{Title: "title"}
+	bl, _ := c.GetLinkToObjectBlock(nil, nil)
+	assert.NotNil(t, bl)
+	content, ok := bl.Content.(*model.BlockContentOfText)
+	assert.True(t, ok)
+	assert.Equal(t, content.Text.Text, notFoundPageMessage)
+
+	nameToID := map[string]string{"id": "title"}
+	bl, _ = c.GetLinkToObjectBlock(nameToID, nil)
+	assert.NotNil(t, bl)
+	content, ok = bl.Content.(*model.BlockContentOfText)
+	assert.True(t, ok)
+	assert.Equal(t, content.Text.Text, notFoundPageMessage)
+}

core/block/import/notion/api/block/mapper.go

@@ -0,0 +1,40 @@
+package block
+
+import (
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+)
+
+// MapRequest is a data object with all necessary structures for blocks
+type MapRequest struct {
+	Blocks []interface{}
+	// Need all these maps for correct mapping of pages and databases from notion to anytype
+	// for such blocks as mentions or links to objects
+	NotionPageIdsToAnytype     map[string]string
+	NotionDatabaseIdsToAnytype map[string]string
+	PageNameToID               map[string]string
+	DatabaseNameToID           map[string]string
+}
+
+type MapResponse struct {
+	Blocks   []*model.Block
+	BlockIDs []string
+}
+
+func (m *MapResponse) Merge(mergedResp *MapResponse) {
+	if mergedResp != nil {
+		m.BlockIDs = append(m.BlockIDs, mergedResp.BlockIDs...)
+		m.Blocks = append(m.Blocks, mergedResp.Blocks...)
+	}
+}
+
+func MapBlocks(req *MapRequest) *MapResponse {
+	resp := &MapResponse{}
+	for _, bl := range req.Blocks {
+		if ba, ok := bl.(Getter); ok {
+			textResp := ba.GetBlocks(req)
+			resp.Merge(textResp)
+			continue
+		}
+	}
+	return resp
+}

core/block/import/notion/api/block/retrieve.go

@@ -0,0 +1,407 @@
+package block
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+
+	"go.uber.org/zap"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/converter"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/client"
+	"github.com/anytypeio/go-anytype-middleware/pb"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/logging"
+)
+
+var logger = logging.Logger("notion-get-blocks")
+
+const (
+	// Page is also a block, so we use endpoint to retrieve its children
+	endpoint    = "/blocks/%s/children"
+	startCursor = "start_cursor"
+	pageSize    = "page_size"
+)
+
+type Service struct {
+	client *client.Client
+}
+
+func New(client *client.Client) *Service {
+	return &Service{
+		client: client,
+	}
+}
+
+type Response struct {
+	Results    []interface{} `json:"results"`
+	HasMore    bool          `json:"has_more"`
+	NextCursor *string       `json:"next_cursor"`
+	Block      Block         `json:"block"`
+}
+
+func (s *Service) GetBlocksAndChildren(ctx context.Context,
+	pageID, apiKey string,
+	pageSize int64,
+	mode pb.RpcObjectImportRequestMode) ([]interface{}, converter.ConvertError) {
+	ce := converter.ConvertError{}
+	allBlocks := make([]interface{}, 0)
+	blocks, err := s.getBlocks(ctx, pageID, apiKey, pageSize)
+	if err != nil {
+		ce.Add(endpoint, err)
+		if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
+			return nil, ce
+		}
+	}
+	for _, b := range blocks {
+		cs, ok := b.(ChildSetter)
+		if !ok {
+			allBlocks = append(allBlocks, b)
+			continue
+		}
+		if cs.HasChild() {
+			children, err := s.GetBlocksAndChildren(ctx, cs.GetID(), apiKey, pageSize, mode)
+			if err != nil {
+				ce.Merge(err)
+			}
+			if err != nil && mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
+				return nil, ce
+			}
+			cs.SetChildren(children)
+		}
+		allBlocks = append(allBlocks, b)
+	}
+	return allBlocks, nil
+}
+
+func (s *Service) MapNotionBlocksToAnytype(req *MapRequest) *MapResponse {
+	return MapBlocks(req)
+}
+
+func (s *Service) getBlocks(ctx context.Context, pageID, apiKey string, pagination int64) ([]interface{}, error) {
+	var (
+		hasMore = true
+		blocks  = make([]interface{}, 0)
+		cursor  string
+	)
+
+	for hasMore {
+		objects, err := s.getBlocksResponse(ctx, pageID, apiKey, cursor, pagination)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, b := range objects.Results {
+			buffer, err := json.Marshal(b)
+			if err != nil {
+				logger.Errorf("GetBlocks: failed to marshal: %s", err)
+				continue
+			}
+			blockMap := b.(map[string]interface{})
+			blocks = append(blocks, s.fillBlocks(Type(blockMap["type"].(string)), buffer)...)
+		}
+
+		if !objects.HasMore {
+			hasMore = false
+			continue
+		}
+
+		cursor = *objects.NextCursor
+
+	}
+	return blocks, nil
+}
+
+//nolint:funlen
+func (*Service) fillBlocks(blockType Type, buffer []byte) []interface{} {
+	blocks := make([]interface{}, 0)
+	switch blockType {
+	case Paragraph:
+		var p ParagraphBlock
+		err := json.Unmarshal(buffer, &p)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &p)
+	case Heading1:
+		var h Heading1Block
+		err := json.Unmarshal(buffer, &h)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &h)
+	case Heading2:
+		var h Heading2Block
+		err := json.Unmarshal(buffer, &h)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &h)
+	case Heading3:
+		var h Heading3Block
+		err := json.Unmarshal(buffer, &h)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &h)
+	case Callout:
+		var c CalloutBlock
+		err := json.Unmarshal(buffer, &c)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &c)
+	case Quote:
+		var q QuoteBlock
+		err := json.Unmarshal(buffer, &q)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &q)
+	case BulletList:
+		var list BulletedListBlock
+		err := json.Unmarshal(buffer, &list)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &list)
+	case NumberList:
+		var nl NumberedListBlock
+		err := json.Unmarshal(buffer, &nl)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &nl)
+	case Toggle:
+		var t ToggleBlock
+		err := json.Unmarshal(buffer, &t)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &t)
+	case Code:
+		var c CodeBlock
+		err := json.Unmarshal(buffer, &c)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &c)
+	case Equation:
+		var e EquationBlock
+		err := json.Unmarshal(buffer, &e)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &e)
+	case ToDo:
+		var t ToDoBlock
+		err := json.Unmarshal(buffer, &t)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &t)
+	case File:
+		var f FileBlock
+		err := json.Unmarshal(buffer, &f)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &f)
+	case Image:
+		var i ImageBlock
+		err := json.Unmarshal(buffer, &i)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &i)
+	case Video:
+		var v VideoBlock
+		err := json.Unmarshal(buffer, &v)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &v)
+	case Pdf:
+		var p PdfBlock
+		err := json.Unmarshal(buffer, &p)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &p)
+	case Bookmark:
+		var b BookmarkBlock
+		err := json.Unmarshal(buffer, &b)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &b)
+	case Divider:
+		var d DividerBlock
+		err := json.Unmarshal(buffer, &d)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &d)
+	case TableOfContents:
+		var t TableOfContentsBlock
+		err := json.Unmarshal(buffer, &t)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &t)
+	case Embed:
+		var e EmbedBlock
+		err := json.Unmarshal(buffer, &e)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &e)
+	case LinkPreview:
+		var lp LinkPreviewBlock
+		err := json.Unmarshal(buffer, &lp)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &lp)
+	case ChildDatabase:
+		var c ChildDatabaseBlock
+		err := json.Unmarshal(buffer, &c)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &c)
+	case ChildPage:
+		var c ChildPageBlock
+		err := json.Unmarshal(buffer, &c)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &c)
+	case LinkToPage:
+		var l LinkToPageBlock
+		err := json.Unmarshal(buffer, &l)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &l)
+	case Unsupported, Template, SyncedBlock:
+		var u UnsupportedBlock
+		err := json.Unmarshal(buffer, &u)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &u)
+	case Table:
+		var t TableBlock
+		err := json.Unmarshal(buffer, &t)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &t)
+	case TableRow:
+		var t TableRowBlock
+		err := json.Unmarshal(buffer, &t)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &t)
+	case ColumnList:
+		var cl ColumnListBlock
+		err := json.Unmarshal(buffer, &cl)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &cl)
+	case Column:
+		var cb ColumnBlock
+		err := json.Unmarshal(buffer, &cb)
+		if err != nil {
+			logger.With(zap.String("method", "getBlocks")).Error(err)
+			return blocks
+		}
+		blocks = append(blocks, &cb)
+	}
+	return blocks
+}
+
+func (s *Service) getBlocksResponse(ctx context.Context,
+	pageID, apiKey, cursor string,
+	pagination int64) (Response, error) {
+	body := &bytes.Buffer{}
+
+	url := fmt.Sprintf(endpoint, pageID)
+
+	req, err := s.client.PrepareRequest(ctx, apiKey, http.MethodGet, url, body)
+
+	if err != nil {
+		return Response{}, fmt.Errorf("GetBlocks: %s", err)
+	}
+	query := req.URL.Query()
+
+	if cursor != "" {
+		query.Add(startCursor, cursor)
+	}
+
+	query.Add(pageSize, strconv.FormatInt(pagination, 10))
+
+	req.URL.RawQuery = query.Encode()
+
+	res, err := s.client.HTTPClient.Do(req)
+
+	if err != nil {
+		return Response{}, fmt.Errorf("GetBlocks: %s", err)
+	}
+	defer res.Body.Close()
+
+	b, err := ioutil.ReadAll(res.Body)
+
+	if err != nil {
+		return Response{}, err
+	}
+	var objects Response
+	if res.StatusCode != http.StatusOK {
+		notionErr := client.TransformHTTPCodeToError(b)
+		if notionErr == nil {
+			return Response{}, fmt.Errorf("GetBlocks: failed http request, %d code", res.StatusCode)
+		}
+		return Response{}, notionErr
+	}
+
+	err = json.Unmarshal(b, &objects)
+
+	if err != nil {
+		return Response{}, err
+	}
+	return objects, nil
+}

core/block/import/notion/api/block/retrieve_test.go

@@ -0,0 +1,991 @@
+package block
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/client"
+	"github.com/anytypeio/go-anytype-middleware/pb"
+)
+
+func Test_GetBlocksAndChildrenSuccessParagraph(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`
+		{
+			"object": "list",
+			"results": [
+				{
+					"object": "block",
+					"id": "a80ae792-b87e-48d2-b24c-c32c1e14d509",
+					"parent": {
+						"type": "page_id",
+						"page_id": "088b08d5-b692-4805-8338-1b147a3bff4a"
+					},
+					"created_time": "2022-11-14T11:52:00.000Z",
+					"last_edited_time": "2022-11-14T12:18:00.000Z",
+					"created_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"last_edited_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"has_children": false,
+					"archived": false,
+					"type": "paragraph",
+					"paragraph": {
+						"rich_text": [
+							{
+								"type": "text",
+								"text": {
+									"content": "dsadasd sdasd\n",
+									"link": null
+								},
+								"annotations": {
+									"bold": false,
+									"italic": false,
+									"strikethrough": false,
+									"underline": false,
+									"code": false,
+									"color": "default"
+								},
+								"plain_text": "dsadasd sdasd\n",
+								"href": null
+							},
+							{
+								"type": "text",
+								"text": {
+									"content": "asd ",
+									"link": null
+								},
+								"annotations": {
+									"bold": true,
+									"italic": false,
+									"strikethrough": false,
+									"underline": false,
+									"code": false,
+									"color": "default"
+								},
+								"plain_text": "asd ",
+								"href": null
+							},
+							{
+								"type": "text",
+								"text": {
+									"content": "asdasd.  \n",
+									"link": null
+								},
+								"annotations": {
+									"bold": true,
+									"italic": true,
+									"strikethrough": false,
+									"underline": true,
+									"code": false,
+									"color": "default"
+								},
+								"plain_text": "asdasd.  \n",
+								"href": null
+							},
+							{
+								"type": "text",
+								"text": {
+									"content": "asdasd",
+									"link": null
+								},
+								"annotations": {
+									"bold": true,
+									"italic": true,
+									"strikethrough": false,
+									"underline": true,
+									"code": false,
+									"color": "orange_background"
+								},
+								"plain_text": "asdasd",
+								"href": null
+							}
+						],
+						"color": "green_background"
+					}
+				}
+				
+			],
+			"next_cursor": null,
+			"has_more": false,
+			"type": "block",
+			"block": {}
+		}
+		`))
+	}))
+
+	defer s.Close()
+	pageSize := int64(100)
+	c := client.NewClient()
+	c.BasePath = s.URL
+
+	blockService := New(c)
+	bl, err := blockService.GetBlocksAndChildren(context.TODO(), "id", "key", pageSize, pb.RpcObjectImportRequest_ALL_OR_NOTHING)
+	assert.Nil(t, err)
+	assert.Len(t, bl, 1)
+	_, ok := bl[0].(*ParagraphBlock)
+	assert.True(t, ok)
+}
+
+func Test_GetBlocksAndChildrenSuccessHeading3(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`
+		{
+			"object": "list",
+			"results": [
+				{
+					"object": "block",
+					"id": "968c10fd-39f5-4a31-8a47-719778d9cb22",
+					"parent": {
+						"type": "page_id",
+						"page_id": "088b08d5-b692-4805-8338-1b147a3bff4a"
+					},
+					"created_time": "2022-11-14T11:54:00.000Z",
+					"last_edited_time": "2022-11-14T11:54:00.000Z",
+					"created_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"last_edited_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"has_children": false,
+					"archived": false,
+					"type": "heading_3",
+					"heading_3": {
+						"rich_text": [
+							{
+								"type": "text",
+								"text": {
+									"content": "Heading 3",
+									"link": null
+								},
+								"annotations": {
+									"bold": false,
+									"italic": false,
+									"strikethrough": true,
+									"underline": true,
+									"code": false,
+									"color": "blue_background"
+								},
+								"plain_text": "Heading 3",
+								"href": null
+							}
+						],
+						"is_toggleable": false,
+						"color": "default"
+					}
+				}
+			],
+			"next_cursor": null,
+			"has_more": false,
+			"type": "block",
+			"block": {}
+		}
+		`))
+	}))
+
+	defer s.Close()
+	pageSize := int64(100)
+	c := client.NewClient()
+	c.BasePath = s.URL
+
+	blockService := New(c)
+	bl, err := blockService.GetBlocksAndChildren(context.TODO(), "id", "key", pageSize, pb.RpcObjectImportRequest_ALL_OR_NOTHING)
+	assert.Nil(t, err)
+	assert.Len(t, bl, 1)
+	_, ok := bl[0].(*Heading3Block)
+	assert.True(t, ok)
+}
+
+func Test_GetBlocksAndChildrenSuccessTodo(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`
+		{
+			"object": "list",
+			"results": [
+				{
+					"object": "block",
+					"id": "c0c29ebb-3064-466c-b058-54f07128a1e9",
+					"parent": {
+						"type": "page_id",
+						"page_id": "088b08d5-b692-4805-8338-1b147a3bff4a"
+					},
+					"created_time": "2022-11-14T11:53:00.000Z",
+					"last_edited_time": "2022-11-14T11:54:00.000Z",
+					"created_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"last_edited_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"has_children": false,
+					"archived": false,
+					"type": "to_do",
+					"to_do": {
+						"rich_text": [],
+						"checked": false,
+						"color": "default"
+					}
+				}
+			],
+			"next_cursor": null,
+			"has_more": false,
+			"type": "block",
+			"block": {}
+		}
+		`))
+	}))
+
+	defer s.Close()
+	pageSize := int64(100)
+	c := client.NewClient()
+	c.BasePath = s.URL
+
+	blockService := New(c)
+	bl, err := blockService.GetBlocksAndChildren(context.TODO(), "id", "key", pageSize, pb.RpcObjectImportRequest_ALL_OR_NOTHING)
+	assert.Nil(t, err)
+	assert.Len(t, bl, 1)
+	_, ok := bl[0].(*ToDoBlock)
+	assert.True(t, ok)
+}
+
+func Test_GetBlocksAndChildrenSuccessHeading2(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`
+		{
+			"object": "list",
+			"results": [
+				{
+					"object": "block",
+					"id": "1d5f7d59-32aa-46dc-aec7-e275fdd56752",
+					"parent": {
+						"type": "page_id",
+						"page_id": "088b08d5-b692-4805-8338-1b147a3bff4a"
+					},
+					"created_time": "2022-11-14T11:54:00.000Z",
+					"last_edited_time": "2022-11-14T11:54:00.000Z",
+					"created_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"last_edited_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"has_children": false,
+					"archived": false,
+					"type": "heading_2",
+					"heading_2": {
+						"rich_text": [
+							{
+								"type": "text",
+								"text": {
+									"content": "Heading 2",
+									"link": null
+								},
+								"annotations": {
+									"bold": true,
+									"italic": true,
+									"strikethrough": false,
+									"underline": false,
+									"code": false,
+									"color": "default"
+								},
+								"plain_text": "Heading 2",
+								"href": null
+							}
+						],
+						"is_toggleable": false,
+						"color": "default"
+					}
+				}
+			],
+			"next_cursor": null,
+			"has_more": false,
+			"type": "block",
+			"block": {}
+		}
+		`))
+	}))
+
+	defer s.Close()
+	pageSize := int64(100)
+	c := client.NewClient()
+	c.BasePath = s.URL
+
+	blockService := New(c)
+	bl, err := blockService.GetBlocksAndChildren(context.TODO(), "id", "key", pageSize, pb.RpcObjectImportRequest_ALL_OR_NOTHING)
+	assert.Nil(t, err)
+	assert.Len(t, bl, 1)
+	_, ok := bl[0].(*Heading2Block)
+	assert.True(t, ok)
+}
+
+func Test_GetBlocksAndChildrenSuccessBulletList(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`
+		{
+			"object": "list",
+			"results": [
+				{
+					"object": "block",
+					"id": "152b978d-ee32-498c-a9db-985677a6dce6",
+					"parent": {
+						"type": "page_id",
+						"page_id": "088b08d5-b692-4805-8338-1b147a3bff4a"
+					},
+					"created_time": "2022-11-14T11:55:00.000Z",
+					"last_edited_time": "2022-11-14T11:55:00.000Z",
+					"created_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"last_edited_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"has_children": false,
+					"archived": false,
+					"type": "bulleted_list_item",
+					"bulleted_list_item": {
+						"rich_text": [
+							{
+								"type": "text",
+								"text": {
+									"content": "buller",
+									"link": null
+								},
+								"annotations": {
+									"bold": false,
+									"italic": false,
+									"strikethrough": false,
+									"underline": false,
+									"code": false,
+									"color": "default"
+								},
+								"plain_text": "buller",
+								"href": null
+							}
+						],
+						"color": "default"
+					}
+				}
+			],
+			"next_cursor": null,
+			"has_more": false,
+			"type": "block",
+			"block": {}
+		}
+		`))
+	}))
+
+	defer s.Close()
+	pageSize := int64(100)
+	c := client.NewClient()
+	c.BasePath = s.URL
+
+	blockService := New(c)
+	bl, err := blockService.GetBlocksAndChildren(context.TODO(), "id", "key", pageSize, pb.RpcObjectImportRequest_ALL_OR_NOTHING)
+	assert.Nil(t, err)
+	assert.Len(t, bl, 1)
+	_, ok := bl[0].(*BulletedListBlock)
+	assert.True(t, ok)
+}
+
+func Test_GetBlocksAndChildrenSuccessNumberedList(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`
+		{
+			"object": "list",
+			"results": [
+				{
+					"object": "block",
+					"id": "38fc4773-8b28-445c-83bf-cb8c06badf10",
+					"parent": {
+						"type": "page_id",
+						"page_id": "088b08d5-b692-4805-8338-1b147a3bff4a"
+					},
+					"created_time": "2022-11-14T11:55:00.000Z",
+					"last_edited_time": "2022-11-14T12:17:00.000Z",
+					"created_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"last_edited_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"has_children": false,
+					"archived": false,
+					"type": "numbered_list_item",
+					"numbered_list_item": {
+						"rich_text": [
+							{
+								"type": "text",
+								"text": {
+									"content": "Number",
+									"link": null
+								},
+								"annotations": {
+									"bold": false,
+									"italic": false,
+									"strikethrough": false,
+									"underline": false,
+									"code": false,
+									"color": "default"
+								},
+								"plain_text": "Number",
+								"href": null
+							}
+						],
+						"color": "default"
+					}
+				}
+			],
+			"next_cursor": null,
+			"has_more": false,
+			"type": "block",
+			"block": {}
+		}
+		`))
+	}))
+
+	defer s.Close()
+	pageSize := int64(100)
+	c := client.NewClient()
+	c.BasePath = s.URL
+
+	blockService := New(c)
+	bl, err := blockService.GetBlocksAndChildren(context.TODO(), "id", "key", pageSize, pb.RpcObjectImportRequest_ALL_OR_NOTHING)
+	assert.Nil(t, err)
+	assert.Len(t, bl, 1)
+	_, ok := bl[0].(*NumberedListBlock)
+	assert.True(t, ok)
+}
+
+func Test_GetBlocksAndChildrenSuccessToggle(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`
+		{
+			"object": "list",
+			"results": [
+				{
+					"object": "block",
+					"id": "ac80ca02-f09c-49f9-bc6f-2079058f1923",
+					"parent": {
+						"type": "page_id",
+						"page_id": "088b08d5-b692-4805-8338-1b147a3bff4a"
+					},
+					"created_time": "2022-11-14T11:55:00.000Z",
+					"last_edited_time": "2022-11-14T11:55:00.000Z",
+					"created_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"last_edited_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"has_children": false,
+					"archived": false,
+					"type": "toggle",
+					"toggle": {
+						"rich_text": [
+							{
+								"type": "text",
+								"text": {
+									"content": "Toggle",
+									"link": null
+								},
+								"annotations": {
+									"bold": true,
+									"italic": false,
+									"strikethrough": false,
+									"underline": false,
+									"code": true,
+									"color": "default"
+								},
+								"plain_text": "Toggle",
+								"href": null
+							}
+						],
+						"color": "default"
+					}
+				}
+			],
+			"next_cursor": null,
+			"has_more": false,
+			"type": "block",
+			"block": {}
+		}
+		`))
+	}))
+
+	defer s.Close()
+	pageSize := int64(100)
+	c := client.NewClient()
+	c.BasePath = s.URL
+
+	blockService := New(c)
+	bl, err := blockService.GetBlocksAndChildren(context.TODO(), "id", "key", pageSize, pb.RpcObjectImportRequest_ALL_OR_NOTHING)
+	assert.Nil(t, err)
+	assert.Len(t, bl, 1)
+	_, ok := bl[0].(*ToggleBlock)
+	assert.True(t, ok)
+}
+
+func Test_GetBlocksAndChildrenSuccessQuote(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`
+		{
+			"object": "list",
+			"results": [
+				{
+					"object": "block",
+					"id": "659d5d72-2a8d-4df2-9475-9bd2ac816ddd",
+					"parent": {
+						"type": "page_id",
+						"page_id": "088b08d5-b692-4805-8338-1b147a3bff4a"
+					},
+					"created_time": "2022-11-14T11:55:00.000Z",
+					"last_edited_time": "2022-11-14T11:56:00.000Z",
+					"created_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"last_edited_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"has_children": false,
+					"archived": false,
+					"type": "quote",
+					"quote": {
+						"rich_text": [
+							{
+								"type": "text",
+								"text": {
+									"content": "Quote",
+									"link": {
+										"url": "ref"
+									}
+								},
+								"annotations": {
+									"bold": true,
+									"italic": false,
+									"strikethrough": true,
+									"underline": false,
+									"code": false,
+									"color": "yellow_background"
+								},
+								"plain_text": "Quote",
+								"href": "ref"
+							}
+						],
+						"color": "default"
+					}
+				}
+			],
+			"next_cursor": null,
+			"has_more": false,
+			"type": "block",
+			"block": {}
+		}
+		`))
+	}))
+
+	defer s.Close()
+	pageSize := int64(100)
+	c := client.NewClient()
+	c.BasePath = s.URL
+
+	blockService := New(c)
+	bl, err := blockService.GetBlocksAndChildren(context.TODO(), "id", "key", pageSize, pb.RpcObjectImportRequest_ALL_OR_NOTHING)
+	assert.Nil(t, err)
+	assert.Len(t, bl, 1)
+	_, ok := bl[0].(*QuoteBlock)
+	assert.True(t, ok)
+}
+
+func Test_GetBlocksAndChildrenSuccessCallout(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`
+		{
+			"object": "list",
+			"results": [
+				{
+					"object": "block",
+					"id": "b17fb388-715c-4f3d-841c-7492d4c29e39",
+					"parent": {
+						"type": "page_id",
+						"page_id": "088b08d5-b692-4805-8338-1b147a3bff4a"
+					},
+					"created_time": "2022-11-14T11:56:00.000Z",
+					"last_edited_time": "2022-11-14T12:17:00.000Z",
+					"created_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"last_edited_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"has_children": false,
+					"archived": false,
+					"type": "callout",
+					"callout": {
+						"rich_text": [
+							{
+								"type": "text",
+								"text": {
+									"content": "BBBBBBB",
+									"link": null
+								},
+								"annotations": {
+									"bold": false,
+									"italic": false,
+									"strikethrough": false,
+									"underline": false,
+									"code": false,
+									"color": "default"
+								},
+								"plain_text": "BBBBBBB",
+								"href": null
+							}
+						],
+						"icon": {
+							"type": "file",
+							"file": {
+								"url": "url",
+								"expiry_time": "2022-11-14T14:38:56.733Z"
+							}
+						},
+						"color": "gray_background"
+					}
+				}
+			],
+			"next_cursor": null,
+			"has_more": false,
+			"type": "block",
+			"block": {}
+		}
+		`))
+	}))
+
+	defer s.Close()
+	pageSize := int64(100)
+	c := client.NewClient()
+	c.BasePath = s.URL
+
+	blockService := New(c)
+	bl, err := blockService.GetBlocksAndChildren(context.TODO(), "id", "key", pageSize, pb.RpcObjectImportRequest_ALL_OR_NOTHING)
+	assert.Nil(t, err)
+	assert.Len(t, bl, 1)
+	_, ok := bl[0].(*CalloutBlock)
+	assert.True(t, ok)
+}
+
+func Test_GetBlocksAndChildrenSuccessCode(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`
+		{
+			"object": "list",
+			"results": [
+				{
+					"object": "block",
+					"id": "b4b694d7-aed7-4ceb-aa22-f48026b07f5d",
+					"parent": {
+						"type": "page_id",
+						"page_id": "088b08d5-b692-4805-8338-1b147a3bff4a"
+					},
+					"created_time": "2022-11-14T12:22:00.000Z",
+					"last_edited_time": "2022-11-14T12:22:00.000Z",
+					"created_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"last_edited_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"has_children": false,
+					"archived": false,
+					"type": "code",
+					"code": {
+						"caption": [],
+						"rich_text": [
+							{
+								"type": "text",
+								"text": {
+									"content": "Code",
+									"link": null
+								},
+								"annotations": {
+									"bold": false,
+									"italic": false,
+									"strikethrough": false,
+									"underline": false,
+									"code": false,
+									"color": "default"
+								},
+								"plain_text": "Code",
+								"href": null
+							}
+						],
+						"language": "html"
+					}
+				}
+			],
+			"next_cursor": null,
+			"has_more": false,
+			"type": "block",
+			"block": {}
+		}
+		`))
+	}))
+
+	defer s.Close()
+	pageSize := int64(100)
+	c := client.NewClient()
+	c.BasePath = s.URL
+
+	blockService := New(c)
+	bl, err := blockService.GetBlocksAndChildren(context.TODO(), "id", "key", pageSize, pb.RpcObjectImportRequest_ALL_OR_NOTHING)
+	assert.Nil(t, err)
+	assert.Len(t, bl, 1)
+	_, ok := bl[0].(*CodeBlock)
+	assert.True(t, ok)
+}
+
+func Test_GetBlocksAndChildrenSuccessError(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusBadRequest)
+		w.Write([]byte(`{"object":"error","status":404,"code":"object_not_found","message":"Could not find block with ID: d6917e78-3212-444d-ae46-97499c021f2d. Make sure the relevant pages and databases are shared with your integration."}`))
+	}))
+
+	defer s.Close()
+	pageSize := int64(100)
+	c := client.NewClient()
+	c.BasePath = s.URL
+
+	blockService := New(c)
+	bl, err := blockService.GetBlocksAndChildren(context.TODO(), "id", "key", pageSize, pb.RpcObjectImportRequest_ALL_OR_NOTHING)
+	assert.NotNil(t, err)
+	assert.Empty(t, bl)
+}
+
+func TestTableBlocks(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`
+		{
+			"object": "list",
+			"results": [
+				{
+					"object": "block",
+					"id": "25377f65-71cf-4779-9829-de3717767148",
+					"parent": {
+						"type": "block_id",
+						"block_id": "049ab49c-17a5-4c03-bdbf-71811b4524b7"
+					},
+					"created_time": "2022-12-09T08:39:00.000Z",
+					"last_edited_time": "2022-12-09T08:40:00.000Z",
+					"created_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"last_edited_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"has_children": false,
+					"archived": false,
+					"type": "table_row",
+					"table_row": {
+						"cells": [
+							[
+								{
+									"type": "text",
+									"text": {
+										"content": "1",
+										"link": null
+									},
+									"annotations": {
+										"bold": false,
+										"italic": false,
+										"strikethrough": false,
+										"underline": false,
+										"code": false,
+										"color": "default"
+									},
+									"plain_text": "1",
+									"href": null
+								}
+							],
+							[],
+							[
+								{
+									"type": "text",
+									"text": {
+										"content": "dddd",
+										"link": null
+									},
+									"annotations": {
+										"bold": false,
+										"italic": false,
+										"strikethrough": false,
+										"underline": false,
+										"code": false,
+										"color": "pink"
+									},
+									"plain_text": "dddd",
+									"href": null
+								}
+							],
+							[]
+						]
+					}
+				},
+				{
+					"object": "block",
+					"id": "f9e3bf51-eb64-45c7-bf68-2776d808e503",
+					"parent": {
+						"type": "block_id",
+						"block_id": "049ab49c-17a5-4c03-bdbf-71811b4524b7"
+					},
+					"created_time": "2022-12-09T08:39:00.000Z",
+					"last_edited_time": "2022-12-09T08:40:00.000Z",
+					"created_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"last_edited_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"has_children": false,
+					"archived": false,
+					"type": "table_row",
+					"table_row": {
+						"cells": [
+							[],
+							[
+								{
+									"type": "text",
+									"text": {
+										"content": "fsdf",
+										"link": null
+									},
+									"annotations": {
+										"bold": false,
+										"italic": true,
+										"strikethrough": false,
+										"underline": false,
+										"code": false,
+										"color": "default"
+									},
+									"plain_text": "fsdf",
+									"href": null
+								}
+							],
+							[],
+							[]
+						]
+					}
+				},
+				{
+					"object": "block",
+					"id": "1f68a81c-ba09-4f1a-ae99-a999dee96b07",
+					"parent": {
+						"type": "block_id",
+						"block_id": "049ab49c-17a5-4c03-bdbf-71811b4524b7"
+					},
+					"created_time": "2022-12-09T08:39:00.000Z",
+					"last_edited_time": "2022-12-09T08:40:00.000Z",
+					"created_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"last_edited_by": {
+						"object": "user",
+						"id": "60faafc6-0c5c-4479-a3f7-67d77cd8a56d"
+					},
+					"has_children": false,
+					"archived": false,
+					"type": "table_row",
+					"table_row": {
+						"cells": [
+							[
+								{
+									"type": "text",
+									"text": {
+										"content": "fdsdf",
+										"link": null
+									},
+									"annotations": {
+										"bold": true,
+										"italic": false,
+										"strikethrough": false,
+										"underline": false,
+										"code": false,
+										"color": "default"
+									},
+									"plain_text": "fdsdf",
+									"href": null
+								}
+							],
+							[],
+							[
+								{
+									"type": "text",
+									"text": {
+										"content": "sdf",
+										"link": null
+									},
+									"annotations": {
+										"bold": false,
+										"italic": false,
+										"strikethrough": false,
+										"underline": false,
+										"code": false,
+										"color": "gray_background"
+									},
+									"plain_text": "sdf",
+									"href": null
+								}
+							],
+							[]
+						]
+					}
+				}
+			],
+			"next_cursor": null,
+			"has_more": false,
+			"type": "block",
+			"block": {}
+		}
+		`))
+	}))
+
+	defer s.Close()
+	pageSize := int64(100)
+	c := client.NewClient()
+	c.BasePath = s.URL
+
+	blockService := New(c)
+	bl, err := blockService.GetBlocksAndChildren(context.TODO(), "id", "key", pageSize, pb.RpcObjectImportRequest_ALL_OR_NOTHING)
+	assert.Nil(t, err)
+	assert.NotNil(t, bl)
+	assert.Len(t, bl, 3)
+}

core/block/import/notion/api/block/table.go

@@ -0,0 +1,183 @@
+package block
+
+import (
+	"github.com/globalsign/mgo/bson"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/editor/table"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+)
+
+// columnLayoutBlock + tableRowLayoutBlock + table
+const numberOfDefaultTableBlocks = 3
+
+type TableBlock struct {
+	Block
+	Table TableObject `json:"table"`
+}
+
+type TableObject struct {
+	Width           int64            `json:"table_width"`
+	HasColumnHeader bool             `json:"has_column_header"`
+	HasRowHeader    bool             `json:"has_row_header"`
+	Children        []*TableRowBlock `json:"children"`
+}
+
+type TableRowBlock struct {
+	Block
+	TableRowObject TableRowObject `json:"table_row"`
+}
+
+type TableRowObject struct {
+	Cells [][]api.RichText `json:"cells"`
+}
+
+func (t *TableBlock) GetID() string {
+	return t.ID
+}
+
+func (t *TableBlock) HasChild() bool {
+	return t.HasChildren
+}
+
+func (t *TableBlock) SetChildren(children []interface{}) {
+	t.Table.Children = make([]*TableRowBlock, 0, len(children))
+	for _, ch := range children {
+		t.Table.Children = append(t.Table.Children, ch.(*TableRowBlock))
+	}
+}
+
+func (t *TableBlock) GetBlocks(req *MapRequest) *MapResponse {
+	columnsBlocks, columnsBlocksIDs, columnLayoutBlockID, columnLayoutBlock := t.getColumns()
+
+	tableResponse := &MapResponse{}
+	tableRowBlocks, tableRowBlocksIDs, rowTextBlocks := t.getRows(req, columnsBlocksIDs, tableResponse)
+
+	tableRowBlockID, tableRowLayoutBlock := t.getLayoutRowBlock(tableRowBlocksIDs)
+
+	rootID, table := t.getTableBlock(columnLayoutBlockID, tableRowBlockID)
+
+	resultNumberOfBlocks := len(columnsBlocks) + len(rowTextBlocks) + len(tableRowBlocks) + numberOfDefaultTableBlocks
+	allBlocks := make([]*model.Block, 0, resultNumberOfBlocks)
+	allBlocks = append(allBlocks, table)
+	allBlocks = append(allBlocks, columnLayoutBlock)
+	allBlocks = append(allBlocks, columnsBlocks...)
+	allBlocks = append(allBlocks, tableRowLayoutBlock)
+	allBlocks = append(allBlocks, tableRowBlocks...)
+	allBlocks = append(allBlocks, rowTextBlocks...)
+
+	allBlocksIDs := make([]string, 0, resultNumberOfBlocks)
+	allBlocksIDs = append(allBlocksIDs, rootID)
+	allBlocksIDs = append(allBlocksIDs, columnLayoutBlockID)
+	allBlocksIDs = append(allBlocksIDs, columnsBlocksIDs...)
+	allBlocksIDs = append(allBlocksIDs, tableRowBlockID)
+	for _, b := range rowTextBlocks {
+		allBlocksIDs = append(allBlocksIDs, b.Id)
+	}
+
+	tableResponse.BlockIDs = allBlocksIDs
+	tableResponse.Blocks = allBlocks
+	return tableResponse
+}
+
+func (*TableBlock) getTableBlock(columnLayoutBlockID string, tableRowBlockID string) (string, *model.Block) {
+	rootID := bson.NewObjectId().Hex()
+	table := &model.Block{
+		Id:          rootID,
+		ChildrenIds: []string{columnLayoutBlockID, tableRowBlockID},
+		Content: &model.BlockContentOfTable{
+			Table: &model.BlockContentTable{},
+		},
+	}
+	return rootID, table
+}
+
+func (*TableBlock) getLayoutRowBlock(children []string) (string, *model.Block) {
+	tableRowBlockID := bson.NewObjectId().Hex()
+	tableRowLayoutBlock := &model.Block{
+		Id:          tableRowBlockID,
+		ChildrenIds: children,
+		Content: &model.BlockContentOfLayout{
+			Layout: &model.BlockContentLayout{
+				Style: model.BlockContentLayout_TableRows,
+			},
+		},
+	}
+	return tableRowBlockID, tableRowLayoutBlock
+}
+
+func (t *TableBlock) getRows(req *MapRequest,
+	columnsBlocksIDs []string,
+	tableResponse *MapResponse) ([]*model.Block, []string, []*model.Block) {
+	var (
+		needHeader        = true
+		tableRowBlocks    = make([]*model.Block, 0, len(t.Table.Children))
+		tableRowBlocksIDs = make([]string, 0, len(t.Table.Children))
+		rowTextBlocks     = make([]*model.Block, 0)
+	)
+	for _, trb := range t.Table.Children {
+		var childBlockIDsCurrRow []string
+		id := bson.NewObjectId().Hex()
+		for i, c := range trb.TableRowObject.Cells {
+			to := &TextObject{
+				RichText: c,
+			}
+			resp := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, req)
+
+			resp.BlockIDs = make([]string, 0, len(c))
+			for _, b := range resp.Blocks {
+				b.Id = table.MakeCellID(id, columnsBlocksIDs[i])
+				resp.BlockIDs = append(resp.BlockIDs, b.Id)
+			}
+			rowTextBlocks = append(rowTextBlocks, resp.Blocks...)
+			childBlockIDsCurrRow = append(childBlockIDsCurrRow, resp.BlockIDs...)
+		}
+
+		var isHeader bool
+		if needHeader {
+			isHeader = t.Table.HasRowHeader
+			needHeader = false
+		}
+
+		tableRowBlocks = append(tableRowBlocks, &model.Block{
+			Id:          id,
+			ChildrenIds: childBlockIDsCurrRow,
+			Content: &model.BlockContentOfTableRow{
+				TableRow: &model.BlockContentTableRow{
+					IsHeader: isHeader,
+				},
+			},
+		})
+		tableRowBlocksIDs = append(tableRowBlocksIDs, id)
+	}
+
+	return tableRowBlocks, tableRowBlocksIDs, rowTextBlocks
+}
+
+func (t *TableBlock) getColumns() ([]*model.Block, []string, string, *model.Block) {
+	columnsBlocks := make([]*model.Block, 0, t.Table.Width)
+	columnsBlocksIDs := make([]string, 0, t.Table.Width)
+	for i := 0; i < int(t.Table.Width); i++ {
+		id := bson.NewObjectId().Hex()
+		columnsBlocks = append(columnsBlocks, &model.Block{
+			Id:          id,
+			ChildrenIds: []string{},
+			Content: &model.BlockContentOfTableColumn{
+				TableColumn: &model.BlockContentTableColumn{},
+			},
+		})
+		columnsBlocksIDs = append(columnsBlocksIDs, id)
+	}
+
+	columnLayoutBlockID := bson.NewObjectId().Hex()
+	columnLayoutBlock := &model.Block{
+		Id:          columnLayoutBlockID,
+		ChildrenIds: columnsBlocksIDs,
+		Content: &model.BlockContentOfLayout{
+			Layout: &model.BlockContentLayout{
+				Style: model.BlockContentLayout_TableColumns,
+			},
+		},
+	}
+	return columnsBlocks, columnsBlocksIDs, columnLayoutBlockID, columnLayoutBlock
+}

core/block/import/notion/api/block/table_test.go

@@ -0,0 +1,91 @@
+package block
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api"
+)
+
+func Test_TableWithOneColumnAndRow(t *testing.T) {
+
+	tb := &TableBlock{
+		Table: TableObject{
+			Width:           1,
+			HasColumnHeader: false,
+			HasRowHeader:    true,
+			Children: []*TableRowBlock{
+				{
+					TableRowObject: TableRowObject{},
+				},
+			},
+		},
+	}
+
+	resp := tb.GetBlocks(&MapRequest{})
+
+	assert.NotNil(t, resp)
+	assert.Len(t, resp.Blocks, 5) // table block + column block + row block + 2 empty text blocks
+}
+
+func Test_TableWithoutContent(t *testing.T) {
+
+	tb := &TableBlock{
+		Table: TableObject{
+			Width:           3,
+			HasColumnHeader: false,
+			HasRowHeader:    true,
+			Children: []*TableRowBlock{
+				{
+					TableRowObject: TableRowObject{},
+				},
+				{
+					TableRowObject: TableRowObject{},
+				},
+			},
+		},
+	}
+
+	assert.Len(t, tb.Table.Children, 2)
+
+	resp := tb.GetBlocks(&MapRequest{})
+
+	assert.NotNil(t, resp)
+	assert.Len(t, resp.Blocks, 8) // table block + 3 * column block + 1 column layout + 1 row layout + 3 * row block
+}
+
+func Test_TableWithDifferentText(t *testing.T) {
+
+	tb := &TableBlock{
+		Table: TableObject{
+			Width:           3,
+			HasColumnHeader: false,
+			HasRowHeader:    true,
+			Children: []*TableRowBlock{
+				{
+					TableRowObject: TableRowObject{
+						Cells: [][]api.RichText{
+							{
+								{
+									Type:      api.Text,
+									PlainText: "Text",
+								},
+							},
+						},
+					},
+				},
+				{
+					TableRowObject: TableRowObject{},
+				},
+			},
+		},
+	}
+
+	assert.Len(t, tb.Table.Children, 2)
+
+	resp := tb.GetBlocks(&MapRequest{})
+
+	assert.NotNil(t, resp)
+	assert.Len(t, resp.Blocks, 9) // table block + 3 * column block + 1 column layout + 1 row layout + 3 * row block + 1 text block
+}

core/block/import/notion/api/block/tableofcontent.go

@@ -0,0 +1,40 @@
+package block
+
+import (
+	"strings"
+
+	"github.com/globalsign/mgo/bson"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+)
+
+type TableOfContentsBlock struct {
+	Block
+	TableOfContent TableOfContentsObject `json:"table_of_contents"`
+}
+
+type TableOfContentsObject struct {
+	Color string `json:"color"`
+}
+
+func (t *TableOfContentsBlock) GetBlocks(req *MapRequest) *MapResponse {
+	id := bson.NewObjectId().Hex()
+	var color string
+	// Anytype Table Of Content doesn't support different colors of text, only background
+	if strings.HasSuffix(t.TableOfContent.Color, api.NotionBackgroundColorSuffix) {
+		color = api.NotionColorToAnytype[t.TableOfContent.Color]
+	}
+
+	block := &model.Block{
+		Id:              id,
+		BackgroundColor: color,
+		Content: &model.BlockContentOfTableOfContents{
+			TableOfContents: &model.BlockContentTableOfContents{},
+		},
+	}
+	return &MapResponse{
+		Blocks:   []*model.Block{block},
+		BlockIDs: []string{id},
+	}
+}

core/block/import/notion/api/block/text.go

@@ -0,0 +1,320 @@
+package block
+
+import (
+	"github.com/globalsign/mgo/bson"
+	"github.com/gogo/protobuf/types"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+	"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
+	textUtil "github.com/anytypeio/go-anytype-middleware/util/text"
+)
+
+type ParagraphBlock struct {
+	Block
+	Paragraph TextObjectWithChildren `json:"paragraph"`
+}
+
+func (p *ParagraphBlock) GetBlocks(req *MapRequest) *MapResponse {
+	childResp := &MapResponse{}
+	if p.HasChildren {
+		mapper := ChildrenMapper(&p.Paragraph)
+		childResp = mapper.MapChildren(req)
+	}
+	resp := p.Paragraph.GetTextBlocks(model.BlockContentText_Paragraph, childResp.BlockIDs, req)
+	resp.Merge(childResp)
+	return resp
+}
+
+func (p *ParagraphBlock) HasChild() bool {
+	return p.HasChildren
+}
+
+func (p *ParagraphBlock) SetChildren(children []interface{}) {
+	p.Paragraph.Children = children
+}
+
+func (p *ParagraphBlock) GetID() string {
+	return p.ID
+}
+
+type Heading1Block struct {
+	Block
+	Heading1 HeadingObject `json:"heading_1"`
+}
+
+func (h *Heading1Block) GetBlocks(req *MapRequest) *MapResponse {
+	return h.Heading1.GetTextBlocks(model.BlockContentText_Header1, nil, req)
+}
+
+type Heading2Block struct {
+	Block
+	Heading2 HeadingObject `json:"heading_2"`
+}
+
+func (h *Heading2Block) GetBlocks(req *MapRequest) *MapResponse {
+	return h.Heading2.GetTextBlocks(model.BlockContentText_Header2, nil, req)
+}
+
+type Heading3Block struct {
+	Block
+	Heading3 HeadingObject `json:"heading_3"`
+}
+
+func (p *Heading3Block) GetBlocks(req *MapRequest) *MapResponse {
+	return p.Heading3.GetTextBlocks(model.BlockContentText_Header3, nil, req)
+}
+
+type HeadingObject struct {
+	TextObject
+	IsToggleable bool `json:"is_toggleable"`
+}
+
+type CalloutBlock struct {
+	Block
+	Callout CalloutObject `json:"callout"`
+}
+
+type CalloutObject struct {
+	TextObjectWithChildren
+	Icon *api.Icon `json:"icon"`
+}
+
+func (c *CalloutBlock) GetBlocks(req *MapRequest) *MapResponse {
+	calloutResp := c.Callout.GetTextBlocks(model.BlockContentText_Callout, nil, req)
+	extendedBlocks := make([]*model.Block, 0, len(calloutResp.Blocks))
+	for _, cb := range calloutResp.Blocks {
+		text, ok := cb.Content.(*model.BlockContentOfText)
+		if !ok {
+			extendedBlocks = append(extendedBlocks, cb)
+			continue
+		}
+		if c.Callout.Icon != nil {
+			if c.Callout.Icon.Emoji != nil {
+				text.Text.IconEmoji = *c.Callout.Icon.Emoji
+			}
+			if c.Callout.Icon.Type == api.External && c.Callout.Icon.External != nil {
+				text.Text.IconImage = c.Callout.Icon.External.URL
+			}
+			if c.Callout.Icon.Type == api.File && c.Callout.Icon.File != nil {
+				text.Text.IconImage = c.Callout.Icon.File.URL
+			}
+		}
+		cb.Content = text
+		extendedBlocks = append(extendedBlocks, cb)
+	}
+	calloutResp.Blocks = extendedBlocks
+	return calloutResp
+}
+
+func (c *CalloutBlock) HasChild() bool {
+	return c.HasChildren
+}
+
+func (c *CalloutBlock) SetChildren(children []interface{}) {
+	c.Callout.Children = children
+}
+
+func (c *CalloutBlock) GetID() string {
+	return c.ID
+}
+
+type QuoteBlock struct {
+	Block
+	Quote TextObjectWithChildren `json:"quote"`
+}
+
+func (q *QuoteBlock) GetBlocks(req *MapRequest) *MapResponse {
+	childResp := &MapResponse{}
+	if q.HasChildren {
+		mapper := ChildrenMapper(&q.Quote)
+		childResp = mapper.MapChildren(req)
+	}
+	resp := q.Quote.GetTextBlocks(model.BlockContentText_Quote, childResp.BlockIDs, req)
+	resp.Merge(childResp)
+	return resp
+}
+
+func (q *QuoteBlock) HasChild() bool {
+	return q.HasChildren
+}
+
+func (q *QuoteBlock) SetChildren(children []interface{}) {
+	q.Quote.Children = children
+}
+
+func (q *QuoteBlock) GetID() string {
+	return q.ID
+}
+
+type NumberedListBlock struct {
+	Block
+	NumberedList TextObjectWithChildren `json:"numbered_list_item"`
+}
+
+func (n *NumberedListBlock) GetBlocks(req *MapRequest) *MapResponse {
+	childResp := &MapResponse{}
+	if n.HasChildren {
+		mapper := ChildrenMapper(&n.NumberedList)
+		childResp = mapper.MapChildren(req)
+	}
+	resp := n.NumberedList.GetTextBlocks(model.BlockContentText_Numbered, childResp.BlockIDs, req)
+	resp.Merge(childResp)
+	return resp
+}
+
+func (n *NumberedListBlock) HasChild() bool {
+	return n.HasChildren
+}
+
+func (n *NumberedListBlock) SetChildren(children []interface{}) {
+	n.NumberedList.Children = children
+}
+
+func (n *NumberedListBlock) GetID() string {
+	return n.ID
+}
+
+type ToDoBlock struct {
+	Block
+	ToDo ToDoObject `json:"to_do"`
+}
+
+func (t *ToDoBlock) GetBlocks(req *MapRequest) *MapResponse {
+	childResp := &MapResponse{}
+	if t.HasChildren {
+		mapper := ChildrenMapper(&t.ToDo)
+		childResp = mapper.MapChildren(req)
+	}
+	resp := t.ToDo.GetTextBlocks(model.BlockContentText_Checkbox, childResp.BlockIDs, req)
+	resp.Merge(childResp)
+	return resp
+}
+
+func (t *ToDoBlock) HasChild() bool {
+	return t.HasChildren
+}
+
+func (t *ToDoBlock) SetChildren(children []interface{}) {
+	t.ToDo.Children = children
+}
+
+func (t *ToDoBlock) GetID() string {
+	return t.ID
+}
+
+type ToDoObject struct {
+	TextObjectWithChildren
+	Checked bool `json:"checked"`
+}
+
+type BulletedListBlock struct {
+	Block
+	BulletedList TextObjectWithChildren `json:"bulleted_list_item"`
+}
+
+func (b *BulletedListBlock) GetBlocks(req *MapRequest) *MapResponse {
+	childResp := &MapResponse{}
+	if b.HasChildren {
+		mapper := ChildrenMapper(&b.BulletedList)
+		childResp = mapper.MapChildren(req)
+	}
+	resp := b.BulletedList.GetTextBlocks(model.BlockContentText_Marked, childResp.BlockIDs, req)
+	resp.Merge(childResp)
+	return resp
+}
+
+func (b *BulletedListBlock) HasChild() bool {
+	return b.HasChildren
+}
+
+func (b *BulletedListBlock) SetChildren(children []interface{}) {
+	b.BulletedList.Children = children
+}
+
+func (b *BulletedListBlock) GetID() string {
+	return b.ID
+}
+
+type ToggleBlock struct {
+	Block
+	Toggle TextObjectWithChildren `json:"toggle"`
+}
+
+func (t *ToggleBlock) GetBlocks(req *MapRequest) *MapResponse {
+	childResp := &MapResponse{}
+	if t.HasChildren {
+		mapper := ChildrenMapper(&t.Toggle)
+		childResp = mapper.MapChildren(req)
+	}
+	resp := t.Toggle.GetTextBlocks(model.BlockContentText_Toggle, childResp.BlockIDs, req)
+	resp.Merge(childResp)
+	return resp
+}
+
+func (t *ToggleBlock) HasChild() bool {
+	return t.HasChildren
+}
+
+func (t *ToggleBlock) SetChildren(children []interface{}) {
+	t.Toggle.Children = children
+}
+
+func (t *ToggleBlock) GetID() string {
+	return t.ID
+}
+
+type CodeBlock struct {
+	Block
+	Code CodeObject `json:"code"`
+}
+
+type CodeObject struct {
+	RichText []api.RichText `json:"rich_text"`
+	Caption  []api.RichText `json:"caption"`
+	Language string         `json:"language"`
+}
+
+func (c *CodeBlock) GetBlocks(req *MapRequest) *MapResponse {
+	id := bson.NewObjectId().Hex()
+	bl := &model.Block{
+		Id: id,
+		Fields: &types.Struct{
+			Fields: map[string]*types.Value{"lang": pbtypes.String(c.Code.Language)},
+		},
+	}
+	marks := []*model.BlockContentTextMark{}
+	var code string
+	for _, rt := range c.Code.RichText {
+		from := textUtil.UTF16RuneCountString(code)
+		code += rt.PlainText
+		to := textUtil.UTF16RuneCountString(code)
+		marks = append(marks, rt.BuildMarkdownFromAnnotations(int32(from), int32(to))...)
+	}
+	bl.Content = &model.BlockContentOfText{
+		Text: &model.BlockContentText{
+			Text:  code,
+			Style: model.BlockContentText_Code,
+			Marks: &model.BlockContentTextMarks{
+				Marks: marks,
+			},
+		},
+	}
+	return &MapResponse{
+		Blocks:   []*model.Block{bl},
+		BlockIDs: []string{id},
+	}
+}
+
+type EquationBlock struct {
+	Block
+	Equation api.EquationObject `json:"equation"`
+}
+
+func (e *EquationBlock) GetBlocks(req *MapRequest) *MapResponse {
+	bl := e.Equation.HandleEquation()
+	return &MapResponse{
+		Blocks:   []*model.Block{bl},
+		BlockIDs: []string{bl.Id},
+	}
+}

core/block/import/notion/api/block/text_test.go

@@ -0,0 +1,191 @@
+package block
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+)
+
+func Test_GetTextBlocksTextSuccess(t *testing.T) {
+	to := &TextObject{
+		RichText: []api.RichText{
+			{
+				Type:      api.Text,
+				PlainText: "test",
+			},
+			{
+				Type:      api.Text,
+				PlainText: "test2",
+			},
+		},
+		Color: api.RedBackGround,
+	}
+
+	bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{})
+	assert.Len(t, bl.Blocks, 1)
+	assert.Equal(t, bl.Blocks[0].GetText().Style, model.BlockContentText_Paragraph)
+	assert.Equal(t, bl.Blocks[0].BackgroundColor, api.AnytypeRed)
+	assert.Equal(t, bl.Blocks[0].GetText().Text, "testtest2")
+}
+
+func Test_GetTextBlocksTextUserMention(t *testing.T) {
+	to := &TextObject{
+		RichText: []api.RichText{
+			{
+				Type: api.Mention,
+				Mention: &api.MentionObject{
+					Type: api.UserMention,
+					User: &api.User{
+						ID:   "id",
+						Name: "Nastya",
+					},
+				},
+				PlainText: "Nastya",
+			},
+		},
+	}
+
+	bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{})
+	assert.Len(t, bl.Blocks, 1)
+	assert.Equal(t, bl.Blocks[0].GetText().Style, model.BlockContentText_Paragraph)
+	assert.Len(t, bl.Blocks[0].GetText().Marks.Marks, 0)
+	assert.Equal(t, bl.Blocks[0].GetText().Text, "Nastya")
+}
+
+func Test_GetTextBlocksTextPageMention(t *testing.T) {
+	to := &TextObject{
+		RichText: []api.RichText{
+			{
+				Type: api.Mention,
+				Mention: &api.MentionObject{
+					Type: api.Page,
+					Page: &api.PageMention{
+						ID: "notionID",
+					},
+				},
+			},
+		},
+	}
+
+	bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{
+		NotionPageIdsToAnytype: map[string]string{"notionID": "anytypeID"},
+		PageNameToID:           map[string]string{"notionID": "Page"},
+	})
+	assert.Len(t, bl.Blocks, 1)
+	assert.Equal(t, bl.Blocks[0].GetText().Style, model.BlockContentText_Paragraph)
+	assert.Len(t, bl.Blocks[0].GetText().Marks.Marks, 1)
+	assert.Equal(t, bl.Blocks[0].GetText().Marks.Marks[0].Type, model.BlockContentTextMark_Mention)
+	assert.Equal(t, bl.Blocks[0].GetText().Text, "Page")
+}
+
+func Test_GetTextBlocksDatabaseMention(t *testing.T) {
+	to := &TextObject{
+		RichText: []api.RichText{
+			{
+				Type: api.Mention,
+				Mention: &api.MentionObject{
+					Type: api.Database,
+					Database: &api.DatabaseMention{
+						ID: "notionID",
+					},
+				},
+			},
+		},
+	}
+
+	bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{
+		NotionDatabaseIdsToAnytype: map[string]string{"notionID": "anytypeID"},
+		DatabaseNameToID:           map[string]string{"notionID": "Database"},
+	})
+	assert.Len(t, bl.Blocks, 1)
+	assert.Equal(t, bl.Blocks[0].GetText().Style, model.BlockContentText_Paragraph)
+	assert.Len(t, bl.Blocks[0].GetText().Marks.Marks, 1)
+	assert.Equal(t, bl.Blocks[0].GetText().Marks.Marks[0].Type, model.BlockContentTextMark_Mention)
+	assert.Equal(t, bl.Blocks[0].GetText().Text, "Database")
+}
+
+func Test_GetTextBlocksDateMention(t *testing.T) {
+	to := &TextObject{
+		RichText: []api.RichText{
+			{
+				Type: api.Mention,
+				Mention: &api.MentionObject{
+					Type: api.Date,
+					Date: &api.DateObject{
+						Start: "2022-11-14",
+					},
+				},
+			},
+		},
+	}
+
+	bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{})
+	assert.Len(t, bl.Blocks, 1)
+	assert.Equal(t, bl.Blocks[0].GetText().Style, model.BlockContentText_Paragraph)
+	assert.Len(t, bl.Blocks[0].GetText().Marks.Marks, 1)
+	assert.Equal(t, bl.Blocks[0].GetText().Marks.Marks[0].Type, model.BlockContentTextMark_Mention)
+	assert.Equal(t, bl.Blocks[0].GetText().Marks.Marks[0].Param, "_date_2022-11-14")
+}
+
+func Test_GetTextBlocksLinkPreview(t *testing.T) {
+	to := &TextObject{
+		RichText: []api.RichText{
+			{
+				Type: api.Mention,
+				Mention: &api.MentionObject{
+					Type: api.LinkPreview,
+					LinkPreview: &api.Link{
+						URL: "ref",
+					},
+				},
+			},
+		},
+	}
+
+	bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{})
+	assert.Len(t, bl.Blocks, 1)
+	assert.Equal(t, bl.Blocks[0].GetText().Style, model.BlockContentText_Paragraph)
+	assert.NotNil(t, bl.Blocks[0].GetText().Marks)
+	assert.Len(t, bl.Blocks[0].GetText().Marks.Marks, 1)
+	assert.Equal(t, bl.Blocks[0].GetText().Marks.Marks[0].Type, model.BlockContentTextMark_Link)
+	assert.Equal(t, bl.Blocks[0].GetText().Text, "ref")
+}
+
+func Test_GetTextBlocksEquation(t *testing.T) {
+	to := &TextObject{
+		RichText: []api.RichText{
+			{
+				Type: api.Equation,
+				Equation: &api.EquationObject{
+					Expression: "Equation",
+				},
+			},
+		},
+	}
+
+	bl := to.GetTextBlocks(model.BlockContentText_Paragraph, nil, &MapRequest{})
+	assert.Len(t, bl.Blocks, 1)
+	assert.NotNil(t, bl.Blocks[0].GetLatex())
+	assert.Equal(t, bl.Blocks[0].GetLatex().Text, "Equation")
+}
+
+func Test_GetCodeBlocksSuccess(t *testing.T) {
+	co := &CodeBlock{
+		Code: CodeObject{
+			RichText: []api.RichText{
+				{
+					Type:      api.Text,
+					PlainText: "Code",
+				},
+			},
+			Language: "Go",
+		},
+	}
+	bl := co.GetBlocks(&MapRequest{})
+	assert.NotNil(t, bl)
+	assert.Len(t, bl.Blocks, 1)
+	assert.Equal(t, bl.Blocks[0].GetText().Text, "Code")
+}

core/block/import/notion/api/block/textobject.go

@@ -0,0 +1,242 @@
+package block
+
+import (
+	"strings"
+	"time"
+
+	"github.com/globalsign/mgo/bson"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+	textUtil "github.com/anytypeio/go-anytype-middleware/util/text"
+)
+
+const DateMentionTimeFormat = "2006-01-02"
+
+type ChildrenMapper interface {
+	MapChildren(req *MapRequest) *MapResponse
+}
+
+type TextObject struct {
+	RichText []api.RichText `json:"rich_text"`
+	Color    string         `json:"color"`
+}
+
+func (t *TextObject) GetTextBlocks(style model.BlockContentTextStyle, childIds []string, req *MapRequest) *MapResponse {
+	marks := []*model.BlockContentTextMark{}
+	id := bson.NewObjectId().Hex()
+	allBlocks := make([]*model.Block, 0)
+	allIds := make([]string, 0)
+	var (
+		text strings.Builder
+	)
+	for _, rt := range t.RichText {
+		if rt.Type == api.Text {
+			marks = append(marks, t.handleTextType(rt, &text, req.NotionPageIdsToAnytype, req.NotionDatabaseIdsToAnytype)...)
+		}
+		if rt.Type == api.Mention {
+			marks = append(marks, t.handleMentionType(rt, &text, req)...)
+		}
+		if rt.Type == api.Equation {
+			eqBlock := rt.Equation.HandleEquation()
+			allBlocks = append(allBlocks, eqBlock)
+			allIds = append(allIds, eqBlock.Id)
+		}
+	}
+	var backgroundColor, textColor string
+	if strings.Contains(t.Color, api.NotionBackgroundColorSuffix) {
+		backgroundColor = api.NotionColorToAnytype[t.Color]
+	} else {
+		textColor = api.NotionColorToAnytype[t.Color]
+	}
+
+	if t.isNotTextBlocks() {
+		return &MapResponse{
+			Blocks:   allBlocks,
+			BlockIDs: allIds,
+		}
+	}
+	allBlocks = append(allBlocks, &model.Block{
+		Id:              id,
+		ChildrenIds:     childIds,
+		BackgroundColor: backgroundColor,
+		Content: &model.BlockContentOfText{
+			Text: &model.BlockContentText{
+				Text:    text.String(),
+				Style:   style,
+				Marks:   &model.BlockContentTextMarks{Marks: marks},
+				Checked: false,
+				Color:   textColor,
+			},
+		},
+	})
+	for _, b := range allBlocks {
+		allIds = append(allIds, b.Id)
+	}
+	return &MapResponse{
+		Blocks:   allBlocks,
+		BlockIDs: allIds,
+	}
+}
+
+func (t *TextObject) handleTextType(rt api.RichText,
+	text *strings.Builder,
+	notionPageIdsToAnytype,
+	notionDatabaseIdsToAnytype map[string]string) []*model.BlockContentTextMark {
+	marks := []*model.BlockContentTextMark{}
+	from := textUtil.UTF16RuneCountString(text.String())
+	if rt.Text != nil && rt.Text.Link != nil && rt.Text.Link.URL != "" {
+		text.WriteString(rt.Text.Content)
+	} else {
+		text.WriteString(rt.PlainText)
+	}
+	to := textUtil.UTF16RuneCountString(text.String())
+	if rt.Text != nil && rt.Text.Link != nil && rt.Text.Link.URL != "" {
+		url := strings.Trim(rt.Text.Link.URL, "/")
+		if databaseID, ok := notionDatabaseIdsToAnytype[url]; ok {
+			url = databaseID
+		}
+		if pageID, ok := notionPageIdsToAnytype[url]; ok {
+			url = pageID
+		}
+		marks = append(marks, &model.BlockContentTextMark{
+			Range: &model.Range{
+				From: int32(from),
+				To:   int32(to),
+			},
+			Type:  model.BlockContentTextMark_Link,
+			Param: url,
+		})
+	}
+	marks = append(marks, rt.BuildMarkdownFromAnnotations(int32(from), int32(to))...)
+	return marks
+}
+
+func (t *TextObject) handleMentionType(rt api.RichText,
+	text *strings.Builder,
+	req *MapRequest) []*model.BlockContentTextMark {
+	if rt.Mention.Type == api.UserMention {
+		return t.handleUserMention(rt, text)
+	}
+	if rt.Mention.Type == api.Database {
+		return t.handleDatabaseMention(rt, text, req.NotionDatabaseIdsToAnytype, req.DatabaseNameToID)
+	}
+	if rt.Mention.Type == api.Page {
+		return t.handlePageMention(rt, text, req.NotionPageIdsToAnytype, req.PageNameToID)
+	}
+	if rt.Mention.Type == api.LinkPreview {
+		return t.handleLinkPreviewMention(rt, text)
+	}
+	if rt.Mention.Type == api.Date {
+		return t.handleDateMention(rt, text)
+	}
+	return nil
+}
+
+func (t *TextObject) handleUserMention(rt api.RichText, text *strings.Builder) []*model.BlockContentTextMark {
+	from := textUtil.UTF16RuneCountString(text.String())
+	text.WriteString(rt.PlainText)
+	to := textUtil.UTF16RuneCountString(text.String())
+	return rt.BuildMarkdownFromAnnotations(int32(from), int32(to))
+}
+
+func (t *TextObject) handleDatabaseMention(rt api.RichText,
+	text *strings.Builder,
+	notionDatabaseIdsToAnytype, databaseNameToID map[string]string) []*model.BlockContentTextMark {
+	if notionDatabaseIdsToAnytype == nil {
+		return nil
+	}
+	from := textUtil.UTF16RuneCountString(text.String())
+	text.WriteString(databaseNameToID[rt.Mention.Database.ID])
+	to := textUtil.UTF16RuneCountString(text.String())
+	marks := rt.BuildMarkdownFromAnnotations(int32(from), int32(to))
+	marks = append(marks, &model.BlockContentTextMark{
+		Range: &model.Range{
+			From: int32(from),
+			To:   int32(to),
+		},
+		Type:  model.BlockContentTextMark_Mention,
+		Param: notionDatabaseIdsToAnytype[rt.Mention.Database.ID],
+	})
+	return marks
+}
+
+func (t *TextObject) handlePageMention(rt api.RichText,
+	text *strings.Builder,
+	notionPageIdsToAnytype, pageNameToID map[string]string) []*model.BlockContentTextMark {
+	if notionPageIdsToAnytype == nil {
+		return nil
+	}
+	from := textUtil.UTF16RuneCountString(text.String())
+	text.WriteString(pageNameToID[rt.Mention.Page.ID])
+	to := textUtil.UTF16RuneCountString(text.String())
+	marks := rt.BuildMarkdownFromAnnotations(int32(from), int32(to))
+	marks = append(marks, &model.BlockContentTextMark{
+		Range: &model.Range{
+			From: int32(from),
+			To:   int32(to),
+		},
+		Type:  model.BlockContentTextMark_Mention,
+		Param: notionPageIdsToAnytype[rt.Mention.Page.ID],
+	})
+	return marks
+}
+
+func (t *TextObject) handleDateMention(rt api.RichText,
+	text *strings.Builder) []*model.BlockContentTextMark {
+	var textDate string
+	if rt.Mention.Date.Start != "" {
+		textDate = rt.Mention.Date.Start
+	}
+	if rt.Mention.Date.End != "" {
+		textDate = rt.Mention.Date.End
+	}
+	date, err := time.Parse(DateMentionTimeFormat, textDate)
+	if err != nil {
+		return nil
+	}
+	from := textUtil.UTF16RuneCountString(text.String())
+	to := textUtil.UTF16RuneCountString(text.String())
+	return []*model.BlockContentTextMark{
+		{
+			Range: &model.Range{
+				From: int32(from),
+				To:   int32(to),
+			},
+			Type:  model.BlockContentTextMark_Mention,
+			Param: addr.TimeToID(date),
+		},
+	}
+}
+
+func (t *TextObject) handleLinkPreviewMention(rt api.RichText, text *strings.Builder) []*model.BlockContentTextMark {
+	from := textUtil.UTF16RuneCountString(text.String())
+	text.WriteString(rt.Mention.LinkPreview.URL)
+	to := textUtil.UTF16RuneCountString(text.String())
+	marks := rt.BuildMarkdownFromAnnotations(int32(from), int32(to))
+	marks = append(marks, &model.BlockContentTextMark{
+		Range: &model.Range{
+			From: int32(from),
+			To:   int32(to),
+		},
+		Type: model.BlockContentTextMark_Link,
+	})
+	return marks
+}
+
+func (t *TextObject) isNotTextBlocks() bool {
+	return len(t.RichText) == 1 && t.RichText[0].Type == api.Equation
+}
+
+type TextObjectWithChildren struct {
+	TextObject
+	Children []interface{} `json:"children"`
+}
+
+func (t *TextObjectWithChildren) MapChildren(req *MapRequest) *MapResponse {
+	childReq := *req
+	childReq.Blocks = t.Children
+	resp := MapBlocks(&childReq)
+	return resp
+}

core/block/import/notion/api/client/client.go

@@ -0,0 +1,48 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+)
+
+const (
+	notionURL  = "https://api.notion.com/v1"
+	apiVersion = "2022-06-28"
+)
+
+type Client struct {
+	HTTPClient *http.Client
+	BasePath   string
+}
+
+// NewClient is a constructor for Client
+func NewClient() *Client {
+	c := &Client{
+		HTTPClient: &http.Client{Timeout: time.Minute},
+		BasePath:   notionURL,
+	}
+	return c
+}
+
+// PrepareRequest create http.Request based on given method, url and body
+func (c *Client) PrepareRequest(ctx context.Context,
+	apiKey, method, url string,
+	body *bytes.Buffer) (*http.Request, error) {
+	resultURL := c.BasePath + url
+	req, err := http.NewRequestWithContext(ctx, method, resultURL, body)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", apiKey))
+	req.Header.Set("Notion-Version", apiVersion)
+
+	if body != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+
+	return req, nil
+}

core/block/import/notion/api/client/error.go

@@ -0,0 +1,24 @@
+package client
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/logging"
+)
+
+type NotionErrorResponse struct {
+	Status  int    `json:"status,omitempty"`
+	Code    string `json:"code,omitempty"`
+	Message string `json:"message,omitempty"`
+}
+
+// and creates error based on NotionErrorResponse
+func TransformHTTPCodeToError(response []byte) error {
+	var notionErr NotionErrorResponse
+	if err := json.Unmarshal(response, &notionErr); err != nil {
+		logging.Logger("client").Error("failed to parse error response from notion %s", err)
+		return nil
+	}
+	return fmt.Errorf("status: %d, code: %s, message: %s", notionErr.Status, notionErr.Code, notionErr.Message)
+}

core/block/import/notion/api/commonobjects.go

@@ -0,0 +1,316 @@
+package api
+
+import (
+	"encoding/json"
+	"strings"
+	"time"
+
+	"github.com/globalsign/mgo/bson"
+
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+)
+
+type RichTextType string
+
+const (
+	Text     RichTextType = "text"
+	Mention  RichTextType = "mention"
+	Equation RichTextType = "equation"
+)
+
+const NotionBackgroundColorSuffix = "background"
+
+// RichText represent RichText object from Notion https://developers.notion.com/reference/rich-text
+type RichText struct {
+	Type        RichTextType    `json:"type,omitempty"`
+	Text        *TextObject     `json:"text,omitempty"`
+	Mention     *MentionObject  `json:"mention,omitempty"`
+	Equation    *EquationObject `json:"equation,omitempty"`
+	Annotations *Annotations    `json:"annotations,omitempty"`
+	PlainText   string          `json:"plain_text,omitempty"`
+	Href        string          `json:"href,omitempty"`
+}
+type TextObject struct {
+	Content string `json:"content"`
+	Link    *Link  `json:"link,omitempty"`
+}
+type EquationObject struct {
+	Expression string `json:"expression"`
+}
+
+func (e *EquationObject) HandleEquation() *model.Block {
+	id := bson.NewObjectId().Hex()
+	return &model.Block{
+		Id:          id,
+		ChildrenIds: []string{},
+		Content: &model.BlockContentOfLatex{
+			Latex: &model.BlockContentLatex{
+				Text: e.Expression,
+			},
+		},
+	}
+}
+
+type Link struct {
+	URL string `json:"url,omitempty"`
+}
+
+func (rt *RichText) BuildMarkdownFromAnnotations(from, to int32) []*model.BlockContentTextMark {
+	marks := []*model.BlockContentTextMark{}
+	if rt.Annotations == nil {
+		return marks
+	}
+	if rt.Annotations.Bold {
+		marks = append(marks, &model.BlockContentTextMark{
+			Range: &model.Range{
+				From: from,
+				To:   to,
+			},
+			Type: model.BlockContentTextMark_Bold,
+		})
+	}
+	if rt.Annotations.Italic {
+		marks = append(marks, &model.BlockContentTextMark{
+			Range: &model.Range{
+				From: from,
+				To:   to,
+			},
+			Type: model.BlockContentTextMark_Italic,
+		})
+	}
+	if rt.Annotations.Strikethrough {
+		marks = append(marks, &model.BlockContentTextMark{
+			Range: &model.Range{
+				From: from,
+				To:   to,
+			},
+			Type: model.BlockContentTextMark_Strikethrough,
+		})
+	}
+	if rt.Annotations.Underline {
+		marks = append(marks, &model.BlockContentTextMark{
+			Range: &model.Range{
+				From: from,
+				To:   to,
+			},
+			Type: model.BlockContentTextMark_Underscored,
+		})
+	}
+	if rt.Annotations.Color != "" {
+		markType := model.BlockContentTextMark_TextColor
+		if strings.HasSuffix(rt.Annotations.Color, NotionBackgroundColorSuffix) {
+			markType = model.BlockContentTextMark_BackgroundColor
+		}
+		marks = append(marks, &model.BlockContentTextMark{
+			Range: &model.Range{
+				From: from,
+				To:   to,
+			},
+			Type:  markType,
+			Param: NotionColorToAnytype[rt.Annotations.Color],
+		})
+	}
+
+	if rt.Annotations.Code {
+		marks = append(marks, &model.BlockContentTextMark{
+			Range: &model.Range{
+				From: from,
+				To:   to,
+			},
+			Type: model.BlockContentTextMark_Keyboard,
+		})
+	}
+
+	return marks
+}
+
+type mentionType string
+
+const (
+	UserMention mentionType = "user"
+	Page        mentionType = "page"
+	Database    mentionType = "database"
+	Date        mentionType = "date"
+	LinkPreview mentionType = "link_preview"
+)
+
+type MentionObject struct {
+	Type        mentionType      `json:"type,omitempty"`
+	User        *User            `json:"user,omitempty"`
+	Page        *PageMention     `json:"page,omitempty"`
+	Database    *DatabaseMention `json:"database,omitempty"`
+	Date        *DateObject      `json:"date,omitempty"`
+	LinkPreview *Link            `json:"link_preview,omitempty"`
+}
+
+type PageMention struct {
+	ID string `json:"id"`
+}
+
+type DatabaseMention struct {
+	ID string `json:"id"`
+}
+
+type DateObject struct {
+	Start    string `json:"start"`
+	End      string `json:"end"`
+	TimeZone string `json:"time_zone"`
+}
+
+const (
+	DefaultColor string = "default"
+	Gray         string = "gray"
+	Brown        string = "brown"
+	Orange       string = "orange"
+	Yellow       string = "yellow"
+	Green        string = "green"
+	Blue         string = "blue"
+	Purple       string = "purple"
+	Pink         string = "pink"
+	Red          string = "red"
+
+	GrayBackGround   string = "gray_background"
+	BrownBackGround  string = "brown_background"
+	OrangeBackGround string = "orange_background"
+	YellowBackGround string = "yellow_background"
+	GreenBackGround  string = "green_background"
+	BlueBackGround   string = "blue_background"
+	PurpleBackGround string = "purple_background"
+	PinkBackGround   string = "pink_background"
+	RedBackGround    string = "red_background"
+
+	AnytypeGray    string = "gray"
+	AnytypeOrange  string = "orange"
+	AnytypeYellow  string = "yellow"
+	AnytypeGreen   string = "lime"
+	AnytypeBlue    string = "blue"
+	AnytypePurple  string = "purple"
+	AnytypePink    string = "pink"
+	AnytypeRed     string = "red"
+	AnytypeDefault string = "default"
+)
+
+var NotionColorToAnytype = map[string]string{
+	DefaultColor: AnytypeDefault,
+	Gray:         AnytypeGray,
+	Brown:        "",
+	Orange:       AnytypeOrange,
+	Yellow:       AnytypeYellow,
+	Green:        AnytypeGreen,
+	Blue:         AnytypeBlue,
+	Purple:       AnytypePurple,
+	Pink:         AnytypePink,
+	Red:          AnytypeRed,
+
+	GrayBackGround:   AnytypeGray,
+	BrownBackGround:  "",
+	OrangeBackGround: AnytypeOrange,
+	YellowBackGround: AnytypeYellow,
+	GreenBackGround:  AnytypeGreen,
+	BlueBackGround:   AnytypeBlue,
+	PurpleBackGround: AnytypePurple,
+	PinkBackGround:   AnytypePink,
+	RedBackGround:    AnytypeRed,
+}
+
+type Annotations struct {
+	Bold          bool   `json:"bold"`
+	Italic        bool   `json:"italic"`
+	Strikethrough bool   `json:"strikethrough"`
+	Underline     bool   `json:"underline"`
+	Code          bool   `json:"code"`
+	Color         string `json:"color"`
+}
+
+type FileType string
+
+const (
+	External FileType = "external"
+	File     FileType = "file"
+)
+
+// FileObject represent File Object object from Notion https://developers.notion.com/reference/file-object
+type FileObject struct {
+	Name     string       `json:"name"`
+	Type     FileType     `json:"type"`
+	File     FileProperty `json:"file,omitempty"`
+	External FileProperty `json:"external,omitempty"`
+}
+
+func (f *FileObject) GetFileBlock(fileType model.BlockContentFileType) (*model.Block, string) {
+	id := bson.NewObjectId().Hex()
+	name := f.External.URL
+	if name == "" {
+		name = f.File.URL
+	}
+	return &model.Block{
+		Id: id,
+		Content: &model.BlockContentOfFile{
+			File: &model.BlockContentFile{
+				Name:    name,
+				AddedAt: time.Now().Unix(),
+				Type:    fileType,
+			},
+		},
+	}, id
+}
+
+type FileProperty struct {
+	URL        string     `json:"url,omitempty"`
+	ExpiryTime *time.Time `json:"expiry_time,omitempty"`
+}
+
+func (o *FileProperty) UnmarshalJSON(data []byte) error {
+	fp := make(map[string]interface{}, 0)
+	if err := json.Unmarshal(data, &fp); err != nil {
+		return err
+	}
+	if url, ok := fp["url"].(string); ok {
+		o.URL = url
+	}
+	if t, ok := fp["expiry_time"].(*time.Time); ok {
+		o.ExpiryTime = t
+	}
+	return nil
+}
+
+type Icon struct {
+	Type     FileType      `json:"type"`
+	Emoji    *string       `json:"emoji,omitempty"`
+	File     *FileProperty `json:"file,omitempty"`
+	External *FileProperty `json:"external,omitempty"`
+}
+
+type userType string
+
+// User represent User Object object from Notion https://developers.notion.com/reference/user
+type User struct {
+	Object    string    `json:"object,omitempty"`
+	ID        string    `json:"id"`
+	Type      userType  `json:"type,omitempty"`
+	Name      string    `json:"name,omitempty"`
+	AvatarURL string    `json:"avatar_url,omitempty"`
+	Person    *Person   `json:"person,omitempty"`
+	Bot       *struct{} `json:"bot,omitempty"`
+}
+
+type Person struct {
+	Email string `json:"email"`
+}
+
+type Parent struct {
+	Type       string `json:"type,omitempty"`
+	PageID     string `json:"page_id"`
+	DatabaseID string `json:"database_id"`
+}
+
+func RichTextToDescription(rt []*RichText) string {
+	var richText strings.Builder
+	for i, title := range rt {
+		richText.WriteString(title.PlainText)
+		if i != len(rt)-1 {
+			richText.WriteString("\n")
+		}
+	}
+	return richText.String()
+}

core/block/import/notion/api/commonobjects_test.go

@@ -0,0 +1,243 @@
+package api
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+)
+
+func Test_BuildMarkdownFromAnnotationsBold(t *testing.T) {
+	rt := &RichText{
+		Annotations: &Annotations{
+			Bold:          true,
+			Italic:        false,
+			Strikethrough: false,
+			Underline:     false,
+			Code:          false,
+			Color:         "",
+		},
+	}
+	marks := rt.BuildMarkdownFromAnnotations(0, 5)
+	assert.Len(t, marks, 1)
+	assert.Equal(t, marks[0].Type, model.BlockContentTextMark_Bold)
+}
+
+func Test_BuildMarkdownFromAnnotationsItalic(t *testing.T) {
+	rt := &RichText{
+		Annotations: &Annotations{
+			Bold:          false,
+			Italic:        true,
+			Strikethrough: false,
+			Underline:     false,
+			Code:          false,
+			Color:         "",
+		},
+	}
+	marks := rt.BuildMarkdownFromAnnotations(0, 5)
+	assert.Len(t, marks, 1)
+	assert.Equal(t, marks[0].Type, model.BlockContentTextMark_Italic)
+}
+
+func Test_BuildMarkdownFromAnnotationsStrikethrough(t *testing.T) {
+	rt := &RichText{
+		Annotations: &Annotations{
+			Bold:          false,
+			Italic:        false,
+			Strikethrough: true,
+			Underline:     false,
+			Code:          false,
+			Color:         "",
+		},
+	}
+	marks := rt.BuildMarkdownFromAnnotations(0, 5)
+	assert.Len(t, marks, 1)
+	assert.Equal(t, marks[0].Type, model.BlockContentTextMark_Strikethrough)
+}
+
+func Test_BuildMarkdownFromAnnotationsUnderline(t *testing.T) {
+	rt := &RichText{
+		Annotations: &Annotations{
+			Bold:          false,
+			Italic:        false,
+			Strikethrough: false,
+			Underline:     true,
+			Code:          false,
+			Color:         "",
+		},
+	}
+	marks := rt.BuildMarkdownFromAnnotations(0, 5)
+	assert.Len(t, marks, 1)
+	assert.Equal(t, marks[0].Type, model.BlockContentTextMark_Underscored)
+}
+
+func Test_BuildMarkdownFromAnnotationsTwoMarks(t *testing.T) {
+	rt := &RichText{
+		Annotations: &Annotations{
+			Bold:          true,
+			Italic:        true,
+			Strikethrough: false,
+			Underline:     false,
+			Code:          false,
+			Color:         "",
+		},
+	}
+	marks := rt.BuildMarkdownFromAnnotations(0, 5)
+	assert.Len(t, marks, 2)
+	assert.Equal(t, marks[0].Type, model.BlockContentTextMark_Bold)
+	assert.Equal(t, marks[1].Type, model.BlockContentTextMark_Italic)
+}
+
+func Test_BuildMarkdownFromAnnotationsColor(t *testing.T) {
+	rt := &RichText{
+		Annotations: &Annotations{
+			Bold:          false,
+			Italic:        false,
+			Strikethrough: false,
+			Underline:     false,
+			Code:          false,
+			Color:         "red",
+		},
+	}
+	marks := rt.BuildMarkdownFromAnnotations(0, 5)
+	assert.Len(t, marks, 1)
+	assert.Equal(t, marks[0].Param, "red")
+}
+
+func Test_GetFileBlockImage(t *testing.T) {
+	f := &FileObject{
+		Name: "file",
+		File: FileProperty{
+			URL:        "",
+			ExpiryTime: &time.Time{},
+		},
+		External: FileProperty{
+			URL:        "https:/example.ru/",
+			ExpiryTime: &time.Time{},
+		},
+	}
+	imageBlock, _ := f.GetFileBlock(model.BlockContentFile_Image)
+	assert.NotNil(t, imageBlock.GetFile())
+	assert.Equal(t, imageBlock.GetFile().Name, "https:/example.ru/")
+	assert.Equal(t, imageBlock.GetFile().Type, model.BlockContentFile_Image)
+
+	f = &FileObject{
+		Name: "file",
+		File: FileProperty{
+			URL:        "https:/example.ru/",
+			ExpiryTime: &time.Time{},
+		},
+		External: FileProperty{
+			URL:        "",
+			ExpiryTime: &time.Time{},
+		},
+	}
+	imageBlock, _ = f.GetFileBlock(model.BlockContentFile_Image)
+	assert.NotNil(t, imageBlock.GetFile())
+	assert.Equal(t, imageBlock.GetFile().Name, "https:/example.ru/")
+	assert.Equal(t, imageBlock.GetFile().Type, model.BlockContentFile_Image)
+}
+
+func Test_GetFileBlockPdf(t *testing.T) {
+	f := &FileObject{
+		Name: "file",
+		File: FileProperty{
+			URL:        "",
+			ExpiryTime: &time.Time{},
+		},
+		External: FileProperty{
+			URL:        "https:/example.ru/",
+			ExpiryTime: &time.Time{},
+		},
+	}
+	imageBlock, _ := f.GetFileBlock(model.BlockContentFile_PDF)
+	assert.NotNil(t, imageBlock.GetFile())
+	assert.Equal(t, imageBlock.GetFile().Name, "https:/example.ru/")
+	assert.Equal(t, imageBlock.GetFile().Type, model.BlockContentFile_PDF)
+
+	f = &FileObject{
+		Name: "file",
+		File: FileProperty{
+			URL:        "https:/example.ru/",
+			ExpiryTime: &time.Time{},
+		},
+		External: FileProperty{
+			URL:        "",
+			ExpiryTime: &time.Time{},
+		},
+	}
+	imageBlock, _ = f.GetFileBlock(model.BlockContentFile_PDF)
+	assert.NotNil(t, imageBlock.GetFile())
+	assert.Equal(t, imageBlock.GetFile().Name, "https:/example.ru/")
+	assert.Equal(t, imageBlock.GetFile().Type, model.BlockContentFile_PDF)
+}
+
+func Test_GetFileBlockFile(t *testing.T) {
+	f := &FileObject{
+		Name: "file",
+		File: FileProperty{
+			URL:        "",
+			ExpiryTime: &time.Time{},
+		},
+		External: FileProperty{
+			URL:        "https:/example.ru/",
+			ExpiryTime: &time.Time{},
+		},
+	}
+	imageBlock, _ := f.GetFileBlock(model.BlockContentFile_File)
+	assert.NotNil(t, imageBlock.GetFile())
+	assert.Equal(t, imageBlock.GetFile().Name, "https:/example.ru/")
+	assert.Equal(t, imageBlock.GetFile().Type, model.BlockContentFile_File)
+
+	f = &FileObject{
+		Name: "file",
+		File: FileProperty{
+			URL:        "https:/example.ru/",
+			ExpiryTime: &time.Time{},
+		},
+		External: FileProperty{
+			URL:        "",
+			ExpiryTime: &time.Time{},
+		},
+	}
+	imageBlock, _ = f.GetFileBlock(model.BlockContentFile_File)
+	assert.NotNil(t, imageBlock.GetFile())
+	assert.Equal(t, imageBlock.GetFile().Name, "https:/example.ru/")
+	assert.Equal(t, imageBlock.GetFile().Type, model.BlockContentFile_File)
+}
+
+func Test_GetFileBlockVideo(t *testing.T) {
+	f := &FileObject{
+		Name: "file",
+		File: FileProperty{
+			URL:        "",
+			ExpiryTime: &time.Time{},
+		},
+		External: FileProperty{
+			URL:        "https:/example.ru/",
+			ExpiryTime: &time.Time{},
+		},
+	}
+	imageBlock, _ := f.GetFileBlock(model.BlockContentFile_Video)
+	assert.NotNil(t, imageBlock.GetFile())
+	assert.Equal(t, imageBlock.GetFile().Name, "https:/example.ru/")
+	assert.Equal(t, imageBlock.GetFile().Type, model.BlockContentFile_Video)
+
+	f = &FileObject{
+		Name: "file",
+		File: FileProperty{
+			URL:        "https:/example.ru/",
+			ExpiryTime: &time.Time{},
+		},
+		External: FileProperty{
+			URL:        "",
+			ExpiryTime: &time.Time{},
+		},
+	}
+	imageBlock, _ = f.GetFileBlock(model.BlockContentFile_Video)
+	assert.NotNil(t, imageBlock.GetFile())
+	assert.Equal(t, imageBlock.GetFile().Name, "https:/example.ru/")
+	assert.Equal(t, imageBlock.GetFile().Type, model.BlockContentFile_Video)
+}

core/block/import/notion/api/database/database.go

@@ -0,0 +1,151 @@
+package database
+
+import (
+	"context"
+	"time"
+
+	"github.com/gogo/protobuf/types"
+	"github.com/textileio/go-threads/core/thread"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/converter"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api"
+	"github.com/anytypeio/go-anytype-middleware/core/block/process"
+	"github.com/anytypeio/go-anytype-middleware/pb"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/threads"
+	"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
+)
+
+const ObjectType = "database"
+
+type Service struct{}
+
+// New is a constructor for Service
+func New() *Service {
+	return &Service{}
+}
+
+// Database represent Database object from Notion https://developers.notion.com/reference/database
+type Database struct {
+	Object         string          `json:"object"`
+	ID             string          `json:"id"`
+	CreatedTime    time.Time       `json:"created_time"`
+	LastEditedTime time.Time       `json:"last_edited_time"`
+	CreatedBy      api.User        `json:"created_by,omitempty"`
+	LastEditedBy   api.User        `json:"last_edited_by,omitempty"`
+	Title          []api.RichText  `json:"title"`
+	Parent         api.Parent      `json:"parent"`
+	URL            string          `json:"url"`
+	Properties     interface{}     `json:"properties"` // can't support it for databases yet
+	Description    []*api.RichText `json:"description"`
+	IsInline       bool            `json:"is_inline"`
+	Archived       bool            `json:"archived"`
+	Icon           *api.Icon       `json:"icon,omitempty"`
+	Cover          *api.FileObject `json:"cover,omitempty"`
+}
+
+func (p *Database) GetObjectType() string {
+	return ObjectType
+}
+
+// GetDatabase makes snaphots from notion Database objects
+func (ds *Service) GetDatabase(ctx context.Context,
+	mode pb.RpcObjectImportRequestMode,
+	databases []Database,
+	progress *process.Progress) (*converter.Response, map[string]string, map[string]string, converter.ConvertError) {
+	var (
+		allSnapshots       = make([]*converter.Snapshot, 0)
+		notionIdsToAnytype = make(map[string]string, 0)
+		databaseNameToID   = make(map[string]string, 0)
+		convereterError    = converter.ConvertError{}
+	)
+
+	progress.SetProgressMessage("Start creating pages from notion databases")
+	for _, d := range databases {
+		if err := progress.TryStep(1); err != nil {
+			ce := converter.NewFromError(d.ID, err)
+			return nil, nil, nil, ce
+		}
+
+		tid, err := threads.ThreadCreateID(thread.AccessControlled, smartblock.SmartBlockTypePage)
+		if err != nil {
+			convereterError.Add(d.ID, err)
+			if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
+				return nil, nil, nil, convereterError
+			}
+			continue
+		}
+		snapshot := ds.transformDatabase(d)
+
+		allSnapshots = append(allSnapshots, &converter.Snapshot{
+			Id:       tid.String(),
+			FileName: d.URL,
+			Snapshot: snapshot,
+		})
+		notionIdsToAnytype[d.ID] = tid.String()
+		databaseNameToID[d.ID] = pbtypes.GetString(snapshot.Details, bundle.RelationKeyName.String())
+	}
+	if convereterError.IsEmpty() {
+		return &converter.Response{Snapshots: allSnapshots}, notionIdsToAnytype, databaseNameToID, nil
+	}
+
+	return &converter.Response{Snapshots: allSnapshots}, notionIdsToAnytype, databaseNameToID, convereterError
+}
+
+func (ds *Service) transformDatabase(d Database) *model.SmartBlockSnapshotBase {
+	details := make(map[string]*types.Value, 0)
+	relations := make([]*converter.Relation, 0)
+	details[bundle.RelationKeySource.String()] = pbtypes.String(d.URL)
+	if len(d.Title) > 0 {
+		details[bundle.RelationKeyName.String()] = pbtypes.String(d.Title[0].PlainText)
+	}
+	if d.Icon != nil && d.Icon.Emoji != nil {
+		details[bundle.RelationKeyIconEmoji.String()] = pbtypes.String(*d.Icon.Emoji)
+	}
+
+	if d.Cover != nil {
+		var relation *converter.Relation
+
+		if d.Cover.Type == api.External {
+			details[bundle.RelationKeyCoverId.String()] = pbtypes.String(d.Cover.External.URL)
+			details[bundle.RelationKeyCoverType.String()] = pbtypes.Float64(1)
+			relation = &converter.Relation{
+				Relation: &model.Relation{
+					Name:   bundle.RelationKeyCoverId.String(),
+					Format: model.RelationFormat_file,
+				},
+			}
+		}
+
+		if d.Cover.Type == api.File {
+			details[bundle.RelationKeyCoverId.String()] = pbtypes.String(d.Cover.File.URL)
+			details[bundle.RelationKeyCoverType.String()] = pbtypes.Float64(1)
+			relation = &converter.Relation{
+				Relation: &model.Relation{
+					Name:   bundle.RelationKeyCoverId.String(),
+					Format: model.RelationFormat_file,
+				},
+			}
+		}
+
+		relations = append(relations, relation)
+	}
+	details[bundle.RelationKeyCreatedDate.String()] = pbtypes.String(d.CreatedTime.String())
+	details[bundle.RelationKeyCreator.String()] = pbtypes.String(d.CreatedBy.Name)
+	details[bundle.RelationKeyIsArchived.String()] = pbtypes.Bool(d.Archived)
+	details[bundle.RelationKeyLastModifiedDate.String()] = pbtypes.String(d.LastEditedTime.String())
+	details[bundle.RelationKeyLastModifiedBy.String()] = pbtypes.String(d.LastEditedBy.Name)
+	details[bundle.RelationKeyDescription.String()] = pbtypes.String(api.RichTextToDescription(d.Description))
+	details[bundle.RelationKeyIsFavorite.String()] = pbtypes.Bool(true)
+
+	snapshot := &model.SmartBlockSnapshotBase{
+		Blocks:      []*model.Block{},
+		Details:     &types.Struct{Fields: details},
+		ObjectTypes: []string{bundle.TypeKeyPage.URL()},
+		Collections: nil,
+	}
+
+	return snapshot
+}

core/block/import/notion/api/database/database_test.go

@@ -0,0 +1 @@
+package database

core/block/import/notion/api/page/page.go

@@ -0,0 +1,321 @@
+package page
+
+import (
+	"context"
+
+	"github.com/gogo/protobuf/types"
+	"github.com/textileio/go-threads/core/thread"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/converter"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/block"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/client"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/property"
+	"github.com/anytypeio/go-anytype-middleware/core/block/process"
+	"github.com/anytypeio/go-anytype-middleware/pb"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/logging"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/threads"
+	"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
+)
+
+var logger = logging.Logger("notion-page")
+
+const (
+	ObjectType = "page"
+	pageSize   = 100
+)
+
+type Service struct {
+	blockService    *block.Service
+	client          *client.Client
+	propertyService *property.Service
+}
+
+// New is a constructor for Service
+func New(client *client.Client) *Service {
+	return &Service{
+		blockService:    block.New(client),
+		client:          client,
+		propertyService: property.New(client),
+	}
+}
+
+// Page represents Page object from notion https://developers.notion.com/reference/page
+type Page struct {
+	Object         string              `json:"object"`
+	ID             string              `json:"id"`
+	CreatedTime    string              `json:"created_time"`
+	LastEditedTime string              `json:"last_edited_time"`
+	CreatedBy      api.User            `json:"created_by,omitempty"`
+	LastEditedBy   api.User            `json:"last_edited_by,omitempty"`
+	Parent         api.Parent          `json:"parent"`
+	Properties     property.Properties `json:"properties"`
+	Archived       bool                `json:"archived"`
+	Icon           *api.Icon           `json:"icon,omitempty"`
+	Cover          *api.FileObject     `json:"cover,omitempty"`
+	URL            string              `json:"url,omitempty"`
+}
+
+func (p *Page) GetObjectType() string {
+	return ObjectType
+}
+
+// GetPages transform Page objects from Notion to snaphots
+func (ds *Service) GetPages(ctx context.Context,
+	apiKey string,
+	mode pb.RpcObjectImportRequestMode,
+	pages []Page,
+	request *block.MapRequest,
+	progress *process.Progress) (*converter.Response, map[string]string, converter.ConvertError) {
+	var (
+		allSnapshots            = make([]*converter.Snapshot, 0)
+		convereterError         converter.ConvertError
+		notionPagesIdsToAnytype = make(map[string]string, 0)
+	)
+
+	progress.SetProgressMessage("Start creating pages from notion")
+
+	for _, p := range pages {
+		if err := progress.TryStep(1); err != nil {
+			ce := converter.NewFromError(p.ID, err)
+			return nil, nil, ce
+		}
+
+		tid, err := threads.ThreadCreateID(thread.AccessControlled, smartblock.SmartBlockTypePage)
+		if err != nil {
+			convereterError.Add(p.ID, err)
+			if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
+				return nil, nil, convereterError
+			}
+			continue
+		}
+		notionPagesIdsToAnytype[p.ID] = tid.String()
+	}
+
+	progress.SetProgressMessage("Start creating blocks")
+	relationsToPageID := make(map[string][]*converter.Relation)
+	// Need to collect pages title and notion ids mapping for such blocks as ChildPage and ChildDatabase,
+	// because we only get title in those blocks from API
+	pageNameToID := make(map[string]string, 0)
+	for _, p := range pages {
+		for _, v := range p.Properties {
+			if t, ok := v.(*property.TitleItem); ok {
+				properties, err := ds.propertyService.GetPropertyObject(ctx, p.ID, t.GetID(), apiKey, t.GetPropertyType())
+				if err != nil {
+					logger.With("method", "handlePageProperties").Errorf("failed to get paginated property, %s", v.GetPropertyType())
+					continue
+				}
+				title := make([]*api.RichText, 0, len(properties))
+				for _, o := range properties {
+					if t, ok := o.(*api.RichText); ok {
+						title = append(title, t)
+					}
+				}
+				t.Title = title
+			}
+		}
+	}
+	request.NotionPageIdsToAnytype = notionPagesIdsToAnytype
+	request.PageNameToID = pageNameToID
+	for _, p := range pages {
+		if err := progress.TryStep(1); err != nil {
+			ce := converter.NewFromError(p.ID, err)
+			return nil, nil, ce
+		}
+		snapshot, relations, ce := ds.transformPages(ctx, apiKey, p, mode, request)
+		if ce != nil {
+			convereterError.Merge(ce)
+			if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
+				return nil, nil, convereterError
+			}
+			continue
+		}
+		pageID := notionPagesIdsToAnytype[p.ID]
+		allSnapshots = append(allSnapshots, &converter.Snapshot{
+			Id:       pageID,
+			FileName: p.URL,
+			Snapshot: snapshot,
+		})
+		relationsToPageID[pageID] = relations
+	}
+	if convereterError.IsEmpty() {
+		return &converter.Response{Snapshots: allSnapshots, Relations: relationsToPageID}, notionPagesIdsToAnytype, nil
+	}
+
+	return &converter.Response{Snapshots: allSnapshots}, notionPagesIdsToAnytype, convereterError
+}
+
+func (ds *Service) transformPages(ctx context.Context,
+	apiKey string,
+	p Page,
+	mode pb.RpcObjectImportRequestMode,
+	request *block.MapRequest) (*model.SmartBlockSnapshotBase, []*converter.Relation, converter.ConvertError) {
+	details := make(map[string]*types.Value, 0)
+	details[bundle.RelationKeySource.String()] = pbtypes.String(p.URL)
+	if p.Icon != nil && p.Icon.Emoji != nil {
+		details[bundle.RelationKeyIconEmoji.String()] = pbtypes.String(*p.Icon.Emoji)
+	}
+	details[bundle.RelationKeyIsArchived.String()] = pbtypes.Bool(p.Archived)
+	details[bundle.RelationKeyIsFavorite.String()] = pbtypes.Bool(true)
+
+	allErrors := converter.ConvertError{}
+	relations := ds.handlePageProperties(ctx, apiKey, p.ID, p.Properties, details, request)
+	addFCoverDetail(p, details)
+	notionBlocks, blocksAndChildrenErr := ds.blockService.GetBlocksAndChildren(ctx, p.ID, apiKey, pageSize, mode)
+	if blocksAndChildrenErr != nil {
+		allErrors.Merge(blocksAndChildrenErr)
+		if mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
+			return nil, nil, allErrors
+		}
+	}
+
+	request.Blocks = notionBlocks
+	resp := ds.blockService.MapNotionBlocksToAnytype(request)
+	snapshot := &model.SmartBlockSnapshotBase{
+		Blocks:      resp.Blocks,
+		Details:     &types.Struct{Fields: details},
+		ObjectTypes: []string{bundle.TypeKeyPage.URL()},
+	}
+
+	return snapshot, relations, nil
+}
+
+// handlePageProperties gets properties values by their ids from notion api
+// and transforms them to Details and RelationLinks
+func (ds *Service) handlePageProperties(ctx context.Context,
+	apiKey, pageID string,
+	p property.Properties,
+	d map[string]*types.Value,
+	req *block.MapRequest) []*converter.Relation {
+	relations := make([]*converter.Relation, 0)
+	for k, v := range p {
+		if isPropertyPaginated(v) {
+			properties, err := ds.propertyService.GetPropertyObject(ctx, pageID, v.GetID(), apiKey, v.GetPropertyType())
+			if err != nil {
+				logger.With("method", "handlePageProperties").Errorf("failed to get paginated property, %s", v.GetPropertyType())
+				continue
+			}
+			ds.handlePaginatedProperty(v, properties)
+		}
+		if r, ok := v.(*property.RelationItem); ok {
+			linkRelationsIDWithAnytypeID(r, req.NotionPageIdsToAnytype, req.NotionDatabaseIdsToAnytype)
+		}
+		var (
+			ds property.DetailSetter
+			ok bool
+		)
+		if ds, ok = v.(property.DetailSetter); !ok {
+			logger.With("method", "handlePageProperties").
+				Errorf("failed to convert to interface DetailSetter, %s", v.GetPropertyType())
+			continue
+		}
+		ds.SetDetail(k, d)
+
+		rel := &converter.Relation{
+			Relation: &model.Relation{
+				Name:   k,
+				Format: v.GetFormat(),
+			},
+		}
+		if isPropertyTag(v) {
+			setOptionsForListRelation(v, rel.Relation)
+		}
+		relations = append(relations, rel)
+	}
+	return relations
+}
+
+func (*Service) handlePaginatedProperty(v property.Object, properties []interface{}) {
+	switch pr := v.(type) {
+	case *property.RelationItem:
+		relationItems := make([]*property.Relation, 0, len(properties))
+		for _, o := range properties {
+			relationItems = append(relationItems, o.(*property.Relation))
+		}
+		pr.Relation = relationItems
+	case *property.RichTextItem:
+		richText := make([]*api.RichText, 0, len(properties))
+		for _, o := range properties {
+			richText = append(richText, o.(*api.RichText))
+		}
+		pr.RichText = richText
+	case *property.PeopleItem:
+		pList := make([]*api.User, 0, len(properties))
+		for _, o := range properties {
+			pList = append(pList, o.(*api.User))
+		}
+		pr.People = pList
+	}
+}
+
+// linkRelationsIDWithAnytypeID take anytype ID based on page/database ID from Notin.
+// In property we get id from Notion, so we somehow need to map this ID with anytype for correct Relation.
+// We use two maps notionPagesIdsToAnytype, notionDatabaseIdsToAnytype for this
+func linkRelationsIDWithAnytypeID(rel *property.RelationItem,
+	notionPagesIdsToAnytype, notionDatabaseIdsToAnytype map[string]string) {
+	for _, r := range rel.Relation {
+		if anytypeID, ok := notionPagesIdsToAnytype[r.ID]; ok {
+			r.ID = anytypeID
+		}
+		if anytypeID, ok := notionDatabaseIdsToAnytype[r.ID]; ok {
+			r.ID = anytypeID
+		}
+	}
+}
+
+func addFCoverDetail(p Page, details map[string]*types.Value) {
+	if p.Cover != nil {
+		if p.Cover.Type == api.External {
+			details[bundle.RelationKeyCoverId.String()] = pbtypes.String(p.Cover.External.URL)
+			details[bundle.RelationKeyCoverType.String()] = pbtypes.Float64(1)
+		}
+
+		if p.Cover.Type == api.File {
+			details[bundle.RelationKeyCoverId.String()] = pbtypes.String(p.Cover.File.URL)
+			details[bundle.RelationKeyCoverType.String()] = pbtypes.Float64(1)
+		}
+
+	}
+
+}
+
+func isPropertyPaginated(pr property.Object) bool {
+	if r, ok := pr.(*property.RelationItem); ok && r.HasMore {
+		return true
+	}
+	return pr.GetPropertyType() == property.PropertyConfigTypeRichText ||
+		pr.GetPropertyType() == property.PropertyConfigTypePeople
+}
+
+func isPropertyTag(pr property.Object) bool {
+	return pr.GetPropertyType() == property.PropertyConfigTypeMultiSelect ||
+		pr.GetPropertyType() == property.PropertyConfigTypeSelect ||
+		pr.GetPropertyType() == property.PropertyConfigStatus
+}
+
+func setOptionsForListRelation(pr property.Object, rel *model.Relation) {
+	var text, color []string
+	switch property := pr.(type) {
+	case *property.StatusItem:
+		text = append(text, property.Status.Name)
+		color = append(color, api.NotionColorToAnytype[property.Status.Color])
+	case *property.SelectItem:
+		text = append(text, property.Select.Name)
+		color = append(color, api.NotionColorToAnytype[property.Select.Color])
+	case *property.MultiSelectItem:
+		for _, so := range property.MultiSelect {
+			text = append(text, so.Name)
+			color = append(color, api.NotionColorToAnytype[so.Color])
+		}
+	}
+
+	for i := 0; i < len(text); i++ {
+		rel.SelectDict = append(rel.SelectDict, &model.RelationOption{
+			Text:  text[i],
+			Color: color[i],
+		})
+	}
+}

core/block/import/notion/api/page/page_test.go

@@ -0,0 +1,306 @@
+package page
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gogo/protobuf/types"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/block"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/client"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/property"
+	"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
+)
+
+func Test_handlePagePropertiesSelect(t *testing.T) {
+	details := make(map[string]*types.Value, 0)
+	c := client.NewClient()
+	ps := New(c)
+
+	p := property.SelectItem{
+		Object: "",
+		ID:     "id",
+		Type:   string(property.PropertyConfigTypeSelect),
+		Select: property.SelectOption{
+			ID:    "id",
+			Name:  "Name",
+			Color: api.Blue,
+		},
+	}
+	pr := property.Properties{"Select": &p}
+	_ = ps.handlePageProperties(context.TODO(), "key", "id", pr, details, &block.MapRequest{})
+
+	assert.NotEmpty(t, details["Select"])
+}
+
+func Test_handlePagePropertiesLastEditedTime(t *testing.T) {
+	c := client.NewClient()
+	ps := New(c)
+
+	details := make(map[string]*types.Value, 0)
+
+	p := property.LastEditedTimeItem{
+		ID:             "id",
+		Type:           string(property.PropertyConfigLastEditedTime),
+		LastEditedTime: "2022-10-24T22:56:00.000Z",
+	}
+	pr := property.Properties{"LastEditedTime": &p}
+	_ = ps.handlePageProperties(context.TODO(), "key", "id", pr, details, &block.MapRequest{})
+
+	assert.NotEmpty(t, details["LastEditedTime"])
+}
+
+func Test_handlePagePropertiesRichText(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`{"object":"list","results":[{"object":"property_item","type":"rich_text","id":"RPBv","rich_text":{"type":"text","text":{"content":"sdfsdfsdfsdfsdfsdf","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"example text","href":null}}],"next_cursor":null,"has_more":false,"type":"property_item","property_item":{"id":"RPBv","next_url":null,"type":"rich_text","rich_text":{}}}`))
+	}))
+
+	c := client.NewClient()
+	c.BasePath = s.URL
+	ps := New(c)
+
+	details := make(map[string]*types.Value, 0)
+
+	p := property.RichTextItem{ID: "id", Type: string(property.PropertyConfigLastEditedTime)}
+	pr := property.Properties{"RichText": &p}
+	_ = ps.handlePageProperties(context.TODO(), "key", "id", pr, details, &block.MapRequest{})
+
+	assert.NotEmpty(t, details["RichText"])
+	assert.Equal(t, details["RichText"].GetStringValue(), "example text")
+}
+
+func Test_handlePagePropertiesStatus(t *testing.T) {
+	c := client.NewClient()
+	ps := New(c)
+
+	details := make(map[string]*types.Value, 0)
+
+	p := property.StatusItem{
+		ID:   "id",
+		Type: property.PropertyConfigStatus,
+		Status: &property.Status{
+			Name:  "Done",
+			ID:    "id",
+			Color: api.Pink,
+		},
+	}
+	pr := property.Properties{"Status": &p}
+	_ = ps.handlePageProperties(context.TODO(), "key", "id", pr, details, &block.MapRequest{})
+
+	assert.NotEmpty(t, details["Status"])
+}
+
+func Test_handlePagePropertiesNumber(t *testing.T) {
+	c := client.NewClient()
+	ps := New(c)
+
+	details := make(map[string]*types.Value, 0)
+
+	num := float64(12)
+	p := property.NumberItem{
+		ID:     "id",
+		Type:   string(property.PropertyConfigTypeNumber),
+		Number: &num,
+	}
+	pr := property.Properties{"Number": &p}
+	_ = ps.handlePageProperties(context.TODO(), "key", "id", pr, details, &block.MapRequest{})
+
+	assert.NotEmpty(t, details["Number"])
+}
+
+func Test_handlePagePropertiesMultiSelect(t *testing.T) {
+	c := client.NewClient()
+	ps := New(c)
+
+	details := make(map[string]*types.Value, 0)
+
+	p := property.MultiSelectItem{
+		ID:   "id",
+		Type: string(property.PropertyConfigTypeMultiSelect),
+		MultiSelect: []*property.SelectOption{
+			{
+				ID:    "id",
+				Name:  "Name",
+				Color: api.Blue,
+			},
+		},
+	}
+	pr := property.Properties{"MultiSelect": &p}
+	_ = ps.handlePageProperties(context.TODO(), "key", "id", pr, details, &block.MapRequest{})
+
+	assert.NotEmpty(t, details["MultiSelect"])
+}
+
+func Test_handlePagePropertiesCheckbox(t *testing.T) {
+	c := client.NewClient()
+	ps := New(c)
+
+	details := make(map[string]*types.Value, 0)
+
+	p := property.CheckboxItem{
+		ID:       "id",
+		Type:     string(property.PropertyConfigTypeCheckbox),
+		Checkbox: true,
+	}
+	pr := property.Properties{"Checkbox": &p}
+	_ = ps.handlePageProperties(context.TODO(), "key", "id", pr, details, &block.MapRequest{})
+
+	assert.NotEmpty(t, details["Checkbox"])
+}
+
+func Test_handlePagePropertiesEmail(t *testing.T) {
+	c := client.NewClient()
+	ps := New(c)
+
+	details := make(map[string]*types.Value, 0)
+
+	email := "a@mail.com"
+	p := property.EmailItem{
+		ID:    "id",
+		Type:  string(property.PropertyConfigTypeEmail),
+		Email: &email,
+	}
+	pr := property.Properties{"Email": &p}
+	_ = ps.handlePageProperties(context.TODO(), "key", "id", pr, details, &block.MapRequest{})
+
+	assert.NotEmpty(t, details["Email"])
+}
+
+func Test_handlePagePropertiesRelation(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`{"object":"list","results":[{"object":"property_item","type":"relation","id":"cm~~","relation":{"id":"id"}}],"next_cursor":null,"has_more":false,"type":"property_item","property_item":{"id":"cm~~","next_url":null,"type":"relation","relation":{}}}`))
+	}))
+
+	c := client.NewClient()
+	c.BasePath = s.URL
+	ps := New(c)
+
+	details := make(map[string]*types.Value, 0)
+
+	p := property.RelationItem{ID: "id", Type: string(property.PropertyConfigTypeRelation), HasMore: true, Relation: []*property.Relation{{ID: "id"}}}
+	pr := property.Properties{"Relation": &p}
+	notionPageIdsToAnytype := map[string]string{"id": "anytypeID"}
+	notionDatabaseIdsToAnytype := map[string]string{"id": "anytypeID"}
+	req := &block.MapRequest{NotionPageIdsToAnytype: notionPageIdsToAnytype, NotionDatabaseIdsToAnytype: notionDatabaseIdsToAnytype}
+	_ = ps.handlePageProperties(context.TODO(), "key", "id", pr, details, req)
+
+	assert.NotNil(t, details["Relation"].GetListValue())
+	assert.Len(t, details["Relation"].GetListValue().Values, 1)
+	assert.Equal(t, pbtypes.GetStringListValue(details["Relation"])[0], "anytypeID")
+}
+
+func Test_handlePagePropertiesPeople(t *testing.T) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`{"object":"list","results":[{"object":"property_item","type":"people","id":"id","people":{"object":"user","id":"1","name":"Example","avatar_url":"https://example1.com","type":"person","person":{"email":"email1@.com"}}},{"object":"property_item","type":"people","id":"id","people":{"object":"user","id":"2","name":"Example 2","avatar_url":"https://example2.com","type":"person","person":{"email":"email2@.com"}}}],"next_cursor":null,"has_more":false,"type":"property_item","property_item":{"id":"id","next_url":null,"type":"people","people":{}}}`))
+	}))
+	c := client.NewClient()
+	c.BasePath = s.URL
+	ps := New(c)
+
+	details := make(map[string]*types.Value, 0)
+
+	p := property.PeopleItem{
+		Object: "",
+		ID:     "id",
+		Type:   string(property.PropertyConfigTypePeople),
+	}
+	pr := property.Properties{"People": &p}
+	_ = ps.handlePageProperties(context.TODO(), "key", "id", pr, details, &block.MapRequest{})
+
+	assert.NotEmpty(t, details["People"])
+	assert.Len(t, pbtypes.GetStringListValue(details["People"]), 2)
+	people := pbtypes.GetStringListValue(details["People"])
+	assert.Equal(t, people[0], "Example")
+	assert.Equal(t, people[1], "Example 2")
+}
+
+func Test_handlePagePropertiesFormula(t *testing.T) {
+	c := client.NewClient()
+	ps := New(c)
+
+	details := make(map[string]*types.Value, 0)
+
+	p := property.FormulaItem{
+		ID:      "id",
+		Type:    string(property.PropertyConfigTypeFormula),
+		Formula: map[string]interface{}{"type": property.NumberFormula, "number": float64(1)},
+	}
+	pr := property.Properties{"Formula": &p}
+	_ = ps.handlePageProperties(context.TODO(), "key", "id", pr, details, &block.MapRequest{})
+
+	assert.NotEmpty(t, details["Formula"])
+}
+
+func Test_handlePagePropertiesTitle(t *testing.T) {
+	c := client.NewClient()
+	ps := New(c)
+
+	details := make(map[string]*types.Value, 0)
+
+	p := property.TitleItem{
+		ID:    "id",
+		Type:  string(property.PropertyConfigTypeTitle),
+		Title: []*api.RichText{{PlainText: "Title"}},
+	}
+	pr := property.Properties{"Title": &p}
+	_ = ps.handlePageProperties(context.TODO(), "key", "id", pr, details, &block.MapRequest{})
+
+	assert.NotEmpty(t, details["name"])
+	assert.NotEmpty(t, details["name"].GetStringValue(), "Title")
+}
+
+func Test_handleRollupProperties(t *testing.T) {
+	c := client.NewClient()
+	ps := New(c)
+
+	details := make(map[string]*types.Value, 0)
+
+	p1 := property.RollupItem{
+		ID:   "id1",
+		Type: string(property.PropertyConfigTypeRollup),
+		Rollup: property.RollupObject{
+			Type:   "number",
+			Number: 2,
+		},
+	}
+
+	p2 := property.RollupItem{
+		ID:   "id2",
+		Type: string(property.PropertyConfigTypeRollup),
+		Rollup: property.RollupObject{
+			Type: "date",
+			Date: &api.DateObject{
+				Start: "12-12-2022",
+			},
+		},
+	}
+
+	p3 := property.RollupItem{
+		ID:   "id3",
+		Type: string(property.PropertyConfigTypeRollup),
+		Rollup: property.RollupObject{
+			Type: "array",
+			Array: []interface{}{
+				map[string]interface{}{"type": "title", "title": []map[string]string{{"plain_text": "Title"}}},
+			},
+		},
+	}
+
+	pr := property.Properties{"Rollup1": &p1, "Rollup2": &p2, "Rollup3": &p3}
+	_ = ps.handlePageProperties(context.TODO(), "key", "id", pr, details, &block.MapRequest{})
+
+	assert.NotEmpty(t, details["Rollup1"])
+	assert.NotEmpty(t, details["Rollup1"].GetNumberValue(), float64(2))
+
+	assert.NotEmpty(t, details["Rollup2"])
+	assert.NotEmpty(t, details["Rollup2"].GetStringValue(), "12-12-2022")
+
+	assert.NotEmpty(t, details["Rollup3"])
+	assert.Len(t, pbtypes.GetStringListValue(details["Rollup3"]), 1)
+	rollup := pbtypes.GetStringListValue(details["Rollup3"])
+	assert.Equal(t, rollup[0], "Title")
+}

core/block/import/notion/api/page/pageslink.go

@@ -0,0 +1,52 @@
+package page
+
+import (
+	"github.com/globalsign/mgo/bson"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/converter"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/database"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+)
+
+func SetPageLinksInDatabase(databaseSnaphots *converter.Response,
+	pages []Page,
+	databases []database.Database,
+	notionPageIdsToAnytype, notionDatabaseIdsToAnytype map[string]string) {
+	snapshots := makeSnapshotMapFromArray(databaseSnaphots.Snapshots)
+
+	for _, p := range pages {
+		if p.Parent.DatabaseID != "" {
+			if parentID, ok := notionDatabaseIdsToAnytype[p.Parent.DatabaseID]; ok {
+				addLinkBlockToDatabase(snapshots[parentID], notionPageIdsToAnytype[p.ID])
+			}
+		}
+	}
+	for _, d := range databases {
+		if d.Parent.DatabaseID != "" {
+			if parentID, ok := notionDatabaseIdsToAnytype[d.Parent.DatabaseID]; ok {
+				addLinkBlockToDatabase(snapshots[parentID], notionDatabaseIdsToAnytype[d.ID])
+			}
+		}
+	}
+}
+
+func addLinkBlockToDatabase(snapshots *converter.Snapshot, targetID string) {
+	id := bson.NewObjectId().Hex()
+	link := &model.Block{
+		Id:          id,
+		ChildrenIds: []string{},
+		Content: &model.BlockContentOfLink{
+			Link: &model.BlockContentLink{
+				TargetBlockId: targetID,
+			},
+		}}
+	snapshots.Snapshot.Blocks = append(snapshots.Snapshot.Blocks, link)
+}
+
+func makeSnapshotMapFromArray(snapshots []*converter.Snapshot) map[string]*converter.Snapshot {
+	snapshotsMap := make(map[string]*converter.Snapshot, len(snapshots))
+	for _, s := range snapshots {
+		snapshotsMap[s.Id] = s
+	}
+	return snapshotsMap
+}

core/block/import/notion/api/property/propertyitem.go

@@ -0,0 +1,683 @@
+package property
+
+// This file represent property item from Notion https://developers.notion.com/reference/property-item-object
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gogo/protobuf/types"
+	"go.uber.org/zap"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+	"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
+)
+
+type ConfigType string
+
+type Object interface {
+	GetPropertyType() ConfigType
+	GetID() string
+	GetFormat() model.RelationFormat
+}
+
+const (
+	PropertyConfigTypeTitle       ConfigType = "title"
+	PropertyConfigTypeRichText    ConfigType = "rich_text"
+	PropertyConfigTypeNumber      ConfigType = "number"
+	PropertyConfigTypeSelect      ConfigType = "select"
+	PropertyConfigTypeMultiSelect ConfigType = "multi_select"
+	PropertyConfigTypeDate        ConfigType = "date"
+	PropertyConfigTypePeople      ConfigType = "people"
+	PropertyConfigTypeFiles       ConfigType = "files"
+	PropertyConfigTypeCheckbox    ConfigType = "checkbox"
+	PropertyConfigTypeURL         ConfigType = "url"
+	PropertyConfigTypeEmail       ConfigType = "email"
+	PropertyConfigTypePhoneNumber ConfigType = "phone_number"
+	PropertyConfigTypeFormula     ConfigType = "formula"
+	PropertyConfigTypeRelation    ConfigType = "relation"
+	PropertyConfigTypeRollup      ConfigType = "rollup"
+	PropertyConfigCreatedTime     ConfigType = "created_time"
+	PropertyConfigCreatedBy       ConfigType = "created_by"
+	PropertyConfigLastEditedTime  ConfigType = "last_edited_time"
+	PropertyConfigLastEditedBy    ConfigType = "last_edited_by"
+	PropertyConfigStatus          ConfigType = "status"
+)
+
+type DetailSetter interface {
+	SetDetail(key string, details map[string]*types.Value)
+}
+
+type TitleItem struct {
+	Object string          `json:"object"`
+	ID     string          `json:"id"`
+	Type   string          `json:"type"`
+	Title  []*api.RichText `json:"title"`
+}
+
+func (t *TitleItem) SetDetail(key string, details map[string]*types.Value) {
+	var richText strings.Builder
+	for i, title := range t.Title {
+		richText.WriteString(title.PlainText)
+		if i != len(t.Title)-1 {
+			richText.WriteString("\n")
+		}
+	}
+	details[bundle.RelationKeyName.String()] = pbtypes.String(richText.String())
+}
+
+func (t *TitleItem) GetPropertyType() ConfigType {
+	return PropertyConfigTypeTitle
+}
+
+func (t *TitleItem) GetID() string {
+	return t.ID
+}
+
+func (t *TitleItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_shorttext
+}
+
+type RichTextItem struct {
+	Object   string          `json:"object"`
+	ID       string          `json:"id"`
+	Type     string          `json:"type"`
+	RichText []*api.RichText `json:"rich_text"`
+}
+
+func (rt *RichTextItem) SetDetail(key string, details map[string]*types.Value) {
+	var richText strings.Builder
+	for i, r := range rt.RichText {
+		richText.WriteString(r.PlainText)
+		if i != len(rt.RichText)-1 {
+			richText.WriteString("\n")
+		}
+	}
+	details[key] = pbtypes.String(richText.String())
+}
+
+func (rt *RichTextItem) GetPropertyType() ConfigType {
+	return PropertyConfigTypeRichText
+}
+
+func (rt *RichTextItem) GetID() string {
+	return rt.ID
+}
+
+func (rt *RichTextItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_longtext
+}
+
+type NumberItem struct {
+	Object string   `json:"object"`
+	ID     string   `json:"id"`
+	Type   string   `json:"type"`
+	Number *float64 `json:"number"`
+}
+
+func (np *NumberItem) SetDetail(key string, details map[string]*types.Value) {
+	if np.Number != nil {
+		details[key] = pbtypes.Float64(*np.Number)
+	}
+}
+
+func (np *NumberItem) GetPropertyType() ConfigType {
+	return PropertyConfigTypeNumber
+}
+
+func (np *NumberItem) GetID() string {
+	return np.ID
+}
+
+func (np *NumberItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_number
+}
+
+type SelectItem struct {
+	Object string       `json:"object"`
+	ID     string       `json:"id"`
+	Type   string       `json:"type"`
+	Select SelectOption `json:"select"`
+}
+
+type SelectOption struct {
+	ID    string `json:"id,omitempty"`
+	Name  string `json:"name"`
+	Color string `json:"color"`
+}
+
+func (sp *SelectItem) SetDetail(key string, details map[string]*types.Value) {
+	details[key] = pbtypes.StringList([]string{sp.Select.Name})
+}
+
+func (sp *SelectItem) GetPropertyType() ConfigType {
+	return PropertyConfigTypeSelect
+}
+
+func (sp *SelectItem) GetID() string {
+	return sp.ID
+}
+
+func (sp *SelectItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_tag
+}
+
+type MultiSelectItem struct {
+	Object      string          `json:"object"`
+	ID          string          `json:"id"`
+	Type        string          `json:"type"`
+	MultiSelect []*SelectOption `json:"multi_select"`
+}
+
+func (ms *MultiSelectItem) SetDetail(key string, details map[string]*types.Value) {
+	msList := make([]string, 0)
+	for _, so := range ms.MultiSelect {
+		msList = append(msList, so.Name)
+	}
+	details[key] = pbtypes.StringList(msList)
+}
+
+func (ms *MultiSelectItem) GetPropertyType() ConfigType {
+	return PropertyConfigTypeMultiSelect
+}
+
+func (ms *MultiSelectItem) GetID() string {
+	return ms.ID
+}
+
+func (ms *MultiSelectItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_tag
+}
+
+type DateItem struct {
+	Object string          `json:"object"`
+	ID     string          `json:"id"`
+	Type   string          `json:"type"`
+	Date   *api.DateObject `json:"date"`
+}
+
+func (dp *DateItem) SetDetail(key string, details map[string]*types.Value) {
+	if dp.Date != nil {
+		var date strings.Builder
+		if dp.Date.Start != "" {
+			date.WriteString(dp.Date.Start)
+		}
+		if dp.Date.End != "" {
+			if dp.Date.Start != "" {
+				date.WriteString(" ")
+			}
+			date.WriteString(dp.Date.End)
+		}
+		details[key] = pbtypes.String(date.String())
+	}
+}
+
+func (dp *DateItem) GetPropertyType() ConfigType {
+	return PropertyConfigTypeDate
+}
+
+func (dp *DateItem) GetID() string {
+	return dp.ID
+}
+
+func (dp *DateItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_longtext
+}
+
+const (
+	NumberFormula  string = "number"
+	StringFormula  string = "string"
+	BooleanFormula string = "boolean"
+	DateFormula    string = "date"
+)
+
+type FormulaItem struct {
+	Object  string                 `json:"object"`
+	ID      string                 `json:"id"`
+	Type    string                 `json:"type"`
+	Formula map[string]interface{} `json:"formula"`
+}
+
+func (f *FormulaItem) SetDetail(key string, details map[string]*types.Value) {
+	if f.Formula == nil {
+		return
+	}
+	switch f.Formula["type"].(string) {
+	case StringFormula:
+		if f.Formula["string"] != nil {
+			details[key] = pbtypes.String(f.Formula["string"].(string))
+		}
+	case NumberFormula:
+		if f.Formula["number"] != nil {
+			stringNumber := strconv.FormatFloat(f.Formula["number"].(float64), 'f', 6, 64)
+			details[key] = pbtypes.String(stringNumber)
+		}
+	case BooleanFormula:
+		if f.Formula["boolean"] != nil {
+			stringBool := strconv.FormatBool(f.Formula["boolean"].(bool))
+			details[key] = pbtypes.String(stringBool)
+		}
+	default:
+		return
+	}
+}
+
+func (f *FormulaItem) GetPropertyType() ConfigType {
+	return PropertyConfigTypeFormula
+}
+
+func (f *FormulaItem) GetID() string {
+	return f.ID
+}
+
+func (f *FormulaItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_shorttext
+}
+
+type RelationItem struct {
+	Object   string      `json:"object"`
+	ID       string      `json:"id"`
+	Type     string      `json:"type"`
+	Relation []*Relation `json:"relation"`
+	HasMore  bool        `json:"has_more"`
+}
+
+type Relation struct {
+	ID string `json:"id"`
+}
+
+func (rp *RelationItem) SetDetail(key string, details map[string]*types.Value) {
+	relation := make([]string, 0, len(rp.Relation))
+	for _, rel := range rp.Relation {
+		relation = append(relation, rel.ID)
+	}
+	details[key] = pbtypes.StringList(relation)
+}
+
+func (rp *RelationItem) GetPropertyType() ConfigType {
+	return PropertyConfigTypeRelation
+}
+
+func (rp *RelationItem) GetID() string {
+	return rp.ID
+}
+
+func (rp *RelationItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_object
+}
+
+type PeopleItem struct {
+	Object string      `json:"object"`
+	ID     string      `json:"id"`
+	Type   string      `json:"type"`
+	People []*api.User `json:"people"`
+}
+
+func (p *PeopleItem) SetDetail(key string, details map[string]*types.Value) {
+	peopleList := make([]string, 0, len(p.People))
+	for _, people := range p.People {
+		peopleList = append(peopleList, people.Name)
+	}
+	details[key] = pbtypes.StringList(peopleList)
+}
+
+func (p *PeopleItem) GetPropertyType() ConfigType {
+	return PropertyConfigTypePeople
+}
+
+func (p *PeopleItem) GetID() string {
+	return p.ID
+}
+
+func (p *PeopleItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_tag
+}
+
+type FileItem struct {
+	Object string           `json:"object"`
+	ID     string           `json:"id"`
+	Type   string           `json:"type"`
+	File   []api.FileObject `json:"files"`
+}
+
+func (f *FileItem) GetPropertyType() ConfigType {
+	return PropertyConfigTypeFiles
+}
+
+func (f *FileItem) GetID() string {
+	return f.ID
+}
+
+func (f *FileItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_file
+}
+
+func (f *FileItem) SetDetail(key string, details map[string]*types.Value) {
+	fileList := make([]string, len(f.File))
+	for i, fo := range f.File {
+		if fo.External.URL != "" {
+			fileList[i] = fo.External.URL
+		} else if fo.File.URL != "" {
+			fileList[i] = fo.File.URL
+		}
+	}
+	details[key] = pbtypes.StringList(fileList)
+}
+
+type CheckboxItem struct {
+	Object   string `json:"object"`
+	ID       string `json:"id"`
+	Type     string `json:"type"`
+	Checkbox bool   `json:"checkbox"`
+}
+
+func (c *CheckboxItem) SetDetail(key string, details map[string]*types.Value) {
+	details[key] = pbtypes.Bool(c.Checkbox)
+}
+
+func (c *CheckboxItem) GetPropertyType() ConfigType {
+	return PropertyConfigTypeCheckbox
+}
+
+func (c *CheckboxItem) GetID() string {
+	return c.ID
+}
+
+func (c *CheckboxItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_checkbox
+}
+
+type URLItem struct {
+	Object string  `json:"object"`
+	ID     string  `json:"id"`
+	Type   string  `json:"type"`
+	URL    *string `json:"url"`
+}
+
+func (u *URLItem) SetDetail(key string, details map[string]*types.Value) {
+	if u.URL != nil {
+		details[key] = pbtypes.String(*u.URL)
+	}
+}
+
+func (u *URLItem) GetPropertyType() ConfigType {
+	return PropertyConfigTypeURL
+}
+
+func (u *URLItem) GetID() string {
+	return u.ID
+}
+
+func (u *URLItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_url
+}
+
+type EmailItem struct {
+	Object string  `json:"object"`
+	ID     string  `json:"id"`
+	Type   string  `json:"type"`
+	Email  *string `json:"email"`
+}
+
+func (e *EmailItem) SetDetail(key string, details map[string]*types.Value) {
+	if e.Email != nil {
+		details[key] = pbtypes.String(*e.Email)
+	}
+}
+
+func (e *EmailItem) GetPropertyType() ConfigType {
+	return PropertyConfigTypeURL
+}
+
+func (e *EmailItem) GetID() string {
+	return e.ID
+}
+
+func (e *EmailItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_email
+}
+
+type PhoneItem struct {
+	Object string  `json:"object"`
+	ID     string  `json:"id"`
+	Type   string  `json:"type"`
+	Phone  *string `json:"phone_number"`
+}
+
+func (p *PhoneItem) SetDetail(key string, details map[string]*types.Value) {
+	if p.Phone != nil {
+		details[key] = pbtypes.String(*p.Phone)
+	}
+}
+
+func (p *PhoneItem) GetPropertyType() ConfigType {
+	return PropertyConfigTypePhoneNumber
+}
+
+func (p *PhoneItem) GetID() string {
+	return p.ID
+}
+
+func (p *PhoneItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_phone
+}
+
+type CreatedTimeItem struct {
+	Object      string `json:"object"`
+	ID          string `json:"id"`
+	Type        string `json:"type"`
+	CreatedTime string `json:"created_time"`
+}
+
+func (ct *CreatedTimeItem) SetDetail(key string, details map[string]*types.Value) {
+	t, err := time.Parse(time.RFC3339, ct.CreatedTime)
+	if err != nil {
+		logger.With(zap.String("method", "SetDetail")).Errorf("failed to parse time %v", err)
+		return
+	}
+	details[key] = pbtypes.Int64(t.Unix())
+}
+
+func (ct *CreatedTimeItem) GetPropertyType() ConfigType {
+	return PropertyConfigCreatedTime
+}
+
+func (ct *CreatedTimeItem) GetID() string {
+	return ct.ID
+}
+
+func (ct *CreatedTimeItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_date
+}
+
+type CreatedByItem struct {
+	Object    string   `json:"object"`
+	ID        string   `json:"id"`
+	Type      string   `json:"type"`
+	CreatedBy api.User `json:"created_by"`
+}
+
+func (cb *CreatedByItem) SetDetail(key string, details map[string]*types.Value) {
+	details[key] = pbtypes.String(cb.CreatedBy.Name)
+}
+
+func (cb *CreatedByItem) GetPropertyType() ConfigType {
+	return PropertyConfigCreatedBy
+}
+
+func (cb *CreatedByItem) GetID() string {
+	return cb.ID
+}
+
+func (cb *CreatedByItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_shorttext
+}
+
+type LastEditedTimeItem struct {
+	Object         string `json:"object"`
+	ID             string `json:"id"`
+	Type           string `json:"type"`
+	LastEditedTime string `json:"last_edited_time"`
+}
+
+func (le *LastEditedTimeItem) SetDetail(key string, details map[string]*types.Value) {
+	t, err := time.Parse(time.RFC3339, le.LastEditedTime)
+	if err != nil {
+		logger.With(zap.String("method", "SetDetail")).Errorf("failed to parse time %v", err)
+		return
+	}
+	details[key] = pbtypes.Int64(t.Unix())
+}
+
+func (le *LastEditedTimeItem) GetPropertyType() ConfigType {
+	return PropertyConfigLastEditedTime
+}
+
+func (le *LastEditedTimeItem) GetID() string {
+	return le.ID
+}
+
+func (le *LastEditedTimeItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_date
+}
+
+type LastEditedByItem struct {
+	Object       string   `json:"object"`
+	ID           string   `json:"id"`
+	Type         string   `json:"type"`
+	LastEditedBy api.User `json:"last_edited_by"`
+}
+
+func (lb *LastEditedByItem) SetDetail(key string, details map[string]*types.Value) {
+	details[key] = pbtypes.String(lb.LastEditedBy.Name)
+}
+
+func (lb *LastEditedByItem) GetPropertyType() ConfigType {
+	return PropertyConfigLastEditedBy
+}
+
+func (lb *LastEditedByItem) GetID() string {
+	return lb.ID
+}
+
+func (lb *LastEditedByItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_shorttext
+}
+
+type StatusItem struct {
+	ID     string     `json:"id"`
+	Type   ConfigType `json:"type"`
+	Status *Status    `json:"status"`
+}
+
+type Status struct {
+	Name  string `json:"name,omitempty"`
+	ID    string `json:"id,omitempty"`
+	Color string `json:"color,omitempty"`
+}
+
+func (sp *StatusItem) SetDetail(key string, details map[string]*types.Value) {
+	details[key] = pbtypes.StringList([]string{sp.Status.Name})
+}
+
+func (sp *StatusItem) GetPropertyType() ConfigType {
+	return PropertyConfigStatus
+}
+
+func (sp *StatusItem) GetID() string {
+	return sp.ID
+}
+
+func (sp *StatusItem) GetFormat() model.RelationFormat {
+	return model.RelationFormat_status
+}
+
+type rollupType string
+
+const (
+	rollupNumber rollupType = "number"
+	rollupDate   rollupType = "date"
+	rollupArray  rollupType = "array"
+)
+
+type propertyObjects []interface{}
+type RollupItem struct {
+	ID     string       `json:"id"`
+	Type   string       `json:"type"`
+	Rollup RollupObject `json:"rollup"`
+}
+
+type RollupObject struct {
+	Type   rollupType      `json:"type"`
+	Number float64         `json:"number"`
+	Date   *api.DateObject `json:"date"`
+	Array  propertyObjects `json:"array"`
+}
+
+func (r *RollupItem) SetDetail(key string, details map[string]*types.Value) {
+	switch r.Rollup.Type {
+	case rollupNumber:
+		details[key] = pbtypes.Float64(r.Rollup.Number)
+	case rollupDate:
+		di := DateItem{Date: r.Rollup.Date}
+		di.SetDetail(key, details)
+	case rollupArray:
+		r.handleArrayType(key, details)
+	}
+}
+
+func (r *RollupItem) handleArrayType(key string, details map[string]*types.Value) {
+	result := make([]string, 0)
+	for _, pr := range r.Rollup.Array {
+		tempDetails := make(map[string]*types.Value, 0)
+		object, err := getPropertyObject(pr)
+		if err != nil {
+			logger.With(zap.String("method", "RollupItem.SetDetail")).Error(err)
+			continue
+		}
+		if ds, ok := object.(DetailSetter); ok {
+			ds.SetDetail(key, tempDetails)
+		}
+		if _, ok := object.(*TitleItem); ok {
+			name := tempDetails[bundle.RelationKeyName.String()]
+			result = append(result, name.GetStringValue())
+		}
+		if value, ok := tempDetails[key]; ok && value != nil {
+			switch value.GetKind().(type) {
+			case *types.Value_StringValue:
+				res := value.GetStringValue()
+				result = append(result, res)
+			case *types.Value_BoolValue:
+				res := value.GetBoolValue()
+				result = append(result, strconv.FormatBool(res))
+			case *types.Value_NumberValue:
+				res := value.GetNumberValue()
+				result = append(result, strconv.FormatFloat(res, 'f', 0, 64))
+			}
+		}
+	}
+	details[key] = pbtypes.StringList(result)
+}
+
+func (r *RollupItem) GetPropertyType() ConfigType {
+	return PropertyConfigTypeRollup
+}
+
+func (r *RollupItem) GetFormat() model.RelationFormat {
+	switch r.Rollup.Type {
+	case rollupNumber:
+		return model.RelationFormat_number
+	case rollupDate:
+		return model.RelationFormat_longtext
+	case rollupArray:
+		return model.RelationFormat_tag
+	}
+	return model.RelationFormat_longtext
+}
+
+func (r *RollupItem) GetID() string {
+	return r.ID
+}

core/block/import/notion/api/property/propertyobject.go

@@ -0,0 +1,249 @@
+package property
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/client"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/logging"
+)
+
+var logger = logging.Logger("notion-property-retriever")
+
+const endpoint = "/pages/%s/properties/%s"
+
+const endpointWithStartCursor = "/pages/%s/properties/%s?start_cursor=%s"
+
+type TitleObject struct {
+	Title api.RichText `json:"title"`
+}
+
+type RichTextObject struct {
+	RichText api.RichText `json:"rich_text"`
+}
+
+type RelationObject struct {
+	Relation Relation `json:"relation"`
+}
+
+type PeopleObject struct {
+	People api.User `json:"people"`
+}
+
+type Properties map[string]Object
+
+func (p *Properties) UnmarshalJSON(data []byte) error {
+	var raw map[string]interface{}
+	if err := json.Unmarshal(data, &raw); err != nil {
+		return err
+	}
+	props, err := parsePropertyConfigs(raw)
+	if err != nil {
+		return err
+	}
+
+	*p = props
+	return nil
+}
+
+func parsePropertyConfigs(raw map[string]interface{}) (Properties, error) {
+	result := make(Properties)
+	for k, v := range raw {
+		p, err := getPropertyObject(v)
+		if err != nil {
+			return nil, err
+		}
+		result[k] = p
+	}
+
+	return result, nil
+}
+
+func getPropertyObject(v interface{}) (Object, error) {
+	var p Object
+	switch rawProperty := v.(type) {
+	case map[string]interface{}:
+		switch ConfigType(rawProperty["type"].(string)) {
+		case PropertyConfigTypeTitle:
+			p = &TitleItem{}
+		case PropertyConfigTypeRichText:
+			p = &RichTextItem{}
+		case PropertyConfigTypeNumber:
+			p = &NumberItem{}
+		case PropertyConfigTypeSelect:
+			p = &SelectItem{}
+		case PropertyConfigTypeMultiSelect:
+			p = &MultiSelectItem{}
+		case PropertyConfigTypeDate:
+			p = &DateItem{}
+		case PropertyConfigTypePeople:
+			p = &PeopleItem{}
+		case PropertyConfigTypeFiles:
+			p = &FileItem{}
+		case PropertyConfigTypeCheckbox:
+			p = &CheckboxItem{}
+		case PropertyConfigTypeURL:
+			p = &URLItem{}
+		case PropertyConfigTypeEmail:
+			p = &EmailItem{}
+		case PropertyConfigTypePhoneNumber:
+			p = &PhoneItem{}
+		case PropertyConfigTypeFormula:
+			p = &FormulaItem{}
+		case PropertyConfigTypeRelation:
+			p = &RelationItem{}
+		case PropertyConfigTypeRollup:
+			p = &RollupItem{}
+		case PropertyConfigCreatedTime:
+			p = &CreatedTimeItem{}
+		case PropertyConfigCreatedBy:
+			p = &CreatedByItem{}
+		case PropertyConfigLastEditedTime:
+			p = &LastEditedTimeItem{}
+		case PropertyConfigLastEditedBy:
+			p = &LastEditedByItem{}
+		case PropertyConfigStatus:
+			p = &StatusItem{}
+		default:
+			return nil, fmt.Errorf("unsupported property type: %s", rawProperty["type"].(string))
+		}
+		b, err := json.Marshal(rawProperty)
+		if err != nil {
+			return nil, err
+		}
+
+		if err = json.Unmarshal(b, &p); err != nil {
+			return nil, err
+		}
+
+	default:
+		return nil, fmt.Errorf("unsupported property format %T", v)
+	}
+	return p, nil
+}
+
+type Service struct {
+	client *client.Client
+}
+
+func New(client *client.Client) *Service {
+	return &Service{
+		client: client,
+	}
+}
+
+type propertyPaginatedRespone struct {
+	Object     string        `json:"object"`
+	ID         string        `json:"id"`
+	Type       string        `json:"type"`
+	Results    []interface{} `json:"results"`
+	Item       interface{}   `json:"property_item"`
+	HasMore    bool          `json:"has_more"`
+	NextCursor string        `json:"next_cursor"`
+}
+
+// GetPropertyObject get from Notion properties values with tyoe People, Title, Relations and Rich text
+// because they have pagination
+func (s *Service) GetPropertyObject(ctx context.Context,
+	pageID, propertyID, apiKey string,
+	propertyType ConfigType) ([]interface{}, error) {
+	var (
+		hasMore     = true
+		body        = &bytes.Buffer{}
+		startCursor string
+		response    propertyPaginatedRespone
+		properties  = make([]interface{}, 0)
+	)
+
+	for hasMore {
+		request := fmt.Sprintf(endpoint, pageID, propertyID)
+		if startCursor != "" {
+			request = fmt.Sprintf(endpointWithStartCursor, pageID, propertyID, startCursor)
+		}
+		req, err := s.client.PrepareRequest(ctx, apiKey, http.MethodGet, request, body)
+
+		if err != nil {
+			return nil, fmt.Errorf("GetPropertyObject: %s", err)
+		}
+		res, err := s.client.HTTPClient.Do(req)
+
+		if err != nil {
+			return nil, fmt.Errorf("GetPropertyObject: %s", err)
+		}
+		defer res.Body.Close()
+
+		b, err := ioutil.ReadAll(res.Body)
+
+		if err != nil {
+			return nil, err
+		}
+
+		if res.StatusCode != http.StatusOK {
+			notionErr := client.TransformHTTPCodeToError(b)
+			if notionErr == nil {
+				return nil, fmt.Errorf("GetPropertyObject: failed http request, %d code", res.StatusCode)
+			}
+			return nil, notionErr
+		}
+
+		err = json.Unmarshal(b, &response)
+		if err != nil {
+			continue
+		}
+		result := response.Results
+		for _, v := range result {
+			buffer, err := json.Marshal(v)
+			if err != nil {
+				logger.Errorf("GetPropertyObject: failed to marshal: %s", err)
+				continue
+			}
+			if propertyType == PropertyConfigTypeTitle {
+				p := TitleObject{}
+				err = json.Unmarshal(buffer, &p)
+				if err != nil {
+					logger.Errorf("GetPropertyObject: failed to marshal TitleItem: %s", err)
+					continue
+				}
+				properties = append(properties, &p.Title)
+			}
+			if propertyType == PropertyConfigTypeRichText {
+				p := RichTextObject{}
+				err = json.Unmarshal(buffer, &p)
+				if err != nil {
+					logger.Errorf("GetPropertyObject: failed to marshal RichTextItem: %s", err)
+					continue
+				}
+				properties = append(properties, &p.RichText)
+			}
+			if propertyType == PropertyConfigTypeRelation {
+				p := RelationObject{}
+				err = json.Unmarshal(buffer, &p)
+				if err != nil {
+					logger.Errorf("GetPropertyObject: failed to marshal RelationItem: %s", err)
+					continue
+				}
+				properties = append(properties, &p.Relation)
+			}
+			if propertyType == PropertyConfigTypePeople {
+				p := PeopleObject{}
+				err = json.Unmarshal(buffer, &p)
+				if err != nil {
+					logger.Errorf("GetPropertyObject: failed to marshal PeopleItem: %s", err)
+					continue
+				}
+				properties = append(properties, &p.People)
+			}
+		}
+		if response.HasMore {
+			startCursor = response.NextCursor
+			continue
+		}
+		hasMore = false
+	}
+	return properties, nil
+}

core/block/import/notion/api/search/search.go

@@ -0,0 +1,151 @@
+package search
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"time"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/client"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/database"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/page"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/logging"
+)
+
+var logger = logging.Logger("notion-search")
+
+const (
+	endpoint = "/search"
+)
+
+type Service struct {
+	client *client.Client
+}
+
+type Response struct {
+	Results    []interface{} `json:"results"`
+	HasMore    bool          `json:"has_more"`
+	NextCursor *string       `json:"next_cursor"`
+}
+
+// New is a constructor for Service
+func New(client *client.Client) *Service {
+	return &Service{
+		client: client,
+	}
+}
+
+type Effector func(ctx context.Context, apiKey string, pageSize int64) ([]database.Database, []page.Page, error)
+
+// Retry is an implementation for retry pattern
+func Retry(effector Effector, retries int, delay time.Duration) Effector {
+	return func(ctx context.Context, apiKey string, pageSize int64) ([]database.Database, []page.Page, error) {
+		for r := 0; ; r++ {
+			database, pages, err := effector(ctx, apiKey, pageSize)
+			if err == nil || r >= retries {
+				return database, pages, err
+			}
+
+			logger.Infof("Attempt %d failed; retrying in %v", r+1, delay)
+
+			select {
+			case <-time.After(delay):
+			case <-ctx.Done():
+				return nil, nil, ctx.Err()
+			}
+		}
+	}
+}
+
+// Search calls /search endoint from Notion, which return all databases and pages from user integration
+func (s *Service) Search(ctx context.Context, apiKey string, pageSize int64) ([]database.Database, []page.Page, error) {
+	var (
+		hasMore         = true
+		body            = &bytes.Buffer{}
+		resultDatabases = make([]database.Database, 0)
+		resultPages     = make([]page.Page, 0)
+		startCursor     string
+	)
+	type Option struct {
+		PageSize    int64  `json:"page_size,omitempty"`
+		StartCursor string `json:"start_cursor,omitempty"`
+	}
+
+	for hasMore {
+		err := json.NewEncoder(body).Encode(&Option{PageSize: pageSize, StartCursor: startCursor})
+
+		if err != nil {
+			return nil, nil, fmt.Errorf("ListDatabases: %s", err)
+		}
+
+		req, err := s.client.PrepareRequest(ctx, apiKey, http.MethodPost, endpoint, body)
+
+		if err != nil {
+			return nil, nil, fmt.Errorf("ListDatabases: %s", err)
+		}
+		res, err := s.client.HTTPClient.Do(req)
+
+		if err != nil {
+			return nil, nil, fmt.Errorf("ListDatabases: %s", err)
+		}
+		defer res.Body.Close()
+
+		b, err := ioutil.ReadAll(res.Body)
+
+		if err != nil {
+			return nil, nil, err
+		}
+		var objects Response
+		if res.StatusCode != http.StatusOK {
+			notionErr := client.TransformHTTPCodeToError(b)
+			if notionErr == nil {
+				return nil, nil, fmt.Errorf("failed http request, %d code", res.StatusCode)
+			}
+			return nil, nil, notionErr
+		}
+
+		err = json.Unmarshal(b, &objects)
+
+		if err != nil {
+			return nil, nil, err
+		}
+		for _, o := range objects.Results {
+			if o.(map[string]interface{})["object"] == database.ObjectType {
+				db, err := json.Marshal(o)
+				if err != nil {
+					return nil, nil, fmt.Errorf("ListDatabases: %s", err)
+				}
+				d := database.Database{}
+				err = json.Unmarshal(db, &d)
+				if err != nil {
+					return nil, nil, fmt.Errorf("ListDatabases: %s", err)
+				}
+				resultDatabases = append(resultDatabases, d)
+			}
+			if o.(map[string]interface{})["object"] == page.ObjectType {
+				pg, err := json.Marshal(o)
+				if err != nil {
+					return nil, nil, fmt.Errorf("ListDatabases: %s", err)
+				}
+				p := page.Page{}
+				err = json.Unmarshal(pg, &p)
+				if err != nil {
+					return nil, nil, fmt.Errorf("ListDatabases: %s", err)
+				}
+				resultPages = append(resultPages, p)
+			}
+		}
+
+		if !objects.HasMore {
+			hasMore = false
+			continue
+		}
+
+		startCursor = *objects.NextCursor
+
+	}
+	return resultDatabases, resultPages, nil
+}

core/block/import/notion/converter.go

@@ -0,0 +1,124 @@
+package notion
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/converter"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/block"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/client"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/database"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/page"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/notion/api/search"
+	"github.com/anytypeio/go-anytype-middleware/core/block/process"
+	"github.com/anytypeio/go-anytype-middleware/pb"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/core"
+)
+
+const (
+	name                      = "Notion"
+	pageSize                  = 100
+	retryDelay                = time.Second
+	retryAmount               = 5
+	numberOfStepsForPages     = 3 // 2 cycles to get snapshots and 1 cycle to create objects
+	numberOfStepsForDatabases = 2 // 1 cycles to get snapshots and 1 cycle to create objects
+)
+
+func init() {
+	converter.RegisterFunc(New)
+}
+
+type Notion struct {
+	search    *search.Service
+	dbService *database.Service
+	pgService *page.Service
+}
+
+func New(core.Service) converter.Converter {
+	cl := client.NewClient()
+	return &Notion{
+		search:    search.New(cl),
+		dbService: database.New(),
+		pgService: page.New(cl),
+	}
+}
+
+func (n *Notion) GetSnapshots(req *pb.RpcObjectImportRequest,
+	progress *process.Progress) (*converter.Response, converter.ConvertError) {
+	ce := converter.NewError()
+	apiKey := n.getParams(req)
+	if apiKey == "" {
+		ce.Add("apiKey", fmt.Errorf("failed to extract apikey"))
+		return nil, ce
+	}
+	db, pages, err := search.Retry(n.search.Search, retryAmount, retryDelay)(context.TODO(), apiKey, pageSize)
+
+	if err != nil {
+		ce.Add("/search", fmt.Errorf("failed to get pages and databases %s", err))
+		return nil, ce
+	}
+
+	progress.SetTotal(int64(len(db)*numberOfStepsForDatabases + len(pages)*numberOfStepsForPages))
+	dbSnapshots, notionIdsToAnytype, dbNameToID, dbErr := n.dbService.GetDatabase(context.TODO(), req.Mode, db, progress)
+
+	if dbErr != nil && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
+		ce.Merge(dbErr)
+		return nil, ce
+	}
+
+	r := &block.MapRequest{
+		NotionDatabaseIdsToAnytype: notionIdsToAnytype,
+		DatabaseNameToID:           dbNameToID,
+	}
+	pgSnapshots, notionPageIDToAnytype, pgErr := n.pgService.GetPages(context.TODO(), apiKey, req.Mode, pages, r, progress)
+	if pgErr != nil && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
+		ce.Merge(pgErr)
+		return nil, ce
+	}
+
+	page.SetPageLinksInDatabase(dbSnapshots, pages, db, notionPageIDToAnytype, notionIdsToAnytype)
+
+	allSnaphots := make([]*converter.Snapshot, 0, len(pgSnapshots.Snapshots)+len(dbSnapshots.Snapshots))
+	allSnaphots = append(allSnaphots, pgSnapshots.Snapshots...)
+	allSnaphots = append(allSnaphots, dbSnapshots.Snapshots...)
+	relations := mergeMaps(dbSnapshots.Relations, pgSnapshots.Relations)
+
+	if pgErr != nil {
+		ce.Merge(pgErr)
+	}
+
+	if dbErr != nil {
+		ce.Merge(dbErr)
+	}
+	if !ce.IsEmpty() {
+		return &converter.Response{Snapshots: allSnaphots, Relations: relations}, ce
+	}
+
+	return &converter.Response{Snapshots: allSnaphots, Relations: relations}, nil
+}
+
+func (n *Notion) getParams(param *pb.RpcObjectImportRequest) string {
+	if p := param.GetNotionParams(); p != nil {
+		return p.GetApiKey()
+	}
+	return ""
+}
+
+func (n *Notion) Name() string {
+	return name
+}
+
+func mergeMaps(first, second map[string][]*converter.Relation) map[string][]*converter.Relation {
+	res := make(map[string][]*converter.Relation, 0)
+
+	for pageID, rel := range first {
+		res[pageID] = rel
+	}
+
+	for pageID, rel := range second {
+		res[pageID] = rel
+	}
+
+	return res
+}

core/block/import/objectcreator.go

@@ -10,9 +10,9 @@ import (
 
 	"github.com/anytypeio/go-anytype-middleware/core/block"
 	"github.com/anytypeio/go-anytype-middleware/core/block/editor/state"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/converter"
 	"github.com/anytypeio/go-anytype-middleware/core/block/import/syncer"
 	"github.com/anytypeio/go-anytype-middleware/core/block/simple"
-	"github.com/anytypeio/go-anytype-middleware/core/block/simple/relation"
 	"github.com/anytypeio/go-anytype-middleware/core/session"
 	"github.com/anytypeio/go-anytype-middleware/pb"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle"
@@ -24,32 +24,57 @@ import (
 	"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
 )
 
-type ObjectCreator struct {
-	service       *block.Service
-	objectCreator objectCreator
-	core          core.Service
-	updater       Updater
-	syncFactory   *syncer.Factory
-}
-
 type objectCreator interface {
 	CreateSmartBlockFromState(ctx context.Context, sbType coresb.SmartBlockType, details *types.Struct, relationIds []string, createState *state.State) (id string, newDetails *types.Struct, err error)
+	CreateSubObjectInWorkspace(details *types.Struct, workspaceID string) (id string, newDetails *types.Struct, err error)
+	CreateSubObjectsInWorkspace(details []*types.Struct) (ids []string, objects []*types.Struct, err error)
 }
 
-func NewCreator(service *block.Service, objCreator objectCreator, core core.Service, updater Updater, syncFactory *syncer.Factory) Creator {
-	return &ObjectCreator{service: service, objectCreator: objCreator, core: core, updater: updater, syncFactory: syncFactory}
+type ObjectCreator struct {
+	service         *block.Service
+	objCreator      objectCreator
+	core            core.Service
+	updater         Updater
+	relationCreator RelationCreator
+	syncFactory     *syncer.Factory
+}
+
+func NewCreator(service *block.Service,
+	objCreator objectCreator,
+	updater Updater,
+	syncFactory *syncer.Factory,
+	relationCreator RelationCreator) Creator {
+	return &ObjectCreator{
+		service:         service,
+		objCreator:      objCreator,
+		updater:         updater,
+		syncFactory:     syncFactory,
+		relationCreator: relationCreator,
+	}
 }
 
 // Create creates smart blocks from given snapshots
-func (oc *ObjectCreator) Create(ctx *session.Context, snapshot *model.SmartBlockSnapshotBase, pageID string, sbType smartblock.SmartBlockType, updateExisting bool) (*types.Struct, error) {
+func (oc *ObjectCreator) Create(ctx *session.Context,
+	snapshot *model.SmartBlockSnapshotBase,
+	relations []*converter.Relation,
+	pageID string,
+	sbType smartblock.SmartBlockType,
+	updateExisting bool) (*types.Struct, error) {
 	isFavorite := pbtypes.GetBool(snapshot.Details, bundle.RelationKeyIsFavorite.String())
 
 	var err error
 
 	if updateExisting {
-		if details, err := oc.updater.Update(ctx, snapshot, pageID); err == nil {
+		var (
+			filesToDelete []string
+			details       *types.Struct
+		)
+		if details, filesToDelete, err = oc.updater.Update(ctx, snapshot, relations, pageID); err == nil {
 			return details, nil
 		}
+		for _, hash := range filesToDelete {
+			oc.deleteFile(hash)
+		}
 		log.Warn("failed to update existing object: %s", err)
 	}
 
@@ -73,24 +98,30 @@ func (oc *ObjectCreator) Create(ctx *session.Context, snapshot *model.SmartBlock
 	st.SetLocalDetail(bundle.RelationKeyLastModifiedBy.String(), pbtypes.String(addr.AnytypeProfileId))
 	st.InjectDerivedDetails()
 
-	if err = oc.validate(st); err != nil {
-		return nil, fmt.Errorf("new id not found for '%s'", st.RootId())
-	}
-
+	var filesToDelete []string
 	defer func() {
 		// delete file in ipfs if there is error after creation
 		if err != nil {
 			for _, bl := range st.Blocks() {
 				if f := bl.GetFile(); f != nil {
-					oc.deleteFile(f)
+					oc.deleteFile(f.Hash)
+				}
+				for _, hash := range filesToDelete {
+					oc.deleteFile(hash)
 				}
 			}
 		}
 	}()
 
-	newID, details, err := oc.objectCreator.CreateSmartBlockFromState(context.TODO(), sbType, nil, nil, st)
+	newID, details, err := oc.objCreator.CreateSmartBlockFromState(context.TODO(), sbType, nil, nil, st)
 	if err != nil {
-		return nil, fmt.Errorf("crear object '%s'", st.RootId())
+		return nil, fmt.Errorf("create object '%s'", st.RootId())
+	}
+
+	var oldRelationBlocksToNew map[string]*model.Block
+	filesToDelete, oldRelationBlocksToNew, err = oc.relationCreator.CreateRelations(ctx, snapshot, pageID, relations)
+	if err != nil {
+		return nil, fmt.Errorf("relation create '%s'", err)
 	}
 
 	if isFavorite {
@@ -101,6 +132,8 @@ func (oc *ObjectCreator) Create(ctx *session.Context, snapshot *model.SmartBlock
 		}
 	}
 
+	oc.relationCreator.ReplaceRelationBlock(ctx, oldRelationBlocksToNew, pageID)
+
 	st.Iterate(func(bl simple.Block) (isContinue bool) {
 		s := oc.syncFactory.GetSyncer(bl)
 		if s != nil {
@@ -110,28 +143,8 @@ func (oc *ObjectCreator) Create(ctx *session.Context, snapshot *model.SmartBlock
 		}
 		return true
 	})
-	return details, nil
-}
 
-func (oc *ObjectCreator) validate(st *state.State) (err error) {
-	var relKeys []string
-	for _, rel := range st.OldExtraRelations() {
-		if !bundle.HasRelation(rel.Key) {
-			log.Errorf("builtin objects should not contain custom relations, got %s in %s(%s)", rel.Name, st.RootId(), pbtypes.GetString(st.Details(), bundle.RelationKeyName.String()))
-		}
-	}
-	st.Iterate(func(b simple.Block) (isContinue bool) {
-		if rb, ok := b.(relation.Block); ok {
-			relKeys = append(relKeys, rb.Model().GetRelation().Key)
-		}
-		return true
-	})
-	for _, rk := range relKeys {
-		if !st.HasRelation(rk) {
-			return fmt.Errorf("bundled template validation: relation '%v' exists in block but not in extra relations", rk)
-		}
-	}
-	return nil
+	return details, nil
 }
 
 func (oc *ObjectCreator) addRootBlock(snapshot *model.SmartBlockSnapshotBase, pageID string) {
@@ -152,8 +165,16 @@ func (oc *ObjectCreator) addRootBlock(snapshot *model.SmartBlockSnapshotBase, pa
 		}
 	}
 	if err != nil {
+		notRootBlockChild := make(map[string]bool, 0)
 		for _, b := range snapshot.Blocks {
-			childrenIds = append(childrenIds, b.Id)
+			if len(b.ChildrenIds) != 0 {
+				for _, id := range b.ChildrenIds {
+					notRootBlockChild[id] = true
+				}
+			}
+			if _, ok := notRootBlockChild[b.Id]; !ok {
+				childrenIds = append(childrenIds, b.Id)
+			}
 		}
 		snapshot.Blocks = append(snapshot.Blocks, &model.Block{
 			Id:          pageID,
@@ -163,24 +184,24 @@ func (oc *ObjectCreator) addRootBlock(snapshot *model.SmartBlockSnapshotBase, pa
 	}
 }
 
-func (oc *ObjectCreator) deleteFile(f *model.BlockContentFile) {
-	inboundLinks, err := oc.core.ObjectStore().GetOutboundLinksById(f.Hash)
+func (oc *ObjectCreator) deleteFile(hash string) {
+	inboundLinks, err := oc.core.ObjectStore().GetOutboundLinksById(hash)
 	if err != nil {
-		log.With("file", f.Hash).Errorf("failed to get inbound links for file: %s", err.Error())
+		log.With("file", hash).Errorf("failed to get inbound links for file: %s", err.Error())
 		return
 	}
 	if len(inboundLinks) == 0 {
-		if err = oc.core.ObjectStore().DeleteObject(f.Hash); err != nil {
-			log.With("file", f.Hash).Errorf("failed to delete file from objectstore: %s", err.Error())
+		if err = oc.core.ObjectStore().DeleteObject(hash); err != nil {
+			log.With("file", hash).Errorf("failed to delete file from objectstore: %s", err.Error())
 		}
-		if err = oc.core.FileStore().DeleteByHash(f.Hash); err != nil {
-			log.With("file", f.Hash).Errorf("failed to delete file from filestore: %s", err.Error())
+		if err = oc.core.FileStore().DeleteByHash(hash); err != nil {
+			log.With("file", hash).Errorf("failed to delete file from filestore: %s", err.Error())
 		}
-		if _, err = oc.core.FileOffload(f.Hash); err != nil {
-			log.With("file", f.Hash).Errorf("failed to offload file: %s", err.Error())
+		if _, err = oc.core.FileOffload(hash); err != nil {
+			log.With("file", hash).Errorf("failed to offload file: %s", err.Error())
 		}
-		if err = oc.core.FileStore().DeleteFileKeys(f.Hash); err != nil {
-			log.With("file", f.Hash).Errorf("failed to delete file keys: %s", err.Error())
+		if err = oc.core.FileStore().DeleteFileKeys(hash); err != nil {
+			log.With("file", hash).Errorf("failed to delete file keys: %s", err.Error())
 		}
 	}
 }

core/block/import/objectupdater.go

@@ -4,10 +4,12 @@ import (
 	"fmt"
 
 	"github.com/gogo/protobuf/types"
+	"go.uber.org/zap"
 
 	"github.com/anytypeio/go-anytype-middleware/core/block"
 	"github.com/anytypeio/go-anytype-middleware/core/block/editor/basic"
 	sb "github.com/anytypeio/go-anytype-middleware/core/block/editor/smartblock"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/converter"
 	"github.com/anytypeio/go-anytype-middleware/core/block/import/syncer"
 	"github.com/anytypeio/go-anytype-middleware/core/block/simple"
 	"github.com/anytypeio/go-anytype-middleware/core/session"
@@ -19,20 +21,28 @@ import (
 )
 
 type ObjectUpdater struct {
-	service     *block.Service
-	core        core.Service
-	syncFactory *syncer.Factory
+	service         *block.Service
+	core            core.Service
+	syncFactory     *syncer.Factory
+	relationCreator RelationCreator
 }
 
-func NewObjectUpdater(service *block.Service, core core.Service, syncFactory *syncer.Factory) Updater {
+func NewObjectUpdater(service *block.Service,
+	core core.Service,
+	syncFactory *syncer.Factory,
+	relationCreator RelationCreator) Updater {
 	return &ObjectUpdater{
-		service:     service,
-		core:        core,
-		syncFactory: syncFactory,
+		service:         service,
+		core:            core,
+		syncFactory:     syncFactory,
+		relationCreator: relationCreator,
 	}
 }
 
-func (ou *ObjectUpdater) Update(ctx *session.Context, snapshot *model.SmartBlockSnapshotBase, pageID string) (*types.Struct, error) {
+func (ou *ObjectUpdater) Update(ctx *session.Context,
+	snapshot *model.SmartBlockSnapshotBase,
+	relations []*converter.Relation,
+	pageID string) (*types.Struct, []string, error) {
 	if snapshot.Details != nil && snapshot.Details.Fields[bundle.RelationKeySource.String()] != nil {
 		source := snapshot.Details.Fields[bundle.RelationKeySource.String()].GetStringValue()
 		records, _, err := ou.core.ObjectStore().Query(nil, database.Query{
@@ -47,7 +57,8 @@ func (ou *ObjectUpdater) Update(ctx *session.Context, snapshot *model.SmartBlock
 		})
 		if err == nil {
 			if len(records) > 0 {
-				return records[0].Details, ou.update(ctx, snapshot, records, pageID)
+				filesToDelete, err := ou.update(ctx, snapshot, records[0].Details, relations, pageID)
+				return records[0].Details, filesToDelete, err
 			}
 		}
 	}
@@ -65,61 +76,72 @@ func (ou *ObjectUpdater) Update(ctx *session.Context, snapshot *model.SmartBlock
 		})
 		if err == nil {
 			if len(records) > 0 {
-				return records[0].Details, ou.update(ctx, snapshot, records, pageID)
+				filesToDelete, err := ou.update(ctx, snapshot, records[0].Details, relations, pageID)
+				return records[0].Details, filesToDelete, err
 			}
 		}
 	}
-	return nil, fmt.Errorf("no source or id details")
+	return nil, nil, fmt.Errorf("no source or id details")
 }
 
 func (ou *ObjectUpdater) update(ctx *session.Context,
 	snapshot *model.SmartBlockSnapshotBase,
-	records []database.Record,
-	pageID string) error {
-	details := records[0]
+	details *types.Struct,
+	relations []*converter.Relation,
+	pageID string) ([]string, error) {
 	simpleBlocks := make([]simple.Block, 0)
-	id := details.Details.Fields[bundle.RelationKeyId.String()].GetStringValue()
-	if details.Details != nil {
-		allBlocksIds := make([]string, 0)
-		if err := ou.service.Do(id, func(b sb.SmartBlock) error {
-			s := b.NewStateCtx(ctx)
-			if err := b.Iterate(func(b simple.Block) (isContinue bool) {
-				if b.Model().GetLink() == nil && id != b.Model().Id {
-					allBlocksIds = append(allBlocksIds, b.Model().Id)
-				}
-				return true
-			}); err != nil {
-				return err
+	var (
+		filesToDelete         = make([]string, 0)
+		oldRelationBlockToNew = make(map[string]*model.Block, 0)
+	)
+	id := details.Fields[bundle.RelationKeyId.String()].GetStringValue()
+	allBlocksIds := make([]string, 0)
+	if err := ou.service.Do(id, func(b sb.SmartBlock) error {
+		s := b.NewStateCtx(ctx)
+		if err := b.Iterate(func(b simple.Block) (isContinue bool) {
+			if b.Model().GetLink() == nil && id != b.Model().Id {
+				allBlocksIds = append(allBlocksIds, b.Model().Id)
 			}
-			for _, v := range allBlocksIds {
-				s.Unlink(v)
-			}
-			for _, block := range snapshot.Blocks {
-				if block.GetLink() != nil {
-					// we don't add link to non-existing object,so checking existence of the object with TargetBlockId in Do
-					if err := ou.service.Do(block.GetLink().TargetBlockId, func(b sb.SmartBlock) error {
-						return nil
-					}); err != nil {
-						continue
-					}
-				}
-				if block.Id != pageID {
-					simpleBlocks = append(simpleBlocks, simple.New(block))
-				}
-			}
-			if err := basic.NewBasic(b).PasteBlocks(s, "", model.Block_Bottom, simpleBlocks); err != nil {
-				return err
-			}
-			return b.Apply(s)
+			return true
 		}); err != nil {
 			return err
 		}
-		for _, b := range simpleBlocks {
-			s := ou.syncFactory.GetSyncer(b)
-			if s != nil {
-				s.Sync(ctx, id, b)
+		for _, v := range allBlocksIds {
+			s.Unlink(v)
+		}
+		for _, block := range snapshot.Blocks {
+			if block.GetLink() != nil {
+				// we don't add link to non-existing object,so checking existence of the object with TargetBlockId in Do
+				if err := ou.service.Do(block.GetLink().TargetBlockId, func(b sb.SmartBlock) error {
+					return nil
+				}); err != nil {
+					continue
+				}
+			}
+			if block.Id != pageID {
+				simpleBlocks = append(simpleBlocks, simple.New(block))
+			}
+		}
+		if err := basic.NewBasic(b).PasteBlocks(s, "", model.Block_Bottom, simpleBlocks); err != nil {
+			return err
+		}
+		return b.Apply(s)
+	}); err != nil {
+		return nil, err
+	}
+	var err error
+	filesToDelete, oldRelationBlockToNew, err = ou.relationCreator.CreateRelations(ctx, snapshot, id, relations)
+	if err != nil {
+		return nil, err
+	}
+	ou.relationCreator.ReplaceRelationBlock(ctx, oldRelationBlockToNew, id)
+	for _, b := range simpleBlocks {
+		s := ou.syncFactory.GetSyncer(b)
+		if s != nil {
+			if err := s.Sync(ctx, id, b); err != nil {
+				log.With(zap.String("object id", pageID)).Errorf("failed to sync %s", err.Error())
 			}
 		}
 	}
-	return nil
+	return filesToDelete, nil
 }

core/block/import/pb/converter.go

@@ -12,6 +12,7 @@ import (
 	"github.com/textileio/go-threads/core/thread"
 
 	"github.com/anytypeio/go-anytype-middleware/core/block/import/converter"
+	"github.com/anytypeio/go-anytype-middleware/core/block/process"
 	"github.com/anytypeio/go-anytype-middleware/pb"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/core"
@@ -33,21 +34,30 @@ func New(core.Service) converter.Converter {
 	return new(Pb)
 }
 
-func (p *Pb) GetSnapshots(req *pb.RpcObjectImportRequest) *converter.Response {
+func (p *Pb) GetSnapshots(req *pb.RpcObjectImportRequest,
+	progress *process.Progress) (*converter.Response, converter.ConvertError) {
 	path, e := p.GetParams(req.Params)
 	allErrors := converter.NewError()
 	if e != nil {
 		allErrors.Add(path, e)
-		return &converter.Response{Error: allErrors}
+		return nil, allErrors
 	}
 	pbFiles, err := p.readFile(path, req.Mode.String())
 	if err != nil && req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
-		return &converter.Response{Error: err}
+		allErrors.Merge(err)
+		return nil, allErrors
 	}
-
 	allSnapshots := make([]*converter.Snapshot, 0)
-	allErrors.Merge(err)
+
+	progress.SetProgressMessage("Start creating snapshots from files")
+	progress.SetTotal(int64(len(pbFiles) * 2))
+
 	for name, file := range pbFiles {
+		if err := progress.TryStep(1); err != nil {
+			ce := converter.NewFromError(name, err)
+			return nil, ce
+		}
+
 		id := strings.TrimSuffix(file.Name, filepath.Ext(file.Name))
 		var (
 			snapshot *model.SmartBlockSnapshotBase
@@ -59,7 +69,7 @@ func (p *Pb) GetSnapshots(req *pb.RpcObjectImportRequest) *converter.Response {
 		if errGS != nil {
 			allErrors.Add(file.Name, errGS)
 			if req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
-				return &converter.Response{Error: allErrors}
+				return nil, allErrors
 			} else {
 				continue
 			}
@@ -68,7 +78,7 @@ func (p *Pb) GetSnapshots(req *pb.RpcObjectImportRequest) *converter.Response {
 		if err != nil {
 			allErrors.Add(path, e)
 			if req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
-				return &converter.Response{Error: allErrors}
+				return nil, allErrors
 			} else {
 				continue
 			}
@@ -77,7 +87,7 @@ func (p *Pb) GetSnapshots(req *pb.RpcObjectImportRequest) *converter.Response {
 		if err != nil {
 			allErrors.Add(path, e)
 			if req.Mode == pb.RpcObjectImportRequest_ALL_OR_NOTHING {
-				return &converter.Response{Error: allErrors}
+				return nil, allErrors
 			} else {
 				continue
 			}
@@ -92,10 +102,10 @@ func (p *Pb) GetSnapshots(req *pb.RpcObjectImportRequest) *converter.Response {
 	}
 
 	if allErrors.IsEmpty() {
-		return &converter.Response{Snapshots: allSnapshots, Error: nil}
+		return &converter.Response{Snapshots: allSnapshots}, nil
 	}
 
-	return &converter.Response{Snapshots: allSnapshots, Error: allErrors}
+	return &converter.Response{Snapshots: allSnapshots}, allErrors
 }
 
 func (p *Pb) Name() string {
@@ -107,8 +117,8 @@ func (p *Pb) GetImage() ([]byte, int64, int64, error) {
 }
 
 func (p *Pb) GetParams(params pb.IsRpcObjectImportRequestParams) (string, error) {
-	if p, ok := params.(*pb.RpcObjectImportRequestParamsOfNotionParams); ok {
-		return p.NotionParams.GetPath(), nil
+	if p, ok := params.(*pb.RpcObjectImportRequestParamsOfMarkdownParams); ok {
+		return p.MarkdownParams.GetPath(), nil
 	}
 	return "", errors.New("PB: GetParams wrong parameters format")
 }

core/block/import/pb/converter_test.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
+	"github.com/anytypeio/go-anytype-middleware/core/block/process"
 	"github.com/anytypeio/go-anytype-middleware/pb"
 )
 
@@ -32,14 +33,14 @@ func Test_GetSnapshotsSuccess(t *testing.T) {
 
 	p := &Pb{}
 
-	res := p.GetSnapshots(&pb.RpcObjectImportRequest{
-		Params: &pb.RpcObjectImportRequestParamsOfNotionParams{NotionParams: &pb.RpcObjectImportRequestNotionParams{Path: wr.Path()}},
+	res, ce := p.GetSnapshots(&pb.RpcObjectImportRequest{
+		Params:                &pb.RpcObjectImportRequestParamsOfMarkdownParams{MarkdownParams: &pb.RpcObjectImportRequestMarkdownParams{Path: wr.Path()}},
 		UpdateExistingObjects: false,
 		Type:                  0,
 		Mode:                  0,
-	})
+	}, process.NewProgress(pb.ModelProcess_Import))
 
-	assert.Nil(t, res.Error)
+	assert.Nil(t, ce)
 	assert.NotNil(t, res.Snapshots)
 	assert.Len(t, res.Snapshots, 1)
 }
@@ -58,14 +59,14 @@ func Test_GetSnapshotsFailedReadZip(t *testing.T) {
 
 	p := &Pb{}
 
-	res := p.GetSnapshots(&pb.RpcObjectImportRequest{
-		Params: &pb.RpcObjectImportRequestParamsOfNotionParams{NotionParams: &pb.RpcObjectImportRequestNotionParams{Path: "not exists"}},
+	_, ce := p.GetSnapshots(&pb.RpcObjectImportRequest{
+		Params:                &pb.RpcObjectImportRequestParamsOfMarkdownParams{MarkdownParams: &pb.RpcObjectImportRequestMarkdownParams{Path: "not exists"}},
 		UpdateExistingObjects: false,
 		Type:                  0,
 		Mode:                  0,
-	})
+	}, process.NewProgress(pb.ModelProcess_Import))
 
-	assert.NotNil(t, res.Error)
+	assert.NotNil(t, ce)
 }
 
 func Test_GetSnapshotsFailedToGetSnapshot(t *testing.T) {
@@ -83,16 +84,16 @@ func Test_GetSnapshotsFailedToGetSnapshot(t *testing.T) {
 
 	p := &Pb{}
 
-	res := p.GetSnapshots(&pb.RpcObjectImportRequest{
-		Params: &pb.RpcObjectImportRequestParamsOfNotionParams{NotionParams: &pb.RpcObjectImportRequestNotionParams{Path: "notexist.zip"}},
+	_, ce := p.GetSnapshots(&pb.RpcObjectImportRequest{
+		Params:                &pb.RpcObjectImportRequestParamsOfMarkdownParams{MarkdownParams: &pb.RpcObjectImportRequestMarkdownParams{Path: "notexist.zip"}},
 		UpdateExistingObjects: false,
 		Type:                  0,
 		Mode:                  0,
-	})
+	}, process.NewProgress(pb.ModelProcess_Import))
 
-	assert.NotNil(t, res.Error)
-	assert.Len(t, res.Error, 1)
-	assert.NotEmpty(t, res.Error.Get("notexist.zip"))
+	assert.NotNil(t, ce)
+	assert.Len(t, ce, 1)
+	assert.NotEmpty(t, ce.Get("notexist.zip"))
 }
 
 func Test_GetSnapshotsFailedToGetSnapshotForTwoFiles(t *testing.T) {
@@ -117,30 +118,30 @@ func Test_GetSnapshotsFailedToGetSnapshotForTwoFiles(t *testing.T) {
 	p := &Pb{}
 
 	// ALL_OR_NOTHING mode
-	res := p.GetSnapshots(&pb.RpcObjectImportRequest{
-		Params: &pb.RpcObjectImportRequestParamsOfNotionParams{NotionParams: &pb.RpcObjectImportRequestNotionParams{Path: wr.Path()}},
+	res, ce := p.GetSnapshots(&pb.RpcObjectImportRequest{
+		Params:                &pb.RpcObjectImportRequestParamsOfMarkdownParams{MarkdownParams: &pb.RpcObjectImportRequestMarkdownParams{Path: wr.Path()}},
 		UpdateExistingObjects: false,
 		Type:                  0,
 		Mode:                  0,
-	})
+	}, process.NewProgress(pb.ModelProcess_Import))
 
-	assert.NotNil(t, res.Error)
-	assert.Nil(t, res.Snapshots)
-	assert.NotEmpty(t, res.Error.Get("test.pb"))
+	assert.NotNil(t, ce)
+	assert.Nil(t, res)
+	assert.NotEmpty(t, ce.Get("test.pb"))
 
 	// IGNORE_ERRORS mode
-	res = p.GetSnapshots(&pb.RpcObjectImportRequest{
-		Params: &pb.RpcObjectImportRequestParamsOfNotionParams{NotionParams: &pb.RpcObjectImportRequestNotionParams{Path: wr.Path()}},
+	res, ce = p.GetSnapshots(&pb.RpcObjectImportRequest{
+		Params:                &pb.RpcObjectImportRequestParamsOfMarkdownParams{MarkdownParams: &pb.RpcObjectImportRequestMarkdownParams{Path: wr.Path()}},
 		UpdateExistingObjects: false,
 		Type:                  0,
 		Mode:                  1,
-	})
+	}, process.NewProgress(pb.ModelProcess_Import))
 
-	assert.NotNil(t, res.Error)
+	assert.NotNil(t, ce)
 	assert.NotNil(t, res.Snapshots)
 	assert.Len(t, res.Snapshots, 1)
-	assert.Len(t, res.Error, 1)
-	assert.NotEmpty(t, res.Error.Get("test.pb"))
+	assert.Len(t, ce, 1)
+	assert.NotEmpty(t, ce.Get("test.pb"))
 }
 
 func newZipWriter(path string) (*zipWriter, error) {

core/block/import/relationcreator.go

@@ -0,0 +1,431 @@
+package importer
+
+import (
+	"strings"
+
+	"github.com/globalsign/mgo/bson"
+	"github.com/gogo/protobuf/types"
+	"go.uber.org/zap"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block"
+	editor "github.com/anytypeio/go-anytype-middleware/core/block/editor/smartblock"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/converter"
+	"github.com/anytypeio/go-anytype-middleware/core/block/simple"
+	"github.com/anytypeio/go-anytype-middleware/core/session"
+	"github.com/anytypeio/go-anytype-middleware/pb"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/core"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/database"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/filestore"
+	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
+	"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
+)
+
+type relationIDFormat struct {
+	ID     string
+	Format model.RelationFormat
+}
+
+type relations []relationIDFormat
+
+type RelationService struct {
+	core             core.Service
+	service          *block.Service
+	objCreator       objectCreator
+	createdRelations map[string]relations // need this field to avoid creation of the same relations
+	store            filestore.FileStore
+}
+
+// NewRelationCreator constructor for RelationService
+func NewRelationCreator(service *block.Service,
+	objCreator objectCreator,
+	store filestore.FileStore,
+	core core.Service) RelationCreator {
+	return &RelationService{
+		service:          service,
+		objCreator:       objCreator,
+		core:             core,
+		createdRelations: make(map[string]relations, 0),
+		store:            store,
+	}
+}
+
+func (rc *RelationService) CreateRelations(ctx *session.Context,
+	snapshot *model.SmartBlockSnapshotBase,
+	pageID string,
+	relations []*converter.Relation) ([]string, map[string]*model.Block, error) {
+	notExistedRelations := make([]*converter.Relation, 0)
+	existedRelations := make(map[string]*converter.Relation, 0)
+	for _, r := range relations {
+		if strings.EqualFold(r.Name, bundle.RelationKeyName.String()) {
+			continue
+		}
+		if rel, ok := rc.createdRelations[r.Name]; ok {
+			var exist bool
+			for _, v := range rel {
+				if v.Format == r.Format {
+					existedRelations[v.ID] = r
+					exist = true
+					break
+				}
+			}
+			if exist {
+				continue
+			}
+		}
+		records, _, err := rc.core.ObjectStore().Query(nil, database.Query{
+			Filters: []*model.BlockContentDataviewFilter{
+				{
+					Condition:   model.BlockContentDataviewFilter_Equal,
+					RelationKey: bundle.RelationKeyName.String(),
+					Value:       pbtypes.String(r.Name),
+					Format:      r.Format,
+				},
+			},
+			Limit: 1,
+		})
+		if err == nil && len(records) > 0 {
+			id := pbtypes.GetString(records[0].Details, bundle.RelationKeyRelationKey.String())
+			existedRelations[id] = r
+			continue
+		}
+		notExistedRelations = append(notExistedRelations, r)
+	}
+
+	filesToDelete, oldRelationBlockToNewUpdate, failedRelations, err := rc.update(ctx, snapshot, existedRelations, pageID)
+	if err != nil {
+		return nil, nil, err
+	}
+	notExistedRelations = append(notExistedRelations, failedRelations...)
+	createfilesToDelete, oldRelationBlockToNewCreate, err := rc.create(ctx, snapshot, notExistedRelations, pageID)
+	if err != nil {
+		return nil, nil, err
+	}
+	filesToDelete = append(filesToDelete, createfilesToDelete...)
+	totalNumberOfRelationBlocks := len(oldRelationBlockToNewCreate) + len(oldRelationBlockToNewUpdate)
+	oldRelationBlockToNewTotal := make(map[string]*model.Block, totalNumberOfRelationBlocks)
+	if len(oldRelationBlockToNewUpdate) == 0 {
+		for k, b := range oldRelationBlockToNewUpdate {
+			oldRelationBlockToNewTotal[k] = b
+		}
+	}
+	if len(oldRelationBlockToNewCreate) == 0 {
+		for k, b := range oldRelationBlockToNewCreate {
+			oldRelationBlockToNewTotal[k] = b
+		}
+	}
+	return filesToDelete, oldRelationBlockToNewTotal, nil
+}
+
+// Create read relations link from snaphot and create according relations in anytype,
+// also set details for according relation in object for files it loads them in ipfs
+func (rc *RelationService) create(ctx *session.Context,
+	snapshot *model.SmartBlockSnapshotBase,
+	relations []*converter.Relation,
+	pageID string) ([]string, map[string]*model.Block, error) {
+	var (
+		err                   error
+		filesToDelete         = make([]string, 0)
+		oldRelationBlockToNew = make(map[string]*model.Block, 0)
+		createRequest         = make([]*types.Struct, 0)
+		existedRelationsIDs   = make([]string, 0)
+		setDetailsRequest     = make([]*pb.RpcObjectSetDetailsDetail, 0)
+	)
+
+	for _, r := range relations {
+		detail := &types.Struct{
+			Fields: map[string]*types.Value{
+				bundle.RelationKeyName.String():           pbtypes.String(r.Name),
+				bundle.RelationKeyRelationFormat.String(): pbtypes.Int64(int64(r.Format)),
+				bundle.RelationKeyType.String():           pbtypes.String(bundle.TypeKeyRelation.URL()),
+				bundle.RelationKeyLayout.String():         pbtypes.Float64(float64(model.ObjectType_relation)),
+			},
+		}
+		createRequest = append(createRequest, detail)
+	}
+	var objects []*types.Struct
+	if _, objects, err = rc.objCreator.CreateSubObjectsInWorkspace(createRequest); err != nil {
+		log.Errorf("create relation %s", err)
+	}
+
+	ids := make([]string, 0, len(existedRelationsIDs)+len(objects))
+	ids = append(ids, existedRelationsIDs...)
+
+	for _, s := range objects {
+		name := pbtypes.GetString(s, bundle.RelationKeyName.String())
+		id := pbtypes.GetString(s, bundle.RelationKeyRelationKey.String())
+		format := model.RelationFormat(pbtypes.GetFloat64(s, bundle.RelationKeyRelationFormat.String()))
+		rc.createdRelations[name] = append(rc.createdRelations[name], relationIDFormat{
+			ID:     id,
+			Format: format,
+		})
+		ids = append(ids, id)
+	}
+
+	if err = rc.service.AddExtraRelations(ctx, pageID, ids); err != nil {
+		log.Errorf("add extra relation %s", err)
+	}
+
+	for _, r := range relations {
+		var relationID string
+		if cr, ok := rc.createdRelations[r.Name]; ok {
+			for _, rel := range cr {
+				if rel.Format == r.Format {
+					relationID = rel.ID
+				}
+			}
+		}
+		if relationID == "" {
+			continue
+		}
+		if snapshot.Details != nil && snapshot.Details.Fields != nil {
+			if snapshot.Details.Fields[r.Name].GetListValue() != nil {
+				rc.handleListValue(ctx, snapshot, r, relationID)
+			}
+			if r.Format == model.RelationFormat_file {
+				filesToDelete = append(filesToDelete, rc.handleFileRelation(ctx, snapshot, r.Name)...)
+			}
+		}
+		setDetailsRequest = append(setDetailsRequest, &pb.RpcObjectSetDetailsDetail{
+			Key:   relationID,
+			Value: snapshot.Details.Fields[r.Name],
+		})
+		if r.BlockID != "" {
+			original, new := rc.linkRelationsBlocks(snapshot, r.BlockID, relationID)
+			if original != nil && new != nil {
+				oldRelationBlockToNew[original.GetId()] = new
+			}
+		}
+	}
+
+	err = rc.service.SetDetails(ctx, pb.RpcObjectSetDetailsRequest{
+		ContextId: pageID,
+		Details:   setDetailsRequest,
+	})
+	if err != nil {
+		log.Errorf("set details %s", err)
+	}
+
+	if ftd, err := rc.handleCoverRelation(ctx, snapshot, pageID); err != nil {
+		log.Errorf("failed to upload cover image %s", err)
+	} else {
+		filesToDelete = append(filesToDelete, ftd...)
+	}
+
+	return filesToDelete, oldRelationBlockToNew, nil
+}
+
+func (rc *RelationService) update(ctx *session.Context,
+	snapshot *model.SmartBlockSnapshotBase,
+	relations map[string]*converter.Relation,
+	pageID string) ([]string, map[string]*model.Block, []*converter.Relation, error) {
+	var (
+		err                   error
+		filesToDelete         = make([]string, 0)
+		oldRelationBlockToNew = make(map[string]*model.Block, 0)
+		failedRelations       = make([]*converter.Relation, 0)
+	)
+
+	// to get failed relations and fallback them to create function
+	for key, r := range relations {
+		if err = rc.service.AddExtraRelations(ctx, pageID, []string{key}); err != nil {
+			log.Errorf("add extra relation %s", err)
+			failedRelations = append(failedRelations, r)
+			continue
+		}
+		if snapshot.Details != nil && snapshot.Details.Fields != nil {
+			if snapshot.Details.Fields[r.Name].GetListValue() != nil {
+				rc.handleListValue(ctx, snapshot, r, key)
+			}
+			if r.Format == model.RelationFormat_file {
+				filesToDelete = append(filesToDelete, rc.handleFileRelation(ctx, snapshot, r.Name)...)
+			}
+		}
+		setDetailsRequest := make([]*pb.RpcObjectSetDetailsDetail, 0)
+		setDetailsRequest = append(setDetailsRequest, &pb.RpcObjectSetDetailsDetail{
+			Key:   key,
+			Value: snapshot.Details.Fields[r.Name],
+		})
+		if r.BlockID != "" {
+			original, new := rc.linkRelationsBlocks(snapshot, r.BlockID, key)
+			if original != nil && new != nil {
+				oldRelationBlockToNew[original.GetId()] = new
+			}
+		}
+		err = rc.service.SetDetails(ctx, pb.RpcObjectSetDetailsRequest{
+			ContextId: pageID,
+			Details:   setDetailsRequest,
+		})
+		if err != nil {
+			log.Errorf("set details %s", err)
+			failedRelations = append(failedRelations, r)
+		}
+	}
+
+	if ftd, err := rc.handleCoverRelation(ctx, snapshot, pageID); err != nil {
+		log.Errorf("failed to upload cover image %s", err)
+	} else {
+		filesToDelete = append(filesToDelete, ftd...)
+	}
+
+	return filesToDelete, oldRelationBlockToNew, failedRelations, nil
+
+}
+
+func (rc *RelationService) ReplaceRelationBlock(ctx *session.Context,
+	oldRelationBlocksToNew map[string]*model.Block,
+	pageID string) {
+	if sbErr := rc.service.Do(pageID, func(sb editor.SmartBlock) error {
+		s := sb.NewStateCtx(ctx)
+		if err := s.Iterate(func(b simple.Block) (isContinue bool) {
+			if b.Model().GetRelation() == nil {
+				return true
+			}
+			bl, ok := oldRelationBlocksToNew[b.Model().GetId()]
+			if !ok {
+				return true
+			}
+			simpleBlock := simple.New(bl)
+			s.Add(simpleBlock)
+			if err := s.InsertTo(b.Model().GetId(), model.Block_Replace, simpleBlock.Model().GetId()); err != nil {
+				log.With(zap.String("object id", pageID)).Errorf("failed to insert: %w", err)
+			}
+			return true
+		}); err != nil {
+			return err
+		}
+		if err := sb.Apply(s); err != nil {
+			log.With(zap.String("object id", pageID)).Errorf("failed to apply state: %w", err)
+			return err
+		}
+		return nil
+	}); sbErr != nil {
+		log.With(zap.String("object id", pageID)).Errorf("failed to replace relation block: %w", sbErr)
+	}
+}
+
+func (rc *RelationService) handleCoverRelation(ctx *session.Context,
+	snapshot *model.SmartBlockSnapshotBase,
+	pageID string) ([]string, error) {
+	filesToDelete := rc.handleFileRelation(ctx, snapshot, bundle.RelationKeyCoverId.String())
+	details := make([]*pb.RpcObjectSetDetailsDetail, 0)
+	details = append(details, &pb.RpcObjectSetDetailsDetail{
+		Key:   bundle.RelationKeyCoverId.String(),
+		Value: snapshot.Details.Fields[bundle.RelationKeyCoverId.String()],
+	})
+	err := rc.service.SetDetails(ctx, pb.RpcObjectSetDetailsRequest{
+		ContextId: pageID,
+		Details:   details,
+	})
+
+	return filesToDelete, err
+}
+
+func (rc *RelationService) handleListValue(ctx *session.Context,
+	snapshot *model.SmartBlockSnapshotBase,
+	r *converter.Relation,
+	relationID string) {
+	var (
+		optionIDs      = make([]string, 0)
+		id             string
+		err            error
+		existedOptions = make(map[string]string, 0)
+	)
+	options, err := rc.service.Anytype().ObjectStore().GetAggregatedOptions(relationID)
+	if err != nil {
+		log.Errorf("get relations options %s", err)
+	}
+	for _, ro := range options {
+		existedOptions[ro.Text] = ro.Id
+	}
+	for _, tag := range r.SelectDict {
+		if optionID, ok := existedOptions[tag.Text]; ok {
+			optionIDs = append(optionIDs, optionID)
+			continue
+		}
+		if id, _, err = rc.objCreator.CreateSubObjectInWorkspace(&types.Struct{
+			Fields: map[string]*types.Value{
+				bundle.RelationKeyName.String():                pbtypes.String(tag.Text),
+				bundle.RelationKeyRelationKey.String():         pbtypes.String(relationID),
+				bundle.RelationKeyType.String():                pbtypes.String(bundle.TypeKeyRelationOption.URL()),
+				bundle.RelationKeyLayout.String():              pbtypes.Float64(float64(model.ObjectType_relationOption)),
+				bundle.RelationKeyRelationOptionColor.String(): pbtypes.String(tag.Color),
+			},
+		}, rc.core.PredefinedBlocks().Account); err != nil {
+			log.Errorf("add extra relation %s", err)
+		}
+		optionIDs = append(optionIDs, id)
+	}
+	snapshot.Details.Fields[r.Name] = pbtypes.StringList(optionIDs)
+}
+
+func (rc *RelationService) handleFileRelation(ctx *session.Context,
+	snapshot *model.SmartBlockSnapshotBase,
+	name string) []string {
+	var allFiles []string
+	if files := snapshot.Details.Fields[name].GetListValue(); files != nil {
+		for _, f := range files.Values {
+			allFiles = append(allFiles, f.GetStringValue())
+		}
+	}
+
+	if files := snapshot.Details.Fields[name].GetStringValue(); files != "" {
+		allFiles = append(allFiles, files)
+	}
+
+	allFilesHashes := make([]string, 0)
+
+	filesToDelete := make([]string, 0, len(allFiles))
+	for _, f := range allFiles {
+		if f == "" {
+			continue
+		}
+		if _, err := rc.store.GetByHash(f); err == nil {
+			allFilesHashes = append(allFilesHashes, f)
+			continue
+		}
+
+		req := pb.RpcFileUploadRequest{LocalPath: f}
+
+		if strings.HasPrefix(f, "http://") || strings.HasPrefix(f, "https://") {
+			req.Url = f
+			req.LocalPath = ""
+		}
+
+		hash, err := rc.service.UploadFile(req)
+		if err != nil {
+			log.Errorf("file uploading %s", err)
+		} else {
+			f = hash
+		}
+
+		filesToDelete = append(filesToDelete, f)
+		allFilesHashes = append(allFilesHashes, f)
+	}
+
+	if snapshot.Details.Fields[name].GetListValue() != nil {
+		snapshot.Details.Fields[name] = pbtypes.StringList(allFilesHashes)
+	}
+
+	if snapshot.Details.Fields[name].GetStringValue() != "" && len(allFilesHashes) != 0 {
+		snapshot.Details.Fields[name] = pbtypes.String(allFilesHashes[0])
+	}
+
+	return filesToDelete
+}
+
+func (rc *RelationService) linkRelationsBlocks(snapshot *model.SmartBlockSnapshotBase,
+	oldID, newID string) (*model.Block, *model.Block) {
+	for _, b := range snapshot.Blocks {
+		if rel, ok := b.Content.(*model.BlockContentOfRelation); ok && rel.Relation.GetKey() == oldID {
+			return b, &model.Block{
+				Id: bson.NewObjectId().Hex(),
+				Content: &model.BlockContentOfRelation{
+					Relation: &model.BlockContentRelation{
+						Key: newID,
+					},
+				}}
+		}
+	}
+	return nil, nil
+}

core/block/import/syncer/bookmark.go

@@ -20,7 +20,7 @@ func NewBookmarkSyncer(service *block.Service) *BookmarkSyncer {
 func (bs *BookmarkSyncer) Sync(ctx *session.Context, id string, b simple.Block) error {
 	err := bs.service.BookmarkFetch(ctx, pb.RpcBlockBookmarkFetchRequest{
 		ContextId: id,
-		BlockId:   b.Model().GetBookmark().TargetObjectId,
+		BlockId:   b.Model().GetId(),
 		Url:       b.Model().GetBookmark().Url,
 	})
 	if err != nil {

core/block/import/syncer/icon.go

@@ -0,0 +1,51 @@
+package syncer
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/anytypeio/go-anytype-middleware/core/block"
+	"github.com/anytypeio/go-anytype-middleware/core/block/editor/basic"
+	"github.com/anytypeio/go-anytype-middleware/core/block/editor/smartblock"
+	"github.com/anytypeio/go-anytype-middleware/core/block/simple"
+	"github.com/anytypeio/go-anytype-middleware/core/session"
+	"github.com/anytypeio/go-anytype-middleware/pb"
+)
+
+type IconSyncer struct {
+	service *block.Service
+}
+
+func NewIconSyncer(service *block.Service) *IconSyncer {
+	return &IconSyncer{service: service}
+}
+
+func (is *IconSyncer) Sync(ctx *session.Context, id string, b simple.Block) error {
+	fileName := b.Model().GetText().GetIconImage()
+	req := pb.RpcFileUploadRequest{LocalPath: fileName}
+	if strings.HasPrefix(fileName, "http://") || strings.HasPrefix(fileName, "https://") {
+		req = pb.RpcFileUploadRequest{Url: fileName}
+	}
+	hash, err := is.service.UploadFile(req)
+	if err != nil {
+		return fmt.Errorf("failed uploading icon image file: %s", err)
+	}
+
+	err = is.service.Do(id, func(sb smartblock.SmartBlock) error {
+		bs := basic.NewBasic(sb)
+		upErr := bs.Update(ctx, func(simpleBlock simple.Block) error {
+			simpleBlock.Model().GetText().IconImage = hash
+			return nil
+		}, b.Model().Id)
+		if upErr != nil {
+			return fmt.Errorf("failed to update block: %s", err)
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed to update block: %s", err)
+	}
+	os.Remove(fileName)
+	return nil
+}

core/block/import/syncer/syncer.go

@@ -5,10 +5,11 @@ import "github.com/anytypeio/go-anytype-middleware/core/block/simple"
 type Factory struct {
 	fs Syncer
 	bs Syncer
+	is Syncer
 }
 
-func New(fs *FileSyncer, bs *BookmarkSyncer) *Factory {
-	return &Factory{fs:fs, bs: bs}
+func New(fs *FileSyncer, bs *BookmarkSyncer, is *IconSyncer) *Factory {
+	return &Factory{fs: fs, bs: bs, is: is}
 }
 
 func (f *Factory) GetSyncer(b simple.Block) Syncer {
@@ -18,5 +19,8 @@ func (f *Factory) GetSyncer(b simple.Block) Syncer {
 	if file := b.Model().GetFile(); file != nil {
 		return f.fs
 	}
+	if b.Model().GetText() != nil && b.Model().GetText().GetIconImage() != "" {
+		return f.is
+	}
 	return nil
 }

core/block/import/types.go

@@ -1,14 +1,18 @@
 package importer
 
 import (
+	"github.com/gogo/protobuf/types"
+
 	"github.com/anytypeio/go-anytype-middleware/app"
-	_ "github.com/anytypeio/go-anytype-middleware/core/block/import/markdown"
-	_ "github.com/anytypeio/go-anytype-middleware/core/block/import/pb"
+	"github.com/anytypeio/go-anytype-middleware/core/block/import/converter"
 	"github.com/anytypeio/go-anytype-middleware/core/session"
 	"github.com/anytypeio/go-anytype-middleware/pb"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
-	"github.com/gogo/protobuf/types"
+	// import plugins
+	_ "github.com/anytypeio/go-anytype-middleware/core/block/import/markdown"
+	_ "github.com/anytypeio/go-anytype-middleware/core/block/import/notion"
+	_ "github.com/anytypeio/go-anytype-middleware/core/block/import/pb"
 )
 
 // Importer incapsulate logic with import
@@ -21,10 +25,20 @@ type Importer interface {
 
 // Creator incapsulate logic with creation of given smartblocks
 type Creator interface {
-	Create(ctx *session.Context, cs *model.SmartBlockSnapshotBase, pageID string, sbType smartblock.SmartBlockType, updateExisting bool) (*types.Struct, error)
+	//nolint: lll
+	Create(ctx *session.Context, cs *model.SmartBlockSnapshotBase, relations []*converter.Relation, pageID string, sbType smartblock.SmartBlockType, updateExisting bool) (*types.Struct, error)
 }
 
 // Updater is interface for updating existing objects
 type Updater interface {
-	Update(ctx *session.Context, cs *model.SmartBlockSnapshotBase, pageID string) (*types.Struct, error)
+	//nolint: lll
+	Update(ctx *session.Context, cs *model.SmartBlockSnapshotBase, relations []*converter.Relation, pageID string) (*types.Struct, []string, error)
+}
+
+// RelationCreator incapsulates logic for creation of relations
+type RelationCreator interface {
+	//nolint: lll
+	ReplaceRelationBlock(ctx *session.Context, oldRelationBlocksToNew map[string]*model.Block, pageID string)
+	//nolint: lll
+	CreateRelations(ctx *session.Context, snapshot *model.SmartBlockSnapshotBase, pageID string, relations []*converter.Relation) ([]string, map[string]*model.Block, error)
 }

core/block/import/web/converter.go

@@ -3,12 +3,14 @@ package web
 import (
 	"fmt"
 
+	"github.com/textileio/go-threads/core/thread"
+
 	"github.com/anytypeio/go-anytype-middleware/core/block/import/converter"
 	"github.com/anytypeio/go-anytype-middleware/core/block/import/web/parsers"
+	"github.com/anytypeio/go-anytype-middleware/core/block/process"
 	"github.com/anytypeio/go-anytype-middleware/pb"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/threads"
-	"github.com/textileio/go-threads/core/thread"
 )
 
 const name = "web"
@@ -29,28 +31,37 @@ func (*Converter) GetParser(url string) parsers.Parser {
 	return nil
 }
 
-func (c *Converter) GetSnapshots(req *pb.RpcObjectImportRequest) *converter.Response {
+func (c *Converter) GetSnapshots(req *pb.RpcObjectImportRequest,
+	progress *process.Progress) (*converter.Response, converter.ConvertError) {
 	we := converter.NewError()
 	url, err := c.getParams(req.Params)
+
+	progress.SetTotal(1)
+
 	if err != nil {
 		we.Add(url, err)
-		return &converter.Response{Error: we}
+		return nil, we
 	}
 	p := c.GetParser(url)
 	if p == nil {
 		we.Add(url, fmt.Errorf("unknown url format"))
-		return &converter.Response{Error: we}
+		return nil, we
 	}
+
+	progress.SetProgressMessage("Start parsing url to snapshot")
 	snapshots, err := p.ParseUrl(url)
+
+	progress.AddDone(1)
+
 	if err != nil {
 		we.Add(url, err)
-		return &converter.Response{Error: we}
+		return nil, we
 	}
 
 	tid, err := threads.ThreadCreateID(thread.AccessControlled, smartblock.SmartBlockTypePage)
 	if err != nil {
 		we.Add(url, err)
-		return &converter.Response{Error: we}
+		return nil, we
 	}
 	s := &converter.Snapshot{
 		Id:       tid.String(),
@@ -58,10 +69,10 @@ func (c *Converter) GetSnapshots(req *pb.RpcObjectImportRequest) *converter.Resp
 		Snapshot: snapshots,
 	}
 	res := &converter.Response{
-		Snapshots:      []*converter.Snapshot{s},
-		Error:          nil,
+		Snapshots: []*converter.Snapshot{s},
 	}
-	return res
+
+	return res, nil
 }
 
 func (p *Converter) Name() string {
@@ -73,4 +84,4 @@ func (p *Converter) getParams(params pb.IsRpcObjectImportRequestParams) (string,
 		return p.BookmarksParams.GetUrl(), nil
 	}
 	return "", fmt.Errorf("PB: GetParams wrong parameters format")
-}
+}

core/block/import/web/parsers/mock.go

@@ -7,8 +7,9 @@ package parsers
 import (
 	reflect "reflect"
 
-	model "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
 	gomock "github.com/golang/mock/gomock"
+
+	model "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
 )
 
 // MockParser is a mock of Parser interface.

core/block/object/creator.go

@@ -251,7 +251,7 @@ func (c *Creator) CreateSet(req *pb.RpcObjectCreateSetRequest) (setID string, ne
 	}
 
 	// TODO: here can be a deadlock if this is somehow created from workspace (as set)
-	return c.CreateSmartBlockFromState(context.TODO(), coresb.SmartBlockTypeSet, req.Details, nil, newState)
+	return c.CreateSmartBlockFromState(context.TODO(), coresb.SmartBlockTypeSet, nil, nil, newState)
 }
 
 // TODO: it must be in another component
@@ -280,7 +280,7 @@ func (c *Creator) CreateSubObjectsInWorkspace(details []*types.Struct) (ids []st
 
 // ObjectCreateBookmark creates a new Bookmark object for provided URL or returns id of existing one
 func (c *Creator) ObjectCreateBookmark(req *pb.RpcObjectCreateBookmarkRequest) (objectID string, newDetails *types.Struct, err error) {
-	u, err := uri.ProcessURI(pbtypes.GetString(req.Details, bundle.RelationKeySource.String()))
+	u, err := uri.NormalizeURI(pbtypes.GetString(req.Details, bundle.RelationKeySource.String()))
 	if err != nil {
 		return "", nil, fmt.Errorf("process uri: %w", err)
 	}

core/block/process/progress.go

@@ -1,11 +1,13 @@
 package process
 
 import (
+	"fmt"
 	"sync"
 	"sync/atomic"
 
-	"github.com/anytypeio/go-anytype-middleware/pb"
 	"github.com/globalsign/mgo/bson"
+
+	"github.com/anytypeio/go-anytype-middleware/pb"
 )
 
 func NewProgress(pType pb.ModelProcessType) *Progress {
@@ -106,3 +108,15 @@ func (p *Progress) Info() pb.ModelProcess {
 func (p *Progress) Done() chan struct{} {
 	return p.done
 }
+
+func (p *Progress) TryStep(delta int64) error {
+	select {
+	case <-p.Canceled():
+		return fmt.Errorf("cancelled import")
+	default:
+	}
+
+	p.AddDone(delta)
+
+	return nil
+}

core/block/service.go

@@ -429,13 +429,13 @@ func (s *Service) AddSubObjectsToWorkspace(
 	return
 }
 
-func (s *Service) RemoveSubObjectsInWorkspace(objectIds []string, workspaceId string) (err error) {
+func (s *Service) RemoveSubObjectsInWorkspace(objectIds []string, workspaceId string, orphansGC bool) (err error) {
 	err = s.Do(workspaceId, func(b smartblock.SmartBlock) error {
 		ws, ok := b.(*editor.Workspaces)
 		if !ok {
 			return fmt.Errorf("incorrect workspace id")
 		}
-		err = ws.RemoveSubObjects(objectIds)
+		err = ws.RemoveSubObjects(objectIds, orphansGC)
 		return err
 	})
 
@@ -1173,7 +1173,7 @@ func (s *Service) ResetToState(pageId string, state *state.State) (err error) {
 }
 
 func (s *Service) ObjectBookmarkFetch(req pb.RpcObjectBookmarkFetchRequest) (err error) {
-	url, err := uri.ProcessURI(req.Url)
+	url, err := uri.NormalizeURI(req.Url)
 	if err != nil {
 		return fmt.Errorf("process uri: %w", err)
 	}

core/block/simple/text/text.go

@@ -234,7 +234,7 @@ func (t *Text) SetText(text string, marks *model.BlockContentTextMarks) (err err
 	} else {
 		for mI, _ := range marks.Marks {
 			if marks.Marks[mI].Type == model.BlockContentTextMark_Link {
-				m, err := uri.ProcessURI(marks.Marks[mI].Param)
+				m, err := uri.NormalizeURI(marks.Marks[mI].Param)
 				if err == nil {
 					marks.Marks[mI].Param = m
 				}
@@ -700,10 +700,10 @@ func (t *Text) IsEmpty() bool {
 	return false
 }
 
-func isIncompatibleType(firstType, secondType model.BlockContentTextMarkType ) bool {
-	if (firstType == model.BlockContentTextMark_Link && secondType == model.BlockContentTextMark_Object) || 
-	(secondType == model.BlockContentTextMark_Link && firstType == model.BlockContentTextMark_Object) {
+func isIncompatibleType(firstType, secondType model.BlockContentTextMarkType) bool {
+	if (firstType == model.BlockContentTextMark_Link && secondType == model.BlockContentTextMark_Object) ||
+		(secondType == model.BlockContentTextMark_Link && firstType == model.BlockContentTextMark_Object) {
 		return true
 	}
 	return false
-}
+}

core/block/source/source.go

@@ -136,6 +136,7 @@ type source struct {
 	sb                       core.SmartBlock
 	tree                     *change.Tree
 	lastSnapshotId           string
+	changesSinceSnapshot     int
 	logHeads                 map[string]*change.Change
 	receiver                 ChangeReceiver
 	unsubscribe              func()
@@ -314,10 +315,11 @@ func (s *source) buildState() (doc state.Doc, err error) {
 	s.lastSnapshotId = root.Id
 	doc = state.NewDocFromSnapshot(s.id, root.GetSnapshot()).(*state.State)
 	doc.(*state.State).SetChangeId(root.Id)
-	st, err := change.BuildStateSimpleCRDT(doc.(*state.State), s.tree)
+	st, changesApplied, err := change.BuildStateSimpleCRDT(doc.(*state.State), s.tree)
 	if err != nil {
 		return
 	}
+	s.changesSinceSnapshot = changesApplied
 
 	if s.sb.Type() != smartblock.SmartBlockTypeArchive && !s.Virtual() {
 		if verr := st.Validate(); verr != nil {
@@ -406,6 +408,9 @@ func (s *source) PushChange(params PushChangeParams) (id string, err error) {
 			},
 			FileKeys: s.getFileHashesForSnapshot(params.FileChangedHashes),
 		}
+		if s.tree.Len() > 0 {
+			log.With("thread", s.id).With("len", s.tree.Len(), "lenSnap", s.changesSinceSnapshot, "changes", len(params.Changes), "doSnap", params.DoSnapshot).Warnf("do the snapshot")
+		}
 	}
 	c.Content = params.Changes
 	c.FileKeys = s.getFileKeysByHashes(params.FileChangedHashes)
@@ -418,8 +423,10 @@ func (s *source) PushChange(params PushChangeParams) (id string, err error) {
 	s.logHeads[s.logId] = ch
 	if c.Snapshot != nil {
 		s.lastSnapshotId = id
+		s.changesSinceSnapshot = 0
 		log.Infof("%s: pushed snapshot", s.id)
 	} else {
+		s.changesSinceSnapshot++
 		log.Debugf("%s: pushed %d changes", s.id, len(ch.Content))
 	}
 	return
@@ -444,13 +451,31 @@ func (v *source) ListIds() ([]string, error) {
 	return ids, nil
 }
 
+func snapshotChance(changesSinceSnapshot int) bool {
+	v := 2000
+	if changesSinceSnapshot <= 100 {
+		return false
+	}
+
+	d := changesSinceSnapshot/50 + 1
+
+	min := (v / 2) - d
+	max := (v / 2) + d
+
+	r := rand.Intn(v)
+	if r >= min && r <= max {
+		return true
+	}
+
+	return false
+}
+
 func (s *source) needSnapshot() bool {
 	if s.tree.Len() == 0 {
 		// starting tree with snapshot
 		return true
 	}
-	// TODO: think about a more smart way
-	return rand.Intn(500) == 42
+	return snapshotChance(s.changesSinceSnapshot)
 }
 
 func (s *source) changeListener(batch *mb.MB) {
@@ -515,11 +540,17 @@ func (s *source) applyRecords(records []core.SmartblockRecordEnvelope) error {
 	case change.Append:
 		changesContent := make([]*pb.ChangeContent, 0, len(changes))
 		for _, ch := range changes {
+			if ch.Snapshot != nil {
+				s.changesSinceSnapshot = 0
+			} else {
+				s.changesSinceSnapshot++
+			}
 			changesContent = append(changesContent, ch.Content...)
 		}
 		s.lastSnapshotId = s.tree.LastSnapshotId(context.TODO())
 		return s.receiver.StateAppend(func(d state.Doc) (*state.State, error) {
-			return change.BuildStateSimpleCRDT(d.(*state.State), s.tree)
+			st, _, err := change.BuildStateSimpleCRDT(d.(*state.State), s.tree)
+			return st, err
 		}, changesContent)
 	case change.Rebuild:
 		s.lastSnapshotId = s.tree.LastSnapshotId(context.TODO())

core/block/source/source_test.go

@@ -0,0 +1,44 @@
+package source
+
+import (
+	"fmt"
+	"os"
+	"testing"
+)
+
+func Test_snapshotChance(t *testing.T) {
+	if os.Getenv("ANYTYPE_TEST_SNAPSHOT_CHANCE") == "" {
+		t.Skip()
+		return
+	}
+	for i := 0; i <= 500; i++ {
+		for s := 0; s <= 10000; s++ {
+			if snapshotChance(s) {
+				fmt.Println(s)
+				break
+			}
+		}
+	}
+	fmt.Println()
+	// here is an example of distribution histogram
+	// https://docs.google.com/spreadsheets/d/1xgH7fUxno5Rm-0VEaSD4LsTHeGeUXQFmHsOm29M6paI
+}
+
+func Test_snapshotChance2(t *testing.T) {
+	if os.Getenv("ANYTYPE_TEST_SNAPSHOT_CHANCE") == "" {
+		t.Skip()
+		return
+	}
+	for s := 0; s <= 10000; s++ {
+		total := 0
+		for i := 0; i <= 50000; i++ {
+			if snapshotChance(s) {
+				total++
+			}
+		}
+		fmt.Printf("%d\t%.5f\n", s, float64(total)/50000)
+	}
+
+	// here is an example of distribution histogram
+	// https://docs.google.com/spreadsheets/d/1xgH7fUxno5Rm-0VEaSD4LsTHeGeUXQFmHsOm29M6paI
+}

core/block/source/threaddb.go

@@ -192,7 +192,11 @@ func (v *threadDB) listenToChanges() (err error) {
 				// we don't have new messages for at least deadline and we have something to flush
 				processBuffer()
 
-			case c := <-listenerCh:
+			case c, ok := <-listenerCh:
+				if !ok {
+					processBuffer()
+					return
+				}
 				log.With("thread id", c.ID.String()).
 					Debugf("received new thread through channel")
 				// as per docs the timer should be stopped or expired with drained channel

core/converter/md/md.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"html"
 	"io"
-	"net/url"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -21,6 +20,7 @@ import (
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/core"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
 	"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
+	"github.com/anytypeio/go-anytype-middleware/util/uri"
 )
 
 type FileNamer interface {
@@ -217,7 +217,7 @@ func (h *MD) renderBookmark(buf writer, in *renderState, b *model.Block) {
 	bm := b.GetBookmark()
 	if bm != nil && bm.Url != "" {
 		buf.WriteString(in.indent)
-		url, e := url.Parse(bm.Url)
+		url, e := uri.ParseURI(bm.Url)
 		if e == nil {
 			fmt.Fprintf(buf, "[%s](%s)    \n", escape.MarkdownCharacters(html.EscapeString(bm.Title)), url.String())
 		}
@@ -411,7 +411,7 @@ func (mw *marksWriter) writeMarks(buf writer, pos int) {
 			if start {
 				buf.WriteString("[")
 			} else {
-				urlP, e := url.Parse(m.Param)
+				urlP, e := uri.ParseURI(m.Param)
 				urlS := m.Param
 				if e == nil {
 					urlS = urlP.String()

core/debug/debugtree/debugtree.go

@@ -213,7 +213,7 @@ func (r *debugTree) BuildStateByTree(t *change.Tree) (*state.State, error) {
 	}
 	s := state.NewDocFromSnapshot("", root.GetSnapshot()).(*state.State)
 	s.SetChangeId(root.Id)
-	st, err := change.BuildStateSimpleCRDT(s, t)
+	st, _, err := change.BuildStateSimpleCRDT(s, t)
 	if err != nil {
 		return nil, err
 	}

core/history/history.go

@@ -218,7 +218,7 @@ func (h *history) buildState(pageId, versionId string) (s *state.State, ver *pb.
 	}
 	s = state.NewDocFromSnapshot(pageId, root.GetSnapshot()).(*state.State)
 	s.SetChangeId(root.Id)
-	st, err := change.BuildStateSimpleCRDT(s, tree)
+	st, _, err := change.BuildStateSimpleCRDT(s, tree)
 	if err != nil {
 		return
 	}

core/indexer/indexer.go

@@ -801,7 +801,7 @@ func (i *indexer) index(ctx context.Context, info doc.DocInfo) error {
 	}
 
 	indexDetails, indexLinks := sbType.Indexable()
-	if sbType != smartblock.SmartBlockTypeSubObject && sbType != smartblock.SmartBlockTypeWorkspace {
+	if sbType != smartblock.SmartBlockTypeSubObject && sbType != smartblock.SmartBlockTypeWorkspace && sbType != smartblock.SmartBlockTypeBreadcrumbs {
 		// avoid recursions
 
 		if pbtypes.GetString(info.State.CombinedDetails(), bundle.RelationKeyCreator.String()) != addr.AnytypeProfileId {

core/linkpreview.go

@@ -2,7 +2,7 @@ package core
 
 import (
 	"context"
-	"net/url"
+	"fmt"
 	"strings"
 	"time"
 
@@ -15,22 +15,12 @@ func (mw *Middleware) LinkPreview(cctx context.Context, req *pb.RpcLinkPreviewRe
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
 
-	urlStr, err := uri.ProcessURI(req.Url)
+	u, err := uri.NormalizeAndParseURI(req.Url)
 	if err != nil {
 		return &pb.RpcLinkPreviewResponse{
 			Error: &pb.RpcLinkPreviewResponseError{
 				Code:        pb.RpcLinkPreviewResponseError_UNKNOWN_ERROR,
-				Description: err.Error(),
-			},
-		}
-	}
-
-	u, err := url.Parse(urlStr)
-	if err != nil {
-		return &pb.RpcLinkPreviewResponse{
-			Error: &pb.RpcLinkPreviewResponseError{
-				Code:        pb.RpcLinkPreviewResponseError_UNKNOWN_ERROR,
-				Description: "failed to parse url",
+				Description: fmt.Sprintf("failed to parse url: %v", err),
 			},
 		}
 	}

core/object.go

@@ -15,6 +15,7 @@ import (
 	"github.com/anytypeio/go-anytype-middleware/core/block"
 	importer "github.com/anytypeio/go-anytype-middleware/core/block/import"
 	"github.com/anytypeio/go-anytype-middleware/core/indexer"
+	"github.com/anytypeio/go-anytype-middleware/core/relation"
 	"github.com/anytypeio/go-anytype-middleware/core/subscription"
 	"github.com/anytypeio/go-anytype-middleware/pb"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle"
@@ -397,12 +398,19 @@ func (mw *Middleware) ObjectGraph(cctx context.Context, req *pb.RpcObjectGraphRe
 	}
 
 	at := mw.app.MustComponent(core.CName).(core.Service)
+	rs := mw.app.MustComponent(relation.CName).(relation.Service)
 
 	records, _, err := at.ObjectStore().Query(nil, database.Query{
 		Filters:          req.Filters,
 		Limit:            int(req.Limit),
 		ObjectTypeFilter: req.ObjectTypeFilter,
 	})
+
+	if err != nil {
+		return response(pb.RpcObjectGraphResponseError_UNKNOWN_ERROR, nil, nil, err)
+	}
+
+	relations, err := rs.ListAll(relation.WithWorkspaceId(at.PredefinedBlocks().Account))
 	if err != nil {
 		return response(pb.RpcObjectGraphResponseError_UNKNOWN_ERROR, nil, nil, err)
 	}
@@ -440,10 +448,8 @@ func (mw *Middleware) ObjectGraph(cctx context.Context, req *pb.RpcObjectGraphRe
 			if list := pbtypes.GetStringListValue(v); len(list) == 0 {
 				continue
 			} else {
-
-				rel, err := at.ObjectStore().GetRelationByKey(k)
-				if err != nil {
-					log.Errorf("ObjectGraph failed to get relation %s: %s", k, err.Error())
+				rel := relations.GetByKey(k)
+				if rel == nil {
 					continue
 				}
 

core/relation/service.go

@@ -3,7 +3,6 @@ package relation
 import (
 	"errors"
 	"fmt"
-	"net/url"
 	"strings"
 	"sync"
 
@@ -19,6 +18,7 @@ import (
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/logging"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
 	"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
+	"github.com/anytypeio/go-anytype-middleware/util/uri"
 )
 
 const CName = "relation"
@@ -35,6 +35,8 @@ func New() Service {
 type Service interface {
 	FetchKeys(keys ...string) (relations relationutils.Relations, err error)
 	FetchKey(key string, opts ...FetchOption) (relation *relationutils.Relation, err error)
+	ListAll(opts ...FetchOption) (relations relationutils.Relations, err error)
+
 	FetchLinks(links pbtypes.RelationLinks) (relations relationutils.Relations, err error)
 	CreateBulkMigration() BulkMigration
 	MigrateRelations(relations []*model.Relation) error
@@ -144,8 +146,6 @@ func (b *bulkMigration) Commit() error {
 type service struct {
 	objectStore     objectstore.ObjectStore
 	relationCreator subObjectCreator
-
-	mu sync.RWMutex
 }
 
 func (s *service) MigrateRelations(relations []*model.Relation) error {
@@ -184,8 +184,6 @@ func (s *service) FetchLinks(links pbtypes.RelationLinks) (relations relationuti
 }
 
 func (s *service) FetchKeys(keys ...string) (relations relationutils.Relations, err error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
 	return s.fetchKeys(keys...)
 }
 
@@ -208,6 +206,43 @@ func (s *service) fetchKeys(keys ...string) (relations []*relationutils.Relation
 	return
 }
 
+func (s *service) ListAll(opts ...FetchOption) (relations relationutils.Relations, err error) {
+	return s.listAll(opts...)
+}
+
+func (s *service) listAll(opts ...FetchOption) (relations relationutils.Relations, err error) {
+	filters := []*model.BlockContentDataviewFilter{
+		{
+			RelationKey: bundle.RelationKeyType.String(),
+			Condition:   model.BlockContentDataviewFilter_Equal,
+			Value:       pbtypes.String(bundle.TypeKeyRelation.URL()),
+		},
+	}
+	o := &fetchOptions{}
+	for _, apply := range opts {
+		apply(o)
+	}
+	if o.workspaceId != nil {
+		filters = append(filters, &model.BlockContentDataviewFilter{
+			RelationKey: bundle.RelationKeyWorkspaceId.String(),
+			Condition:   model.BlockContentDataviewFilter_Equal,
+			Value:       pbtypes.String(*o.workspaceId),
+		})
+	}
+
+	relations2, _, err := s.objectStore.Query(nil, database.Query{
+		Filters: filters,
+	})
+	if err != nil {
+		return
+	}
+
+	for _, rec := range relations2 {
+		relations = append(relations, relationutils.RelationFromStruct(rec.Details))
+	}
+	return
+}
+
 type fetchOptions struct {
 	workspaceId *string
 }
@@ -215,8 +250,6 @@ type fetchOptions struct {
 type FetchOption func(options *fetchOptions)
 
 func (s *service) FetchKey(key string, opts ...FetchOption) (relation *relationutils.Relation, err error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
 	return s.fetchKey(key, opts...)
 }
 
@@ -341,7 +374,7 @@ func (s *service) ValidateFormat(key string, v *types.Value) error {
 			return fmt.Errorf("incorrect type: %T instead of string", v.Kind)
 		}
 
-		_, err := url.Parse(strings.TrimSpace(v.GetStringValue()))
+		err := uri.ValidateURI(strings.TrimSpace(v.GetStringValue()))
 		if err != nil {
 			return fmt.Errorf("failed to parse URL: %s", err.Error())
 		}

core/relations.go

@@ -178,6 +178,13 @@ func (mw *Middleware) ObjectCreateSet(cctx context.Context, req *pb.RpcObjectCre
 	)
 	err := mw.doBlockService(func(bs *block.Service) error {
 		var err error
+		if req.Details == nil {
+			req.Details = &types.Struct{}
+		}
+		if req.Details.Fields == nil {
+			req.Details.Fields = map[string]*types.Value{}
+		}
+		req.Details.Fields[bundle.RelationKeySetOf.String()] = pbtypes.StringList(req.Source)
 		id, newDetails, err = bs.CreateObject(req, bundle.TypeKeySet)
 		return err
 	})

core/workspace.go

@@ -184,7 +184,7 @@ func (mw *Middleware) WorkspaceObjectListRemove(cctx context.Context, req *pb.Rp
 	)
 
 	err := mw.doBlockService(func(bs *block.Service) (err error) {
-		err = bs.RemoveSubObjectsInWorkspace(req.ObjectIds, mw.GetAnytype().PredefinedBlocks().Account)
+		err = bs.RemoveSubObjectsInWorkspace(req.ObjectIds, mw.GetAnytype().PredefinedBlocks().Account, true)
 		return
 	})
 

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/anytypeio/go-slip21 v0.0.0-20200218204727-e2e51e20ab51
 	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
 	github.com/blevesearch/bleve/v2 v2.3.6
+	github.com/chai2010/webp v1.1.1
 	github.com/cheggaaa/mb v1.0.3
 	github.com/dave/jennifer v1.4.1
 	github.com/davecgh/go-spew v1.1.1
@@ -87,6 +88,7 @@ require (
 	github.com/yuin/goldmark v1.4.13
 	go.uber.org/zap v1.24.0
 	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db
+	golang.org/x/image v0.0.0-20211028202545-6944b10bf410
 	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b
 	golang.org/x/text v0.5.0
 	google.golang.org/grpc v1.47.0
@@ -275,7 +277,6 @@ require (
 	go.uber.org/fx v1.18.2 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	golang.org/x/crypto v0.3.0 // indirect
-	golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
 	golang.org/x/mod v0.7.0 // indirect
 	golang.org/x/net v0.3.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect

go.sum

@@ -185,6 +185,8 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk=
+github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/cheggaaa/mb v1.0.3 h1:03ksWum+6kHclB+kjwKMaBtgl5gtNYUwNpxsHQciKe8=
 github.com/cheggaaa/mb v1.0.3/go.mod h1:NUl0GBtFLlfg2o6iZwxzcG7Lslc2wV/ADTFbLXtVPE4=
@@ -1567,8 +1569,9 @@ golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86h
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg=
 golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
+golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=

pb/protos/commands.proto

@@ -1735,13 +1735,18 @@ message Rpc {
                 oneof params {
                     NotionParams notionParams = 1;
                     BookmarksParams bookmarksParams = 2; //for internal use
+                    MarkdownParams markdownParams = 3;
                 }
-                repeated Snapshot snapshots = 3; // optional, for external developers usage
-                bool updateExistingObjects = 4;
-                Type type = 5;
-                Mode mode = 6;
+                repeated Snapshot snapshots = 4; // optional, for external developers usage
+                bool updateExistingObjects = 5;
+                Type type = 6;
+                Mode mode = 7;
 
                 message NotionParams {
+                    string apiKey = 1;
+                }
+
+                message MarkdownParams {
                     string path = 1;
                 }
 
@@ -1760,7 +1765,8 @@ message Rpc {
                 };
                 enum Type {
                     Notion = 0;
-                    External = 1; // external developers use it
+                    Markdown = 1; 
+                    External = 2; // external developers use it
                 };
 
             }
@@ -1805,6 +1811,7 @@ message Rpc {
                 Type type = 1; 
                 enum Type {
                     Notion = 0;
+                    Markdown = 1;
                 };
             }
         }

pkg/lib/core/core.go

@@ -3,6 +3,17 @@ package core
 import (
 	"context"
 	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/libp2p/go-libp2p/core/peer"
+	pstore "github.com/libp2p/go-libp2p/core/peerstore"
+	"github.com/libp2p/go-libp2p/p2p/discovery/mdns"
+	"github.com/textileio/go-threads/core/net"
+
 	"github.com/anytypeio/go-anytype-middleware/app"
 	"github.com/anytypeio/go-anytype-middleware/core/anytype/config"
 	"github.com/anytypeio/go-anytype-middleware/core/configfetcher"
@@ -23,14 +34,6 @@ import (
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pin"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/threads"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/util"
-	"github.com/libp2p/go-libp2p/core/peer"
-	pstore "github.com/libp2p/go-libp2p/core/peerstore"
-	"github.com/libp2p/go-libp2p/p2p/discovery/mdns"
-	"github.com/textileio/go-threads/core/net"
-	"os"
-	"path/filepath"
-	"strings"
-	"sync"
 )
 
 var log = logging.Logger("anytype-core")

pkg/lib/core/images.go

@@ -3,6 +3,8 @@ package core
 import (
 	"context"
 	"fmt"
+	"io"
+
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/files"
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/storage"
 )

pkg/lib/database/database.go

@@ -80,7 +80,6 @@ type Query struct {
 	WithSystemObjects bool
 	ObjectTypeFilter  []string
 	WorkspaceId       string
-	SearchInWorkspace bool
 }
 
 func (q Query) DSQuery(sch schema.Schema) (qq query.Query, err error) {

pkg/lib/database/subscription.go

@@ -9,10 +9,11 @@ import (
 )
 
 type subscription struct {
-	ids    []string
-	quit   chan struct{}
-	closed bool
-	ch     chan *types.Struct
+	ids        []string
+	quit       chan struct{}
+	closed     bool
+	closedOnce sync.Once
+	ch         chan *types.Struct
 	sync.RWMutex
 }
 
@@ -29,21 +30,21 @@ func (sub *subscription) RecordChan() chan *types.Struct {
 }
 
 func (sub *subscription) Close() {
-	sub.Lock()
-	defer sub.Unlock()
-	if sub.closed {
-		return
-	}
-
-	close(sub.quit)
-	close(sub.ch)
-
-	sub.closed = true
+	sub.closedOnce.Do(func() {
+		close(sub.quit)
+		close(sub.ch)
+		sub.Lock()
+		sub.closed = true
+		sub.Unlock()
+	})
 }
 
 func (sub *subscription) Subscribe(ids []string) (added []string) {
 	sub.Lock()
 	defer sub.Unlock()
+	if sub.closed {
+		return nil
+	}
 loop:
 	for _, id := range ids {
 		for _, idEx := range sub.ids {

pkg/lib/mill/image_resize.go

@@ -3,9 +3,6 @@ package mill
 import (
 	"bytes"
 	"fmt"
-	"github.com/disintegration/imaging"
-	"github.com/dsoprea/go-exif/v3"
-	jpegstructure "github.com/dsoprea/go-jpeg-image-structure/v2"
 	"image"
 	"image/color/palette"
 	"image/draw"
@@ -15,7 +12,16 @@ import (
 	"io"
 	"strconv"
 
+	"github.com/chai2010/webp"
+	"github.com/disintegration/imaging"
+	"github.com/dsoprea/go-exif/v3"
+	jpegstructure "github.com/dsoprea/go-jpeg-image-structure/v2"
+	"golang.org/x/exp/slices"
+
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/mill/ico"
+
+	// Import for image.DecodeConfig to support .webp format
+	_ "golang.org/x/image/webp"
 )
 
 // Format enumerates the type of images currently supported
@@ -30,6 +36,7 @@ const (
 	PNG  Format = "png"
 	GIF  Format = "gif"
 	ICO  Format = "ico"
+	WEBP Format = "webp"
 )
 
 type ImageSize struct {
@@ -150,7 +157,7 @@ func (m *ImageResize) Mill(r io.ReadSeeker, name string) (*Result, error) {
 			r2 = r
 		}
 		// here is an optimisation
-		// lets return the original picture in case it has not been resized or normilised
+		// lets return the original picture in case it has not been resized or normalized
 		return &Result{
 			File: r2,
 			Meta: map[string]interface{}{
@@ -160,32 +167,36 @@ func (m *ImageResize) Mill(r io.ReadSeeker, name string) (*Result, error) {
 		}, nil
 	}
 
-	if format == JPEG || format == PNG || format == ICO {
-		if format == JPEG && img == nil {
+	if slices.Contains([]Format{JPEG, PNG, ICO, WEBP}, format) {
+		switch {
+		case format == JPEG && img == nil:
 			// we already have img decoded if we have orientation <= 1
 			img, err = jpeg.Decode(r)
-			if err != nil {
-				return nil, err
-			}
-		} else if format != JPEG {
+		case format == WEBP:
+			img, err = webp.Decode(r)
+		case format != JPEG:
 			img, err = png.Decode(r)
-			if err != nil {
-				return nil, err
-			}
+		}
+
+		if err != nil {
+			return nil, err
 		}
 
 		resized := imaging.Resize(img, width, 0, imaging.Lanczos)
 		width, height = resized.Rect.Max.X, resized.Rect.Max.Y
 
 		buff := &bytes.Buffer{}
-		if format == JPEG {
-			if err = jpeg.Encode(buff, resized, &jpeg.Options{Quality: quality}); err != nil {
-				return nil, err
-			}
-		} else {
-			if err = png.Encode(buff, resized); err != nil {
-				return nil, err
-			}
+		switch format {
+		case JPEG:
+			err = jpeg.Encode(buff, resized, &jpeg.Options{Quality: quality})
+		case WEBP:
+			err = webp.Encode(buff, resized, &webp.Options{Quality: float32(quality)})
+		default:
+			err = png.Encode(buff, resized)
+		}
+
+		if err != nil {
+			return nil, err
 		}
 
 		return &Result{
@@ -195,7 +206,9 @@ func (m *ImageResize) Mill(r io.ReadSeeker, name string) (*Result, error) {
 				"height": height,
 			},
 		}, nil
-	} else if format == GIF {
+	}
+
+	if format == GIF {
 		gifImg, err := gif.DecodeAll(r)
 		if err != nil {
 			return nil, err

pkg/lib/mill/image_resize_test.go

@@ -3,9 +3,6 @@ package mill
 import (
 	"bytes"
 	"fmt"
-	"github.com/davecgh/go-spew/spew"
-	exif2 "github.com/dsoprea/go-exif/v3"
-	"github.com/stretchr/testify/require"
 	"image"
 	"image/jpeg"
 	"io"
@@ -13,7 +10,10 @@ import (
 	"os"
 	"testing"
 
+	"github.com/davecgh/go-spew/spew"
+	exif2 "github.com/dsoprea/go-exif/v3"
 	"github.com/rwcarlsen/goexif/exif"
+	"github.com/stretchr/testify/require"
 
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/mill/testdata"
 )

pkg/lib/mill/testdata/images.go

@@ -44,4 +44,11 @@ var Images = []TestImage{
 		Width:   300,
 		Height:  187,
 	},
+	{
+		Path:    "testdata/image.webp",
+		Format:  "webp",
+		HasExif: true,
+		Width:   320,
+		Height:  214,
+	},
 }

pkg/lib/pb/model/protos/models.proto

@@ -406,8 +406,8 @@ message Block {
                     LessOrEqual = 6;
                     Like = 7;
                     NotLike = 8;
-                    In = 9;
-                    NotIn = 10;
+                    In = 9; // "at least one value(from the provided list) is IN"
+                    NotIn = 10; // "none of provided values are IN"
                     Empty = 11;
                     NotEmpty = 12;
                     AllIn = 13;

util/linkpreview/linkpreview.go

@@ -5,7 +5,6 @@ import (
 	"github.com/anytypeio/go-anytype-middleware/util/text"
 	"io"
 	"net/http"
-	"net/url"
 	"path/filepath"
 	"strings"
 	"unicode/utf8"
@@ -87,7 +86,7 @@ func (l *linkPreview) convertOGToInfo(fetchUrl string, og *opengraph.OpenGraph)
 	}
 
 	if len(og.Image) != 0 {
-		url, err := uri.ProcessURI(og.Image[0].URL)
+		url, err := uri.NormalizeURI(og.Image[0].URL)
 		if err == nil {
 			i.ImageUrl = url
 		}
@@ -127,11 +126,11 @@ func (l *linkPreview) makeNonHtml(fetchUrl string, resp *http.Response) (i model
 	} else {
 		i.Type = model.LinkPreview_Unknown
 	}
-	pUrl, e := url.Parse(fetchUrl)
+	pURL, e := uri.ParseURI(fetchUrl)
 	if e == nil {
-		pUrl.Path = "favicon.ico"
-		pUrl.RawQuery = ""
-		i.FaviconUrl = pUrl.String()
+		pURL.Path = "favicon.ico"
+		pURL.RawQuery = ""
+		i.FaviconUrl = pURL.String()
 	}
 	return
 }

util/unsplash/unsplash.go

@@ -8,6 +8,7 @@ import (
 	"github.com/anytypeio/go-anytype-middleware/pkg/lib/logging"
 	"github.com/anytypeio/go-anytype-middleware/util/ocache"
 	"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
+	"github.com/anytypeio/go-anytype-middleware/util/uri"
 	"github.com/dsoprea/go-exif/v3"
 	jpegstructure "github.com/dsoprea/go-jpeg-image-structure/v2"
 	"github.com/hbagdi/go-unsplash/unsplash"
@@ -15,7 +16,6 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -111,8 +111,8 @@ func newFromPhoto(v unsplash.Photo) (Result, error) {
 		fUrl := v.Urls.Regular.String()
 		// hack to have full hd instead of 1080w,
 		// in case unsplash will change the URL format it will not break things
-		u, _ := url.Parse(fUrl)
-		if u != nil {
+		u, err := uri.ParseURI(fUrl)
+		if err == nil {
 			if q := u.Query(); q.Get("w") != "" {
 				q.Set("w", "1920")
 				u.RawQuery = q.Encode()

util/uri/uri.go

@@ -2,6 +2,7 @@ package uri
 
 import (
 	"fmt"
+	"net/url"
 	"os"
 	"regexp"
 	"strings"
@@ -15,31 +16,62 @@ var (
 	noPrefixHttpRegex      = regexp.MustCompile(`^[\pL\d.-]+(?:\.[\pL\\d.-]+)+[\pL\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.\/\d]+$`)
 	haveUriSchemeRegex     = regexp.MustCompile(`^([a-zA-Z][A-Za-z0-9+.-]*):[\S]+`)
 	winFilepathPrefixRegex = regexp.MustCompile(`^[a-zA-Z]:[\\\/]`)
+
+	// errors
+	errURLEmpty             = fmt.Errorf("url is empty")
+	errFilepathNotSupported = fmt.Errorf("filepath not supported")
 )
 
-// ProcessURI tries to verify the web URI and return the normalized URI
-func ProcessURI(url string) (urlOut string, err error) {
-	if len(url) == 0 {
-		return url, fmt.Errorf("url is empty")
-
-	} else if noPrefixEmailRegexp.MatchString(url) {
-		return "mailto:" + url, nil
-
-	} else if noPrefixTelRegexp.MatchString(url) {
-		return "tel:" + url, nil
-
-	} else if winFilepathPrefixRegex.MatchString(url) {
-		return "", fmt.Errorf("filepath not supported")
-
-	} else if strings.HasPrefix(url, string(os.PathSeparator)) || strings.HasPrefix(url, ".") {
-		return "", fmt.Errorf("filepath not supported")
-
-	} else if noPrefixHttpRegex.MatchString(url) {
-		return "http://" + url, nil
-
-	} else if haveUriSchemeRegex.MatchString(url) {
-		return url, nil
+func excludePathAndEmptyURIs(uri string) error {
+	switch {
+	case len(uri) == 0:
+		return errURLEmpty
+	case winFilepathPrefixRegex.MatchString(uri):
+		return errFilepathNotSupported
+	case strings.HasPrefix(uri, string(os.PathSeparator)):
+		return errFilepathNotSupported
+	case strings.HasPrefix(uri, "."):
+		return errFilepathNotSupported
 	}
 
-	return url, fmt.Errorf("not a uri")
+	return nil
+}
+
+func normalizeURI(uri string) string {
+	switch {
+	case noPrefixEmailRegexp.MatchString(uri):
+		return "mailto:" + uri
+	case noPrefixTelRegexp.MatchString(uri):
+		return "tel:" + uri
+	case noPrefixHttpRegex.MatchString(uri):
+		return "http://" + uri
+	}
+
+	return uri
+}
+
+func ParseURI(uri string) (*url.URL, error) {
+	uri = strings.TrimSpace(uri)
+	if err := excludePathAndEmptyURIs(uri); err != nil {
+		return nil, err
+	}
+
+	return url.Parse(uri)
+}
+
+func NormalizeURI(uri string) (string, error) {
+	if err := ValidateURI(uri); err != nil {
+		return "", err
+	}
+
+	return normalizeURI(uri), nil
+}
+
+func NormalizeAndParseURI(uri string) (*url.URL, error) {
+	uri = strings.TrimSpace(uri)
+	if err := excludePathAndEmptyURIs(uri); err != nil {
+		return nil, err
+	}
+
+	return url.Parse(normalizeURI(uri))
 }

util/uri/uri_test.go

@@ -6,95 +6,115 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestURI_ProcessURI(t *testing.T) {
+func TestURI_NormalizeURI(t *testing.T) {
 	t.Run("should process mailto uri", func(t *testing.T) {
 		uri := "john@doe.com"
-		processedUri, err := ProcessURI(uri)
-		assert.Equal(t, "mailto:"+uri, processedUri)
+		processedURI, err := NormalizeURI(uri)
 		assert.NoError(t, err)
+		assert.Equal(t, "mailto:"+uri, processedURI)
 	})
 
 	t.Run("should process tel uri", func(t *testing.T) {
 		uri := "+491234567"
-		processedUri, err := ProcessURI(uri)
-		assert.Equal(t, "tel:"+uri, processedUri)
+		processedURI, err := NormalizeURI(uri)
 		assert.NoError(t, err)
+		assert.Equal(t, "tel:"+uri, processedURI)
 	})
 
 	t.Run("should process url", func(t *testing.T) {
 		uri := "website.com"
-		processedUri, err := ProcessURI(uri)
-		assert.Equal(t, "http://"+uri, processedUri)
+		processedURI, err := NormalizeURI(uri)
 		assert.NoError(t, err)
+		assert.Equal(t, "http://"+uri, processedURI)
 	})
 
 	t.Run("should process url with additional content 1", func(t *testing.T) {
 		uri := "website.com/123/456"
-		processedUri, err := ProcessURI(uri)
-		assert.Equal(t, "http://"+uri, processedUri)
+		processedURI, err := NormalizeURI(uri)
 		assert.NoError(t, err)
+		assert.Equal(t, "http://"+uri, processedURI)
 	})
 
 	t.Run("should process url with additional content 2", func(t *testing.T) {
 		uri := "website.com?content=11"
-		processedUri, err := ProcessURI(uri)
-		assert.Equal(t, "http://"+uri, processedUri)
+		processedURI, err := NormalizeURI(uri)
 		assert.NoError(t, err)
+		assert.Equal(t, "http://"+uri, processedURI)
 	})
 
 	t.Run("should process url with additional content and numbers", func(t *testing.T) {
 		uri := "webs1te.com/123/456"
-		processedUri, err := ProcessURI(uri)
-		assert.Equal(t, "http://"+uri, processedUri)
+		processedURI, err := NormalizeURI(uri)
 		assert.NoError(t, err)
+		assert.Equal(t, "http://"+uri, processedURI)
 	})
 
-	t.Run("should return error if it is not a uri", func(t *testing.T) {
-		uri := "website"
-		processedUri, err := ProcessURI(uri)
-		assert.Equal(t, uri, processedUri)
-		assert.Error(t, err)
-	})
-
-	t.Run("should not process url with http://", func(t *testing.T) {
+	t.Run("should not modify url with http://", func(t *testing.T) {
 		uri := "http://website.com"
-		processedUri, err := ProcessURI(uri)
-		assert.Equal(t, uri, processedUri)
+		processedURI, err := NormalizeURI(uri)
 		assert.NoError(t, err)
+		assert.Equal(t, uri, processedURI)
 	})
 
-	t.Run("should not process url with https://", func(t *testing.T) {
+	t.Run("should not modify url with https://", func(t *testing.T) {
 		uri := "https://website.com"
-		processedUri, err := ProcessURI(uri)
-		assert.Equal(t, uri, processedUri)
+		processedURI, err := NormalizeURI(uri)
 		assert.NoError(t, err)
+		assert.Equal(t, uri, processedURI)
 	})
 
-	t.Run("should not process non url/tel/mailto uri", func(t *testing.T) {
+	t.Run("should not modify non url/tel/mailto uri", func(t *testing.T) {
 		uri := "type:content"
-		processedUri, err := ProcessURI(uri)
-		assert.Equal(t, uri, processedUri)
+		processedURI, err := NormalizeURI(uri)
+		assert.NoError(t, err)
+		assert.Equal(t, uri, processedURI)
+	})
+}
+
+func TestURI_ValidateURI(t *testing.T) {
+	t.Run("should return error on empty string", func(t *testing.T) {
+		uri := ""
+		err := ValidateURI(uri)
+		assert.Error(t, err)
+		assert.Equal(t, err, errURLEmpty)
+	})
+
+	t.Run("should return error on win filepath", func(t *testing.T) {
+		uri := "D://folder//file.txt"
+		err := ValidateURI(uri)
+		assert.Error(t, err)
+		assert.Equal(t, err, errFilepathNotSupported)
+	})
+
+	t.Run("should return error on unix abs filepath", func(t *testing.T) {
+		uri := "/folder/file.txt"
+		err := ValidateURI(uri)
+		assert.Error(t, err)
+		assert.Equal(t, err, errFilepathNotSupported)
+	})
+
+	t.Run("should return error on unix rel filepath", func(t *testing.T) {
+		uri := "../folder/file.txt"
+		err := ValidateURI(uri)
+		assert.Error(t, err)
+		assert.Equal(t, err, errFilepathNotSupported)
+	})
+
+	t.Run("should not return error if url is surrounded by whitespaces", func(t *testing.T) {
+		uri := " \t\n\v\r\f https://brutal-site.org \t\n\v\r\f "
+		err := ValidateURI(uri)
+		assert.NoError(t, err)
+	})
+
+	t.Run("should not return error if url has spaces inside", func(t *testing.T) {
+		uri := "I do love enough space.org"
+		err := ValidateURI(uri)
+		assert.NoError(t, err)
+	})
+
+	t.Run("should not return error if url contains emojis", func(t *testing.T) {
+		uri := "merry 🎄 and a happy 🎁.kevin.blog"
+		err := ValidateURI(uri)
 		assert.NoError(t, err)
-	})
-
-	t.Run("should gives error on win filepath", func(t *testing.T) {
-		uri := "D://folder//file.txt"
-		processedUri, err := ProcessURI(uri)
-		assert.Equal(t, "", processedUri)
-		assert.NotNil(t, err)
-	})
-
-	t.Run("should gives error on unix abs filepath", func(t *testing.T) {
-		uri := "/folder/file.txt"
-		processedUri, err := ProcessURI(uri)
-		assert.Equal(t, "", processedUri)
-		assert.NotNil(t, err)
-	})
-
-	t.Run("should gives error on unix rel filepath", func(t *testing.T) {
-		uri := "../folder/file.txt"
-		processedUri, err := ProcessURI(uri)
-		assert.Equal(t, "", processedUri)
-		assert.NotNil(t, err)
 	})
 }