GO-4459 Merge branch 'go-5364-export-markdown-in-memory' into GO-4459-rest-api-docs

core/block/export/export.go

@@ -3,8 +3,10 @@ package export
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"fmt"
 	"math/rand"
+	"net/url"
 	"os"
 	"path/filepath"
 	"slices"
@@ -39,6 +41,7 @@ import (
 	"github.com/anyproto/anytype-heart/pkg/lib/bundle"
 	"github.com/anyproto/anytype-heart/pkg/lib/core/smartblock"
 	"github.com/anyproto/anytype-heart/pkg/lib/database"
+	"github.com/anyproto/anytype-heart/pkg/lib/gateway"
 	"github.com/anyproto/anytype-heart/pkg/lib/localstore/addr"
 	"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore"
 	"github.com/anyproto/anytype-heart/pkg/lib/logging"
@@ -69,6 +72,7 @@ var log = logging.Logger("anytype-mw-export")
 
 type Export interface {
 	Export(ctx context.Context, req pb.RpcObjectListExportRequest) (path string, succeed int, err error)
+	ExportSingleInMemory(ctx context.Context, spaceId string, objectId string, format model.ExportFormat) (res string, err error)
 	app.Component
 }
 
@@ -81,6 +85,7 @@ type export struct {
 	accountService      account.Service
 	notificationService notifications.Notifications
 	processService      process.Service
+	gatewayService      gateway.Gateway
 }
 
 func New() Export {
@@ -96,6 +101,7 @@ func (e *export) Init(a *app.App) (err error) {
 	e.spaceService = app.MustComponent[space.Service](a)
 	e.accountService = app.MustComponent[account.Service](a)
 	e.notificationService = app.MustComponent[notifications.Notifications](a)
+	e.gatewayService = app.MustComponent[gateway.Gateway](a)
 	return
 }
 
@@ -118,6 +124,20 @@ func (e *export) Export(ctx context.Context, req pb.RpcObjectListExportRequest)
 	return exportCtx.exportObjects(ctx, queue)
 }
 
+func (e *export) ExportSingleInMemory(ctx context.Context, spaceId string, objectId string, format model.ExportFormat) (res string, err error) {
+	req := pb.RpcObjectListExportRequest{
+		SpaceId:         spaceId,
+		ObjectIds:       []string{objectId},
+		IncludeFiles:    true,
+		Format:          format,
+		IncludeNested:   true,
+		IncludeArchived: true,
+	}
+
+	exportCtx := newExportContext(e, req)
+	return exportCtx.exportObject(ctx, objectId)
+}
+
 func (e *export) finishWithNotification(spaceId string, exportFormat model.ExportFormat, queue process.Queue, err error) {
 	errCode := model.NotificationExport_NULL
 	if err != nil {
@@ -168,7 +188,7 @@ type exportContext struct {
 	relations        map[string]struct{}
 	setOfList        map[string]struct{}
 	objectTypes      map[string]struct{}
-
+	gatewayUrl       string
 	*export
 }
 
@@ -190,8 +210,8 @@ func newExportContext(e *export, req pb.RpcObjectListExportRequest) *exportConte
 		setOfList:        make(map[string]struct{}),
 		objectTypes:      make(map[string]struct{}),
 		relations:        make(map[string]struct{}),
-
-		export: e,
+		gatewayUrl:       "http://" + e.gatewayService.Addr(),
+		export:           e,
 	}
 	return ec
 }
@@ -224,6 +244,41 @@ func (e *exportContext) getStateFilters(id string) *state.Filters {
 	return nil
 }
 
+// exportObject synchronously exports a single object and return the bytes slice
+func (e *exportContext) exportObject(ctx context.Context, objectId string) (string, error) {
+	e.reqIds = []string{objectId}
+	e.includeArchive = true
+	err := e.docsForExport(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	var docNamer Namer
+	if e.format == model.Export_Markdown && e.gatewayUrl != "" {
+		u, err := url.Parse(e.gatewayUrl)
+		if err != nil {
+			return "", err
+		}
+		docNamer = &deepLinkNamer{gatewayUrl: *u}
+	} else {
+		docNamer = newNamer()
+	}
+	inMemoryWriter := &InMemoryWriter{fn: docNamer}
+	err = e.writeDoc(ctx, inMemoryWriter, objectId, e.docs.transformToDetailsMap())
+	if err != nil {
+		return "", err
+	}
+
+	for _, v := range inMemoryWriter.data {
+		if e.format == model.Export_Protobuf {
+			return base64.StdEncoding.EncodeToString(v), nil
+		}
+		return string(v), nil
+	}
+
+	return "", fmt.Errorf("failed to find data in writer")
+}
+
 func (e *exportContext) exportObjects(ctx context.Context, queue process.Queue) (string, int, error) {
 	var (
 		err  error

core/block/export/writer.go

@@ -4,22 +4,28 @@ import (
 	"archive/zip"
 	"fmt"
 	"io"
+	"net/url"
 	"os"
 	"path"
 	"path/filepath"
 	"sync"
 	"time"
 
+	"github.com/anyproto/anytype-heart/pkg/lib/mill"
 	"github.com/anyproto/anytype-heart/util/anyerror"
 )
 
 type writer interface {
 	Path() string
-	Namer() *namer
+	Namer() Namer
 	WriteFile(filename string, r io.Reader, lastModifiedDate int64) (err error)
 	Close() (err error)
 }
 
+type Namer interface {
+	Get(path, hash, title, ext string) (name string)
+}
+
 func uniqName() string {
 	return time.Now().Format("Anytype.20060102.150405.99")
 }
@@ -44,7 +50,7 @@ type dirWriter struct {
 	m    sync.Mutex
 }
 
-func (d *dirWriter) Namer() *namer {
+func (d *dirWriter) Namer() Namer {
 	d.m.Lock()
 	defer d.m.Unlock()
 	if d.fn == nil {
@@ -108,7 +114,7 @@ type zipWriter struct {
 	fn   *namer
 }
 
-func (d *zipWriter) Namer() *namer {
+func (d *zipWriter) Namer() Namer {
 	d.m.Lock()
 	defer d.m.Unlock()
 	if d.fn == nil {
@@ -149,3 +155,66 @@ func (d *zipWriter) Close() (err error) {
 func getZipName(path string) string {
 	return filepath.Join(path, uniqName()+".zip")
 }
+
+type InMemoryWriter struct {
+	data map[string][]byte
+	fn   Namer
+	m    sync.Mutex
+}
+
+func (d *InMemoryWriter) Namer() Namer {
+	return d.fn
+}
+
+func (d *InMemoryWriter) Path() string {
+	return ""
+}
+
+func (d *InMemoryWriter) WriteFile(filename string, r io.Reader, lastModifiedDate int64) (err error) {
+	d.m.Lock()
+	defer d.m.Unlock()
+	if d.data == nil {
+		d.data = make(map[string][]byte)
+	}
+	b, err := io.ReadAll(r)
+	if err != nil {
+		return
+	}
+	d.data[filename] = b
+	return
+}
+
+func (d *InMemoryWriter) Close() (err error) {
+	return nil
+}
+
+func (d *InMemoryWriter) GetData(id string) []byte {
+	d.m.Lock()
+	defer d.m.Unlock()
+	return d.data[id]
+}
+
+// deepLinkNamer used to render a single-object export, in md format
+type deepLinkNamer struct {
+	gatewayUrl url.URL
+}
+
+func (fn *deepLinkNamer) Get(path, hash, title, ext string) (name string) {
+	if ext == ".md" {
+		// object links via deeplink to the app
+		return "anytype://object?objectId=" + hash
+	}
+
+	// files links via gateway
+	if fn.gatewayUrl.Host == "" {
+		return "anytype://object?objectId=" + hash
+	}
+	u := fn.gatewayUrl
+	if mill.IsImageExt(ext) {
+		u.Path = "image/" + hash
+	} else {
+		u.Path = "file/" + hash
+	}
+
+	return u.String()
+}

core/block/import/markdown/anymark/anyblocks.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/gogo/protobuf/types"
 
+	"github.com/anyproto/anytype-heart/pkg/lib/mill"
 	"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
 	"github.com/anyproto/anytype-heart/util/pbtypes"
 )
@@ -98,7 +99,6 @@ func ConvertTextToFile(filePath string) *model.BlockContentOfFile {
 		return nil
 	}
 
-	imageFormats := []string{"jpg", "jpeg", "png", "gif", "webp"}
 	videoFormats := []string{"mp4", "m4v", "mov"}
 	audioFormats := []string{"mp3", "ogg", "wav", "m4a", "flac"}
 	pdfFormat := "pdf"
@@ -107,11 +107,8 @@ func ConvertTextToFile(filePath string) *model.BlockContentOfFile {
 	fileExt := filepath.Ext(filePath)
 	if fileExt != "" {
 		fileExt = fileExt[1:]
-		for _, ext := range imageFormats {
-			if strings.EqualFold(fileExt, ext) {
-				fileType = model.BlockContentFile_Image
-				break
-			}
+		if mill.IsImageExt(fileExt) {
+			fileType = model.BlockContentFile_Image
 		}
 
 		for _, ext := range videoFormats {

core/export.go

@@ -37,3 +37,31 @@ func (mw *Middleware) ObjectListExport(cctx context.Context, req *pb.RpcObjectLi
 	})
 	return response(path, succeed, err)
 }
+
+func (mw *Middleware) ObjectExport(cctx context.Context, req *pb.RpcObjectExportRequest) *pb.RpcObjectExportResponse {
+	response := func(result string, err error) (res *pb.RpcObjectExportResponse) {
+		res = &pb.RpcObjectExportResponse{
+			Error: &pb.RpcObjectExportResponseError{
+				Code: pb.RpcObjectExportResponseError_NULL,
+			},
+		}
+		if err != nil {
+			res.Error.Code = pb.RpcObjectExportResponseError_UNKNOWN_ERROR
+			res.Error.Description = getErrorDescription(err)
+			return
+		} else {
+			res.Result = result
+		}
+		return res
+	}
+	var (
+		result string
+		err    error
+	)
+	err = mw.doBlockService(func(_ *block.Service) error {
+		es := mw.applicationService.GetApp().MustComponent(export.CName).(export.Export)
+		result, err = es.ExportSingleInMemory(cctx, req.SpaceId, req.ObjectId, req.Format)
+		return err
+	})
+	return response(result, err)
+}

docs/proto.md

@@ -909,6 +909,10 @@
     - [Rpc.Object.Duplicate.Request](#anytype-Rpc-Object-Duplicate-Request)
     - [Rpc.Object.Duplicate.Response](#anytype-Rpc-Object-Duplicate-Response)
     - [Rpc.Object.Duplicate.Response.Error](#anytype-Rpc-Object-Duplicate-Response-Error)
+    - [Rpc.Object.Export](#anytype-Rpc-Object-Export)
+    - [Rpc.Object.Export.Request](#anytype-Rpc-Object-Export-Request)
+    - [Rpc.Object.Export.Response](#anytype-Rpc-Object-Export-Response)
+    - [Rpc.Object.Export.Response.Error](#anytype-Rpc-Object-Export-Response-Error)
     - [Rpc.Object.Graph](#anytype-Rpc-Object-Graph)
     - [Rpc.Object.Graph.Edge](#anytype-Rpc-Object-Graph-Edge)
     - [Rpc.Object.Graph.Request](#anytype-Rpc-Object-Graph-Request)
@@ -1552,6 +1556,7 @@
     - [Rpc.Object.CrossSpaceSearchUnsubscribe.Response.Error.Code](#anytype-Rpc-Object-CrossSpaceSearchUnsubscribe-Response-Error-Code)
     - [Rpc.Object.DateByTimestamp.Response.Error.Code](#anytype-Rpc-Object-DateByTimestamp-Response-Error-Code)
     - [Rpc.Object.Duplicate.Response.Error.Code](#anytype-Rpc-Object-Duplicate-Response-Error-Code)
+    - [Rpc.Object.Export.Response.Error.Code](#anytype-Rpc-Object-Export-Response-Error-Code)
     - [Rpc.Object.Graph.Edge.Type](#anytype-Rpc-Object-Graph-Edge-Type)
     - [Rpc.Object.Graph.Response.Error.Code](#anytype-Rpc-Object-Graph-Response-Error-Code)
     - [Rpc.Object.GroupsSubscribe.Response.Error.Code](#anytype-Rpc-Object-GroupsSubscribe-Response-Error-Code)
@@ -2181,6 +2186,7 @@
 | ObjectUndo | [Rpc.Object.Undo.Request](#anytype-Rpc-Object-Undo-Request) | [Rpc.Object.Undo.Response](#anytype-Rpc-Object-Undo-Response) |  |
 | ObjectRedo | [Rpc.Object.Redo.Request](#anytype-Rpc-Object-Redo-Request) | [Rpc.Object.Redo.Response](#anytype-Rpc-Object-Redo-Response) |  |
 | ObjectListExport | [Rpc.Object.ListExport.Request](#anytype-Rpc-Object-ListExport-Request) | [Rpc.Object.ListExport.Response](#anytype-Rpc-Object-ListExport-Response) |  |
+| ObjectExport | [Rpc.Object.Export.Request](#anytype-Rpc-Object-Export-Request) | [Rpc.Object.Export.Response](#anytype-Rpc-Object-Export-Response) |  |
 | ObjectBookmarkFetch | [Rpc.Object.BookmarkFetch.Request](#anytype-Rpc-Object-BookmarkFetch-Request) | [Rpc.Object.BookmarkFetch.Response](#anytype-Rpc-Object-BookmarkFetch-Response) |  |
 | ObjectToBookmark | [Rpc.Object.ToBookmark.Request](#anytype-Rpc-Object-ToBookmark-Request) | [Rpc.Object.ToBookmark.Response](#anytype-Rpc-Object-ToBookmark-Response) |  |
 | ObjectImport | [Rpc.Object.Import.Request](#anytype-Rpc-Object-Import-Request) | [Rpc.Object.Import.Response](#anytype-Rpc-Object-Import-Response) |  |
@@ -15532,6 +15538,66 @@ Get the info for page alongside with info for all inbound and outbound links fro
 
 
 
+<a name="anytype-Rpc-Object-Export"></a>
+
+### Rpc.Object.Export
+
+
+
+
+
+
+
+<a name="anytype-Rpc-Object-Export-Request"></a>
+
+### Rpc.Object.Export.Request
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| spaceId | [string](#string) |  |  |
+| objectId | [string](#string) |  | ids of documents for export, when empty - will export all available docs |
+| format | [model.Export.Format](#anytype-model-Export-Format) |  | export format |
+
+
+
+
+
+
+<a name="anytype-Rpc-Object-Export-Response"></a>
+
+### Rpc.Object.Export.Response
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| error | [Rpc.Object.Export.Response.Error](#anytype-Rpc-Object-Export-Response-Error) |  |  |
+| result | [string](#string) |  |  |
+| event | [ResponseEvent](#anytype-ResponseEvent) |  |  |
+
+
+
+
+
+
+<a name="anytype-Rpc-Object-Export-Response-Error"></a>
+
+### Rpc.Object.Export.Response.Error
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| code | [Rpc.Object.Export.Response.Error.Code](#anytype-Rpc-Object-Export-Response-Error-Code) |  |  |
+| description | [string](#string) |  |  |
+
+
+
+
+
+
 <a name="anytype-Rpc-Object-Graph"></a>
 
 ### Rpc.Object.Graph
@@ -24731,6 +24797,19 @@ Middleware-to-front-end response, that can contain a NULL error or a non-NULL er
 
 
 
+<a name="anytype-Rpc-Object-Export-Response-Error-Code"></a>
+
+### Rpc.Object.Export.Response.Error.Code
+
+
+| Name | Number | Description |
+| ---- | ------ | ----------- |
+| NULL | 0 |  |
+| UNKNOWN_ERROR | 1 |  |
+| BAD_INPUT | 2 | ... |
+
+
+
 <a name="anytype-Rpc-Object-Graph-Edge-Type"></a>
 
 ### Rpc.Object.Graph.Edge.Type

pb/protos/commands.proto

@@ -2908,6 +2908,35 @@ message Rpc {
 
         }
 
+        message Export {
+            message Request {
+                string spaceId = 10;
+                // ids of documents for export, when empty - will export all available docs
+                string objectId = 2;
+                // export format
+                anytype.model.Export.Format format = 3;
+            }
+
+            message Response {
+                Error error = 1;
+                string result = 2;
+                ResponseEvent event = 3;
+
+                message Error {
+                    Code code = 1;
+                    string description = 2;
+
+                    enum Code {
+                        NULL = 0;
+                        UNKNOWN_ERROR = 1;
+                        BAD_INPUT = 2;
+                        // ...
+                    }
+                }
+            }
+
+        }
+
         message Import {
             message Request {
                 option (no_auth) = true;

pb/protos/service/service.proto

@@ -127,6 +127,7 @@ service ClientCommands {
     rpc ObjectUndo (anytype.Rpc.Object.Undo.Request) returns (anytype.Rpc.Object.Undo.Response);
     rpc ObjectRedo (anytype.Rpc.Object.Redo.Request) returns (anytype.Rpc.Object.Redo.Response);
     rpc ObjectListExport (anytype.Rpc.Object.ListExport.Request) returns (anytype.Rpc.Object.ListExport.Response);
+    rpc ObjectExport (anytype.Rpc.Object.Export.Request) returns (anytype.Rpc.Object.Export.Response);
     rpc ObjectBookmarkFetch (anytype.Rpc.Object.BookmarkFetch.Request) returns (anytype.Rpc.Object.BookmarkFetch.Response);
     rpc ObjectToBookmark (anytype.Rpc.Object.ToBookmark.Request) returns (anytype.Rpc.Object.ToBookmark.Response);
     rpc ObjectImport (anytype.Rpc.Object.Import.Request) returns (anytype.Rpc.Object.Import.Response);

pkg/lib/mill/image_resize.go

@@ -42,6 +42,14 @@ const (
 	TIFF     Format = "tiff"
 )
 
+func IsImageExt(ext string) bool {
+	switch strings.ToLower(strings.TrimPrefix(ext, ".")) {
+	case "jpg", "jpeg", "png", "gif", "ico", "webp", "heic", "heif", "bmp", "tiff", "psd":
+		return true
+	}
+	return false
+}
+
 func IsImage(mime string) bool {
 	parts := strings.SplitN(mime, "/", 2)
 	if len(parts) == 1 {