From 9cf7e4e18c5cdcb9bbbcc7de8157621cf9a1aab9 Mon Sep 17 00:00:00 2001 From: AnastasiaShemyakinskaya Date: Tue, 6 Aug 2024 14:47:36 +0200 Subject: [PATCH 1/4] GO-2746: add export structure Signed-off-by: AnastasiaShemyakinskaya --- core/block/export/export.go | 34 +++++++++++++--- core/block/export/export_test.go | 67 ++++++++++++++++++++++++++++++++ core/block/export/writer.go | 7 ++++ tests/integration/export_test.go | 14 ++++++- 4 files changed, 115 insertions(+), 7 deletions(-) diff --git a/core/block/export/export.go b/core/block/export/export.go index 8f4ddb83e..a7c47b5a4 100644 --- a/core/block/export/export.go +++ b/core/block/export/export.go @@ -55,8 +55,14 @@ import ( const CName = "export" const ( - tempFileName = "temp_anytype_backup" - spaceDirectory = "spaces" + tempFileName = "temp_anytype_backup" + spaceDirectory = "spaces" + typesDirectory = "types" + objectsDirectory = "objects" + relationsDirectory = "relations" + relationsOptionsDirectory = "relationsOptions" + templatesDirectory = "templates" + filesObjects = "filesObjects" ) var log = logging.Logger("anytype-mw-export") @@ -491,7 +497,7 @@ func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportReques } conv.SetKnownDocs(docInfo) result := conv.Convert(b.Type().ToProto()) - filename := e.provideFileName(docID, req.SpaceId, conv, st) + filename := e.provideFileName(docID, req.SpaceId, conv, st, b.Type()) if req.Format == model.Export_Markdown { filename = e.provideMarkdownName(st, wr, docID, conv, req.SpaceId) } @@ -519,8 +525,9 @@ func (e *export) provideMarkdownName(s *state.State, wr writer, docID string, co return wr.Namer().Get(path, docID, name, conv.Ext()) } -func (e *export) provideFileName(docID, spaceId string, conv converter.Converter, st *state.State) string { - filename := docID + conv.Ext() +func (e *export) provideFileName(docID, spaceId string, conv converter.Converter, st *state.State, blockType smartblock.SmartBlockType) string { + dir := e.provideFileDirectory(blockType) + filename := filepath.Join(dir, docID+conv.Ext()) if spaceId == "" { spaceId := pbtypes.GetString(st.LocalDetails(), bundle.RelationKeySpaceId.String()) filename = filepath.Join(spaceDirectory, spaceId, filename) @@ -528,6 +535,23 @@ func (e *export) provideFileName(docID, spaceId string, conv converter.Converter return filename } +func (e *export) provideFileDirectory(blockType smartblock.SmartBlockType) string { + switch blockType { + case smartblock.SmartBlockTypeRelation: + return relationsDirectory + case smartblock.SmartBlockTypeRelationOption: + return relationsOptionsDirectory + case smartblock.SmartBlockTypeObjectType: + return typesDirectory + case smartblock.SmartBlockTypeTemplate: + return templatesDirectory + case smartblock.SmartBlockTypeFile, smartblock.SmartBlockTypeFileObject: + return filesObjects + default: + return objectsDirectory + } +} + func (e *export) saveFile(ctx context.Context, wr writer, fileObject sb.SmartBlock, exportAllSpaces bool) (fileName string, err error) { fullId := domain.FullFileId{ SpaceId: fileObject.Space().Id(), diff --git a/core/block/export/export_test.go b/core/block/export/export_test.go index c9b427032..bd616af07 100644 --- a/core/block/export/export_test.go +++ b/core/block/export/export_test.go @@ -11,6 +11,8 @@ import ( "github.com/anyproto/anytype-heart/core/block/cache/mock_cache" "github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest" + "github.com/anyproto/anytype-heart/core/block/editor/state" + "github.com/anyproto/anytype-heart/core/converter/pbjson" "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/pb" "github.com/anyproto/anytype-heart/pkg/lib/bundle" @@ -495,3 +497,68 @@ func Test_docsForExport(t *testing.T) { assert.Equal(t, 2, len(docsForExport)) }) } + +func Test_provideFileName(t *testing.T) { + t.Run("file dir for relation", func(t *testing.T) { + // given + e := &export{} + + // when + fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeRelation) + + // then + assert.Equal(t, relationsDirectory+string(filepath.Separator)+"docId.pb.json", fileName) + }) + t.Run("file dir for relation option", func(t *testing.T) { + // given + e := &export{} + + // when + fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeRelationOption) + + // then + assert.Equal(t, relationsOptionsDirectory+string(filepath.Separator)+"docId.pb.json", fileName) + }) + t.Run("file dir for types", func(t *testing.T) { + // given + e := &export{} + + // when + fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeObjectType) + + // then + assert.Equal(t, typesDirectory+string(filepath.Separator)+"docId.pb.json", fileName) + }) + t.Run("file dir for objects", func(t *testing.T) { + // given + e := &export{} + + // when + fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypePage) + + // then + assert.Equal(t, objectsDirectory+string(filepath.Separator)+"docId.pb.json", fileName) + }) + t.Run("file dir for files objects", func(t *testing.T) { + // given + e := &export{} + + // when + fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeFileObject) + + // then + assert.Equal(t, filesObjects+string(filepath.Separator)+"docId.pb.json", fileName) + }) + t.Run("space is not provided", func(t *testing.T) { + // given + e := &export{} + st := state.NewDoc("root", nil).(*state.State) + st.SetDetail(bundle.RelationKeySpaceId.String(), pbtypes.String("spaceId")) + + // when + fileName := e.provideFileName("docId", "", pbjson.NewConverter(st), st, smartblock.SmartBlockTypeFileObject) + + // then + assert.Equal(t, spaceDirectory+string(filepath.Separator)+"spaceId"+string(filepath.Separator)+filesObjects+string(filepath.Separator)+"docId.pb.json", fileName) + }) +} diff --git a/core/block/export/writer.go b/core/block/export/writer.go index bbfdd95a7..8493cd419 100644 --- a/core/block/export/writer.go +++ b/core/block/export/writer.go @@ -58,6 +58,13 @@ func (d *dirWriter) Path() string { } func (d *dirWriter) WriteFile(filename string, r io.Reader, lastModifiedDate int64) (err error) { + dir := filepath.Dir(filename) + if dir != "" { + err = os.MkdirAll(filepath.Join(d.path, dir), 0700) + if err != nil { + return err + } + } filename = path.Join(d.path, filename) f, err := os.Create(filename) if err != nil { diff --git a/tests/integration/export_test.go b/tests/integration/export_test.go index 8971a5cce..f2ede4fd8 100644 --- a/tests/integration/export_test.go +++ b/tests/integration/export_test.go @@ -84,8 +84,18 @@ func TestExportFiles(t *testing.T) { var foundPbFiles int for _, entry := range entries { - if filepath.Ext(entry.Name()) == ".pb" { - foundPbFiles++ + if entry.IsDir() { + files, err := os.ReadDir(filepath.Join(exportPath, entry.Name())) + require.NoError(t, err) + for _, file := range files { + if filepath.Ext(file.Name()) == ".pb" { + foundPbFiles++ + } + } + } else { + if filepath.Ext(entry.Name()) == ".pb" { + foundPbFiles++ + } } } // 4 objects total: Page object + Page type + File object From 669c280dab25384b2bf9d2a4c730a9ef835c0c9c Mon Sep 17 00:00:00 2001 From: AnastasiaShemyakinskaya Date: Wed, 7 Aug 2024 12:50:19 +0200 Subject: [PATCH 2/4] GO-2746: fix comments Signed-off-by: AnastasiaShemyakinskaya --- core/block/export/export.go | 14 +++++++------- core/block/export/writer.go | 8 +++----- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/core/block/export/export.go b/core/block/export/export.go index a7c47b5a4..ce63f2e1e 100644 --- a/core/block/export/export.go +++ b/core/block/export/export.go @@ -467,8 +467,8 @@ func (e *export) writeMultiDoc(ctx context.Context, return } -func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportRequest, wr writer, docInfo map[string]*types.Struct, queue process.Queue, docID string) (err error) { - return cache.Do(e.picker, docID, func(b sb.SmartBlock) error { +func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportRequest, wr writer, docInfo map[string]*types.Struct, queue process.Queue, docId string) (err error) { + return cache.Do(e.picker, docId, func(b sb.SmartBlock) error { st := b.NewState() if pbtypes.GetBool(st.CombinedDetails(), bundle.RelationKeyIsDeleted.String()) { return nil @@ -497,11 +497,11 @@ func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportReques } conv.SetKnownDocs(docInfo) result := conv.Convert(b.Type().ToProto()) - filename := e.provideFileName(docID, req.SpaceId, conv, st, b.Type()) + filename := e.provideFileName(docId, req.SpaceId, conv, st, b.Type()) if req.Format == model.Export_Markdown { - filename = e.provideMarkdownName(st, wr, docID, conv, req.SpaceId) + filename = e.provideMarkdownName(st, wr, docId, conv, req.SpaceId) } - if docID == b.Space().DerivedIDs().Home { + if docId == b.Space().DerivedIDs().Home { filename = "index" + conv.Ext() } lastModifiedDate := pbtypes.GetInt64(st.LocalDetails(), bundle.RelationKeyLastModifiedDate.String()) @@ -525,9 +525,9 @@ func (e *export) provideMarkdownName(s *state.State, wr writer, docID string, co return wr.Namer().Get(path, docID, name, conv.Ext()) } -func (e *export) provideFileName(docID, spaceId string, conv converter.Converter, st *state.State, blockType smartblock.SmartBlockType) string { +func (e *export) provideFileName(docId, spaceId string, conv converter.Converter, st *state.State, blockType smartblock.SmartBlockType) string { dir := e.provideFileDirectory(blockType) - filename := filepath.Join(dir, docID+conv.Ext()) + filename := filepath.Join(dir, docId+conv.Ext()) if spaceId == "" { spaceId := pbtypes.GetString(st.LocalDetails(), bundle.RelationKeySpaceId.String()) filename = filepath.Join(spaceDirectory, spaceId, filename) diff --git a/core/block/export/writer.go b/core/block/export/writer.go index 8493cd419..ec49b9a10 100644 --- a/core/block/export/writer.go +++ b/core/block/export/writer.go @@ -59,11 +59,9 @@ func (d *dirWriter) Path() string { func (d *dirWriter) WriteFile(filename string, r io.Reader, lastModifiedDate int64) (err error) { dir := filepath.Dir(filename) - if dir != "" { - err = os.MkdirAll(filepath.Join(d.path, dir), 0700) - if err != nil { - return err - } + err = os.MkdirAll(filepath.Join(d.path, dir), 0700) + if err != nil { + return err } filename = path.Join(d.path, filename) f, err := os.Create(filename) From 533f2ec5d21c73bd0a4dc8d191ed6739348ff1e3 Mon Sep 17 00:00:00 2001 From: AnastasiaShemyakinskaya Date: Thu, 8 Aug 2024 11:26:22 +0200 Subject: [PATCH 3/4] GO-2746: refactoring Signed-off-by: AnastasiaShemyakinskaya --- core/block/export/export.go | 13 +++++++------ core/block/export/export_test.go | 12 ++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/core/block/export/export.go b/core/block/export/export.go index ce63f2e1e..21a9e660f 100644 --- a/core/block/export/export.go +++ b/core/block/export/export.go @@ -497,12 +497,13 @@ func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportReques } conv.SetKnownDocs(docInfo) result := conv.Convert(b.Type().ToProto()) - filename := e.provideFileName(docId, req.SpaceId, conv, st, b.Type()) + var filename string if req.Format == model.Export_Markdown { - filename = e.provideMarkdownName(st, wr, docId, conv, req.SpaceId) - } - if docId == b.Space().DerivedIDs().Home { + filename = e.makeMarkdownName(st, wr, docId, conv, req.SpaceId) + } else if docId == b.Space().DerivedIDs().Home { filename = "index" + conv.Ext() + } else { + filename = e.makeFileName(docId, req.SpaceId, conv, st, b.Type()) } lastModifiedDate := pbtypes.GetInt64(st.LocalDetails(), bundle.RelationKeyLastModifiedDate.String()) if err = wr.WriteFile(filename, bytes.NewReader(result), lastModifiedDate); err != nil { @@ -512,7 +513,7 @@ func (e *export) writeDoc(ctx context.Context, req *pb.RpcObjectListExportReques }) } -func (e *export) provideMarkdownName(s *state.State, wr writer, docID string, conv converter.Converter, spaceId string) string { +func (e *export) makeMarkdownName(s *state.State, wr writer, docID string, conv converter.Converter, spaceId string) string { name := pbtypes.GetString(s.Details(), bundle.RelationKeyName.String()) if name == "" { name = s.Snippet() @@ -525,7 +526,7 @@ func (e *export) provideMarkdownName(s *state.State, wr writer, docID string, co return wr.Namer().Get(path, docID, name, conv.Ext()) } -func (e *export) provideFileName(docId, spaceId string, conv converter.Converter, st *state.State, blockType smartblock.SmartBlockType) string { +func (e *export) makeFileName(docId, spaceId string, conv converter.Converter, st *state.State, blockType smartblock.SmartBlockType) string { dir := e.provideFileDirectory(blockType) filename := filepath.Join(dir, docId+conv.Ext()) if spaceId == "" { diff --git a/core/block/export/export_test.go b/core/block/export/export_test.go index bd616af07..c5b697e94 100644 --- a/core/block/export/export_test.go +++ b/core/block/export/export_test.go @@ -504,7 +504,7 @@ func Test_provideFileName(t *testing.T) { e := &export{} // when - fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeRelation) + fileName := e.makeFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeRelation) // then assert.Equal(t, relationsDirectory+string(filepath.Separator)+"docId.pb.json", fileName) @@ -514,7 +514,7 @@ func Test_provideFileName(t *testing.T) { e := &export{} // when - fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeRelationOption) + fileName := e.makeFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeRelationOption) // then assert.Equal(t, relationsOptionsDirectory+string(filepath.Separator)+"docId.pb.json", fileName) @@ -524,7 +524,7 @@ func Test_provideFileName(t *testing.T) { e := &export{} // when - fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeObjectType) + fileName := e.makeFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeObjectType) // then assert.Equal(t, typesDirectory+string(filepath.Separator)+"docId.pb.json", fileName) @@ -534,7 +534,7 @@ func Test_provideFileName(t *testing.T) { e := &export{} // when - fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypePage) + fileName := e.makeFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypePage) // then assert.Equal(t, objectsDirectory+string(filepath.Separator)+"docId.pb.json", fileName) @@ -544,7 +544,7 @@ func Test_provideFileName(t *testing.T) { e := &export{} // when - fileName := e.provideFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeFileObject) + fileName := e.makeFileName("docId", "spaceId", pbjson.NewConverter(nil), nil, smartblock.SmartBlockTypeFileObject) // then assert.Equal(t, filesObjects+string(filepath.Separator)+"docId.pb.json", fileName) @@ -556,7 +556,7 @@ func Test_provideFileName(t *testing.T) { st.SetDetail(bundle.RelationKeySpaceId.String(), pbtypes.String("spaceId")) // when - fileName := e.provideFileName("docId", "", pbjson.NewConverter(st), st, smartblock.SmartBlockTypeFileObject) + fileName := e.makeFileName("docId", "", pbjson.NewConverter(st), st, smartblock.SmartBlockTypeFileObject) // then assert.Equal(t, spaceDirectory+string(filepath.Separator)+"spaceId"+string(filepath.Separator)+filesObjects+string(filepath.Separator)+"docId.pb.json", fileName) From b4f8757e1fd3914661e926a4345cd8830a66e2d7 Mon Sep 17 00:00:00 2001 From: AnastasiaShemyakinskaya Date: Thu, 8 Aug 2024 12:35:51 +0200 Subject: [PATCH 4/4] GO-2746: add comments Signed-off-by: AnastasiaShemyakinskaya --- core/block/export/export.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/block/export/export.go b/core/block/export/export.go index 21a9e660f..7e75e5e76 100644 --- a/core/block/export/export.go +++ b/core/block/export/export.go @@ -519,6 +519,7 @@ func (e *export) makeMarkdownName(s *state.State, wr writer, docID string, conv name = s.Snippet() } path := "" + // space can be empty in case user want to export all spaces if spaceId == "" { spaceId := pbtypes.GetString(s.LocalDetails(), bundle.RelationKeySpaceId.String()) path = filepath.Join(spaceDirectory, spaceId) @@ -529,6 +530,7 @@ func (e *export) makeMarkdownName(s *state.State, wr writer, docID string, conv func (e *export) makeFileName(docId, spaceId string, conv converter.Converter, st *state.State, blockType smartblock.SmartBlockType) string { dir := e.provideFileDirectory(blockType) filename := filepath.Join(dir, docId+conv.Ext()) + // space can be empty in case user want to export all spaces if spaceId == "" { spaceId := pbtypes.GetString(st.LocalDetails(), bundle.RelationKeySpaceId.String()) filename = filepath.Join(spaceDirectory, spaceId, filename)