From 5c421d429b6b19596954652638a091b8a80c2be1 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Fri, 9 Apr 2021 15:13:10 +0400 Subject: [PATCH] fix tests --- cmd/cli/cafe.go | 21 +- core/account.go | 8 +- core/anytype/bootstrap.go | 31 +-- core/block/database/objects/objects.go | 6 +- core/block/editor.go | 6 +- core/block/editor/dataview/dataview.go | 10 +- core/block/source/source.go | 6 +- core/history/history_test.go | 2 +- core/indexer/indexer.go | 13 +- core/indexer/indexer_test.go | 44 +++- core/recordsbatcher/recordsbatcher.go | 24 +- core/relations.go | 4 +- core/relations_test.go | 16 +- core/wallet/wallet.go | 37 +-- go.mod | 4 +- pkg/lib/core/common_test.go | 58 ----- pkg/lib/core/core.go | 21 +- pkg/lib/core/core_test.go | 217 ------------------ pkg/lib/core/files.go | 12 +- pkg/lib/core/files_test.go | 56 ----- pkg/lib/core/images.go | 4 +- pkg/lib/core/images_test.go | 102 -------- pkg/lib/core/migration.go | 51 ++-- pkg/lib/core/pages.go | 10 +- pkg/lib/core/pages_test.go | 168 -------------- pkg/lib/core/profile.go | 2 +- pkg/lib/core/smartblocks.go | 2 +- pkg/lib/core/smartblocks_test.go | 1 - pkg/lib/core/wallet_test.go | 40 ---- pkg/lib/datastore/clientds/clientds.go | 23 +- pkg/lib/files/files.go | 9 +- pkg/lib/files/images.go | 4 +- pkg/lib/ipfs/helpers/helpers.go | 17 +- pkg/lib/localstore/{ => filestore}/files.go | 111 +++++---- pkg/lib/localstore/ftsearch/ftsearch.go | 7 +- pkg/lib/localstore/ftsearch/ftsearch_test.go | 37 ++- .../localstore/{ => objectstore}/objects.go | 188 ++++++++------- .../{ => objectstore}/objects_test.go | 47 ++-- pkg/lib/localstore/stores.go | 111 ++------- pkg/lib/pin/service.go | 4 +- util/slice/slice.go | 23 ++ util/testMock/anytype.go | 11 +- 42 files changed, 509 insertions(+), 1059 deletions(-) delete mode 100644 pkg/lib/core/common_test.go delete mode 100644 pkg/lib/core/core_test.go delete mode 100644 pkg/lib/core/files_test.go delete mode 100644 pkg/lib/core/images_test.go delete mode 100644 pkg/lib/core/pages_test.go delete mode 100644 pkg/lib/core/smartblocks_test.go delete mode 100644 pkg/lib/core/wallet_test.go rename pkg/lib/localstore/{ => filestore}/files.go (73%) rename pkg/lib/localstore/{ => objectstore}/objects.go (88%) rename pkg/lib/localstore/{ => objectstore}/objects_test.go (85%) diff --git a/cmd/cli/cafe.go b/cmd/cli/cafe.go index 1a31e82de..56eb2528b 100644 --- a/cmd/cli/cafe.go +++ b/cmd/cli/cafe.go @@ -2,7 +2,9 @@ package main import ( "context" - "fmt" + app2 "github.com/anytypeio/go-anytype-middleware/app" + wallet2 "github.com/anytypeio/go-anytype-middleware/core/wallet" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/cafe" core2 "github.com/anytypeio/go-anytype-middleware/pkg/lib/core" "github.com/anytypeio/go-anytype-middleware/pkg/lib/wallet" "github.com/anytypeio/go-anytype-middleware/util/console" @@ -53,15 +55,14 @@ var findProfiles = &cobra.Command{ appAccount, err = core2.WalletAccountAt(appMnemonic, 0, "") rootPath, err := ioutil.TempDir(os.TempDir(), "anytype_*") - rawSeed, err := appAccount.Raw() - - err = core2.WalletInitRepo(rootPath, rawSeed) - - var opts = []core2.ServiceOption{core2.WithRootPathAndAccount(rootPath, appAccount.Address())} - a, _ := core2.New(opts...) - err = a.Start() + app := new(app2.App) + app.Register(wallet2.NewWithRepoPathAndKeys(rootPath, appAccount, nil)) + app.Register(cafe.New()) + at := core2.New() + app.Register(at) + err = app.Start() if err != nil { - fmt.Println("failed to start: "+ err.Error()) + console.Fatal("failed to start anytype: %s", err.Error()) return } var found bool @@ -78,7 +79,7 @@ var findProfiles = &cobra.Command{ console.Success("got profile: id=%s name=%s", profile.AccountAddr, profile.Name) } }() - err = a.FindProfilesByAccountIDs(context.Background(), accountsToFind, ch) + err = at.FindProfilesByAccountIDs(context.Background(), accountsToFind, ch) if err != nil { console.Fatal("failed to query cafe: " + err.Error()) } diff --git a/core/account.go b/core/account.go index d260ca5c7..9531f0699 100644 --- a/core/account.go +++ b/core/account.go @@ -155,7 +155,7 @@ func (mw *Middleware) AccountCreate(req *pb.RpcAccountCreateRequest) *pb.RpcAcco newAcc := &model.Account{Id: account.Address()} - comps, err := anytype.DefaultClientComponents(true, mw.rootPath, account.Address()) + comps, err := anytype.BootstrapConfigAndWallet(true, mw.rootPath, account.Address()) if err != nil { return response(nil, pb.RpcAccountCreateResponseError_UNKNOWN_ERROR, err) } @@ -275,7 +275,7 @@ func (mw *Middleware) AccountRecover(_ *pb.RpcAccountRecoverRequest) *pb.RpcAcco return response(pb.RpcAccountRecoverResponseError_FAILED_TO_STOP_RUNNING_NODE, err) } - comps, err := anytype.DefaultClientComponents(false, mw.rootPath, zeroAccount.Address()) + comps, err := anytype.BootstrapConfigAndWallet(false, mw.rootPath, zeroAccount.Address()) if err != nil { return response(pb.RpcAccountRecoverResponseError_UNKNOWN_ERROR, err) } @@ -293,7 +293,7 @@ func (mw *Middleware) AccountRecover(_ *pb.RpcAccountRecoverRequest) *pb.RpcAcco mw.accountSearchCancel = func() { searchQueryCancel() } defer searchQueryCancel() - comps, err = anytype.DefaultClientComponents(false, mw.rootPath, zeroAccount.Address()) + comps, err = anytype.BootstrapConfigAndWallet(false, mw.rootPath, zeroAccount.Address()) if err != nil { return response(pb.RpcAccountRecoverResponseError_UNKNOWN_ERROR, err) } @@ -465,7 +465,7 @@ func (mw *Middleware) AccountSelect(req *pb.RpcAccountSelectRequest) *pb.RpcAcco } } - comps, err := anytype.DefaultClientComponents(false, mw.rootPath, req.Id) + comps, err := anytype.BootstrapConfigAndWallet(false, mw.rootPath, req.Id) if err != nil { return response(nil, pb.RpcAccountSelectResponseError_UNKNOWN_ERROR, err) } diff --git a/core/anytype/bootstrap.go b/core/anytype/bootstrap.go index ad839b71c..4be7bfd97 100644 --- a/core/anytype/bootstrap.go +++ b/core/anytype/bootstrap.go @@ -18,32 +18,21 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/files" "github.com/anytypeio/go-anytype-middleware/pkg/lib/gateway" "github.com/anytypeio/go-anytype-middleware/pkg/lib/ipfs/ipfslite" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/filestore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/ftsearch" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pin" "github.com/anytypeio/go-anytype-middleware/pkg/lib/threads" "github.com/anytypeio/go-anytype-middleware/util/builtintemplate" "github.com/anytypeio/go-anytype-middleware/util/linkpreview" ) -func DefaultClientComponents(newAccount bool, rootPath, accountId string) ([]app.Component, error) { +func BootstrapConfigAndWallet(newAccount bool, rootPath, accountId string) ([]app.Component, error) { return []app.Component{ config.New(func(c *config.Config) { c.NewAccount = newAccount }), - wallet.NewWithAccountRepo(rootPath, accountId), - clientds.New(), - ftsearch.New(), - localstore.New(), - recordsbatcher.New(), - indexer.New(), - ipfslite.New(), - files.New(), - cafe.New(), - threads.New(), - core.New(), - pin.New(), }, nil } @@ -60,7 +49,19 @@ func Bootstrap(a *app.App, components ...app.Component) { for _, c := range components { a.Register(c) } - a.Register(status.New()). + a.Register(clientds.New()). + Register(ftsearch.New()). + Register(objectstore.New()). + Register(filestore.New()). + Register(recordsbatcher.New()). + Register(indexer.New()). + Register(ipfslite.New()). + Register(files.New()). + Register(cafe.New()). + Register(threads.New()). + Register(core.New()). + Register(pin.New()). + Register(status.New()). Register(meta.New()). Register(block.New()). Register(process.New()). diff --git a/core/block/database/objects/objects.go b/core/block/database/objects/objects.go index 507dcec1a..d966e565d 100644 --- a/core/block/database/objects/objects.go +++ b/core/block/database/objects/objects.go @@ -2,13 +2,13 @@ package objects import ( "errors" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/anytypeio/go-anytype-middleware/pb" "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" coresb "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" "github.com/anytypeio/go-anytype-middleware/pkg/lib/database" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore" pbrelation "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/relation" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" "github.com/gogo/protobuf/types" @@ -22,7 +22,7 @@ const ( var log = logging.Logger("anytype-core-db") func New( - pageStore localstore.ObjectStore, + pageStore objectstore.ObjectStore, objectTypeUrl string, setDetails func(req pb.RpcBlockSetDetailsRequest) error, getRelations func(objectId string) (relations []*pbrelation.Relation, err error), @@ -44,7 +44,7 @@ func New( } type setOfObjects struct { - localstore.ObjectStore + objectstore.ObjectStore objectTypeUrl string setDetails func(req pb.RpcBlockSetDetailsRequest) error getRelations func(objectId string) (relations []*pbrelation.Relation, err error) diff --git a/core/block/editor.go b/core/block/editor.go index 16a8114b6..648d1e89f 100644 --- a/core/block/editor.go +++ b/core/block/editor.go @@ -3,6 +3,7 @@ package block import ( "context" "fmt" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/core/block/editor" "github.com/anytypeio/go-anytype-middleware/core/block/editor/basic" @@ -21,7 +22,6 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" coresb "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" "github.com/anytypeio/go-anytype-middleware/pkg/lib/files" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" pbrelation "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/relation" "github.com/anytypeio/go-anytype-middleware/util/pbtypes" @@ -723,7 +723,7 @@ func (s *service) SetObjectTypes(ctx *state.Context, objectId string, objectType } func (s *service) CreateSet(ctx *state.Context, req pb.RpcBlockCreateSetRequest) (linkId string, setId string, err error) { - objType, err := localstore.GetObjectType(s.anytype.ObjectStore(), req.ObjectTypeUrl) + objType, err := objectstore.GetObjectType(s.anytype.ObjectStore(), req.ObjectTypeUrl) if err != nil { return "", "", err } @@ -844,4 +844,4 @@ func (s *service) ListAvailableRelations(objectId string) (aggregatedRelations [ }) return -} \ No newline at end of file +} diff --git a/core/block/editor/dataview/dataview.go b/core/block/editor/dataview/dataview.go index 89b0dfef9..c0cb2b480 100644 --- a/core/block/editor/dataview/dataview.go +++ b/core/block/editor/dataview/dataview.go @@ -3,6 +3,7 @@ package dataview import ( "context" "fmt" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "sync" blockDB "github.com/anytypeio/go-anytype-middleware/core/block/database" @@ -14,7 +15,6 @@ import ( bundle "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" "github.com/anytypeio/go-anytype-middleware/pkg/lib/database" "github.com/anytypeio/go-anytype-middleware/pkg/lib/database/filter" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" pbrelation "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/relation" @@ -291,7 +291,7 @@ func (d *dataviewCollectionImpl) GetAggregatedRelations(blockId string) ([]*pbre return nil, err } - objectType, err := localstore.GetObjectType(d.Anytype().ObjectStore(), tb.GetSource()) + objectType, err := objectstore.GetObjectType(d.Anytype().ObjectStore(), tb.GetSource()) if err != nil { return nil, err } @@ -476,7 +476,7 @@ func (d *dataviewCollectionImpl) CreateView(ctx *state.Context, id string, view } if len(view.Relations) == 0 { - objType, err := localstore.GetObjectType(d.Anytype().ObjectStore(), tb.GetSource()) + objType, err := objectstore.GetObjectType(d.Anytype().ObjectStore(), tb.GetSource()) if err != nil { return nil, fmt.Errorf("object type not found") } @@ -577,7 +577,7 @@ func (d *dataviewCollectionImpl) UpdateRecord(_ *state.Context, blockId string, } dv := d.getDataviewImpl(dvBlock) - objectType, err := localstore.GetObjectType(d.Anytype().ObjectStore(), source) + objectType, err := objectstore.GetObjectType(d.Anytype().ObjectStore(), source) if err != nil { return err } @@ -707,7 +707,7 @@ func (d *dataviewCollectionImpl) fetchAndGetEventsMessages(dv *dataviewImpl, dvB } // todo: inject schema - objectType, err := localstore.GetObjectType(d.Anytype().ObjectStore(), source) + objectType, err := objectstore.GetObjectType(d.Anytype().ObjectStore(), source) if err != nil { return nil, err } diff --git a/core/block/source/source.go b/core/block/source/source.go index ea5804eda..483701991 100644 --- a/core/block/source/source.go +++ b/core/block/source/source.go @@ -247,7 +247,8 @@ func InjectCreationInfo(s Source, st *state.State) (err error) { var ( createdDate = time.Now().Unix() - createdBy string + // todo: remove this default + createdBy = s.Anytype().Account() ) // protect from the big documents with a large trees ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) @@ -255,8 +256,7 @@ func InjectCreationInfo(s Source, st *state.State) (err error) { fc, err := s.FindFirstChange(ctx) if err == change.ErrEmpty { err = nil - // todo: fixme refactor - createdBy = "" + createdBy = s.Anytype().Account() log.Debugf("InjectCreationInfo set for the empty object") } else if err != nil { return fmt.Errorf("failed to find first change to derive creation info") diff --git a/core/history/history_test.go b/core/history/history_test.go index 55f97f061..44f4d5af0 100644 --- a/core/history/history_test.go +++ b/core/history/history_test.go @@ -1,6 +1,7 @@ package history import ( + "github.com/anytypeio/go-anytype-middleware/pkg/lib/threads" "testing" "github.com/anytypeio/go-anytype-middleware/app" @@ -8,7 +9,6 @@ import ( "github.com/anytypeio/go-anytype-middleware/change" "github.com/anytypeio/go-anytype-middleware/core/block/editor/state" "github.com/anytypeio/go-anytype-middleware/pb" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/threads" "github.com/anytypeio/go-anytype-middleware/util/testMock" "github.com/anytypeio/go-anytype-middleware/util/testMock/mockMeta" "github.com/golang/mock/gomock" diff --git a/core/indexer/indexer.go b/core/indexer/indexer.go index 93e46a319..71612ec79 100644 --- a/core/indexer/indexer.go +++ b/core/indexer/indexer.go @@ -2,6 +2,7 @@ package indexer import ( "fmt" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "sync" "time" @@ -12,7 +13,6 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" "github.com/anytypeio/go-anytype-middleware/pkg/lib/core" "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/ftsearch" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" @@ -61,7 +61,7 @@ type batchReader interface { } type indexer struct { - store localstore.ObjectStore + store objectstore.ObjectStore anytype core.Service searchInfo GetSearchInfo cache map[string]*doc @@ -76,6 +76,8 @@ type indexer struct { func (i *indexer) Init(a *app.App) (err error) { i.anytype = a.MustComponent(core.CName).(core.Service) i.searchInfo = a.MustComponent("blockService").(GetSearchInfo) + i.store = a.MustComponent(objectstore.CName).(objectstore.ObjectStore) + i.cache = make(map[string]*doc) i.newRecordsBatcher = a.MustComponent(recordsbatcher.CName).(recordsbatcher.RecordsBatcher) i.quitWG = new(sync.WaitGroup) @@ -88,7 +90,6 @@ func (i *indexer) Name() (name string) { } func (i *indexer) Run() (err error) { - i.store = i.anytype.ObjectStore() if ftErr := i.ftInit(); ftErr != nil { log.Errorf("can't init ft: %v", ftErr) } @@ -144,14 +145,15 @@ func (i *indexer) reindexBundled() { func (i *indexer) detailsLoop() { go func() { defer i.quitWG.Done() - var records []core.SmartblockRecordWithThreadID + var records = make([]core.SmartblockRecordWithThreadID, 100) for { - records = records[:0] + records = records[0:cap(records)] n := i.newRecordsBatcher.Read(records) if n == 0 { // means no more data is available return } + records = records[0:n] i.applyRecords(records) } @@ -287,6 +289,7 @@ func (i *indexer) index(id string, records []core.SmartblockRecordEnvelope, only func (i *indexer) ftLoop() { defer i.quitWG.Done() ticker := time.NewTicker(ftIndexInterval) + i.ftIndex() for { select { case <-i.quit: diff --git a/core/indexer/indexer_test.go b/core/indexer/indexer_test.go index 2aed1ed93..a8c2e1828 100644 --- a/core/indexer/indexer_test.go +++ b/core/indexer/indexer_test.go @@ -1,8 +1,17 @@ package indexer_test import ( + "github.com/anytypeio/go-anytype-middleware/core/anytype/config" + "github.com/anytypeio/go-anytype-middleware/core/recordsbatcher" + "github.com/anytypeio/go-anytype-middleware/core/wallet" "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/datastore/clientds" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/ftsearch" + "io" + "io/ioutil" + "os" "testing" "time" @@ -25,14 +34,14 @@ func TestNewIndexer(t *testing.T) { t.Run("open/close", func(t *testing.T) { fx := newFixture(t) // should add all bundled relations to full text index - defer fx.tearDown() defer fx.Close() + defer fx.tearDown() }) t.Run("indexMeta", func(t *testing.T) { fx := newFixture(t) - defer fx.tearDown() defer fx.Close() + defer fx.tearDown() var ( sbId = "sbId" @@ -54,18 +63,20 @@ func TestNewIndexer(t *testing.T) { updatedCh = make(chan struct{}) ) sb.EXPECT().ID().Return(sbId).AnyTimes() + sb.EXPECT().Type().Return(smartblock.SmartBlockTypePage).AnyTimes() + sb.EXPECT().GetLogs().Return(nil, nil) fx.anytype.EXPECT().GetBlock(sbId).Return(sb, nil) fx.objectStore.EXPECT().AddToIndexQueue(sbId) fx.objectStore.EXPECT().GetDetails(sbId) - fx.objectStore.EXPECT().UpdateObject(sbId, gomock.Any(), gomock.Any(), nil, "").DoAndReturn(func(id string, details *types.Struct, relations *pbrelation.Relations, links []string, snippet string) (err error) { + fx.objectStore.EXPECT().UpdateObjectDetails(sbId, gomock.Any(), gomock.Any()).DoAndReturn(func(id string, details *types.Struct, relations *pbrelation.Relations) (err error) { assert.Equal(t, "value", pbtypes.GetString(det, "key")) close(updatedCh) return }) - fx.ch <- core.SmartblockRecordWithThreadID{ + fx.rb.Add(core.SmartblockRecordWithThreadID{ SmartblockRecordEnvelope: core.SmartblockRecordEnvelope{ SmartblockRecord: core.SmartblockRecord{ ID: "snapshot", @@ -73,7 +84,7 @@ func TestNewIndexer(t *testing.T) { }, }, ThreadID: sbId, - } + }) select { case <-updatedCh: @@ -84,36 +95,45 @@ func TestNewIndexer(t *testing.T) { } func newFixture(t *testing.T) *fixture { + ta := testapp.New() + rb := recordsbatcher.New() fx := &fixture{ ctrl: gomock.NewController(t), ta: ta, + rb: rb, } fx.anytype = testMock.RegisterMockAnytype(fx.ctrl, ta) fx.getSerach = mockIndexer.NewMockGetSearchInfo(fx.ctrl) fx.getSerach.EXPECT().Name().AnyTimes().Return("blockService") fx.getSerach.EXPECT().Init(gomock.Any()) - fx.objectStore = testMock.NewMockObjectStore(fx.ctrl) - fx.objectStore.EXPECT().AddToIndexQueue("_anytype_profile") + fx.objectStore = testMock.RegisterMockObjectStore(fx.ctrl, ta) + + fx.getSerach.EXPECT().GetSearchInfo(gomock.Any()).AnyTimes() for _, rk := range bundle.ListRelationsKeys() { fx.objectStore.EXPECT().AddToIndexQueue(addr.BundledRelationURLPrefix + rk.String()) + } for _, ok := range bundle.ListTypesKeys() { fx.objectStore.EXPECT().AddToIndexQueue(ok.URL()) } + fx.objectStore.EXPECT().AddToIndexQueue("_anytype_profile") fx.objectStore.EXPECT().FTSearch().Return(nil).AnyTimes() - fx.objectStore.EXPECT().CreateObject(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + fx.objectStore.EXPECT().IndexForEach(gomock.Any()).Times(1) + fx.objectStore.EXPECT().CreateObject(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() fx.anytype.EXPECT().ObjectStore().Return(fx.objectStore).AnyTimes() - fx.ch = make(chan core.SmartblockRecordWithThreadID) - fx.anytype.EXPECT().SubscribeForNewRecords(gomock.Any()).Return(fx.ch, nil) fx.Indexer = indexer.New() - ta.With(fx.Indexer).With(fx.getSerach) + + rootPath, err := ioutil.TempDir(os.TempDir(), "anytype_*") + require.NoError(t, err) + + ta.With(&config.DefaultConfig).With(wallet.NewWithRepoPathAndKeys(rootPath, nil, nil)).With(clientds.New()).With(ftsearch.New()).With(fx.rb).With(fx.Indexer).With(fx.getSerach) require.NoError(t, ta.Start()) return fx @@ -126,10 +146,12 @@ type fixture struct { objectStore *testMock.MockObjectStore getSerach *mockIndexer.MockGetSearchInfo ch chan core.SmartblockRecordWithThreadID + rb recordsbatcher.RecordsBatcher ta *testapp.TestApp } func (fx *fixture) tearDown() { + fx.rb.(io.Closer).Close() fx.ta.Close() fx.ctrl.Finish() } diff --git a/core/recordsbatcher/recordsbatcher.go b/core/recordsbatcher/recordsbatcher.go index 12fc92179..644adf7f6 100644 --- a/core/recordsbatcher/recordsbatcher.go +++ b/core/recordsbatcher/recordsbatcher.go @@ -3,6 +3,7 @@ package recordsbatcher import ( "github.com/anytypeio/go-anytype-middleware/app" "github.com/anytypeio/go-anytype-middleware/pkg/lib/core" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/cheggaaa/mb" "sync" "time" @@ -10,6 +11,8 @@ import ( const CName = "recordsbatcher" +var log = logging.Logger("anytype-recordsbatcher") + type recordsBatcher struct { batcher *mb.MB packDelay time.Duration // delay for better packing of msgs @@ -17,6 +20,7 @@ type recordsBatcher struct { } func (r *recordsBatcher) Init(a *app.App) (err error) { + log.Errorf("recordsBatcher %p init", r) r.batcher = mb.New(0) r.packDelay = time.Millisecond * 100 return nil @@ -27,7 +31,12 @@ func (r *recordsBatcher) Name() (name string) { } func (r *recordsBatcher) Add(msgs ...core.SmartblockRecordWithThreadID) error { - return r.batcher.Add(msgs) + var msgsIfaces []interface{} + for _, msg := range msgs { + msgsIfaces = append(msgsIfaces, interface{}(msg)) + } + + return r.batcher.Add(msgsIfaces...) } func (r *recordsBatcher) Read(buffer []core.SmartblockRecordWithThreadID) int { @@ -36,19 +45,22 @@ func (r *recordsBatcher) Read(buffer []core.SmartblockRecordWithThreadID) int { time.Sleep(r.packDelay) r.m.Unlock() }() - msgs := r.batcher.Wait() + + msgs := r.batcher.WaitMax(len(buffer)) if len(msgs) == 0 { return 0 } - var total int + var msgsCasted []core.SmartblockRecordWithThreadID for _, msg := range msgs { - buffer = append(buffer, msg.(core.SmartblockRecordWithThreadID)) - total++ + msgsCasted = append(msgsCasted[0:], msg.(core.SmartblockRecordWithThreadID)) } - return total + + return copy(buffer, msgsCasted) } func (r *recordsBatcher) Close() (err error) { + log.Errorf("recordsBatcher %p close", r) + return r.batcher.Close() } diff --git a/core/relations.go b/core/relations.go index 94cf00912..5f68f3c4f 100644 --- a/core/relations.go +++ b/core/relations.go @@ -2,10 +2,10 @@ package core import ( "fmt" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/threads" "strings" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" "github.com/globalsign/mgo/bson" @@ -315,5 +315,5 @@ func (mw *Middleware) SetCreate(req *pb.RpcSetCreateRequest) *pb.RpcSetCreateRes } func (mw *Middleware) getObjectType(at core.Service, url string) (*pbrelation.ObjectType, error) { - return localstore.GetObjectType(at.ObjectStore(), url) + return objectstore.GetObjectType(at.ObjectStore(), url) } diff --git a/core/relations_test.go b/core/relations_test.go index eb670b245..2d4ec6ebc 100644 --- a/core/relations_test.go +++ b/core/relations_test.go @@ -169,7 +169,7 @@ func TestRelationAdd(t *testing.T) { }}}) require.Equal(t, 0, int(respPageCreate.Error.Code), respPageCreate.Error.Description) - time.Sleep(time.Second * 2) + time.Sleep(time.Millisecond * 200) respOpenNewPage = mw.BlockOpen(&pb.RpcBlockOpenRequest{BlockId: respPageCreate.PageId}) require.Equal(t, 0, int(respOpenNewPage.Error.Code), respOpenNewPage.Error.Description) @@ -388,7 +388,7 @@ func TestRelationAdd(t *testing.T) { RelationKey: respRelCreate.RelationKey, }) require.Equal(t, 0, int(respRelOptCreate.Error.Code), respRelOptCreate.Error.Description) - time.Sleep(time.Second * 1) + time.Sleep(time.Millisecond * 200) respRecordUpdate := mw.BlockDataviewRecordUpdate( &pb.RpcBlockDataviewRecordUpdateRequest{ @@ -423,7 +423,7 @@ func TestRelationAdd(t *testing.T) { }) require.Equal(t, 0, int(respOptAdd.Error.Code), respOptAdd.Error.Description) - time.Sleep(time.Second) + time.Sleep(time.Millisecond * 200) respRecordUpdate2 := mw.BlockDataviewRecordUpdate( &pb.RpcBlockDataviewRecordUpdateRequest{ @@ -520,7 +520,7 @@ func TestRelationAdd(t *testing.T) { }) require.Equal(t, 0, int(respRelAdd3.Error.Code), respRelAdd3.Error.Description) rel3.Key = respRelAdd3.RelationKey - time.Sleep(time.Second) + time.Sleep(time.Millisecond * 200) respOptionAdd1 := mw.ObjectRelationOptionAdd(&pb.RpcObjectRelationOptionAddRequest{ ContextId: respPage1Create.PageId, @@ -577,7 +577,7 @@ func TestRelationAdd(t *testing.T) { }, }) require.Equal(t, 0, int(respOptionAdd5.Error.Code), respOptionAdd5.Error.Description) - time.Sleep(time.Second * 2) + time.Sleep(time.Millisecond * 200) tests := []test{ { rel1.Key, @@ -749,7 +749,7 @@ func TestCustomType(t *testing.T) { } } - time.Sleep(time.Second) + time.Sleep(time.Millisecond * 200) respObjectTypeList = mw.ObjectTypeList(nil) require.Equal(t, 0, int(respObjectTypeList.Error.Code), respObjectTypeList.Error.Description) lastObjType := respObjectTypeList.ObjectTypes[len(respObjectTypeList.ObjectTypes)-1] @@ -860,7 +860,7 @@ func TestBundledType(t *testing.T) { profile := getDetailsForContext(show.Details, mw.GetAnytype().PredefinedBlocks().Profile) require.NotNil(t, profile, fmt.Sprintf("%s got no details for profile", show.RootId)) - time.Sleep(time.Second) + time.Sleep(time.Millisecond * 200) respOpenPagesSet := mw.BlockOpen(&pb.RpcBlockOpenRequest{BlockId: mw.GetAnytype().PredefinedBlocks().SetPages}) require.Equal(t, 0, int(respOpenPagesSet.Error.Code), respOpenPagesSet.Error.Description) @@ -877,7 +877,7 @@ func TestBundledType(t *testing.T) { respCreatePage = mw.PageCreate(&pb.RpcPageCreateRequest{Details: &types2.Struct{Fields: map[string]*types2.Value{"name": pbtypes.String("test2")}}}) require.Equal(t, 0, int(respCreatePage.Error.Code), respCreatePage.Error.Description) - time.Sleep(time.Second) + time.Sleep(time.Millisecond * 200) respOpenPagesSet = mw.BlockOpen(&pb.RpcBlockOpenRequest{BlockId: mw.GetAnytype().PredefinedBlocks().SetPages}) require.Equal(t, 0, int(respOpenPagesSet.Error.Code), respOpenPagesSet.Error.Description) diff --git a/core/wallet/wallet.go b/core/wallet/wallet.go index 373b4f197..d0eccd583 100644 --- a/core/wallet/wallet.go +++ b/core/wallet/wallet.go @@ -40,7 +40,7 @@ func (r *wallet) GetDevicePrivkey() (wallet2.Keypair, error) { func (r *wallet) Init(a *app.App) (err error) { var b []byte - if r.deviceKeyPath != "" { + if r.deviceKeypair == nil && r.deviceKeyPath != "" { b, err = ioutil.ReadFile(r.deviceKeyPath) if err != nil { return fmt.Errorf("failed to read device keyfile: %w", err) @@ -56,21 +56,24 @@ func (r *wallet) Init(a *app.App) (err error) { } } - b, err = ioutil.ReadFile(r.accountKeyPath) - if err != nil { - return fmt.Errorf("failed to read account keyfile: %w", err) + if r.accountKeypair == nil && r.accountKeyPath != "" { + b, err = ioutil.ReadFile(r.accountKeyPath) + if err != nil { + return fmt.Errorf("failed to read account keyfile: %w", err) + } + + r.accountKeypair, err = wallet2.UnmarshalBinary(b) + if err != nil { + return err + } + if r.accountKeypair.KeypairType() != wallet2.KeypairTypeAccount { + return fmt.Errorf("got %s key type instead of %s", r.accountKeypair.KeypairType(), wallet2.KeypairTypeAccount) + } } - r.accountKeypair, err = wallet2.UnmarshalBinary(b) - if err != nil { - return err + if r.deviceKeypair != nil { + logging.SetHost(r.deviceKeypair.Address()) } - if r.accountKeypair.KeypairType() != wallet2.KeypairTypeAccount { - return fmt.Errorf("got %s key type instead of %s", r.accountKeypair.KeypairType(), wallet2.KeypairTypeAccount) - } - - logging.SetHost(r.deviceKeypair.Address()) - return nil } @@ -95,6 +98,14 @@ func NewWithAccountRepo(rootpath, accountId string) Wallet { } } +func NewWithRepoPathAndKeys(repoPath string, accountKeypair, deviceKeypair wallet2.Keypair) Wallet { + return &wallet{ + repoPath: repoPath, + accountKeypair: accountKeypair, + deviceKeypair: deviceKeypair, + } +} + type Wallet interface { RepoPath() string GetAccountPrivkey() (wallet2.Keypair, error) diff --git a/go.mod b/go.mod index 9cc69ef91..d59ba57f5 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 github.com/h2non/filetype v1.1.1 - github.com/hashicorp/go-multierror v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/golang-lru v0.5.4 github.com/hsanjuan/ipfs-lite v1.1.18 github.com/improbable-eng/grpc-web v0.13.0 @@ -67,7 +67,7 @@ require ( github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd github.com/spf13/cobra v0.0.5 github.com/stretchr/testify v1.7.0 - github.com/textileio/go-ds-badger v0.2.7-0.20201204225019-4ee78c4a40e2 // indirect + github.com/textileio/go-ds-badger v0.2.7-0.20201204225019-4ee78c4a40e2 github.com/textileio/go-threads v1.0.2-0.20210304072541-d0f91da84404 github.com/tyler-smith/go-bip39 v1.0.1-0.20190808214741-c55f737395bc github.com/uber/jaeger-client-go v2.25.0+incompatible diff --git a/pkg/lib/core/common_test.go b/pkg/lib/core/common_test.go deleted file mode 100644 index 5643b471b..000000000 --- a/pkg/lib/core/common_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package core - -import ( - "bytes" - "context" - "io/ioutil" - "testing" - "time" - - "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/config" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/wallet" - ipfslite "github.com/hsanjuan/ipfs-lite" - "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/pnet" - ma "github.com/multiformats/go-multiaddr" - "github.com/stretchr/testify/require" -) - -func Benchmark_ConnectCafe(t *testing.B) { - r := bytes.NewReader([]byte(ipfsPrivateNetworkKey)) - pnet, err := pnet.DecodeV1PSK(r) - require.NoError(t, err) - - cafeAddr, err := ma.NewMultiaddr(config.DefaultConfig.CafeP2PAddr) - require.NoError(t, err) - - cafeAddrInfo, err := peer.AddrInfoFromP2pAddr(cafeAddr) - require.NoError(t, err) - - for n := 0; n < t.N; n++ { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - - deviceKP, err := wallet.NewRandomKeypair(wallet.KeypairTypeDevice) - require.NoError(t, err) - - tmpfile, err := ioutil.TempFile("", "ipfslite1") - require.NoError(t, err) - tmpfile.Close() - - m, err := ma.NewMultiaddr("/ip4/0.0.0.0/tcp/0") - require.NoError(t, err) - - h, dht, err := ipfslite.SetupLibp2p( - ctx, - deviceKP, - pnet, - []ma.Multiaddr{m}, - nil, - ipfslite.Libp2pOptionsExtra..., - ) - - err = h.Connect(ctx, *cafeAddrInfo) - require.NoError(t, err) - cancel() - h.Close() - dht.Close() - } -} diff --git a/pkg/lib/core/core.go b/pkg/lib/core/core.go index db67bfc54..b1a98a1a9 100644 --- a/pkg/lib/core/core.go +++ b/pkg/lib/core/core.go @@ -3,6 +3,8 @@ package core import ( "context" "fmt" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/filestore" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "io" "sync" "time" @@ -22,7 +24,6 @@ import ( "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" "github.com/anytypeio/go-anytype-middleware/pkg/lib/datastore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/files" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pin" @@ -73,7 +74,7 @@ type Service interface { ImageAddWithBytes(ctx context.Context, content []byte, filename string) (Image, error) // deprecated ImageAddWithReader(ctx context.Context, content io.ReadSeeker, filename string) (Image, error) // deprecated - ObjectStore() localstore.ObjectStore // deprecated + ObjectStore() objectstore.ObjectStore // deprecated ObjectInfoWithLinks(id string) (*model.ObjectInfoWithLinks, error) ObjectList() ([]*model.ObjectInfo, error) @@ -87,10 +88,12 @@ var _ app.Component = (*Anytype)(nil) var _ Service = (*Anytype)(nil) type Anytype struct { - files *files.Service - cafe cafe.Client - mdns discovery.Service - localStore *localstore.LocalStore + files *files.Service + cafe cafe.Client + mdns discovery.Service + objectStore objectstore.ObjectStore + fileStore filestore.FileStore + localStoreDS ds.TxnDatastore predefinedBlockIds threads.DerivedSmartblockIds @@ -132,7 +135,8 @@ func (a *Anytype) Init(ap *app.App) (err error) { a.wallet = ap.MustComponent(wallet.CName).(wallet.Wallet) a.config = ap.MustComponent(config.CName).(*config.Config) a.recordsbatch = ap.MustComponent("recordsbatcher").(batchAdder) - a.localStore = ap.MustComponent(localstore.CName).(*localstore.LocalStore) + a.objectStore = ap.MustComponent(objectstore.CName).(objectstore.ObjectStore) + a.fileStore = ap.MustComponent(filestore.CName).(filestore.FileStore) a.localStoreDS = ap.MustComponent(datastore.CName).(datastore.Datastore).LocalstoreDS() a.threadService = ap.MustComponent(threads.CName).(threads.Service) a.cafe = ap.MustComponent(cafe.CName).(cafe.Client) @@ -298,7 +302,8 @@ func (a *Anytype) InitNewSmartblocksChan(ch chan<- string) error { // Subscribes to new records for all threads and add them to the batcher func (a *Anytype) subscribeForNewRecords() (err error) { ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + // do not defer cancel, cancel only on shutdown + threadsCh, err := a.threadService.Threads().Subscribe(ctx) if err != nil { return err diff --git a/pkg/lib/core/core_test.go b/pkg/lib/core/core_test.go deleted file mode 100644 index d870e39b0..000000000 --- a/pkg/lib/core/core_test.go +++ /dev/null @@ -1,217 +0,0 @@ -package core - -import ( - "context" - "fmt" - "sync" - "testing" - "time" - - "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/threads" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/database" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" - pbrelation "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/relation" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/schema" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/structs" - "github.com/anytypeio/go-anytype-middleware/util/pbtypes" - "github.com/gogo/protobuf/types" - "github.com/stretchr/testify/require" -) - -var doOnce sync.Once -var s Service - -func getRunningServiceB(t *testing.B) Service { - doOnce.Do(func() { - s = createAccount(t) - err := s.Start() - require.NoError(t, err) - }) - return s -} - -func getRunningService(t *testing.T) Service { - doOnce.Do(func() { - s = createAccount(t) - err := s.Start() - require.NoError(t, err) - - err = s.InitPredefinedBlocks(context.Background(), false) - require.NoError(t, err) - }) - return s -} - -func TestAnytype_IsStarted(t *testing.T) { - s := getRunningService(t) - require.True(t, s.(*Anytype).isStarted) -} - -func TestAnytype_DeviceKeyEquals(t *testing.T) { - s := getRunningService(t) - require.Equal(t, s.(*Anytype).t.Host().ID().String(), s.(*Anytype).opts.Device.Address()) -} - -func TestAnytype_GetDatabaseByID(t *testing.T) { - s := getRunningService(t) - require.NotNil(t, s) - - err := s.InitPredefinedBlocks(context.Background(), false) - require.NoError(t, err) - - block1, err := s.CreateBlock(smartblock.SmartBlockTypePage) - require.NoError(t, err) - - block2, err := s.CreateBlock(smartblock.SmartBlockTypePage) - require.NoError(t, err) - - details1 := &types.Struct{Fields: map[string]*types.Value{"name": structs.String("block1_name")}} - relations1 := &pbrelation.Relations{Relations: []*pbrelation.Relation{bundle.MustGetRelation(bundle.RelationKeyName), bundle.MustGetRelation(bundle.RelationKeyLastModifiedDate)}} - blocks1 := []*model.Block{ - { - Id: "test_id1", - Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "Kademlia is a distributed hash table for decentralized peer-to-peer computer networks designed by Petar Maymounkov and David Mazières in 2002.[1][2] It specifies the structure of the network and the exchange of information through node lookups. Kademlia nodes communicate among themselves using UDP. A virtual or overlay network is formed by the participant nodes. Each node is identified by a number or node ID. The node ID serves not only as identification, but the Kademlia algorithm uses the node ID to locate values (usually file hashes or keywords). In fact, the node ID provides a direct map to file hashes and that node stores information on where to obtain the file or resource."}}, - }, - } - err = block1.(*smartBlock).indexSnapshot(details1, relations1, blocks1) - require.NoError(t, err) - - details2 := &types.Struct{Fields: map[string]*types.Value{"name": structs.String("block2_name")}} - relations2 := &pbrelation.Relations{Relations: []*pbrelation.Relation{bundle.MustGetRelation(bundle.RelationKeyIconImage)}} - - blocks2 := []*model.Block{ - { - Id: "test_id2", - Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "Kademlia is a distributed hash table for decentralized peer-to-peer computer networks designed by Petar Maymounkov and David Mazières in 2002.[1][2] It specifies the structure of the network and the exchange of information through node lookups. Kademlia nodes communicate among themselves using UDP. A virtual or overlay network is formed by the participant nodes. Each node is identified by a number or node ID. The node ID serves not only as identification, but the Kademlia algorithm uses the node ID to locate values (usually file hashes or keywords). In fact, the node ID provides a direct map to file hashes and that node stores information on where to obtain the file or resource."}}, - }, - } - - err = block2.(*smartBlock).indexSnapshot(details2, relations2, blocks2) - require.NoError(t, err) - - var ps = s.ObjectStore() - sch := schema.New(bundle.MustGetType(bundle.TypeKeyPage), nil) - results, total, err := ps.Query(&sch, database.Query{Limit: 1, Sorts: []*model.BlockContentDataviewSort{{RelationKey: "name"}}}) - require.NoError(t, err) - require.Len(t, results, 1) - require.Equal(t, 2, total) - require.Equal(t, details1.Fields["name"].GetStringValue(), results[0].Details.Fields["name"].GetStringValue()) - require.Equal(t, block1.ID(), results[0].Details.Fields["id"].GetStringValue()) - - results, total, err = ps.Query(&sch, database.Query{Limit: 10, Filters: []*model.BlockContentDataviewFilter{{ - Operator: model.BlockContentDataviewFilter_And, - RelationKey: "name", - Condition: model.BlockContentDataviewFilter_Like, - Value: structs.String("lock1"), - }}, - - Sorts: []*model.BlockContentDataviewSort{{RelationKey: "name"}}}) - - require.NoError(t, err) - require.Len(t, results, 1) - require.Equal(t, total, 1) - - n := time.Now() - nowTruncatedToDay := time.Date(n.Year(), n.Month(), n.Day(), 0, 0, 0, 0, n.Location()) - - details1.Fields["lastOpenedDate"] = pbtypes.Float64(float64(time.Now().Unix())) - err = ps.UpdateObjectDetails(block1.ID(), details1, relations1) - require.NoError(t, err) - - results, total, err = ps.Query(&sch, database.Query{Limit: 10, Filters: []*model.BlockContentDataviewFilter{{ - Operator: model.BlockContentDataviewFilter_And, - RelationKey: "lastOpenedDate", - Condition: model.BlockContentDataviewFilter_Equal, - Value: structs.Float64(float64(nowTruncatedToDay.Unix())), - }}, - }) - require.NoError(t, err) - require.Len(t, results, 1) - require.Equal(t, total, 1) - - details1.Fields["lastModifiedDate"] = pbtypes.Float64(float64(time.Now().Unix())) - details2.Fields["lastModifiedDate"] = pbtypes.Float64(float64(time.Now().Unix())) - err = ps.UpdateObjectDetails(block1.ID(), details1, relations1) - require.NoError(t, err) - - err = ps.UpdateObjectDetails(block2.ID(), details2, relations2) - require.NoError(t, err) - - results, total, err = ps.Query(&sch, database.Query{Limit: 10, Filters: []*model.BlockContentDataviewFilter{{ - Operator: model.BlockContentDataviewFilter_And, - RelationKey: "lastModifiedDate", - Condition: model.BlockContentDataviewFilter_Equal, - Value: structs.Float64(float64(nowTruncatedToDay.Unix())), - }}, - }) - require.NoError(t, err) - require.Len(t, results, 2) - require.Equal(t, total, 2) - - nextDay := time.Date(n.Year(), n.Month(), n.Day()+1, 0, 0, 0, 0, time.UTC) - - results, total, err = ps.Query(&sch, database.Query{Limit: 10, Filters: []*model.BlockContentDataviewFilter{{ - Operator: model.BlockContentDataviewFilter_And, - RelationKey: "lastModifiedDate", - Condition: model.BlockContentDataviewFilter_Less, - Value: structs.Float64(float64(nextDay.Unix())), - }}, - }) - require.NoError(t, err) - require.Len(t, results, 2) - require.Equal(t, total, 2) - - prevDay := time.Date(n.Year(), n.Month(), n.Day()-1, 0, 0, 0, 0, time.UTC) - - results, total, err = ps.Query(&sch, database.Query{Limit: 10, Filters: []*model.BlockContentDataviewFilter{{ - Operator: model.BlockContentDataviewFilter_And, - RelationKey: "lastModifiedDate", - Condition: model.BlockContentDataviewFilter_Greater, - Value: structs.Float64(float64(prevDay.Unix())), - }}, - }) - require.NoError(t, err) - require.Len(t, results, 2) - require.Equal(t, total, 2) - - results, total, err = ps.Query(&sch, database.Query{Limit: 10, Filters: []*model.BlockContentDataviewFilter{{ - Operator: model.BlockContentDataviewFilter_And, - RelationKey: "lastModifiedDate", - Condition: model.BlockContentDataviewFilter_Greater, - Value: structs.Float64(float64(nextDay.Unix())), - }}, - }) - require.NoError(t, err) - require.Len(t, results, 0) - require.Equal(t, total, 0) -} - -func TestAnytype_PredefinedBlocks(t *testing.T) { - s := getRunningService(t) - require.NotNil(t, s) - - err := s.InitPredefinedBlocks(context.Background(), false) - require.NoError(t, err) - - fmt.Printf("profile: %s\n", s.PredefinedBlocks().Profile) - fmt.Printf("home: %s\n", s.PredefinedBlocks().Home) - - require.Len(t, s.PredefinedBlocks().Home, 57) - require.Len(t, s.PredefinedBlocks().Profile, 57) - require.Len(t, s.PredefinedBlocks().Archive, 57) - - tid, err := threads.ProfileThreadIDFromAccountAddress(s.Account()) - require.NoError(t, err) - - require.Equal(t, s.PredefinedBlocks().Profile, tid.String()) -} - -func TestAnytype_CreateBlock(t *testing.T) { - s := getRunningService(t) - block, err := s.CreateBlock(smartblock.SmartBlockTypePage) - require.NoError(t, err) - require.Equal(t, block.Type(), smartblock.SmartBlockTypePage) - require.Len(t, block.ID(), 57) -} diff --git a/pkg/lib/core/files.go b/pkg/lib/core/files.go index a41068dd2..baaea8b2f 100644 --- a/pkg/lib/core/files.go +++ b/pkg/lib/core/files.go @@ -3,11 +3,11 @@ package core import ( "context" "fmt" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/filestore" "io" "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" "github.com/anytypeio/go-anytype-middleware/pkg/lib/files" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore" pbrelation "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/relation" ) @@ -18,20 +18,20 @@ func (a *Anytype) FileGetKeys(hash string) (*files.FileKeys, error) { } func (a *Anytype) FileStoreKeys(fileKeys ...files.FileKeys) error { - var fks []localstore.FileKeys + var fks []filestore.FileKeys for _, fk := range fileKeys { - fks = append(fks, localstore.FileKeys{ + fks = append(fks, filestore.FileKeys{ Hash: fk.Hash, Keys: fk.Keys, }) } - return a.localStore.Files.AddFileKeys(fks...) + return a.fileStore.AddFileKeys(fks...) } func (a *Anytype) FileByHash(ctx context.Context, hash string) (File, error) { - fileList, err := a.localStore.Files.ListByTarget(hash) + fileList, err := a.fileStore.ListByTarget(hash) if err != nil { return nil, err } @@ -80,7 +80,7 @@ func (a *Anytype) FileAdd(ctx context.Context, options ...files.AddOption) (File return nil, err } - err = a.localStore.Objects.UpdateObjectDetails(f.hash, details, &pbrelation.Relations{Relations: bundle.MustGetType(bundle.TypeKeyFile).Relations}) + err = a.objectStore.UpdateObjectDetails(f.hash, details, &pbrelation.Relations{Relations: bundle.MustGetType(bundle.TypeKeyFile).Relations}) if err != nil { return nil, err } diff --git a/pkg/lib/core/files_test.go b/pkg/lib/core/files_test.go deleted file mode 100644 index 28cb19e1d..000000000 --- a/pkg/lib/core/files_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package core - -import ( - "bytes" - "context" - "encoding/json" - "io/ioutil" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -func TestAnytype_FileByHash(t *testing.T) { - s := getRunningService(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - nf, err := s.FileAddWithBytes(ctx, []byte("123"), "file.txt") - require.NoError(t, err) - require.Len(t, nf.Hash(), 59) - - f, err := s.FileByHash(ctx, nf.Hash()) - require.NoError(t, err) - require.Equal(t, nf.Hash(), f.Hash()) - - fr, err := f.Reader() - require.NoError(t, err) - - fb, err := ioutil.ReadAll(fr) - require.NoError(t, err) - require.Equal(t, fb, []byte("123")) - - require.NotNil(t, f.Meta()) - require.Equal(t, "file.txt", f.Meta().Name) - require.Equal(t, int64(3), f.Meta().Size) -} - -func Test_smartBlock_FileKeysRestore(t *testing.T) { - s := getRunningService(t) - - f, err := s.FileAddWithReader(context.Background(), bytes.NewReader([]byte("123")), "test") - require.NoError(t, err) - - keys, err := s.(*Anytype).localStore.Files.GetFileKeys(f.Hash()) - require.NoError(t, err) - - keysExpectedJson, _ := json.Marshal(keys) - err = s.(*Anytype).localStore.Files.DeleteFileKeys(f.Hash()) - require.NoError(t, err) - - keysActual, err := s.(*Anytype).files.FileRestoreKeys(context.Background(), f.Hash()) - require.NoError(t, err) - - keysActualJson, _ := json.Marshal(keysActual) - require.Equal(t, keysExpectedJson, keysActualJson) -} diff --git a/pkg/lib/core/images.go b/pkg/lib/core/images.go index c1e9cb0d2..97050eb0c 100644 --- a/pkg/lib/core/images.go +++ b/pkg/lib/core/images.go @@ -14,7 +14,7 @@ import ( var ErrImageNotFound = fmt.Errorf("image not found") func (a *Anytype) ImageByHash(ctx context.Context, hash string) (Image, error) { - files, err := a.localStore.Files.ListByTarget(hash) + files, err := a.fileStore.ListByTarget(hash) if err != nil { return nil, err } @@ -74,7 +74,7 @@ func (a *Anytype) ImageAdd(ctx context.Context, options ...files.AddOption) (Ima return nil, err } - err = a.localStore.Objects.UpdateObjectDetails(img.hash, details, &pbrelation.Relations{Relations: bundle.MustGetType(bundle.TypeKeyImage).Relations}) + err = a.objectStore.UpdateObjectDetails(img.hash, details, &pbrelation.Relations{Relations: bundle.MustGetType(bundle.TypeKeyImage).Relations}) if err != nil { return nil, err } diff --git a/pkg/lib/core/images_test.go b/pkg/lib/core/images_test.go deleted file mode 100644 index 9f1138a6c..000000000 --- a/pkg/lib/core/images_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package core - -import ( - "context" - "encoding/json" - "io/ioutil" - "os" - "testing" - "time" - - "github.com/anytypeio/go-anytype-middleware/pkg/lib/files" - "github.com/stretchr/testify/require" -) - -func TestAnytype_ImageByHash(t *testing.T) { - s := getRunningService(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - fd, err := os.Open("../mill/testdata/image.jpeg") - require.NoError(t, err) - - nf, err := s.ImageAddWithReader(ctx, fd, "image.jpeg") - require.NoError(t, err) - require.Len(t, nf.Hash(), 59) - - f, err := s.ImageByHash(ctx, nf.Hash()) - require.NoError(t, err) - require.Equal(t, nf.Hash(), f.Hash()) - - flargest, err := f.GetFileForLargestWidth(ctx) - require.NoError(t, err) - - flargestr, err := flargest.Reader() - require.NoError(t, err) - - fb, err := ioutil.ReadAll(flargestr) - require.NoError(t, err) - require.True(t, len(fb) > 100) - - require.NotNil(t, flargest.Meta()) - require.Equal(t, "image.jpeg", flargest.Meta().Name) - require.Equal(t, int64(68648), flargest.Meta().Size) -} - -func TestAnytype_ImageByHashUnencrypted(t *testing.T) { - s := getRunningService(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - fd, err := os.Open("../mill/testdata/image.jpeg") - require.NoError(t, err) - - nf, err := s.ImageAdd(ctx, files.WithReader(fd), files.WithName("image.jpeg"), files.WithPlaintext(true)) - require.NoError(t, err) - require.Len(t, nf.Hash(), 59) - - f, err := s.ImageByHash(ctx, nf.Hash()) - require.NoError(t, err) - for _, variant := range f.(*image).variantsByWidth { - require.Equal(t, "", variant.Key) - } - - flargest, err := f.GetFileForLargestWidth(ctx) - require.NoError(t, err) - - require.Equal(t, "", flargest.(*file).info.Key) - - flargestr, err := flargest.Reader() - require.NoError(t, err) - - fb, err := ioutil.ReadAll(flargestr) - require.NoError(t, err) - require.True(t, len(fb) > 100) - - require.NotNil(t, flargest.Meta()) - require.Equal(t, "image.jpeg", flargest.Meta().Name) - require.Equal(t, int64(68648), flargest.Meta().Size) -} - -func TestAnytype_ImageFileKeysRestore(t *testing.T) { - s := getRunningService(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - fd, err := os.Open("../mill/testdata/image.png") - require.NoError(t, err) - - nf, err := s.ImageAddWithReader(ctx, fd, "image.jpeg") - require.NoError(t, err) - require.Len(t, nf.Hash(), 59) - - keys, err := s.(*Anytype).localStore.Files.GetFileKeys(nf.Hash()) - require.NoError(t, err) - - keysExpectedJson, _ := json.Marshal(keys) - err = s.(*Anytype).localStore.Files.DeleteFileKeys(nf.Hash()) - require.NoError(t, err) - - keysActual, err := s.(*Anytype).files.FileRestoreKeys(context.Background(), nf.Hash()) - require.NoError(t, err) - - keysActualJson, _ := json.Marshal(keysActual) - require.Equal(t, keysExpectedJson, keysActualJson) -} diff --git a/pkg/lib/core/migration.go b/pkg/lib/core/migration.go index 6886df2fd..f678b2241 100644 --- a/pkg/lib/core/migration.go +++ b/pkg/lib/core/migration.go @@ -89,8 +89,8 @@ func (a *Anytype) saveCurrentRepoVersion() error { func (a *Anytype) runMigrationsUnsafe() error { // todo: FIXME refactoring - if _, err := os.Stat(filepath.Join(a.wallet.RepoPath(), "ipfslite")); os.IsNotExist(err) { - log.Debugf("repo is not inited, save all migrations as done") + if a.config.NewAccount { + log.Debugf("new account") return a.saveCurrentRepoVersion() } @@ -142,13 +142,6 @@ func doWithRunningNode(a *Anytype, offline bool, stopAfter bool, f func() error) }() a.opts.Offline = offline*/ - if !a.isStarted { - err := a.start() - if err != nil { - return err - } - } - var err error if stopAfter { defer func() { @@ -290,7 +283,7 @@ func alterThreadsDbSchema(a *Anytype, _ bool) error { func addFilesMetaHash(a *Anytype, lastMigration bool) error { // todo: better split into 2 migrations return doWithRunningNode(a, true, !lastMigration, func() error { - files, err := a.localStore.Files.List() + files, err := a.fileStore.List() if err != nil { return err } @@ -328,7 +321,7 @@ func addFilesMetaHash(a *Anytype, lastMigration bool) error { func addFilesToObjects(a *Anytype, lastMigration bool) error { // todo: better split into 2 migrations return doWithRunningNode(a, true, !lastMigration, func() error { - files, err := a.localStore.Files.List() + files, err := a.fileStore.List() if err != nil { return err } @@ -369,7 +362,7 @@ func addFilesToObjects(a *Anytype, lastMigration bool) error { continue } - err = a.localStore.Objects.UpdateObjectDetails(img.Hash(), details, &pbrelation.Relations{Relations: imgObjType.Relations}) + err = a.objectStore.UpdateObjectDetails(img.Hash(), details, &pbrelation.Relations{Relations: imgObjType.Relations}) if err != nil { // this shouldn't fail cancel() @@ -404,7 +397,7 @@ func addFilesToObjects(a *Anytype, lastMigration bool) error { continue } - err = a.localStore.Objects.UpdateObjectDetails(file.Hash(), details, &pbrelation.Relations{Relations: fileObjType.Relations}) + err = a.objectStore.UpdateObjectDetails(file.Hash(), details, &pbrelation.Relations{Relations: fileObjType.Relations}) if err != nil { cancel() return err @@ -428,7 +421,7 @@ func removeBundleRelationsFromDs(a *Anytype, lastMigration bool) error { keys := bundle.ListRelationsKeys() var migrated int for _, key := range keys { - err := a.localStore.Objects.RemoveRelationFromCache(key.String()) + err := a.objectStore.RemoveRelationFromCache(key.String()) if err != nil { continue } @@ -443,7 +436,7 @@ func removeBundleRelationsFromDs(a *Anytype, lastMigration bool) error { } func ReindexAll(a *Anytype) (int, error) { - ids, err := a.localStore.Objects.ListIds() + ids, err := a.objectStore.ListIds() if err != nil { return 0, err } @@ -456,14 +449,14 @@ func ReindexAll(a *Anytype) (int, error) { } if sbt == smartblock.SmartBlockTypeArchive { // remove archive we have accidentally indexed - err = a.localStore.Objects.DeleteObject(id) + err = a.objectStore.DeleteObject(id) if err != nil { log.Errorf("migration reindexAll: failed to delete archive from index: %s", err.Error()) } total-- continue } - for _, idx := range a.localStore.Objects.Indexes() { + for _, idx := range a.objectStore.Indexes() { //if idx.Name == "objtype_relkey_setid" { // skip it because we can't reindex relations in sets for now // continue @@ -474,7 +467,7 @@ func ReindexAll(a *Anytype) (int, error) { log.Errorf("migration reindexAll: failed to delete archive from index: %s", err.Error()) } } - oi, err := a.localStore.Objects.GetByIDs(id) + oi, err := a.objectStore.GetByIDs(id) if err != nil { log.Errorf("migration reindexAll: failed to get objects by id: %s", err.Error()) continue @@ -502,7 +495,7 @@ func ReindexAll(a *Anytype) (int, error) { } if sbt == smartblock.SmartBlockTypeIndexedRelation { - err = a.localStore.Objects.DeleteObject(id) + err = a.objectStore.DeleteObject(id) if err != nil { log.Errorf("deletion of indexed relation failed: %s", err.Error()) } @@ -511,14 +504,14 @@ func ReindexAll(a *Anytype) (int, error) { } o.Details.Fields[bundle.RelationKeyType.String()] = pbtypes.String(objType) - err = a.localStore.Objects.CreateObject(id, o.Details, o.Relations, nil, o.Snippet) + err = a.objectStore.CreateObject(id, o.Details, o.Relations, nil, o.Snippet) if err != nil { log.Errorf("migration reindexAll: createObject failed: %s", err.Error()) continue } migrated++ } - relations, _ := a.localStore.Objects.ListRelations("") + relations, _ := a.objectStore.ListRelations("") for _, rel := range relations { if bundle.HasRelation(rel.Key) { rel.Creator = a.ProfileID() @@ -539,7 +532,7 @@ func ReindexAll(a *Anytype) (int, error) { divided = append(divided, relations[i:end]) } for _, chunk := range divided { - err = a.localStore.Objects.StoreRelations(chunk) + err = a.objectStore.StoreRelations(chunk) if err != nil { log.Errorf("reindex relations failed: %s", err.Error()) } else { @@ -563,7 +556,7 @@ func removeIncorrectlyIndexedRelations(a *Anytype, lastMigration bool) error { var err error for _, rk := range bundle.ListRelationsKeys() { // remove accidentally indexed bundled relations with custom relation prefix - err = a.ObjectStore().DeleteObject(addr.CustomRelationURLPrefix + rk.String()) + err = a.objectStore.DeleteObject(addr.CustomRelationURLPrefix + rk.String()) if err != nil { log.Errorf("migration reindexAll: failed to delete archive from index: %s", err.Error()) } @@ -581,7 +574,7 @@ func reindexAll(a *Anytype, lastMigration bool) error { func reindexStoredRelations(a *Anytype, lastMigration bool) error { return doWithRunningNode(a, true, !lastMigration, func() error { - rels, err := a.localStore.Objects.ListRelations("") + rels, err := a.objectStore.ListRelations("") if err != nil { return err } @@ -615,20 +608,20 @@ func reindexStoredRelations(a *Anytype, lastMigration bool) error { } } - return a.localStore.Objects.StoreRelations(rels) + return a.objectStore.StoreRelations(rels) }) } func addMissingLayout(a *Anytype, lastMigration bool) error { return doWithRunningNode(a, true, !lastMigration, func() error { - ids, err := a.localStore.Objects.ListIds() + ids, err := a.objectStore.ListIds() if err != nil { return err } total := len(ids) var migrated int for _, id := range ids { - oi, err := a.localStore.Objects.GetByIDs(id) + oi, err := a.objectStore.GetByIDs(id) if err != nil { log.Errorf("migration addMissingLayout: failed to get objects by id: %s", err.Error()) continue @@ -665,7 +658,7 @@ func addMissingLayout(a *Anytype, lastMigration bool) error { layout = t.Layout } } else { - oi, err := a.localStore.Objects.GetByIDs(otUrl) + oi, err := a.objectStore.GetByIDs(otUrl) if err != nil { log.Errorf("migration addMissingLayout: failed to get objects type by id: %s", err.Error()) continue @@ -682,7 +675,7 @@ func addMissingLayout(a *Anytype, lastMigration bool) error { } o.Details.Fields[bundle.RelationKeyLayout.String()] = pbtypes.Float64(float64(layout)) - err = a.localStore.Objects.UpdateObjectDetails(id, o.Details, o.Relations) + err = a.objectStore.UpdateObjectDetails(id, o.Details, o.Relations) if err != nil { log.Errorf("migration addMissingLayout: failed to UpdateObject: %s", err.Error()) continue diff --git a/pkg/lib/core/pages.go b/pkg/lib/core/pages.go index 10981e2f4..234f78da3 100644 --- a/pkg/lib/core/pages.go +++ b/pkg/lib/core/pages.go @@ -1,22 +1,22 @@ package core import ( + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "sort" "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" ) // Deprecated, use localstore component directly -func (a *Anytype) ObjectStore() localstore.ObjectStore { - return a.localStore.Objects +func (a *Anytype) ObjectStore() objectstore.ObjectStore { + return a.objectStore } // deprecated, to be removed func (a *Anytype) ObjectInfoWithLinks(id string) (*model.ObjectInfoWithLinks, error) { - return a.localStore.Objects.GetWithLinksInfoByID(id) + return a.objectStore.GetWithLinksInfoByID(id) } // deprecated, to be removed @@ -42,7 +42,7 @@ func (a *Anytype) ObjectList() ([]*model.ObjectInfo, error) { idsS = append(idsS, id.String()) } - pages, err := a.localStore.Objects.GetByIDs(idsS...) + pages, err := a.objectStore.GetByIDs(idsS...) if err != nil { return nil, err } diff --git a/pkg/lib/core/pages_test.go b/pkg/lib/core/pages_test.go deleted file mode 100644 index 63ce06848..000000000 --- a/pkg/lib/core/pages_test.go +++ /dev/null @@ -1,168 +0,0 @@ -package core - -import ( - "testing" - - "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/structs" - "github.com/anytypeio/go-anytype-middleware/util/pbtypes" - "github.com/gogo/protobuf/types" - "github.com/stretchr/testify/require" -) - -func Test_Anytype_ObjectInfoWithLinks(t *testing.T) { - s := getRunningService(t) - block1, err := s.CreateBlock(smartblock.SmartBlockTypePage) - require.NoError(t, err) - - blockID := "test_id1" - - blockContent1 := "Kademlia is a distributed hash table for decentralized peer-to-peer computer networks designed by" + - " Petar Maymounkov and David Mazières in 2002.[1][2] It specifies the structure of the network and the exchange " + - "of information through node lookups. Kademlia nodes communicate among themselves using UDP. A virtual or overlay" + - " network is formed by the participant nodes. Each node is identified by a number or node ID. The node ID serves " + - "not only as identification, but the Kademlia algorithm uses the node ID to locate values (usually file hashes " + - "or keywords). In fact, the node ID provides a direct map to file hashes and that node stores information on " + - "where to obtain the file or resource." - details1 := &types.Struct{Fields: map[string]*types.Value{"name": structs.String("block1_name")}} - blocks1 := []*model.Block{ - { - Id: blockID, - Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: blockContent1}}, - }, - } - err = block1.(*smartBlock).indexSnapshot(details1, nil, blocks1) - require.NoError(t, err) - - block2, err := s.CreateBlock(smartblock.SmartBlockTypePage) - require.NoError(t, err) - - details2 := &types.Struct{Fields: map[string]*types.Value{"name": structs.String("block2_name")}} - blocks2 := []*model.Block{ - { - Id: blockID, - Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "test"}}, - }, - { - Id: blockID, - Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{TargetBlockId: block1.ID()}}, - }, - } - - err = block2.(*smartBlock).indexSnapshot(details2, nil, blocks2) - require.NoError(t, err) - - info2, err := s.ObjectInfoWithLinks(block2.ID()) - require.NoError(t, err) - require.NotNil(t, info2.Links) - require.NotNil(t, info2.Links.Outbound) - require.Len(t, info2.Links.Outbound, 1) - - require.Equal(t, block1.ID(), info2.Links.Outbound[0].Id) - details1.Fields["id"] = pbtypes.String(block1.ID()) - - require.True(t, info2.Links.Outbound[0].Details.Compare(details1) == 0) - - info1, err := s.ObjectInfoWithLinks(block1.ID()) - require.NoError(t, err) - require.NotNil(t, info1.Links) - require.Len(t, info1.Links.Inbound, 1) - - require.Equal(t, block2.ID(), info1.Links.Inbound[0].Id) - details2.Fields["id"] = pbtypes.String(block2.ID()) - - require.True(t, info1.Links.Inbound[0].Details.Compare(details2) == 0) - require.Equal(t, getSnippet(blocks1), info1.Info.Snippet) - - // test change of existing page index - blockContent2 := "newtext" - details2Modified := &types.Struct{Fields: map[string]*types.Value{"name": structs.String("block2_name_modified")}} - blocks2Modified := []*model.Block{ - { - Id: blockID, - Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: blockContent2}}, - }, - } - - details2Modified.Fields["id"] = pbtypes.String(block2.ID()) - err = block2.(*smartBlock).indexSnapshot(details2Modified, nil, blocks2Modified) - require.NoError(t, err) - - info2Modified, err := s.ObjectInfoWithLinks(block2.ID()) - require.NoError(t, err) - - info1Modified, err := s.ObjectInfoWithLinks(block1.ID()) - require.NoError(t, err) - - require.Len(t, info1Modified.Links.Inbound, 1) - require.Len(t, info2Modified.Links.Outbound, 1) - require.Equal(t, getSnippet(blocks2Modified), info2Modified.Info.Snippet) - require.True(t, details2Modified.Compare(info2Modified.Info.Details) == 0) - - err = s.DeleteBlock(block1.ID()) - require.NoError(t, err) - - info1Modified, err = s.ObjectInfoWithLinks(block1.ID()) - require.Error(t, err) - require.Nil(t, info1Modified) - - info2Modified, err = s.ObjectInfoWithLinks(block2.ID()) - require.NoError(t, err) - require.Len(t, info2Modified.Links.Outbound, 0) -} - -func Test_Anytype_PageList(t *testing.T) { - s := getRunningService(t) - block1, err := s.CreateBlock(smartblock.SmartBlockTypePage) - require.NoError(t, err) - - blockID := "test_id1" - - details1 := &types.Struct{Fields: map[string]*types.Value{"name": structs.String("block1_name")}} - blocks1 := []*model.Block{ - { - Id: blockID, - Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "test"}}, - }, - } - err = block1.(*smartBlock).indexSnapshot(details1, nil, blocks1) - - require.NoError(t, err) - block2, err := s.CreateBlock(smartblock.SmartBlockTypePage) - require.NoError(t, err) - - details2 := &types.Struct{Fields: map[string]*types.Value{"name": structs.String("block2_name")}} - blocks2 := []*model.Block{ - { - Id: blockID, - Content: &model.BlockContentOfText{Text: &model.BlockContentText{Text: "test"}}, - }, - { - Id: blockID, - Content: &model.BlockContentOfLink{Link: &model.BlockContentLink{TargetBlockId: block1.ID()}}, - }, - } - - err = block2.(*smartBlock).indexSnapshot(details2, nil, blocks2) - require.NoError(t, err) - - pages, err := s.ObjectList() - require.NoError(t, err) - - var pageById = make(map[string]*model.ObjectInfo) - for _, page := range pages { - pageById[page.Id] = page - } - - require.NotNil(t, pageById[block1.ID()]) - details1.Fields["id"] = pbtypes.String(block1.ID()) - details2.Fields["id"] = pbtypes.String(block2.ID()) - - require.True(t, details1.Compare(pageById[block1.ID()].Details) == 0) - require.Equal(t, "test", pageById[block1.ID()].Snippet) - - require.Equal(t, block2.ID(), pageById[block2.ID()].Id) - require.Equal(t, details2, pageById[block2.ID()].Details) - require.Equal(t, "test", pageById[block2.ID()].Snippet) -} diff --git a/pkg/lib/core/profile.go b/pkg/lib/core/profile.go index a62bd570c..3c2a37a92 100644 --- a/pkg/lib/core/profile.go +++ b/pkg/lib/core/profile.go @@ -79,7 +79,7 @@ func (a *Anytype) LocalProfile() (Profile, error) { profileId = a.predefinedBlockIds.Profile ) - ps := a.localStore.Objects + ps := a.objectStore if ps == nil { return profile, errors.New("no pagestore available") } diff --git a/pkg/lib/core/smartblocks.go b/pkg/lib/core/smartblocks.go index cb5ce773f..e3233c487 100644 --- a/pkg/lib/core/smartblocks.go +++ b/pkg/lib/core/smartblocks.go @@ -31,7 +31,7 @@ func (a *Anytype) DeleteBlock(id string) error { return err } - if err = a.localStore.Objects.DeleteObject(id); err != nil { + if err = a.objectStore.DeleteObject(id); err != nil { return err } diff --git a/pkg/lib/core/smartblocks_test.go b/pkg/lib/core/smartblocks_test.go deleted file mode 100644 index 9a8bc9592..000000000 --- a/pkg/lib/core/smartblocks_test.go +++ /dev/null @@ -1 +0,0 @@ -package core diff --git a/pkg/lib/core/wallet_test.go b/pkg/lib/core/wallet_test.go deleted file mode 100644 index c9946d7f3..000000000 --- a/pkg/lib/core/wallet_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package core - -import ( - "fmt" - "io/ioutil" - "os" - - "github.com/stretchr/testify/require" -) - -func createAccount(t require.TestingT) Service { - mnemonic, err := WalletGenerateMnemonic(12) - fmt.Printf("mnemonic: %s\n", mnemonic) - require.NoError(t, err) - - account, err := WalletAccountAt(mnemonic, 0, "") - fmt.Printf("account 0: %s\n", account.Address()) - - require.NoError(t, err) - rootPath, err := ioutil.TempDir(os.TempDir(), "anytype_*") - require.NoError(t, err) - - rawSeed, err := account.Raw() - require.NoError(t, err) - - err = WalletInitRepo(rootPath, rawSeed) - require.NoError(t, err) - - var opts = []ServiceOption{WithRootPathAndAccount(rootPath, account.Address())} - - if os.Getenv("ANYTYPE_TEST_OFFLINE") == "1" { - opts = append(opts, WithOfflineMode(true)) - opts = append(opts, WithoutCafe()) - } - - anytype, err := New(opts...) - require.NoError(t, err) - - return anytype -} diff --git a/pkg/lib/datastore/clientds/clientds.go b/pkg/lib/datastore/clientds/clientds.go index 17ff86b58..f4474335f 100644 --- a/pkg/lib/datastore/clientds/clientds.go +++ b/pkg/lib/datastore/clientds/clientds.go @@ -115,14 +115,25 @@ func (r *datastore) Name() (name string) { } func (r *datastore) Close() (err error) { - err2 := r.logstoreDS.Close() - if err2 != nil { - err = multierror.Append(err, err2) + if r.logstoreDS != nil { + err2 := r.logstoreDS.Close() + if err2 != nil { + err = multierror.Append(err, err2) + } } - err2 = r.litestoreDS.Close() - if err2 != nil { - err = multierror.Append(err, err2) + if r.litestoreDS != nil { + err2 := r.litestoreDS.Close() + if err2 != nil { + err = multierror.Append(err, err2) + } + } + + if r.threadsDbDS != nil { + err2 := r.threadsDbDS.Close() + if err2 != nil { + err = multierror.Append(err, err2) + } } return err diff --git a/pkg/lib/files/files.go b/pkg/lib/files/files.go index 8f486d1cb..b3c3bfd6d 100644 --- a/pkg/lib/files/files.go +++ b/pkg/lib/files/files.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "fmt" "github.com/anytypeio/go-anytype-middleware/app" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/filestore" "io" "io/ioutil" "strconv" @@ -45,14 +46,14 @@ var ErrorFailedToUnmarhalNotencrypted = fmt.Errorf("failed to unmarshal not-encr var _ app.Component = (*Service)(nil) type Service struct { - store localstore.FileStore + store filestore.FileStore ipfs ipfs.IPFS pins pin.FilePinService } func (s *Service) Init(a *app.App) (err error) { s.ipfs = a.MustComponent("ipfs").(ipfs.Node).GetIpfs() - s.store = a.MustComponent("localstore").(*localstore.LocalStore).Files + s.store = a.MustComponent("filestore").(filestore.FileStore) s.pins = a.MustComponent(pin.CName).(pin.FilePinService) return nil } @@ -98,7 +99,7 @@ func (s *Service) FileAdd(ctx context.Context, opts AddOptions) (string, *storag return "", nil, err } - if err = s.store.AddFileKeys(localstore.FileKeys{ + if err = s.store.AddFileKeys(filestore.FileKeys{ Hash: nodeHash, Keys: keys.KeysByPath, }); err != nil { @@ -162,7 +163,7 @@ func (s *Service) FileRestoreKeys(ctx context.Context, hash string) (map[string] } } - err = s.store.AddFileKeys(localstore.FileKeys{ + err = s.store.AddFileKeys(filestore.FileKeys{ Hash: hash, Keys: fileKeys, }) diff --git a/pkg/lib/files/images.go b/pkg/lib/files/images.go index da5e7b61d..a79dbc74f 100644 --- a/pkg/lib/files/images.go +++ b/pkg/lib/files/images.go @@ -3,8 +3,8 @@ package files import ( "context" "fmt" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/filestore" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/mill/schema/anytype" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/storage" ) @@ -22,7 +22,7 @@ func (s *Service) ImageAdd(ctx context.Context, opts AddOptions) (string, map[in nodeHash := node.Cid().String() - err = s.store.AddFileKeys(localstore.FileKeys{ + err = s.store.AddFileKeys(filestore.FileKeys{ Hash: nodeHash, Keys: keys.KeysByPath, }) diff --git a/pkg/lib/ipfs/helpers/helpers.go b/pkg/lib/ipfs/helpers/helpers.go index 5d8f3f104..b87577106 100644 --- a/pkg/lib/ipfs/helpers/helpers.go +++ b/pkg/lib/ipfs/helpers/helpers.go @@ -405,30 +405,27 @@ func ResolveLinkByNames(nd ipld.Node, names []string) (*ipld.Link, error) { } func PermanentConnection(ctx context.Context, addr ma.Multiaddr, host host.Host, retryInterval time.Duration) error { - pidStr, _ := addr.ValueForProtocol(ma.P_P2P) - pid, err := peer.Decode(pidStr) + addrInfo, err := peer.AddrInfoFromP2pAddr(addr) if err != nil { return fmt.Errorf("PermanentConnection invalid addr: %s", err.Error()) } - log.Errorf("PermanentConnection start %s", pid) + log.Errorf("PermanentConnection start %v", addrInfo.String()) go func() { for { - state := host.Network().Connectedness(pid) + state := host.Network().Connectedness(addrInfo.ID) // do not handle CanConnect purposefully if state == network.NotConnected || state == network.CannotConnect { if swrm, ok := host.Network().(*swarm.Swarm); ok { // clear backoff in order to connect more aggressively - swrm.Backoff().Clear(pid) + swrm.Backoff().Clear(addrInfo.ID) } - err = host.Connect(ctx, peer.AddrInfo{ - ID: pid, - Addrs: []ma.Multiaddr{addr}, - }) + + err = host.Connect(ctx, *addrInfo) if err != nil { log.Warnf("PermanentConnection failed: %s", err.Error()) } else { - log.Debugf("PermanentConnection %s reconnected succesfully", pid.String()) + log.Debugf("PermanentConnection %s reconnected succesfully", addrInfo.ID.String()) } } diff --git a/pkg/lib/localstore/files.go b/pkg/lib/localstore/filestore/files.go similarity index 73% rename from pkg/lib/localstore/files.go rename to pkg/lib/localstore/filestore/files.go index a49f328eb..b8623884d 100644 --- a/pkg/lib/localstore/files.go +++ b/pkg/lib/localstore/filestore/files.go @@ -1,7 +1,11 @@ -package localstore +package filestore import ( "fmt" + "github.com/anytypeio/go-anytype-middleware/app" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/datastore" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "sync" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/storage" @@ -17,26 +21,24 @@ var ( filesInfoBase = ds.NewKey("/" + filesPrefix + "/info") filesKeysBase = ds.NewKey("/" + filesPrefix + "/keys") - _ FileStore = (*dsFileStore)(nil) - - indexMillSourceOpts = Index{ + indexMillSourceOpts = localstore.Index{ Prefix: filesPrefix, Name: "mill_source_opts", - Keys: func(val interface{}) []IndexKeyParts { + Keys: func(val interface{}) []localstore.IndexKeyParts { if v, ok := val.(*storage.FileInfo); ok { - return []IndexKeyParts{[]string{v.Mill, v.Source, v.Opts}} + return []localstore.IndexKeyParts{[]string{v.Mill, v.Source, v.Opts}} } return nil }, Unique: true, } - indexTargets = Index{ + indexTargets = localstore.Index{ Prefix: filesPrefix, Name: "targets", - Keys: func(val interface{}) []IndexKeyParts { + Keys: func(val interface{}) []localstore.IndexKeyParts { if v, ok := val.(*storage.FileInfo); ok { - var keys []IndexKeyParts + var keys []localstore.IndexKeyParts for _, target := range v.Targets { keys = append(keys, []string{target}) } @@ -48,12 +50,12 @@ var ( Unique: true, } - indexMillChecksum = Index{ + indexMillChecksum = localstore.Index{ Prefix: filesPrefix, Name: "mill_checksum", - Keys: func(val interface{}) []IndexKeyParts { + Keys: func(val interface{}) []localstore.IndexKeyParts { if v, ok := val.(*storage.FileInfo); ok { - return []IndexKeyParts{[]string{v.Mill, v.Checksum}} + return []localstore.IndexKeyParts{[]string{v.Mill, v.Checksum}} } return nil }, @@ -66,23 +68,54 @@ type dsFileStore struct { l sync.Mutex } +var log = logging.Logger("anytype-localstore") + +const CName = "filestore" + +type FileStore interface { + app.Component + localstore.Indexable + Add(file *storage.FileInfo) error + AddMulti(upsert bool, files ...*storage.FileInfo) error + AddFileKeys(fileKeys ...FileKeys) error + GetFileKeys(hash string) (map[string]string, error) + GetByHash(hash string) (*storage.FileInfo, error) + GetBySource(mill string, source string, opts string) (*storage.FileInfo, error) + GetByChecksum(mill string, checksum string) (*storage.FileInfo, error) + AddTarget(hash string, target string) error + RemoveTarget(hash string, target string) error + ListTargets() ([]string, error) + ListByTarget(target string) ([]*storage.FileInfo, error) + Count() (int, error) + DeleteByHash(hash string) error + DeleteFileKeys(hash string) error + List() ([]*storage.FileInfo, error) +} + +func New() FileStore { + return &dsFileStore{} +} + +func (ls *dsFileStore) Init(a *app.App) (err error) { + ls.ds = a.MustComponent(datastore.CName).(datastore.Datastore).LocalstoreDS() + return nil +} + +func (ls *dsFileStore) Name() (name string) { + return CName +} + type FileKeys struct { Hash string Keys map[string]string } -func NewFileStore(ds ds.TxnDatastore) FileStore { - return &dsFileStore{ - ds: ds, - } -} - func (m *dsFileStore) Prefix() string { return "files" } -func (m *dsFileStore) Indexes() []Index { - return []Index{ +func (m *dsFileStore) Indexes() []localstore.Index { + return []localstore.Index{ indexMillChecksum, indexMillSourceOpts, indexTargets, @@ -104,7 +137,7 @@ func (m *dsFileStore) Add(file *storage.FileInfo) error { return err } if exists { - return ErrDuplicateKey + return localstore.ErrDuplicateKey } b, err := proto.Marshal(file) @@ -117,7 +150,7 @@ func (m *dsFileStore) Add(file *storage.FileInfo) error { return err } - err = AddIndexesWithTxn(m, txn, file, file.Hash) + err = localstore.AddIndexesWithTxn(m, txn, file, file.Hash) if err != nil { return err } @@ -153,7 +186,7 @@ func (m *dsFileStore) AddMulti(upsert bool, files ...*storage.FileInfo) error { return err } - err = AddIndexesWithTxn(m, txn, file, file.Hash) + err = localstore.AddIndexesWithTxn(m, txn, file, file.Hash) if err != nil { return err } @@ -172,7 +205,7 @@ func (m *dsFileStore) AddFileKeys(fileKeys ...FileKeys) error { for _, fk := range fileKeys { err = m.addSingleFileKeys(txn, fk.Hash, fk.Keys) if err != nil { - if err == ErrDuplicateKey { + if err == localstore.ErrDuplicateKey { continue } return err @@ -205,7 +238,7 @@ func (m *dsFileStore) addSingleFileKeys(txn ds.Txn, hash string, keys map[string return err } if exists { - return ErrDuplicateKey + return localstore.ErrDuplicateKey } b, err := proto.Marshal(&storage.FileKeys{ @@ -230,7 +263,7 @@ func (m *dsFileStore) GetFileKeys(hash string) (map[string]string, error) { b, err := txn.Get(fileKeysKey) if err != nil { if err == ds.ErrNotFound { - return nil, ErrNotFound + return nil, localstore.ErrNotFound } return nil, err } @@ -269,7 +302,7 @@ func (m *dsFileStore) AddTarget(hash string, target string) error { } fileInfoKey := filesInfoBase.ChildString(file.Hash) - err = AddIndex(indexTargets, m.ds, file, file.Hash) + err = localstore.AddIndex(indexTargets, m.ds, file, file.Hash) if err != nil { return err } @@ -314,7 +347,7 @@ func (m *dsFileStore) GetByHash(hash string) (*storage.FileInfo, error) { b, err := m.ds.Get(fileInfoKey) if err != nil { if err == ds.ErrNotFound { - return nil, ErrNotFound + return nil, localstore.ErrNotFound } return nil, err } @@ -334,7 +367,7 @@ func (m *dsFileStore) GetByChecksum(mill string, checksum string) (*storage.File } defer txn.Discard() - key, err := GetKeyByIndex(indexMillChecksum, txn, &storage.FileInfo{Mill: mill, Checksum: checksum}) + key, err := localstore.GetKeyByIndex(indexMillChecksum, txn, &storage.FileInfo{Mill: mill, Checksum: checksum}) if err != nil { return nil, err } @@ -360,7 +393,7 @@ func (m *dsFileStore) GetBySource(mill string, source string, opts string) (*sto } defer txn.Discard() - key, err := GetKeyByIndex(indexMillSourceOpts, txn, &storage.FileInfo{Mill: mill, Source: source, Opts: opts}) + key, err := localstore.GetKeyByIndex(indexMillSourceOpts, txn, &storage.FileInfo{Mill: mill, Source: source, Opts: opts}) if err != nil { return nil, err } @@ -386,21 +419,21 @@ func (m *dsFileStore) ListTargets() ([]string, error) { } defer txn.Discard() - targetPrefix := indexBase.ChildString(indexTargets.Prefix).ChildString(indexTargets.Name).String() + targetPrefix := localstore.IndexBase.ChildString(indexTargets.Prefix).ChildString(indexTargets.Name).String() - res, err := GetKeys(txn, targetPrefix, 0) + res, err := localstore.GetKeys(txn, targetPrefix, 0) if err != nil { return nil, err } - keys, err := ExtractKeysFromResults(res) + keys, err := localstore.ExtractKeysFromResults(res) if err != nil { return nil, err } var targets = make([]string, len(keys)) for i, key := range keys { - target, err := CarveKeyParts(key, -2, -1) + target, err := localstore.CarveKeyParts(key, -2, -1) if err != nil { return nil, err } @@ -417,12 +450,12 @@ func (m *dsFileStore) ListByTarget(target string) ([]*storage.FileInfo, error) { } defer txn.Discard() - results, err := GetKeysByIndexParts(txn, indexTargets.Prefix, indexTargets.Name, []string{target}, "", indexTargets.Hash, 0) + results, err := localstore.GetKeysByIndexParts(txn, indexTargets.Prefix, indexTargets.Name, []string{target}, "", indexTargets.Hash, 0) if err != nil { return nil, err } - keys, err := GetLeavesFromResults(results) + keys, err := localstore.GetLeavesFromResults(results) if err != nil { return nil, err } @@ -463,12 +496,12 @@ func (m *dsFileStore) List() ([]*storage.FileInfo, error) { } defer txn.Discard() - res, err := GetKeys(txn, filesInfoBase.String(), 0) + res, err := localstore.GetKeys(txn, filesInfoBase.String(), 0) if err != nil { return nil, err } - hashes, err := GetLeavesFromResults(res) + hashes, err := localstore.GetLeavesFromResults(res) if err != nil { return nil, err } @@ -491,7 +524,7 @@ func (m *dsFileStore) DeleteByHash(hash string) error { return fmt.Errorf("failed to find file by hash to remove") } - err = RemoveIndexes(m, m.ds, file, file.Hash) + err = localstore.RemoveIndexes(m, m.ds, file, file.Hash) if err != nil { return err } diff --git a/pkg/lib/localstore/ftsearch/ftsearch.go b/pkg/lib/localstore/ftsearch/ftsearch.go index 196874511..8ddc881a7 100644 --- a/pkg/lib/localstore/ftsearch/ftsearch.go +++ b/pkg/lib/localstore/ftsearch/ftsearch.go @@ -26,7 +26,7 @@ func New() FTSearch { } type FTSearch interface { - app.Component + app.ComponentRunnable Index(d SearchDoc) (err error) Search(query string) (results []string, err error) Delete(id string) error @@ -65,6 +65,11 @@ func (f *ftSearch) init() (err error) { return } +func (f *ftSearch) Run() (err error) { + // todo: move bleve init here? + return nil +} + func (f *ftSearch) Index(d SearchDoc) (err error) { return f.index.Index(d.Id, d) } diff --git a/pkg/lib/localstore/ftsearch/ftsearch_test.go b/pkg/lib/localstore/ftsearch/ftsearch_test.go index 9c0c59cda..f73865cad 100644 --- a/pkg/lib/localstore/ftsearch/ftsearch_test.go +++ b/pkg/lib/localstore/ftsearch/ftsearch_test.go @@ -1,19 +1,39 @@ package ftsearch import ( + "github.com/anytypeio/go-anytype-middleware/app/testapp" + "github.com/anytypeio/go-anytype-middleware/core/wallet" + "github.com/golang/mock/gomock" "io/ioutil" - "os" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +type fixture struct { + ft FTSearch + ta *testapp.TestApp + ctrl *gomock.Controller +} + +func newFixture(path string, t *testing.T) *fixture { + ft := New() + ta := testapp.New(). + With(wallet.NewWithRepoPathAndKeys(path, nil, nil)). + With(ft) + + require.NoError(t, ta.Start()) + return &fixture{ + ft: ft, + ta: ta, + } +} + func TestNewFTSearch(t *testing.T) { tmpDir, _ := ioutil.TempDir("", "") - defer os.RemoveAll(tmpDir) - ft, err := NewFTSearch(tmpDir) - require.NoError(t, err) + fixture := newFixture(tmpDir, t) + ft := fixture.ft require.NoError(t, ft.Index(SearchDoc{ Id: "test", Title: "one", @@ -23,7 +43,9 @@ func TestNewFTSearch(t *testing.T) { require.NoError(t, err) assert.Len(t, res, 1) ft.Close() - ft, err = NewFTSearch(tmpDir) + fixture = newFixture(tmpDir, t) + ft = fixture.ft + require.NoError(t, err) res, err = ft.Search("one") require.NoError(t, err) @@ -33,9 +55,8 @@ func TestNewFTSearch(t *testing.T) { func TestFtSearch_Search(t *testing.T) { tmpDir, _ := ioutil.TempDir("", "") - defer os.RemoveAll(tmpDir) - ft, err := NewFTSearch(tmpDir) - require.NoError(t, err) + fixture := newFixture(tmpDir, t) + ft := fixture.ft defer ft.Close() var docs = [...]SearchDoc{ { diff --git a/pkg/lib/localstore/objects.go b/pkg/lib/localstore/objectstore/objects.go similarity index 88% rename from pkg/lib/localstore/objects.go rename to pkg/lib/localstore/objectstore/objects.go index 2fde2d968..78a5e9b84 100644 --- a/pkg/lib/localstore/objects.go +++ b/pkg/lib/localstore/objectstore/objects.go @@ -1,9 +1,14 @@ -package localstore +package objectstore import ( "encoding/binary" "fmt" + "github.com/anytypeio/go-anytype-middleware/app" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/datastore" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/addr" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" + "github.com/anytypeio/go-anytype-middleware/util/slice" "strings" "sync" "time" @@ -23,6 +28,10 @@ import ( "github.com/ipfs/go-datastore/query" ) +var log = logging.Logger("anytype-localstore") + +const CName = "objectstore" + var ( // ObjectInfo is stored in db key pattern: pagesPrefix = "pages" @@ -40,12 +49,12 @@ var ( // /relations/relations/: relation model relationsBase = ds.NewKey("/" + relationsPrefix + "/relations") - indexObjectTypeRelationObjectId = Index{ + indexObjectTypeRelationObjectId = localstore.Index{ Prefix: relationsPrefix, Name: "objtype_relkey_objid", - Keys: func(val interface{}) []IndexKeyParts { + Keys: func(val interface{}) []localstore.IndexKeyParts { if v, ok := val.(*relationObjectType); ok { - var indexes []IndexKeyParts + var indexes []localstore.IndexKeyParts for _, rk := range v.relationKeys { for _, ot := range v.objectTypes { otCompact, err := objTypeCompactEncode(ot) @@ -54,7 +63,7 @@ var ( continue } - indexes = append(indexes, IndexKeyParts([]string{otCompact, rk})) + indexes = append(indexes, localstore.IndexKeyParts([]string{otCompact, rk})) } } return indexes @@ -65,12 +74,12 @@ var ( SplitIndexKeyParts: true, } - indexObjectTypeRelationSetId = Index{ + indexObjectTypeRelationSetId = localstore.Index{ Prefix: relationsPrefix, Name: "objtype_relkey_setid", - Keys: func(val interface{}) []IndexKeyParts { + Keys: func(val interface{}) []localstore.IndexKeyParts { if v, ok := val.(*relationObjectType); ok { - var indexes []IndexKeyParts + var indexes []localstore.IndexKeyParts for _, rk := range v.relationKeys { for _, ot := range v.objectTypes { otCompact, err := objTypeCompactEncode(ot) @@ -79,7 +88,7 @@ var ( continue } - indexes = append(indexes, IndexKeyParts([]string{otCompact, rk})) + indexes = append(indexes, localstore.IndexKeyParts([]string{otCompact, rk})) } } return indexes @@ -90,12 +99,12 @@ var ( SplitIndexKeyParts: true, } - indexRelationOptionObject = Index{ + indexRelationOptionObject = localstore.Index{ Prefix: pagesPrefix, Name: "relkey_optid", - Keys: func(val interface{}) []IndexKeyParts { + Keys: func(val interface{}) []localstore.IndexKeyParts { if v, ok := val.(*pbrelation.Relation); ok { - var indexes []IndexKeyParts + var indexes []localstore.IndexKeyParts if v.Format != pbrelation.RelationFormat_tag && v.Format != pbrelation.RelationFormat_status { return nil } @@ -104,7 +113,7 @@ var ( } for _, opt := range v.SelectDict { - indexes = append(indexes, IndexKeyParts([]string{v.Key, opt.Id})) + indexes = append(indexes, localstore.IndexKeyParts([]string{v.Key, opt.Id})) } return indexes } @@ -114,24 +123,24 @@ var ( SplitIndexKeyParts: true, } - indexRelationObject = Index{ + indexRelationObject = localstore.Index{ Prefix: pagesPrefix, Name: "relkey", - Keys: func(val interface{}) []IndexKeyParts { + Keys: func(val interface{}) []localstore.IndexKeyParts { if v, ok := val.(*pbrelation.Relation); ok { - return []IndexKeyParts{[]string{v.Key}} + return []localstore.IndexKeyParts{[]string{v.Key}} } return nil }, Unique: false, } - indexFormatOptionObject = Index{ + indexFormatOptionObject = localstore.Index{ Prefix: pagesPrefix, Name: "format_relkey_optid", - Keys: func(val interface{}) []IndexKeyParts { + Keys: func(val interface{}) []localstore.IndexKeyParts { if v, ok := val.(*pbrelation.Relation); ok { - var indexes []IndexKeyParts + var indexes []localstore.IndexKeyParts if v.Format != pbrelation.RelationFormat_tag && v.Format != pbrelation.RelationFormat_status { return nil } @@ -140,7 +149,7 @@ var ( } for _, opt := range v.SelectDict { - indexes = append(indexes, IndexKeyParts([]string{v.Format.String(), v.Key, opt.Id})) + indexes = append(indexes, localstore.IndexKeyParts([]string{v.Format.String(), v.Key, opt.Id})) } return indexes } @@ -150,12 +159,12 @@ var ( SplitIndexKeyParts: true, } - indexObjectTypeObject = Index{ + indexObjectTypeObject = localstore.Index{ Prefix: pagesPrefix, Name: "type", - Keys: func(val interface{}) []IndexKeyParts { + Keys: func(val interface{}) []localstore.IndexKeyParts { if v, ok := val.(*model.ObjectDetails); ok { - var indexes []IndexKeyParts + var indexes []localstore.IndexKeyParts types := pbtypes.GetStringList(v.Details, bundle.RelationKeyType.String()) for _, ot := range types { @@ -164,7 +173,7 @@ var ( log.Errorf("type index construction error('%s'): %s", ot, err.Error()) continue } - indexes = append(indexes, IndexKeyParts([]string{otCompact})) + indexes = append(indexes, localstore.IndexKeyParts([]string{otCompact})) } return indexes } @@ -177,6 +186,56 @@ var ( _ ObjectStore = (*dsObjectStore)(nil) ) +func New() ObjectStore { + return &dsObjectStore{} +} + +func (ls *dsObjectStore) Init(a *app.App) (err error) { + ls.ds = a.MustComponent(datastore.CName).(datastore.Datastore).LocalstoreDS() + fts := a.Component(ftsearch.CName) + if fts == nil { + log.Warnf("init objectstore without fulltext") + } else { + ls.fts = fts.(ftsearch.FTSearch) + } + return nil +} + +func (ls *dsObjectStore) Name() (name string) { + return CName +} + +type ObjectStore interface { + app.Component + localstore.Indexable + database.Reader + + CreateObject(id string, details *types.Struct, relations *pbrelation.Relations, links []string, snippet string) error + UpdateObjectDetails(id string, details *types.Struct, relations *pbrelation.Relations) error + UpdateObjectLinksAndSnippet(id string, links []string, snippet string) error + + StoreRelations(relations []*pbrelation.Relation) error + + DeleteObject(id string) error + RemoveRelationFromCache(key string) error + + UpdateRelationsInSet(setId, objTypeBefore, objTypeAfter string, relationsBefore, relationsAfter *pbrelation.Relations) error + + GetWithLinksInfoByID(id string) (*model.ObjectInfoWithLinks, error) + GetWithOutboundLinksInfoById(id string) (*model.ObjectInfoWithOutboundLinks, error) + GetDetails(id string) (*model.ObjectDetails, error) + GetAggregatedOptions(relationKey string, relationFormat pbrelation.RelationFormat, objectType string) (options []*pbrelation.RelationOption, err error) + + GetByIDs(ids ...string) ([]*model.ObjectInfo, error) + List() ([]*model.ObjectInfo, error) + ListIds() ([]string, error) + + QueryObjectInfo(q database.Query, objectTypes []smartblock.SmartBlockType) (results []*model.ObjectInfo, total int, err error) + AddToIndexQueue(id string) error + IndexForEach(f func(id string, tm time.Time) error) error + FTSearch() ftsearch.FTSearch +} + type relationOption struct { relationKey string optionId string @@ -225,10 +284,6 @@ func (m *filterObjectTypes) Filter(e query.Entry) bool { return m.not } -func NewObjectStore(ds ds.TxnDatastore, fts ftsearch.FTSearch) ObjectStore { - return &dsObjectStore{ds: ds, fts: fts} -} - type dsObjectStore struct { // underlying storage ds ds.TxnDatastore @@ -245,12 +300,12 @@ func (m *dsObjectStore) AggregateObjectIdsByOptionForRelation(relationKey string txn, err := m.ds.NewTransaction(true) defer txn.Discard() - res, err := GetKeysByIndexParts(txn, pagesPrefix, indexRelationOptionObject.Name, []string{relationKey}, "/", false, 100) + res, err := localstore.GetKeysByIndexParts(txn, pagesPrefix, indexRelationOptionObject.Name, []string{relationKey}, "/", false, 100) if err != nil { return nil, err } - keys, err := ExtractKeysFromResults(res) + keys, err := localstore.ExtractKeysFromResults(res) if err != nil { return nil, err } @@ -258,11 +313,11 @@ func (m *dsObjectStore) AggregateObjectIdsByOptionForRelation(relationKey string objectsByOptionId = make(map[string][]string) for _, key := range keys { - optionId, err := CarveKeyParts(key, -2, -1) + optionId, err := localstore.CarveKeyParts(key, -2, -1) if err != nil { return nil, err } - objId, err := CarveKeyParts(key, -1, 0) + objId, err := localstore.CarveKeyParts(key, -1, 0) if err != nil { return nil, err } @@ -280,23 +335,23 @@ func (m *dsObjectStore) getAggregatedOptionsForFormat(format pbrelation.Relation txn, err := m.ds.NewTransaction(true) defer txn.Discard() - res, err := GetKeysByIndexParts(txn, pagesPrefix, indexFormatOptionObject.Name, []string{format.String()}, "/", false, 100) + res, err := localstore.GetKeysByIndexParts(txn, pagesPrefix, indexFormatOptionObject.Name, []string{format.String()}, "/", false, 100) if err != nil { return nil, err } - keys, err := ExtractKeysFromResults(res) + keys, err := localstore.ExtractKeysFromResults(res) if err != nil { return nil, err } var ex = make(map[string]struct{}) for _, key := range keys { - optionId, err := CarveKeyParts(key, -2, -1) + optionId, err := localstore.CarveKeyParts(key, -2, -1) if err != nil { return nil, err } - relKey, err := CarveKeyParts(key, -3, -2) + relKey, err := localstore.CarveKeyParts(key, -3, -2) if err != nil { return nil, err } @@ -727,12 +782,12 @@ func (m *dsObjectStore) AggregateRelationsFromObjectsOfType(objType string) ([]* if err != nil { return nil, fmt.Errorf("failed to encode object type '%s': %s", objType, err.Error()) } - res, err := GetKeysByIndexParts(txn, indexObjectTypeRelationObjectId.Prefix, indexObjectTypeRelationObjectId.Name, []string{objTypeCompact}, "/", false, 0) + res, err := localstore.GetKeysByIndexParts(txn, indexObjectTypeRelationObjectId.Prefix, indexObjectTypeRelationObjectId.Name, []string{objTypeCompact}, "/", false, 0) if err != nil { return nil, err } - relKeys, err := GetKeyPartFromResults(res, -2, -1, true) + relKeys, err := localstore.GetKeyPartFromResults(res, -2, -1, true) if err != nil { return nil, err } @@ -763,12 +818,12 @@ func (m *dsObjectStore) AggregateRelationsFromSetsOfType(objType string) ([]*pbr if err != nil { return nil, err } - res, err := GetKeysByIndexParts(txn, indexObjectTypeRelationSetId.Prefix, indexObjectTypeRelationSetId.Name, []string{objTypeCompact}, "/", false, 0) + res, err := localstore.GetKeysByIndexParts(txn, indexObjectTypeRelationSetId.Prefix, indexObjectTypeRelationSetId.Name, []string{objTypeCompact}, "/", false, 0) if err != nil { return nil, err } - relKeys, err := GetKeyPartFromResults(res, -2, -1, true) + relKeys, err := localstore.GetKeyPartFromResults(res, -2, -1, true) if err != nil { return nil, err } @@ -963,29 +1018,6 @@ func (m *dsObjectStore) GetByIDs(ids ...string) ([]*model.ObjectInfo, error) { return getObjectsInfo(txn, ids) } -func diffSlices(a, b []string) (removed []string, added []string) { - var amap = map[string]struct{}{} - var bmap = map[string]struct{}{} - - for _, item := range a { - amap[item] = struct{}{} - } - - for _, item := range b { - if _, exists := amap[item]; !exists { - added = append(added, item) - } - bmap[item] = struct{}{} - } - - for _, item := range a { - if _, exists := bmap[item]; !exists { - removed = append(removed, item) - } - } - return -} - func (m *dsObjectStore) CreateObject(id string, details *types.Struct, relations *pbrelation.Relations, links []string, snippet string) error { m.l.Lock() defer m.l.Unlock() @@ -1067,7 +1099,7 @@ func (m *dsObjectStore) UpdateObjectDetails(id string, details *types.Struct, re func (m *dsObjectStore) updateArchive(txn ds.Txn, id string, links []string) error { exLinks, _ := findOutboundLinks(txn, id) - removedLinks, addedLinks := diffSlices(exLinks, links) + removedLinks, addedLinks := slice.DifferenceRemovedAdded(exLinks, links) getCurrentDetails := func(id string) (*types.Struct, error) { det, err := m.GetDetails(id) if err != nil { @@ -1128,7 +1160,7 @@ func (m *dsObjectStore) updateObjectLinksAndSnippet(txn ds.Txn, id string, links var addedLinks, removedLinks []string exLinks, _ := findOutboundLinks(txn, id) - removedLinks, addedLinks = diffSlices(exLinks, links) + removedLinks, addedLinks = slice.DifferenceRemovedAdded(exLinks, links) if len(addedLinks) > 0 { for _, k := range pageLinkKeys(id, nil, addedLinks) { if err := txn.Put(k, nil); err != nil { @@ -1266,7 +1298,7 @@ func (m *dsObjectStore) updateDetails(txn ds.Txn, id string, oldDetails *model.O return err } - err = UpdateIndexesWithTxn(m, txn, oldDetails, newDetails, id) + err = localstore.UpdateIndexesWithTxn(m, txn, oldDetails, newDetails, id) if err != nil { return err } @@ -1401,13 +1433,13 @@ func (m *dsObjectStore) updateRelations(txn ds.Txn, objTypesBefore []string, obj } } - err := AddIndexesWithTxn(m, txn, relation, id) + err := localstore.AddIndexesWithTxn(m, txn, relation, id) if err != nil { return err } } - err := UpdateIndexWithTxn(indexObjectTypeRelationObjectId, txn, &relationObjectType{ + err := localstore.UpdateIndexWithTxn(indexObjectTypeRelationObjectId, txn, &relationObjectType{ relationKeys: pbtypes.GetRelationKeys(relationsBefore.Relations), objectTypes: objTypesBefore, }, &relationObjectType{ @@ -1431,7 +1463,7 @@ func (m *dsObjectStore) updateRelations(txn ds.Txn, objTypesBefore []string, obj } func (m *dsObjectStore) updateRelationsInSet(txn ds.Txn, setId, objTypesBefore, objTypesAfter string, relationsBefore, relationsAfter *pbrelation.Relations) error { - return UpdateIndexWithTxn(indexObjectTypeRelationSetId, txn, &relationObjectType{ + return localstore.UpdateIndexWithTxn(indexObjectTypeRelationSetId, txn, &relationObjectType{ relationKeys: pbtypes.GetRelationKeys(relationsBefore.Relations), objectTypes: []string{objTypesBefore}, }, &relationObjectType{ @@ -1449,8 +1481,8 @@ func (m *dsObjectStore) Prefix() string { return pagesPrefix } -func (m *dsObjectStore) Indexes() []Index { - return []Index{indexObjectTypeRelationObjectId, indexObjectTypeRelationSetId, indexRelationOptionObject, indexRelationObject, indexFormatOptionObject, indexObjectTypeObject} +func (m *dsObjectStore) Indexes() []localstore.Index { + return []localstore.Index{indexObjectTypeRelationObjectId, indexObjectTypeRelationSetId, indexRelationOptionObject, indexRelationObject, indexFormatOptionObject, indexObjectTypeObject} } func (m *dsObjectStore) FTSearch() ftsearch.FTSearch { @@ -1472,12 +1504,12 @@ func (m *dsObjectStore) makeFTSQuery(text string, dsq query.Query) (query.Query, } func (m *dsObjectStore) listIdsOfType(txn ds.Txn, ot string) ([]string, error) { - res, err := GetKeysByIndexParts(txn, pagesPrefix, indexObjectTypeObject.Name, []string{ot}, "", false, 100) + res, err := localstore.GetKeysByIndexParts(txn, pagesPrefix, indexObjectTypeObject.Name, []string{ot}, "", false, 100) if err != nil { return nil, err } - return GetLeavesFromResults(res) + return localstore.GetLeavesFromResults(res) } func (m *dsObjectStore) listRelationsKeys(txn ds.Txn) ([]string, error) { @@ -1540,18 +1572,18 @@ func isObjectBelongToType(txn ds.Txn, id, objType string) (bool, error) { return false, err } - return HasPrimaryKeyByIndexParts(txn, pagesPrefix, indexObjectTypeObject.Name, []string{objTypeCompact}, "", false, id) + return localstore.HasPrimaryKeyByIndexParts(txn, pagesPrefix, indexObjectTypeObject.Name, []string{objTypeCompact}, "", false, id) } func isRelationBelongToType(txn ds.Txn, relKey, objectType string) (bool, error) { - res, err := GetKeysByIndexParts(txn, pagesPrefix, indexRelationObject.Name, []string{relKey}, "", false, 0) + res, err := localstore.GetKeysByIndexParts(txn, pagesPrefix, indexRelationObject.Name, []string{relKey}, "", false, 0) if err != nil { return false, err } i := 0 for v := range res.Next() { i++ - objId, err := CarveKeyParts(v.Key, -1, 0) + objId, err := localstore.CarveKeyParts(v.Key, -1, 0) if err != nil { return false, err } @@ -1696,7 +1728,7 @@ func hasInboundLinks(txn ds.Txn, id string) (bool, error) { } // max is 1 - inboundLinks, err := CountAllKeysFromResults(inboundResults) + inboundLinks, err := localstore.CountAllKeysFromResults(inboundResults) return inboundLinks > 0, err } @@ -1720,7 +1752,7 @@ func findByPrefix(txn ds.Txn, prefix string, limit int) ([]string, error) { return nil, err } - return GetLeavesFromResults(results) + return localstore.GetLeavesFromResults(results) } func pageLinkKeys(id string, in []string, out []string) []ds.Key { diff --git a/pkg/lib/localstore/objects_test.go b/pkg/lib/localstore/objectstore/objects_test.go similarity index 85% rename from pkg/lib/localstore/objects_test.go rename to pkg/lib/localstore/objectstore/objects_test.go index fbfa3e92b..8d1a06264 100644 --- a/pkg/lib/localstore/objects_test.go +++ b/pkg/lib/localstore/objectstore/objects_test.go @@ -1,16 +1,19 @@ -package localstore +package objectstore import ( "fmt" + "github.com/anytypeio/go-anytype-middleware/app/testapp" + "github.com/anytypeio/go-anytype-middleware/core/anytype/config" + "github.com/anytypeio/go-anytype-middleware/core/wallet" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/datastore/clientds" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/threads" "io/ioutil" "os" - "path/filepath" "testing" "time" "github.com/anytypeio/go-anytype-middleware/pkg/lib/bundle" "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/threads" "github.com/anytypeio/go-anytype-middleware/pkg/lib/database" "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/ftsearch" "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" @@ -21,7 +24,6 @@ import ( ds "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" "github.com/ipfs/go-datastore/sync" - badger "github.com/ipfs/go-ds-badger" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/textileio/go-threads/core/thread" @@ -31,11 +33,11 @@ func TestDsObjectStore_IndexQueue(t *testing.T) { tmpDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(tmpDir) - bds, err := badger.NewDatastore(tmpDir, nil) + app := testapp.New() + ds := New() + err := app.With(&config.DefaultConfig).With(wallet.NewWithRepoPathAndKeys(tmpDir, nil, nil)).With(clientds.New()).With(ds).Start() require.NoError(t, err) - ds := NewObjectStore(bds, nil) - require.NoError(t, ds.AddToIndexQueue("one")) require.NoError(t, ds.AddToIndexQueue("one")) require.NoError(t, ds.AddToIndexQueue("two")) @@ -84,14 +86,12 @@ func TestDsObjectStore_Query(t *testing.T) { tmpDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(tmpDir) - fts, err := ftsearch.NewFTSearch(filepath.Join(tmpDir, "fts")) + app := testapp.New() + ds := New() + err := app.With(&config.DefaultConfig).With(wallet.NewWithRepoPathAndKeys(tmpDir, nil, nil)).With(clientds.New()).With(ftsearch.New()).With(ds).Start() require.NoError(t, err) + fts := app.MustComponent(ftsearch.CName).(ftsearch.FTSearch) - bds, err := badger.NewDatastore(tmpDir, nil) - require.NoError(t, err) - - ds := NewObjectStore(bds, fts) - defer ds.Close() newDet := func(name string) *types.Struct { return &types.Struct{ Fields: map[string]*types.Value{ @@ -105,9 +105,9 @@ func TestDsObjectStore_Query(t *testing.T) { id1 := tid1.String() id2 := tid2.String() id3 := tid3.String() - require.NoError(t, ds.UpdateObject(id1, newDet("one"), nil, nil, "s1")) - require.NoError(t, ds.UpdateObject(id2, newDet("two"), nil, nil, "s2")) - require.NoError(t, ds.UpdateObject(id3, newDet("three"), nil, nil, "s3")) + require.NoError(t, ds.CreateObject(id1, newDet("one"), nil, nil, "s1")) + require.NoError(t, ds.CreateObject(id2, newDet("two"), nil, nil, "s2")) + require.NoError(t, ds.CreateObject(id3, newDet("three"), nil, nil, "s3")) require.NoError(t, fts.Index(ftsearch.SearchDoc{ Id: id1, Title: "one", @@ -201,14 +201,11 @@ func TestDsObjectStore_RelationsIndex(t *testing.T) { tmpDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(tmpDir) - fts, err := ftsearch.NewFTSearch(filepath.Join(tmpDir, "fts")) + app := testapp.New() + ds := New() + err := app.With(&config.DefaultConfig).With(wallet.NewWithRepoPathAndKeys(tmpDir, nil, nil)).With(clientds.New()).With(ftsearch.New()).With(ds).Start() require.NoError(t, err) - bds, err := badger.NewDatastore(tmpDir, nil) - require.NoError(t, err) - - ds := NewObjectStore(bds, fts) - defer ds.Close() newDet := func(name, objtype string) *types.Struct { return &types.Struct{ Fields: map[string]*types.Value{ @@ -220,7 +217,7 @@ func TestDsObjectStore_RelationsIndex(t *testing.T) { id1 := getId() id2 := getId() id3 := getId() - require.NoError(t, ds.UpdateObject(id1, newDet("one", "_ota1"), &pbrelation.Relations{Relations: []*pbrelation.Relation{ + require.NoError(t, ds.CreateObject(id1, newDet("one", "_ota1"), &pbrelation.Relations{Relations: []*pbrelation.Relation{ { Key: "rel1", Format: pbrelation.RelationFormat_status, @@ -240,7 +237,7 @@ func TestDsObjectStore_RelationsIndex(t *testing.T) { }, }}, nil, "s1")) - require.NoError(t, ds.UpdateObject(id2, newDet("two", "_ota2"), &pbrelation.Relations{Relations: []*pbrelation.Relation{ + require.NoError(t, ds.CreateObject(id2, newDet("two", "_ota2"), &pbrelation.Relations{Relations: []*pbrelation.Relation{ { Key: "rel1", Format: pbrelation.RelationFormat_status, @@ -272,7 +269,7 @@ func TestDsObjectStore_RelationsIndex(t *testing.T) { }, }, }}, nil, "s2")) - require.NoError(t, ds.UpdateObject(id3, newDet("three", "_ota2"), nil, nil, "s3")) + require.NoError(t, ds.CreateObject(id3, newDet("three", "_ota2"), nil, nil, "s3")) restOpts, err := ds.GetAggregatedOptions("rel1", pbrelation.RelationFormat_status, "_otffff") require.NoError(t, err) diff --git a/pkg/lib/localstore/stores.go b/pkg/lib/localstore/stores.go index 8c4b1acb9..7b478726c 100644 --- a/pkg/lib/localstore/stores.go +++ b/pkg/lib/localstore/stores.go @@ -3,22 +3,12 @@ package localstore import ( "crypto/sha256" "fmt" - "github.com/anytypeio/go-anytype-middleware/app" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/datastore" - "strings" - "time" - - "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/database" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/ftsearch" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model" - pbrelation "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/relation" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/storage" + "strings" + "github.com/anytypeio/go-anytype-middleware/util/slice" "github.com/dgtony/collections/polymorph" "github.com/dgtony/collections/slices" - "github.com/gogo/protobuf/types" ds "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" "github.com/multiformats/go-base32" @@ -27,86 +17,11 @@ import ( var ErrDuplicateKey = fmt.Errorf("duplicate key") var ErrNotFound = fmt.Errorf("not found") -var log = logging.Logger("anytype-localstore") - var ( - indexBase = ds.NewKey("/idx") + log = logging.Logger("anytype-localstore") + IndexBase = ds.NewKey("/idx") ) -const CName = "localstore" - -type LocalStore struct { - app.Component - Files FileStore - Objects ObjectStore -} - -type FileStore interface { - Indexable - Add(file *storage.FileInfo) error - AddMulti(upsert bool, files ...*storage.FileInfo) error - AddFileKeys(fileKeys ...FileKeys) error - GetFileKeys(hash string) (map[string]string, error) - GetByHash(hash string) (*storage.FileInfo, error) - GetBySource(mill string, source string, opts string) (*storage.FileInfo, error) - GetByChecksum(mill string, checksum string) (*storage.FileInfo, error) - AddTarget(hash string, target string) error - RemoveTarget(hash string, target string) error - ListTargets() ([]string, error) - ListByTarget(target string) ([]*storage.FileInfo, error) - Count() (int, error) - DeleteByHash(hash string) error - DeleteFileKeys(hash string) error - List() ([]*storage.FileInfo, error) -} - -type ObjectStore interface { - Indexable - database.Reader - - CreateObject(id string, details *types.Struct, relations *pbrelation.Relations, links []string, snippet string) error - UpdateObjectDetails(id string, details *types.Struct, relations *pbrelation.Relations) error - UpdateObjectLinksAndSnippet(id string, links []string, snippet string) error - - StoreRelations(relations []*pbrelation.Relation) error - - DeleteObject(id string) error - RemoveRelationFromCache(key string) error - - UpdateRelationsInSet(setId, objTypeBefore, objTypeAfter string, relationsBefore, relationsAfter *pbrelation.Relations) error - - GetWithLinksInfoByID(id string) (*model.ObjectInfoWithLinks, error) - GetWithOutboundLinksInfoById(id string) (*model.ObjectInfoWithOutboundLinks, error) - GetDetails(id string) (*model.ObjectDetails, error) - GetAggregatedOptions(relationKey string, relationFormat pbrelation.RelationFormat, objectType string) (options []*pbrelation.RelationOption, err error) - - GetByIDs(ids ...string) ([]*model.ObjectInfo, error) - List() ([]*model.ObjectInfo, error) - ListIds() ([]string, error) - - QueryObjectInfo(q database.Query, objectTypes []smartblock.SmartBlockType) (results []*model.ObjectInfo, total int, err error) - AddToIndexQueue(id string) error - IndexForEach(f func(id string, tm time.Time) error) error - FTSearch() ftsearch.FTSearch -} - -func New() *LocalStore { - return &LocalStore{} -} - -func (ls *LocalStore) Init(a *app.App) (err error) { - store := a.MustComponent(datastore.CName).(datastore.Datastore).LocalstoreDS() - fts := a.MustComponent(ftsearch.CName).(ftsearch.FTSearch) - - ls.Objects = NewObjectStore(store, fts) - ls.Files = NewFileStore(store) - return nil -} - -func (ls *LocalStore) Name() (name string) { - return CName -} - type Indexable interface { Indexes() []Index } @@ -160,7 +75,7 @@ func AddIndex(index Index, ds ds.TxnDatastore, newVal interface{}, newValPrimary func UpdateIndexWithTxn(index Index, ds ds.Txn, oldVal interface{}, newVal interface{}, newValPrimary string) error { oldKeys := index.JoinedKeys(oldVal) hasKey := func(key string) (exists bool, err error) { - return ds.Has(indexBase.ChildString(index.Prefix).ChildString(index.Name).ChildString(key)) + return ds.Has(IndexBase.ChildString(index.Prefix).ChildString(index.Name).ChildString(key)) } if len(oldKeys) > 0 { @@ -176,10 +91,10 @@ func UpdateIndexWithTxn(index Index, ds ds.Txn, oldVal interface{}, newVal inter newKeys := index.JoinedKeys(newVal) - removed, added := diffSlices(oldKeys, newKeys) + removed, added := slice.DifferenceRemovedAdded(oldKeys, newKeys) for _, removedKey := range removed { - key := indexBase.ChildString(index.Prefix).ChildString(index.Name).ChildString(removedKey).ChildString(newValPrimary) + key := IndexBase.ChildString(index.Prefix).ChildString(index.Name).ChildString(removedKey).ChildString(newValPrimary) exists, err := ds.Has(key) if err != nil { return err @@ -197,7 +112,7 @@ func UpdateIndexWithTxn(index Index, ds ds.Txn, oldVal interface{}, newVal inter } for _, addedKey := range added { - key := indexBase.ChildString(index.Prefix).ChildString(index.Name).ChildString(addedKey).ChildString(newValPrimary) + key := IndexBase.ChildString(index.Prefix).ChildString(index.Name).ChildString(addedKey).ChildString(newValPrimary) exists, err := ds.Has(key) if err != nil { return err @@ -228,7 +143,7 @@ func AddIndexWithTxn(index Index, ds ds.Txn, newVal interface{}, newValPrimary s keyStr = base32.RawStdEncoding.EncodeToString(keyBytesF[:]) } - key := indexBase.ChildString(index.Prefix).ChildString(index.Name).ChildString(keyStr) + key := IndexBase.ChildString(index.Prefix).ChildString(index.Name).ChildString(keyStr) if index.Unique { exists, err := ds.Has(key) if err != nil { @@ -266,7 +181,7 @@ func EraseIndex(index Index, ds ds.TxnDatastore) error { // EraseIndexWithTxn deletes the whole index func EraseIndexWithTxn(index Index, txn ds.Txn) error { - key := indexBase.ChildString(index.Prefix).ChildString(index.Name) + key := IndexBase.ChildString(index.Prefix).ChildString(index.Name) res, err := GetKeys(txn, key.String(), 0) if err != nil { return err @@ -294,7 +209,7 @@ func RemoveIndexWithTxn(index Index, txn ds.Txn, val interface{}, valPrimary str keyStr = base32.RawStdEncoding.EncodeToString(keyBytesF[:]) } - key := indexBase.ChildString(index.Prefix).ChildString(index.Name).ChildString(keyStr) + key := IndexBase.ChildString(index.Prefix).ChildString(index.Name).ChildString(keyStr) exists, err := txn.Has(key.ChildString(valPrimary)) if err != nil { @@ -399,7 +314,7 @@ func GetKeyByIndex(index Index, txn ds.Txn, val interface{}) (string, error) { } func getDsKeyByIndexParts(prefix string, keyIndexName string, keyIndexValue []string, separator string, hash bool) ds.Key { - key := indexBase.ChildString(prefix).ChildString(keyIndexName) + key := IndexBase.ChildString(prefix).ChildString(keyIndexName) if len(keyIndexValue) == 0 { return key } @@ -533,7 +448,7 @@ func GetKeysByIndex(index Index, txn ds.Txn, val interface{}, limit int) (query. keyStr = base32.RawStdEncoding.EncodeToString(keyBytesF[:]) } - key := indexBase.ChildString(index.Prefix).ChildString(index.Name).ChildString(keyStr) + key := IndexBase.ChildString(index.Prefix).ChildString(index.Name).ChildString(keyStr) if index.Unique { limit = 1 } diff --git a/pkg/lib/pin/service.go b/pkg/lib/pin/service.go index 12ee680b4..eef6ec644 100644 --- a/pkg/lib/pin/service.go +++ b/pkg/lib/pin/service.go @@ -4,12 +4,12 @@ import ( "context" "errors" "github.com/anytypeio/go-anytype-middleware/app" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/filestore" "sync" "time" "github.com/anytypeio/go-anytype-middleware/pkg/lib/cafe" cafepb "github.com/anytypeio/go-anytype-middleware/pkg/lib/cafe/pb" - "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore" "github.com/anytypeio/go-anytype-middleware/pkg/lib/logging" "github.com/dgtony/collections/hashset" ) @@ -44,7 +44,7 @@ type filePinService struct { ctx context.Context ctxCancel context.CancelFunc cafe cafe.Client - store localstore.FileStore + store filestore.FileStore files map[string]FilePinInfo activate chan struct{} diff --git a/util/slice/slice.go b/util/slice/slice.go index 18014891d..414509d09 100644 --- a/util/slice/slice.go +++ b/util/slice/slice.go @@ -6,6 +6,29 @@ import ( "sort" ) +func DifferenceRemovedAdded(a, b []string) (removed []string, added []string) { + var amap = map[string]struct{}{} + var bmap = map[string]struct{}{} + + for _, item := range a { + amap[item] = struct{}{} + } + + for _, item := range b { + if _, exists := amap[item]; !exists { + added = append(added, item) + } + bmap[item] = struct{}{} + } + + for _, item := range a { + if _, exists := bmap[item]; !exists { + removed = append(removed, item) + } + } + return +} + func FindPos(s []string, v string) int { for i, sv := range s { if sv == v { diff --git a/util/testMock/anytype.go b/util/testMock/anytype.go index f8eb7d92f..977fb8a8d 100644 --- a/util/testMock/anytype.go +++ b/util/testMock/anytype.go @@ -1,11 +1,12 @@ //go:generate mockgen -package testMock -destination anytype_mock.go github.com/anytypeio/go-anytype-middleware/pkg/lib/core Service,SmartBlock,SmartBlockSnapshot,File,Image -//go:generate mockgen -package testMock -destination objectstore_mock.go github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore ObjectStore +//go:generate mockgen -package testMock -destination objectstore_mock.go github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore ObjectStore //go:generate mockgen -package testMock -destination history_mock.go github.com/anytypeio/go-anytype-middleware/core/block/undo History package testMock import ( "github.com/anytypeio/go-anytype-middleware/app/testapp" "github.com/anytypeio/go-anytype-middleware/pkg/lib/core" + "github.com/anytypeio/go-anytype-middleware/pkg/lib/localstore/objectstore" "github.com/golang/mock/gomock" ) @@ -19,6 +20,14 @@ func RegisterMockAnytype(ctrl *gomock.Controller, ta *testapp.TestApp) *MockServ return ms } +func RegisterMockObjectStore(ctrl *gomock.Controller, ta *testapp.TestApp) *MockObjectStore { + ms := NewMockObjectStore(ctrl) + ms.EXPECT().Name().AnyTimes().Return(objectstore.CName) + ms.EXPECT().Init(gomock.Any()).AnyTimes() + ta.Register(ms) + return ms +} + func GetMockAnytype(ta *testapp.TestApp) *MockService { return ta.MustComponent(core.CName).(*MockService) }