anyproto/anytype-heart
1
0
Fork 0
mirror of https://github.com/anyproto/anytype-heart.git synced 2025-06-11 18:20:33 +09:00
Code Activity

Use common ocache

Browse source
This commit is contained in:
mcrakhman 2023-04-25 18:17:59 +02:00 • committed by Mikhail Iudin
parent f5174adf06
commit 3d953ccb92
No known key found for this signature in database
GPG key ID: FAAAA8BAABDFF1C0
22 changed files with 357 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

10
core/anytype/bootstrap.go
View file

@ -2,6 +2,10 @@ package anytype
import (
"context"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/net/rpc/server"
"github.com/anytypeio/go-anytype-middleware/space"
"github.com/anytypeio/go-anytype-middleware/space/storage"
"os"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/app"
@ -104,9 +108,13 @@ func Bootstrap(a *app.App, components ...app.Component) {
a.Register(c)
}
a.Register(clientds.New()).
Register(server.New()).
Register(storage.New()).
Register(commonspace.New()).
Register(space.New()).
Register(relation.New()).
Register(ftsearch.New()).
Register(objectstore.New()).
Register(relation.New()).
Register(filestore.New()).
Register(recordsbatcher.New()).
Register(ipfslite.New()).

2
core/block/editor.go
View file

@ -7,6 +7,7 @@ import (
"github.com/gogo/protobuf/types"
ds "github.com/ipfs/go-datastore"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/ocache"
"github.com/anytypeio/go-anytype-middleware/core/block/doc"
"github.com/anytypeio/go-anytype-middleware/core/block/editor"
"github.com/anytypeio/go-anytype-middleware/core/block/editor/basic"
@ -28,7 +29,6 @@ import (
"github.com/anytypeio/go-anytype-middleware/pb"
coresb "github.com/anytypeio/go-anytype-middleware/pkg/lib/core/smartblock"
"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
"github.com/anytypeio/go-anytype-middleware/util/ocache"
"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
)

2
core/block/editor/smartblock/smartblock.go
View file

@ -11,6 +11,7 @@ import (
"github.com/gogo/protobuf/types"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/app"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/ocache"
"github.com/anytypeio/go-anytype-middleware/core/block/doc"
"github.com/anytypeio/go-anytype-middleware/core/block/editor/state"
"github.com/anytypeio/go-anytype-middleware/core/block/editor/template"
@ -32,7 +33,6 @@ import (
"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
"github.com/anytypeio/go-anytype-middleware/util/internalflag"
"github.com/anytypeio/go-anytype-middleware/util/mutex"
"github.com/anytypeio/go-anytype-middleware/util/ocache"
"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
"github.com/anytypeio/go-anytype-middleware/util/slice"
)

2
core/block/service.go
View file

@ -16,6 +16,7 @@ import (
"github.com/textileio/go-threads/core/thread"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/app"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/ocache"
bookmarksvc "github.com/anytypeio/go-anytype-middleware/core/block/bookmark"
"github.com/anytypeio/go-anytype-middleware/core/block/doc"
"github.com/anytypeio/go-anytype-middleware/core/block/editor"
@ -48,7 +49,6 @@ import (
"github.com/anytypeio/go-anytype-middleware/pkg/lib/threads"
"github.com/anytypeio/go-anytype-middleware/util/internalflag"
"github.com/anytypeio/go-anytype-middleware/util/linkpreview"
"github.com/anytypeio/go-anytype-middleware/util/ocache"
"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
"github.com/anytypeio/go-anytype-middleware/util/uri"

9
core/indexer/indexer.go
View file

@ -32,7 +32,6 @@ import (
"github.com/anytypeio/go-anytype-middleware/pkg/lib/logging"
"github.com/anytypeio/go-anytype-middleware/pkg/lib/pb/model"
"github.com/anytypeio/go-anytype-middleware/pkg/lib/threads"
"github.com/anytypeio/go-anytype-middleware/util/ocache"
"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
"github.com/anytypeio/go-anytype-middleware/util/slice"
)
@ -320,7 +319,7 @@ func (i *indexer) reindexOutdatedThreads() (toReindex, success int, err error) {
// }
// we do this instead of context.WithTimeout in order to continue loading in case of timeout in background
ctx = context.WithValue(context.Background(), ocache.CacheTimeout, cacheTimeout)
//ctx = context.WithValue(context.Background(), ocache.CacheTimeout, cacheTimeout)
ctx = context.WithValue(ctx, metrics.CtxKeyRequest, "reindexOutdatedThreads")
d, err := i.doc.GetDocInfo(ctx, id)
if err != nil {
@ -428,7 +427,7 @@ func (i *indexer) Reindex(ctx context.Context, reindex reindexFlags) (err error)
}
// for all ids except home and archive setting cache timeout for reindexing
ctx = context.WithValue(ctx, ocache.CacheTimeout, cacheTimeout)
//ctx = context.WithValue(ctx, ocache.CacheTimeout, cacheTimeout)
if reindex&reindexThreadObjects != 0 {
ids, err := i.getIdsForTypes(
smartblock.SmartBlockTypePage,
@ -915,8 +914,8 @@ func (i *indexer) ftIndex() {
func (i *indexer) ftIndexDoc(id string, _ time.Time) (err error) {
st := time.Now()
ctx := context.WithValue(context.Background(), ocache.CacheTimeout, cacheTimeout)
ctx = context.WithValue(ctx, metrics.CtxKeyRequest, "index_fulltext")
//ctx := context.WithValue(context.Background(), ocache.CacheTimeout, cacheTimeout)
ctx := context.WithValue(context.Background(), metrics.CtxKeyRequest, "index_fulltext")
info, err := i.doc.GetDocInfo(ctx, id)
if err != nil {

2
go.mod
View file

@ -102,7 +102,7 @@ require (
github.com/alecthomas/jsonschema v0.0.0-20191017121752-4bb6e3fae4f2 // indirect
github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/anytypeio/go-anytype-infrastructure-experiments/common v0.0.0-20230102121649-8680c2432933 // indirect
github.com/anytypeio/go-anytype-infrastructure-experiments/common v0.0.0-20230102132724-987bd0154f4a // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect

2
go.sum
View file

@ -82,6 +82,8 @@ github.com/anytypeio/go-anytype-infrastructure-experiments/common v0.0.0-2023010
github.com/anytypeio/go-anytype-infrastructure-experiments/common v0.0.0-20230102103143-1a2ceb6e2bd6/go.mod h1:2vafONFWOYl9XTv9vW5KzOmuqdxhC6PAlRfkT7TseZA=
github.com/anytypeio/go-anytype-infrastructure-experiments/common v0.0.0-20230102121649-8680c2432933 h1:s0yyWe4o+UQRoMWuoFkZHoLAsfEc7yPyOP0v1bKQT0A=
github.com/anytypeio/go-anytype-infrastructure-experiments/common v0.0.0-20230102121649-8680c2432933/go.mod h1:2vafONFWOYl9XTv9vW5KzOmuqdxhC6PAlRfkT7TseZA=
github.com/anytypeio/go-anytype-infrastructure-experiments/common v0.0.0-20230102132724-987bd0154f4a h1:A9xzoo/Dy2PN7rbxsfvvBw6g1xhc8/DHQbmhFfcuFVQ=
github.com/anytypeio/go-anytype-infrastructure-experiments/common v0.0.0-20230102132724-987bd0154f4a/go.mod h1:2vafONFWOYl9XTv9vW5KzOmuqdxhC6PAlRfkT7TseZA=
github.com/anytypeio/go-ds-badger3 v0.3.1-0.20221103102622-3233d4e13cb8 h1:LC9w0M0SbA5VuZeBtUdq+uR4mdjbJhxurNtovmRiOrU=
github.com/anytypeio/go-ds-badger3 v0.3.1-0.20221103102622-3233d4e13cb8/go.mod h1:R5tqrpzflXnpE1v91BNZrz62AVox66+rWM51LXNoNik=
github.com/anytypeio/go-gelf v0.0.0-20210418191311-774bd5b016e7 h1:YBmcug6mOdwjJcmzA/Fpjtl2oo78+fMlr5Xj0dzwg78=

109
space/clientcache/treecache.go Normal file
View file

@ -0,0 +1,109 @@
package clientcache
import (
"context"
"errors"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/accountservice"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/app"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/logger"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/ocache"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/object/tree/objecttree"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/object/treegetter"
"github.com/anytypeio/go-anytype-middleware/space"
"go.uber.org/zap"
"time"
)
var log = logger.NewNamed("treecache")
var ErrCacheObjectWithoutTree = errors.New("cache object contains no tree")
type ctxKey int
const spaceKey ctxKey = 0
type treeCache struct {
gcttl int
cache ocache.OCache
account accountservice.Service
clientService space.Service
}
type TreeCache interface {
treegetter.TreeGetter
}
type updateListener struct {
}
func (u *updateListener) Update(tree objecttree.ObjectTree) {
log.With(
zap.Strings("heads", tree.Heads()),
zap.String("tree id", tree.Id())).
Debug("updating tree")
}
func (u *updateListener) Rebuild(tree objecttree.ObjectTree) {
log.With(
zap.Strings("heads", tree.Heads()),
zap.String("tree id", tree.Id())).
Debug("rebuilding tree")
}
func New(ttl int) TreeCache {
return &treeCache{
gcttl: ttl,
}
}
func (c *treeCache) Run(ctx context.Context) (err error) {
return nil
}
func (c *treeCache) Close(ctx context.Context) (err error) {
return c.cache.Close()
}
func (c *treeCache) Init(a *app.App) (err error) {
c.clientService = a.MustComponent(space.CName).(space.Service)
c.account = a.MustComponent(accountservice.CName).(accountservice.Service)
c.cache = ocache.New(
func(ctx context.Context, id string) (value ocache.Object, err error) {
spaceId := ctx.Value(spaceKey).(string)
_, err = c.clientService.GetSpace(ctx, spaceId)
if err != nil {
return
}
return
},
ocache.WithLogger(log.Sugar()),
ocache.WithGCPeriod(time.Minute),
ocache.WithTTL(time.Duration(c.gcttl)*time.Second),
)
return nil
}
func (c *treeCache) Name() (name string) {
return treegetter.CName
}
func (c *treeCache) GetTree(ctx context.Context, spaceId, id string) (tr objecttree.ObjectTree, err error) {
ctx = context.WithValue(ctx, spaceKey, spaceId)
v, err := c.cache.Get(ctx, id)
if err != nil {
return
}
return v.(objecttree.ObjectTree), nil
}
func (c *treeCache) DeleteTree(ctx context.Context, spaceId, treeId string) (err error) {
tr, err := c.GetTree(ctx, spaceId, treeId)
if err != nil {
return
}
err = tr.Delete()
if err != nil {
return
}
_, err = c.cache.Remove(treeId)
return
}

75
space/rpchandler.go Normal file
View file

@ -0,0 +1,75 @@
package space
import (
"context"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto"
)
type rpcHandler struct {
s *service
}
func (r *rpcHandler) SpacePull(ctx context.Context, request *spacesyncproto.SpacePullRequest) (resp *spacesyncproto.SpacePullResponse, err error) {
sp, err := r.s.GetSpace(ctx, request.Id)
if err != nil {
if err != spacesyncproto.ErrSpaceMissing {
err = spacesyncproto.ErrUnexpected
}
return
}
spaceDesc, err := sp.Description()
if err != nil {
err = spacesyncproto.ErrUnexpected
return
}
resp = &spacesyncproto.SpacePullResponse{
Payload: &spacesyncproto.SpacePayload{
SpaceHeader: spaceDesc.SpaceHeader,
AclPayloadId: spaceDesc.AclId,
AclPayload: spaceDesc.AclPayload,
SpaceSettingsPayload: spaceDesc.SpaceSettingsPayload,
SpaceSettingsPayloadId: spaceDesc.SpaceSettingsId,
},
}
return
}
func (r *rpcHandler) SpacePush(ctx context.Context, req *spacesyncproto.SpacePushRequest) (resp *spacesyncproto.SpacePushResponse, err error) {
description := commonspace.SpaceDescription{
SpaceHeader: req.Payload.SpaceHeader,
AclId: req.Payload.AclPayloadId,
AclPayload: req.Payload.AclPayload,
SpaceSettingsPayload: req.Payload.SpaceSettingsPayload,
SpaceSettingsId: req.Payload.SpaceSettingsPayloadId,
}
ctx = context.WithValue(ctx, commonspace.AddSpaceCtxKey, description)
_, err = r.s.GetSpace(ctx, description.SpaceHeader.GetId())
if err != nil {
return
}
resp = &spacesyncproto.SpacePushResponse{}
return
}
func (r *rpcHandler) HeadSync(ctx context.Context, req *spacesyncproto.HeadSyncRequest) (*spacesyncproto.HeadSyncResponse, error) {
sp, err := r.s.GetSpace(ctx, req.SpaceId)
if err != nil {
return nil, spacesyncproto.ErrSpaceMissing
}
return sp.SpaceSyncRpc().HeadSync(ctx, req)
}
func (r *rpcHandler) ObjectSyncStream(stream spacesyncproto.DRPCSpaceSync_ObjectSyncStreamStream) error {
msg, err := stream.Recv()
if err != nil {
return err
}
sp, err := r.s.GetSpace(stream.Context(), msg.SpaceId)
if err != nil {
return spacesyncproto.ErrSpaceMissing
}
return sp.SpaceSyncRpc().Stream(stream)
}

111
space/service.go Normal file
View file

@ -0,0 +1,111 @@
package space
import (
"context"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/app"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/logger"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/ocache"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacestorage"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/spacesyncproto"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/syncstatus"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/net/rpc/server"
"time"
)
const CName = "client.clientspace"
var log = logger.NewNamed(CName)
func New() Service {
return &service{}
}
type Service interface {
GetSpace(ctx context.Context, id string) (commonspace.Space, error)
CreateSpace(ctx context.Context, payload commonspace.SpaceCreatePayload) (commonspace.Space, error)
DeriveSpace(ctx context.Context, payload commonspace.SpaceDerivePayload) (commonspace.Space, error)
app.ComponentRunnable
}
type service struct {
conf commonspace.Config
spaceCache ocache.OCache
commonSpace commonspace.SpaceService
spaceStorageProvider spacestorage.SpaceStorageProvider
}
func (s *service) Init(a *app.App) (err error) {
s.conf = a.MustComponent("config").(commonspace.ConfigGetter).GetSpace()
s.commonSpace = a.MustComponent(commonspace.CName).(commonspace.SpaceService)
s.spaceStorageProvider = a.MustComponent(spacestorage.CName).(spacestorage.SpaceStorageProvider)
s.spaceCache = ocache.New(
s.loadSpace,
ocache.WithLogger(log.Sugar()),
ocache.WithGCPeriod(time.Minute),
ocache.WithTTL(time.Duration(s.conf.GCTTL)*time.Second),
)
return spacesyncproto.DRPCRegisterSpaceSync(a.MustComponent(server.CName).(server.DRPCServer), &rpcHandler{s})
}
func (s *service) Name() (name string) {
return CName
}
func (s *service) Run(ctx context.Context) (err error) {
return
}
func (s *service) CreateSpace(ctx context.Context, payload commonspace.SpaceCreatePayload) (container commonspace.Space, err error) {
id, err := s.commonSpace.CreateSpace(ctx, payload)
if err != nil {
return
}
obj, err := s.spaceCache.Get(ctx, id)
if err != nil {
return
}
return obj.(commonspace.Space), nil
}
func (s *service) DeriveSpace(ctx context.Context, payload commonspace.SpaceDerivePayload) (container commonspace.Space, err error) {
id, err := s.commonSpace.DeriveSpace(ctx, payload)
if err != nil {
return
}
obj, err := s.spaceCache.Get(ctx, id)
if err != nil {
return
}
return obj.(commonspace.Space), nil
}
func (s *service) GetSpace(ctx context.Context, id string) (container commonspace.Space, err error) {
v, err := s.spaceCache.Get(ctx, id)
if err != nil {
return
}
return v.(commonspace.Space), nil
}
func (s *service) loadSpace(ctx context.Context, id string) (value ocache.Object, err error) {
cc, err := s.commonSpace.NewSpace(ctx, id)
if err != nil {
return
}
ns, err := newClientSpace(cc)
if err != nil {
return
}
ns.SyncStatus().(syncstatus.StatusWatcher).SetUpdateReceiver(&statusReceiver{})
if err = ns.Init(ctx); err != nil {
return
}
return ns, nil
}
func (s *service) Close(ctx context.Context) (err error) {
return s.spaceCache.Close()
}

22
space/space.go Normal file
View file

@ -0,0 +1,22 @@
package space
import (
"context"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace"
)
func newClientSpace(cc commonspace.Space) (commonspace.Space, error) {
return &clientSpace{cc}, nil
}
type clientSpace struct {
commonspace.Space
}
func (s *clientSpace) Init(ctx context.Context) (err error) {
return s.Space.Init(ctx)
}
func (s *clientSpace) Close() (err error) {
return s.Space.Close()
}

20
space/statusreceiver.go Normal file
View file

@ -0,0 +1,20 @@
package space
import (
"context"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/commonspace/syncstatus"
"go.uber.org/zap"
)
type statusReceiver struct {
}
func (s *statusReceiver) UpdateTree(ctx context.Context, treeId string, status syncstatus.SyncStatus) (err error) {
log.With(zap.String("treeId", treeId), zap.Bool("synced", status == syncstatus.StatusSynced)).
Debug("updating sync status")
return nil
}
func (s *statusReceiver) UpdateNodeConnection(online bool) {
log.With(zap.Bool("nodes online", online)).Debug("updating node connection")
}

0
core/storage/helpers.go → space/storage/helpers.go
View file

0
core/storage/keys.go → space/storage/keys.go
View file

0
core/storage/liststorage.go → space/storage/liststorage.go
View file

0
core/storage/liststorage_test.go → space/storage/liststorage_test.go
View file

0
core/storage/spacestorage.go → space/storage/spacestorage.go
View file

0
core/storage/spacestorage_test.go → space/storage/spacestorage_test.go
View file

0
core/storage/storageservice.go → space/storage/storageservice.go
View file

0
core/storage/treestorage.go → space/storage/treestorage.go
View file

0
core/storage/treestorage_test.go → space/storage/treestorage_test.go
View file

2
util/unsplash/unsplash.go
View file

@ -4,9 +4,9 @@ import (
"context"
"fmt"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/app"
"github.com/anytypeio/go-anytype-infrastructure-experiments/common/app/ocache"
"github.com/anytypeio/go-anytype-middleware/core/configfetcher"
"github.com/anytypeio/go-anytype-middleware/pkg/lib/logging"
"github.com/anytypeio/go-anytype-middleware/util/ocache"
"github.com/anytypeio/go-anytype-middleware/util/pbtypes"
"github.com/anytypeio/go-anytype-middleware/util/uri"
"github.com/dsoprea/go-exif/v3"