1
0
Fork 0
mirror of https://github.com/anyproto/anytype-heart.git synced 2025-06-08 05:47:07 +09:00

Merge pull request #2185 from anyproto/GO-4146-new-spacestore

GO-4146: New spacestore merging once again
This commit is contained in:
Mikhail Rakhmanov 2025-03-03 23:02:31 +01:00 committed by GitHub
commit b243f1f4e2
Signed by: github
GPG key ID: B5690EEEBB952194
104 changed files with 3886 additions and 1546 deletions

View file

@ -97,6 +97,9 @@ packages:
github.com/anyproto/anytype-heart/space/spacecore/storage:
interfaces:
ClientStorage:
github.com/anyproto/anytype-heart/space/spacecore/storage/anystorage:
interfaces:
ClientSpaceStorage:
github.com/anyproto/anytype-heart/space/techspace:
interfaces:
TechSpace:

View file

@ -9,16 +9,19 @@ import (
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/goccy/go-graphviz"
"github.com/gogo/protobuf/jsonpb"
"github.com/gogo/protobuf/proto"
"github.com/anyproto/anytype-heart/core/block/editor/state"
"github.com/anyproto/anytype-heart/core/debug/treearchive"
"github.com/anyproto/anytype-heart/core/debug/exporter"
"github.com/anyproto/anytype-heart/pkg/lib/core/smartblock"
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
"github.com/anyproto/anytype-heart/util/pbtypes"
@ -42,22 +45,20 @@ func main() {
return
}
fmt.Println("opening file...")
st := time.Now()
archive, err := treearchive.Open(*file)
var (
st = time.Now()
ctx = context.Background()
)
res, err := exporter.ImportStorage(ctx, *file)
if err != nil {
log.Fatal("can't open debug file:", err)
log.Fatal("can't import the tree:", err)
}
objectTree, err := res.CreateReadableTree(*fromRoot, "")
if err != nil {
log.Fatal("can't create readable tree:", err)
}
defer archive.Close()
fmt.Printf("open archive done in %.1fs\n", time.Since(st).Seconds())
importer := treearchive.NewTreeImporter(archive.ListStorage(), archive.TreeStorage())
st = time.Now()
err = importer.Import(*fromRoot, "")
if err != nil {
log.Fatal("can't import the tree", err)
}
fmt.Printf("import tree done in %.1fs\n", time.Since(st).Seconds())
importer := exporter.NewTreeImporter(objectTree)
if *makeJson {
treeJson, err := importer.Json()
if err != nil {
@ -83,10 +84,11 @@ func main() {
}
fmt.Println("Change:")
fmt.Println(pbtypes.Sprint(ch.Model))
err = importer.Import(*fromRoot, ch.Id)
objectTree, err = res.CreateReadableTree(*fromRoot, ch.Id)
if err != nil {
log.Fatal("can't import the tree before", ch.Id, err)
log.Fatal("can't create readable tree:", err)
}
importer = exporter.NewTreeImporter(objectTree)
}
ot := importer.ObjectTree()
di, err := ot.Debug(state.ChangeParser{})
@ -126,12 +128,16 @@ func main() {
if *objectStore {
fmt.Println("fetch object store info..")
ls, err := archive.LocalStore()
f, err := os.Open(filepath.Join(res.FolderPath, "localstore.json"))
if err != nil {
fmt.Println("can't open objectStore info:", err)
} else {
fmt.Println(pbtypes.Sprint(ls))
log.Fatal("can't open objectStore info:", err)
}
info := &model.ObjectInfo{}
if err = jsonpb.Unmarshal(f, info); err != nil {
log.Fatal("can't unmarshal objectStore info:", err)
}
defer f.Close()
fmt.Println(pbtypes.Sprint(info))
}
if *makeTree {

View file

@ -306,7 +306,7 @@ func getTableSizes(mw *core.Middleware) (tables map[string]uint64) {
tables = make(map[string]uint64)
cfg := mw.GetApp().MustComponent(config.CName).(*config.Config)
db, err := sql.Open("sqlite3", cfg.GetSpaceStorePath())
db, err := sql.Open("sqlite3", cfg.GetSqliteStorePath())
if err != nil {
fmt.Println("Error opening database:", err)
return

View file

@ -10,6 +10,7 @@ import (
"github.com/anyproto/anytype-heart/core/application"
"github.com/anyproto/anytype-heart/core/session"
"github.com/anyproto/anytype-heart/pb"
"github.com/anyproto/anytype-heart/space/spacecore/storage/migrator"
)
func (mw *Middleware) AccountCreate(cctx context.Context, req *pb.RpcAccountCreateRequest) *pb.RpcAccountCreateResponse {
@ -48,24 +49,32 @@ func (mw *Middleware) AccountRecover(cctx context.Context, _ *pb.RpcAccountRecov
},
}
}
func (mw *Middleware) AccountMigrate(_ context.Context, _ *pb.RpcAccountMigrateRequest) *pb.RpcAccountMigrateResponse {
// this is for clients API compatibility, because there is no migration in this branch
// todo: remove this
func (mw *Middleware) AccountMigrate(cctx context.Context, req *pb.RpcAccountMigrateRequest) *pb.RpcAccountMigrateResponse {
err := mw.applicationService.AccountMigrate(cctx, req)
code := mapErrorCode(err,
errToCode(application.ErrBadInput, pb.RpcAccountMigrateResponseError_BAD_INPUT),
errToCode(application.ErrAccountNotFound, pb.RpcAccountMigrateResponseError_ACCOUNT_NOT_FOUND),
errToCode(context.Canceled, pb.RpcAccountMigrateResponseError_CANCELED),
errTypeToCode(&migrator.NotEnoughFreeSpaceError{}, pb.RpcAccountMigrateResponseError_NOT_ENOUGH_FREE_SPACE),
)
return &pb.RpcAccountMigrateResponse{
Error: &pb.RpcAccountMigrateResponseError{
Code: 0,
Description: "",
Code: code,
Description: getErrorDescription(err),
},
}
}
func (mw *Middleware) AccountMigrateCancel(_ context.Context, _ *pb.RpcAccountMigrateCancelRequest) *pb.RpcAccountMigrateCancelResponse {
// this is for clients API compatibility, because there is no migration in this branch
// todo: remove this
func (mw *Middleware) AccountMigrateCancel(cctx context.Context, req *pb.RpcAccountMigrateCancelRequest) *pb.RpcAccountMigrateCancelResponse {
err := mw.applicationService.AccountMigrateCancel(cctx, req)
code := mapErrorCode(err,
errToCode(application.ErrBadInput, pb.RpcAccountMigrateCancelResponseError_BAD_INPUT),
)
return &pb.RpcAccountMigrateCancelResponse{
Error: &pb.RpcAccountMigrateCancelResponseError{
Code: 0,
Description: "",
Code: code,
Description: getErrorDescription(err),
},
}
}
@ -85,6 +94,7 @@ func (mw *Middleware) AccountSelect(cctx context.Context, req *pb.RpcAccountSele
errToCode(application.ErrAnotherProcessIsRunning, pb.RpcAccountSelectResponseError_ANOTHER_ANYTYPE_PROCESS_IS_RUNNING),
errToCode(application.ErrIncompatibleVersion, pb.RpcAccountSelectResponseError_FAILED_TO_FETCH_REMOTE_NODE_HAS_INCOMPATIBLE_PROTO_VERSION),
errToCode(application.ErrFailedToStartApplication, pb.RpcAccountSelectResponseError_FAILED_TO_RUN_NODE),
errToCode(application.ErrAccountStoreIsNotMigrated, pb.RpcAccountSelectResponseError_ACCOUNT_STORE_NOT_MIGRATED),
)
return &pb.RpcAccountSelectResponse{
Config: nil,

View file

@ -10,7 +10,6 @@ import (
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/acl/aclclient"
"github.com/anyproto/any-sync/commonspace/object/acl/list"
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
"github.com/anyproto/any-sync/coordinator/coordinatorclient"
"github.com/anyproto/any-sync/coordinator/coordinatorproto"
"github.com/anyproto/any-sync/nodeconf"
@ -392,7 +391,7 @@ func (a *aclService) ViewInvite(ctx context.Context, inviteCid cid.Cid, inviteFi
if len(recs) == 0 {
return domain.InviteView{}, fmt.Errorf("no acl records found for space: %s, %w", res.SpaceId, ErrAclRequestFailed)
}
store, err := liststorage.NewInMemoryAclListStorage(recs[0].Id, recs)
store, err := list.NewInMemoryStorage(recs[0].Id, recs)
if err != nil {
return domain.InviteView{}, convertedOrAclRequestError(err)
}

View file

@ -143,9 +143,6 @@ func (m mockSyncAcl) HandleRequest(ctx context.Context, senderId string, request
return nil, nil
}
func (m mockSyncAcl) SetHeadUpdater(updater headupdater.HeadUpdater) {
}
func (m mockSyncAcl) SetAclUpdater(updater headupdater.AclUpdater) {
}
@ -326,7 +323,7 @@ func TestService_ViewInvite(t *testing.T) {
keys, err := accountdata.NewRandom()
require.NoError(t, err)
fx.mockAccountService.EXPECT().Keys().Return(keys)
aclList, err := list.NewTestDerivedAcl("spaceId", keys)
aclList, err := list.NewInMemoryDerivedAcl("spaceId", keys)
require.NoError(t, err)
inv, err := aclList.RecordBuilder().BuildInvite()
require.NoError(t, err)
@ -357,7 +354,7 @@ func TestService_ViewInvite(t *testing.T) {
keys, err := accountdata.NewRandom()
require.NoError(t, err)
fx.mockAccountService.EXPECT().Keys().Return(keys)
aclList, err := list.NewTestDerivedAcl("spaceId", keys)
aclList, err := list.NewInMemoryDerivedAcl("spaceId", keys)
require.NoError(t, err)
inv, err := aclList.RecordBuilder().BuildInvite()
require.NoError(t, err)

View file

@ -108,9 +108,12 @@ import (
"github.com/anyproto/anytype-heart/space/spacecore/clientserver"
"github.com/anyproto/anytype-heart/space/spacecore/credentialprovider"
"github.com/anyproto/anytype-heart/space/spacecore/localdiscovery"
"github.com/anyproto/anytype-heart/space/spacecore/oldstorage"
"github.com/anyproto/anytype-heart/space/spacecore/peermanager"
"github.com/anyproto/anytype-heart/space/spacecore/peerstore"
"github.com/anyproto/anytype-heart/space/spacecore/storage"
"github.com/anyproto/anytype-heart/space/spacecore/storage/migrator"
"github.com/anyproto/anytype-heart/space/spacecore/storage/migratorfinisher"
"github.com/anyproto/anytype-heart/space/spacecore/typeprovider"
"github.com/anyproto/anytype-heart/space/spacefactory"
"github.com/anyproto/anytype-heart/space/virtualspaceservice"
@ -191,6 +194,18 @@ func appVersion(a *app.App, clientWithVersion string) string {
return clientWithVersion + "/middle:" + middleVersion + "/any-sync:" + anySyncVersion
}
func BootstrapMigration(a *app.App, components ...app.Component) {
for _, c := range components {
a.Register(c)
}
a.Register(migratorfinisher.New()).
Register(clientds.New()).
Register(oldstorage.New()).
Register(storage.New()).
Register(process.New()).
Register(migrator.New())
}
func Bootstrap(a *app.App, components ...app.Component) {
for _, c := range components {
a.Register(c)

View file

@ -43,6 +43,7 @@ const (
const (
SpaceStoreBadgerPath = "spacestore"
SpaceStoreSqlitePath = "spaceStore.db"
SpaceStoreNewPath = "spaceStoreNew"
)
var (
@ -69,6 +70,7 @@ type Config struct {
DisableThreadsSyncEvents bool
DontStartLocalNetworkSyncAutomatically bool
PeferYamuxTransport bool
DisableNetworkIdCheck bool
SpaceStorageMode storage.SpaceStorageMode
NetworkMode pb.RpcAccountNetworkMode
NetworkCustomConfigFilePath string `json:",omitempty"` // not saved to config
@ -192,7 +194,7 @@ func (c *Config) initFromFileAndEnv(repoPath string) error {
if len(split) == 1 {
return fmt.Errorf("failed to split repo path: %s", repoPath)
}
c.SqliteTempPath = filepath.Join(split[0], "cache")
c.SqliteTempPath = filepath.Join(split[0], "files")
c.AnyStoreConfig.SQLiteConnectionOptions = make(map[string]string)
c.AnyStoreConfig.SQLiteConnectionOptions["temp_store_directory"] = "'" + c.SqliteTempPath + "'"
}
@ -293,12 +295,27 @@ func (c *Config) FSConfig() (FSConfig, error) {
return FSConfig{IPFSStorageAddr: res.CustomFileStorePath}, nil
}
func (c *Config) GetRepoPath() string {
return c.RepoPath
}
func (c *Config) GetConfigPath() string {
return filepath.Join(c.RepoPath, ConfigFileName)
}
func (c *Config) GetSpaceStorePath() string {
return filepath.Join(c.RepoPath, "spaceStore.db")
func (c *Config) GetSqliteStorePath() string {
return filepath.Join(c.RepoPath, SpaceStoreSqlitePath)
}
func (c *Config) GetOldSpaceStorePath() string {
if c.GetSpaceStorageMode() == storage.SpaceStorageModeBadger {
return filepath.Join(c.RepoPath, SpaceStoreBadgerPath)
}
return c.GetSqliteStorePath()
}
func (c *Config) GetNewSpaceStorePath() string {
return filepath.Join(c.RepoPath, SpaceStoreNewPath)
}
func (c *Config) GetTempDirPath() string {
@ -391,7 +408,7 @@ func (c *Config) GetNodeConfWithError() (conf nodeconf.Configuration, err error)
if err := yaml.Unmarshal(confBytes, &conf); err != nil {
return nodeconf.Configuration{}, errors.Join(ErrNetworkFileFailedToRead, err)
}
if c.NetworkId != "" && c.NetworkId != conf.NetworkId {
if !c.DisableNetworkIdCheck && c.NetworkId != "" && c.NetworkId != conf.NetworkId {
log.Warnf("Network id mismatch: %s != %s", c.NetworkId, conf.NetworkId)
return nodeconf.Configuration{}, errors.Join(ErrNetworkIdMismatch, fmt.Errorf("network id mismatch: %s != %s", c.NetworkId, conf.NetworkId))
}

View file

@ -31,15 +31,23 @@ var (
ErrNoMnemonicProvided = errors.New("no mnemonic provided")
ErrIncompatibleVersion = errors.New("can't fetch account's data because remote nodes have incompatible protocol version. Please update anytype to the latest version")
ErrAnotherProcessIsRunning = errors.New("another anytype process is running")
ErrFailedToFindAccountInfo = errors.New("failed to find account info")
ErrAccountIsDeleted = errors.New("account is deleted")
ErrAnotherProcessIsRunning = errors.New("another anytype process is running")
ErrFailedToFindAccountInfo = errors.New("failed to find account info")
ErrAccountIsDeleted = errors.New("account is deleted")
ErrAccountStoreIsNotMigrated = errors.New("account store is not migrated")
)
func (s *Service) AccountSelect(ctx context.Context, req *pb.RpcAccountSelectRequest) (*model.Account, error) {
if req.Id == "" {
return nil, ErrEmptyAccountID
}
curMigration := s.migrationManager.getOrCreateMigration(req.RootPath, req.Id)
if !curMigration.successful() {
return nil, ErrAccountStoreIsNotMigrated
}
if s.migrationManager.isRunning() {
return nil, ErrMigrationRunning
}
if runtime.GOOS != "android" && runtime.GOOS != "ios" {
s.traceRecorder.start()

View file

@ -0,0 +1,222 @@
package application
import (
"context"
"errors"
"os"
"path/filepath"
"sync"
"github.com/anyproto/any-sync/app"
"go.uber.org/zap"
"github.com/anyproto/anytype-heart/core/anytype"
"github.com/anyproto/anytype-heart/core/anytype/config"
"github.com/anyproto/anytype-heart/pb"
"github.com/anyproto/anytype-heart/pkg/lib/core"
)
var (
ErrAccountNotFound = errors.New("account not found")
ErrMigrationRunning = errors.New("migration is running")
)
func (s *Service) AccountMigrate(ctx context.Context, req *pb.RpcAccountMigrateRequest) error {
if s.rootPath == "" {
s.rootPath = req.RootPath
}
return s.migrationManager.getOrCreateMigration(req.RootPath, req.Id).wait()
}
func (s *Service) AccountMigrateCancel(ctx context.Context, req *pb.RpcAccountMigrateCancelRequest) error {
m := s.migrationManager.getMigration(req.Id)
if m == nil {
return nil
}
m.cancelMigration()
return nil
}
func (s *Service) migrate(ctx context.Context, id string) error {
res, err := core.WalletAccountAt(s.mnemonic, 0)
if err != nil {
return err
}
if _, err := os.Stat(filepath.Join(s.rootPath, id)); err != nil {
if os.IsNotExist(err) {
return ErrAccountNotFound
}
return err
}
cfg := anytype.BootstrapConfig(false, os.Getenv("ANYTYPE_STAGING") == "1")
cfg.PeferYamuxTransport = true
cfg.DisableNetworkIdCheck = true
comps := []app.Component{
cfg,
anytype.BootstrapWallet(s.rootPath, res),
s.eventSender,
}
a := &app.App{}
anytype.BootstrapMigration(a, comps...)
err = a.Start(ctx)
if err != nil {
return err
}
return a.Close(ctx)
}
type migration struct {
mx sync.Mutex
isStarted bool
isFinished bool
ctx context.Context
cancel context.CancelFunc
manager *migrationManager
err error
id string
done chan struct{}
}
func newMigration(m *migrationManager, id string) *migration {
ctx, cancel := context.WithCancel(context.Background())
return &migration{
ctx: ctx,
cancel: cancel,
done: make(chan struct{}),
id: id,
manager: m,
}
}
func newSuccessfulMigration(manager *migrationManager, id string) *migration {
m := newMigration(manager, id)
m.setFinished(nil, false)
return m
}
func (m *migration) setFinished(err error, notify bool) {
m.mx.Lock()
defer m.mx.Unlock()
m.isFinished = true
m.err = err
close(m.done)
if notify {
m.manager.setMigrationRunning(m.id, false)
}
}
func (m *migration) cancelMigration() {
m.cancel()
err := m.wait()
if err != nil {
log.Warn("failed to wait for migration to finish", zap.Error(err))
}
}
func (m *migration) wait() error {
m.mx.Lock()
if !m.manager.setMigrationRunning(m.id, true) {
m.mx.Unlock()
return ErrMigrationRunning
}
if !m.isStarted {
m.isStarted = true
} else {
m.mx.Unlock()
<-m.done
return m.err
}
m.mx.Unlock()
err := m.manager.service.migrate(m.ctx, m.id)
if err != nil {
m.setFinished(err, true)
return err
}
m.setFinished(nil, true)
return nil
}
func (m *migration) successful() bool {
m.mx.Lock()
defer m.mx.Unlock()
return m.isFinished && m.err == nil
}
func (m *migration) finished() bool {
m.mx.Lock()
defer m.mx.Unlock()
return m.isFinished
}
type migrationManager struct {
migrations map[string]*migration
service *Service
runningMigration string
sync.Mutex
}
func newMigrationManager(s *Service) *migrationManager {
return &migrationManager{
service: s,
}
}
func (m *migrationManager) setMigrationRunning(id string, isRunning bool) bool {
m.Lock()
defer m.Unlock()
if (m.runningMigration != "" && m.runningMigration != id) && isRunning {
return false
}
if m.runningMigration == "" && !isRunning {
panic("migration is not running")
}
if isRunning {
m.runningMigration = id
} else {
m.runningMigration = ""
}
return true
}
func (m *migrationManager) isRunning() bool {
m.Lock()
defer m.Unlock()
return m.runningMigration != ""
}
func (m *migrationManager) getOrCreateMigration(rootPath, id string) *migration {
m.Lock()
defer m.Unlock()
if m.migrations == nil {
m.migrations = make(map[string]*migration)
}
if m.migrations[id] == nil {
sqlitePath := filepath.Join(rootPath, id, config.SpaceStoreSqlitePath)
baderPath := filepath.Join(rootPath, id, config.SpaceStoreBadgerPath)
if anyPathExists([]string{sqlitePath, baderPath}) {
m.migrations[id] = newMigration(m, id)
} else {
m.migrations[id] = newSuccessfulMigration(m, id)
}
}
if m.migrations[id].finished() && !m.migrations[id].successful() {
// resetting migration
m.migrations[id] = newMigration(m, id)
}
return m.migrations[id]
}
func (m *migrationManager) getMigration(id string) *migration {
m.Lock()
defer m.Unlock()
return m.migrations[id]
}
func anyPathExists(paths []string) bool {
for _, path := range paths {
if _, err := os.Stat(path); err == nil {
return true
}
}
return false
}

View file

@ -30,16 +30,20 @@ type Service struct {
eventSender event.Sender
sessions session.Service
traceRecorder *traceRecorder
migrationManager *migrationManager
appAccountStartInProcessCancel context.CancelFunc
appAccountStartInProcessCancelMutex sync.Mutex
}
func New() *Service {
return &Service{
s := &Service{
sessions: session.New(),
traceRecorder: &traceRecorder{},
}
m := newMigrationManager(s)
s.migrationManager = m
return s
}
func (s *Service) GetApp() *app.App {

View file

@ -1,6 +1,7 @@
package block
import (
"context"
"encoding/json"
"fmt"
"net/http"
@ -22,6 +23,7 @@ import (
func (s *Service) DebugRouter(r chi.Router) {
r.Get("/objects", debug.JSONHandler(s.debugListObjects))
r.Get("/tree/{id}", debug.JSONHandler(s.debugTree))
r.Get("/tree_in_space/{spaceId}/{id}", debug.JSONHandler(s.debugTreeInSpace))
r.Get("/objects_per_space/{spaceId}", debug.JSONHandler(s.debugListObjectsPerSpace))
r.Get("/objects/{id}", debug.JSONHandler(s.debugGetObject))
}
@ -122,6 +124,44 @@ func (s *Service) debugTree(req *http.Request) (debugTree, error) {
return result, err
}
// TODO Refactor
func (s *Service) debugTreeInSpace(req *http.Request) (debugTree, error) {
spaceId := chi.URLParam(req, "spaceId")
id := chi.URLParam(req, "id")
result := debugTree{
Id: id,
}
spc, err := s.spaceService.Get(context.Background(), spaceId)
if err != nil {
return result, fmt.Errorf("get space: %w", err)
}
err = spc.Do(id, func(sb smartblock.SmartBlock) error {
ot := sb.Tree()
return ot.IterateRoot(source.UnmarshalChange, func(change *objecttree.Change) bool {
change.Next = nil
raw, err := json.Marshal(change)
if err != nil {
log.Error("debug tree: marshal change", zap.Error(err))
return false
}
ts := time.Unix(change.Timestamp, 0)
ch := debugChange{
Change: raw,
Timestamp: ts.Format(time.RFC3339),
}
if change.Identity != nil {
ch.Identity = change.Identity.Account()
}
result.Changes = append(result.Changes, ch)
return true
})
})
return result, err
}
func (s *Service) getDebugObject(id string) (debugObject, error) {
var obj debugObject
err := cache.Do(s, id, func(sb smartblock.SmartBlock) error {

View file

@ -39,6 +39,7 @@ type StoreObject interface {
ToggleMessageReaction(ctx context.Context, messageId string, emoji string) error
DeleteMessage(ctx context.Context, messageId string) error
SubscribeLastMessages(ctx context.Context, limit int) ([]*model.ChatMessage, int, error)
MarkSeenHeads(heads []string)
Unsubscribe() error
}
@ -112,6 +113,10 @@ func (s *storeObject) onUpdate() {
s.subscription.flush()
}
func (s *storeObject) MarkSeenHeads(heads []string) {
s.storeSource.MarkSeenHeads(heads)
}
func (s *storeObject) GetMessagesByIds(ctx context.Context, messageIds []string) ([]*model.ChatMessage, error) {
coll, err := s.store.Collection(ctx, collectionName)
if err != nil {

View file

@ -1,72 +1,60 @@
package debug
import (
"io/ioutil"
"log"
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/anyproto/anytype-heart/core/debug/treearchive"
)
func TestBuildFast(t *testing.T) {
// Specify the directory you want to iterate
dir := "./testdata"
// Read the directory
files, err := ioutil.ReadDir(dir)
if err != nil {
t.Fatalf("Failed to read dir: %s", err)
}
// Iterate over the files
for _, file := range files {
t.Run(file.Name(), func(t *testing.T) {
filePath := filepath.Join(dir, file.Name())
// open the file
f, err := os.Open(filePath)
if err != nil {
t.Fatalf("Failed to open file: %s", err)
}
defer f.Close()
testBuildFast(t, filePath)
})
}
}
func testBuildFast(b *testing.T, filepath string) {
// todo: replace with less heavy tree
archive, err := treearchive.Open(filepath)
if err != nil {
require.NoError(b, err)
}
defer archive.Close()
importer := treearchive.NewTreeImporter(archive.ListStorage(), archive.TreeStorage())
err = importer.Import(false, "")
if err != nil {
log.Fatal("can't import the tree", err)
}
start := time.Now()
_, err = importer.State()
if err != nil {
log.Fatal("can't build state:", err)
}
b.Logf("fast build took %s", time.Since(start))
importer2 := treearchive.NewTreeImporter(archive.ListStorage(), archive.TreeStorage())
err = importer2.Import(false, "")
if err != nil {
log.Fatal("can't import the tree", err)
}
}
// TODO: revive at some point
// func TestBuildFast(t *testing.T) {
// // Specify the directory you want to iterate
// dir := "./testdata"
//
// // Read the directory
// files, err := ioutil.ReadDir(dir)
// if err != nil {
// t.Fatalf("Failed to read dir: %s", err)
// }
//
// // Iterate over the files
// for _, file := range files {
// t.Run(file.Name(), func(t *testing.T) {
// filePath := filepath.Join(dir, file.Name())
//
// // open the file
// f, err := os.Open(filePath)
// if err != nil {
// t.Fatalf("Failed to open file: %s", err)
// }
// defer f.Close()
//
// testBuildFast(t, filePath)
// })
// }
// }
//
// func testBuildFast(b *testing.T, filepath string) {
// // todo: replace with less heavy tree
// archive, err := treearchive.Open(filepath)
// if err != nil {
// require.NoError(b, err)
// }
// defer archive.Close()
//
// importer := exporter.NewTreeImporter(archive.ListStorage(), archive.TreeStorage())
//
// err = importer.Import(false, "")
// if err != nil {
// log.Fatal("can't import the tree", err)
// }
//
// start := time.Now()
// _, err = importer.State()
// if err != nil {
// log.Fatal("can't build state:", err)
// }
// b.Logf("fast build took %s", time.Since(start))
//
// importer2 := exporter.NewTreeImporter(archive.ListStorage(), archive.TreeStorage())
//
// err = importer2.Import(false, "")
// if err != nil {
// log.Fatal("can't import the tree", err)
// }
//
// }

View file

@ -47,10 +47,6 @@ func (stx *StoreStateTx) NextOrder(prev string) string {
return lexId.Next(prev)
}
func (stx *StoreStateTx) NextBeforeOrder(prev string, before string) (string, error) {
return lexId.NextBefore(prev, before)
}
func (stx *StoreStateTx) SetOrder(changeId, order string) (err error) {
stx.arena.Reset()
obj := stx.arena.NewObject()

View file

@ -4,9 +4,8 @@ import (
"sync"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/anytype-heart/space/spacecore/storage"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore"
)
const CName = "block.object.resolver"
@ -21,12 +20,12 @@ func New() Resolver {
}
type resolver struct {
storage storage.ClientStorage
objectStore objectstore.ObjectStore
sync.Mutex
}
func (r *resolver) Init(a *app.App) (err error) {
r.storage = a.MustComponent(spacestorage.CName).(storage.ClientStorage)
r.objectStore = a.MustComponent(objectstore.CName).(objectstore.ObjectStore)
return
}
@ -35,5 +34,5 @@ func (r *resolver) Name() (name string) {
}
func (r *resolver) ResolveSpaceID(objectID string) (string, error) {
return r.storage.GetSpaceID(objectID)
return r.objectStore.GetSpaceId(objectID)
}

View file

@ -20,7 +20,7 @@ func Test_Payloads(t *testing.T) {
changePayload := []byte("some")
keys, err := accountdata.NewRandom()
require.NoError(t, err)
aclList, err := list.NewTestDerivedAcl("spaceId", keys)
aclList, err := list.NewInMemoryDerivedAcl("spaceId", keys)
require.NoError(t, err)
timestamp := time.Now().Add(time.Hour).Unix()

View file

@ -102,7 +102,7 @@ func NewTreeSyncer(spaceId string) treesyncer.TreeSyncer {
func (t *treeSyncer) Init(a *app.App) (err error) {
t.isSyncing = true
spaceStorage := app.MustComponent[spacestorage.SpaceStorage](a)
t.spaceSettingsId = spaceStorage.SpaceSettingsId()
t.spaceSettingsId = spaceStorage.StateStorage().SettingsId()
t.treeManager = app.MustComponent[treemanager.TreeManager](a)
t.nodeConf = app.MustComponent[nodeconf.NodeConf](a)
t.syncedTreeRemover = app.MustComponent[SyncedTreeRemover](a)

View file

@ -8,6 +8,7 @@ import (
"time"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/headsync/statestorage/mock_statestorage"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
"github.com/anyproto/any-sync/commonspace/object/tree/synctree/mock_synctree"
"github.com/anyproto/any-sync/commonspace/object/treemanager/mock_treemanager"
@ -32,6 +33,7 @@ type fixture struct {
nodeConf *mock_nodeconf.MockService
syncStatus *mock_treesyncer.MockSyncedTreeRemover
syncDetailsUpdater *mock_treesyncer.MockSyncDetailsUpdater
stateStorage *mock_statestorage.MockStateStorage
}
func newFixture(t *testing.T, spaceId string) *fixture {
@ -44,7 +46,9 @@ func newFixture(t *testing.T, spaceId string) *fixture {
syncStatus := mock_treesyncer.NewMockSyncedTreeRemover(t)
syncDetailsUpdater := mock_treesyncer.NewMockSyncDetailsUpdater(t)
spaceStorage := mock_spacestorage.NewMockSpaceStorage(ctrl)
spaceStorage.EXPECT().SpaceSettingsId().Return("spaceSettingsId").AnyTimes()
stateStorage := mock_statestorage.NewMockStateStorage(ctrl)
spaceStorage.EXPECT().StateStorage().AnyTimes().Return(stateStorage)
stateStorage.EXPECT().SettingsId().AnyTimes().Return("settingsId")
a := new(app.App)
a.Register(testutil.PrepareMock(context.Background(), a, treeManager)).
@ -64,6 +68,7 @@ func newFixture(t *testing.T, spaceId string) *fixture {
nodeConf: nodeConf,
syncStatus: syncStatus,
syncDetailsUpdater: syncDetailsUpdater,
stateStorage: stateStorage,
}
}

View file

@ -342,7 +342,7 @@ func (s *Service) SpaceInitChat(ctx context.Context, spaceId string) error {
return err
}
if spaceChatExists, err := spc.Storage().HasTree(chatId); err != nil {
if spaceChatExists, err := spc.Storage().HasTree(ctx, chatId); err != nil {
return err
} else if !spaceChatExists {
_, err = s.objectCreator.AddChatDerivedObject(ctx, spc, workspaceId)

View file

@ -276,6 +276,39 @@ func (_c *MockStore_Id_Call) RunAndReturn(run func() string) *MockStore_Id_Call
return _c
}
// MarkSeenHeads provides a mock function with given fields: heads
func (_m *MockStore) MarkSeenHeads(heads []string) {
_m.Called(heads)
}
// MockStore_MarkSeenHeads_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkSeenHeads'
type MockStore_MarkSeenHeads_Call struct {
*mock.Call
}
// MarkSeenHeads is a helper method to define mock.On call
// - heads []string
func (_e *MockStore_Expecter) MarkSeenHeads(heads interface{}) *MockStore_MarkSeenHeads_Call {
return &MockStore_MarkSeenHeads_Call{Call: _e.mock.On("MarkSeenHeads", heads)}
}
func (_c *MockStore_MarkSeenHeads_Call) Run(run func(heads []string)) *MockStore_MarkSeenHeads_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].([]string))
})
return _c
}
func (_c *MockStore_MarkSeenHeads_Call) Return() *MockStore_MarkSeenHeads_Call {
_c.Call.Return()
return _c
}
func (_c *MockStore_MarkSeenHeads_Call) RunAndReturn(run func([]string)) *MockStore_MarkSeenHeads_Call {
_c.Call.Return(run)
return _c
}
// PushChange provides a mock function with given fields: params
func (_m *MockStore) PushChange(params source.PushChangeParams) (string, error) {
ret := _m.Called(params)

View file

@ -11,7 +11,6 @@ import (
"github.com/anyproto/any-sync/commonspace/object/acl/list"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
"github.com/anyproto/any-sync/commonspace/object/tree/synctree/updatelistener"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/commonspace/objecttreebuilder"
"github.com/anyproto/any-sync/commonspace/spacestorage"
@ -100,7 +99,7 @@ type BuildOptions struct {
func (b *BuildOptions) BuildTreeOpts() objecttreebuilder.BuildTreeOpts {
return objecttreebuilder.BuildTreeOpts{
Listener: b.Listener,
TreeBuilder: func(treeStorage treestorage.TreeStorage, aclList list.AclList) (objecttree.ObjectTree, error) {
TreeBuilder: func(treeStorage objecttree.Storage, aclList list.AclList) (objecttree.ObjectTree, error) {
ot, err := objecttree.BuildKeyFilterableObjectTree(treeStorage, aclList)
if err != nil {
return nil, err
@ -123,7 +122,7 @@ func (s *service) NewSource(ctx context.Context, space Space, id string, buildOp
if err != nil {
return nil, err
}
err = s.storageService.BindSpaceID(src.SpaceID(), src.Id())
err = s.objectStore.BindSpaceId(src.SpaceID(), src.Id())
if err != nil {
return nil, fmt.Errorf("store space id for object: %w", err)
}
@ -246,7 +245,7 @@ func (s *service) RegisterStaticSource(src Source) error {
s.mu.Lock()
defer s.mu.Unlock()
s.staticIds[src.Id()] = src
err := s.storageService.BindSpaceID(src.SpaceID(), src.Id())
err := s.objectStore.BindSpaceId(src.SpaceID(), src.Id())
if err != nil {
return fmt.Errorf("store space id for object: %w", err)
}

View file

@ -11,7 +11,7 @@ import (
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
"github.com/anyproto/any-sync/commonspace/object/tree/synctree"
"github.com/anyproto/any-sync/commonspace/object/tree/synctree/updatelistener"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/objecttreebuilder"
"github.com/gogo/protobuf/proto"
"github.com/golang/snappy"
@ -33,6 +33,7 @@ type Store interface {
ReadStoreDoc(ctx context.Context, stateStore *storestate.StoreState, onUpdateHook func()) (err error)
PushStoreChange(ctx context.Context, params PushStoreChangeParams) (changeId string, err error)
SetPushChangeHook(onPushChange PushChangeHook)
MarkSeenHeads(heads []string)
}
type PushStoreChangeParams struct {
@ -51,6 +52,7 @@ type store struct {
store *storestate.StoreState
onUpdateHook func()
onPushChange PushChangeHook
diffManager *objecttree.DiffManager
sbType smartblock.SmartBlockType
}
@ -62,6 +64,17 @@ func (s *store) SetPushChangeHook(onPushChange PushChangeHook) {
s.onPushChange = onPushChange
}
func (s *store) createDiffManager(ctx context.Context, curTreeHeads, seenHeads []string) (err error) {
buildTree := func(heads []string) (objecttree.ReadableObjectTree, error) {
return s.space.TreeBuilder().BuildHistoryTree(ctx, s.Id(), objecttreebuilder.HistoryTreeOpts{
Heads: heads,
Include: true,
})
}
s.diffManager, err = objecttree.NewDiffManager(seenHeads, curTreeHeads, buildTree, func(ids []string) {})
return
}
func (s *store) ReadDoc(ctx context.Context, receiver ChangeReceiver, empty bool) (doc state.Doc, err error) {
s.receiver = receiver
setter, ok := s.ObjectTree.(synctree.ListenerSetter)
@ -100,7 +113,10 @@ func (s *store) PushChange(params PushChangeParams) (id string, err error) {
func (s *store) ReadStoreDoc(ctx context.Context, storeState *storestate.StoreState, onUpdateHook func()) (err error) {
s.onUpdateHook = onUpdateHook
s.store = storeState
err = s.createDiffManager(ctx, s.source.Tree().Heads(), nil)
if err != nil {
return err
}
tx, err := s.store.NewTx(ctx)
if err != nil {
return
@ -143,11 +159,11 @@ func (s *store) PushStoreChange(ctx context.Context, params PushStoreChangeParam
IsEncrypted: true,
DataType: dataType,
Timestamp: params.Time.Unix(),
}, func(change *treechangeproto.RawTreeChangeWithId) error {
order := tx.NextOrder(tx.GetMaxOrder())
}, func(change objecttree.StorageChange) error {
// TODO: get order here
err = tx.ApplyChangeSet(storestate.ChangeSet{
Id: change.Id,
Order: order,
Order: change.OrderId,
Changes: params.Changes,
Creator: s.accountService.AccountID(),
Timestamp: params.Time.Unix(),
@ -169,6 +185,14 @@ func (s *store) PushStoreChange(ctx context.Context, params PushStoreChangeParam
if err == nil {
s.onUpdateHook()
}
ch, err := s.ObjectTree.GetChange(changeId)
if err != nil {
return "", err
}
s.diffManager.Add(&objecttree.Change{
Id: changeId,
PreviousIds: ch.PreviousIds,
})
return changeId, err
}
@ -185,12 +209,17 @@ func (s *store) update(ctx context.Context, tree objecttree.ObjectTree) error {
return errors.Join(tx.Rollback(), err)
}
err = tx.Commit()
s.diffManager.Update(tree)
if err == nil {
s.onUpdateHook()
}
return err
}
func (s *store) MarkSeenHeads(heads []string) {
s.diffManager.Remove(heads)
}
func (s *store) Update(tree objecttree.ObjectTree) error {
err := s.update(context.Background(), tree)
if err != nil {

View file

@ -24,62 +24,14 @@ type storeApply struct {
}
func (a *storeApply) Apply() (err error) {
maxOrder := a.tx.GetMaxOrder()
isEmpty := maxOrder == ""
iterErr := a.ot.IterateRoot(UnmarshalStoreChange, func(change *objecttree.Change) bool {
// not a new change - remember and continue
if !a.allIsNew && !change.IsNew && !isEmpty {
a.prevChange = change
a.prevOrder = ""
if !a.allIsNew && !change.IsNew {
return true
}
currOrder, curOrdErr := a.tx.GetOrder(change.Id)
if curOrdErr != nil {
if !errors.Is(curOrdErr, storestate.ErrOrderNotFound) {
err = curOrdErr
return false
}
} else {
// change has been handled before
a.prevChange = change
a.prevOrder = currOrder
return true
}
prevOrder, prevOrdErr := a.getPrevOrder()
if prevOrdErr != nil {
if !errors.Is(prevOrdErr, storestate.ErrOrderNotFound) {
err = prevOrdErr
return false
}
if !isEmpty {
// it should not happen, consistency with tree and store broken
err = fmt.Errorf("unable to find previous order")
return false
}
}
if prevOrder == a.tx.GetMaxOrder() {
// insert on top - just create next id
currOrder = a.tx.NextOrder(prevOrder)
} else {
// insert in the middle - find next order and create id between
nextOrder, nextOrdErr := a.findNextOrder(change.Id)
if nextOrdErr != nil {
// it should not happen, consistency with tree and store broken
err = errors.Join(nextOrdErr, fmt.Errorf("unable to find next order"))
return false
}
if currOrder, err = a.tx.NextBeforeOrder(prevOrder, nextOrder); err != nil {
return false
}
}
if err = a.applyChange(change, currOrder); err != nil {
if err = a.applyChange(change); err != nil {
return false
}
a.prevOrder = currOrder
a.prevChange = change
return true
})
if err == nil && iterErr != nil {
@ -88,18 +40,18 @@ func (a *storeApply) Apply() (err error) {
return
}
func (a *storeApply) applyChange(change *objecttree.Change, order string) (err error) {
func (a *storeApply) applyChange(change *objecttree.Change) (err error) {
storeChange, ok := change.Model.(*pb.StoreChange)
if !ok {
// if it is root
if _, ok := change.Model.(*treechangeproto.TreeChangeInfo); ok {
return a.tx.SetOrder(change.Id, order)
return a.tx.SetOrder(change.Id, change.OrderId)
}
return fmt.Errorf("unexpected change content type: %T", change.Model)
}
set := storestate.ChangeSet{
Id: change.Id,
Order: order,
Order: change.OrderId,
Changes: storeChange.ChangeSet,
Creator: change.Identity.Account(),
Timestamp: change.Timestamp,
@ -111,56 +63,3 @@ func (a *storeApply) applyChange(change *objecttree.Change, order string) (err e
}
return err
}
func (a *storeApply) getPrevOrder() (order string, err error) {
if a.prevOrder != "" {
return a.prevOrder, nil
}
if a.prevChange == nil {
return "", nil
}
if order, err = a.tx.GetOrder(a.prevChange.Id); err != nil {
return
}
a.prevOrder = order
return
}
func (a *storeApply) findNextOrder(changeId string) (order string, err error) {
if order = a.findNextInCache(changeId); order != "" {
return
}
a.nextCacheChange = map[string]struct{}{}
iterErr := a.ot.IterateFrom(changeId, UnmarshalStoreChange, func(change *objecttree.Change) bool {
order, err = a.tx.GetOrder(change.Id)
if err != nil {
if errors.Is(err, storestate.ErrOrderNotFound) {
// no order - remember id and move forward
a.nextCacheChange[change.Id] = struct{}{}
return true
} else {
return false
}
}
// order found
a.nextCachedOrder = order
return false
})
if err == nil && iterErr != nil {
return "", iterErr
}
return
}
func (a *storeApply) findNextInCache(changeId string) (order string) {
if a.nextCacheChange == nil {
return ""
}
if _, ok := a.nextCacheChange[changeId]; ok {
return a.nextCachedOrder
}
a.nextCachedOrder = ""
a.nextCacheChange = nil
return ""
}

View file

@ -2,12 +2,10 @@ package source
import (
"context"
"fmt"
"os"
"path/filepath"
"sort"
"testing"
"time"
anystore "github.com/anyproto/any-store"
"github.com/anyproto/any-sync/commonspace/object/accountdata"
@ -19,6 +17,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"golang.org/x/sys/unix"
"github.com/anyproto/anytype-heart/core/block/editor/storestate"
"github.com/anyproto/anytype-heart/pb"
@ -98,90 +97,6 @@ func TestStoreApply_RealTree(t *testing.T) {
})
}
func TestStoreApply_Apply(t *testing.T) {
t.Run("new tree", func(t *testing.T) {
fx := newMockTreeStoreFx(t)
tx := fx.RequireTx(t)
changes := []*objecttree.Change{
testChange("1", false),
testChange("2", false),
testChange("3", false),
}
fx.ApplyChanges(t, tx, changes...)
require.NoError(t, tx.Commit())
})
t.Run("insert middle", func(t *testing.T) {
fx := newMockTreeStoreFx(t)
tx := fx.RequireTx(t)
changes := []*objecttree.Change{
testChange("1", false),
testChange("2", false),
testChange("3", false),
}
fx.ApplyChanges(t, tx, changes...)
require.NoError(t, tx.Commit())
tx = fx.RequireTx(t)
changes = []*objecttree.Change{
testChange("1", false),
testChange("1.1", true),
testChange("1.2", true),
testChange("1.3", true),
testChange("2", false),
testChange("2.2", true),
testChange("3", false),
}
fx.ExpectTreeFrom("1.1", changes[1:]...)
fx.ExpectTreeFrom("2.2", changes[6:]...)
fx.ApplyChanges(t, tx, changes...)
require.NoError(t, tx.Commit())
})
t.Run("append", func(t *testing.T) {
fx := newMockTreeStoreFx(t)
tx := fx.RequireTx(t)
changes := []*objecttree.Change{
testChange("1", false),
testChange("2", false),
testChange("3", false),
}
fx.ApplyChanges(t, tx, changes...)
require.NoError(t, tx.Commit())
tx = fx.RequireTx(t)
changes = []*objecttree.Change{
testChange("1", false),
testChange("2", false),
testChange("3", false),
testChange("4", true),
testChange("5", true),
testChange("6", true),
}
fx.ApplyChanges(t, tx, changes...)
require.NoError(t, tx.Commit())
})
}
func TestStoreApply_Apply10000(t *testing.T) {
fx := newMockTreeStoreFx(t)
tx := fx.RequireTx(t)
changes := make([]*objecttree.Change, 100000)
for i := range changes {
changes[i] = testChange(fmt.Sprint(i), false)
}
st := time.Now()
applier := &storeApply{
tx: tx,
ot: fx.mockTree,
}
fx.ExpectTree(changes...)
require.NoError(t, applier.Apply())
t.Logf("apply dur: %v;", time.Since(st))
st = time.Now()
require.NoError(t, tx.Commit())
t.Logf("commit dur: %v;", time.Since(st))
}
type storeFx struct {
state *storestate.StoreState
mockTree *mock_objecttree.MockObjectTree
@ -243,37 +158,6 @@ func (fx *storeFx) ApplyChanges(t *testing.T, tx *storestate.StoreStateTx, chang
fx.AssertOrder(t, tx, changes...)
}
func newMockTreeStoreFx(t testing.TB) *storeFx {
tmpDir, err := os.MkdirTemp("", "source_store_*")
require.NoError(t, err)
db, err := anystore.Open(ctx, filepath.Join(tmpDir, "test.db"), nil)
require.NoError(t, err)
ctrl := gomock.NewController(t)
t.Cleanup(func() {
if db != nil {
_ = db.Close()
}
ctrl.Finish()
if tmpDir != "" {
_ = os.RemoveAll(tmpDir)
}
})
state, err := storestate.New(ctx, "source_test", db, storestate.DefaultHandler{Name: "default"})
require.NoError(t, err)
tree := mock_objecttree.NewMockObjectTree(ctrl)
tree.EXPECT().Id().Return("root").AnyTimes()
return &storeFx{
state: state,
mockTree: tree,
db: db,
}
}
func newRealTreeStoreFx(t testing.TB) *storeFx {
tmpDir, err := os.MkdirTemp("", "source_store_*")
require.NoError(t, err)
@ -295,14 +179,16 @@ func newRealTreeStoreFx(t testing.TB) *storeFx {
state, err := storestate.New(ctx, "source_test", db, storestate.DefaultHandler{Name: "default"})
require.NoError(t, err)
aclList, _ := prepareAclList(t)
objTree, err := buildTree(aclList)
objTree, err := buildTree(t, aclList)
require.NoError(t, err)
fx := &storeFx{
state: state,
realTree: objTree,
aclList: aclList,
changeCreator: objecttree.NewMockChangeCreator(),
db: db,
state: state,
realTree: objTree,
aclList: aclList,
changeCreator: objecttree.NewMockChangeCreator(func() anystore.DB {
return createStore(ctx, t)
}),
db: db,
}
tx := fx.RequireTx(t)
defer tx.Rollback()
@ -321,6 +207,7 @@ func testChange(id string, isNew bool) *objecttree.Change {
return &objecttree.Change{
Id: id,
OrderId: id,
IsNew: isNew,
Model: &pb.StoreChange{},
Identity: pub,
@ -330,15 +217,17 @@ func testChange(id string, isNew bool) *objecttree.Change {
func prepareAclList(t testing.TB) (list.AclList, *accountdata.AccountKeys) {
randKeys, err := accountdata.NewRandom()
require.NoError(t, err)
aclList, err := list.NewTestDerivedAcl("spaceId", randKeys)
aclList, err := list.NewInMemoryDerivedAcl("spaceId", randKeys)
require.NoError(t, err, "building acl list should be without error")
return aclList, randKeys
}
func buildTree(aclList list.AclList) (objecttree.ObjectTree, error) {
changeCreator := objecttree.NewMockChangeCreator()
treeStorage := changeCreator.CreateNewTreeStorage("0", aclList.Head().Id, false)
func buildTree(t testing.TB, aclList list.AclList) (objecttree.ObjectTree, error) {
changeCreator := objecttree.NewMockChangeCreator(func() anystore.DB {
return createStore(ctx, t)
})
treeStorage := changeCreator.CreateNewTreeStorage(t.(*testing.T), "0", aclList.Head().Id, false)
tree, err := objecttree.BuildTestableTree(treeStorage, aclList)
if err != nil {
return nil, err
@ -346,3 +235,22 @@ func buildTree(aclList list.AclList) (objecttree.ObjectTree, error) {
tree.SetFlusher(objecttree.MarkNewChangeFlusher())
return tree, nil
}
func createStore(ctx context.Context, t testing.TB) anystore.DB {
return createNamedStore(ctx, t, "changes.db")
}
func createNamedStore(ctx context.Context, t testing.TB, name string) anystore.DB {
path := filepath.Join(t.TempDir(), name)
db, err := anystore.Open(ctx, path, nil)
require.NoError(t, err)
t.Cleanup(func() {
err := db.Close()
require.NoError(t, err)
unix.Rmdir(path)
})
return objecttree.TestStore{
DB: db,
Path: path,
}
}

View file

@ -83,13 +83,13 @@ func (c *configFetcher) updateStatus(ctx context.Context) (err error) {
techSpace := c.getter.TechSpace()
res, err := c.client.StatusCheck(ctx, techSpace.Id())
if errors.Is(err, coordinatorproto.ErrSpaceNotExists) {
header, sErr := techSpace.Storage().SpaceHeader()
state, sErr := techSpace.Storage().StateStorage().GetState(ctx)
if sErr != nil {
return sErr
}
payload := coordinatorclient.SpaceSignPayload{
SpaceId: header.Id,
SpaceHeader: header.RawHeader,
SpaceId: techSpace.Id(),
SpaceHeader: state.SpaceHeader,
OldAccount: c.wallet.GetOldAccountKey(),
Identity: c.wallet.GetAccountPrivkey(),
}

View file

@ -0,0 +1,111 @@
package exporter
import (
"context"
"fmt"
anystore "github.com/anyproto/any-store"
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
"github.com/anyproto/any-sync/commonspace/object/accountdata"
"github.com/anyproto/any-sync/commonspace/object/acl/list"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/util/crypto"
"golang.org/x/exp/slices"
)
type DataConverter interface {
Unmarshall(dataType string, decrypted []byte) (any, error)
Marshall(model any) (data []byte, dataType string, err error)
}
func prepareExport(ctx context.Context, readable objecttree.ReadableObjectTree, store anystore.DB) (objecttree.ObjectTree, error) {
headStorage, err := headstorage.New(ctx, store)
if err != nil {
return nil, err
}
acl := readable.AclList()
root := acl.Root()
listStorage, err := list.CreateStorage(ctx, root, headStorage, store)
if err != nil {
return nil, err
}
keys, err := accountdata.NewRandom()
if err != nil {
return nil, err
}
newAcl, err := list.BuildAclListWithIdentity(keys, listStorage, list.NoOpAcceptorVerifier{})
if err != nil {
return nil, err
}
treeStorage, err := objecttree.CreateStorage(ctx, readable.Header(), headStorage, store)
if err != nil {
return nil, err
}
writeTree, err := objecttree.BuildTestableTree(treeStorage, newAcl)
if err != nil {
return nil, err
}
return writeTree, nil
}
type ExportParams struct {
Readable objecttree.ReadableObjectTree
Store anystore.DB
Converter DataConverter
}
func ExportTree(ctx context.Context, params ExportParams) error {
writeTree, err := prepareExport(ctx, params.Readable, params.Store)
if err != nil {
return err
}
var (
changeBuilder = objecttree.NewChangeBuilder(crypto.NewKeyStorage(), writeTree.Header())
converter = params.Converter
changes []*treechangeproto.RawTreeChangeWithId
)
err = writeTree.IterateRoot(
func(change *objecttree.Change, decrypted []byte) (any, error) {
return converter.Unmarshall(change.DataType, decrypted)
},
func(change *objecttree.Change) bool {
if change.Id == writeTree.Id() {
return true
}
var (
data []byte
dataType string
)
data, dataType, err = converter.Marshall(change.Model)
if err != nil {
return false
}
// that means that change is unencrypted
change.ReadKeyId = ""
change.Data = data
change.DataType = dataType
var raw *treechangeproto.RawTreeChangeWithId
raw, err = changeBuilder.Marshall(change)
if err != nil {
return false
}
changes = append(changes, raw)
return true
})
if err != nil {
return err
}
payload := objecttree.RawChangesPayload{
NewHeads: writeTree.Heads(),
RawChanges: changes,
}
res, err := writeTree.AddRawChanges(ctx, payload)
if err != nil {
return err
}
if !slices.Equal(res.Heads, writeTree.Heads()) {
return fmt.Errorf("heads mismatch: %v != %v", res.Heads, writeTree.Heads())
}
return nil
}

View file

@ -0,0 +1,96 @@
package exporter
import (
"context"
"os"
"path/filepath"
"strings"
anystore "github.com/anyproto/any-store"
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
"github.com/anyproto/any-sync/commonspace/object/accountdata"
"github.com/anyproto/any-sync/commonspace/object/acl/list"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
"github.com/anyproto/anytype-heart/util/ziputil"
)
type ImportResult struct {
List list.AclList
Storage objecttree.Storage
FolderPath string
}
func (i ImportResult) CreateReadableTree(fullTree bool, beforeId string) (objecttree.ReadableObjectTree, error) {
return objecttree.BuildNonVerifiableHistoryTree(objecttree.HistoryTreeParams{
Storage: i.Storage,
AclList: i.List,
Heads: i.Heads(fullTree, beforeId),
IncludeBeforeId: true,
})
}
func (i ImportResult) Heads(fullTree bool, beforeId string) []string {
if fullTree {
return nil
}
return []string{beforeId}
}
func ImportStorage(ctx context.Context, path string) (res ImportResult, err error) {
targetDir := strings.TrimSuffix(path, filepath.Ext(path))
if _, err = os.Stat(targetDir); err == nil {
err = os.RemoveAll(targetDir)
if err != nil {
return
}
}
if err = ziputil.UnzipFolder(path, targetDir); err != nil {
return
}
anyStore, err := anystore.Open(ctx, targetDir, nil)
if err != nil {
return
}
defer anyStore.Close()
var (
aclId string
treeId string
)
headStorage, err := headstorage.New(ctx, anyStore)
if err != nil {
return
}
err = headStorage.IterateEntries(ctx, headstorage.IterOpts{}, func(entry headstorage.HeadsEntry) (bool, error) {
if entry.CommonSnapshot == "" {
aclId = entry.Id
return true, nil
}
treeId = entry.Id
return true, nil
})
if err != nil {
return
}
listStorage, err := list.NewStorage(ctx, aclId, headStorage, anyStore)
if err != nil {
return
}
randomKeys, err := accountdata.NewRandom()
if err != nil {
return
}
acl, err := list.BuildAclListWithIdentity(randomKeys, listStorage, list.NoOpAcceptorVerifier{})
if err != nil {
return
}
treeStorage, err := objecttree.NewStorage(ctx, treeId, headStorage, anyStore)
if err != nil {
return
}
return ImportResult{
List: acl,
Storage: treeStorage,
FolderPath: targetDir,
}, nil
}

View file

@ -1,13 +1,10 @@
package treearchive
package exporter
import (
"errors"
"fmt"
"github.com/anyproto/any-sync/commonspace/object/acl/list"
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/anytype-heart/core/block/editor/state"
"github.com/anyproto/anytype-heart/core/block/source"
@ -43,22 +40,18 @@ func (m MarshalledJsonChange) MarshalJSON() ([]byte, error) {
type TreeImporter interface {
ObjectTree() objecttree.ReadableObjectTree
State() (*state.State, error) // set fullStateChain to true to get full state chain, otherwise only the last state will be returned
Import(fromRoot bool, beforeId string) error
State() (*state.State, error)
Json() (TreeJson, error)
ChangeAt(idx int) (IdChange, error)
}
type treeImporter struct {
listStorage liststorage.ListStorage
treeStorage treestorage.TreeStorage
objectTree objecttree.ReadableObjectTree
objectTree objecttree.ReadableObjectTree
}
func NewTreeImporter(listStorage liststorage.ListStorage, treeStorage treestorage.TreeStorage) TreeImporter {
func NewTreeImporter(objectTree objecttree.ReadableObjectTree) TreeImporter {
return &treeImporter{
listStorage: listStorage,
treeStorage: treeStorage,
objectTree: objectTree,
}
}
@ -83,25 +76,6 @@ func (t *treeImporter) State() (*state.State, error) {
return st, nil
}
func (t *treeImporter) Import(fullTree bool, beforeId string) (err error) {
aclList, err := list.BuildAclList(t.listStorage, list.NoOpAcceptorVerifier{})
if err != nil {
return
}
var heads []string
if !fullTree {
heads = []string{beforeId}
}
t.objectTree, err = objecttree.BuildNonVerifiableHistoryTree(objecttree.HistoryTreeParams{
TreeStorage: t.treeStorage,
AclList: aclList,
Heads: heads,
IncludeBeforeId: true,
})
return
}
func (t *treeImporter) Json() (treeJson TreeJson, err error) {
treeJson = TreeJson{
Id: t.objectTree.Id(),

View file

@ -1,99 +0,0 @@
package treearchive
import (
"archive/zip"
"encoding/json"
"io/fs"
"os"
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/consensus/consensusproto"
"github.com/anyproto/anytype-heart/core/debug/treearchive/zipaclstorage"
"github.com/anyproto/anytype-heart/core/debug/treearchive/ziptreestorage"
)
type ExportedObjectsJson struct {
AclId string `json:"aclId"`
TreeId string `json:"treeId"`
}
type ArchiveWriter struct {
zw *zip.Writer
zf fs.File
treeId string
aclId string
storages []flushableStorage
}
type flushableStorage interface {
FlushStorage() error
}
func NewArchiveWriter(path string) (*ArchiveWriter, error) {
zf, err := os.Create(path)
if err != nil {
return nil, err
}
zw := zip.NewWriter(zf)
return &ArchiveWriter{
zw: zw,
zf: zf,
}, nil
}
func (e *ArchiveWriter) ZipWriter() *zip.Writer {
return e.zw
}
func (e *ArchiveWriter) TreeStorage(root *treechangeproto.RawTreeChangeWithId) (treestorage.TreeStorage, error) {
e.treeId = root.Id
st, err := ziptreestorage.NewZipTreeWriteStorage(root, e.zw)
if err != nil {
return nil, err
}
e.storages = append(e.storages, st.(flushableStorage))
return st, nil
}
func (e *ArchiveWriter) ListStorage(root *consensusproto.RawRecordWithId) (liststorage.ListStorage, error) {
e.aclId = root.Id
st, err := zipaclstorage.NewACLWriteStorage(root, e.zw)
if err != nil {
return nil, err
}
e.storages = append(e.storages, st.(flushableStorage))
return st, nil
}
func (e *ArchiveWriter) Close() (err error) {
for _, st := range e.storages {
err = st.FlushStorage()
if err != nil {
return
}
}
exportedHeader, err := e.zw.CreateHeader(&zip.FileHeader{
Name: "exported.json",
Method: zip.Deflate,
})
if err != nil {
return
}
enc := json.NewEncoder(exportedHeader)
enc.SetIndent("", "\t")
err = enc.Encode(ExportedObjectsJson{
TreeId: e.treeId,
AclId: e.aclId,
})
if err != nil {
return
}
err = e.zw.Close()
if err != nil {
return
}
return e.zf.Close()
}

View file

@ -1,91 +0,0 @@
package treearchive
import (
"archive/zip"
"encoding/json"
"fmt"
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/gogo/protobuf/jsonpb"
"github.com/anyproto/anytype-heart/core/debug/treearchive/zipaclstorage"
"github.com/anyproto/anytype-heart/core/debug/treearchive/ziptreestorage"
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
)
type TreeArchive interface {
ListStorage() liststorage.ListStorage
TreeStorage() treestorage.TreeStorage
LocalStore() (*model.ObjectInfo, error)
Close() error
}
type treeArchive struct {
listStorage liststorage.ListStorage
treeStorage treestorage.TreeStorage
zr *zip.ReadCloser
}
// Open expects debug tree zip file
// returns TreeArchive that has ListStorage and TreeStorage
func Open(filename string) (tr TreeArchive, err error) {
zr, err := zip.OpenReader(filename)
if err != nil {
return nil, err
}
exported, err := zr.Open("exported.json")
if err != nil {
return
}
defer exported.Close()
expJson := &ExportedObjectsJson{}
if err = json.NewDecoder(exported).Decode(expJson); err != nil {
return
}
listStorage, err := zipaclstorage.NewZipAclReadStorage(expJson.AclId, zr)
if err != nil {
return
}
treeStorage, err := ziptreestorage.NewZipTreeReadStorage(expJson.TreeId, zr)
if err != nil {
return
}
return &treeArchive{
listStorage: listStorage,
treeStorage: treeStorage,
zr: zr,
}, nil
}
func (a *treeArchive) ListStorage() liststorage.ListStorage {
return a.listStorage
}
func (a *treeArchive) TreeStorage() treestorage.TreeStorage {
return a.treeStorage
}
func (a *treeArchive) LocalStore() (*model.ObjectInfo, error) {
for _, f := range a.zr.File {
if f.Name == "localstore.json" {
rd, err := f.Open()
if err != nil {
return nil, err
}
defer rd.Close()
var oi = &model.ObjectInfo{}
if err = jsonpb.Unmarshal(rd, oi); err != nil {
return nil, err
}
return oi, nil
}
}
return nil, fmt.Errorf("block logs file not found")
}
func (a *treeArchive) Close() (err error) {
return a.zr.Close()
}

View file

@ -1,77 +0,0 @@
package zipaclstorage
import (
"archive/zip"
"context"
"fmt"
"io"
"strings"
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
"github.com/anyproto/any-sync/consensus/consensusproto"
)
type zipAclReadStorage struct {
id string
files map[string]*zip.File
}
func NewZipAclReadStorage(id string, zr *zip.ReadCloser) (ls liststorage.ListStorage, err error) {
aclStorage := &zipAclReadStorage{
id: id,
files: map[string]*zip.File{},
}
for _, f := range zr.Reader.File {
if len(f.Name) > len(id) && strings.Contains(f.Name, id) {
split := strings.SplitAfter(id, "/")
last := split[len(split)-1]
aclStorage.files[last] = f
}
}
ls = aclStorage
return
}
func (z *zipAclReadStorage) Id() string {
return z.id
}
func (z *zipAclReadStorage) Root() (*consensusproto.RawRecordWithId, error) {
return z.readRecord(z.id)
}
func (z *zipAclReadStorage) Head() (string, error) {
return z.id, nil
}
func (z *zipAclReadStorage) SetHead(headId string) error {
panic("should not be called")
}
func (z *zipAclReadStorage) GetRawRecord(_ context.Context, id string) (*consensusproto.RawRecordWithId, error) {
return z.readRecord(id)
}
func (z *zipAclReadStorage) AddRawRecord(_ context.Context, _ *consensusproto.RawRecordWithId) (err error) {
panic("should not be called")
}
func (z *zipAclReadStorage) readRecord(id string) (rec *consensusproto.RawRecordWithId, err error) {
file, ok := z.files[id]
if !ok {
err = fmt.Errorf("object not found in storage")
return
}
opened, err := file.Open()
if err != nil {
return
}
defer opened.Close()
buf, err := io.ReadAll(opened)
if err != nil {
return
}
rec = &consensusproto.RawRecordWithId{Payload: buf, Id: id}
return
}

View file

@ -1,59 +0,0 @@
package zipaclstorage
import (
"archive/zip"
"context"
"strings"
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
"github.com/anyproto/any-sync/consensus/consensusproto"
)
type zipACLWriteStorage struct {
id string
zw *zip.Writer
}
func NewACLWriteStorage(root *consensusproto.RawRecordWithId, zw *zip.Writer) (ls liststorage.ListStorage, err error) {
ls = &zipACLWriteStorage{
id: root.Id,
zw: zw,
}
err = ls.AddRawRecord(context.Background(), root)
return
}
// nolint:revive
func (z *zipACLWriteStorage) Id() string {
return z.id
}
func (z *zipACLWriteStorage) Root() (*consensusproto.RawRecordWithId, error) {
panic("should not be called")
}
func (z *zipACLWriteStorage) Head() (string, error) {
return z.id, nil
}
func (z *zipACLWriteStorage) SetHead(_ string) error {
// TODO: As soon as our acls are writeable, this should be implemented
panic("should not be called")
}
func (z *zipACLWriteStorage) GetRawRecord(_ context.Context, _ string) (*consensusproto.RawRecordWithId, error) {
panic("should not be called")
}
func (z *zipACLWriteStorage) AddRawRecord(_ context.Context, rec *consensusproto.RawRecordWithId) (err error) {
wr, err := z.zw.Create(strings.Join([]string{z.id, rec.Id}, "/"))
if err != nil {
return
}
_, err = wr.Write(rec.Payload)
return
}
func (z *zipACLWriteStorage) FlushStorage() error {
return nil
}

View file

@ -1,116 +0,0 @@
package ziptreestorage
import (
"archive/zip"
"context"
"encoding/json"
"fmt"
"io"
"strings"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
)
type zipTreeReadStorage struct {
id string
heads []string
files map[string]*zip.File
}
func NewZipTreeReadStorage(id string, zr *zip.ReadCloser) (st treestorage.TreeStorage, err error) {
zrs := &zipTreeReadStorage{
id: id,
heads: nil,
files: map[string]*zip.File{},
}
for _, f := range zr.Reader.File {
if len(f.Name) > len(id) && strings.Contains(f.Name, id) {
split := strings.Split(f.Name, "/")
last := split[len(split)-1]
zrs.files[last] = f
}
}
data, ok := zrs.files["data.json"]
if !ok {
err = fmt.Errorf("no data.json in archive")
return
}
dataOpened, err := data.Open()
if err != nil {
return
}
defer dataOpened.Close()
headsEntry := &HeadsJsonEntry{}
if err = json.NewDecoder(dataOpened).Decode(headsEntry); err != nil {
return
}
zrs.heads = headsEntry.Heads
st = zrs
return
}
func (t *zipTreeReadStorage) GetAllChangeIds() (chs []string, err error) {
return nil, fmt.Errorf("get all change ids should not be called")
}
func (z *zipTreeReadStorage) Id() string {
return z.id
}
func (z *zipTreeReadStorage) Root() (root *treechangeproto.RawTreeChangeWithId, err error) {
return z.readChange(z.id)
}
func (z *zipTreeReadStorage) Heads() ([]string, error) {
return z.heads, nil
}
func (z *zipTreeReadStorage) SetHeads(heads []string) (err error) {
panic("should not be called")
}
func (z *zipTreeReadStorage) AddRawChange(change *treechangeproto.RawTreeChangeWithId) (err error) {
panic("should not be called")
}
func (z *zipTreeReadStorage) AddRawChangesSetHeads(changes []*treechangeproto.RawTreeChangeWithId, heads []string) (err error) {
panic("should not be called")
}
func (z *zipTreeReadStorage) GetRawChange(ctx context.Context, id string) (*treechangeproto.RawTreeChangeWithId, error) {
return z.readChange(id)
}
func (z *zipTreeReadStorage) GetAppendRawChange(ctx context.Context, buf []byte, id string) (*treechangeproto.RawTreeChangeWithId, error) {
return z.readChange(id)
}
func (z *zipTreeReadStorage) HasChange(ctx context.Context, id string) (ok bool, err error) {
_, ok = z.files[id]
return
}
func (z *zipTreeReadStorage) Delete() error {
panic("should not be called")
}
func (z *zipTreeReadStorage) readChange(id string) (change *treechangeproto.RawTreeChangeWithId, err error) {
file, ok := z.files[id]
if !ok {
err = fmt.Errorf("object not found in storage")
return
}
opened, err := file.Open()
if err != nil {
return
}
defer opened.Close()
buf, err := io.ReadAll(opened)
if err != nil {
return
}
change = &treechangeproto.RawTreeChangeWithId{RawChange: buf, Id: id}
return
}

View file

@ -1,110 +0,0 @@
package ziptreestorage
import (
"archive/zip"
"context"
"encoding/json"
"fmt"
"strings"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
)
type HeadsJsonEntry struct {
Heads []string `json:"heads"`
RootId string `json:"rootId"`
}
type zipTreeWriteStorage struct {
id string
heads []string
zw *zip.Writer
}
func NewZipTreeWriteStorage(root *treechangeproto.RawTreeChangeWithId, zw *zip.Writer) (st treestorage.TreeStorage, err error) {
z := &zipTreeWriteStorage{
id: root.Id,
zw: zw,
}
err = z.SetHeads([]string{root.Id})
if err != nil {
return
}
err = z.AddRawChange(root)
if err != nil {
return
}
st = z
return
}
func (z *zipTreeWriteStorage) Id() string {
return z.id
}
func (t *zipTreeWriteStorage) GetAllChangeIds() (chs []string, err error) {
return nil, fmt.Errorf("get all change ids should not be called")
}
func (z *zipTreeWriteStorage) Root() (*treechangeproto.RawTreeChangeWithId, error) {
panic("should not be implemented")
}
func (z *zipTreeWriteStorage) Heads() ([]string, error) {
return z.heads, nil
}
func (z *zipTreeWriteStorage) SetHeads(heads []string) (err error) {
z.heads = heads
return
}
func (z *zipTreeWriteStorage) AddRawChange(change *treechangeproto.RawTreeChangeWithId) (err error) {
wr, err := z.zw.Create(strings.Join([]string{z.id, change.Id}, "/"))
if err != nil {
return
}
_, err = wr.Write(change.RawChange)
return
}
func (z *zipTreeWriteStorage) AddRawChangesSetHeads(changes []*treechangeproto.RawTreeChangeWithId, heads []string) (err error) {
for _, ch := range changes {
err = z.AddRawChange(ch)
if err != nil {
return
}
}
return z.SetHeads(heads)
}
func (z *zipTreeWriteStorage) GetRawChange(ctx context.Context, id string) (*treechangeproto.RawTreeChangeWithId, error) {
panic("should not be called")
}
func (z *zipTreeWriteStorage) GetAppendRawChange(ctx context.Context, buf []byte, id string) (*treechangeproto.RawTreeChangeWithId, error) {
panic("should not be called")
}
func (z *zipTreeWriteStorage) HasChange(ctx context.Context, id string) (ok bool, err error) {
panic("should not be called")
}
func (z *zipTreeWriteStorage) Delete() error {
panic("should not be called")
}
func (z *zipTreeWriteStorage) FlushStorage() (err error) {
chw, err := z.zw.CreateHeader(&zip.FileHeader{
Name: strings.Join([]string{z.id, "data.json"}, "/"),
Method: zip.Deflate,
})
enc := json.NewEncoder(chw)
enc.SetIndent("", "\t")
err = enc.Encode(HeadsJsonEntry{
Heads: z.heads,
RootId: z.id,
})
return
}

View file

@ -1,7 +1,6 @@
package debug
import (
"archive/zip"
"bytes"
"context"
"fmt"
@ -11,14 +10,15 @@ import (
"path/filepath"
"time"
"github.com/anyproto/any-sync/commonspace/object/tree/exporter"
anystore "github.com/anyproto/any-store"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
"github.com/anyproto/anytype-heart/core/debug/treearchive"
"github.com/anyproto/anytype-heart/core/debug/exporter"
"github.com/anyproto/anytype-heart/core/domain"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore"
"github.com/anyproto/anytype-heart/util/anonymize"
"github.com/anyproto/anytype-heart/util/pbtypes"
"github.com/anyproto/anytype-heart/util/ziputil"
)
type treeExporter struct {
@ -26,66 +26,78 @@ type treeExporter struct {
s objectstore.ObjectStore
anonymized bool
id domain.FullID
zw *zip.Writer
}
func (e *treeExporter) Export(ctx context.Context, path string, tree objecttree.ReadableObjectTree) (filename string, err error) {
filename = filepath.Join(path, fmt.Sprintf("at.dbg.%s.%s.zip", e.id, time.Now().Format("20060102.150405.99")))
archiveWriter, err := treearchive.NewArchiveWriter(filename)
var (
exportDirPath = filepath.Join(path, fmt.Sprintf("at.dbg.%s.%s", e.id, time.Now().Format("20060102.150405.99")))
dbPath = filepath.Join(exportDirPath, "db")
localStorePath = filepath.Join(exportDirPath, "localstore.json")
logPath = filepath.Join(exportDirPath, "creation.log")
)
filename = exportDirPath + ".zip"
err = os.MkdirAll(exportDirPath, 0755)
if err != nil {
return
}
defer archiveWriter.Close()
e.zw = archiveWriter.ZipWriter()
params := exporter.TreeExporterParams{
ListStorageExporter: archiveWriter,
TreeStorageExporter: archiveWriter,
DataConverter: &changeDataConverter{anonymize: e.anonymized},
defer func() {
_ = os.RemoveAll(exportDirPath)
}()
err = os.Mkdir(dbPath, 0755)
if err != nil {
return
}
anyStore, err := anystore.Open(ctx, dbPath, nil)
if err != nil {
return
}
defer func() {
_ = anyStore.Close()
}()
exportParams := exporter.ExportParams{
Readable: tree,
Store: anyStore,
Converter: &changeDataConverter{anonymize: e.anonymized},
}
anySyncExporter := exporter.NewTreeExporter(params)
logBuf := bytes.NewBuffer(nil)
e.log = stdlog.New(io.MultiWriter(logBuf, os.Stderr), "", stdlog.LstdFlags)
e.log.Printf("exporting tree and acl")
st := time.Now()
err = anySyncExporter.ExportUnencrypted(tree)
err = exporter.ExportTree(ctx, exportParams)
if err != nil {
e.log.Printf("export tree in zip error: %v", err)
return
}
err = anyStore.Checkpoint(ctx, true)
if err != nil {
return
}
logBuf := bytes.NewBuffer(nil)
e.log = stdlog.New(io.MultiWriter(logBuf, os.Stderr), "", stdlog.LstdFlags)
e.log.Printf("exported tree for a %v", time.Since(st))
data, err := e.s.SpaceIndex(e.id.SpaceID).GetInfosByIds([]string{e.id.ObjectID})
if err != nil {
e.log.Printf("can't fetch localstore info: %v", err)
} else {
if len(data) > 0 {
data[0].Details = transform(data[0].Details, e.anonymized, anonymize.Details)
// TODO: [storage] fix details, take from main
// data[0].Details = transform(data[0].Details, e.anonymized, anonymize.Struct)
data[0].Snippet = transform(data[0].Snippet, e.anonymized, anonymize.Text)
for i, r := range data[0].Relations {
data[0].Relations[i] = transform(r, e.anonymized, anonymize.Relation)
}
osData := pbtypes.Sprint(data[0].ToProto())
lsWr, er := e.zw.Create("localstore.json")
er := os.WriteFile(localStorePath, []byte(osData), 0600)
if er != nil {
e.log.Printf("create file in zip error: %v", er)
e.log.Printf("localstore.json write error: %v", err)
} else {
if _, err := lsWr.Write([]byte(osData)); err != nil {
e.log.Printf("localstore.json write error: %v", err)
} else {
e.log.Printf("localstore.json wrote")
}
e.log.Printf("localstore.json wrote")
}
} else {
e.log.Printf("not data in objectstore")
e.log.Printf("no data in objectstore")
}
}
logW, err := e.zw.Create("creation.log")
err = os.WriteFile(logPath, logBuf.Bytes(), 0600)
if err != nil {
return
}
io.Copy(logW, logBuf)
err = ziputil.ZipFolder(exportDirPath, filename)
return
}

View file

@ -56,7 +56,6 @@ func DownloadManifest(url string, checkWhitelist bool) (info *model.ManifestInfo
if err != nil {
return nil, err
}
schemaResp := schemaResponse{}
err = json.Unmarshal(raw, &schemaResp)
if err != nil {
@ -119,7 +118,7 @@ func getRawJson(url string) ([]byte, error) {
if err != nil {
return nil, err
}
req.Close = true
res, err := client.Do(req)
if err != nil {
return nil, err

View file

@ -4,6 +4,7 @@ import (
_ "embed"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
@ -11,8 +12,6 @@ import (
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
)
const port = ":7070"
//go:embed testdata/schema.json
var schemaJSON []byte
@ -37,13 +36,13 @@ func TestIsInWhitelist(t *testing.T) {
}
func TestDownloadManifestAndValidateSchema(t *testing.T) {
schema := schemaResponse{Schema: "http://localhost" + port + "/schema.json"}
server := startHttpServer()
defer server.Shutdown(nil)
defer server.Close()
schema := schemaResponse{Schema: server.URL + "/schema.json"}
t.Run("download knowledge base manifest", func(t *testing.T) {
// given
url := "http://localhost" + port + "/manifest.json"
url := server.URL + "/manifest.json"
// when
info, err := DownloadManifest(url, false)
@ -116,7 +115,7 @@ func TestDownloadManifestAndValidateSchema(t *testing.T) {
})
}
func startHttpServer() *http.Server {
func startHttpServer() *httptest.Server {
handler := http.NewServeMux()
handler.HandleFunc("/manifest.json", func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
@ -128,9 +127,7 @@ func startHttpServer() *http.Server {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write(schemaJSON)
})
server := &http.Server{Addr: port, Handler: handler}
go server.ListenAndServe()
return server
return httptest.NewServer(handler)
}
func buildInfo() *model.ManifestInfo {

View file

@ -45,7 +45,6 @@ func NewIndexerFixture(t *testing.T) *IndexerFixture {
walletService := mock_wallet.NewMockWallet(t)
walletService.EXPECT().Name().Return(wallet.CName)
objectStore := objectstore.NewStoreFixture(t)
clientStorage := mock_storage.NewMockClientStorage(t)

View file

@ -143,7 +143,7 @@ func (i *indexer) Index(info smartblock.DocInfo, options ...smartblock.IndexOpti
i.lock.Lock()
spaceInd, ok := i.spaceIndexers[info.Space.Id()]
if !ok {
spaceInd = newSpaceIndexer(i.runCtx, i.store.SpaceIndex(info.Space.Id()), i.store, i.storageService)
spaceInd = newSpaceIndexer(i.runCtx, i.store.SpaceIndex(info.Space.Id()), i.store)
i.spaceIndexers[info.Space.Id()] = spaceInd
}
i.lock.Unlock()

View file

@ -5,7 +5,6 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/anyproto/anytype-heart/core/block/editor/smartblock"
"github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest"
@ -50,7 +49,6 @@ func TestIndexer(t *testing.T) {
)))
smartTest.SetType(coresb.SmartBlockTypePage)
indexerFx.storageServiceFx.EXPECT().BindSpaceID(mock.Anything, mock.Anything).Return(nil)
indexerFx.store.SpaceIndex("spaceId1").SaveLastIndexedHeadsHash(ctx, "objectId1", "7f40bc2814f5297818461f889780a870ea033fe64c5a261117f2b662515a3dba")
// when
@ -78,7 +76,6 @@ func TestIndexer(t *testing.T) {
)))
smartTest.SetType(coresb.SmartBlockTypePage)
indexerFx.storageServiceFx.EXPECT().BindSpaceID(mock.Anything, mock.Anything).Return(nil)
indexerFx.store.SpaceIndex("spaceId1").SaveLastIndexedHeadsHash(ctx, "objectId1", "randomHash")
// when
@ -107,7 +104,6 @@ func TestIndexer(t *testing.T) {
)))
smartTest.SetType(coresb.SmartBlockTypePage)
indexerFx.storageServiceFx.EXPECT().BindSpaceID(mock.Anything, mock.Anything).Return(nil)
indexerFx.store.SpaceIndex("spaceId1").SaveLastIndexedHeadsHash(ctx, "objectId1", "7f40bc2814f5297818461f889780a870ea033fe64c5a261117f2b662515a3dba")
// when

View file

@ -7,6 +7,7 @@ import (
"time"
anystore "github.com/anyproto/any-store"
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
"go.uber.org/zap"
"github.com/anyproto/anytype-heart/core/block/editor/smartblock"
@ -51,7 +52,7 @@ const (
)
type allDeletedIdsProvider interface {
AllDeletedTreeIds() (ids []string, err error)
AllDeletedTreeIds(ctx context.Context) (ids []string, err error)
}
func (i *indexer) buildFlags(spaceID string) (reindexFlags, error) {
@ -247,11 +248,7 @@ func (i *indexer) addSyncDetails(space clientspace.Space) {
func (i *indexer) reindexDeletedObjects(space clientspace.Space) error {
store := i.store.SpaceIndex(space.Id())
storage, ok := space.Storage().(allDeletedIdsProvider)
if !ok {
return fmt.Errorf("space storage doesn't implement allDeletedIdsProvider")
}
allIds, err := storage.AllDeletedTreeIds()
allIds, err := space.Storage().AllDeletedTreeIds(i.runCtx)
if err != nil {
return fmt.Errorf("get deleted tree ids: %w", err)
}
@ -440,35 +437,34 @@ func (i *indexer) reindexIDs(ctx context.Context, space smartblock.Space, reinde
func (i *indexer) reindexOutdatedObjects(ctx context.Context, space clientspace.Space) (toReindex, success int, err error) {
store := i.store.SpaceIndex(space.Id())
tids := space.StoredIds()
var entries []headstorage.HeadsEntry
err = space.Storage().HeadStorage().IterateEntries(ctx, headstorage.IterOpts{}, func(entry headstorage.HeadsEntry) (bool, error) {
// skipping Acl
if entry.CommonSnapshot != "" {
entries = append(entries, entry)
}
return true, nil
})
if err != nil {
return
}
var idsToReindex []string
for _, tid := range tids {
for _, entry := range entries {
id := entry.Id
logErr := func(err error) {
log.With("tree", tid).Errorf("reindexOutdatedObjects failed to get tree to reindex: %s", err)
log.With("tree", entry.Id).Errorf("reindexOutdatedObjects failed to get tree to reindex: %s", err)
}
lastHash, err := store.GetLastIndexedHeadsHash(ctx, tid)
lastHash, err := store.GetLastIndexedHeadsHash(ctx, id)
if err != nil {
logErr(err)
continue
}
info, err := space.Storage().TreeStorage(tid)
if err != nil {
logErr(err)
continue
}
heads, err := info.Heads()
if err != nil {
logErr(err)
continue
}
hh := headsHash(heads)
hh := headsHash(entry.Heads)
if lastHash != hh {
if lastHash != "" {
log.With("tree", tid).Warnf("not equal indexed heads hash: %s!=%s (%d logs)", lastHash, hh, len(heads))
log.With("tree", id).Warnf("not equal indexed heads hash: %s!=%s (%d logs)", lastHash, hh, len(entry.Heads))
}
idsToReindex = append(idsToReindex, tid)
idsToReindex = append(idsToReindex, id)
}
}

View file

@ -4,9 +4,12 @@ import (
"context"
"testing"
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
"github.com/anyproto/any-sync/commonspace/headsync/headstorage/mock_headstorage"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"github.com/anyproto/anytype-heart/core/block/editor"
"github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest"
@ -22,6 +25,7 @@ import (
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
"github.com/anyproto/anytype-heart/space/clientspace"
mock_space "github.com/anyproto/anytype-heart/space/clientspace/mock_clientspace"
"github.com/anyproto/anytype-heart/space/spacecore/storage/anystorage/mock_anystorage"
"github.com/anyproto/anytype-heart/space/spacecore/storage/mock_storage"
)
@ -54,7 +58,6 @@ func TestReindexMarketplaceSpace(t *testing.T) {
virtualSpace := getMockSpace(indexerFx)
storage := mock_storage.NewMockClientStorage(t)
storage.EXPECT().BindSpaceID(mock.Anything, mock.Anything).Return(nil)
indexerFx.storageService = storage
// when
@ -92,7 +95,6 @@ func TestReindexMarketplaceSpace(t *testing.T) {
require.NoError(t, err)
storage := mock_storage.NewMockClientStorage(t)
storage.EXPECT().BindSpaceID(mock.Anything, mock.Anything).Return(nil)
fx.storageService = storage
// when
@ -128,7 +130,6 @@ func TestReindexMarketplaceSpace(t *testing.T) {
require.NoError(t, err)
storage := mock_storage.NewMockClientStorage(t)
storage.EXPECT().BindSpaceID(mock.Anything, mock.Anything).Return(nil)
fx.storageService = storage
fx.sourceFx.EXPECT().IDsListerBySmartblockType(mock.Anything, mock.Anything).Return(idsLister{Ids: []string{}}, nil).Maybe()
@ -189,6 +190,15 @@ func TestIndexer_ReindexSpace_RemoveParticipants(t *testing.T) {
err = fx.objectStore.SaveChecksums(spaceId2, &checksums)
require.NoError(t, err)
ctrl := gomock.NewController(t)
headStorage := mock_headstorage.NewMockHeadStorage(ctrl)
storage := mock_anystorage.NewMockClientSpaceStorage(t)
storage.EXPECT().HeadStorage().Return(headStorage)
headStorage.EXPECT().IterateEntries(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().
DoAndReturn(func(ctx context.Context, opts headstorage.IterOpts, entryIter headstorage.EntryIterator) error {
return nil
})
for _, space := range []string{spaceId1, spaceId2} {
t.Run("reindex - participants deleted - when flag doesn't match", func(t *testing.T) {
// given
@ -196,7 +206,7 @@ func TestIndexer_ReindexSpace_RemoveParticipants(t *testing.T) {
spc := mock_space.NewMockSpace(t)
spc.EXPECT().Id().Return(space)
spc.EXPECT().StoredIds().Return([]string{}).Maybe()
spc.EXPECT().Storage().Return(storage)
fx.sourceFx.EXPECT().IDsListerBySmartblockType(mock.Anything, mock.Anything).Return(idsLister{Ids: []string{}}, nil).Maybe()
// when
@ -275,12 +285,21 @@ func TestIndexer_ReindexSpace_EraseLinks(t *testing.T) {
require.NoError(t, err)
err = fx.objectStore.SaveChecksums(spaceId2, &checksums)
require.NoError(t, err)
ctrl := gomock.NewController(t)
headStorage := mock_headstorage.NewMockHeadStorage(ctrl)
storage := mock_anystorage.NewMockClientSpaceStorage(t)
storage.EXPECT().HeadStorage().Return(headStorage)
headStorage.EXPECT().IterateEntries(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().
DoAndReturn(func(ctx context.Context, opts headstorage.IterOpts, entryIter headstorage.EntryIterator) error {
return nil
})
t.Run("links from archive and home are deleted", func(t *testing.T) {
// given
favs := []string{"fav1", "fav2"}
trash := []string{"trash1", "trash2"}
store := fx.store.SpaceIndex("space1")
err = store.UpdateObjectLinks(ctx, "home", favs)
require.NoError(t, err)
err = store.UpdateObjectLinks(ctx, "bin", trash)
@ -294,7 +313,7 @@ func TestIndexer_ReindexSpace_EraseLinks(t *testing.T) {
space1 := mock_space.NewMockSpace(t)
space1.EXPECT().Id().Return(spaceId1)
space1.EXPECT().StoredIds().Return([]string{}).Maybe()
space1.EXPECT().Storage().Return(storage)
// when
err = fx.ReindexSpace(space1)
@ -335,8 +354,7 @@ func TestIndexer_ReindexSpace_EraseLinks(t *testing.T) {
space1 := mock_space.NewMockSpace(t)
space1.EXPECT().Id().Return(spaceId2)
space1.EXPECT().StoredIds().Return([]string{}).Maybe()
space1.EXPECT().Storage().Return(storage)
// when
err = fx.ReindexSpace(space1)
assert.NoError(t, err)

View file

@ -14,24 +14,21 @@ import (
"github.com/anyproto/anytype-heart/core/block/editor/smartblock"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceindex"
"github.com/anyproto/anytype-heart/space/spacecore/storage"
)
type spaceIndexer struct {
runCtx context.Context
spaceIndex spaceindex.Store
objectStore objectstore.ObjectStore
storageService storage.ClientStorage
batcher *mb.MB[indexTask]
runCtx context.Context
spaceIndex spaceindex.Store
objectStore objectstore.ObjectStore
batcher *mb.MB[indexTask]
}
func newSpaceIndexer(runCtx context.Context, spaceIndex spaceindex.Store, objectStore objectstore.ObjectStore, storageService storage.ClientStorage) *spaceIndexer {
func newSpaceIndexer(runCtx context.Context, spaceIndex spaceindex.Store, objectStore objectstore.ObjectStore) *spaceIndexer {
ind := &spaceIndexer{
runCtx: runCtx,
spaceIndex: spaceIndex,
objectStore: objectStore,
storageService: storageService,
batcher: mb.New[indexTask](100),
runCtx: runCtx,
spaceIndex: spaceIndex,
objectStore: objectStore,
batcher: mb.New[indexTask](100),
}
go ind.indexBatchLoop()
return ind
@ -123,7 +120,7 @@ func (i *spaceIndexer) index(ctx context.Context, info smartblock.DocInfo, optio
for _, o := range options {
o(opts)
}
err := i.storageService.BindSpaceID(info.Space.Id(), info.Id)
err := i.objectStore.BindSpaceId(info.Space.Id(), info.Id)
if err != nil {
log.Error("failed to bind space id", zap.Error(err), zap.String("id", info.Id))
return err

View file

@ -189,7 +189,7 @@ func (s *service) publishToPublishServer(ctx context.Context, spaceId, pageId, u
return err
}
version, err := s.evaluateDocumentVersion(spc, pageId, joinSpace)
version, err := s.evaluateDocumentVersion(ctx, spc, pageId, joinSpace)
if err != nil {
return err
}
@ -380,12 +380,12 @@ func (s *service) extractInviteLink(ctx context.Context, spaceId string, joinSpa
return inviteLink, nil
}
func (s *service) evaluateDocumentVersion(spc clientspace.Space, pageId string, joinSpace bool) (string, error) {
treeStorage, err := spc.Storage().TreeStorage(pageId)
func (s *service) evaluateDocumentVersion(ctx context.Context, spc clientspace.Space, pageId string, joinSpace bool) (string, error) {
treeStorage, err := spc.Storage().TreeStorage(ctx, pageId)
if err != nil {
return "", err
}
heads, err := treeStorage.Heads()
heads, err := treeStorage.Heads(ctx)
if err != nil {
return "", err
}

View file

@ -11,15 +11,17 @@ import (
"path/filepath"
"testing"
anystore "github.com/anyproto/any-store"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
"github.com/anyproto/any-sync/consensus/consensusproto"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree/mock_objecttree"
"github.com/anyproto/anytype-publish-server/publishclient/publishapi"
"github.com/gogo/protobuf/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"golang.org/x/sys/unix"
"github.com/anyproto/anytype-heart/core/anytype/account/mock_account"
"github.com/anyproto/anytype-heart/core/block/cache/mock_cache"
@ -46,6 +48,7 @@ import (
"github.com/anyproto/anytype-heart/pkg/lib/threads"
"github.com/anyproto/anytype-heart/space/clientspace/mock_clientspace"
"github.com/anyproto/anytype-heart/space/mock_space"
"github.com/anyproto/anytype-heart/space/spacecore/storage/anystorage/mock_anystorage"
"github.com/anyproto/anytype-heart/space/spacecore/typeprovider/mock_typeprovider"
"github.com/anyproto/anytype-heart/tests/testutil"
"github.com/anyproto/anytype-heart/util/pbtypes"
@ -135,7 +138,7 @@ func TestPublish(t *testing.T) {
t.Run("success", func(t *testing.T) {
// given
isPersonal := true
spaceService, err := prepaeSpaceService(t, isPersonal)
spaceService, err := prepareSpaceService(t, isPersonal)
objectTypeId := "customObjectType"
expectedUri := "test"
@ -178,7 +181,7 @@ func TestPublish(t *testing.T) {
t.Run("success with space sharing", func(t *testing.T) {
// given
isPersonal := false
spaceService, err := prepaeSpaceService(t, isPersonal)
spaceService, err := prepareSpaceService(t, isPersonal)
objectTypeId := "customObjectType"
expectedUri := "test"
@ -231,7 +234,7 @@ func TestPublish(t *testing.T) {
})
t.Run("success with space sharing - invite not exists", func(t *testing.T) {
isPersonal := false
spaceService, err := prepaeSpaceService(t, isPersonal)
spaceService, err := prepareSpaceService(t, isPersonal)
objectTypeId := "customObjectType"
expectedUri := "test"
@ -279,7 +282,7 @@ func TestPublish(t *testing.T) {
t.Run("success for member", func(t *testing.T) {
// given
isPersonal := false
spaceService, err := prepaeSpaceService(t, isPersonal)
spaceService, err := prepareSpaceService(t, isPersonal)
objectTypeId := "customObjectType"
expectedUri := "test"
@ -337,7 +340,7 @@ func TestPublish(t *testing.T) {
t.Run("internal error", func(t *testing.T) {
// given
isPersonal := true
spaceService, err := prepaeSpaceService(t, isPersonal)
spaceService, err := prepareSpaceService(t, isPersonal)
objectTypeId := "customObjectType"
expectedUri := "test"
@ -660,28 +663,22 @@ func TestService_PublishingList(t *testing.T) {
})
}
func prepaeSpaceService(t *testing.T, isPersonal bool) (*mock_space.MockService, error) {
var ctx = context.Background()
func prepareSpaceService(t *testing.T, isPersonal bool) (*mock_space.MockService, error) {
spaceService := mock_space.NewMockService(t)
space := mock_clientspace.NewMockSpace(t)
ctrl := gomock.NewController(t)
space.EXPECT().IsPersonal().Return(isPersonal)
space.EXPECT().Id().Return(spaceId)
storage, err := spacestorage.NewInMemorySpaceStorage(spacestorage.SpaceStorageCreatePayload{
AclWithId: &consensusproto.RawRecordWithId{Id: "aclId"},
SpaceHeaderWithId: &spacesyncproto.RawSpaceHeaderWithId{Id: spaceId},
SpaceSettingsWithId: &treechangeproto.RawTreeChangeWithId{Id: "settingsId"},
},
)
assert.NoError(t, err)
objectHeads := []string{"heads"}
_, err = storage.CreateTreeStorage(treestorage.TreeStorageCreatePayload{
RootRawChange: &treechangeproto.RawTreeChangeWithId{Id: objectId},
Heads: objectHeads,
})
assert.NoError(t, err)
space.EXPECT().Storage().Return(storage)
st := mock_anystorage.NewMockClientSpaceStorage(t)
mockSt := mock_objecttree.NewMockStorage(ctrl)
st.EXPECT().TreeStorage(mock.Anything, mock.Anything).Return(mockSt, nil)
mockSt.EXPECT().Heads(gomock.Any()).Return([]string{"heads"}, nil)
space.EXPECT().Storage().Return(st)
spaceService.EXPECT().Get(context.Background(), spaceId).Return(space, nil)
return spaceService, err
return spaceService, nil
}
func prepareExporter(t *testing.T, objectTypeId string, spaceService *mock_space.MockService) export.Export {
@ -895,3 +892,22 @@ func createTestFile(fileName string, size int64) error {
file.Close()
return nil
}
func createStore(ctx context.Context, t testing.TB) anystore.DB {
return createNamedStore(ctx, t, "changes.db")
}
func createNamedStore(ctx context.Context, t testing.TB, name string) anystore.DB {
path := filepath.Join(t.TempDir(), name)
db, err := anystore.Open(ctx, path, nil)
require.NoError(t, err)
t.Cleanup(func() {
err := db.Close()
require.NoError(t, err)
unix.Rmdir(path)
})
return objecttree.TestStore{
DB: db,
Path: path,
}
}

View file

@ -33,7 +33,7 @@ const CName = "subscription"
var log = logging.Logger("anytype-mw-subscription")
var batchTime = 50 * time.Millisecond
var batchTime = 250 * time.Millisecond
func New() Service {
return &service{}

View file

@ -94,7 +94,7 @@ func (s *syncStatusService) Init(a *app.App) (err error) {
s.updateIntervalSecs = syncUpdateInterval
s.updateTimeout = syncTimeout
s.spaceId = sharedState.SpaceId
s.spaceSettingsId = spaceStorage.SpaceSettingsId()
s.spaceSettingsId = spaceStorage.StateStorage().SettingsId()
s.periodicSync = periodicsync.NewPeriodicSync(
s.updateIntervalSecs,
s.updateTimeout,

View file

@ -5,6 +5,7 @@ import (
"testing"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/headsync/statestorage/mock_statestorage"
"github.com/anyproto/any-sync/commonspace/spacestate"
"github.com/anyproto/any-sync/commonspace/spacestorage/mock_spacestorage"
"github.com/anyproto/any-sync/nodeconf/mock_nodeconf"
@ -174,7 +175,9 @@ func newFixture(t *testing.T, spaceId string) *fixture {
ctrl := gomock.NewController(t)
service := mock_nodeconf.NewMockService(ctrl)
storage := mock_spacestorage.NewMockSpaceStorage(ctrl)
storage.EXPECT().SpaceSettingsId().Return(testSpaceSettingsId)
stateStorage := mock_statestorage.NewMockStateStorage(ctrl)
storage.EXPECT().StateStorage().AnyTimes().Return(stateStorage)
stateStorage.EXPECT().SettingsId().AnyTimes().Return(testSpaceSettingsId)
spaceState := &spacestate.SpaceState{SpaceId: spaceId}
config := &config.Config{}

View file

@ -44,7 +44,7 @@ func TestSyncSubscriptions(t *testing.T) {
subs.(*syncSubscriptions).service = testSubs
err := subs.Run(context.Background())
require.NoError(t, err)
time.Sleep(100 * time.Millisecond)
time.Sleep(500 * time.Millisecond)
spaceSub, err := subs.GetSubscription("spaceId")
require.NoError(t, err)
syncCnt := spaceSub.SyncingObjectsCount([]string{"1", "2"})
@ -59,7 +59,7 @@ func TestSyncSubscriptions(t *testing.T) {
objects[i][bundle.RelationKeySyncStatus] = domain.Int64(int64(domain.ObjectSyncStatusSynced))
testSubs.AddObjects(t, "spaceId", []objectstore.TestObject{objects[i]})
}
time.Sleep(100 * time.Millisecond)
time.Sleep(500 * time.Millisecond)
syncCnt = spaceSub.SyncingObjectsCount([]string{"1", "2"})
require.Equal(t, 2, syncCnt)
err = subs.Close(context.Background())

View file

@ -24,10 +24,16 @@ func (mw *Middleware) getResponseEvent(ctx session.Context) *pb.ResponseEvent {
type errToCodeTuple[T ~int32] struct {
err error
code T
checkErrorType any
}
func errToCode[T ~int32](err error, code T) errToCodeTuple[T] {
return errToCodeTuple[T]{err, code}
return errToCodeTuple[T]{err: err, code: code}
}
func errTypeToCode[T ~int32](errTypeProto any, code T) errToCodeTuple[T] {
return errToCodeTuple[T]{code: code, checkErrorType: errTypeProto}
}
func mapErrorCode[T ~int32](err error, mappings ...errToCodeTuple[T]) T {
@ -35,8 +41,15 @@ func mapErrorCode[T ~int32](err error, mappings ...errToCodeTuple[T]) T {
return 0
}
for _, m := range mappings {
if errors.Is(err, m.err) {
return m.code
if m.err != nil {
if errors.Is(err, m.err) {
return m.code
}
}
if m.checkErrorType != nil {
if errors.As(err, m.checkErrorType) {
return m.code
}
}
}
// Unknown error

View file

@ -8,12 +8,20 @@ import (
"github.com/stretchr/testify/assert"
)
type testErrorType struct {
}
func (t testErrorType) Error() string {
return "error type!"
}
type testCode int32
func TestErrorCodeMapping(t *testing.T) {
err1 := errors.New("err1")
err2 := errors.New("err2")
err3 := errors.New("err3")
err4 := testErrorType{}
wrapped1 := errors.Join(err1, fmt.Errorf("description of error"))
wrapped2 := fmt.Errorf("description of error: %w", err2)
@ -23,12 +31,14 @@ func TestErrorCodeMapping(t *testing.T) {
errToCode(err1, testCode(2)),
errToCode(err2, testCode(3)),
errToCode(err3, testCode(4)),
errTypeToCode(&testErrorType{}, testCode(5)),
)
}
assert.True(t, 0 == mapper(nil))
assert.True(t, 1 == mapper(errors.New("unknown error")))
assert.True(t, 2 == mapper(wrapped1))
assert.True(t, 3 == mapper(wrapped2))
assert.True(t, 4 == mapper(err3))
assert.Equal(t, testCode(0), mapper(nil))
assert.Equal(t, testCode(1), mapper(errors.New("unknown error")))
assert.Equal(t, testCode(2), mapper(wrapped1))
assert.Equal(t, testCode(3), mapper(wrapped2))
assert.Equal(t, testCode(4), mapper(err3))
assert.Equal(t, testCode(5), mapper(err4))
}

23
go.mod
View file

@ -8,7 +8,7 @@ require (
github.com/VividCortex/ewma v1.2.0
github.com/adrium/goheif v0.0.0-20230113233934-ca402e77a786
github.com/anyproto/any-store v0.1.8
github.com/anyproto/any-sync v0.5.26
github.com/anyproto/any-sync v0.6.1
github.com/anyproto/anytype-publish-server/publishclient v0.0.0-20250131145601-de288583ff2a
github.com/anyproto/go-chash v0.1.0
github.com/anyproto/go-naturaldate/v2 v2.0.2-0.20230524105841-9829cfd13438
@ -17,7 +17,7 @@ require (
github.com/anyproto/protobuf v1.3.3-0.20240814124528-72b8c7e0e0f5
github.com/anyproto/tantivy-go v0.3.1
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
github.com/avast/retry-go/v4 v4.6.1
github.com/avast/retry-go/v4 v4.6.0
github.com/chai2010/webp v1.1.2-0.20240612091223-aa1b379218b7
github.com/cheggaaa/mb/v3 v3.0.2
github.com/dave/jennifer v1.7.1
@ -51,7 +51,7 @@ require (
github.com/hbagdi/go-unsplash v0.0.0-20230414214043-474fc02c9119
github.com/huandu/skiplist v1.2.1
github.com/improbable-eng/grpc-web v0.15.0
github.com/ipfs/boxo v0.27.4
github.com/ipfs/boxo v0.28.0
github.com/ipfs/go-block-format v0.2.0
github.com/ipfs/go-cid v0.5.0
github.com/ipfs/go-datastore v0.8.1
@ -107,7 +107,8 @@ require (
golang.org/x/image v0.24.0
golang.org/x/mobile v0.0.0-20241108191957-fa514ef75a0f
golang.org/x/net v0.35.0
golang.org/x/oauth2 v0.27.0
golang.org/x/oauth2 v0.26.0
golang.org/x/sys v0.30.0
golang.org/x/text v0.22.0
google.golang.org/grpc v1.70.0
gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20180125164251-1832d8546a9f
@ -115,7 +116,6 @@ require (
gopkg.in/yaml.v3 v3.0.1
storj.io/drpc v0.0.34
zombiezen.com/go/sqlite v1.4.0
)
require (
@ -188,7 +188,7 @@ require (
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/flatbuffers v1.12.1 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/pprof v0.0.0-20250202011525-fc3143867406 // indirect
github.com/google/pprof v0.0.0-20250208200701-d0013a598941 // indirect
github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/gosimple/unidecode v1.0.1 // indirect
@ -215,7 +215,7 @@ require (
github.com/klauspost/cpuid/v2 v2.2.9 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/libp2p/go-libp2p v0.39.0 // indirect
github.com/libp2p/go-libp2p v0.40.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
@ -247,7 +247,7 @@ require (
github.com/prometheus/common v0.62.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/pseudomuto/protokit v0.2.1 // indirect
github.com/quic-go/quic-go v0.49.0 // indirect
github.com/quic-go/quic-go v0.50.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rs/cors v1.11.0 // indirect
github.com/rs/zerolog v1.29.0 // indirect
@ -279,16 +279,15 @@ require (
go.opentelemetry.io/otel/metric v1.34.0 // indirect
go.opentelemetry.io/otel/trace v1.34.0 // indirect
golang.org/x/arch v0.8.0 // indirect
golang.org/x/crypto v0.33.0 // indirect
golang.org/x/crypto v0.35.0 // indirect
golang.org/x/mod v0.23.0 // indirect
golang.org/x/sync v0.11.0 // indirect
golang.org/x/sys v0.30.0 // indirect
golang.org/x/term v0.29.0 // indirect
golang.org/x/time v0.10.0 // indirect
golang.org/x/tools v0.30.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a // indirect
google.golang.org/protobuf v1.36.4 // indirect
google.golang.org/protobuf v1.36.5 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
lukechampine.com/blake3 v1.3.0 // indirect
@ -322,3 +321,5 @@ replace google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/
replace github.com/btcsuite/btcutil => github.com/btcsuite/btcd/btcutil v1.1.5
replace github.com/dsoprea/go-jpeg-image-structure/v2 => github.com/dchesterton/go-jpeg-image-structure/v2 v2.0.0-20240318203529-c3eea088bd38
replace zombiezen.com/go/sqlite => github.com/anyproto/go-sqlite v0.0.0-20250226111550-9b81a8e3cff4

58
go.sum
View file

@ -84,8 +84,8 @@ github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kk
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/anyproto/any-store v0.1.8 h1:/bxUVq6sBTwYkmPL2g1xUAWNb3axF+zPhP2dvdEBH68=
github.com/anyproto/any-store v0.1.8/go.mod h1:GpnVhcGm5aUQtOwCnKeTt4jsWgVXZ773WbQVLFdeCFo=
github.com/anyproto/any-sync v0.5.26 h1:JWcR/RFGQ22CYWrdh2Ain6uEWNePtYHCLs+LXsm8hhU=
github.com/anyproto/any-sync v0.5.26/go.mod h1:Ljftoz6/mCM/2wP2tK9H1/jtVAxfgqzYplBA4MbiUs0=
github.com/anyproto/any-sync v0.6.1 h1:Dasbp7qGQme8diGGpzaDQfSDs5o7PAK3E5rxHHrB/+4=
github.com/anyproto/any-sync v0.6.1/go.mod h1:5js8TNBdqe75zwlr9XEQSVDtwhsvEU2qLeC2wTnT/Fo=
github.com/anyproto/anytype-publish-server/publishclient v0.0.0-20250131145601-de288583ff2a h1:ZZM+0OUCQMWSLSflpkf0ZMVo3V76qEDDIXPpQOClNs0=
github.com/anyproto/anytype-publish-server/publishclient v0.0.0-20250131145601-de288583ff2a/go.mod h1:4fkueCZcGniSMXkrwESO8zzERrh/L7WHimRNWecfGM0=
github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580 h1:Ba80IlCCxkZ9H1GF+7vFu/TSpPvbpDCxXJ5ogc4euYc=
@ -104,6 +104,8 @@ github.com/anyproto/go-slip10 v1.0.0 h1:uAEtSuudR3jJBOfkOXf3bErxVoxbuKwdoJN55M1i
github.com/anyproto/go-slip10 v1.0.0/go.mod h1:BCmIlM1KB8wX6K4/8pOvxPl9oVKfEvZ5vsmO5rkK6vg=
github.com/anyproto/go-slip21 v1.0.0 h1:CI7lUqTIwmPOEGVAj4jyNLoICvueh++0U2HoAi3m2ZY=
github.com/anyproto/go-slip21 v1.0.0/go.mod h1:gbIJt7HAdr5DuT4f2pFTKCBSUWYsm/fysHBNqgsuxT0=
github.com/anyproto/go-sqlite v0.0.0-20250226111550-9b81a8e3cff4 h1:HzVjm45VOUVFUrxh2s0cRR4lqfCr/VAee6wNzPLcApI=
github.com/anyproto/go-sqlite v0.0.0-20250226111550-9b81a8e3cff4/go.mod h1:0w9F1DN9IZj9AcLS9YDKMboubCACkwYCGkzoy3eG5ik=
github.com/anyproto/html-to-markdown v0.0.0-20231025221133-830bf0a6f139 h1:Wp9z0Q2kAstznWUmTZyOb9UgpVmUgYt1LXRvK/cg10E=
github.com/anyproto/html-to-markdown v0.0.0-20231025221133-830bf0a6f139/go.mod h1:1zaDDQVWTRwNksmTUTkcVXqgNF28YHiEUIm8FL9Z+II=
github.com/anyproto/lexid v0.0.4 h1:2ztI0y5pNdtojd3vChw/YV/P6IO9pB7PccYysImDxWI=
@ -124,8 +126,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
github.com/avast/retry-go/v4 v4.6.1 h1:VkOLRubHdisGrHnTu89g08aQEWEgRU7LVEop3GbIcMk=
github.com/avast/retry-go/v4 v4.6.1/go.mod h1:V6oF8njAwxJ5gRo1Q7Cxab24xs5NCWZBeaHHBklR8mA=
github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA=
github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE=
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
@ -483,8 +485,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20250202011525-fc3143867406 h1:wlQI2cYY0BsWmmPPAnxfQ8SDW0S3Jasn+4B8kXFxprg=
github.com/google/pprof v0.0.0-20250202011525-fc3143867406/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
github.com/google/pprof v0.0.0-20250208200701-d0013a598941 h1:43XjGa6toxLpeksjcxs1jIoIyr+vUfOqY2c6HB4bpoc=
github.com/google/pprof v0.0.0-20250208200701-d0013a598941/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@ -579,8 +581,8 @@ github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.27.4 h1:6nC8lY5GnR6whAbW88hFz6L13wZUj2vr5BRe3iTvYBI=
github.com/ipfs/boxo v0.27.4/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
github.com/ipfs/boxo v0.28.0 h1:io6nXqN8XMOstB7dQGG5GWnMk4WssoMvva9OADErZdI=
github.com/ipfs/boxo v0.28.0/go.mod h1:eY9w3iTpmZGKzDfEYjm3oK8f+xjv8KJhhNXJwicmd3I=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
@ -687,12 +689,12 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw=
github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:st3qqfu8+pMfh+9Mzqb2GTiwrAGjIPszEjZmtksN8Jc=
github.com/libp2p/go-libp2p v0.39.0 h1:LmrhDRud4eDkQCSB4l5NfoIFDqvDwAyANmfeYkgnKgs=
github.com/libp2p/go-libp2p v0.39.0/go.mod h1:3zicI8Lp7Isun+Afo/JOACUbbJqqR2owK6RQWFsVAbI=
github.com/libp2p/go-libp2p v0.40.0 h1:1LOMO3gigxeXFs50HGEc1U79OINewUQB7o4gTKGPC3U=
github.com/libp2p/go-libp2p v0.40.0/go.mod h1:hOzj2EAIYsXpVpBnyA1pRHzpUJGF9nbWiDLjgasnbF0=
github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94=
github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8=
github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0=
github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk=
github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg=
github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E=
github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA=
github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg=
github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0=
@ -701,8 +703,8 @@ github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk=
github.com/libp2p/go-nat v0.2.0/go.mod h1:3MJr+GRpRkyT65EpVPBstXLvOlAPzUVlG6Pwg9ohLJk=
github.com/libp2p/go-netroute v0.2.2 h1:Dejd8cQ47Qx2kRABg6lPwknU7+nBnFRpko45/fFPuZ8=
github.com/libp2p/go-netroute v0.2.2/go.mod h1:Rntq6jUAH0l9Gg17w5bFGhcC9a+vk4KNXs6s7IljKYE=
github.com/libp2p/go-yamux/v4 v4.0.2 h1:nrLh89LN/LEiqcFiqdKDRHjGstN300C1269K/EX0CPU=
github.com/libp2p/go-yamux/v4 v4.0.2/go.mod h1:C808cCRgOs1iBwY4S71T5oxgMxgLmqUw56qh4AeBW2o=
github.com/libp2p/go-yamux/v5 v5.0.0 h1:2djUh96d3Jiac/JpGkKs4TO49YhsfLopAoryfPmf+Po=
github.com/libp2p/go-yamux/v5 v5.0.0/go.mod h1:en+3cdX51U0ZslwRdRLrvQsdayFt3TSUKvBGErzpWbU=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
@ -867,16 +869,12 @@ github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk=
github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE=
github.com/pion/dtls/v3 v3.0.4 h1:44CZekewMzfrn9pmGrj5BNnTMDCFwr+6sLH+cCuLM7U=
github.com/pion/dtls/v3 v3.0.4/go.mod h1:R373CsjxWqNPf6MEkfdy3aSe9niZvL/JaKlGeFphtMg=
github.com/pion/ice/v2 v2.3.37 h1:ObIdaNDu1rCo7hObhs34YSBcO7fjslJMZV0ux+uZWh0=
github.com/pion/ice/v2 v2.3.37/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ=
github.com/pion/ice/v4 v4.0.6 h1:jmM9HwI9lfetQV/39uD0nY4y++XZNPhvzIPCb8EwxUM=
github.com/pion/ice/v4 v4.0.6/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw=
github.com/pion/interceptor v0.1.37 h1:aRA8Zpab/wE7/c0O3fh1PqY0AJI3fCSEM5lRWJVorwI=
github.com/pion/interceptor v0.1.37/go.mod h1:JzxbJ4umVTlZAf+/utHzNesY8tmRkM2lVmkS82TTj8Y=
github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI=
github.com/pion/logging v0.2.3/go.mod h1:z8YfknkquMe1csOrxK5kc+5/ZPAzMxbKLX5aXpbpC90=
github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8=
github.com/pion/mdns v0.0.12/go.mod h1:VExJjv8to/6Wqm1FXK+Ii/Z9tsVk/F5sD/N70cnYFbk=
github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM=
github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA=
github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
@ -899,12 +897,10 @@ github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQp
github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0=
github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo=
github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc=
github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
github.com/pion/turn/v4 v4.0.0 h1:qxplo3Rxa9Yg1xXDxxH8xaqcyGUtbHYw4QSCvmFWvhM=
github.com/pion/turn/v4 v4.0.0/go.mod h1:MuPDkm15nYSklKpN8vWJ9W2M0PlyQZqYt1McGuxG7mA=
github.com/pion/webrtc/v4 v4.0.8 h1:T1ZmnT9qxIJIt4d8XoiMOBrTClGHDDXNg9e/fh018Qc=
github.com/pion/webrtc/v4 v4.0.8/go.mod h1:HHBeUVBAC+j4ZFnYhovEFStF02Arb1EyD4G7e7HBTJw=
github.com/pion/webrtc/v4 v4.0.9 h1:PyOYMRKJgfy0dzPcYtFD/4oW9zaw3Ze3oZzzbj2LV9E=
github.com/pion/webrtc/v4 v4.0.9/go.mod h1:ViHLVaNpiuvaH8pdiuQxuA9awuE6KVzAXx3vVWilOck=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@ -954,8 +950,8 @@ github.com/pseudomuto/protokit v0.2.1 h1:kCYpE3thoR6Esm0CUvd5xbrDTOZPvQPTDeyXpZf
github.com/pseudomuto/protokit v0.2.1/go.mod h1:gt7N5Rz2flBzYafvaxyIxMZC0TTF5jDZfRnw25hAAyo=
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
github.com/quic-go/quic-go v0.49.0 h1:w5iJHXwHxs1QxyBv1EHKuC50GX5to8mJAxvtnttJp94=
github.com/quic-go/quic-go v0.49.0/go.mod h1:s2wDnmCdooUQBmQfpUSTCYBl1/D4FcqbULMMkASvR6s=
github.com/quic-go/quic-go v0.50.0 h1:3H/ld1pa3CYhkcc20TPIyG1bNsdhn9qZBGN3b9/UyUo=
github.com/quic-go/quic-go v0.50.0/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E=
github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg=
github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
@ -1194,8 +1190,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -1322,8 +1318,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M=
golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
golang.org/x/oauth2 v0.26.0 h1:afQXWNNaeC4nvZ0Ed9XvCCzXM6UHJG7iCg0W4fPqSBE=
golang.org/x/oauth2 v0.26.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -1644,8 +1640,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM=
google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@ -1725,5 +1721,3 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
storj.io/drpc v0.0.34 h1:q9zlQKfJ5A7x8NQNFk8x7eKUF78FMhmAbZLnFK+og7I=
storj.io/drpc v0.0.34/go.mod h1:Y9LZaa8esL1PW2IDMqJE7CFSNq7d5bQ3RI7mGPtmKMg=
zombiezen.com/go/sqlite v1.4.0 h1:N1s3RIljwtp4541Y8rM880qgGIgq3fTD2yks1xftnKU=
zombiezen.com/go/sqlite v1.4.0/go.mod h1:0w9F1DN9IZj9AcLS9YDKMboubCACkwYCGkzoy3eG5ik=

View file

@ -7,8 +7,8 @@ import (
"time"
"github.com/cheggaaa/mb/v3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/valyala/fastjson"
"github.com/anyproto/anytype-heart/metrics/anymetry"
@ -79,19 +79,19 @@ func TestClient_SendEvents(t *testing.T) {
go c.startSendingBatchMessages(&testAppInfoProvider{})
c.send(&testEvent{})
time.Sleep(1 * time.Millisecond)
time.Sleep(100 * time.Millisecond)
assert.Equal(t, 1, c.batcher.Len())
require.Equal(t, 1, c.batcher.Len())
telemetry.AssertNotCalled(t, "SendEvents", mock.Anything, mock.Anything)
c.send(&testEvent{})
time.Sleep(1 * time.Millisecond)
time.Sleep(100 * time.Millisecond)
mutex.Lock()
assert.Equal(t, 0, c.batcher.Len())
assert.Equal(t, 2, len(events))
require.Equal(t, 0, c.batcher.Len())
require.Equal(t, 2, len(events))
mutex.Unlock()
assert.True(t, events[0].GetTimestamp() > 0)
require.True(t, events[0].GetTimestamp() > 0)
telemetry.AssertCalled(t, "SendEvents", mock.Anything, mock.Anything)
}

View file

@ -732,7 +732,7 @@ message Event {
FaviconHash faviconHash = 6;
Type type = 7;
TargetObjectId targetObjectId = 8;
message Url {
string value = 1;

View file

@ -170,10 +170,13 @@ func (r *clientds) Init(a *app.App) (err error) {
r.localstoreDS, err = openBadgerWithRecover(opts)
err = anyerror.CleanupError(err)
if err != nil && isBadgerCorrupted(err) {
log.With("error", err).Error("badger db is corrupted")
// because localstore contains mostly recoverable info (with th only exception of objects' lastOpenedDate)
// we can just remove and recreate it
err2 := os.Rename(opts.Dir, filepath.Join(opts.Dir, "-corrupted"))
log.Errorf("failed to rename corrupted localstore: %s", err2)
err2 := os.Rename(opts.Dir, opts.Dir+"-corrupted")
if err2 != nil {
log.Errorf("failed to rename corrupted localstore: %s", err2)
}
var errAfterRemove error
r.localstoreDS, errAfterRemove = openBadgerWithRecover(opts)
errAfterRemove = anyerror.CleanupError(errAfterRemove)

View file

@ -75,16 +75,14 @@ func (w *walletStub) Name() string { return wallet.CName }
func NewStoreFixture(t testing.TB) *StoreFixture {
ctx, cancel := context.WithCancel(context.Background())
walletService := newWalletStub(t)
fullText := ftsearch.TantivyNew()
testApp := &app.App{}
dataStore, err := datastore.NewInMemory()
require.NoError(t, err)
testApp.Register(newWalletStub(t))
testApp.Register(dataStore)
testApp.Register(walletService)
err = fullText.Init(testApp)
require.NoError(t, err)
err = fullText.Run(context.Background())
@ -100,7 +98,7 @@ func NewStoreFixture(t testing.TB) *StoreFixture {
fts: fullText,
sourceService: &detailsFromId{},
arenaPool: &anyenc.ArenaPool{},
repoPath: walletService.RepoPath(),
objectStorePath: t.TempDir(),
oldStore: oldStore,
spaceIndexes: map[string]spaceindex.Store{},
techSpaceIdProvider: &stubTechSpaceIdProvider{},

View file

@ -20,6 +20,7 @@ import (
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/anystorehelper"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/oldstore"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceindex"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceresolverstore"
"github.com/anyproto/anytype-heart/pkg/lib/logging"
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
)
@ -52,6 +53,7 @@ type ObjectStore interface {
GetCrdtDb(spaceId string) anystore.DB
SpaceNameGetter
spaceresolverstore.Store
CrossSpace
}
@ -86,9 +88,11 @@ type TechSpaceIdProvider interface {
}
type dsObjectStore struct {
repoPath string
techSpaceId string
anyStoreConfig anystore.Config
spaceresolverstore.Store
objectStorePath string
techSpaceId string
anyStoreConfig anystore.Config
anyStore anystore.DB
anyStoreLockRemove func() error
@ -145,6 +149,9 @@ func New() ObjectStore {
func (s *dsObjectStore) Init(a *app.App) (err error) {
s.sourceService = app.MustComponent[spaceindex.SourceDetailsFromID](a)
repoPath := app.MustComponent[wallet.Wallet](a).RepoPath()
fts := a.Component(ftsearch.CName)
if fts == nil {
log.Warnf("init objectstore without fulltext")
@ -152,8 +159,10 @@ func (s *dsObjectStore) Init(a *app.App) (err error) {
s.fts = fts.(ftsearch.FTSearch)
}
s.arenaPool = &anyenc.ArenaPool{}
s.repoPath = app.MustComponent[wallet.Wallet](a).RepoPath()
s.anyStoreConfig = *app.MustComponent[configProvider](a).GetAnyStoreConfig()
cfg := app.MustComponent[configProvider](a)
s.objectStorePath = filepath.Join(repoPath, "objectstore")
s.anyStoreConfig = *cfg.GetAnyStoreConfig()
s.setDefaultConfig()
s.oldStore = app.MustComponent[oldstore.Service](a)
s.techSpaceIdProvider = app.MustComponent[TechSpaceIdProvider](a)
@ -168,12 +177,23 @@ func (s *dsObjectStore) Name() (name string) {
func (s *dsObjectStore) Run(ctx context.Context) error {
s.techSpaceId = s.techSpaceIdProvider.TechSpaceId()
dbDir := s.storeRootDir()
err := ensureDirExists(dbDir)
err := ensureDirExists(s.objectStorePath)
if err != nil {
return err
}
return s.openDatabase(ctx, filepath.Join(dbDir, "objects.db"))
err = s.openDatabase(ctx, filepath.Join(s.objectStorePath, "objects.db"))
if err != nil {
return fmt.Errorf("open db: %w", err)
}
store, err := spaceresolverstore.New(s.componentCtx, s.anyStore)
if err != nil {
return fmt.Errorf("new space resolver store: %w", err)
}
s.Store = store
return err
}
func (s *dsObjectStore) setDefaultConfig() {
@ -184,10 +204,6 @@ func (s *dsObjectStore) setDefaultConfig() {
s.anyStoreConfig.SQLiteConnectionOptions["synchronous"] = "off"
}
func (s *dsObjectStore) storeRootDir() string {
return filepath.Join(s.repoPath, "objectstore")
}
func ensureDirExists(dir string) error {
_, err := os.Stat(dir)
if errors.Is(err, os.ErrNotExist) {
@ -238,7 +254,7 @@ func (s *dsObjectStore) preloadExistingObjectStores() error {
var err error
s.spaceStoreDirsCheck.Do(func() {
var entries []os.DirEntry
entries, err = os.ReadDir(s.storeRootDir())
entries, err = os.ReadDir(s.objectStorePath)
s.Lock()
defer s.Unlock()
for _, entry := range entries {
@ -300,7 +316,7 @@ func (s *dsObjectStore) SpaceIndex(spaceId string) spaceindex.Store {
func (s *dsObjectStore) getOrInitSpaceIndex(spaceId string) spaceindex.Store {
store, ok := s.spaceIndexes[spaceId]
if !ok {
dir := filepath.Join(s.storeRootDir(), spaceId)
dir := filepath.Join(s.objectStorePath, spaceId)
err := ensureDirExists(dir)
if err != nil {
return spaceindex.NewInvalidStore(err)
@ -334,7 +350,7 @@ func (s *dsObjectStore) GetCrdtDb(spaceId string) anystore.DB {
db, ok := s.crdtDbs[spaceId]
if !ok {
dir := filepath.Join(s.storeRootDir(), spaceId)
dir := filepath.Join(s.objectStorePath, spaceId)
err := ensureDirExists(dir)
if err != nil {
return nil

View file

@ -0,0 +1,71 @@
package spaceresolverstore
import (
"context"
"errors"
"fmt"
anystore "github.com/anyproto/any-store"
"github.com/anyproto/any-store/anyenc"
"github.com/anyproto/any-store/query"
"github.com/anyproto/anytype-heart/core/domain"
)
const bindKey = "b"
type Store interface {
BindSpaceId(spaceId, objectId string) error
GetSpaceId(objectId string) (spaceId string, err error)
}
type dsObjectStore struct {
componentCtx context.Context
collection anystore.Collection
arenaPool *anyenc.ArenaPool
}
func New(componentCtx context.Context, db anystore.DB) (Store, error) {
collection, err := db.Collection(componentCtx, "bindId")
if err != nil {
return nil, fmt.Errorf("open bindId collection: %w", err)
}
return &dsObjectStore{
componentCtx: componentCtx,
arenaPool: &anyenc.ArenaPool{},
collection: collection,
}, nil
}
func (d *dsObjectStore) BindSpaceId(spaceId, objectId string) error {
return d.modifyBind(d.componentCtx, objectId, spaceId)
}
func (d *dsObjectStore) GetSpaceId(objectId string) (spaceId string, err error) {
doc, err := d.collection.FindId(d.componentCtx, objectId)
if err != nil {
if errors.Is(err, anystore.ErrDocNotFound) {
return "", domain.ErrObjectNotFound
}
return "", err
}
return doc.Value().GetString(bindKey), nil
}
func (d *dsObjectStore) modifyBind(ctx context.Context, objectId, spaceId string) error {
tx, err := d.collection.WriteTx(ctx)
if err != nil {
return err
}
arena := d.arenaPool.Get()
defer d.arenaPool.Put(arena)
mod := query.ModifyFunc(func(a *anyenc.Arena, v *anyenc.Value) (result *anyenc.Value, modified bool, err error) {
v.Set(bindKey, arena.NewString(spaceId))
return v, true, nil
})
_, err = d.collection.UpsertId(tx.Context(), objectId, mod)
if err != nil {
return errors.Join(err, tx.Rollback())
}
return tx.Commit()
}

View file

@ -3,10 +3,12 @@
package mock_clientspace
import (
context "context"
anystorage "github.com/anyproto/anytype-heart/space/spacecore/storage/anystorage"
commonspace "github.com/anyproto/any-sync/commonspace"
context "context"
domain "github.com/anyproto/anytype-heart/core/domain"
headsync "github.com/anyproto/any-sync/commonspace/headsync"
@ -21,8 +23,6 @@ import (
smartblock "github.com/anyproto/anytype-heart/core/block/editor/smartblock"
spacestorage "github.com/anyproto/any-sync/commonspace/spacestorage"
threads "github.com/anyproto/anytype-heart/pkg/lib/threads"
treestorage "github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
@ -1511,19 +1511,19 @@ func (_c *MockSpace_Remove_Call) RunAndReturn(run func(context.Context, string)
}
// Storage provides a mock function with given fields:
func (_m *MockSpace) Storage() spacestorage.SpaceStorage {
func (_m *MockSpace) Storage() anystorage.ClientSpaceStorage {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for Storage")
}
var r0 spacestorage.SpaceStorage
if rf, ok := ret.Get(0).(func() spacestorage.SpaceStorage); ok {
var r0 anystorage.ClientSpaceStorage
if rf, ok := ret.Get(0).(func() anystorage.ClientSpaceStorage); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(spacestorage.SpaceStorage)
r0 = ret.Get(0).(anystorage.ClientSpaceStorage)
}
}
@ -1547,12 +1547,12 @@ func (_c *MockSpace_Storage_Call) Run(run func()) *MockSpace_Storage_Call {
return _c
}
func (_c *MockSpace_Storage_Call) Return(_a0 spacestorage.SpaceStorage) *MockSpace_Storage_Call {
func (_c *MockSpace_Storage_Call) Return(_a0 anystorage.ClientSpaceStorage) *MockSpace_Storage_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockSpace_Storage_Call) RunAndReturn(run func() spacestorage.SpaceStorage) *MockSpace_Storage_Call {
func (_c *MockSpace_Storage_Call) RunAndReturn(run func() anystorage.ClientSpaceStorage) *MockSpace_Storage_Call {
_c.Call.Return(run)
return _c
}

View file

@ -11,7 +11,6 @@ import (
"github.com/anyproto/any-sync/commonspace"
"github.com/anyproto/any-sync/commonspace/headsync"
"github.com/anyproto/any-sync/commonspace/objecttreebuilder"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/any-sync/net/peer"
"github.com/anyproto/any-sync/util/crypto"
"go.uber.org/zap"
@ -29,6 +28,7 @@ import (
"github.com/anyproto/anytype-heart/space/spacecore"
"github.com/anyproto/anytype-heart/space/spacecore/peermanager"
"github.com/anyproto/anytype-heart/space/spacecore/storage"
"github.com/anyproto/anytype-heart/space/spacecore/storage/anystorage"
)
type Space interface {
@ -40,7 +40,7 @@ type Space interface {
DebugAllHeads() []headsync.TreeHeads
DeleteTree(ctx context.Context, id string) (err error)
StoredIds() []string
Storage() spacestorage.SpaceStorage
Storage() anystorage.ClientSpaceStorage
DerivedIDs() threads.DerivedSmartblockIds
@ -124,12 +124,16 @@ func BuildSpace(ctx context.Context, deps SpaceDeps) (Space, error) {
if err != nil {
return nil, fmt.Errorf("derive object ids: %w", err)
}
if deps.StorageService.IsSpaceCreated(deps.CommonSpace.Id()) {
isSpaceCreated, err := sp.Storage().IsSpaceCreated(ctx)
if err != nil {
return nil, fmt.Errorf("is space created: %w", err)
}
if isSpaceCreated {
err = sp.ObjectProvider.CreateMandatoryObjects(ctx, sp)
if err != nil {
return nil, fmt.Errorf("create mandatory objects: %w", err)
}
err = deps.StorageService.UnmarkSpaceCreated(deps.CommonSpace.Id())
err = sp.Storage().UnmarkSpaceCreated(ctx)
if err != nil {
return nil, fmt.Errorf("unmark space created: %w", err)
}
@ -207,8 +211,8 @@ func (s *space) StoredIds() []string {
return s.common.StoredIds()
}
func (s *space) Storage() spacestorage.SpaceStorage {
return s.common.Storage()
func (s *space) Storage() anystorage.ClientSpaceStorage {
return s.common.Storage().(anystorage.ClientSpaceStorage)
}
func (s *space) DerivedIDs() threads.DerivedSmartblockIds {
@ -305,10 +309,7 @@ func (s *space) TryLoadBundledObjects(ctx context.Context) (missingSourceIds []s
if err != nil {
return nil, err
}
storedIds, err := s.Storage().StoredIds()
if err != nil {
return nil, err
}
storedIds := s.StoredIds()
missingIds := bundledObjectIds.Filter(func(bo domain.BundledObjectId) bool {
return !slices.Contains(storedIds, bo.DerivedObjectId)
@ -318,11 +319,7 @@ func (s *space) TryLoadBundledObjects(ctx context.Context) (missingSourceIds []s
s.LoadObjectsIgnoreErrs(ctx, missingIds.DerivedObjectIds())
// todo: make LoadObjectsIgnoreErrs return list of loaded ids
storedIds, err = s.Storage().StoredIds()
if err != nil {
return nil, err
}
storedIds = s.StoredIds()
missingIds = bundledObjectIds.Filter(func(bo domain.BundledObjectId) bool {
return !slices.Contains(storedIds, bo.DerivedObjectId)
})
@ -348,7 +345,10 @@ func (s *space) migrationProfileObject(ctx context.Context) error {
return err
}
extractedProfileExists, _ := s.Storage().HasTree(extractedProfileId)
extractedProfileExists, err := s.Storage().HasTree(ctx, extractedProfileId)
if err != nil {
return err
}
if extractedProfileExists {
return nil
}

View file

@ -77,10 +77,6 @@ func (c *virtualCommonSpace) HandleStreamSyncRequest(ctx context.Context, req *s
return nil
}
func (c *virtualCommonSpace) HandleDeprecatedObjectSyncRequest(ctx context.Context, req *spacesyncproto.ObjectSyncMessage) (resp *spacesyncproto.ObjectSyncMessage, err error) {
return
}
func (c *virtualCommonSpace) HandleStream(stream spacesyncproto.DRPCSpaceSync_ObjectSyncStreamStream) error {
return nil
}
@ -113,7 +109,7 @@ func (c *virtualCommonSpace) DebugAllHeads() []headsync.TreeHeads {
return nil
}
func (c *virtualCommonSpace) Description() (desc commonspace.SpaceDescription, err error) {
func (c *virtualCommonSpace) Description(ctx context.Context) (desc commonspace.SpaceDescription, err error) {
return
}

View file

@ -238,10 +238,6 @@ func (s *syncAclStub) HandleRequest(ctx context.Context, senderId string, reques
return
}
func (s *syncAclStub) SetHeadUpdater(updater headupdater.HeadUpdater) {
return
}
func (s *syncAclStub) SetAclUpdater(updater headupdater.AclUpdater) {
s.updater = updater
return

View file

@ -0,0 +1,65 @@
package oldstorage
import (
"context"
"fmt"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
"github.com/anyproto/anytype-heart/space/spacecore/storage"
"github.com/anyproto/anytype-heart/space/spacecore/storage/badgerstorage"
"github.com/anyproto/anytype-heart/space/spacecore/storage/sqlitestorage"
)
type SpaceStorageMode int
const CName = "client.spacecore.oldstorage"
const (
SpaceStorageModeSqlite SpaceStorageMode = iota // used for new account repos
SpaceStorageModeBadger // used for existing account repos
)
type ClientStorage interface {
oldstorage.SpaceStorageProvider
app.ComponentRunnable
// GetBoundObjectIds returns list of object ids bound (mapped) to space id
GetBoundObjectIds(spaceId string) ([]string, error)
AllSpaceIds() (ids []string, err error)
DeleteSpaceStorage(ctx context.Context, spaceId string) error
GetSpaceID(objectID string) (spaceID string, err error)
EstimateSize() (uint64, error)
}
// storageService is a proxy for the actual storage implementation
type storageService struct {
ClientStorage
}
func New() ClientStorage {
return &storageService{}
}
type configGetter interface {
GetSpaceStorageMode() storage.SpaceStorageMode
}
func (s *storageService) Name() (name string) {
return CName
}
func (s *storageService) Init(a *app.App) (err error) {
mode := a.MustComponent("config").(configGetter).GetSpaceStorageMode()
if mode == storage.SpaceStorageModeBadger {
// for already existing account repos
s.ClientStorage = badgerstorage.New()
} else if mode == storage.SpaceStorageModeSqlite {
// sqlite used for new account repos
s.ClientStorage = sqlitestorage.New()
} else {
return fmt.Errorf("unknown storage mode %d", mode)
}
return s.ClientStorage.Init(a)
}

View file

@ -85,7 +85,7 @@ func (r *rpcHandler) SpacePull(ctx context.Context, request *spacesyncproto.Spac
return
}
spaceDesc, err := sp.Description()
spaceDesc, err := sp.Description(ctx)
if err != nil {
err = spacesyncproto.ErrUnexpected
return

View file

@ -0,0 +1,132 @@
package anystorage
import (
"context"
"errors"
anystore "github.com/anyproto/any-store"
"github.com/anyproto/any-store/anyenc"
"github.com/anyproto/any-store/query"
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/spacestorage"
)
type ClientSpaceStorage interface {
spacestorage.SpaceStorage
HasTree(ctx context.Context, id string) (has bool, err error)
TreeRoot(ctx context.Context, id string) (root *treechangeproto.RawTreeChangeWithId, err error)
MarkSpaceCreated(ctx context.Context) error
IsSpaceCreated(ctx context.Context) (created bool, err error)
UnmarkSpaceCreated(ctx context.Context) error
AllDeletedTreeIds(ctx context.Context) (ids []string, err error)
}
var _ ClientSpaceStorage = (*clientStorage)(nil)
const (
clientCollectionKey = "_client"
clientDocumentKey = "space"
createdKey = "created"
rawChangeKey = "r"
)
type clientStorage struct {
spacestorage.SpaceStorage
clientColl anystore.Collection
}
func (r *clientStorage) AllDeletedTreeIds(ctx context.Context) (ids []string, err error) {
err = r.SpaceStorage.HeadStorage().IterateEntries(ctx, headstorage.IterOpts{Deleted: true}, func(entry headstorage.HeadsEntry) (bool, error) {
ids = append(ids, entry.Id)
return true, nil
})
return
}
func NewClientStorage(ctx context.Context, spaceStorage spacestorage.SpaceStorage) (*clientStorage, error) {
storage := &clientStorage{
SpaceStorage: spaceStorage,
}
anyStore := storage.AnyStore()
client, err := anyStore.Collection(ctx, clientCollectionKey)
if err != nil {
return nil, err
}
storage.clientColl = client
return storage, nil
}
func (r *clientStorage) Close(ctx context.Context) (err error) {
spaceStorageErr := r.SpaceStorage.Close(ctx)
anyStoreErr := r.SpaceStorage.AnyStore().Close()
return errors.Join(spaceStorageErr, anyStoreErr)
}
func (r *clientStorage) HasTree(ctx context.Context, id string) (has bool, err error) {
_, err = r.SpaceStorage.HeadStorage().GetEntry(ctx, id)
isNotFound := errors.Is(err, anystore.ErrDocNotFound)
if err != nil && !isNotFound {
return false, err
}
return !isNotFound, nil
}
func (r *clientStorage) TreeRoot(ctx context.Context, id string) (root *treechangeproto.RawTreeChangeWithId, err error) {
// it should be faster to do it that way, instead of calling TreeStorage
coll, err := r.SpaceStorage.AnyStore().OpenCollection(ctx, objecttree.CollName)
if err != nil {
return nil, err
}
doc, err := coll.FindId(ctx, id)
if err != nil {
return nil, err
}
return &treechangeproto.RawTreeChangeWithId{
Id: id,
RawChange: doc.Value().GetBytes(rawChangeKey),
}, nil
}
func (r *clientStorage) MarkSpaceCreated(ctx context.Context) error {
return r.modifyState(ctx, true)
}
func (r *clientStorage) IsSpaceCreated(ctx context.Context) (isCreated bool, err error) {
doc, err := r.clientColl.FindId(ctx, clientDocumentKey)
isNotFound := errors.Is(err, anystore.ErrDocNotFound)
if err != nil && !isNotFound {
return false, err
}
if isNotFound {
return false, nil
}
return doc.Value().GetBool(createdKey), nil
}
func (r *clientStorage) UnmarkSpaceCreated(ctx context.Context) error {
return r.modifyState(ctx, false)
}
func (r *clientStorage) modifyState(ctx context.Context, isCreated bool) error {
tx, err := r.clientColl.WriteTx(ctx)
if err != nil {
return err
}
arena := &anyenc.Arena{}
val := arena.NewTrue()
if !isCreated {
val = arena.NewFalse()
}
mod := query.ModifyFunc(func(a *anyenc.Arena, v *anyenc.Value) (result *anyenc.Value, modified bool, err error) {
v.Set(createdKey, val)
return v, true, nil
})
_, err = r.clientColl.UpsertId(tx.Context(), clientDocumentKey, mod)
if err != nil {
rollErr := tx.Rollback()
return errors.Join(err, rollErr)
}
return tx.Commit()
}

View file

@ -0,0 +1,917 @@
// Code generated by mockery. DO NOT EDIT.
package mock_anystorage
import (
anystore "github.com/anyproto/any-store"
app "github.com/anyproto/any-sync/app"
context "context"
headstorage "github.com/anyproto/any-sync/commonspace/headsync/headstorage"
list "github.com/anyproto/any-sync/commonspace/object/acl/list"
mock "github.com/stretchr/testify/mock"
objecttree "github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
statestorage "github.com/anyproto/any-sync/commonspace/headsync/statestorage"
treechangeproto "github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
treestorage "github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
)
// MockClientSpaceStorage is an autogenerated mock type for the ClientSpaceStorage type
type MockClientSpaceStorage struct {
mock.Mock
}
type MockClientSpaceStorage_Expecter struct {
mock *mock.Mock
}
func (_m *MockClientSpaceStorage) EXPECT() *MockClientSpaceStorage_Expecter {
return &MockClientSpaceStorage_Expecter{mock: &_m.Mock}
}
// AclStorage provides a mock function with given fields:
func (_m *MockClientSpaceStorage) AclStorage() (list.Storage, error) {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for AclStorage")
}
var r0 list.Storage
var r1 error
if rf, ok := ret.Get(0).(func() (list.Storage, error)); ok {
return rf()
}
if rf, ok := ret.Get(0).(func() list.Storage); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(list.Storage)
}
}
if rf, ok := ret.Get(1).(func() error); ok {
r1 = rf()
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockClientSpaceStorage_AclStorage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AclStorage'
type MockClientSpaceStorage_AclStorage_Call struct {
*mock.Call
}
// AclStorage is a helper method to define mock.On call
func (_e *MockClientSpaceStorage_Expecter) AclStorage() *MockClientSpaceStorage_AclStorage_Call {
return &MockClientSpaceStorage_AclStorage_Call{Call: _e.mock.On("AclStorage")}
}
func (_c *MockClientSpaceStorage_AclStorage_Call) Run(run func()) *MockClientSpaceStorage_AclStorage_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *MockClientSpaceStorage_AclStorage_Call) Return(_a0 list.Storage, _a1 error) *MockClientSpaceStorage_AclStorage_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockClientSpaceStorage_AclStorage_Call) RunAndReturn(run func() (list.Storage, error)) *MockClientSpaceStorage_AclStorage_Call {
_c.Call.Return(run)
return _c
}
// AllDeletedTreeIds provides a mock function with given fields: ctx
func (_m *MockClientSpaceStorage) AllDeletedTreeIds(ctx context.Context) ([]string, error) {
ret := _m.Called(ctx)
if len(ret) == 0 {
panic("no return value specified for AllDeletedTreeIds")
}
var r0 []string
var r1 error
if rf, ok := ret.Get(0).(func(context.Context) ([]string, error)); ok {
return rf(ctx)
}
if rf, ok := ret.Get(0).(func(context.Context) []string); ok {
r0 = rf(ctx)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
if rf, ok := ret.Get(1).(func(context.Context) error); ok {
r1 = rf(ctx)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockClientSpaceStorage_AllDeletedTreeIds_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AllDeletedTreeIds'
type MockClientSpaceStorage_AllDeletedTreeIds_Call struct {
*mock.Call
}
// AllDeletedTreeIds is a helper method to define mock.On call
// - ctx context.Context
func (_e *MockClientSpaceStorage_Expecter) AllDeletedTreeIds(ctx interface{}) *MockClientSpaceStorage_AllDeletedTreeIds_Call {
return &MockClientSpaceStorage_AllDeletedTreeIds_Call{Call: _e.mock.On("AllDeletedTreeIds", ctx)}
}
func (_c *MockClientSpaceStorage_AllDeletedTreeIds_Call) Run(run func(ctx context.Context)) *MockClientSpaceStorage_AllDeletedTreeIds_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context))
})
return _c
}
func (_c *MockClientSpaceStorage_AllDeletedTreeIds_Call) Return(ids []string, err error) *MockClientSpaceStorage_AllDeletedTreeIds_Call {
_c.Call.Return(ids, err)
return _c
}
func (_c *MockClientSpaceStorage_AllDeletedTreeIds_Call) RunAndReturn(run func(context.Context) ([]string, error)) *MockClientSpaceStorage_AllDeletedTreeIds_Call {
_c.Call.Return(run)
return _c
}
// AnyStore provides a mock function with given fields:
func (_m *MockClientSpaceStorage) AnyStore() anystore.DB {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for AnyStore")
}
var r0 anystore.DB
if rf, ok := ret.Get(0).(func() anystore.DB); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(anystore.DB)
}
}
return r0
}
// MockClientSpaceStorage_AnyStore_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AnyStore'
type MockClientSpaceStorage_AnyStore_Call struct {
*mock.Call
}
// AnyStore is a helper method to define mock.On call
func (_e *MockClientSpaceStorage_Expecter) AnyStore() *MockClientSpaceStorage_AnyStore_Call {
return &MockClientSpaceStorage_AnyStore_Call{Call: _e.mock.On("AnyStore")}
}
func (_c *MockClientSpaceStorage_AnyStore_Call) Run(run func()) *MockClientSpaceStorage_AnyStore_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *MockClientSpaceStorage_AnyStore_Call) Return(_a0 anystore.DB) *MockClientSpaceStorage_AnyStore_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockClientSpaceStorage_AnyStore_Call) RunAndReturn(run func() anystore.DB) *MockClientSpaceStorage_AnyStore_Call {
_c.Call.Return(run)
return _c
}
// Close provides a mock function with given fields: ctx
func (_m *MockClientSpaceStorage) Close(ctx context.Context) error {
ret := _m.Called(ctx)
if len(ret) == 0 {
panic("no return value specified for Close")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context) error); ok {
r0 = rf(ctx)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockClientSpaceStorage_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close'
type MockClientSpaceStorage_Close_Call struct {
*mock.Call
}
// Close is a helper method to define mock.On call
// - ctx context.Context
func (_e *MockClientSpaceStorage_Expecter) Close(ctx interface{}) *MockClientSpaceStorage_Close_Call {
return &MockClientSpaceStorage_Close_Call{Call: _e.mock.On("Close", ctx)}
}
func (_c *MockClientSpaceStorage_Close_Call) Run(run func(ctx context.Context)) *MockClientSpaceStorage_Close_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context))
})
return _c
}
func (_c *MockClientSpaceStorage_Close_Call) Return(err error) *MockClientSpaceStorage_Close_Call {
_c.Call.Return(err)
return _c
}
func (_c *MockClientSpaceStorage_Close_Call) RunAndReturn(run func(context.Context) error) *MockClientSpaceStorage_Close_Call {
_c.Call.Return(run)
return _c
}
// CreateTreeStorage provides a mock function with given fields: ctx, payload
func (_m *MockClientSpaceStorage) CreateTreeStorage(ctx context.Context, payload treestorage.TreeStorageCreatePayload) (objecttree.Storage, error) {
ret := _m.Called(ctx, payload)
if len(ret) == 0 {
panic("no return value specified for CreateTreeStorage")
}
var r0 objecttree.Storage
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, treestorage.TreeStorageCreatePayload) (objecttree.Storage, error)); ok {
return rf(ctx, payload)
}
if rf, ok := ret.Get(0).(func(context.Context, treestorage.TreeStorageCreatePayload) objecttree.Storage); ok {
r0 = rf(ctx, payload)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(objecttree.Storage)
}
}
if rf, ok := ret.Get(1).(func(context.Context, treestorage.TreeStorageCreatePayload) error); ok {
r1 = rf(ctx, payload)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockClientSpaceStorage_CreateTreeStorage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateTreeStorage'
type MockClientSpaceStorage_CreateTreeStorage_Call struct {
*mock.Call
}
// CreateTreeStorage is a helper method to define mock.On call
// - ctx context.Context
// - payload treestorage.TreeStorageCreatePayload
func (_e *MockClientSpaceStorage_Expecter) CreateTreeStorage(ctx interface{}, payload interface{}) *MockClientSpaceStorage_CreateTreeStorage_Call {
return &MockClientSpaceStorage_CreateTreeStorage_Call{Call: _e.mock.On("CreateTreeStorage", ctx, payload)}
}
func (_c *MockClientSpaceStorage_CreateTreeStorage_Call) Run(run func(ctx context.Context, payload treestorage.TreeStorageCreatePayload)) *MockClientSpaceStorage_CreateTreeStorage_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(treestorage.TreeStorageCreatePayload))
})
return _c
}
func (_c *MockClientSpaceStorage_CreateTreeStorage_Call) Return(_a0 objecttree.Storage, _a1 error) *MockClientSpaceStorage_CreateTreeStorage_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockClientSpaceStorage_CreateTreeStorage_Call) RunAndReturn(run func(context.Context, treestorage.TreeStorageCreatePayload) (objecttree.Storage, error)) *MockClientSpaceStorage_CreateTreeStorage_Call {
_c.Call.Return(run)
return _c
}
// HasTree provides a mock function with given fields: ctx, id
func (_m *MockClientSpaceStorage) HasTree(ctx context.Context, id string) (bool, error) {
ret := _m.Called(ctx, id)
if len(ret) == 0 {
panic("no return value specified for HasTree")
}
var r0 bool
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string) (bool, error)); ok {
return rf(ctx, id)
}
if rf, ok := ret.Get(0).(func(context.Context, string) bool); ok {
r0 = rf(ctx, id)
} else {
r0 = ret.Get(0).(bool)
}
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, id)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockClientSpaceStorage_HasTree_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'HasTree'
type MockClientSpaceStorage_HasTree_Call struct {
*mock.Call
}
// HasTree is a helper method to define mock.On call
// - ctx context.Context
// - id string
func (_e *MockClientSpaceStorage_Expecter) HasTree(ctx interface{}, id interface{}) *MockClientSpaceStorage_HasTree_Call {
return &MockClientSpaceStorage_HasTree_Call{Call: _e.mock.On("HasTree", ctx, id)}
}
func (_c *MockClientSpaceStorage_HasTree_Call) Run(run func(ctx context.Context, id string)) *MockClientSpaceStorage_HasTree_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(string))
})
return _c
}
func (_c *MockClientSpaceStorage_HasTree_Call) Return(has bool, err error) *MockClientSpaceStorage_HasTree_Call {
_c.Call.Return(has, err)
return _c
}
func (_c *MockClientSpaceStorage_HasTree_Call) RunAndReturn(run func(context.Context, string) (bool, error)) *MockClientSpaceStorage_HasTree_Call {
_c.Call.Return(run)
return _c
}
// HeadStorage provides a mock function with given fields:
func (_m *MockClientSpaceStorage) HeadStorage() headstorage.HeadStorage {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for HeadStorage")
}
var r0 headstorage.HeadStorage
if rf, ok := ret.Get(0).(func() headstorage.HeadStorage); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(headstorage.HeadStorage)
}
}
return r0
}
// MockClientSpaceStorage_HeadStorage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'HeadStorage'
type MockClientSpaceStorage_HeadStorage_Call struct {
*mock.Call
}
// HeadStorage is a helper method to define mock.On call
func (_e *MockClientSpaceStorage_Expecter) HeadStorage() *MockClientSpaceStorage_HeadStorage_Call {
return &MockClientSpaceStorage_HeadStorage_Call{Call: _e.mock.On("HeadStorage")}
}
func (_c *MockClientSpaceStorage_HeadStorage_Call) Run(run func()) *MockClientSpaceStorage_HeadStorage_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *MockClientSpaceStorage_HeadStorage_Call) Return(_a0 headstorage.HeadStorage) *MockClientSpaceStorage_HeadStorage_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockClientSpaceStorage_HeadStorage_Call) RunAndReturn(run func() headstorage.HeadStorage) *MockClientSpaceStorage_HeadStorage_Call {
_c.Call.Return(run)
return _c
}
// Id provides a mock function with given fields:
func (_m *MockClientSpaceStorage) Id() string {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for Id")
}
var r0 string
if rf, ok := ret.Get(0).(func() string); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// MockClientSpaceStorage_Id_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Id'
type MockClientSpaceStorage_Id_Call struct {
*mock.Call
}
// Id is a helper method to define mock.On call
func (_e *MockClientSpaceStorage_Expecter) Id() *MockClientSpaceStorage_Id_Call {
return &MockClientSpaceStorage_Id_Call{Call: _e.mock.On("Id")}
}
func (_c *MockClientSpaceStorage_Id_Call) Run(run func()) *MockClientSpaceStorage_Id_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *MockClientSpaceStorage_Id_Call) Return(_a0 string) *MockClientSpaceStorage_Id_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockClientSpaceStorage_Id_Call) RunAndReturn(run func() string) *MockClientSpaceStorage_Id_Call {
_c.Call.Return(run)
return _c
}
// Init provides a mock function with given fields: a
func (_m *MockClientSpaceStorage) Init(a *app.App) error {
ret := _m.Called(a)
if len(ret) == 0 {
panic("no return value specified for Init")
}
var r0 error
if rf, ok := ret.Get(0).(func(*app.App) error); ok {
r0 = rf(a)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockClientSpaceStorage_Init_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Init'
type MockClientSpaceStorage_Init_Call struct {
*mock.Call
}
// Init is a helper method to define mock.On call
// - a *app.App
func (_e *MockClientSpaceStorage_Expecter) Init(a interface{}) *MockClientSpaceStorage_Init_Call {
return &MockClientSpaceStorage_Init_Call{Call: _e.mock.On("Init", a)}
}
func (_c *MockClientSpaceStorage_Init_Call) Run(run func(a *app.App)) *MockClientSpaceStorage_Init_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(*app.App))
})
return _c
}
func (_c *MockClientSpaceStorage_Init_Call) Return(err error) *MockClientSpaceStorage_Init_Call {
_c.Call.Return(err)
return _c
}
func (_c *MockClientSpaceStorage_Init_Call) RunAndReturn(run func(*app.App) error) *MockClientSpaceStorage_Init_Call {
_c.Call.Return(run)
return _c
}
// IsSpaceCreated provides a mock function with given fields: ctx
func (_m *MockClientSpaceStorage) IsSpaceCreated(ctx context.Context) (bool, error) {
ret := _m.Called(ctx)
if len(ret) == 0 {
panic("no return value specified for IsSpaceCreated")
}
var r0 bool
var r1 error
if rf, ok := ret.Get(0).(func(context.Context) (bool, error)); ok {
return rf(ctx)
}
if rf, ok := ret.Get(0).(func(context.Context) bool); ok {
r0 = rf(ctx)
} else {
r0 = ret.Get(0).(bool)
}
if rf, ok := ret.Get(1).(func(context.Context) error); ok {
r1 = rf(ctx)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockClientSpaceStorage_IsSpaceCreated_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IsSpaceCreated'
type MockClientSpaceStorage_IsSpaceCreated_Call struct {
*mock.Call
}
// IsSpaceCreated is a helper method to define mock.On call
// - ctx context.Context
func (_e *MockClientSpaceStorage_Expecter) IsSpaceCreated(ctx interface{}) *MockClientSpaceStorage_IsSpaceCreated_Call {
return &MockClientSpaceStorage_IsSpaceCreated_Call{Call: _e.mock.On("IsSpaceCreated", ctx)}
}
func (_c *MockClientSpaceStorage_IsSpaceCreated_Call) Run(run func(ctx context.Context)) *MockClientSpaceStorage_IsSpaceCreated_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context))
})
return _c
}
func (_c *MockClientSpaceStorage_IsSpaceCreated_Call) Return(created bool, err error) *MockClientSpaceStorage_IsSpaceCreated_Call {
_c.Call.Return(created, err)
return _c
}
func (_c *MockClientSpaceStorage_IsSpaceCreated_Call) RunAndReturn(run func(context.Context) (bool, error)) *MockClientSpaceStorage_IsSpaceCreated_Call {
_c.Call.Return(run)
return _c
}
// MarkSpaceCreated provides a mock function with given fields: ctx
func (_m *MockClientSpaceStorage) MarkSpaceCreated(ctx context.Context) error {
ret := _m.Called(ctx)
if len(ret) == 0 {
panic("no return value specified for MarkSpaceCreated")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context) error); ok {
r0 = rf(ctx)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockClientSpaceStorage_MarkSpaceCreated_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkSpaceCreated'
type MockClientSpaceStorage_MarkSpaceCreated_Call struct {
*mock.Call
}
// MarkSpaceCreated is a helper method to define mock.On call
// - ctx context.Context
func (_e *MockClientSpaceStorage_Expecter) MarkSpaceCreated(ctx interface{}) *MockClientSpaceStorage_MarkSpaceCreated_Call {
return &MockClientSpaceStorage_MarkSpaceCreated_Call{Call: _e.mock.On("MarkSpaceCreated", ctx)}
}
func (_c *MockClientSpaceStorage_MarkSpaceCreated_Call) Run(run func(ctx context.Context)) *MockClientSpaceStorage_MarkSpaceCreated_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context))
})
return _c
}
func (_c *MockClientSpaceStorage_MarkSpaceCreated_Call) Return(_a0 error) *MockClientSpaceStorage_MarkSpaceCreated_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockClientSpaceStorage_MarkSpaceCreated_Call) RunAndReturn(run func(context.Context) error) *MockClientSpaceStorage_MarkSpaceCreated_Call {
_c.Call.Return(run)
return _c
}
// Name provides a mock function with given fields:
func (_m *MockClientSpaceStorage) Name() string {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for Name")
}
var r0 string
if rf, ok := ret.Get(0).(func() string); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// MockClientSpaceStorage_Name_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Name'
type MockClientSpaceStorage_Name_Call struct {
*mock.Call
}
// Name is a helper method to define mock.On call
func (_e *MockClientSpaceStorage_Expecter) Name() *MockClientSpaceStorage_Name_Call {
return &MockClientSpaceStorage_Name_Call{Call: _e.mock.On("Name")}
}
func (_c *MockClientSpaceStorage_Name_Call) Run(run func()) *MockClientSpaceStorage_Name_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *MockClientSpaceStorage_Name_Call) Return(name string) *MockClientSpaceStorage_Name_Call {
_c.Call.Return(name)
return _c
}
func (_c *MockClientSpaceStorage_Name_Call) RunAndReturn(run func() string) *MockClientSpaceStorage_Name_Call {
_c.Call.Return(run)
return _c
}
// Run provides a mock function with given fields: ctx
func (_m *MockClientSpaceStorage) Run(ctx context.Context) error {
ret := _m.Called(ctx)
if len(ret) == 0 {
panic("no return value specified for Run")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context) error); ok {
r0 = rf(ctx)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockClientSpaceStorage_Run_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Run'
type MockClientSpaceStorage_Run_Call struct {
*mock.Call
}
// Run is a helper method to define mock.On call
// - ctx context.Context
func (_e *MockClientSpaceStorage_Expecter) Run(ctx interface{}) *MockClientSpaceStorage_Run_Call {
return &MockClientSpaceStorage_Run_Call{Call: _e.mock.On("Run", ctx)}
}
func (_c *MockClientSpaceStorage_Run_Call) Run(run func(ctx context.Context)) *MockClientSpaceStorage_Run_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context))
})
return _c
}
func (_c *MockClientSpaceStorage_Run_Call) Return(err error) *MockClientSpaceStorage_Run_Call {
_c.Call.Return(err)
return _c
}
func (_c *MockClientSpaceStorage_Run_Call) RunAndReturn(run func(context.Context) error) *MockClientSpaceStorage_Run_Call {
_c.Call.Return(run)
return _c
}
// StateStorage provides a mock function with given fields:
func (_m *MockClientSpaceStorage) StateStorage() statestorage.StateStorage {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for StateStorage")
}
var r0 statestorage.StateStorage
if rf, ok := ret.Get(0).(func() statestorage.StateStorage); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(statestorage.StateStorage)
}
}
return r0
}
// MockClientSpaceStorage_StateStorage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'StateStorage'
type MockClientSpaceStorage_StateStorage_Call struct {
*mock.Call
}
// StateStorage is a helper method to define mock.On call
func (_e *MockClientSpaceStorage_Expecter) StateStorage() *MockClientSpaceStorage_StateStorage_Call {
return &MockClientSpaceStorage_StateStorage_Call{Call: _e.mock.On("StateStorage")}
}
func (_c *MockClientSpaceStorage_StateStorage_Call) Run(run func()) *MockClientSpaceStorage_StateStorage_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *MockClientSpaceStorage_StateStorage_Call) Return(_a0 statestorage.StateStorage) *MockClientSpaceStorage_StateStorage_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockClientSpaceStorage_StateStorage_Call) RunAndReturn(run func() statestorage.StateStorage) *MockClientSpaceStorage_StateStorage_Call {
_c.Call.Return(run)
return _c
}
// TreeRoot provides a mock function with given fields: ctx, id
func (_m *MockClientSpaceStorage) TreeRoot(ctx context.Context, id string) (*treechangeproto.RawTreeChangeWithId, error) {
ret := _m.Called(ctx, id)
if len(ret) == 0 {
panic("no return value specified for TreeRoot")
}
var r0 *treechangeproto.RawTreeChangeWithId
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string) (*treechangeproto.RawTreeChangeWithId, error)); ok {
return rf(ctx, id)
}
if rf, ok := ret.Get(0).(func(context.Context, string) *treechangeproto.RawTreeChangeWithId); ok {
r0 = rf(ctx, id)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*treechangeproto.RawTreeChangeWithId)
}
}
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, id)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockClientSpaceStorage_TreeRoot_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'TreeRoot'
type MockClientSpaceStorage_TreeRoot_Call struct {
*mock.Call
}
// TreeRoot is a helper method to define mock.On call
// - ctx context.Context
// - id string
func (_e *MockClientSpaceStorage_Expecter) TreeRoot(ctx interface{}, id interface{}) *MockClientSpaceStorage_TreeRoot_Call {
return &MockClientSpaceStorage_TreeRoot_Call{Call: _e.mock.On("TreeRoot", ctx, id)}
}
func (_c *MockClientSpaceStorage_TreeRoot_Call) Run(run func(ctx context.Context, id string)) *MockClientSpaceStorage_TreeRoot_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(string))
})
return _c
}
func (_c *MockClientSpaceStorage_TreeRoot_Call) Return(root *treechangeproto.RawTreeChangeWithId, err error) *MockClientSpaceStorage_TreeRoot_Call {
_c.Call.Return(root, err)
return _c
}
func (_c *MockClientSpaceStorage_TreeRoot_Call) RunAndReturn(run func(context.Context, string) (*treechangeproto.RawTreeChangeWithId, error)) *MockClientSpaceStorage_TreeRoot_Call {
_c.Call.Return(run)
return _c
}
// TreeStorage provides a mock function with given fields: ctx, id
func (_m *MockClientSpaceStorage) TreeStorage(ctx context.Context, id string) (objecttree.Storage, error) {
ret := _m.Called(ctx, id)
if len(ret) == 0 {
panic("no return value specified for TreeStorage")
}
var r0 objecttree.Storage
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string) (objecttree.Storage, error)); ok {
return rf(ctx, id)
}
if rf, ok := ret.Get(0).(func(context.Context, string) objecttree.Storage); ok {
r0 = rf(ctx, id)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(objecttree.Storage)
}
}
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, id)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockClientSpaceStorage_TreeStorage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'TreeStorage'
type MockClientSpaceStorage_TreeStorage_Call struct {
*mock.Call
}
// TreeStorage is a helper method to define mock.On call
// - ctx context.Context
// - id string
func (_e *MockClientSpaceStorage_Expecter) TreeStorage(ctx interface{}, id interface{}) *MockClientSpaceStorage_TreeStorage_Call {
return &MockClientSpaceStorage_TreeStorage_Call{Call: _e.mock.On("TreeStorage", ctx, id)}
}
func (_c *MockClientSpaceStorage_TreeStorage_Call) Run(run func(ctx context.Context, id string)) *MockClientSpaceStorage_TreeStorage_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(string))
})
return _c
}
func (_c *MockClientSpaceStorage_TreeStorage_Call) Return(_a0 objecttree.Storage, _a1 error) *MockClientSpaceStorage_TreeStorage_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockClientSpaceStorage_TreeStorage_Call) RunAndReturn(run func(context.Context, string) (objecttree.Storage, error)) *MockClientSpaceStorage_TreeStorage_Call {
_c.Call.Return(run)
return _c
}
// UnmarkSpaceCreated provides a mock function with given fields: ctx
func (_m *MockClientSpaceStorage) UnmarkSpaceCreated(ctx context.Context) error {
ret := _m.Called(ctx)
if len(ret) == 0 {
panic("no return value specified for UnmarkSpaceCreated")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context) error); ok {
r0 = rf(ctx)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockClientSpaceStorage_UnmarkSpaceCreated_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UnmarkSpaceCreated'
type MockClientSpaceStorage_UnmarkSpaceCreated_Call struct {
*mock.Call
}
// UnmarkSpaceCreated is a helper method to define mock.On call
// - ctx context.Context
func (_e *MockClientSpaceStorage_Expecter) UnmarkSpaceCreated(ctx interface{}) *MockClientSpaceStorage_UnmarkSpaceCreated_Call {
return &MockClientSpaceStorage_UnmarkSpaceCreated_Call{Call: _e.mock.On("UnmarkSpaceCreated", ctx)}
}
func (_c *MockClientSpaceStorage_UnmarkSpaceCreated_Call) Run(run func(ctx context.Context)) *MockClientSpaceStorage_UnmarkSpaceCreated_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context))
})
return _c
}
func (_c *MockClientSpaceStorage_UnmarkSpaceCreated_Call) Return(_a0 error) *MockClientSpaceStorage_UnmarkSpaceCreated_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockClientSpaceStorage_UnmarkSpaceCreated_Call) RunAndReturn(run func(context.Context) error) *MockClientSpaceStorage_UnmarkSpaceCreated_Call {
_c.Call.Return(run)
return _c
}
// NewMockClientSpaceStorage creates a new instance of MockClientSpaceStorage. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockClientSpaceStorage(t interface {
mock.TestingT
Cleanup(func())
}) *MockClientSpaceStorage {
mock := &MockClientSpaceStorage{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}

View file

@ -0,0 +1,144 @@
package anystorage
import (
"context"
"errors"
"fmt"
"os"
"path"
"strings"
"sync"
anystore "github.com/anyproto/any-store"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/app/logger"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"golang.org/x/exp/maps"
)
// nolint: unused
var log = logger.NewNamed(spacestorage.CName)
func New(rootPath string, anyStoreConfig *anystore.Config) *storageService {
return &storageService{
rootPath: rootPath,
config: anyStoreConfig,
}
}
type storageService struct {
rootPath string
config *anystore.Config
sync.Mutex
}
func (s *storageService) AllSpaceIds() (ids []string, err error) {
var files []string
fileInfo, err := os.ReadDir(s.rootPath)
if err != nil {
return files, fmt.Errorf("can't read datadir '%v': %w", s.rootPath, err)
}
for _, file := range fileInfo {
if !strings.HasPrefix(file.Name(), ".") {
files = append(files, file.Name())
}
}
return files, nil
}
func (s *storageService) Run(ctx context.Context) (err error) {
return nil
}
func (s *storageService) openDb(ctx context.Context, id string) (db anystore.DB, err error) {
dbPath := path.Join(s.rootPath, id, "store.db")
if _, err := os.Stat(dbPath); err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, spacestorage.ErrSpaceStorageMissing
}
return nil, err
}
return anystore.Open(ctx, dbPath, s.anyStoreConfig())
}
func (s *storageService) createDb(ctx context.Context, id string) (db anystore.DB, err error) {
dirPath := path.Join(s.rootPath, id)
err = os.MkdirAll(dirPath, 0755)
if err != nil {
return nil, err
}
dbPath := path.Join(dirPath, "store.db")
return anystore.Open(ctx, dbPath, s.anyStoreConfig())
}
func (s *storageService) Close(ctx context.Context) (err error) {
return nil
}
func (s *storageService) Init(a *app.App) (err error) {
if _, err = os.Stat(s.rootPath); err != nil {
err = os.MkdirAll(s.rootPath, 0755)
if err != nil {
return err
}
}
return nil
}
func (s *storageService) Name() (name string) {
return spacestorage.CName
}
func (s *storageService) WaitSpaceStorage(ctx context.Context, id string) (spacestorage.SpaceStorage, error) {
db, err := s.openDb(ctx, id)
if err != nil {
return nil, err
}
st, err := spacestorage.New(ctx, id, db)
if err != nil {
return nil, err
}
return NewClientStorage(ctx, st)
}
func (s *storageService) SpaceExists(id string) bool {
if id == "" {
return false
}
dbPath := path.Join(s.rootPath, id)
if _, err := os.Stat(dbPath); err != nil {
return false
}
return true
}
func (s *storageService) CreateSpaceStorage(ctx context.Context, payload spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error) {
db, err := s.createDb(ctx, payload.SpaceHeaderWithId.Id)
if err != nil {
return nil, err
}
st, err := spacestorage.Create(ctx, db, payload)
if err != nil {
return nil, err
}
return NewClientStorage(ctx, st)
}
func (s *storageService) DeleteSpaceStorage(ctx context.Context, spaceId string) error {
dbPath := path.Join(s.rootPath, spaceId)
return os.RemoveAll(dbPath)
}
func (s *storageService) anyStoreConfig() *anystore.Config {
s.Lock()
defer s.Unlock()
opts := maps.Clone(s.config.SQLiteConnectionOptions)
if opts == nil {
opts = make(map[string]string)
}
opts["synchronous"] = "off"
return &anystore.Config{
ReadConnections: 4,
SQLiteConnectionOptions: opts,
}
}

View file

@ -60,6 +60,10 @@ func (t treeKeys) RawChangeKey(id string) []byte {
return treestorage.JoinStringsToBytes("space", t.spaceId, "t", t.id, id)
}
func (t treeKeys) RawChangesPrefix() []byte {
return treestorage.JoinStringsToBytes("space", t.spaceId, "t", t.id)
}
func (t treeKeys) RawChangePrefix() []byte {
return t.rawChangePrefix
}

View file

@ -4,12 +4,15 @@ import (
"context"
"errors"
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
"github.com/anyproto/any-sync/consensus/consensusproto"
"github.com/dgraph-io/badger/v4"
)
var ErrIncorrectKey = errors.New("key format is incorrect")
var (
ErrIncorrectKey = errors.New("key format is incorrect")
ErrUnknownRecord = errors.New("record does not exist")
)
type listStorage struct {
db *badger.DB
@ -18,7 +21,7 @@ type listStorage struct {
root *consensusproto.RawRecordWithId
}
func newListStorage(spaceId string, db *badger.DB, txn *badger.Txn) (ls liststorage.ListStorage, err error) {
func newListStorage(spaceId string, db *badger.DB, txn *badger.Txn) (ls oldstorage.ListStorage, err error) {
keys := newAclKeys(spaceId)
rootId, err := getTxn(txn, keys.RootIdKey())
if err != nil {
@ -45,7 +48,7 @@ func newListStorage(spaceId string, db *badger.DB, txn *badger.Txn) (ls liststor
return
}
func createListStorage(spaceID string, db *badger.DB, txn *badger.Txn, root *consensusproto.RawRecordWithId) (ls liststorage.ListStorage, err error) {
func createListStorage(spaceID string, db *badger.DB, txn *badger.Txn, root *consensusproto.RawRecordWithId) (ls oldstorage.ListStorage, err error) {
keys := newAclKeys(spaceID)
_, err = getTxn(txn, keys.RootIdKey())
if err != badger.ErrKeyNotFound {
@ -99,7 +102,7 @@ func (l *listStorage) GetRawRecord(_ context.Context, id string) (raw *consensus
res, err := getDB(l.db, l.keys.RawRecordKey(id))
if err != nil {
if err == badger.ErrKeyNotFound {
err = liststorage.ErrUnknownRecord
err = ErrUnknownRecord
}
return
}

View file

@ -4,13 +4,13 @@ import (
"context"
"testing"
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
"github.com/anyproto/any-sync/consensus/consensusproto"
"github.com/dgraph-io/badger/v4"
"github.com/stretchr/testify/require"
)
func testList(t *testing.T, store liststorage.ListStorage, root *consensusproto.RawRecordWithId, head string) {
func testList(t *testing.T, store oldstorage.ListStorage, root *consensusproto.RawRecordWithId, head string) {
require.Equal(t, store.Id(), root.Id)
aclRoot, err := store.Root()
@ -35,7 +35,7 @@ func TestListStorage(t *testing.T) {
return nil
})
var listStore liststorage.ListStorage
var listStore oldstorage.ListStorage
fx.db.View(func(txn *badger.Txn) (err error) {
listStore, err = newListStorage(spaceId, fx.db, txn)
require.NoError(t, err)

View file

@ -7,10 +7,10 @@ import (
"fmt"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
"github.com/dgraph-io/badger/v4"
"golang.org/x/exp/slices"
@ -21,7 +21,7 @@ type spaceStorage struct {
spaceSettingsId string
objDb *badger.DB
keys spaceKeys
aclStorage liststorage.ListStorage
aclStorage oldstorage.ListStorage
header *spacesyncproto.RawSpaceHeaderWithId
service *storageService
}
@ -38,7 +38,7 @@ func (s *spaceStorage) Name() (name string) {
return spacestorage.CName
}
func newSpaceStorage(objDb *badger.DB, spaceId string, service *storageService) (store spacestorage.SpaceStorage, err error) {
func newSpaceStorage(objDb *badger.DB, spaceId string, service *storageService) (store oldstorage.SpaceStorage, err error) {
keys := newSpaceKeys(spaceId)
err = objDb.View(func(txn *badger.Txn) error {
header, err := getTxn(txn, keys.HeaderKey())
@ -75,7 +75,7 @@ func newSpaceStorage(objDb *badger.DB, spaceId string, service *storageService)
return
}
func createSpaceStorage(db *badger.DB, payload spacestorage.SpaceStorageCreatePayload, service *storageService) (store spacestorage.SpaceStorage, err error) {
func createSpaceStorage(db *badger.DB, payload spacestorage.SpaceStorageCreatePayload, service *storageService) (store oldstorage.SpaceStorage, err error) {
keys := newSpaceKeys(payload.SpaceHeaderWithId.Id)
if hasDB(db, keys.HeaderKey()) {
err = spacestorage.ErrSpaceStorageExists
@ -133,15 +133,15 @@ func (s *spaceStorage) HasTree(id string) (bool, error) {
return hasDB(s.objDb, keys.RootIdKey()), nil
}
func (s *spaceStorage) TreeStorage(id string) (treestorage.TreeStorage, error) {
func (s *spaceStorage) TreeStorage(id string) (oldstorage.TreeStorage, error) {
return newTreeStorage(s.objDb, s.spaceId, id)
}
func (s *spaceStorage) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (ts treestorage.TreeStorage, err error) {
func (s *spaceStorage) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (ts oldstorage.TreeStorage, err error) {
return createTreeStorage(s.objDb, s.spaceId, payload)
}
func (s *spaceStorage) AclStorage() (liststorage.ListStorage, error) {
func (s *spaceStorage) AclStorage() (oldstorage.ListStorage, error) {
return s.aclStorage, nil
}
@ -186,7 +186,7 @@ func (s *spaceStorage) AllDeletedTreeIds() (ids []string, err error) {
var isDeleted bool
err = item.Value(func(val []byte) error {
if bytes.Equal(val, []byte(spacestorage.TreeDeletedStatusDeleted)) {
if bytes.Equal(val, []byte(oldstorage.TreeDeletedStatusDeleted)) {
isDeleted = true
}
return nil

View file

@ -7,7 +7,8 @@ import (
"testing"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
spacestorage "github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
"github.com/anyproto/any-sync/consensus/consensusproto"
"github.com/stretchr/testify/assert"
@ -36,7 +37,7 @@ func spaceTestPayload() spacestorage.SpaceStorageCreatePayload {
}
}
func testSpace(t *testing.T, store spacestorage.SpaceStorage, payload spacestorage.SpaceStorageCreatePayload) {
func testSpace(t *testing.T, store oldstorage.SpaceStorage, payload spacestorage.SpaceStorageCreatePayload) {
header, err := store.SpaceHeader()
require.NoError(t, err)
require.Equal(t, payload.SpaceHeaderWithId, header)
@ -139,3 +140,58 @@ func TestSpaceStorage_StoredIds_BigTxn(t *testing.T) {
require.NoError(t, err)
require.Len(t, storedIds, 0)
}
func newServiceFixture(t *testing.T) *storageService {
fx := newFixture(t)
fx.open(t)
t.Cleanup(func() {
fx.stop(t)
})
s := &storageService{
db: fx.db,
keys: newStorageServiceKeys(),
lockedSpaces: map[string]*lockSpace{},
}
return s
}
func TestStorageService_BindSpaceID(t *testing.T) {
fx := newServiceFixture(t)
err := fx.BindSpaceID("spaceId1", "objectId1")
require.NoError(t, err)
spaceId, err := fx.GetSpaceID("objectId1")
require.NoError(t, err)
require.Equal(t, spaceId, "spaceId1")
}
func TestStorageService_GetBoundObjectIds(t *testing.T) {
t.Run("with no bindings", func(t *testing.T) {
fx := newServiceFixture(t)
ids, err := fx.GetBoundObjectIds("spaceId")
require.NoError(t, err)
assert.Empty(t, ids)
})
t.Run("ok", func(t *testing.T) {
fx := newServiceFixture(t)
spaceId := "spaceId1"
err := fx.BindSpaceID(spaceId, "objectId1")
require.NoError(t, err)
err = fx.BindSpaceID(spaceId, "objectId2")
require.NoError(t, err)
ids, err := fx.GetBoundObjectIds(spaceId)
require.NoError(t, err)
assert.ElementsMatch(t, []string{"objectId1", "objectId2"}, ids)
})
}

View file

@ -1,12 +1,15 @@
package badgerstorage
import (
"bytes"
"context"
"errors"
"fmt"
"sync"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
"github.com/dgraph-io/badger/v4"
"github.com/anyproto/anytype-heart/core/domain"
@ -46,11 +49,11 @@ func (s *storageService) Name() (name string) {
return spacestorage.CName
}
func (s *storageService) SpaceStorage(id string) (spacestorage.SpaceStorage, error) {
func (s *storageService) SpaceStorage(id string) (oldstorage.SpaceStorage, error) {
return newSpaceStorage(s.db, id, s)
}
func (s *storageService) WaitSpaceStorage(ctx context.Context, id string) (store spacestorage.SpaceStorage, err error) {
func (s *storageService) WaitSpaceStorage(ctx context.Context, id string) (store oldstorage.SpaceStorage, err error) {
var ls *lockSpace
ls, err = s.checkLock(id, func() error {
store, err = newSpaceStorage(s.db, id, s)
@ -152,7 +155,7 @@ func (s *storageService) unlockSpaceStorage(id string) {
}
}
func (s *storageService) CreateSpaceStorage(payload spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error) {
func (s *storageService) CreateSpaceStorage(payload spacestorage.SpaceStorageCreatePayload) (oldstorage.SpaceStorage, error) {
return createSpaceStorage(s.db, payload, s)
}
@ -166,6 +169,37 @@ func (s *storageService) GetSpaceID(objectID string) (spaceID string, err error)
return spaceID, err
}
func (s *storageService) GetBoundObjectIds(spaceId string) (ids []string, err error) {
prefix := []byte("bind/")
spaceIdBytes := []byte(spaceId)
err = s.db.View(func(txn *badger.Txn) error {
opts := badger.DefaultIteratorOptions
opts.PrefetchValues = false
opts.Prefix = prefix
it := txn.NewIterator(opts)
defer it.Close()
for it.Rewind(); it.Valid(); it.Next() {
item := it.Item()
id := item.Key()
err = item.Value(func(val []byte) error {
if bytes.Equal(spaceIdBytes, val) {
idStr := string(id)
ids = append(ids, idStr[len(prefix):])
}
return nil
})
if err != nil {
return fmt.Errorf("read value: %w", err)
}
}
return nil
})
return
}
func (s *storageService) BindSpaceID(spaceID, objectID string) (err error) {
return badgerhelper.SetValue(s.db, s.keys.BindObjectIDKey(objectID), []byte(spaceID))
}
@ -193,6 +227,11 @@ func (s *storageService) AllSpaceIds() (ids []string, err error) {
return
}
func (s *storageService) EstimateSize() (uint64, error) {
onDiskSize, _ := s.db.EstimateSize(nil)
return onDiskSize, nil
}
func (s *storageService) Run(ctx context.Context) (err error) {
s.db, err = s.provider.SpaceStorage()
if err != nil {

View file

@ -1,11 +1,13 @@
package badgerstorage
import (
"bytes"
"context"
"fmt"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
"github.com/dgraph-io/badger/v4"
)
@ -16,7 +18,7 @@ type treeStorage struct {
root *treechangeproto.RawTreeChangeWithId
}
func newTreeStorage(db *badger.DB, spaceId, treeId string) (ts treestorage.TreeStorage, err error) {
func newTreeStorage(db *badger.DB, spaceId, treeId string) (ts oldstorage.TreeStorage, err error) {
keys := newTreeKeys(spaceId, treeId)
err = db.View(func(txn *badger.Txn) error {
_, err := txn.Get(keys.RootIdKey())
@ -48,7 +50,7 @@ func newTreeStorage(db *badger.DB, spaceId, treeId string) (ts treestorage.TreeS
return
}
func createTreeStorage(db *badger.DB, spaceId string, payload treestorage.TreeStorageCreatePayload) (ts treestorage.TreeStorage, err error) {
func createTreeStorage(db *badger.DB, spaceId string, payload treestorage.TreeStorageCreatePayload) (ts oldstorage.TreeStorage, err error) {
keys := newTreeKeys(spaceId, payload.RootRawChange.Id)
if hasDB(db, keys.RootIdKey()) {
err = treestorage.ErrTreeExists
@ -57,7 +59,7 @@ func createTreeStorage(db *badger.DB, spaceId string, payload treestorage.TreeSt
return forceCreateTreeStorage(db, spaceId, payload)
}
func forceCreateTreeStorage(db *badger.DB, spaceId string, payload treestorage.TreeStorageCreatePayload) (ts treestorage.TreeStorage, err error) {
func forceCreateTreeStorage(db *badger.DB, spaceId string, payload treestorage.TreeStorageCreatePayload) (ts oldstorage.TreeStorage, err error) {
keys := newTreeKeys(spaceId, payload.RootRawChange.Id)
err = db.Update(func(txn *badger.Txn) error {
err = txn.Set(keys.RawChangeKey(payload.RootRawChange.Id), payload.RootRawChange.GetRawChange())
@ -108,7 +110,74 @@ func (t *treeStorage) Heads() (heads []string, err error) {
}
func (t *treeStorage) GetAllChangeIds() (chs []string, err error) {
return nil, fmt.Errorf("get all change ids should not be called")
prefix := t.keys.RawChangesPrefix()
err = t.db.View(func(txn *badger.Txn) error {
opts := badger.DefaultIteratorOptions
opts.PrefetchValues = false
opts.Prefix = prefix
it := txn.NewIterator(opts)
defer it.Close()
for it.Rewind(); it.Valid(); it.Next() {
item := it.Item()
id := item.Key()
changeId := string(id[len(prefix)+1:])
// Special case
if changeId == "heads" {
continue
}
chs = append(chs, changeId)
if err != nil {
return fmt.Errorf("read value: %w", err)
}
}
return nil
})
return chs, err
}
func (t *treeStorage) GetAllChanges() ([]*treechangeproto.RawTreeChangeWithId, error) {
var changes []*treechangeproto.RawTreeChangeWithId
err := t.IterateChanges(func(id string, rawChange []byte) error {
changes = append(changes, &treechangeproto.RawTreeChangeWithId{
Id: id,
RawChange: bytes.Clone(rawChange),
})
return nil
})
return changes, err
}
func (t *treeStorage) IterateChanges(proc func(id string, rawChange []byte) error) error {
prefix := t.keys.RawChangesPrefix()
return t.db.View(func(txn *badger.Txn) error {
opts := badger.DefaultIteratorOptions
opts.PrefetchValues = false
opts.Prefix = prefix
it := txn.NewIterator(opts)
defer it.Close()
for it.Rewind(); it.Valid(); it.Next() {
item := it.Item()
id := item.Key()
changeId := string(id[len(prefix)+1:])
// Special case
if changeId == "heads" {
continue
}
err := item.Value(func(val []byte) error {
return proc(changeId, val)
})
if err != nil {
return fmt.Errorf("read value: %w", err)
}
}
return nil
})
}
func (t *treeStorage) SetHeads(heads []string) (err error) {

View file

@ -1,16 +1,24 @@
package badgerstorage
import (
"bytes"
"context"
"os"
"testing"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
"github.com/dgraph-io/badger/v4"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type oldTreeStorage interface {
oldstorage.ChangesIterator
oldstorage.TreeStorage
}
func treeTestPayload() treestorage.TreeStorageCreatePayload {
rootRawChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("some"), Id: "someRootId"}
otherChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("some other"), Id: "otherId"}
@ -27,7 +35,7 @@ type fixture struct {
db *badger.DB
}
func testTreePayload(t *testing.T, store treestorage.TreeStorage, payload treestorage.TreeStorageCreatePayload) {
func testTreePayload(t *testing.T, store oldstorage.TreeStorage, payload treestorage.TreeStorageCreatePayload) {
require.Equal(t, payload.RootRawChange.Id, store.Id())
root, err := store.Root()
@ -126,8 +134,9 @@ func TestTreeStorage_Methods(t *testing.T) {
fx.open(t)
defer fx.stop(t)
store, err := newTreeStorage(fx.db, spaceId, payload.RootRawChange.Id)
treeStore, err := newTreeStorage(fx.db, spaceId, payload.RootRawChange.Id)
require.NoError(t, err)
store := treeStore.(oldTreeStorage)
testTreePayload(t, store, payload)
t.Run("update heads", func(t *testing.T) {
@ -139,7 +148,7 @@ func TestTreeStorage_Methods(t *testing.T) {
})
t.Run("add raw change, get change and has change", func(t *testing.T) {
newChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("ab"), Id: "newId"}
newChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("ab"), Id: "id10"}
require.NoError(t, store.AddRawChange(newChange))
rawCh, err := store.GetRawChange(context.Background(), newChange.Id)
require.NoError(t, err)
@ -157,6 +166,50 @@ func TestTreeStorage_Methods(t *testing.T) {
require.NoError(t, err)
require.False(t, has)
})
t.Run("iterate changes", func(t *testing.T) {
newChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("foo"), Id: "id01"}
require.NoError(t, store.AddRawChange(newChange))
newChange = &treechangeproto.RawTreeChangeWithId{RawChange: []byte("bar"), Id: "id20"}
require.NoError(t, store.AddRawChange(newChange))
var collected []*treechangeproto.RawTreeChangeWithId
require.NoError(t, store.IterateChanges(func(id string, rawChange []byte) error {
collected = append(collected, &treechangeproto.RawTreeChangeWithId{
Id: id,
RawChange: bytes.Clone(rawChange),
})
return nil
}))
want := []*treechangeproto.RawTreeChangeWithId{
{Id: "id01", RawChange: []byte("foo")},
{Id: "id10", RawChange: []byte("ab")},
{Id: "id20", RawChange: []byte("bar")},
{Id: "otherId", RawChange: []byte("some other")},
{Id: "someRootId", RawChange: []byte("some")},
}
assert.Equal(t, want, collected)
got, err := store.GetAllChanges()
require.NoError(t, err)
assert.Equal(t, want, got)
})
t.Run("get all change ids", func(t *testing.T) {
got, err := store.GetAllChangeIds()
require.NoError(t, err)
want := []string{"id01",
"id10",
"id20",
"otherId",
"someRootId",
}
assert.Equal(t, want, got)
})
}
func TestTreeStorage_Delete(t *testing.T) {

View file

@ -0,0 +1,247 @@
package migrator
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
anystore "github.com/anyproto/any-store"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/spacestorage/migration"
"github.com/anyproto/anytype-heart/core/block/process"
"github.com/anyproto/anytype-heart/pb"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/anystorehelper"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceresolverstore"
"github.com/anyproto/anytype-heart/space/spacecore/oldstorage"
"github.com/anyproto/anytype-heart/space/spacecore/storage"
"github.com/anyproto/anytype-heart/space/spacecore/storage/migratorfinisher"
"github.com/anyproto/anytype-heart/util/freespace"
)
type NotEnoughFreeSpaceError struct {
Free uint64
Required uint64
}
func (e NotEnoughFreeSpaceError) Error() string {
if e.Required == 0 {
return fmt.Sprintf("not enough free space: %d", e.Free)
}
return fmt.Sprintf("Not enough free space: %d, required: %d", e.Free, e.Required)
}
const CName = "client.storage.migration"
type migrator struct {
oldStorage oldstorage.ClientStorage
newStorage storage.ClientStorage
process process.Service
path string
objectStorePath string
finisher migratorfinisher.Service
anyStoreConfig *anystore.Config
}
type pathProvider interface {
GetNewSpaceStorePath() string
GetOldSpaceStorePath() string
GetRepoPath() string
GetAnyStoreConfig() *anystore.Config
}
func New() app.ComponentRunnable {
return &migrator{}
}
func (m *migrator) Init(a *app.App) (err error) {
cfg := a.MustComponent("config").(pathProvider)
m.path = cfg.GetNewSpaceStorePath()
m.objectStorePath = filepath.Join(cfg.GetRepoPath(), "objectstore")
m.oldStorage = app.MustComponent[oldstorage.ClientStorage](a)
m.newStorage = app.MustComponent[storage.ClientStorage](a)
m.process = app.MustComponent[process.Service](a)
m.finisher = app.MustComponent[migratorfinisher.Service](a)
m.anyStoreConfig = cfg.GetAnyStoreConfig()
return nil
}
func (m *migrator) Name() (name string) {
return CName
}
func (m *migrator) Run(ctx context.Context) (err error) {
oldSize, err := m.oldStorage.EstimateSize()
if err != nil {
return fmt.Errorf("estimate size: %w", err)
}
free, err := freespace.GetFreeDiskSpace(m.path)
if err != nil {
return fmt.Errorf("get free disk space: %w", err)
}
requiredDiskSpace := oldSize * 15 / 10
if requiredDiskSpace > free {
return NotEnoughFreeSpaceError{
Free: free,
Required: requiredDiskSpace,
}
}
err = m.run(ctx)
if err != nil {
if strings.Contains(err.Error(), "disk is full") {
return NotEnoughFreeSpaceError{
Free: free,
}
}
return err
}
return nil
}
func (m *migrator) run(ctx context.Context) (err error) {
progress := process.NewProgress(&pb.ModelProcessMessageOfMigration{Migration: &pb.ModelProcessMigration{}})
progress.SetProgressMessage("Migrating spaces")
err = m.process.Add(progress)
if err != nil {
return err
}
defer func() {
progress.Finish(err)
}()
migrator := migration.NewSpaceMigrator(m.oldStorage, m.newStorage, 40, m.path)
allIds, err := m.oldStorage.AllSpaceIds()
if err != nil {
return err
}
var (
total int64
totalMap = make(map[string]int64)
)
for _, id := range allIds {
store, err := m.oldStorage.WaitSpaceStorage(ctx, id)
if err != nil {
return err
}
storedIds, err := store.StoredIds()
if err != nil {
return err
}
total += int64(len(storedIds))
totalMap[id] = int64(len(storedIds))
err = store.Close(ctx)
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
}
progress.SetTotal(total)
for _, id := range allIds {
err := migrator.MigrateId(ctx, id, progress)
if err != nil {
if errors.Is(err, migration.ErrAlreadyMigrated) {
progress.AddDone(totalMap[id])
continue
}
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
}
err = m.doObjectStoreDb(ctx, func(db anystore.DB) error {
resolverStore, err := spaceresolverstore.New(ctx, db)
if err != nil {
return fmt.Errorf("new resolver store: %w", err)
}
for _, spaceId := range allIds {
objectIds, err := m.oldStorage.GetBoundObjectIds(spaceId)
if err != nil {
return fmt.Errorf("get bound object ids: %w", err)
}
for _, objectId := range objectIds {
err = resolverStore.BindSpaceId(spaceId, objectId)
if err != nil {
return fmt.Errorf("bind space id: %w", err)
}
}
}
return nil
})
if err != nil {
return fmt.Errorf("migrate space id bindings: %w", err)
}
// TODO Maybe add some condition?
m.finisher.SetMigrationDone()
return nil
}
// nolint:unused
func (m *migrator) verify(ctx context.Context, fast bool) ([]*verificationReport, error) {
var reports []*verificationReport
err := m.doObjectStoreDb(ctx, func(db anystore.DB) error {
resolverStore, err := spaceresolverstore.New(ctx, db)
if err != nil {
return fmt.Errorf("new resolver store: %w", err)
}
v := &verifier{
fast: fast,
oldStorage: m.oldStorage,
newStorage: m.newStorage,
resolverStore: resolverStore,
}
reports, err = v.verify(ctx)
return err
})
if err != nil {
return nil, err
}
return reports, nil
}
func (m *migrator) doObjectStoreDb(ctx context.Context, proc func(db anystore.DB) error) error {
err := ensureDirExists(m.objectStorePath)
if err != nil {
return fmt.Errorf("ensure dir exists: %w", err)
}
store, lockRemove, err := anystorehelper.OpenDatabaseWithLockCheck(ctx, filepath.Join(m.objectStorePath, "objects.db"), m.anyStoreConfig)
if err != nil {
return fmt.Errorf("open database: %w", err)
}
err = proc(store)
return errors.Join(err, store.Close(), lockRemove())
}
func ensureDirExists(dir string) error {
_, err := os.Stat(dir)
if errors.Is(err, os.ErrNotExist) {
err = os.MkdirAll(dir, 0700)
if err != nil {
return fmt.Errorf("create db dir: %w", err)
}
}
return nil
}
func (m *migrator) Close(ctx context.Context) (err error) {
return nil
}

View file

@ -0,0 +1,203 @@
package migrator
import (
"context"
"io"
"os"
"path/filepath"
"testing"
"github.com/anyproto/any-sync/app"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/anyproto/anytype-heart/core/anytype/config"
"github.com/anyproto/anytype-heart/core/block/process"
"github.com/anyproto/anytype-heart/core/event/mock_event"
"github.com/anyproto/anytype-heart/core/wallet"
"github.com/anyproto/anytype-heart/pb"
"github.com/anyproto/anytype-heart/pkg/lib/datastore/clientds"
"github.com/anyproto/anytype-heart/space/spacecore/oldstorage"
"github.com/anyproto/anytype-heart/space/spacecore/storage"
"github.com/anyproto/anytype-heart/space/spacecore/storage/migratorfinisher"
"github.com/anyproto/anytype-heart/tests/testutil"
)
type fixture struct {
migrator *migrator
app *app.App
cfg *config.Config
}
type quicPreferenceSetterStub struct {
}
func (q *quicPreferenceSetterStub) Init(a *app.App) (err error) {
return nil
}
func (q *quicPreferenceSetterStub) Name() (name string) {
return "quicPreferenceSetterStub"
}
func (q *quicPreferenceSetterStub) PreferQuic(b bool) {
}
func newFixture(t *testing.T, mode storage.SpaceStorageMode) *fixture {
cfg := config.New()
cfg.SpaceStorageMode = mode
cfg.RepoPath = t.TempDir()
fx := &fixture{
cfg: cfg,
}
return fx
}
func (fx *fixture) start(t *testing.T) {
walletService := wallet.NewWithRepoDirAndRandomKeys(fx.cfg.RepoPath)
oldStorage := oldstorage.New()
newStorage := storage.New()
processService := process.New()
eventSender := mock_event.NewMockSender(t)
eventSender.EXPECT().Broadcast(mock.Anything).Run(func(ev *pb.Event) {
}).Maybe()
eventSender.EXPECT().BroadcastExceptSessions(mock.Anything, mock.Anything).Run(func(ev *pb.Event, exceptSessions []string) {
t.Log(ev)
}).Maybe()
migrator := New().(*migrator)
ctx := context.Background()
testApp := &app.App{}
testApp.Register(migratorfinisher.New())
testApp.Register(clientds.New())
testApp.Register(testutil.PrepareMock(ctx, testApp, eventSender))
testApp.Register(&quicPreferenceSetterStub{})
testApp.Register(walletService)
testApp.Register(fx.cfg)
testApp.Register(oldStorage)
testApp.Register(newStorage)
testApp.Register(processService)
testApp.Register(migrator)
fx.app = testApp
fx.migrator = migrator
err := testApp.Start(ctx)
require.NoError(t, err)
}
func assertReports(t *testing.T, reports []*verificationReport) {
for _, report := range reports {
for _, err := range report.errors {
assert.NoError(t, err.err, err.id)
}
}
}
func TestMigration(t *testing.T) {
t.Run("no old storage", func(t *testing.T) {
fx := newFixture(t, storage.SpaceStorageModeSqlite)
fx.start(t)
})
t.Run("with sqlite, fast verification", func(t *testing.T) {
fx := newFixture(t, storage.SpaceStorageModeSqlite)
err := copyFile("testdata/spaceStore.db", fx.cfg.GetOldSpaceStorePath())
require.NoError(t, err)
// TODO Test object->space bindings were populated
fx.start(t)
reports, err := fx.migrator.verify(context.Background(), true)
require.NoError(t, err)
assertReports(t, reports)
})
t.Run("with sqlite, full verification", func(t *testing.T) {
fx := newFixture(t, storage.SpaceStorageModeSqlite)
err := copyFile("testdata/spaceStore.db", fx.cfg.GetOldSpaceStorePath())
require.NoError(t, err)
// TODO Test object->space bindings were populated
fx.start(t)
reports, err := fx.migrator.verify(context.Background(), false)
require.NoError(t, err)
assertReports(t, reports)
})
t.Run("with badger, fast verification", func(t *testing.T) {
fx := newFixture(t, storage.SpaceStorageModeBadger)
err := copyDir("testdata/badger_spacestore", fx.cfg.GetOldSpaceStorePath())
require.NoError(t, err)
// TODO Test object->space bindings were populated
fx.start(t)
reports, err := fx.migrator.verify(context.Background(), true)
require.NoError(t, err)
assertReports(t, reports)
})
t.Run("with badger, full verification", func(t *testing.T) {
fx := newFixture(t, storage.SpaceStorageModeBadger)
err := copyDir("testdata/badger_spacestore", fx.cfg.GetOldSpaceStorePath())
require.NoError(t, err)
// TODO Test object->space bindings were populated
fx.start(t)
reports, err := fx.migrator.verify(context.Background(), false)
require.NoError(t, err)
assertReports(t, reports)
})
}
func copyFile(srcPath string, destPath string) error {
src, err := os.Open(srcPath)
if err != nil {
return err
}
defer src.Close()
dest, err := os.Create(destPath)
if err != nil {
return err
}
defer dest.Close()
_, err = io.Copy(dest, src)
return err
}
func copyDir(srcPath string, destPath string) error {
dir, err := os.ReadDir(srcPath)
if err != nil {
return err
}
err = os.MkdirAll(destPath, os.ModePerm)
if err != nil {
return err
}
for _, entry := range dir {
src := filepath.Join(srcPath, entry.Name())
dst := filepath.Join(destPath, entry.Name())
err := copyFile(src, dst)
if err != nil {
return err
}
}
return nil
}

Binary file not shown.

View file

@ -0,0 +1 @@
/À‰µ 7¤¼¯…püHello Badger

Binary file not shown.

View file

@ -0,0 +1,224 @@
// nolint:unused
package migrator
import (
"bytes"
"context"
"errors"
"fmt"
"slices"
"time"
anystore "github.com/anyproto/any-store"
"github.com/anyproto/any-store/query"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
"github.com/anyproto/any-sync/commonspace/spacestorage"
oldstorage2 "github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
"github.com/anyproto/anytype-heart/core/domain"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceresolverstore"
"github.com/anyproto/anytype-heart/space/spacecore/oldstorage"
"github.com/anyproto/anytype-heart/space/spacecore/storage"
)
type verifier struct {
fast bool
oldStorage oldstorage.ClientStorage
newStorage storage.ClientStorage
resolverStore spaceresolverstore.Store
}
type errorEntry struct {
id string
err error
}
type verificationReport struct {
spaceId string
treesCompared int
errors []errorEntry
totalBytesCompared int
duration time.Duration
}
func (v *verifier) verify(ctx context.Context) ([]*verificationReport, error) {
allSpaceIds, err := v.oldStorage.AllSpaceIds()
if err != nil {
return nil, fmt.Errorf("list all space ids: %w", err)
}
reports := make([]*verificationReport, 0, len(allSpaceIds))
for _, spaceId := range allSpaceIds {
report, err := v.verifySpace(ctx, spaceId)
if err != nil {
return nil, fmt.Errorf("verify space: %w", err)
}
report.spaceId = spaceId
reports = append(reports, report)
}
return reports, nil
}
func (v *verifier) verifySpace(ctx context.Context, spaceId string) (*verificationReport, error) {
oldStore, err := v.oldStorage.WaitSpaceStorage(ctx, spaceId)
if err != nil {
return nil, fmt.Errorf("open old store: %w", err)
}
newStore, err := v.newStorage.WaitSpaceStorage(ctx, spaceId)
if err != nil {
return nil, fmt.Errorf("open new store: %w", err)
}
storedIds, err := oldStore.StoredIds()
if err != nil {
return nil, err
}
newStoreCollection, err := newStore.AnyStore().Collection(ctx, "changes")
if err != nil {
return nil, fmt.Errorf("get new store collection: %w", err)
}
report := &verificationReport{}
now := time.Now()
for _, treeId := range storedIds {
bytesCompared, err := v.verifyTree(ctx, oldStore, newStore, newStoreCollection, treeId)
if err != nil {
report.errors = append(report.errors, errorEntry{id: treeId, err: err})
}
report.treesCompared++
report.totalBytesCompared += bytesCompared
}
report.duration = time.Since(now)
err = oldStore.Close(ctx)
if err != nil {
return nil, err
}
return report, nil
}
func (v *verifier) verifyTree(ctx context.Context, oldStore oldstorage2.SpaceStorage, newStore spacestorage.SpaceStorage, newStoreCollection anystore.Collection, treeId string) (int, error) {
newHeadStorage := newStore.HeadStorage()
entry, err := newHeadStorage.GetEntry(ctx, treeId)
if err != nil {
return 0, fmt.Errorf("get heads entry: %w", err)
}
oldTreeStorage, err := oldStore.TreeStorage(treeId)
if err != nil {
return 0, fmt.Errorf("open old tree storage: %w", err)
}
oldHeads, err := oldTreeStorage.Heads()
if err != nil {
return 0, fmt.Errorf("open old heads storage: %w", err)
}
if !slices.Equal(oldHeads, entry.Heads) {
return 0, fmt.Errorf("old heads doesn't match tree storage")
}
err = v.verifySpaceBindings(oldStore, treeId)
if err != nil {
return 0, fmt.Errorf("verify space: %w", err)
}
newTreeStorage, err := newStore.TreeStorage(ctx, treeId)
if err != nil {
return 0, fmt.Errorf("open new tree storage: %w", err)
}
var bytesCompared int
if v.fast {
err = v.verifyChangesFast(ctx, oldTreeStorage, newTreeStorage)
if err != nil {
return 0, fmt.Errorf("verify tree fast: %w", err)
}
} else {
bytesCompared, err = v.verifyChangesFull(ctx, newStoreCollection, oldTreeStorage)
if err != nil {
return 0, fmt.Errorf("verify tree fast: %w", err)
}
}
return bytesCompared, nil
}
func (v *verifier) verifySpaceBindings(oldStore oldstorage2.SpaceStorage, treeId string) error {
gotSpaceId, err := v.resolverStore.GetSpaceId(treeId)
// If it's not found in new store, check that it's not found in old store either
if errors.Is(err, domain.ErrObjectNotFound) {
_, err = v.oldStorage.GetSpaceID(treeId)
if errors.Is(err, domain.ErrObjectNotFound) {
return nil
}
if err == nil {
return fmt.Errorf("binding is not found in new store")
}
return fmt.Errorf("check binding in old store: %w", err)
} else if err != nil {
return fmt.Errorf("resolve space id for object: %w", err)
}
if gotSpaceId != oldStore.Id() {
return fmt.Errorf("resolved spaced id mismatch")
}
return nil
}
// verifyChangesFast checks only existence of changes
func (v *verifier) verifyChangesFast(ctx context.Context, oldTreeStorage oldstorage2.TreeStorage, newTreeStorage objecttree.Storage) error {
oldChangeIds, err := oldTreeStorage.GetAllChangeIds()
if err != nil {
return fmt.Errorf("get old change ids: %w", err)
}
if len(oldChangeIds) == 0 {
return fmt.Errorf("old change ids is empty")
}
for _, oldChangeId := range oldChangeIds {
ok, err := newTreeStorage.Has(ctx, oldChangeId)
if err != nil {
return fmt.Errorf("get old change id: %w", err)
}
if !ok {
return fmt.Errorf("old change id doesn't exist")
}
}
return nil
}
// verifyChangesFull checks byte contents of changes
func (v *verifier) verifyChangesFull(ctx context.Context, newStoreCollection anystore.Collection, oldTreeStorage oldstorage2.TreeStorage) (int, error) {
iterator, ok := oldTreeStorage.(oldstorage2.ChangesIterator)
if !ok {
return 0, fmt.Errorf("old tree storage doesn't implement ChangesIterator")
}
var bytesCompared int
iter, err := newStoreCollection.Find(query.Key{Path: []string{"t"}, Filter: query.NewComp(query.CompOpEq, oldTreeStorage.Id())}).Sort("id").Iter(ctx)
if err != nil {
return 0, fmt.Errorf("new store: changes iterator: %w", err)
}
defer iter.Close()
err = iterator.IterateChanges(func(id string, oldChange []byte) error {
if !iter.Next() {
return fmt.Errorf("new store iterator: no more changes")
}
doc, err := iter.Doc()
if err != nil {
return fmt.Errorf("new store iterator: read doc: %w", err)
}
newId := doc.Value().GetString("id")
if newId != id {
return fmt.Errorf("new store iterator: id doesn't match")
}
bytesCompared += len(oldChange)
if !bytes.Equal(oldChange, doc.Value().GetBytes("r")) {
return fmt.Errorf("old tree change doesn't match tree storage")
}
return nil
})
return bytesCompared, err
}

View file

@ -0,0 +1,96 @@
package migratorfinisher
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/anyproto/any-sync/app"
)
const (
migratedName = "space_store_migrated"
objectStoreFolder = "objectstore"
crdtDb = "crdt"
)
const CName = "space.spacecore.storage.migratorfinisher"
type finisher struct {
isMigrationDone bool
newStorePath string
oldPath string
}
type Service interface {
app.ComponentRunnable
SetMigrationDone()
}
func New() Service {
return &finisher{}
}
type pathProvider interface {
GetNewSpaceStorePath() string
GetOldSpaceStorePath() string
}
func (f *finisher) Init(a *app.App) (err error) {
cfg := a.MustComponent("config").(pathProvider)
f.newStorePath = cfg.GetNewSpaceStorePath()
f.oldPath = cfg.GetOldSpaceStorePath()
return nil
}
func (f *finisher) Name() (name string) {
return CName
}
func (f *finisher) Run(ctx context.Context) (err error) {
return nil
}
func (f *finisher) SetMigrationDone() {
f.isMigrationDone = true
}
func (f *finisher) Close(ctx context.Context) error {
if !f.isMigrationDone {
return nil
}
err := f.removeCrdtIndexes()
if err != nil {
return nil
}
return f.renameOldStore()
}
func (f *finisher) renameOldStore() error {
newName := migratedName
newPath := filepath.Join(filepath.Dir(f.oldPath), newName+filepath.Ext(f.oldPath))
err := os.Rename(f.oldPath, newPath)
if err != nil {
return fmt.Errorf("failed to rename: %w", err)
}
return nil
}
func (f *finisher) removeCrdtIndexes() error {
rootDir := filepath.Join(filepath.Dir(f.newStorePath), objectStoreFolder)
prefix := crdtDb
return filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() && strings.HasPrefix(info.Name(), prefix) {
if removeErr := os.Remove(path); removeErr != nil {
return removeErr
}
}
return nil
})
}

View file

@ -82,53 +82,6 @@ func (_c *MockClientStorage_AllSpaceIds_Call) RunAndReturn(run func() ([]string,
return _c
}
// BindSpaceID provides a mock function with given fields: spaceID, objectID
func (_m *MockClientStorage) BindSpaceID(spaceID string, objectID string) error {
ret := _m.Called(spaceID, objectID)
if len(ret) == 0 {
panic("no return value specified for BindSpaceID")
}
var r0 error
if rf, ok := ret.Get(0).(func(string, string) error); ok {
r0 = rf(spaceID, objectID)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockClientStorage_BindSpaceID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BindSpaceID'
type MockClientStorage_BindSpaceID_Call struct {
*mock.Call
}
// BindSpaceID is a helper method to define mock.On call
// - spaceID string
// - objectID string
func (_e *MockClientStorage_Expecter) BindSpaceID(spaceID interface{}, objectID interface{}) *MockClientStorage_BindSpaceID_Call {
return &MockClientStorage_BindSpaceID_Call{Call: _e.mock.On("BindSpaceID", spaceID, objectID)}
}
func (_c *MockClientStorage_BindSpaceID_Call) Run(run func(spaceID string, objectID string)) *MockClientStorage_BindSpaceID_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(string), args[1].(string))
})
return _c
}
func (_c *MockClientStorage_BindSpaceID_Call) Return(err error) *MockClientStorage_BindSpaceID_Call {
_c.Call.Return(err)
return _c
}
func (_c *MockClientStorage_BindSpaceID_Call) RunAndReturn(run func(string, string) error) *MockClientStorage_BindSpaceID_Call {
_c.Call.Return(run)
return _c
}
// Close provides a mock function with given fields: ctx
func (_m *MockClientStorage) Close(ctx context.Context) error {
ret := _m.Called(ctx)
@ -175,9 +128,9 @@ func (_c *MockClientStorage_Close_Call) RunAndReturn(run func(context.Context) e
return _c
}
// CreateSpaceStorage provides a mock function with given fields: payload
func (_m *MockClientStorage) CreateSpaceStorage(payload spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error) {
ret := _m.Called(payload)
// CreateSpaceStorage provides a mock function with given fields: ctx, payload
func (_m *MockClientStorage) CreateSpaceStorage(ctx context.Context, payload spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error) {
ret := _m.Called(ctx, payload)
if len(ret) == 0 {
panic("no return value specified for CreateSpaceStorage")
@ -185,19 +138,19 @@ func (_m *MockClientStorage) CreateSpaceStorage(payload spacestorage.SpaceStorag
var r0 spacestorage.SpaceStorage
var r1 error
if rf, ok := ret.Get(0).(func(spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error)); ok {
return rf(payload)
if rf, ok := ret.Get(0).(func(context.Context, spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error)); ok {
return rf(ctx, payload)
}
if rf, ok := ret.Get(0).(func(spacestorage.SpaceStorageCreatePayload) spacestorage.SpaceStorage); ok {
r0 = rf(payload)
if rf, ok := ret.Get(0).(func(context.Context, spacestorage.SpaceStorageCreatePayload) spacestorage.SpaceStorage); ok {
r0 = rf(ctx, payload)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(spacestorage.SpaceStorage)
}
}
if rf, ok := ret.Get(1).(func(spacestorage.SpaceStorageCreatePayload) error); ok {
r1 = rf(payload)
if rf, ok := ret.Get(1).(func(context.Context, spacestorage.SpaceStorageCreatePayload) error); ok {
r1 = rf(ctx, payload)
} else {
r1 = ret.Error(1)
}
@ -211,14 +164,15 @@ type MockClientStorage_CreateSpaceStorage_Call struct {
}
// CreateSpaceStorage is a helper method to define mock.On call
// - ctx context.Context
// - payload spacestorage.SpaceStorageCreatePayload
func (_e *MockClientStorage_Expecter) CreateSpaceStorage(payload interface{}) *MockClientStorage_CreateSpaceStorage_Call {
return &MockClientStorage_CreateSpaceStorage_Call{Call: _e.mock.On("CreateSpaceStorage", payload)}
func (_e *MockClientStorage_Expecter) CreateSpaceStorage(ctx interface{}, payload interface{}) *MockClientStorage_CreateSpaceStorage_Call {
return &MockClientStorage_CreateSpaceStorage_Call{Call: _e.mock.On("CreateSpaceStorage", ctx, payload)}
}
func (_c *MockClientStorage_CreateSpaceStorage_Call) Run(run func(payload spacestorage.SpaceStorageCreatePayload)) *MockClientStorage_CreateSpaceStorage_Call {
func (_c *MockClientStorage_CreateSpaceStorage_Call) Run(run func(ctx context.Context, payload spacestorage.SpaceStorageCreatePayload)) *MockClientStorage_CreateSpaceStorage_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(spacestorage.SpaceStorageCreatePayload))
run(args[0].(context.Context), args[1].(spacestorage.SpaceStorageCreatePayload))
})
return _c
}
@ -228,7 +182,7 @@ func (_c *MockClientStorage_CreateSpaceStorage_Call) Return(_a0 spacestorage.Spa
return _c
}
func (_c *MockClientStorage_CreateSpaceStorage_Call) RunAndReturn(run func(spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error)) *MockClientStorage_CreateSpaceStorage_Call {
func (_c *MockClientStorage_CreateSpaceStorage_Call) RunAndReturn(run func(context.Context, spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error)) *MockClientStorage_CreateSpaceStorage_Call {
_c.Call.Return(run)
return _c
}
@ -280,62 +234,6 @@ func (_c *MockClientStorage_DeleteSpaceStorage_Call) RunAndReturn(run func(conte
return _c
}
// GetSpaceID provides a mock function with given fields: objectID
func (_m *MockClientStorage) GetSpaceID(objectID string) (string, error) {
ret := _m.Called(objectID)
if len(ret) == 0 {
panic("no return value specified for GetSpaceID")
}
var r0 string
var r1 error
if rf, ok := ret.Get(0).(func(string) (string, error)); ok {
return rf(objectID)
}
if rf, ok := ret.Get(0).(func(string) string); ok {
r0 = rf(objectID)
} else {
r0 = ret.Get(0).(string)
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(objectID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockClientStorage_GetSpaceID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSpaceID'
type MockClientStorage_GetSpaceID_Call struct {
*mock.Call
}
// GetSpaceID is a helper method to define mock.On call
// - objectID string
func (_e *MockClientStorage_Expecter) GetSpaceID(objectID interface{}) *MockClientStorage_GetSpaceID_Call {
return &MockClientStorage_GetSpaceID_Call{Call: _e.mock.On("GetSpaceID", objectID)}
}
func (_c *MockClientStorage_GetSpaceID_Call) Run(run func(objectID string)) *MockClientStorage_GetSpaceID_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(string))
})
return _c
}
func (_c *MockClientStorage_GetSpaceID_Call) Return(spaceID string, err error) *MockClientStorage_GetSpaceID_Call {
_c.Call.Return(spaceID, err)
return _c
}
func (_c *MockClientStorage_GetSpaceID_Call) RunAndReturn(run func(string) (string, error)) *MockClientStorage_GetSpaceID_Call {
_c.Call.Return(run)
return _c
}
// Init provides a mock function with given fields: a
func (_m *MockClientStorage) Init(a *app.App) error {
ret := _m.Called(a)
@ -382,98 +280,6 @@ func (_c *MockClientStorage_Init_Call) RunAndReturn(run func(*app.App) error) *M
return _c
}
// IsSpaceCreated provides a mock function with given fields: id
func (_m *MockClientStorage) IsSpaceCreated(id string) bool {
ret := _m.Called(id)
if len(ret) == 0 {
panic("no return value specified for IsSpaceCreated")
}
var r0 bool
if rf, ok := ret.Get(0).(func(string) bool); ok {
r0 = rf(id)
} else {
r0 = ret.Get(0).(bool)
}
return r0
}
// MockClientStorage_IsSpaceCreated_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IsSpaceCreated'
type MockClientStorage_IsSpaceCreated_Call struct {
*mock.Call
}
// IsSpaceCreated is a helper method to define mock.On call
// - id string
func (_e *MockClientStorage_Expecter) IsSpaceCreated(id interface{}) *MockClientStorage_IsSpaceCreated_Call {
return &MockClientStorage_IsSpaceCreated_Call{Call: _e.mock.On("IsSpaceCreated", id)}
}
func (_c *MockClientStorage_IsSpaceCreated_Call) Run(run func(id string)) *MockClientStorage_IsSpaceCreated_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(string))
})
return _c
}
func (_c *MockClientStorage_IsSpaceCreated_Call) Return(created bool) *MockClientStorage_IsSpaceCreated_Call {
_c.Call.Return(created)
return _c
}
func (_c *MockClientStorage_IsSpaceCreated_Call) RunAndReturn(run func(string) bool) *MockClientStorage_IsSpaceCreated_Call {
_c.Call.Return(run)
return _c
}
// MarkSpaceCreated provides a mock function with given fields: id
func (_m *MockClientStorage) MarkSpaceCreated(id string) error {
ret := _m.Called(id)
if len(ret) == 0 {
panic("no return value specified for MarkSpaceCreated")
}
var r0 error
if rf, ok := ret.Get(0).(func(string) error); ok {
r0 = rf(id)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockClientStorage_MarkSpaceCreated_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkSpaceCreated'
type MockClientStorage_MarkSpaceCreated_Call struct {
*mock.Call
}
// MarkSpaceCreated is a helper method to define mock.On call
// - id string
func (_e *MockClientStorage_Expecter) MarkSpaceCreated(id interface{}) *MockClientStorage_MarkSpaceCreated_Call {
return &MockClientStorage_MarkSpaceCreated_Call{Call: _e.mock.On("MarkSpaceCreated", id)}
}
func (_c *MockClientStorage_MarkSpaceCreated_Call) Run(run func(id string)) *MockClientStorage_MarkSpaceCreated_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(string))
})
return _c
}
func (_c *MockClientStorage_MarkSpaceCreated_Call) Return(err error) *MockClientStorage_MarkSpaceCreated_Call {
_c.Call.Return(err)
return _c
}
func (_c *MockClientStorage_MarkSpaceCreated_Call) RunAndReturn(run func(string) error) *MockClientStorage_MarkSpaceCreated_Call {
_c.Call.Return(run)
return _c
}
// Name provides a mock function with given fields:
func (_m *MockClientStorage) Name() string {
ret := _m.Called()
@ -611,52 +417,6 @@ func (_c *MockClientStorage_SpaceExists_Call) RunAndReturn(run func(string) bool
return _c
}
// UnmarkSpaceCreated provides a mock function with given fields: id
func (_m *MockClientStorage) UnmarkSpaceCreated(id string) error {
ret := _m.Called(id)
if len(ret) == 0 {
panic("no return value specified for UnmarkSpaceCreated")
}
var r0 error
if rf, ok := ret.Get(0).(func(string) error); ok {
r0 = rf(id)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockClientStorage_UnmarkSpaceCreated_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UnmarkSpaceCreated'
type MockClientStorage_UnmarkSpaceCreated_Call struct {
*mock.Call
}
// UnmarkSpaceCreated is a helper method to define mock.On call
// - id string
func (_e *MockClientStorage_Expecter) UnmarkSpaceCreated(id interface{}) *MockClientStorage_UnmarkSpaceCreated_Call {
return &MockClientStorage_UnmarkSpaceCreated_Call{Call: _e.mock.On("UnmarkSpaceCreated", id)}
}
func (_c *MockClientStorage_UnmarkSpaceCreated_Call) Run(run func(id string)) *MockClientStorage_UnmarkSpaceCreated_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(string))
})
return _c
}
func (_c *MockClientStorage_UnmarkSpaceCreated_Call) Return(err error) *MockClientStorage_UnmarkSpaceCreated_Call {
_c.Call.Return(err)
return _c
}
func (_c *MockClientStorage_UnmarkSpaceCreated_Call) RunAndReturn(run func(string) error) *MockClientStorage_UnmarkSpaceCreated_Call {
_c.Call.Return(run)
return _c
}
// WaitSpaceStorage provides a mock function with given fields: ctx, id
func (_m *MockClientStorage) WaitSpaceStorage(ctx context.Context, id string) (spacestorage.SpaceStorage, error) {
ret := _m.Called(ctx, id)

View file

@ -6,12 +6,14 @@ import (
"errors"
"sync"
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
"github.com/anyproto/any-sync/consensus/consensusproto"
)
func newListStorage(ss *spaceStorage, treeId string) (liststorage.ListStorage, error) {
var ErrUnknownRecord = errors.New("record does not exist")
func newListStorage(ss *spaceStorage, treeId string) (oldstorage.ListStorage, error) {
ts := &listStorage{
listId: treeId,
spaceStorage: ss,
@ -38,7 +40,7 @@ type listStorage struct {
func (t *listStorage) Root() (*consensusproto.RawRecordWithId, error) {
tch, err := t.spaceStorage.TreeRoot(t.listId)
if err != nil {
return nil, replaceNoRowsErr(err, liststorage.ErrUnknownRecord)
return nil, replaceNoRowsErr(err, ErrUnknownRecord)
}
return &consensusproto.RawRecordWithId{
Payload: tch.RawChange,
@ -66,7 +68,7 @@ func (t *listStorage) SetHead(headId string) error {
func (t *listStorage) GetRawRecord(ctx context.Context, id string) (*consensusproto.RawRecordWithId, error) {
tch, err := t.spaceStorage.TreeRoot(id)
if err != nil {
return nil, replaceNoRowsErr(err, liststorage.ErrUnknownRecord)
return nil, replaceNoRowsErr(err, ErrUnknownRecord)
}
return &consensusproto.RawRecordWithId{
Payload: tch.RawChange,

View file

@ -3,7 +3,7 @@ package sqlitestorage
import (
"testing"
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
"github.com/anyproto/any-sync/consensus/consensusproto"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -56,7 +56,7 @@ func TestListStorage_SetHead(t *testing.T) {
}
func testList(t *testing.T, store liststorage.ListStorage, root *consensusproto.RawRecordWithId, head string) {
func testList(t *testing.T, store oldstorage.ListStorage, root *consensusproto.RawRecordWithId, head string) {
require.Equal(t, store.Id(), root.Id)
aclRoot, err := store.Root()

View file

@ -5,6 +5,7 @@ import (
"database/sql"
"errors"
"net/url"
"os"
"sync"
"time"
@ -12,6 +13,7 @@ import (
"github.com/anyproto/any-sync/app/logger"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
"github.com/globalsign/mgo/bson"
"github.com/mattn/go-sqlite3"
"go.uber.org/atomic"
@ -30,7 +32,7 @@ var (
)
type configGetter interface {
GetSpaceStorePath() string
GetSqliteStorePath() string
GetTempDirPath() string
}
@ -51,6 +53,8 @@ type storageService struct {
allTreeDelStatus,
change,
hasTree,
listChanges,
iterateChanges,
hasChange,
updateTreeHeads,
deleteTree,
@ -60,6 +64,7 @@ type storageService struct {
spaceIds,
spaceIsCreated,
upsertBind,
getAllBinds,
deleteSpace,
deleteTreesBySpace,
deleteChangesBySpace,
@ -91,7 +96,7 @@ func New() *storageService {
}
func (s *storageService) Init(a *app.App) (err error) {
s.dbPath = a.MustComponent("config").(configGetter).GetSpaceStorePath()
s.dbPath = a.MustComponent("config").(configGetter).GetSqliteStorePath()
s.dbTempPath = a.MustComponent("config").(configGetter).GetTempDirPath()
s.lockedSpaces = map[string]*lockSpace{}
if s.checkpointAfterWrite == 0 {
@ -159,7 +164,7 @@ func (s *storageService) Name() (name string) {
return spacestorage.CName
}
func (s *storageService) WaitSpaceStorage(ctx context.Context, id string) (store spacestorage.SpaceStorage, err error) {
func (s *storageService) WaitSpaceStorage(ctx context.Context, id string) (store oldstorage.SpaceStorage, err error) {
var ls *lockSpace
ls, err = s.checkLock(id, func() error {
store, err = newSpaceStorage(s, id)
@ -277,7 +282,7 @@ func (s *storageService) unlockSpaceStorage(id string) {
}
}
func (s *storageService) CreateSpaceStorage(payload spacestorage.SpaceStorageCreatePayload) (ss spacestorage.SpaceStorage, err error) {
func (s *storageService) CreateSpaceStorage(payload spacestorage.SpaceStorageCreatePayload) (ss oldstorage.SpaceStorage, err error) {
_, err = s.checkLock(payload.SpaceHeaderWithId.Id, func() error {
ss, err = createSpaceStorage(s, payload)
return err
@ -291,6 +296,25 @@ func (s *storageService) GetSpaceID(objectID string) (spaceID string, err error)
return
}
func (s *storageService) GetBoundObjectIds(spaceId string) ([]string, error) {
rows, err := s.stmt.getAllBinds.Query(spaceId)
if err != nil {
return nil, err
}
defer func() {
err = errors.Join(rows.Close())
}()
var ids []string
for rows.Next() {
var id string
if err = rows.Scan(&id); err != nil {
return nil, err
}
ids = append(ids, id)
}
return ids, nil
}
func (s *storageService) BindSpaceID(spaceID, objectID string) (err error) {
_, err = s.stmt.upsertBind.Exec(objectID, spaceID, spaceID)
return
@ -352,6 +376,15 @@ func (s *storageService) checkpoint() (err error) {
return err
}
func (s *storageService) EstimateSize() (uint64, error) {
stat, err := os.Stat(s.dbPath)
if err != nil {
return 0, err
}
// nolint: gosec
return uint64(stat.Size()), nil
}
func (s *storageService) Close(ctx context.Context) (err error) {
if s.ctxCancel != nil {
s.ctxCancel()

View file

@ -28,6 +28,34 @@ func TestStorageService_BindSpaceID(t *testing.T) {
assert.Equal(t, "spaceId2", spaceId)
}
func TestStorageService_GetBoundObjectIds(t *testing.T) {
t.Run("no bindings", func(t *testing.T) {
fx := newFixture(t)
defer fx.finish(t)
spaceId := "spaceId"
ids, err := fx.GetBoundObjectIds(spaceId)
require.NoError(t, err)
assert.Empty(t, ids)
})
t.Run("ok", func(t *testing.T) {
fx := newFixture(t)
defer fx.finish(t)
spaceId := "spaceId"
require.NoError(t, fx.BindSpaceID(spaceId, "objectId1"))
require.NoError(t, fx.BindSpaceID(spaceId, "objectId2"))
ids, err := fx.GetBoundObjectIds(spaceId)
require.NoError(t, err)
assert.ElementsMatch(t, []string{"objectId1", "objectId2"}, ids)
})
}
func TestStorageService_DeleteSpaceStorage(t *testing.T) {
fx := newFixture(t)
defer fx.finish(t)
@ -192,7 +220,7 @@ type testConfig struct {
tmpDir string
}
func (t *testConfig) GetSpaceStorePath() string {
func (t *testConfig) GetSqliteStorePath() string {
return filepath.Join(t.tmpDir, "spaceStore.db")
}
func (t *testConfig) GetTempDirPath() string {

View file

@ -7,10 +7,10 @@ import (
"sync"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
_ "github.com/mattn/go-sqlite3"
)
@ -34,12 +34,12 @@ type spaceStorage struct {
isDeleted bool
aclStorage liststorage.ListStorage
aclStorage oldstorage.ListStorage
header *spacesyncproto.RawSpaceHeaderWithId
}
func newSpaceStorage(s *storageService, spaceId string) (spacestorage.SpaceStorage, error) {
func newSpaceStorage(s *storageService, spaceId string) (oldstorage.SpaceStorage, error) {
ss := &spaceStorage{
spaceId: spaceId,
service: s,
@ -64,7 +64,7 @@ func newSpaceStorage(s *storageService, spaceId string) (spacestorage.SpaceStora
return ss, nil
}
func createSpaceStorage(s *storageService, payload spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error) {
func createSpaceStorage(s *storageService, payload spacestorage.SpaceStorageCreatePayload) (oldstorage.SpaceStorage, error) {
tx, err := s.writeDb.Begin()
if err != nil {
return nil, err
@ -194,7 +194,7 @@ func (s *spaceStorage) TreeDeletedStatus(id string) (status string, err error) {
}
func (s *spaceStorage) AllDeletedTreeIds() (ids []string, err error) {
rows, err := s.service.stmt.allTreeDelStatus.Query(s.spaceId, spacestorage.TreeDeletedStatusDeleted)
rows, err := s.service.stmt.allTreeDelStatus.Query(s.spaceId, oldstorage.TreeDeletedStatusDeleted)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
@ -218,7 +218,7 @@ func (s *spaceStorage) SpaceSettingsId() string {
return s.spaceSettingsId
}
func (s *spaceStorage) AclStorage() (liststorage.ListStorage, error) {
func (s *spaceStorage) AclStorage() (oldstorage.ListStorage, error) {
return s.aclStorage, nil
}
@ -258,7 +258,7 @@ func (s *spaceStorage) TreeRoot(id string) (*treechangeproto.RawTreeChangeWithId
}, nil
}
func (s *spaceStorage) TreeStorage(id string) (treestorage.TreeStorage, error) {
func (s *spaceStorage) TreeStorage(id string) (oldstorage.TreeStorage, error) {
return newTreeStorage(s, id)
}
@ -271,7 +271,7 @@ func (s *spaceStorage) HasTree(id string) (bool, error) {
return res > 0, nil
}
func (s *spaceStorage) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (treestorage.TreeStorage, error) {
func (s *spaceStorage) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (oldstorage.TreeStorage, error) {
return createTreeStorage(s, payload)
}

View file

@ -6,6 +6,7 @@ import (
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
"github.com/anyproto/any-sync/consensus/consensusproto"
"github.com/stretchr/testify/assert"
@ -90,7 +91,7 @@ func TestSpaceStorage_NewAndCreateTree(t *testing.T) {
treeIds, err := store.StoredIds()
require.NoError(t, err)
assert.Equal(t, []string{payload.SpaceSettingsWithId.Id, otherStore.Id()}, treeIds)
assert.Equal(t, []string{payload.SpaceSettingsWithId.Id}, treeIds)
deletedIds, err := store.(*spaceStorage).AllDeletedTreeIds()
require.NoError(t, err)
@ -106,11 +107,11 @@ func TestSpaceStorage_AllDeletedTreeIds(t *testing.T) {
store, err := createSpaceStorage(fx.storageService, payload)
require.NoError(t, err)
err = store.SetTreeDeletedStatus("id1", spacestorage.TreeDeletedStatusDeleted)
err = store.SetTreeDeletedStatus("id1", oldstorage.TreeDeletedStatusDeleted)
require.NoError(t, err)
err = store.SetTreeDeletedStatus("id2", spacestorage.TreeDeletedStatusQueued)
err = store.SetTreeDeletedStatus("id2", oldstorage.TreeDeletedStatusQueued)
require.NoError(t, err)
err = store.SetTreeDeletedStatus("id3", spacestorage.TreeDeletedStatusDeleted)
err = store.SetTreeDeletedStatus("id3", oldstorage.TreeDeletedStatusDeleted)
require.NoError(t, err)
deletedIds, err := store.(*spaceStorage).AllDeletedTreeIds()
@ -127,12 +128,12 @@ func TestSpaceStorage_SetTreeDeletedStatus(t *testing.T) {
store, err := createSpaceStorage(fx.storageService, payload)
require.NoError(t, err)
err = store.SetTreeDeletedStatus("treeId", spacestorage.TreeDeletedStatusDeleted)
err = store.SetTreeDeletedStatus("treeId", oldstorage.TreeDeletedStatusDeleted)
require.NoError(t, err)
status, err := store.TreeDeletedStatus("treeId")
require.NoError(t, err)
require.Equal(t, spacestorage.TreeDeletedStatusDeleted, status)
require.Equal(t, oldstorage.TreeDeletedStatusDeleted, status)
_, err = store.TreeStorage("treeId")
require.ErrorIs(t, err, treestorage.ErrUnknownTreeId)
@ -202,7 +203,7 @@ func TestSpaceStorage_ReadSpaceHash(t *testing.T) {
require.NoError(t, ss.WriteSpaceHash("hash"))
var checkHashes = func(ss spacestorage.SpaceStorage) {
var checkHashes = func(ss oldstorage.SpaceStorage) {
hash, err = ss.ReadSpaceHash()
require.NoError(t, err)
assert.Equal(t, "hash", hash)
@ -238,7 +239,7 @@ func spaceTestPayload() spacestorage.SpaceStorageCreatePayload {
}
}
func testSpace(t *testing.T, store spacestorage.SpaceStorage, payload spacestorage.SpaceStorageCreatePayload) {
func testSpace(t *testing.T, store oldstorage.SpaceStorage, payload spacestorage.SpaceStorageCreatePayload) {
header, err := store.SpaceHeader()
require.NoError(t, err)
require.Equal(t, payload.SpaceHeaderWithId, header)

View file

@ -58,7 +58,7 @@ func initStmts(s *storageService) (err error) {
if s.stmt.updateSpaceIsDeleted, err = s.writeDb.Prepare(`UPDATE spaces SET isDeleted = ? WHERE id = ?`); err != nil {
return
}
if s.stmt.treeIdsBySpace, err = s.readDb.Prepare(`SELECT id FROM trees WHERE spaceId = ? AND type != 1`); err != nil {
if s.stmt.treeIdsBySpace, err = s.readDb.Prepare(`SELECT id FROM trees WHERE spaceId = ? AND type != 1 AND deleteStatus IS NULL`); err != nil {
return
}
if s.stmt.deleteTree, err = s.writeDb.Prepare(`
@ -88,6 +88,12 @@ func initStmts(s *storageService) (err error) {
if s.stmt.hasChange, err = s.readDb.Prepare(`SELECT COUNT(*) FROM changes WHERE id = ? AND treeId = ?`); err != nil {
return
}
if s.stmt.listChanges, err = s.readDb.Prepare(`SELECT id FROM changes WHERE treeId = ? ORDER BY id`); err != nil {
return
}
if s.stmt.iterateChanges, err = s.readDb.Prepare(`SELECT id, data FROM changes WHERE treeId = ? ORDER BY id`); err != nil {
return
}
if s.stmt.updateTreeHeads, err = s.writeDb.Prepare(`UPDATE trees SET heads = ? WHERE id = ?`); err != nil {
return
}
@ -112,6 +118,9 @@ func initStmts(s *storageService) (err error) {
if s.stmt.upsertBind, err = s.writeDb.Prepare(`INSERT INTO binds (objectId, spaceId) VALUES (?, ?) ON CONFLICT (objectId) DO UPDATE SET spaceId = ?`); err != nil {
return
}
if s.stmt.getAllBinds, err = s.writeDb.Prepare(`SELECT objectId FROM binds WHERE spaceId = ?`); err != nil {
return
}
if s.stmt.deleteSpace, err = s.writeDb.Prepare(`DELETE FROM spaces WHERE id = ?`); err != nil {
return
}

View file

@ -1,15 +1,18 @@
package sqlitestorage
import (
"bytes"
"context"
"errors"
"fmt"
"sync"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
)
func newTreeStorage(ss *spaceStorage, treeId string) (treestorage.TreeStorage, error) {
func newTreeStorage(ss *spaceStorage, treeId string) (oldstorage.TreeStorage, error) {
ts := &treeStorage{
treeId: treeId,
spaceStorage: ss,
@ -23,7 +26,7 @@ func newTreeStorage(ss *spaceStorage, treeId string) (treestorage.TreeStorage, e
return ts, nil
}
func createTreeStorage(ss *spaceStorage, payload treestorage.TreeStorageCreatePayload) (ts treestorage.TreeStorage, err error) {
func createTreeStorage(ss *spaceStorage, payload treestorage.TreeStorageCreatePayload) (ts oldstorage.TreeStorage, err error) {
ts = &treeStorage{
treeId: payload.RootRawChange.Id,
spaceStorage: ss,
@ -99,7 +102,54 @@ func (t *treeStorage) Heads() ([]string, error) {
}
func (t *treeStorage) GetAllChangeIds() (chs []string, err error) {
return nil, fmt.Errorf("get all change ids should not be called")
rows, err := t.service.stmt.listChanges.Query(t.treeId)
if err != nil {
return nil, replaceNoRowsErr(err, nil)
}
for rows.Next() {
var id string
err = rows.Scan(&id)
if err != nil {
return nil, errors.Join(rows.Close(), err)
}
chs = append(chs, id)
}
return chs, rows.Close()
}
func (t *treeStorage) GetAllChanges() ([]*treechangeproto.RawTreeChangeWithId, error) {
var changes []*treechangeproto.RawTreeChangeWithId
err := t.IterateChanges(func(id string, rawChange []byte) error {
changes = append(changes, &treechangeproto.RawTreeChangeWithId{
Id: id,
RawChange: bytes.Clone(rawChange),
})
return nil
})
return changes, err
}
func (t *treeStorage) IterateChanges(proc func(id string, rawChange []byte) error) error {
rows, err := t.service.stmt.iterateChanges.Query(t.treeId)
if err != nil {
return replaceNoRowsErr(err, nil)
}
buf := make([]byte, 0, 1024)
for rows.Next() {
var id string
err = rows.Scan(&id, &buf)
if err != nil {
return errors.Join(rows.Close(), err)
}
err = proc(id, buf)
if err != nil {
return errors.Join(rows.Close(), err)
}
buf = buf[:0]
}
return rows.Close()
}
func (t *treeStorage) SetHeads(heads []string) error {

View file

@ -1,17 +1,26 @@
package sqlitestorage
import (
"bytes"
"crypto/rand"
"database/sql"
"encoding/hex"
"slices"
"sort"
"testing"
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type oldTreeStorage interface {
oldstorage.ChangesIterator
oldstorage.TreeStorage
}
func TestTreeStorage_Create(t *testing.T) {
fx := newFixture(t)
defer fx.finish(t)
@ -38,11 +47,13 @@ func TestTreeStorage_Methods(t *testing.T) {
ss, err := createSpaceStorage(fx.storageService, spacePayload)
require.NoError(t, err)
payload := treeTestPayload()
store, err := ss.CreateTreeStorage(payload)
var store oldTreeStorage
treeStore, err := ss.CreateTreeStorage(payload)
require.NoError(t, err)
store, err = ss.TreeStorage(payload.RootRawChange.Id)
treeStore, err = ss.TreeStorage(payload.RootRawChange.Id)
require.NoError(t, err)
store = treeStore.(oldTreeStorage)
testTreePayload(t, store, payload)
t.Run("update heads", func(t *testing.T) {
@ -54,7 +65,7 @@ func TestTreeStorage_Methods(t *testing.T) {
})
t.Run("add raw change, get change and has change", func(t *testing.T) {
newChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("ab"), Id: "newId"}
newChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("ab"), Id: "id10"}
require.NoError(t, store.AddRawChange(newChange))
rawCh, err := store.GetRawChange(ctx, newChange.Id)
require.NoError(t, err)
@ -72,6 +83,55 @@ func TestTreeStorage_Methods(t *testing.T) {
require.NoError(t, err)
require.False(t, has)
})
t.Run("iterate changes", func(t *testing.T) {
newChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("foo"), Id: "id01"}
require.NoError(t, store.AddRawChange(newChange))
newChange = &treechangeproto.RawTreeChangeWithId{RawChange: []byte("bar"), Id: "id20"}
require.NoError(t, store.AddRawChange(newChange))
var collected []*treechangeproto.RawTreeChangeWithId
require.NoError(t, store.IterateChanges(func(id string, rawChange []byte) error {
collected = append(collected, &treechangeproto.RawTreeChangeWithId{
Id: id,
RawChange: bytes.Clone(rawChange),
})
return nil
}))
want := slices.Clone(payload.Changes)
want = append(want, []*treechangeproto.RawTreeChangeWithId{
{Id: "id01", RawChange: []byte("foo")},
{Id: "id10", RawChange: []byte("ab")},
{Id: "id20", RawChange: []byte("bar")},
}...)
sort.Slice(want, func(i, j int) bool {
return want[i].Id < want[j].Id
})
assert.Equal(t, want, collected)
got, err := store.GetAllChanges()
require.NoError(t, err)
assert.Equal(t, want, got)
})
t.Run("get all change ids", func(t *testing.T) {
got, err := store.GetAllChangeIds()
require.NoError(t, err)
want := []string{
payload.Changes[0].Id,
payload.Changes[1].Id,
"id01",
"id10",
"id20",
}
sort.Strings(want)
assert.Equal(t, want, got)
})
}
func TestTreeStorage_AddRawChangesSetHeads(t *testing.T) {
@ -190,7 +250,7 @@ func treeTestPayload() treestorage.TreeStorageCreatePayload {
}
}
func testTreePayload(t *testing.T, store treestorage.TreeStorage, payload treestorage.TreeStorageCreatePayload) {
func testTreePayload(t *testing.T, store oldstorage.TreeStorage, payload treestorage.TreeStorageCreatePayload) {
require.Equal(t, payload.RootRawChange.Id, store.Id())
root, err := store.Root()

View file

@ -2,13 +2,12 @@ package storage
import (
"context"
"fmt"
anystore "github.com/anyproto/any-store"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/spacestorage"
"github.com/anyproto/anytype-heart/space/spacecore/storage/badgerstorage"
"github.com/anyproto/anytype-heart/space/spacecore/storage/sqlitestorage"
"github.com/anyproto/anytype-heart/space/spacecore/storage/anystorage"
)
type SpaceStorageMode int
@ -22,12 +21,7 @@ type ClientStorage interface {
spacestorage.SpaceStorageProvider
app.ComponentRunnable
AllSpaceIds() (ids []string, err error)
GetSpaceID(objectID string) (spaceID string, err error)
BindSpaceID(spaceID, objectID string) (err error)
DeleteSpaceStorage(ctx context.Context, spaceId string) error
MarkSpaceCreated(id string) (err error)
UnmarkSpaceCreated(id string) (err error)
IsSpaceCreated(id string) (created bool)
}
// storageService is a proxy for the actual storage implementation
@ -40,7 +34,8 @@ func New() ClientStorage {
}
type configGetter interface {
GetSpaceStorageMode() SpaceStorageMode
GetNewSpaceStorePath() string
GetAnyStoreConfig() *anystore.Config
}
func (s *storageService) Name() (name string) {
@ -48,16 +43,7 @@ func (s *storageService) Name() (name string) {
}
func (s *storageService) Init(a *app.App) (err error) {
mode := a.MustComponent("config").(configGetter).GetSpaceStorageMode()
if mode == SpaceStorageModeBadger {
// for already existing account repos
s.ClientStorage = badgerstorage.New()
} else if mode == SpaceStorageModeSqlite {
// sqlite used for new account repos
s.ClientStorage = sqlitestorage.New()
} else {
return fmt.Errorf("unknown storage mode %d", mode)
}
getter := a.MustComponent("config").(configGetter)
s.ClientStorage = anystorage.New(getter.GetNewSpaceStorePath(), getter.GetAnyStoreConfig())
return s.ClientStorage.Init(a)
}

View file

@ -25,6 +25,7 @@ import (
"github.com/anyproto/anytype-heart/pkg/lib/logging"
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
"github.com/anyproto/anytype-heart/space/spacecore"
"github.com/anyproto/anytype-heart/space/spacecore/storage/anystorage"
)
const CName = "space.typeprovider"
@ -196,17 +197,15 @@ func (p *provider) objectTypeFromSpace(spaceID string, id string) (tp smartblock
return
}
p.RUnlock()
sp, err := p.spaceService.Get(context.Background(), spaceID)
ctx := context.Background()
sp, err := p.spaceService.Get(ctx, spaceID)
if err != nil {
return
}
store := sp.Storage()
rawRoot, err := store.TreeRoot(id)
rawRoot, err := sp.Storage().(anystorage.ClientSpaceStorage).TreeRoot(ctx, id)
if err != nil {
return
}
tp, _, err = GetTypeAndKeyFromRoot(rawRoot)
if err != nil {
return

View file

@ -17,6 +17,7 @@ import (
"github.com/anyproto/anytype-heart/space/internal/spacecontroller"
"github.com/anyproto/anytype-heart/space/spacecore"
"github.com/anyproto/anytype-heart/space/spacecore/storage"
"github.com/anyproto/anytype-heart/space/spacecore/storage/anystorage"
"github.com/anyproto/anytype-heart/space/spaceinfo"
"github.com/anyproto/anytype-heart/space/techspace"
)
@ -71,7 +72,7 @@ func (s *spaceFactory) CreatePersonalSpace(ctx context.Context, metadata []byte)
if err != nil {
return
}
err = s.storageService.MarkSpaceCreated(coreSpace.Id())
err = coreSpace.Storage().(anystorage.ClientSpaceStorage).MarkSpaceCreated(ctx)
if err != nil {
return
}
@ -192,7 +193,11 @@ func (s *spaceFactory) CreateInvitingSpace(ctx context.Context, id, aclHeadId st
}
func (s *spaceFactory) CreateShareableSpace(ctx context.Context, id string) (sp spacecontroller.SpaceController, err error) {
err = s.storageService.MarkSpaceCreated(id)
coreSpace, err := s.spaceCore.Get(ctx, id)
if err != nil {
return
}
err = coreSpace.Storage().(anystorage.ClientSpaceStorage).MarkSpaceCreated(ctx)
if err != nil {
return
}

Some files were not shown because too many files have changed in this diff Show more