mirror of
https://github.com/anyproto/anytype-heart.git
synced 2025-06-08 05:47:07 +09:00
Merge pull request #2185 from anyproto/GO-4146-new-spacestore
GO-4146: New spacestore merging once again
This commit is contained in:
commit
b243f1f4e2
104 changed files with 3886 additions and 1546 deletions
|
@ -97,6 +97,9 @@ packages:
|
|||
github.com/anyproto/anytype-heart/space/spacecore/storage:
|
||||
interfaces:
|
||||
ClientStorage:
|
||||
github.com/anyproto/anytype-heart/space/spacecore/storage/anystorage:
|
||||
interfaces:
|
||||
ClientSpaceStorage:
|
||||
github.com/anyproto/anytype-heart/space/techspace:
|
||||
interfaces:
|
||||
TechSpace:
|
||||
|
|
|
@ -9,16 +9,19 @@ import (
|
|||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/goccy/go-graphviz"
|
||||
"github.com/gogo/protobuf/jsonpb"
|
||||
"github.com/gogo/protobuf/proto"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/block/editor/state"
|
||||
"github.com/anyproto/anytype-heart/core/debug/treearchive"
|
||||
"github.com/anyproto/anytype-heart/core/debug/exporter"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/core/smartblock"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
|
||||
"github.com/anyproto/anytype-heart/util/pbtypes"
|
||||
|
@ -42,22 +45,20 @@ func main() {
|
|||
return
|
||||
}
|
||||
fmt.Println("opening file...")
|
||||
st := time.Now()
|
||||
archive, err := treearchive.Open(*file)
|
||||
var (
|
||||
st = time.Now()
|
||||
ctx = context.Background()
|
||||
)
|
||||
res, err := exporter.ImportStorage(ctx, *file)
|
||||
if err != nil {
|
||||
log.Fatal("can't open debug file:", err)
|
||||
log.Fatal("can't import the tree:", err)
|
||||
}
|
||||
objectTree, err := res.CreateReadableTree(*fromRoot, "")
|
||||
if err != nil {
|
||||
log.Fatal("can't create readable tree:", err)
|
||||
}
|
||||
defer archive.Close()
|
||||
fmt.Printf("open archive done in %.1fs\n", time.Since(st).Seconds())
|
||||
|
||||
importer := treearchive.NewTreeImporter(archive.ListStorage(), archive.TreeStorage())
|
||||
st = time.Now()
|
||||
err = importer.Import(*fromRoot, "")
|
||||
if err != nil {
|
||||
log.Fatal("can't import the tree", err)
|
||||
}
|
||||
fmt.Printf("import tree done in %.1fs\n", time.Since(st).Seconds())
|
||||
|
||||
importer := exporter.NewTreeImporter(objectTree)
|
||||
if *makeJson {
|
||||
treeJson, err := importer.Json()
|
||||
if err != nil {
|
||||
|
@ -83,10 +84,11 @@ func main() {
|
|||
}
|
||||
fmt.Println("Change:")
|
||||
fmt.Println(pbtypes.Sprint(ch.Model))
|
||||
err = importer.Import(*fromRoot, ch.Id)
|
||||
objectTree, err = res.CreateReadableTree(*fromRoot, ch.Id)
|
||||
if err != nil {
|
||||
log.Fatal("can't import the tree before", ch.Id, err)
|
||||
log.Fatal("can't create readable tree:", err)
|
||||
}
|
||||
importer = exporter.NewTreeImporter(objectTree)
|
||||
}
|
||||
ot := importer.ObjectTree()
|
||||
di, err := ot.Debug(state.ChangeParser{})
|
||||
|
@ -126,12 +128,16 @@ func main() {
|
|||
|
||||
if *objectStore {
|
||||
fmt.Println("fetch object store info..")
|
||||
ls, err := archive.LocalStore()
|
||||
f, err := os.Open(filepath.Join(res.FolderPath, "localstore.json"))
|
||||
if err != nil {
|
||||
fmt.Println("can't open objectStore info:", err)
|
||||
} else {
|
||||
fmt.Println(pbtypes.Sprint(ls))
|
||||
log.Fatal("can't open objectStore info:", err)
|
||||
}
|
||||
info := &model.ObjectInfo{}
|
||||
if err = jsonpb.Unmarshal(f, info); err != nil {
|
||||
log.Fatal("can't unmarshal objectStore info:", err)
|
||||
}
|
||||
defer f.Close()
|
||||
fmt.Println(pbtypes.Sprint(info))
|
||||
}
|
||||
|
||||
if *makeTree {
|
||||
|
|
|
@ -306,7 +306,7 @@ func getTableSizes(mw *core.Middleware) (tables map[string]uint64) {
|
|||
tables = make(map[string]uint64)
|
||||
cfg := mw.GetApp().MustComponent(config.CName).(*config.Config)
|
||||
|
||||
db, err := sql.Open("sqlite3", cfg.GetSpaceStorePath())
|
||||
db, err := sql.Open("sqlite3", cfg.GetSqliteStorePath())
|
||||
if err != nil {
|
||||
fmt.Println("Error opening database:", err)
|
||||
return
|
||||
|
|
|
@ -10,6 +10,7 @@ import (
|
|||
"github.com/anyproto/anytype-heart/core/application"
|
||||
"github.com/anyproto/anytype-heart/core/session"
|
||||
"github.com/anyproto/anytype-heart/pb"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage/migrator"
|
||||
)
|
||||
|
||||
func (mw *Middleware) AccountCreate(cctx context.Context, req *pb.RpcAccountCreateRequest) *pb.RpcAccountCreateResponse {
|
||||
|
@ -48,24 +49,32 @@ func (mw *Middleware) AccountRecover(cctx context.Context, _ *pb.RpcAccountRecov
|
|||
},
|
||||
}
|
||||
}
|
||||
func (mw *Middleware) AccountMigrate(_ context.Context, _ *pb.RpcAccountMigrateRequest) *pb.RpcAccountMigrateResponse {
|
||||
// this is for clients API compatibility, because there is no migration in this branch
|
||||
// todo: remove this
|
||||
|
||||
func (mw *Middleware) AccountMigrate(cctx context.Context, req *pb.RpcAccountMigrateRequest) *pb.RpcAccountMigrateResponse {
|
||||
err := mw.applicationService.AccountMigrate(cctx, req)
|
||||
code := mapErrorCode(err,
|
||||
errToCode(application.ErrBadInput, pb.RpcAccountMigrateResponseError_BAD_INPUT),
|
||||
errToCode(application.ErrAccountNotFound, pb.RpcAccountMigrateResponseError_ACCOUNT_NOT_FOUND),
|
||||
errToCode(context.Canceled, pb.RpcAccountMigrateResponseError_CANCELED),
|
||||
errTypeToCode(&migrator.NotEnoughFreeSpaceError{}, pb.RpcAccountMigrateResponseError_NOT_ENOUGH_FREE_SPACE),
|
||||
)
|
||||
return &pb.RpcAccountMigrateResponse{
|
||||
Error: &pb.RpcAccountMigrateResponseError{
|
||||
Code: 0,
|
||||
Description: "",
|
||||
Code: code,
|
||||
Description: getErrorDescription(err),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (mw *Middleware) AccountMigrateCancel(_ context.Context, _ *pb.RpcAccountMigrateCancelRequest) *pb.RpcAccountMigrateCancelResponse {
|
||||
// this is for clients API compatibility, because there is no migration in this branch
|
||||
// todo: remove this
|
||||
func (mw *Middleware) AccountMigrateCancel(cctx context.Context, req *pb.RpcAccountMigrateCancelRequest) *pb.RpcAccountMigrateCancelResponse {
|
||||
err := mw.applicationService.AccountMigrateCancel(cctx, req)
|
||||
code := mapErrorCode(err,
|
||||
errToCode(application.ErrBadInput, pb.RpcAccountMigrateCancelResponseError_BAD_INPUT),
|
||||
)
|
||||
return &pb.RpcAccountMigrateCancelResponse{
|
||||
Error: &pb.RpcAccountMigrateCancelResponseError{
|
||||
Code: 0,
|
||||
Description: "",
|
||||
Code: code,
|
||||
Description: getErrorDescription(err),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
@ -85,6 +94,7 @@ func (mw *Middleware) AccountSelect(cctx context.Context, req *pb.RpcAccountSele
|
|||
errToCode(application.ErrAnotherProcessIsRunning, pb.RpcAccountSelectResponseError_ANOTHER_ANYTYPE_PROCESS_IS_RUNNING),
|
||||
errToCode(application.ErrIncompatibleVersion, pb.RpcAccountSelectResponseError_FAILED_TO_FETCH_REMOTE_NODE_HAS_INCOMPATIBLE_PROTO_VERSION),
|
||||
errToCode(application.ErrFailedToStartApplication, pb.RpcAccountSelectResponseError_FAILED_TO_RUN_NODE),
|
||||
errToCode(application.ErrAccountStoreIsNotMigrated, pb.RpcAccountSelectResponseError_ACCOUNT_STORE_NOT_MIGRATED),
|
||||
)
|
||||
return &pb.RpcAccountSelectResponse{
|
||||
Config: nil,
|
||||
|
|
|
@ -10,7 +10,6 @@ import (
|
|||
"github.com/anyproto/any-sync/app"
|
||||
"github.com/anyproto/any-sync/commonspace/acl/aclclient"
|
||||
"github.com/anyproto/any-sync/commonspace/object/acl/list"
|
||||
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
|
||||
"github.com/anyproto/any-sync/coordinator/coordinatorclient"
|
||||
"github.com/anyproto/any-sync/coordinator/coordinatorproto"
|
||||
"github.com/anyproto/any-sync/nodeconf"
|
||||
|
@ -392,7 +391,7 @@ func (a *aclService) ViewInvite(ctx context.Context, inviteCid cid.Cid, inviteFi
|
|||
if len(recs) == 0 {
|
||||
return domain.InviteView{}, fmt.Errorf("no acl records found for space: %s, %w", res.SpaceId, ErrAclRequestFailed)
|
||||
}
|
||||
store, err := liststorage.NewInMemoryAclListStorage(recs[0].Id, recs)
|
||||
store, err := list.NewInMemoryStorage(recs[0].Id, recs)
|
||||
if err != nil {
|
||||
return domain.InviteView{}, convertedOrAclRequestError(err)
|
||||
}
|
||||
|
|
|
@ -143,9 +143,6 @@ func (m mockSyncAcl) HandleRequest(ctx context.Context, senderId string, request
|
|||
return nil, nil
|
||||
}
|
||||
|
||||
func (m mockSyncAcl) SetHeadUpdater(updater headupdater.HeadUpdater) {
|
||||
}
|
||||
|
||||
func (m mockSyncAcl) SetAclUpdater(updater headupdater.AclUpdater) {
|
||||
}
|
||||
|
||||
|
@ -326,7 +323,7 @@ func TestService_ViewInvite(t *testing.T) {
|
|||
keys, err := accountdata.NewRandom()
|
||||
require.NoError(t, err)
|
||||
fx.mockAccountService.EXPECT().Keys().Return(keys)
|
||||
aclList, err := list.NewTestDerivedAcl("spaceId", keys)
|
||||
aclList, err := list.NewInMemoryDerivedAcl("spaceId", keys)
|
||||
require.NoError(t, err)
|
||||
inv, err := aclList.RecordBuilder().BuildInvite()
|
||||
require.NoError(t, err)
|
||||
|
@ -357,7 +354,7 @@ func TestService_ViewInvite(t *testing.T) {
|
|||
keys, err := accountdata.NewRandom()
|
||||
require.NoError(t, err)
|
||||
fx.mockAccountService.EXPECT().Keys().Return(keys)
|
||||
aclList, err := list.NewTestDerivedAcl("spaceId", keys)
|
||||
aclList, err := list.NewInMemoryDerivedAcl("spaceId", keys)
|
||||
require.NoError(t, err)
|
||||
inv, err := aclList.RecordBuilder().BuildInvite()
|
||||
require.NoError(t, err)
|
||||
|
|
|
@ -108,9 +108,12 @@ import (
|
|||
"github.com/anyproto/anytype-heart/space/spacecore/clientserver"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/credentialprovider"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/localdiscovery"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/oldstorage"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/peermanager"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/peerstore"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage/migrator"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage/migratorfinisher"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/typeprovider"
|
||||
"github.com/anyproto/anytype-heart/space/spacefactory"
|
||||
"github.com/anyproto/anytype-heart/space/virtualspaceservice"
|
||||
|
@ -191,6 +194,18 @@ func appVersion(a *app.App, clientWithVersion string) string {
|
|||
return clientWithVersion + "/middle:" + middleVersion + "/any-sync:" + anySyncVersion
|
||||
}
|
||||
|
||||
func BootstrapMigration(a *app.App, components ...app.Component) {
|
||||
for _, c := range components {
|
||||
a.Register(c)
|
||||
}
|
||||
a.Register(migratorfinisher.New()).
|
||||
Register(clientds.New()).
|
||||
Register(oldstorage.New()).
|
||||
Register(storage.New()).
|
||||
Register(process.New()).
|
||||
Register(migrator.New())
|
||||
}
|
||||
|
||||
func Bootstrap(a *app.App, components ...app.Component) {
|
||||
for _, c := range components {
|
||||
a.Register(c)
|
||||
|
|
|
@ -43,6 +43,7 @@ const (
|
|||
const (
|
||||
SpaceStoreBadgerPath = "spacestore"
|
||||
SpaceStoreSqlitePath = "spaceStore.db"
|
||||
SpaceStoreNewPath = "spaceStoreNew"
|
||||
)
|
||||
|
||||
var (
|
||||
|
@ -69,6 +70,7 @@ type Config struct {
|
|||
DisableThreadsSyncEvents bool
|
||||
DontStartLocalNetworkSyncAutomatically bool
|
||||
PeferYamuxTransport bool
|
||||
DisableNetworkIdCheck bool
|
||||
SpaceStorageMode storage.SpaceStorageMode
|
||||
NetworkMode pb.RpcAccountNetworkMode
|
||||
NetworkCustomConfigFilePath string `json:",omitempty"` // not saved to config
|
||||
|
@ -192,7 +194,7 @@ func (c *Config) initFromFileAndEnv(repoPath string) error {
|
|||
if len(split) == 1 {
|
||||
return fmt.Errorf("failed to split repo path: %s", repoPath)
|
||||
}
|
||||
c.SqliteTempPath = filepath.Join(split[0], "cache")
|
||||
c.SqliteTempPath = filepath.Join(split[0], "files")
|
||||
c.AnyStoreConfig.SQLiteConnectionOptions = make(map[string]string)
|
||||
c.AnyStoreConfig.SQLiteConnectionOptions["temp_store_directory"] = "'" + c.SqliteTempPath + "'"
|
||||
}
|
||||
|
@ -293,12 +295,27 @@ func (c *Config) FSConfig() (FSConfig, error) {
|
|||
return FSConfig{IPFSStorageAddr: res.CustomFileStorePath}, nil
|
||||
}
|
||||
|
||||
func (c *Config) GetRepoPath() string {
|
||||
return c.RepoPath
|
||||
}
|
||||
|
||||
func (c *Config) GetConfigPath() string {
|
||||
return filepath.Join(c.RepoPath, ConfigFileName)
|
||||
}
|
||||
|
||||
func (c *Config) GetSpaceStorePath() string {
|
||||
return filepath.Join(c.RepoPath, "spaceStore.db")
|
||||
func (c *Config) GetSqliteStorePath() string {
|
||||
return filepath.Join(c.RepoPath, SpaceStoreSqlitePath)
|
||||
}
|
||||
|
||||
func (c *Config) GetOldSpaceStorePath() string {
|
||||
if c.GetSpaceStorageMode() == storage.SpaceStorageModeBadger {
|
||||
return filepath.Join(c.RepoPath, SpaceStoreBadgerPath)
|
||||
}
|
||||
return c.GetSqliteStorePath()
|
||||
}
|
||||
|
||||
func (c *Config) GetNewSpaceStorePath() string {
|
||||
return filepath.Join(c.RepoPath, SpaceStoreNewPath)
|
||||
}
|
||||
|
||||
func (c *Config) GetTempDirPath() string {
|
||||
|
@ -391,7 +408,7 @@ func (c *Config) GetNodeConfWithError() (conf nodeconf.Configuration, err error)
|
|||
if err := yaml.Unmarshal(confBytes, &conf); err != nil {
|
||||
return nodeconf.Configuration{}, errors.Join(ErrNetworkFileFailedToRead, err)
|
||||
}
|
||||
if c.NetworkId != "" && c.NetworkId != conf.NetworkId {
|
||||
if !c.DisableNetworkIdCheck && c.NetworkId != "" && c.NetworkId != conf.NetworkId {
|
||||
log.Warnf("Network id mismatch: %s != %s", c.NetworkId, conf.NetworkId)
|
||||
return nodeconf.Configuration{}, errors.Join(ErrNetworkIdMismatch, fmt.Errorf("network id mismatch: %s != %s", c.NetworkId, conf.NetworkId))
|
||||
}
|
||||
|
|
|
@ -31,15 +31,23 @@ var (
|
|||
ErrNoMnemonicProvided = errors.New("no mnemonic provided")
|
||||
ErrIncompatibleVersion = errors.New("can't fetch account's data because remote nodes have incompatible protocol version. Please update anytype to the latest version")
|
||||
|
||||
ErrAnotherProcessIsRunning = errors.New("another anytype process is running")
|
||||
ErrFailedToFindAccountInfo = errors.New("failed to find account info")
|
||||
ErrAccountIsDeleted = errors.New("account is deleted")
|
||||
ErrAnotherProcessIsRunning = errors.New("another anytype process is running")
|
||||
ErrFailedToFindAccountInfo = errors.New("failed to find account info")
|
||||
ErrAccountIsDeleted = errors.New("account is deleted")
|
||||
ErrAccountStoreIsNotMigrated = errors.New("account store is not migrated")
|
||||
)
|
||||
|
||||
func (s *Service) AccountSelect(ctx context.Context, req *pb.RpcAccountSelectRequest) (*model.Account, error) {
|
||||
if req.Id == "" {
|
||||
return nil, ErrEmptyAccountID
|
||||
}
|
||||
curMigration := s.migrationManager.getOrCreateMigration(req.RootPath, req.Id)
|
||||
if !curMigration.successful() {
|
||||
return nil, ErrAccountStoreIsNotMigrated
|
||||
}
|
||||
if s.migrationManager.isRunning() {
|
||||
return nil, ErrMigrationRunning
|
||||
}
|
||||
|
||||
if runtime.GOOS != "android" && runtime.GOOS != "ios" {
|
||||
s.traceRecorder.start()
|
||||
|
|
222
core/application/account_store_migrate.go
Normal file
222
core/application/account_store_migrate.go
Normal file
|
@ -0,0 +1,222 @@
|
|||
package application
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/anyproto/any-sync/app"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/anytype"
|
||||
"github.com/anyproto/anytype-heart/core/anytype/config"
|
||||
"github.com/anyproto/anytype-heart/pb"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/core"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrAccountNotFound = errors.New("account not found")
|
||||
ErrMigrationRunning = errors.New("migration is running")
|
||||
)
|
||||
|
||||
func (s *Service) AccountMigrate(ctx context.Context, req *pb.RpcAccountMigrateRequest) error {
|
||||
if s.rootPath == "" {
|
||||
s.rootPath = req.RootPath
|
||||
}
|
||||
return s.migrationManager.getOrCreateMigration(req.RootPath, req.Id).wait()
|
||||
}
|
||||
|
||||
func (s *Service) AccountMigrateCancel(ctx context.Context, req *pb.RpcAccountMigrateCancelRequest) error {
|
||||
m := s.migrationManager.getMigration(req.Id)
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
m.cancelMigration()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) migrate(ctx context.Context, id string) error {
|
||||
res, err := core.WalletAccountAt(s.mnemonic, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(s.rootPath, id)); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return ErrAccountNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
cfg := anytype.BootstrapConfig(false, os.Getenv("ANYTYPE_STAGING") == "1")
|
||||
cfg.PeferYamuxTransport = true
|
||||
cfg.DisableNetworkIdCheck = true
|
||||
comps := []app.Component{
|
||||
cfg,
|
||||
anytype.BootstrapWallet(s.rootPath, res),
|
||||
s.eventSender,
|
||||
}
|
||||
a := &app.App{}
|
||||
anytype.BootstrapMigration(a, comps...)
|
||||
err = a.Start(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return a.Close(ctx)
|
||||
}
|
||||
|
||||
type migration struct {
|
||||
mx sync.Mutex
|
||||
isStarted bool
|
||||
isFinished bool
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
manager *migrationManager
|
||||
err error
|
||||
id string
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func newMigration(m *migrationManager, id string) *migration {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &migration{
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
done: make(chan struct{}),
|
||||
id: id,
|
||||
manager: m,
|
||||
}
|
||||
}
|
||||
|
||||
func newSuccessfulMigration(manager *migrationManager, id string) *migration {
|
||||
m := newMigration(manager, id)
|
||||
m.setFinished(nil, false)
|
||||
return m
|
||||
}
|
||||
|
||||
func (m *migration) setFinished(err error, notify bool) {
|
||||
m.mx.Lock()
|
||||
defer m.mx.Unlock()
|
||||
m.isFinished = true
|
||||
m.err = err
|
||||
close(m.done)
|
||||
if notify {
|
||||
m.manager.setMigrationRunning(m.id, false)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *migration) cancelMigration() {
|
||||
m.cancel()
|
||||
err := m.wait()
|
||||
if err != nil {
|
||||
log.Warn("failed to wait for migration to finish", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (m *migration) wait() error {
|
||||
m.mx.Lock()
|
||||
if !m.manager.setMigrationRunning(m.id, true) {
|
||||
m.mx.Unlock()
|
||||
return ErrMigrationRunning
|
||||
}
|
||||
if !m.isStarted {
|
||||
m.isStarted = true
|
||||
} else {
|
||||
m.mx.Unlock()
|
||||
<-m.done
|
||||
return m.err
|
||||
}
|
||||
m.mx.Unlock()
|
||||
err := m.manager.service.migrate(m.ctx, m.id)
|
||||
if err != nil {
|
||||
m.setFinished(err, true)
|
||||
return err
|
||||
}
|
||||
m.setFinished(nil, true)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *migration) successful() bool {
|
||||
m.mx.Lock()
|
||||
defer m.mx.Unlock()
|
||||
return m.isFinished && m.err == nil
|
||||
}
|
||||
|
||||
func (m *migration) finished() bool {
|
||||
m.mx.Lock()
|
||||
defer m.mx.Unlock()
|
||||
return m.isFinished
|
||||
}
|
||||
|
||||
type migrationManager struct {
|
||||
migrations map[string]*migration
|
||||
service *Service
|
||||
runningMigration string
|
||||
sync.Mutex
|
||||
}
|
||||
|
||||
func newMigrationManager(s *Service) *migrationManager {
|
||||
return &migrationManager{
|
||||
service: s,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *migrationManager) setMigrationRunning(id string, isRunning bool) bool {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
if (m.runningMigration != "" && m.runningMigration != id) && isRunning {
|
||||
return false
|
||||
}
|
||||
if m.runningMigration == "" && !isRunning {
|
||||
panic("migration is not running")
|
||||
}
|
||||
if isRunning {
|
||||
m.runningMigration = id
|
||||
} else {
|
||||
m.runningMigration = ""
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (m *migrationManager) isRunning() bool {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
return m.runningMigration != ""
|
||||
}
|
||||
|
||||
func (m *migrationManager) getOrCreateMigration(rootPath, id string) *migration {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
if m.migrations == nil {
|
||||
m.migrations = make(map[string]*migration)
|
||||
}
|
||||
if m.migrations[id] == nil {
|
||||
sqlitePath := filepath.Join(rootPath, id, config.SpaceStoreSqlitePath)
|
||||
baderPath := filepath.Join(rootPath, id, config.SpaceStoreBadgerPath)
|
||||
if anyPathExists([]string{sqlitePath, baderPath}) {
|
||||
m.migrations[id] = newMigration(m, id)
|
||||
} else {
|
||||
m.migrations[id] = newSuccessfulMigration(m, id)
|
||||
}
|
||||
}
|
||||
if m.migrations[id].finished() && !m.migrations[id].successful() {
|
||||
// resetting migration
|
||||
m.migrations[id] = newMigration(m, id)
|
||||
}
|
||||
return m.migrations[id]
|
||||
}
|
||||
|
||||
func (m *migrationManager) getMigration(id string) *migration {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
return m.migrations[id]
|
||||
}
|
||||
|
||||
func anyPathExists(paths []string) bool {
|
||||
for _, path := range paths {
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
|
@ -30,16 +30,20 @@ type Service struct {
|
|||
eventSender event.Sender
|
||||
sessions session.Service
|
||||
traceRecorder *traceRecorder
|
||||
migrationManager *migrationManager
|
||||
|
||||
appAccountStartInProcessCancel context.CancelFunc
|
||||
appAccountStartInProcessCancelMutex sync.Mutex
|
||||
}
|
||||
|
||||
func New() *Service {
|
||||
return &Service{
|
||||
s := &Service{
|
||||
sessions: session.New(),
|
||||
traceRecorder: &traceRecorder{},
|
||||
}
|
||||
m := newMigrationManager(s)
|
||||
s.migrationManager = m
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *Service) GetApp() *app.App {
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package block
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
@ -22,6 +23,7 @@ import (
|
|||
func (s *Service) DebugRouter(r chi.Router) {
|
||||
r.Get("/objects", debug.JSONHandler(s.debugListObjects))
|
||||
r.Get("/tree/{id}", debug.JSONHandler(s.debugTree))
|
||||
r.Get("/tree_in_space/{spaceId}/{id}", debug.JSONHandler(s.debugTreeInSpace))
|
||||
r.Get("/objects_per_space/{spaceId}", debug.JSONHandler(s.debugListObjectsPerSpace))
|
||||
r.Get("/objects/{id}", debug.JSONHandler(s.debugGetObject))
|
||||
}
|
||||
|
@ -122,6 +124,44 @@ func (s *Service) debugTree(req *http.Request) (debugTree, error) {
|
|||
return result, err
|
||||
}
|
||||
|
||||
// TODO Refactor
|
||||
func (s *Service) debugTreeInSpace(req *http.Request) (debugTree, error) {
|
||||
spaceId := chi.URLParam(req, "spaceId")
|
||||
id := chi.URLParam(req, "id")
|
||||
|
||||
result := debugTree{
|
||||
Id: id,
|
||||
}
|
||||
|
||||
spc, err := s.spaceService.Get(context.Background(), spaceId)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("get space: %w", err)
|
||||
}
|
||||
|
||||
err = spc.Do(id, func(sb smartblock.SmartBlock) error {
|
||||
ot := sb.Tree()
|
||||
return ot.IterateRoot(source.UnmarshalChange, func(change *objecttree.Change) bool {
|
||||
change.Next = nil
|
||||
raw, err := json.Marshal(change)
|
||||
if err != nil {
|
||||
log.Error("debug tree: marshal change", zap.Error(err))
|
||||
return false
|
||||
}
|
||||
ts := time.Unix(change.Timestamp, 0)
|
||||
ch := debugChange{
|
||||
Change: raw,
|
||||
Timestamp: ts.Format(time.RFC3339),
|
||||
}
|
||||
if change.Identity != nil {
|
||||
ch.Identity = change.Identity.Account()
|
||||
}
|
||||
result.Changes = append(result.Changes, ch)
|
||||
return true
|
||||
})
|
||||
})
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (s *Service) getDebugObject(id string) (debugObject, error) {
|
||||
var obj debugObject
|
||||
err := cache.Do(s, id, func(sb smartblock.SmartBlock) error {
|
||||
|
|
|
@ -39,6 +39,7 @@ type StoreObject interface {
|
|||
ToggleMessageReaction(ctx context.Context, messageId string, emoji string) error
|
||||
DeleteMessage(ctx context.Context, messageId string) error
|
||||
SubscribeLastMessages(ctx context.Context, limit int) ([]*model.ChatMessage, int, error)
|
||||
MarkSeenHeads(heads []string)
|
||||
Unsubscribe() error
|
||||
}
|
||||
|
||||
|
@ -112,6 +113,10 @@ func (s *storeObject) onUpdate() {
|
|||
s.subscription.flush()
|
||||
}
|
||||
|
||||
func (s *storeObject) MarkSeenHeads(heads []string) {
|
||||
s.storeSource.MarkSeenHeads(heads)
|
||||
}
|
||||
|
||||
func (s *storeObject) GetMessagesByIds(ctx context.Context, messageIds []string) ([]*model.ChatMessage, error) {
|
||||
coll, err := s.store.Collection(ctx, collectionName)
|
||||
if err != nil {
|
||||
|
|
|
@ -1,72 +1,60 @@
|
|||
package debug
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/debug/treearchive"
|
||||
)
|
||||
|
||||
func TestBuildFast(t *testing.T) {
|
||||
// Specify the directory you want to iterate
|
||||
dir := "./testdata"
|
||||
|
||||
// Read the directory
|
||||
files, err := ioutil.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read dir: %s", err)
|
||||
}
|
||||
|
||||
// Iterate over the files
|
||||
for _, file := range files {
|
||||
t.Run(file.Name(), func(t *testing.T) {
|
||||
filePath := filepath.Join(dir, file.Name())
|
||||
|
||||
// open the file
|
||||
f, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to open file: %s", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
testBuildFast(t, filePath)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testBuildFast(b *testing.T, filepath string) {
|
||||
// todo: replace with less heavy tree
|
||||
archive, err := treearchive.Open(filepath)
|
||||
if err != nil {
|
||||
require.NoError(b, err)
|
||||
}
|
||||
defer archive.Close()
|
||||
|
||||
importer := treearchive.NewTreeImporter(archive.ListStorage(), archive.TreeStorage())
|
||||
|
||||
err = importer.Import(false, "")
|
||||
if err != nil {
|
||||
log.Fatal("can't import the tree", err)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
_, err = importer.State()
|
||||
if err != nil {
|
||||
log.Fatal("can't build state:", err)
|
||||
}
|
||||
b.Logf("fast build took %s", time.Since(start))
|
||||
|
||||
importer2 := treearchive.NewTreeImporter(archive.ListStorage(), archive.TreeStorage())
|
||||
|
||||
err = importer2.Import(false, "")
|
||||
if err != nil {
|
||||
log.Fatal("can't import the tree", err)
|
||||
}
|
||||
|
||||
}
|
||||
// TODO: revive at some point
|
||||
// func TestBuildFast(t *testing.T) {
|
||||
// // Specify the directory you want to iterate
|
||||
// dir := "./testdata"
|
||||
//
|
||||
// // Read the directory
|
||||
// files, err := ioutil.ReadDir(dir)
|
||||
// if err != nil {
|
||||
// t.Fatalf("Failed to read dir: %s", err)
|
||||
// }
|
||||
//
|
||||
// // Iterate over the files
|
||||
// for _, file := range files {
|
||||
// t.Run(file.Name(), func(t *testing.T) {
|
||||
// filePath := filepath.Join(dir, file.Name())
|
||||
//
|
||||
// // open the file
|
||||
// f, err := os.Open(filePath)
|
||||
// if err != nil {
|
||||
// t.Fatalf("Failed to open file: %s", err)
|
||||
// }
|
||||
// defer f.Close()
|
||||
//
|
||||
// testBuildFast(t, filePath)
|
||||
// })
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// func testBuildFast(b *testing.T, filepath string) {
|
||||
// // todo: replace with less heavy tree
|
||||
// archive, err := treearchive.Open(filepath)
|
||||
// if err != nil {
|
||||
// require.NoError(b, err)
|
||||
// }
|
||||
// defer archive.Close()
|
||||
//
|
||||
// importer := exporter.NewTreeImporter(archive.ListStorage(), archive.TreeStorage())
|
||||
//
|
||||
// err = importer.Import(false, "")
|
||||
// if err != nil {
|
||||
// log.Fatal("can't import the tree", err)
|
||||
// }
|
||||
//
|
||||
// start := time.Now()
|
||||
// _, err = importer.State()
|
||||
// if err != nil {
|
||||
// log.Fatal("can't build state:", err)
|
||||
// }
|
||||
// b.Logf("fast build took %s", time.Since(start))
|
||||
//
|
||||
// importer2 := exporter.NewTreeImporter(archive.ListStorage(), archive.TreeStorage())
|
||||
//
|
||||
// err = importer2.Import(false, "")
|
||||
// if err != nil {
|
||||
// log.Fatal("can't import the tree", err)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
|
|
|
@ -47,10 +47,6 @@ func (stx *StoreStateTx) NextOrder(prev string) string {
|
|||
return lexId.Next(prev)
|
||||
}
|
||||
|
||||
func (stx *StoreStateTx) NextBeforeOrder(prev string, before string) (string, error) {
|
||||
return lexId.NextBefore(prev, before)
|
||||
}
|
||||
|
||||
func (stx *StoreStateTx) SetOrder(changeId, order string) (err error) {
|
||||
stx.arena.Reset()
|
||||
obj := stx.arena.NewObject()
|
||||
|
|
|
@ -4,9 +4,8 @@ import (
|
|||
"sync"
|
||||
|
||||
"github.com/anyproto/any-sync/app"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage"
|
||||
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore"
|
||||
)
|
||||
|
||||
const CName = "block.object.resolver"
|
||||
|
@ -21,12 +20,12 @@ func New() Resolver {
|
|||
}
|
||||
|
||||
type resolver struct {
|
||||
storage storage.ClientStorage
|
||||
objectStore objectstore.ObjectStore
|
||||
sync.Mutex
|
||||
}
|
||||
|
||||
func (r *resolver) Init(a *app.App) (err error) {
|
||||
r.storage = a.MustComponent(spacestorage.CName).(storage.ClientStorage)
|
||||
r.objectStore = a.MustComponent(objectstore.CName).(objectstore.ObjectStore)
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -35,5 +34,5 @@ func (r *resolver) Name() (name string) {
|
|||
}
|
||||
|
||||
func (r *resolver) ResolveSpaceID(objectID string) (string, error) {
|
||||
return r.storage.GetSpaceID(objectID)
|
||||
return r.objectStore.GetSpaceId(objectID)
|
||||
}
|
||||
|
|
|
@ -20,7 +20,7 @@ func Test_Payloads(t *testing.T) {
|
|||
changePayload := []byte("some")
|
||||
keys, err := accountdata.NewRandom()
|
||||
require.NoError(t, err)
|
||||
aclList, err := list.NewTestDerivedAcl("spaceId", keys)
|
||||
aclList, err := list.NewInMemoryDerivedAcl("spaceId", keys)
|
||||
require.NoError(t, err)
|
||||
timestamp := time.Now().Add(time.Hour).Unix()
|
||||
|
||||
|
|
|
@ -102,7 +102,7 @@ func NewTreeSyncer(spaceId string) treesyncer.TreeSyncer {
|
|||
func (t *treeSyncer) Init(a *app.App) (err error) {
|
||||
t.isSyncing = true
|
||||
spaceStorage := app.MustComponent[spacestorage.SpaceStorage](a)
|
||||
t.spaceSettingsId = spaceStorage.SpaceSettingsId()
|
||||
t.spaceSettingsId = spaceStorage.StateStorage().SettingsId()
|
||||
t.treeManager = app.MustComponent[treemanager.TreeManager](a)
|
||||
t.nodeConf = app.MustComponent[nodeconf.NodeConf](a)
|
||||
t.syncedTreeRemover = app.MustComponent[SyncedTreeRemover](a)
|
||||
|
|
|
@ -8,6 +8,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/anyproto/any-sync/app"
|
||||
"github.com/anyproto/any-sync/commonspace/headsync/statestorage/mock_statestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/synctree/mock_synctree"
|
||||
"github.com/anyproto/any-sync/commonspace/object/treemanager/mock_treemanager"
|
||||
|
@ -32,6 +33,7 @@ type fixture struct {
|
|||
nodeConf *mock_nodeconf.MockService
|
||||
syncStatus *mock_treesyncer.MockSyncedTreeRemover
|
||||
syncDetailsUpdater *mock_treesyncer.MockSyncDetailsUpdater
|
||||
stateStorage *mock_statestorage.MockStateStorage
|
||||
}
|
||||
|
||||
func newFixture(t *testing.T, spaceId string) *fixture {
|
||||
|
@ -44,7 +46,9 @@ func newFixture(t *testing.T, spaceId string) *fixture {
|
|||
syncStatus := mock_treesyncer.NewMockSyncedTreeRemover(t)
|
||||
syncDetailsUpdater := mock_treesyncer.NewMockSyncDetailsUpdater(t)
|
||||
spaceStorage := mock_spacestorage.NewMockSpaceStorage(ctrl)
|
||||
spaceStorage.EXPECT().SpaceSettingsId().Return("spaceSettingsId").AnyTimes()
|
||||
stateStorage := mock_statestorage.NewMockStateStorage(ctrl)
|
||||
spaceStorage.EXPECT().StateStorage().AnyTimes().Return(stateStorage)
|
||||
stateStorage.EXPECT().SettingsId().AnyTimes().Return("settingsId")
|
||||
|
||||
a := new(app.App)
|
||||
a.Register(testutil.PrepareMock(context.Background(), a, treeManager)).
|
||||
|
@ -64,6 +68,7 @@ func newFixture(t *testing.T, spaceId string) *fixture {
|
|||
nodeConf: nodeConf,
|
||||
syncStatus: syncStatus,
|
||||
syncDetailsUpdater: syncDetailsUpdater,
|
||||
stateStorage: stateStorage,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -342,7 +342,7 @@ func (s *Service) SpaceInitChat(ctx context.Context, spaceId string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
if spaceChatExists, err := spc.Storage().HasTree(chatId); err != nil {
|
||||
if spaceChatExists, err := spc.Storage().HasTree(ctx, chatId); err != nil {
|
||||
return err
|
||||
} else if !spaceChatExists {
|
||||
_, err = s.objectCreator.AddChatDerivedObject(ctx, spc, workspaceId)
|
||||
|
|
|
@ -276,6 +276,39 @@ func (_c *MockStore_Id_Call) RunAndReturn(run func() string) *MockStore_Id_Call
|
|||
return _c
|
||||
}
|
||||
|
||||
// MarkSeenHeads provides a mock function with given fields: heads
|
||||
func (_m *MockStore) MarkSeenHeads(heads []string) {
|
||||
_m.Called(heads)
|
||||
}
|
||||
|
||||
// MockStore_MarkSeenHeads_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkSeenHeads'
|
||||
type MockStore_MarkSeenHeads_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// MarkSeenHeads is a helper method to define mock.On call
|
||||
// - heads []string
|
||||
func (_e *MockStore_Expecter) MarkSeenHeads(heads interface{}) *MockStore_MarkSeenHeads_Call {
|
||||
return &MockStore_MarkSeenHeads_Call{Call: _e.mock.On("MarkSeenHeads", heads)}
|
||||
}
|
||||
|
||||
func (_c *MockStore_MarkSeenHeads_Call) Run(run func(heads []string)) *MockStore_MarkSeenHeads_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].([]string))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockStore_MarkSeenHeads_Call) Return() *MockStore_MarkSeenHeads_Call {
|
||||
_c.Call.Return()
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockStore_MarkSeenHeads_Call) RunAndReturn(run func([]string)) *MockStore_MarkSeenHeads_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// PushChange provides a mock function with given fields: params
|
||||
func (_m *MockStore) PushChange(params source.PushChangeParams) (string, error) {
|
||||
ret := _m.Called(params)
|
||||
|
|
|
@ -11,7 +11,6 @@ import (
|
|||
"github.com/anyproto/any-sync/commonspace/object/acl/list"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/synctree/updatelistener"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/objecttreebuilder"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage"
|
||||
|
||||
|
@ -100,7 +99,7 @@ type BuildOptions struct {
|
|||
func (b *BuildOptions) BuildTreeOpts() objecttreebuilder.BuildTreeOpts {
|
||||
return objecttreebuilder.BuildTreeOpts{
|
||||
Listener: b.Listener,
|
||||
TreeBuilder: func(treeStorage treestorage.TreeStorage, aclList list.AclList) (objecttree.ObjectTree, error) {
|
||||
TreeBuilder: func(treeStorage objecttree.Storage, aclList list.AclList) (objecttree.ObjectTree, error) {
|
||||
ot, err := objecttree.BuildKeyFilterableObjectTree(treeStorage, aclList)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -123,7 +122,7 @@ func (s *service) NewSource(ctx context.Context, space Space, id string, buildOp
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = s.storageService.BindSpaceID(src.SpaceID(), src.Id())
|
||||
err = s.objectStore.BindSpaceId(src.SpaceID(), src.Id())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("store space id for object: %w", err)
|
||||
}
|
||||
|
@ -246,7 +245,7 @@ func (s *service) RegisterStaticSource(src Source) error {
|
|||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.staticIds[src.Id()] = src
|
||||
err := s.storageService.BindSpaceID(src.SpaceID(), src.Id())
|
||||
err := s.objectStore.BindSpaceId(src.SpaceID(), src.Id())
|
||||
if err != nil {
|
||||
return fmt.Errorf("store space id for object: %w", err)
|
||||
}
|
||||
|
|
|
@ -11,7 +11,7 @@ import (
|
|||
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/synctree"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/synctree/updatelistener"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||
"github.com/anyproto/any-sync/commonspace/objecttreebuilder"
|
||||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/golang/snappy"
|
||||
|
||||
|
@ -33,6 +33,7 @@ type Store interface {
|
|||
ReadStoreDoc(ctx context.Context, stateStore *storestate.StoreState, onUpdateHook func()) (err error)
|
||||
PushStoreChange(ctx context.Context, params PushStoreChangeParams) (changeId string, err error)
|
||||
SetPushChangeHook(onPushChange PushChangeHook)
|
||||
MarkSeenHeads(heads []string)
|
||||
}
|
||||
|
||||
type PushStoreChangeParams struct {
|
||||
|
@ -51,6 +52,7 @@ type store struct {
|
|||
store *storestate.StoreState
|
||||
onUpdateHook func()
|
||||
onPushChange PushChangeHook
|
||||
diffManager *objecttree.DiffManager
|
||||
sbType smartblock.SmartBlockType
|
||||
}
|
||||
|
||||
|
@ -62,6 +64,17 @@ func (s *store) SetPushChangeHook(onPushChange PushChangeHook) {
|
|||
s.onPushChange = onPushChange
|
||||
}
|
||||
|
||||
func (s *store) createDiffManager(ctx context.Context, curTreeHeads, seenHeads []string) (err error) {
|
||||
buildTree := func(heads []string) (objecttree.ReadableObjectTree, error) {
|
||||
return s.space.TreeBuilder().BuildHistoryTree(ctx, s.Id(), objecttreebuilder.HistoryTreeOpts{
|
||||
Heads: heads,
|
||||
Include: true,
|
||||
})
|
||||
}
|
||||
s.diffManager, err = objecttree.NewDiffManager(seenHeads, curTreeHeads, buildTree, func(ids []string) {})
|
||||
return
|
||||
}
|
||||
|
||||
func (s *store) ReadDoc(ctx context.Context, receiver ChangeReceiver, empty bool) (doc state.Doc, err error) {
|
||||
s.receiver = receiver
|
||||
setter, ok := s.ObjectTree.(synctree.ListenerSetter)
|
||||
|
@ -100,7 +113,10 @@ func (s *store) PushChange(params PushChangeParams) (id string, err error) {
|
|||
func (s *store) ReadStoreDoc(ctx context.Context, storeState *storestate.StoreState, onUpdateHook func()) (err error) {
|
||||
s.onUpdateHook = onUpdateHook
|
||||
s.store = storeState
|
||||
|
||||
err = s.createDiffManager(ctx, s.source.Tree().Heads(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := s.store.NewTx(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
|
@ -143,11 +159,11 @@ func (s *store) PushStoreChange(ctx context.Context, params PushStoreChangeParam
|
|||
IsEncrypted: true,
|
||||
DataType: dataType,
|
||||
Timestamp: params.Time.Unix(),
|
||||
}, func(change *treechangeproto.RawTreeChangeWithId) error {
|
||||
order := tx.NextOrder(tx.GetMaxOrder())
|
||||
}, func(change objecttree.StorageChange) error {
|
||||
// TODO: get order here
|
||||
err = tx.ApplyChangeSet(storestate.ChangeSet{
|
||||
Id: change.Id,
|
||||
Order: order,
|
||||
Order: change.OrderId,
|
||||
Changes: params.Changes,
|
||||
Creator: s.accountService.AccountID(),
|
||||
Timestamp: params.Time.Unix(),
|
||||
|
@ -169,6 +185,14 @@ func (s *store) PushStoreChange(ctx context.Context, params PushStoreChangeParam
|
|||
if err == nil {
|
||||
s.onUpdateHook()
|
||||
}
|
||||
ch, err := s.ObjectTree.GetChange(changeId)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
s.diffManager.Add(&objecttree.Change{
|
||||
Id: changeId,
|
||||
PreviousIds: ch.PreviousIds,
|
||||
})
|
||||
return changeId, err
|
||||
}
|
||||
|
||||
|
@ -185,12 +209,17 @@ func (s *store) update(ctx context.Context, tree objecttree.ObjectTree) error {
|
|||
return errors.Join(tx.Rollback(), err)
|
||||
}
|
||||
err = tx.Commit()
|
||||
s.diffManager.Update(tree)
|
||||
if err == nil {
|
||||
s.onUpdateHook()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *store) MarkSeenHeads(heads []string) {
|
||||
s.diffManager.Remove(heads)
|
||||
}
|
||||
|
||||
func (s *store) Update(tree objecttree.ObjectTree) error {
|
||||
err := s.update(context.Background(), tree)
|
||||
if err != nil {
|
||||
|
|
|
@ -24,62 +24,14 @@ type storeApply struct {
|
|||
}
|
||||
|
||||
func (a *storeApply) Apply() (err error) {
|
||||
maxOrder := a.tx.GetMaxOrder()
|
||||
isEmpty := maxOrder == ""
|
||||
iterErr := a.ot.IterateRoot(UnmarshalStoreChange, func(change *objecttree.Change) bool {
|
||||
// not a new change - remember and continue
|
||||
if !a.allIsNew && !change.IsNew && !isEmpty {
|
||||
a.prevChange = change
|
||||
a.prevOrder = ""
|
||||
if !a.allIsNew && !change.IsNew {
|
||||
return true
|
||||
}
|
||||
currOrder, curOrdErr := a.tx.GetOrder(change.Id)
|
||||
if curOrdErr != nil {
|
||||
if !errors.Is(curOrdErr, storestate.ErrOrderNotFound) {
|
||||
err = curOrdErr
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
// change has been handled before
|
||||
a.prevChange = change
|
||||
a.prevOrder = currOrder
|
||||
return true
|
||||
}
|
||||
|
||||
prevOrder, prevOrdErr := a.getPrevOrder()
|
||||
if prevOrdErr != nil {
|
||||
if !errors.Is(prevOrdErr, storestate.ErrOrderNotFound) {
|
||||
err = prevOrdErr
|
||||
return false
|
||||
}
|
||||
if !isEmpty {
|
||||
// it should not happen, consistency with tree and store broken
|
||||
err = fmt.Errorf("unable to find previous order")
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if prevOrder == a.tx.GetMaxOrder() {
|
||||
// insert on top - just create next id
|
||||
currOrder = a.tx.NextOrder(prevOrder)
|
||||
} else {
|
||||
// insert in the middle - find next order and create id between
|
||||
nextOrder, nextOrdErr := a.findNextOrder(change.Id)
|
||||
if nextOrdErr != nil {
|
||||
// it should not happen, consistency with tree and store broken
|
||||
err = errors.Join(nextOrdErr, fmt.Errorf("unable to find next order"))
|
||||
return false
|
||||
}
|
||||
if currOrder, err = a.tx.NextBeforeOrder(prevOrder, nextOrder); err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if err = a.applyChange(change, currOrder); err != nil {
|
||||
if err = a.applyChange(change); err != nil {
|
||||
return false
|
||||
}
|
||||
a.prevOrder = currOrder
|
||||
a.prevChange = change
|
||||
return true
|
||||
})
|
||||
if err == nil && iterErr != nil {
|
||||
|
@ -88,18 +40,18 @@ func (a *storeApply) Apply() (err error) {
|
|||
return
|
||||
}
|
||||
|
||||
func (a *storeApply) applyChange(change *objecttree.Change, order string) (err error) {
|
||||
func (a *storeApply) applyChange(change *objecttree.Change) (err error) {
|
||||
storeChange, ok := change.Model.(*pb.StoreChange)
|
||||
if !ok {
|
||||
// if it is root
|
||||
if _, ok := change.Model.(*treechangeproto.TreeChangeInfo); ok {
|
||||
return a.tx.SetOrder(change.Id, order)
|
||||
return a.tx.SetOrder(change.Id, change.OrderId)
|
||||
}
|
||||
return fmt.Errorf("unexpected change content type: %T", change.Model)
|
||||
}
|
||||
set := storestate.ChangeSet{
|
||||
Id: change.Id,
|
||||
Order: order,
|
||||
Order: change.OrderId,
|
||||
Changes: storeChange.ChangeSet,
|
||||
Creator: change.Identity.Account(),
|
||||
Timestamp: change.Timestamp,
|
||||
|
@ -111,56 +63,3 @@ func (a *storeApply) applyChange(change *objecttree.Change, order string) (err e
|
|||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *storeApply) getPrevOrder() (order string, err error) {
|
||||
if a.prevOrder != "" {
|
||||
return a.prevOrder, nil
|
||||
}
|
||||
if a.prevChange == nil {
|
||||
return "", nil
|
||||
}
|
||||
if order, err = a.tx.GetOrder(a.prevChange.Id); err != nil {
|
||||
return
|
||||
}
|
||||
a.prevOrder = order
|
||||
return
|
||||
}
|
||||
|
||||
func (a *storeApply) findNextOrder(changeId string) (order string, err error) {
|
||||
if order = a.findNextInCache(changeId); order != "" {
|
||||
return
|
||||
}
|
||||
|
||||
a.nextCacheChange = map[string]struct{}{}
|
||||
iterErr := a.ot.IterateFrom(changeId, UnmarshalStoreChange, func(change *objecttree.Change) bool {
|
||||
order, err = a.tx.GetOrder(change.Id)
|
||||
if err != nil {
|
||||
if errors.Is(err, storestate.ErrOrderNotFound) {
|
||||
// no order - remember id and move forward
|
||||
a.nextCacheChange[change.Id] = struct{}{}
|
||||
return true
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// order found
|
||||
a.nextCachedOrder = order
|
||||
return false
|
||||
})
|
||||
if err == nil && iterErr != nil {
|
||||
return "", iterErr
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (a *storeApply) findNextInCache(changeId string) (order string) {
|
||||
if a.nextCacheChange == nil {
|
||||
return ""
|
||||
}
|
||||
if _, ok := a.nextCacheChange[changeId]; ok {
|
||||
return a.nextCachedOrder
|
||||
}
|
||||
a.nextCachedOrder = ""
|
||||
a.nextCacheChange = nil
|
||||
return ""
|
||||
}
|
||||
|
|
|
@ -2,12 +2,10 @@ package source
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
anystore "github.com/anyproto/any-store"
|
||||
"github.com/anyproto/any-sync/commonspace/object/accountdata"
|
||||
|
@ -19,6 +17,7 @@ import (
|
|||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/mock/gomock"
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/block/editor/storestate"
|
||||
"github.com/anyproto/anytype-heart/pb"
|
||||
|
@ -98,90 +97,6 @@ func TestStoreApply_RealTree(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestStoreApply_Apply(t *testing.T) {
|
||||
t.Run("new tree", func(t *testing.T) {
|
||||
fx := newMockTreeStoreFx(t)
|
||||
tx := fx.RequireTx(t)
|
||||
changes := []*objecttree.Change{
|
||||
testChange("1", false),
|
||||
testChange("2", false),
|
||||
testChange("3", false),
|
||||
}
|
||||
fx.ApplyChanges(t, tx, changes...)
|
||||
require.NoError(t, tx.Commit())
|
||||
})
|
||||
t.Run("insert middle", func(t *testing.T) {
|
||||
fx := newMockTreeStoreFx(t)
|
||||
tx := fx.RequireTx(t)
|
||||
changes := []*objecttree.Change{
|
||||
testChange("1", false),
|
||||
testChange("2", false),
|
||||
testChange("3", false),
|
||||
}
|
||||
fx.ApplyChanges(t, tx, changes...)
|
||||
require.NoError(t, tx.Commit())
|
||||
|
||||
tx = fx.RequireTx(t)
|
||||
changes = []*objecttree.Change{
|
||||
testChange("1", false),
|
||||
testChange("1.1", true),
|
||||
testChange("1.2", true),
|
||||
testChange("1.3", true),
|
||||
testChange("2", false),
|
||||
testChange("2.2", true),
|
||||
testChange("3", false),
|
||||
}
|
||||
fx.ExpectTreeFrom("1.1", changes[1:]...)
|
||||
fx.ExpectTreeFrom("2.2", changes[6:]...)
|
||||
fx.ApplyChanges(t, tx, changes...)
|
||||
require.NoError(t, tx.Commit())
|
||||
})
|
||||
t.Run("append", func(t *testing.T) {
|
||||
fx := newMockTreeStoreFx(t)
|
||||
tx := fx.RequireTx(t)
|
||||
changes := []*objecttree.Change{
|
||||
testChange("1", false),
|
||||
testChange("2", false),
|
||||
testChange("3", false),
|
||||
}
|
||||
fx.ApplyChanges(t, tx, changes...)
|
||||
require.NoError(t, tx.Commit())
|
||||
|
||||
tx = fx.RequireTx(t)
|
||||
changes = []*objecttree.Change{
|
||||
testChange("1", false),
|
||||
testChange("2", false),
|
||||
testChange("3", false),
|
||||
testChange("4", true),
|
||||
testChange("5", true),
|
||||
testChange("6", true),
|
||||
}
|
||||
fx.ApplyChanges(t, tx, changes...)
|
||||
require.NoError(t, tx.Commit())
|
||||
})
|
||||
}
|
||||
|
||||
func TestStoreApply_Apply10000(t *testing.T) {
|
||||
fx := newMockTreeStoreFx(t)
|
||||
tx := fx.RequireTx(t)
|
||||
changes := make([]*objecttree.Change, 100000)
|
||||
for i := range changes {
|
||||
changes[i] = testChange(fmt.Sprint(i), false)
|
||||
}
|
||||
st := time.Now()
|
||||
applier := &storeApply{
|
||||
tx: tx,
|
||||
ot: fx.mockTree,
|
||||
}
|
||||
fx.ExpectTree(changes...)
|
||||
require.NoError(t, applier.Apply())
|
||||
t.Logf("apply dur: %v;", time.Since(st))
|
||||
st = time.Now()
|
||||
require.NoError(t, tx.Commit())
|
||||
t.Logf("commit dur: %v;", time.Since(st))
|
||||
|
||||
}
|
||||
|
||||
type storeFx struct {
|
||||
state *storestate.StoreState
|
||||
mockTree *mock_objecttree.MockObjectTree
|
||||
|
@ -243,37 +158,6 @@ func (fx *storeFx) ApplyChanges(t *testing.T, tx *storestate.StoreStateTx, chang
|
|||
fx.AssertOrder(t, tx, changes...)
|
||||
}
|
||||
|
||||
func newMockTreeStoreFx(t testing.TB) *storeFx {
|
||||
tmpDir, err := os.MkdirTemp("", "source_store_*")
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := anystore.Open(ctx, filepath.Join(tmpDir, "test.db"), nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
ctrl := gomock.NewController(t)
|
||||
t.Cleanup(func() {
|
||||
if db != nil {
|
||||
_ = db.Close()
|
||||
}
|
||||
ctrl.Finish()
|
||||
if tmpDir != "" {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}
|
||||
})
|
||||
|
||||
state, err := storestate.New(ctx, "source_test", db, storestate.DefaultHandler{Name: "default"})
|
||||
require.NoError(t, err)
|
||||
|
||||
tree := mock_objecttree.NewMockObjectTree(ctrl)
|
||||
tree.EXPECT().Id().Return("root").AnyTimes()
|
||||
|
||||
return &storeFx{
|
||||
state: state,
|
||||
mockTree: tree,
|
||||
db: db,
|
||||
}
|
||||
}
|
||||
|
||||
func newRealTreeStoreFx(t testing.TB) *storeFx {
|
||||
tmpDir, err := os.MkdirTemp("", "source_store_*")
|
||||
require.NoError(t, err)
|
||||
|
@ -295,14 +179,16 @@ func newRealTreeStoreFx(t testing.TB) *storeFx {
|
|||
state, err := storestate.New(ctx, "source_test", db, storestate.DefaultHandler{Name: "default"})
|
||||
require.NoError(t, err)
|
||||
aclList, _ := prepareAclList(t)
|
||||
objTree, err := buildTree(aclList)
|
||||
objTree, err := buildTree(t, aclList)
|
||||
require.NoError(t, err)
|
||||
fx := &storeFx{
|
||||
state: state,
|
||||
realTree: objTree,
|
||||
aclList: aclList,
|
||||
changeCreator: objecttree.NewMockChangeCreator(),
|
||||
db: db,
|
||||
state: state,
|
||||
realTree: objTree,
|
||||
aclList: aclList,
|
||||
changeCreator: objecttree.NewMockChangeCreator(func() anystore.DB {
|
||||
return createStore(ctx, t)
|
||||
}),
|
||||
db: db,
|
||||
}
|
||||
tx := fx.RequireTx(t)
|
||||
defer tx.Rollback()
|
||||
|
@ -321,6 +207,7 @@ func testChange(id string, isNew bool) *objecttree.Change {
|
|||
|
||||
return &objecttree.Change{
|
||||
Id: id,
|
||||
OrderId: id,
|
||||
IsNew: isNew,
|
||||
Model: &pb.StoreChange{},
|
||||
Identity: pub,
|
||||
|
@ -330,15 +217,17 @@ func testChange(id string, isNew bool) *objecttree.Change {
|
|||
func prepareAclList(t testing.TB) (list.AclList, *accountdata.AccountKeys) {
|
||||
randKeys, err := accountdata.NewRandom()
|
||||
require.NoError(t, err)
|
||||
aclList, err := list.NewTestDerivedAcl("spaceId", randKeys)
|
||||
aclList, err := list.NewInMemoryDerivedAcl("spaceId", randKeys)
|
||||
require.NoError(t, err, "building acl list should be without error")
|
||||
|
||||
return aclList, randKeys
|
||||
}
|
||||
|
||||
func buildTree(aclList list.AclList) (objecttree.ObjectTree, error) {
|
||||
changeCreator := objecttree.NewMockChangeCreator()
|
||||
treeStorage := changeCreator.CreateNewTreeStorage("0", aclList.Head().Id, false)
|
||||
func buildTree(t testing.TB, aclList list.AclList) (objecttree.ObjectTree, error) {
|
||||
changeCreator := objecttree.NewMockChangeCreator(func() anystore.DB {
|
||||
return createStore(ctx, t)
|
||||
})
|
||||
treeStorage := changeCreator.CreateNewTreeStorage(t.(*testing.T), "0", aclList.Head().Id, false)
|
||||
tree, err := objecttree.BuildTestableTree(treeStorage, aclList)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -346,3 +235,22 @@ func buildTree(aclList list.AclList) (objecttree.ObjectTree, error) {
|
|||
tree.SetFlusher(objecttree.MarkNewChangeFlusher())
|
||||
return tree, nil
|
||||
}
|
||||
|
||||
func createStore(ctx context.Context, t testing.TB) anystore.DB {
|
||||
return createNamedStore(ctx, t, "changes.db")
|
||||
}
|
||||
|
||||
func createNamedStore(ctx context.Context, t testing.TB, name string) anystore.DB {
|
||||
path := filepath.Join(t.TempDir(), name)
|
||||
db, err := anystore.Open(ctx, path, nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
err := db.Close()
|
||||
require.NoError(t, err)
|
||||
unix.Rmdir(path)
|
||||
})
|
||||
return objecttree.TestStore{
|
||||
DB: db,
|
||||
Path: path,
|
||||
}
|
||||
}
|
||||
|
|
|
@ -83,13 +83,13 @@ func (c *configFetcher) updateStatus(ctx context.Context) (err error) {
|
|||
techSpace := c.getter.TechSpace()
|
||||
res, err := c.client.StatusCheck(ctx, techSpace.Id())
|
||||
if errors.Is(err, coordinatorproto.ErrSpaceNotExists) {
|
||||
header, sErr := techSpace.Storage().SpaceHeader()
|
||||
state, sErr := techSpace.Storage().StateStorage().GetState(ctx)
|
||||
if sErr != nil {
|
||||
return sErr
|
||||
}
|
||||
payload := coordinatorclient.SpaceSignPayload{
|
||||
SpaceId: header.Id,
|
||||
SpaceHeader: header.RawHeader,
|
||||
SpaceId: techSpace.Id(),
|
||||
SpaceHeader: state.SpaceHeader,
|
||||
OldAccount: c.wallet.GetOldAccountKey(),
|
||||
Identity: c.wallet.GetAccountPrivkey(),
|
||||
}
|
||||
|
|
111
core/debug/exporter/exporter.go
Normal file
111
core/debug/exporter/exporter.go
Normal file
|
@ -0,0 +1,111 @@
|
|||
package exporter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
anystore "github.com/anyproto/any-store"
|
||||
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
|
||||
"github.com/anyproto/any-sync/commonspace/object/accountdata"
|
||||
"github.com/anyproto/any-sync/commonspace/object/acl/list"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||
"github.com/anyproto/any-sync/util/crypto"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
type DataConverter interface {
|
||||
Unmarshall(dataType string, decrypted []byte) (any, error)
|
||||
Marshall(model any) (data []byte, dataType string, err error)
|
||||
}
|
||||
|
||||
func prepareExport(ctx context.Context, readable objecttree.ReadableObjectTree, store anystore.DB) (objecttree.ObjectTree, error) {
|
||||
headStorage, err := headstorage.New(ctx, store)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
acl := readable.AclList()
|
||||
root := acl.Root()
|
||||
listStorage, err := list.CreateStorage(ctx, root, headStorage, store)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys, err := accountdata.NewRandom()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
newAcl, err := list.BuildAclListWithIdentity(keys, listStorage, list.NoOpAcceptorVerifier{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
treeStorage, err := objecttree.CreateStorage(ctx, readable.Header(), headStorage, store)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
writeTree, err := objecttree.BuildTestableTree(treeStorage, newAcl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return writeTree, nil
|
||||
}
|
||||
|
||||
type ExportParams struct {
|
||||
Readable objecttree.ReadableObjectTree
|
||||
Store anystore.DB
|
||||
Converter DataConverter
|
||||
}
|
||||
|
||||
func ExportTree(ctx context.Context, params ExportParams) error {
|
||||
writeTree, err := prepareExport(ctx, params.Readable, params.Store)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var (
|
||||
changeBuilder = objecttree.NewChangeBuilder(crypto.NewKeyStorage(), writeTree.Header())
|
||||
converter = params.Converter
|
||||
changes []*treechangeproto.RawTreeChangeWithId
|
||||
)
|
||||
err = writeTree.IterateRoot(
|
||||
func(change *objecttree.Change, decrypted []byte) (any, error) {
|
||||
return converter.Unmarshall(change.DataType, decrypted)
|
||||
},
|
||||
func(change *objecttree.Change) bool {
|
||||
if change.Id == writeTree.Id() {
|
||||
return true
|
||||
}
|
||||
var (
|
||||
data []byte
|
||||
dataType string
|
||||
)
|
||||
data, dataType, err = converter.Marshall(change.Model)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
// that means that change is unencrypted
|
||||
change.ReadKeyId = ""
|
||||
change.Data = data
|
||||
change.DataType = dataType
|
||||
var raw *treechangeproto.RawTreeChangeWithId
|
||||
raw, err = changeBuilder.Marshall(change)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
changes = append(changes, raw)
|
||||
return true
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
payload := objecttree.RawChangesPayload{
|
||||
NewHeads: writeTree.Heads(),
|
||||
RawChanges: changes,
|
||||
}
|
||||
res, err := writeTree.AddRawChanges(ctx, payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !slices.Equal(res.Heads, writeTree.Heads()) {
|
||||
return fmt.Errorf("heads mismatch: %v != %v", res.Heads, writeTree.Heads())
|
||||
}
|
||||
return nil
|
||||
}
|
96
core/debug/exporter/importer.go
Normal file
96
core/debug/exporter/importer.go
Normal file
|
@ -0,0 +1,96 @@
|
|||
package exporter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
anystore "github.com/anyproto/any-store"
|
||||
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
|
||||
"github.com/anyproto/any-sync/commonspace/object/accountdata"
|
||||
"github.com/anyproto/any-sync/commonspace/object/acl/list"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
|
||||
|
||||
"github.com/anyproto/anytype-heart/util/ziputil"
|
||||
)
|
||||
|
||||
type ImportResult struct {
|
||||
List list.AclList
|
||||
Storage objecttree.Storage
|
||||
FolderPath string
|
||||
}
|
||||
|
||||
func (i ImportResult) CreateReadableTree(fullTree bool, beforeId string) (objecttree.ReadableObjectTree, error) {
|
||||
return objecttree.BuildNonVerifiableHistoryTree(objecttree.HistoryTreeParams{
|
||||
Storage: i.Storage,
|
||||
AclList: i.List,
|
||||
Heads: i.Heads(fullTree, beforeId),
|
||||
IncludeBeforeId: true,
|
||||
})
|
||||
}
|
||||
|
||||
func (i ImportResult) Heads(fullTree bool, beforeId string) []string {
|
||||
if fullTree {
|
||||
return nil
|
||||
}
|
||||
return []string{beforeId}
|
||||
}
|
||||
|
||||
func ImportStorage(ctx context.Context, path string) (res ImportResult, err error) {
|
||||
targetDir := strings.TrimSuffix(path, filepath.Ext(path))
|
||||
if _, err = os.Stat(targetDir); err == nil {
|
||||
err = os.RemoveAll(targetDir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
if err = ziputil.UnzipFolder(path, targetDir); err != nil {
|
||||
return
|
||||
}
|
||||
anyStore, err := anystore.Open(ctx, targetDir, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer anyStore.Close()
|
||||
var (
|
||||
aclId string
|
||||
treeId string
|
||||
)
|
||||
headStorage, err := headstorage.New(ctx, anyStore)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = headStorage.IterateEntries(ctx, headstorage.IterOpts{}, func(entry headstorage.HeadsEntry) (bool, error) {
|
||||
if entry.CommonSnapshot == "" {
|
||||
aclId = entry.Id
|
||||
return true, nil
|
||||
}
|
||||
treeId = entry.Id
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
listStorage, err := list.NewStorage(ctx, aclId, headStorage, anyStore)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
randomKeys, err := accountdata.NewRandom()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
acl, err := list.BuildAclListWithIdentity(randomKeys, listStorage, list.NoOpAcceptorVerifier{})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
treeStorage, err := objecttree.NewStorage(ctx, treeId, headStorage, anyStore)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return ImportResult{
|
||||
List: acl,
|
||||
Storage: treeStorage,
|
||||
FolderPath: targetDir,
|
||||
}, nil
|
||||
}
|
|
@ -1,13 +1,10 @@
|
|||
package treearchive
|
||||
package exporter
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/acl/list"
|
||||
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/block/editor/state"
|
||||
"github.com/anyproto/anytype-heart/core/block/source"
|
||||
|
@ -43,22 +40,18 @@ func (m MarshalledJsonChange) MarshalJSON() ([]byte, error) {
|
|||
|
||||
type TreeImporter interface {
|
||||
ObjectTree() objecttree.ReadableObjectTree
|
||||
State() (*state.State, error) // set fullStateChain to true to get full state chain, otherwise only the last state will be returned
|
||||
Import(fromRoot bool, beforeId string) error
|
||||
State() (*state.State, error)
|
||||
Json() (TreeJson, error)
|
||||
ChangeAt(idx int) (IdChange, error)
|
||||
}
|
||||
|
||||
type treeImporter struct {
|
||||
listStorage liststorage.ListStorage
|
||||
treeStorage treestorage.TreeStorage
|
||||
objectTree objecttree.ReadableObjectTree
|
||||
objectTree objecttree.ReadableObjectTree
|
||||
}
|
||||
|
||||
func NewTreeImporter(listStorage liststorage.ListStorage, treeStorage treestorage.TreeStorage) TreeImporter {
|
||||
func NewTreeImporter(objectTree objecttree.ReadableObjectTree) TreeImporter {
|
||||
return &treeImporter{
|
||||
listStorage: listStorage,
|
||||
treeStorage: treeStorage,
|
||||
objectTree: objectTree,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -83,25 +76,6 @@ func (t *treeImporter) State() (*state.State, error) {
|
|||
return st, nil
|
||||
}
|
||||
|
||||
func (t *treeImporter) Import(fullTree bool, beforeId string) (err error) {
|
||||
aclList, err := list.BuildAclList(t.listStorage, list.NoOpAcceptorVerifier{})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var heads []string
|
||||
if !fullTree {
|
||||
heads = []string{beforeId}
|
||||
}
|
||||
t.objectTree, err = objecttree.BuildNonVerifiableHistoryTree(objecttree.HistoryTreeParams{
|
||||
TreeStorage: t.treeStorage,
|
||||
AclList: aclList,
|
||||
Heads: heads,
|
||||
IncludeBeforeId: true,
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (t *treeImporter) Json() (treeJson TreeJson, err error) {
|
||||
treeJson = TreeJson{
|
||||
Id: t.objectTree.Id(),
|
|
@ -1,99 +0,0 @@
|
|||
package treearchive
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"os"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
"github.com/anyproto/any-sync/consensus/consensusproto"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/debug/treearchive/zipaclstorage"
|
||||
"github.com/anyproto/anytype-heart/core/debug/treearchive/ziptreestorage"
|
||||
)
|
||||
|
||||
type ExportedObjectsJson struct {
|
||||
AclId string `json:"aclId"`
|
||||
TreeId string `json:"treeId"`
|
||||
}
|
||||
|
||||
type ArchiveWriter struct {
|
||||
zw *zip.Writer
|
||||
zf fs.File
|
||||
treeId string
|
||||
aclId string
|
||||
storages []flushableStorage
|
||||
}
|
||||
|
||||
type flushableStorage interface {
|
||||
FlushStorage() error
|
||||
}
|
||||
|
||||
func NewArchiveWriter(path string) (*ArchiveWriter, error) {
|
||||
zf, err := os.Create(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
zw := zip.NewWriter(zf)
|
||||
return &ArchiveWriter{
|
||||
zw: zw,
|
||||
zf: zf,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (e *ArchiveWriter) ZipWriter() *zip.Writer {
|
||||
return e.zw
|
||||
}
|
||||
|
||||
func (e *ArchiveWriter) TreeStorage(root *treechangeproto.RawTreeChangeWithId) (treestorage.TreeStorage, error) {
|
||||
e.treeId = root.Id
|
||||
st, err := ziptreestorage.NewZipTreeWriteStorage(root, e.zw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e.storages = append(e.storages, st.(flushableStorage))
|
||||
return st, nil
|
||||
}
|
||||
|
||||
func (e *ArchiveWriter) ListStorage(root *consensusproto.RawRecordWithId) (liststorage.ListStorage, error) {
|
||||
e.aclId = root.Id
|
||||
st, err := zipaclstorage.NewACLWriteStorage(root, e.zw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e.storages = append(e.storages, st.(flushableStorage))
|
||||
return st, nil
|
||||
}
|
||||
|
||||
func (e *ArchiveWriter) Close() (err error) {
|
||||
for _, st := range e.storages {
|
||||
err = st.FlushStorage()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
exportedHeader, err := e.zw.CreateHeader(&zip.FileHeader{
|
||||
Name: "exported.json",
|
||||
Method: zip.Deflate,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
enc := json.NewEncoder(exportedHeader)
|
||||
enc.SetIndent("", "\t")
|
||||
err = enc.Encode(ExportedObjectsJson{
|
||||
TreeId: e.treeId,
|
||||
AclId: e.aclId,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = e.zw.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return e.zf.Close()
|
||||
}
|
|
@ -1,91 +0,0 @@
|
|||
package treearchive
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
"github.com/gogo/protobuf/jsonpb"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/debug/treearchive/zipaclstorage"
|
||||
"github.com/anyproto/anytype-heart/core/debug/treearchive/ziptreestorage"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
|
||||
)
|
||||
|
||||
type TreeArchive interface {
|
||||
ListStorage() liststorage.ListStorage
|
||||
TreeStorage() treestorage.TreeStorage
|
||||
LocalStore() (*model.ObjectInfo, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
type treeArchive struct {
|
||||
listStorage liststorage.ListStorage
|
||||
treeStorage treestorage.TreeStorage
|
||||
zr *zip.ReadCloser
|
||||
}
|
||||
|
||||
// Open expects debug tree zip file
|
||||
// returns TreeArchive that has ListStorage and TreeStorage
|
||||
func Open(filename string) (tr TreeArchive, err error) {
|
||||
zr, err := zip.OpenReader(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
exported, err := zr.Open("exported.json")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer exported.Close()
|
||||
expJson := &ExportedObjectsJson{}
|
||||
if err = json.NewDecoder(exported).Decode(expJson); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
listStorage, err := zipaclstorage.NewZipAclReadStorage(expJson.AclId, zr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
treeStorage, err := ziptreestorage.NewZipTreeReadStorage(expJson.TreeId, zr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return &treeArchive{
|
||||
listStorage: listStorage,
|
||||
treeStorage: treeStorage,
|
||||
zr: zr,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *treeArchive) ListStorage() liststorage.ListStorage {
|
||||
return a.listStorage
|
||||
}
|
||||
|
||||
func (a *treeArchive) TreeStorage() treestorage.TreeStorage {
|
||||
return a.treeStorage
|
||||
}
|
||||
|
||||
func (a *treeArchive) LocalStore() (*model.ObjectInfo, error) {
|
||||
for _, f := range a.zr.File {
|
||||
if f.Name == "localstore.json" {
|
||||
rd, err := f.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rd.Close()
|
||||
var oi = &model.ObjectInfo{}
|
||||
if err = jsonpb.Unmarshal(rd, oi); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return oi, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("block logs file not found")
|
||||
}
|
||||
|
||||
func (a *treeArchive) Close() (err error) {
|
||||
return a.zr.Close()
|
||||
}
|
|
@ -1,77 +0,0 @@
|
|||
package zipaclstorage
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
|
||||
"github.com/anyproto/any-sync/consensus/consensusproto"
|
||||
)
|
||||
|
||||
type zipAclReadStorage struct {
|
||||
id string
|
||||
files map[string]*zip.File
|
||||
}
|
||||
|
||||
func NewZipAclReadStorage(id string, zr *zip.ReadCloser) (ls liststorage.ListStorage, err error) {
|
||||
aclStorage := &zipAclReadStorage{
|
||||
id: id,
|
||||
files: map[string]*zip.File{},
|
||||
}
|
||||
for _, f := range zr.Reader.File {
|
||||
if len(f.Name) > len(id) && strings.Contains(f.Name, id) {
|
||||
split := strings.SplitAfter(id, "/")
|
||||
last := split[len(split)-1]
|
||||
aclStorage.files[last] = f
|
||||
}
|
||||
}
|
||||
ls = aclStorage
|
||||
return
|
||||
}
|
||||
|
||||
func (z *zipAclReadStorage) Id() string {
|
||||
return z.id
|
||||
}
|
||||
|
||||
func (z *zipAclReadStorage) Root() (*consensusproto.RawRecordWithId, error) {
|
||||
return z.readRecord(z.id)
|
||||
}
|
||||
|
||||
func (z *zipAclReadStorage) Head() (string, error) {
|
||||
return z.id, nil
|
||||
}
|
||||
|
||||
func (z *zipAclReadStorage) SetHead(headId string) error {
|
||||
panic("should not be called")
|
||||
}
|
||||
|
||||
func (z *zipAclReadStorage) GetRawRecord(_ context.Context, id string) (*consensusproto.RawRecordWithId, error) {
|
||||
return z.readRecord(id)
|
||||
}
|
||||
|
||||
func (z *zipAclReadStorage) AddRawRecord(_ context.Context, _ *consensusproto.RawRecordWithId) (err error) {
|
||||
panic("should not be called")
|
||||
}
|
||||
|
||||
func (z *zipAclReadStorage) readRecord(id string) (rec *consensusproto.RawRecordWithId, err error) {
|
||||
file, ok := z.files[id]
|
||||
if !ok {
|
||||
err = fmt.Errorf("object not found in storage")
|
||||
return
|
||||
}
|
||||
opened, err := file.Open()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer opened.Close()
|
||||
|
||||
buf, err := io.ReadAll(opened)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
rec = &consensusproto.RawRecordWithId{Payload: buf, Id: id}
|
||||
return
|
||||
}
|
|
@ -1,59 +0,0 @@
|
|||
package zipaclstorage
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
|
||||
"github.com/anyproto/any-sync/consensus/consensusproto"
|
||||
)
|
||||
|
||||
type zipACLWriteStorage struct {
|
||||
id string
|
||||
zw *zip.Writer
|
||||
}
|
||||
|
||||
func NewACLWriteStorage(root *consensusproto.RawRecordWithId, zw *zip.Writer) (ls liststorage.ListStorage, err error) {
|
||||
ls = &zipACLWriteStorage{
|
||||
id: root.Id,
|
||||
zw: zw,
|
||||
}
|
||||
err = ls.AddRawRecord(context.Background(), root)
|
||||
return
|
||||
}
|
||||
|
||||
// nolint:revive
|
||||
func (z *zipACLWriteStorage) Id() string {
|
||||
return z.id
|
||||
}
|
||||
|
||||
func (z *zipACLWriteStorage) Root() (*consensusproto.RawRecordWithId, error) {
|
||||
panic("should not be called")
|
||||
}
|
||||
|
||||
func (z *zipACLWriteStorage) Head() (string, error) {
|
||||
return z.id, nil
|
||||
}
|
||||
|
||||
func (z *zipACLWriteStorage) SetHead(_ string) error {
|
||||
// TODO: As soon as our acls are writeable, this should be implemented
|
||||
panic("should not be called")
|
||||
}
|
||||
|
||||
func (z *zipACLWriteStorage) GetRawRecord(_ context.Context, _ string) (*consensusproto.RawRecordWithId, error) {
|
||||
panic("should not be called")
|
||||
}
|
||||
|
||||
func (z *zipACLWriteStorage) AddRawRecord(_ context.Context, rec *consensusproto.RawRecordWithId) (err error) {
|
||||
wr, err := z.zw.Create(strings.Join([]string{z.id, rec.Id}, "/"))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_, err = wr.Write(rec.Payload)
|
||||
return
|
||||
}
|
||||
|
||||
func (z *zipACLWriteStorage) FlushStorage() error {
|
||||
return nil
|
||||
}
|
|
@ -1,116 +0,0 @@
|
|||
package ziptreestorage
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
)
|
||||
|
||||
type zipTreeReadStorage struct {
|
||||
id string
|
||||
heads []string
|
||||
files map[string]*zip.File
|
||||
}
|
||||
|
||||
func NewZipTreeReadStorage(id string, zr *zip.ReadCloser) (st treestorage.TreeStorage, err error) {
|
||||
zrs := &zipTreeReadStorage{
|
||||
id: id,
|
||||
heads: nil,
|
||||
files: map[string]*zip.File{},
|
||||
}
|
||||
for _, f := range zr.Reader.File {
|
||||
if len(f.Name) > len(id) && strings.Contains(f.Name, id) {
|
||||
split := strings.Split(f.Name, "/")
|
||||
last := split[len(split)-1]
|
||||
zrs.files[last] = f
|
||||
}
|
||||
}
|
||||
data, ok := zrs.files["data.json"]
|
||||
if !ok {
|
||||
err = fmt.Errorf("no data.json in archive")
|
||||
return
|
||||
}
|
||||
dataOpened, err := data.Open()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer dataOpened.Close()
|
||||
headsEntry := &HeadsJsonEntry{}
|
||||
if err = json.NewDecoder(dataOpened).Decode(headsEntry); err != nil {
|
||||
return
|
||||
}
|
||||
zrs.heads = headsEntry.Heads
|
||||
st = zrs
|
||||
return
|
||||
}
|
||||
|
||||
func (t *zipTreeReadStorage) GetAllChangeIds() (chs []string, err error) {
|
||||
return nil, fmt.Errorf("get all change ids should not be called")
|
||||
}
|
||||
|
||||
func (z *zipTreeReadStorage) Id() string {
|
||||
return z.id
|
||||
}
|
||||
|
||||
func (z *zipTreeReadStorage) Root() (root *treechangeproto.RawTreeChangeWithId, err error) {
|
||||
return z.readChange(z.id)
|
||||
}
|
||||
|
||||
func (z *zipTreeReadStorage) Heads() ([]string, error) {
|
||||
return z.heads, nil
|
||||
}
|
||||
|
||||
func (z *zipTreeReadStorage) SetHeads(heads []string) (err error) {
|
||||
panic("should not be called")
|
||||
}
|
||||
|
||||
func (z *zipTreeReadStorage) AddRawChange(change *treechangeproto.RawTreeChangeWithId) (err error) {
|
||||
panic("should not be called")
|
||||
}
|
||||
|
||||
func (z *zipTreeReadStorage) AddRawChangesSetHeads(changes []*treechangeproto.RawTreeChangeWithId, heads []string) (err error) {
|
||||
panic("should not be called")
|
||||
}
|
||||
|
||||
func (z *zipTreeReadStorage) GetRawChange(ctx context.Context, id string) (*treechangeproto.RawTreeChangeWithId, error) {
|
||||
return z.readChange(id)
|
||||
}
|
||||
|
||||
func (z *zipTreeReadStorage) GetAppendRawChange(ctx context.Context, buf []byte, id string) (*treechangeproto.RawTreeChangeWithId, error) {
|
||||
return z.readChange(id)
|
||||
}
|
||||
|
||||
func (z *zipTreeReadStorage) HasChange(ctx context.Context, id string) (ok bool, err error) {
|
||||
_, ok = z.files[id]
|
||||
return
|
||||
}
|
||||
|
||||
func (z *zipTreeReadStorage) Delete() error {
|
||||
panic("should not be called")
|
||||
}
|
||||
|
||||
func (z *zipTreeReadStorage) readChange(id string) (change *treechangeproto.RawTreeChangeWithId, err error) {
|
||||
file, ok := z.files[id]
|
||||
if !ok {
|
||||
err = fmt.Errorf("object not found in storage")
|
||||
return
|
||||
}
|
||||
opened, err := file.Open()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer opened.Close()
|
||||
|
||||
buf, err := io.ReadAll(opened)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
change = &treechangeproto.RawTreeChangeWithId{RawChange: buf, Id: id}
|
||||
return
|
||||
}
|
|
@ -1,110 +0,0 @@
|
|||
package ziptreestorage
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
)
|
||||
|
||||
type HeadsJsonEntry struct {
|
||||
Heads []string `json:"heads"`
|
||||
RootId string `json:"rootId"`
|
||||
}
|
||||
|
||||
type zipTreeWriteStorage struct {
|
||||
id string
|
||||
heads []string
|
||||
zw *zip.Writer
|
||||
}
|
||||
|
||||
func NewZipTreeWriteStorage(root *treechangeproto.RawTreeChangeWithId, zw *zip.Writer) (st treestorage.TreeStorage, err error) {
|
||||
z := &zipTreeWriteStorage{
|
||||
id: root.Id,
|
||||
zw: zw,
|
||||
}
|
||||
err = z.SetHeads([]string{root.Id})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = z.AddRawChange(root)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
st = z
|
||||
return
|
||||
}
|
||||
|
||||
func (z *zipTreeWriteStorage) Id() string {
|
||||
return z.id
|
||||
}
|
||||
|
||||
func (t *zipTreeWriteStorage) GetAllChangeIds() (chs []string, err error) {
|
||||
return nil, fmt.Errorf("get all change ids should not be called")
|
||||
}
|
||||
|
||||
func (z *zipTreeWriteStorage) Root() (*treechangeproto.RawTreeChangeWithId, error) {
|
||||
panic("should not be implemented")
|
||||
}
|
||||
|
||||
func (z *zipTreeWriteStorage) Heads() ([]string, error) {
|
||||
return z.heads, nil
|
||||
}
|
||||
|
||||
func (z *zipTreeWriteStorage) SetHeads(heads []string) (err error) {
|
||||
z.heads = heads
|
||||
return
|
||||
}
|
||||
|
||||
func (z *zipTreeWriteStorage) AddRawChange(change *treechangeproto.RawTreeChangeWithId) (err error) {
|
||||
wr, err := z.zw.Create(strings.Join([]string{z.id, change.Id}, "/"))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_, err = wr.Write(change.RawChange)
|
||||
return
|
||||
}
|
||||
|
||||
func (z *zipTreeWriteStorage) AddRawChangesSetHeads(changes []*treechangeproto.RawTreeChangeWithId, heads []string) (err error) {
|
||||
for _, ch := range changes {
|
||||
err = z.AddRawChange(ch)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
return z.SetHeads(heads)
|
||||
}
|
||||
|
||||
func (z *zipTreeWriteStorage) GetRawChange(ctx context.Context, id string) (*treechangeproto.RawTreeChangeWithId, error) {
|
||||
panic("should not be called")
|
||||
}
|
||||
|
||||
func (z *zipTreeWriteStorage) GetAppendRawChange(ctx context.Context, buf []byte, id string) (*treechangeproto.RawTreeChangeWithId, error) {
|
||||
panic("should not be called")
|
||||
}
|
||||
|
||||
func (z *zipTreeWriteStorage) HasChange(ctx context.Context, id string) (ok bool, err error) {
|
||||
panic("should not be called")
|
||||
}
|
||||
|
||||
func (z *zipTreeWriteStorage) Delete() error {
|
||||
panic("should not be called")
|
||||
}
|
||||
|
||||
func (z *zipTreeWriteStorage) FlushStorage() (err error) {
|
||||
chw, err := z.zw.CreateHeader(&zip.FileHeader{
|
||||
Name: strings.Join([]string{z.id, "data.json"}, "/"),
|
||||
Method: zip.Deflate,
|
||||
})
|
||||
enc := json.NewEncoder(chw)
|
||||
enc.SetIndent("", "\t")
|
||||
err = enc.Encode(HeadsJsonEntry{
|
||||
Heads: z.heads,
|
||||
RootId: z.id,
|
||||
})
|
||||
return
|
||||
}
|
|
@ -1,7 +1,6 @@
|
|||
package debug
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
|
@ -11,14 +10,15 @@ import (
|
|||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/exporter"
|
||||
anystore "github.com/anyproto/any-store"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/debug/treearchive"
|
||||
"github.com/anyproto/anytype-heart/core/debug/exporter"
|
||||
"github.com/anyproto/anytype-heart/core/domain"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore"
|
||||
"github.com/anyproto/anytype-heart/util/anonymize"
|
||||
"github.com/anyproto/anytype-heart/util/pbtypes"
|
||||
"github.com/anyproto/anytype-heart/util/ziputil"
|
||||
)
|
||||
|
||||
type treeExporter struct {
|
||||
|
@ -26,66 +26,78 @@ type treeExporter struct {
|
|||
s objectstore.ObjectStore
|
||||
anonymized bool
|
||||
id domain.FullID
|
||||
zw *zip.Writer
|
||||
}
|
||||
|
||||
func (e *treeExporter) Export(ctx context.Context, path string, tree objecttree.ReadableObjectTree) (filename string, err error) {
|
||||
filename = filepath.Join(path, fmt.Sprintf("at.dbg.%s.%s.zip", e.id, time.Now().Format("20060102.150405.99")))
|
||||
archiveWriter, err := treearchive.NewArchiveWriter(filename)
|
||||
var (
|
||||
exportDirPath = filepath.Join(path, fmt.Sprintf("at.dbg.%s.%s", e.id, time.Now().Format("20060102.150405.99")))
|
||||
dbPath = filepath.Join(exportDirPath, "db")
|
||||
localStorePath = filepath.Join(exportDirPath, "localstore.json")
|
||||
logPath = filepath.Join(exportDirPath, "creation.log")
|
||||
)
|
||||
filename = exportDirPath + ".zip"
|
||||
err = os.MkdirAll(exportDirPath, 0755)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer archiveWriter.Close()
|
||||
|
||||
e.zw = archiveWriter.ZipWriter()
|
||||
params := exporter.TreeExporterParams{
|
||||
ListStorageExporter: archiveWriter,
|
||||
TreeStorageExporter: archiveWriter,
|
||||
DataConverter: &changeDataConverter{anonymize: e.anonymized},
|
||||
defer func() {
|
||||
_ = os.RemoveAll(exportDirPath)
|
||||
}()
|
||||
err = os.Mkdir(dbPath, 0755)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
anyStore, err := anystore.Open(ctx, dbPath, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
_ = anyStore.Close()
|
||||
}()
|
||||
exportParams := exporter.ExportParams{
|
||||
Readable: tree,
|
||||
Store: anyStore,
|
||||
Converter: &changeDataConverter{anonymize: e.anonymized},
|
||||
}
|
||||
anySyncExporter := exporter.NewTreeExporter(params)
|
||||
logBuf := bytes.NewBuffer(nil)
|
||||
|
||||
e.log = stdlog.New(io.MultiWriter(logBuf, os.Stderr), "", stdlog.LstdFlags)
|
||||
e.log.Printf("exporting tree and acl")
|
||||
st := time.Now()
|
||||
err = anySyncExporter.ExportUnencrypted(tree)
|
||||
err = exporter.ExportTree(ctx, exportParams)
|
||||
if err != nil {
|
||||
e.log.Printf("export tree in zip error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
err = anyStore.Checkpoint(ctx, true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
logBuf := bytes.NewBuffer(nil)
|
||||
e.log = stdlog.New(io.MultiWriter(logBuf, os.Stderr), "", stdlog.LstdFlags)
|
||||
e.log.Printf("exported tree for a %v", time.Since(st))
|
||||
data, err := e.s.SpaceIndex(e.id.SpaceID).GetInfosByIds([]string{e.id.ObjectID})
|
||||
if err != nil {
|
||||
e.log.Printf("can't fetch localstore info: %v", err)
|
||||
} else {
|
||||
if len(data) > 0 {
|
||||
data[0].Details = transform(data[0].Details, e.anonymized, anonymize.Details)
|
||||
// TODO: [storage] fix details, take from main
|
||||
// data[0].Details = transform(data[0].Details, e.anonymized, anonymize.Struct)
|
||||
data[0].Snippet = transform(data[0].Snippet, e.anonymized, anonymize.Text)
|
||||
for i, r := range data[0].Relations {
|
||||
data[0].Relations[i] = transform(r, e.anonymized, anonymize.Relation)
|
||||
}
|
||||
osData := pbtypes.Sprint(data[0].ToProto())
|
||||
lsWr, er := e.zw.Create("localstore.json")
|
||||
er := os.WriteFile(localStorePath, []byte(osData), 0600)
|
||||
if er != nil {
|
||||
e.log.Printf("create file in zip error: %v", er)
|
||||
e.log.Printf("localstore.json write error: %v", err)
|
||||
} else {
|
||||
if _, err := lsWr.Write([]byte(osData)); err != nil {
|
||||
e.log.Printf("localstore.json write error: %v", err)
|
||||
} else {
|
||||
e.log.Printf("localstore.json wrote")
|
||||
}
|
||||
e.log.Printf("localstore.json wrote")
|
||||
}
|
||||
} else {
|
||||
e.log.Printf("not data in objectstore")
|
||||
e.log.Printf("no data in objectstore")
|
||||
}
|
||||
}
|
||||
logW, err := e.zw.Create("creation.log")
|
||||
err = os.WriteFile(logPath, logBuf.Bytes(), 0600)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
io.Copy(logW, logBuf)
|
||||
err = ziputil.ZipFolder(exportDirPath, filename)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
@ -56,7 +56,6 @@ func DownloadManifest(url string, checkWhitelist bool) (info *model.ManifestInfo
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
schemaResp := schemaResponse{}
|
||||
err = json.Unmarshal(raw, &schemaResp)
|
||||
if err != nil {
|
||||
|
@ -119,7 +118,7 @@ func getRawJson(url string) ([]byte, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Close = true
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
|
@ -4,6 +4,7 @@ import (
|
|||
_ "embed"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
@ -11,8 +12,6 @@ import (
|
|||
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
|
||||
)
|
||||
|
||||
const port = ":7070"
|
||||
|
||||
//go:embed testdata/schema.json
|
||||
var schemaJSON []byte
|
||||
|
||||
|
@ -37,13 +36,13 @@ func TestIsInWhitelist(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestDownloadManifestAndValidateSchema(t *testing.T) {
|
||||
schema := schemaResponse{Schema: "http://localhost" + port + "/schema.json"}
|
||||
server := startHttpServer()
|
||||
defer server.Shutdown(nil)
|
||||
defer server.Close()
|
||||
schema := schemaResponse{Schema: server.URL + "/schema.json"}
|
||||
|
||||
t.Run("download knowledge base manifest", func(t *testing.T) {
|
||||
// given
|
||||
url := "http://localhost" + port + "/manifest.json"
|
||||
url := server.URL + "/manifest.json"
|
||||
|
||||
// when
|
||||
info, err := DownloadManifest(url, false)
|
||||
|
@ -116,7 +115,7 @@ func TestDownloadManifestAndValidateSchema(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func startHttpServer() *http.Server {
|
||||
func startHttpServer() *httptest.Server {
|
||||
handler := http.NewServeMux()
|
||||
handler.HandleFunc("/manifest.json", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
@ -128,9 +127,7 @@ func startHttpServer() *http.Server {
|
|||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write(schemaJSON)
|
||||
})
|
||||
server := &http.Server{Addr: port, Handler: handler}
|
||||
go server.ListenAndServe()
|
||||
return server
|
||||
return httptest.NewServer(handler)
|
||||
}
|
||||
|
||||
func buildInfo() *model.ManifestInfo {
|
||||
|
|
|
@ -45,7 +45,6 @@ func NewIndexerFixture(t *testing.T) *IndexerFixture {
|
|||
|
||||
walletService := mock_wallet.NewMockWallet(t)
|
||||
walletService.EXPECT().Name().Return(wallet.CName)
|
||||
|
||||
objectStore := objectstore.NewStoreFixture(t)
|
||||
clientStorage := mock_storage.NewMockClientStorage(t)
|
||||
|
||||
|
|
|
@ -143,7 +143,7 @@ func (i *indexer) Index(info smartblock.DocInfo, options ...smartblock.IndexOpti
|
|||
i.lock.Lock()
|
||||
spaceInd, ok := i.spaceIndexers[info.Space.Id()]
|
||||
if !ok {
|
||||
spaceInd = newSpaceIndexer(i.runCtx, i.store.SpaceIndex(info.Space.Id()), i.store, i.storageService)
|
||||
spaceInd = newSpaceIndexer(i.runCtx, i.store.SpaceIndex(info.Space.Id()), i.store)
|
||||
i.spaceIndexers[info.Space.Id()] = spaceInd
|
||||
}
|
||||
i.lock.Unlock()
|
||||
|
|
|
@ -5,7 +5,6 @@ import (
|
|||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/block/editor/smartblock"
|
||||
"github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest"
|
||||
|
@ -50,7 +49,6 @@ func TestIndexer(t *testing.T) {
|
|||
)))
|
||||
|
||||
smartTest.SetType(coresb.SmartBlockTypePage)
|
||||
indexerFx.storageServiceFx.EXPECT().BindSpaceID(mock.Anything, mock.Anything).Return(nil)
|
||||
indexerFx.store.SpaceIndex("spaceId1").SaveLastIndexedHeadsHash(ctx, "objectId1", "7f40bc2814f5297818461f889780a870ea033fe64c5a261117f2b662515a3dba")
|
||||
|
||||
// when
|
||||
|
@ -78,7 +76,6 @@ func TestIndexer(t *testing.T) {
|
|||
)))
|
||||
|
||||
smartTest.SetType(coresb.SmartBlockTypePage)
|
||||
indexerFx.storageServiceFx.EXPECT().BindSpaceID(mock.Anything, mock.Anything).Return(nil)
|
||||
indexerFx.store.SpaceIndex("spaceId1").SaveLastIndexedHeadsHash(ctx, "objectId1", "randomHash")
|
||||
|
||||
// when
|
||||
|
@ -107,7 +104,6 @@ func TestIndexer(t *testing.T) {
|
|||
)))
|
||||
|
||||
smartTest.SetType(coresb.SmartBlockTypePage)
|
||||
indexerFx.storageServiceFx.EXPECT().BindSpaceID(mock.Anything, mock.Anything).Return(nil)
|
||||
indexerFx.store.SpaceIndex("spaceId1").SaveLastIndexedHeadsHash(ctx, "objectId1", "7f40bc2814f5297818461f889780a870ea033fe64c5a261117f2b662515a3dba")
|
||||
|
||||
// when
|
||||
|
|
|
@ -7,6 +7,7 @@ import (
|
|||
"time"
|
||||
|
||||
anystore "github.com/anyproto/any-store"
|
||||
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/block/editor/smartblock"
|
||||
|
@ -51,7 +52,7 @@ const (
|
|||
)
|
||||
|
||||
type allDeletedIdsProvider interface {
|
||||
AllDeletedTreeIds() (ids []string, err error)
|
||||
AllDeletedTreeIds(ctx context.Context) (ids []string, err error)
|
||||
}
|
||||
|
||||
func (i *indexer) buildFlags(spaceID string) (reindexFlags, error) {
|
||||
|
@ -247,11 +248,7 @@ func (i *indexer) addSyncDetails(space clientspace.Space) {
|
|||
|
||||
func (i *indexer) reindexDeletedObjects(space clientspace.Space) error {
|
||||
store := i.store.SpaceIndex(space.Id())
|
||||
storage, ok := space.Storage().(allDeletedIdsProvider)
|
||||
if !ok {
|
||||
return fmt.Errorf("space storage doesn't implement allDeletedIdsProvider")
|
||||
}
|
||||
allIds, err := storage.AllDeletedTreeIds()
|
||||
allIds, err := space.Storage().AllDeletedTreeIds(i.runCtx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get deleted tree ids: %w", err)
|
||||
}
|
||||
|
@ -440,35 +437,34 @@ func (i *indexer) reindexIDs(ctx context.Context, space smartblock.Space, reinde
|
|||
|
||||
func (i *indexer) reindexOutdatedObjects(ctx context.Context, space clientspace.Space) (toReindex, success int, err error) {
|
||||
store := i.store.SpaceIndex(space.Id())
|
||||
tids := space.StoredIds()
|
||||
var entries []headstorage.HeadsEntry
|
||||
err = space.Storage().HeadStorage().IterateEntries(ctx, headstorage.IterOpts{}, func(entry headstorage.HeadsEntry) (bool, error) {
|
||||
// skipping Acl
|
||||
if entry.CommonSnapshot != "" {
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var idsToReindex []string
|
||||
for _, tid := range tids {
|
||||
for _, entry := range entries {
|
||||
id := entry.Id
|
||||
logErr := func(err error) {
|
||||
log.With("tree", tid).Errorf("reindexOutdatedObjects failed to get tree to reindex: %s", err)
|
||||
log.With("tree", entry.Id).Errorf("reindexOutdatedObjects failed to get tree to reindex: %s", err)
|
||||
}
|
||||
|
||||
lastHash, err := store.GetLastIndexedHeadsHash(ctx, tid)
|
||||
lastHash, err := store.GetLastIndexedHeadsHash(ctx, id)
|
||||
if err != nil {
|
||||
logErr(err)
|
||||
continue
|
||||
}
|
||||
info, err := space.Storage().TreeStorage(tid)
|
||||
if err != nil {
|
||||
logErr(err)
|
||||
continue
|
||||
}
|
||||
heads, err := info.Heads()
|
||||
if err != nil {
|
||||
logErr(err)
|
||||
continue
|
||||
}
|
||||
|
||||
hh := headsHash(heads)
|
||||
hh := headsHash(entry.Heads)
|
||||
if lastHash != hh {
|
||||
if lastHash != "" {
|
||||
log.With("tree", tid).Warnf("not equal indexed heads hash: %s!=%s (%d logs)", lastHash, hh, len(heads))
|
||||
log.With("tree", id).Warnf("not equal indexed heads hash: %s!=%s (%d logs)", lastHash, hh, len(entry.Heads))
|
||||
}
|
||||
idsToReindex = append(idsToReindex, tid)
|
||||
idsToReindex = append(idsToReindex, id)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -4,9 +4,12 @@ import (
|
|||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
|
||||
"github.com/anyproto/any-sync/commonspace/headsync/headstorage/mock_headstorage"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/mock/gomock"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/block/editor"
|
||||
"github.com/anyproto/anytype-heart/core/block/editor/smartblock/smarttest"
|
||||
|
@ -22,6 +25,7 @@ import (
|
|||
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
|
||||
"github.com/anyproto/anytype-heart/space/clientspace"
|
||||
mock_space "github.com/anyproto/anytype-heart/space/clientspace/mock_clientspace"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage/anystorage/mock_anystorage"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage/mock_storage"
|
||||
)
|
||||
|
||||
|
@ -54,7 +58,6 @@ func TestReindexMarketplaceSpace(t *testing.T) {
|
|||
virtualSpace := getMockSpace(indexerFx)
|
||||
|
||||
storage := mock_storage.NewMockClientStorage(t)
|
||||
storage.EXPECT().BindSpaceID(mock.Anything, mock.Anything).Return(nil)
|
||||
indexerFx.storageService = storage
|
||||
|
||||
// when
|
||||
|
@ -92,7 +95,6 @@ func TestReindexMarketplaceSpace(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
|
||||
storage := mock_storage.NewMockClientStorage(t)
|
||||
storage.EXPECT().BindSpaceID(mock.Anything, mock.Anything).Return(nil)
|
||||
fx.storageService = storage
|
||||
|
||||
// when
|
||||
|
@ -128,7 +130,6 @@ func TestReindexMarketplaceSpace(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
|
||||
storage := mock_storage.NewMockClientStorage(t)
|
||||
storage.EXPECT().BindSpaceID(mock.Anything, mock.Anything).Return(nil)
|
||||
fx.storageService = storage
|
||||
|
||||
fx.sourceFx.EXPECT().IDsListerBySmartblockType(mock.Anything, mock.Anything).Return(idsLister{Ids: []string{}}, nil).Maybe()
|
||||
|
@ -189,6 +190,15 @@ func TestIndexer_ReindexSpace_RemoveParticipants(t *testing.T) {
|
|||
err = fx.objectStore.SaveChecksums(spaceId2, &checksums)
|
||||
require.NoError(t, err)
|
||||
|
||||
ctrl := gomock.NewController(t)
|
||||
headStorage := mock_headstorage.NewMockHeadStorage(ctrl)
|
||||
storage := mock_anystorage.NewMockClientSpaceStorage(t)
|
||||
storage.EXPECT().HeadStorage().Return(headStorage)
|
||||
headStorage.EXPECT().IterateEntries(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().
|
||||
DoAndReturn(func(ctx context.Context, opts headstorage.IterOpts, entryIter headstorage.EntryIterator) error {
|
||||
return nil
|
||||
})
|
||||
|
||||
for _, space := range []string{spaceId1, spaceId2} {
|
||||
t.Run("reindex - participants deleted - when flag doesn't match", func(t *testing.T) {
|
||||
// given
|
||||
|
@ -196,7 +206,7 @@ func TestIndexer_ReindexSpace_RemoveParticipants(t *testing.T) {
|
|||
|
||||
spc := mock_space.NewMockSpace(t)
|
||||
spc.EXPECT().Id().Return(space)
|
||||
spc.EXPECT().StoredIds().Return([]string{}).Maybe()
|
||||
spc.EXPECT().Storage().Return(storage)
|
||||
fx.sourceFx.EXPECT().IDsListerBySmartblockType(mock.Anything, mock.Anything).Return(idsLister{Ids: []string{}}, nil).Maybe()
|
||||
|
||||
// when
|
||||
|
@ -275,12 +285,21 @@ func TestIndexer_ReindexSpace_EraseLinks(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
err = fx.objectStore.SaveChecksums(spaceId2, &checksums)
|
||||
require.NoError(t, err)
|
||||
ctrl := gomock.NewController(t)
|
||||
headStorage := mock_headstorage.NewMockHeadStorage(ctrl)
|
||||
storage := mock_anystorage.NewMockClientSpaceStorage(t)
|
||||
storage.EXPECT().HeadStorage().Return(headStorage)
|
||||
headStorage.EXPECT().IterateEntries(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().
|
||||
DoAndReturn(func(ctx context.Context, opts headstorage.IterOpts, entryIter headstorage.EntryIterator) error {
|
||||
return nil
|
||||
})
|
||||
|
||||
t.Run("links from archive and home are deleted", func(t *testing.T) {
|
||||
// given
|
||||
favs := []string{"fav1", "fav2"}
|
||||
trash := []string{"trash1", "trash2"}
|
||||
store := fx.store.SpaceIndex("space1")
|
||||
|
||||
err = store.UpdateObjectLinks(ctx, "home", favs)
|
||||
require.NoError(t, err)
|
||||
err = store.UpdateObjectLinks(ctx, "bin", trash)
|
||||
|
@ -294,7 +313,7 @@ func TestIndexer_ReindexSpace_EraseLinks(t *testing.T) {
|
|||
|
||||
space1 := mock_space.NewMockSpace(t)
|
||||
space1.EXPECT().Id().Return(spaceId1)
|
||||
space1.EXPECT().StoredIds().Return([]string{}).Maybe()
|
||||
space1.EXPECT().Storage().Return(storage)
|
||||
|
||||
// when
|
||||
err = fx.ReindexSpace(space1)
|
||||
|
@ -335,8 +354,7 @@ func TestIndexer_ReindexSpace_EraseLinks(t *testing.T) {
|
|||
|
||||
space1 := mock_space.NewMockSpace(t)
|
||||
space1.EXPECT().Id().Return(spaceId2)
|
||||
space1.EXPECT().StoredIds().Return([]string{}).Maybe()
|
||||
|
||||
space1.EXPECT().Storage().Return(storage)
|
||||
// when
|
||||
err = fx.ReindexSpace(space1)
|
||||
assert.NoError(t, err)
|
||||
|
|
|
@ -14,24 +14,21 @@ import (
|
|||
"github.com/anyproto/anytype-heart/core/block/editor/smartblock"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceindex"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage"
|
||||
)
|
||||
|
||||
type spaceIndexer struct {
|
||||
runCtx context.Context
|
||||
spaceIndex spaceindex.Store
|
||||
objectStore objectstore.ObjectStore
|
||||
storageService storage.ClientStorage
|
||||
batcher *mb.MB[indexTask]
|
||||
runCtx context.Context
|
||||
spaceIndex spaceindex.Store
|
||||
objectStore objectstore.ObjectStore
|
||||
batcher *mb.MB[indexTask]
|
||||
}
|
||||
|
||||
func newSpaceIndexer(runCtx context.Context, spaceIndex spaceindex.Store, objectStore objectstore.ObjectStore, storageService storage.ClientStorage) *spaceIndexer {
|
||||
func newSpaceIndexer(runCtx context.Context, spaceIndex spaceindex.Store, objectStore objectstore.ObjectStore) *spaceIndexer {
|
||||
ind := &spaceIndexer{
|
||||
runCtx: runCtx,
|
||||
spaceIndex: spaceIndex,
|
||||
objectStore: objectStore,
|
||||
storageService: storageService,
|
||||
batcher: mb.New[indexTask](100),
|
||||
runCtx: runCtx,
|
||||
spaceIndex: spaceIndex,
|
||||
objectStore: objectStore,
|
||||
batcher: mb.New[indexTask](100),
|
||||
}
|
||||
go ind.indexBatchLoop()
|
||||
return ind
|
||||
|
@ -123,7 +120,7 @@ func (i *spaceIndexer) index(ctx context.Context, info smartblock.DocInfo, optio
|
|||
for _, o := range options {
|
||||
o(opts)
|
||||
}
|
||||
err := i.storageService.BindSpaceID(info.Space.Id(), info.Id)
|
||||
err := i.objectStore.BindSpaceId(info.Space.Id(), info.Id)
|
||||
if err != nil {
|
||||
log.Error("failed to bind space id", zap.Error(err), zap.String("id", info.Id))
|
||||
return err
|
||||
|
|
|
@ -189,7 +189,7 @@ func (s *service) publishToPublishServer(ctx context.Context, spaceId, pageId, u
|
|||
return err
|
||||
}
|
||||
|
||||
version, err := s.evaluateDocumentVersion(spc, pageId, joinSpace)
|
||||
version, err := s.evaluateDocumentVersion(ctx, spc, pageId, joinSpace)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -380,12 +380,12 @@ func (s *service) extractInviteLink(ctx context.Context, spaceId string, joinSpa
|
|||
return inviteLink, nil
|
||||
}
|
||||
|
||||
func (s *service) evaluateDocumentVersion(spc clientspace.Space, pageId string, joinSpace bool) (string, error) {
|
||||
treeStorage, err := spc.Storage().TreeStorage(pageId)
|
||||
func (s *service) evaluateDocumentVersion(ctx context.Context, spc clientspace.Space, pageId string, joinSpace bool) (string, error) {
|
||||
treeStorage, err := spc.Storage().TreeStorage(ctx, pageId)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
heads, err := treeStorage.Heads()
|
||||
heads, err := treeStorage.Heads(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
|
@ -11,15 +11,17 @@ import (
|
|||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
anystore "github.com/anyproto/any-store"
|
||||
"github.com/anyproto/any-sync/app"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
|
||||
"github.com/anyproto/any-sync/consensus/consensusproto"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree/mock_objecttree"
|
||||
"github.com/anyproto/anytype-publish-server/publishclient/publishapi"
|
||||
"github.com/gogo/protobuf/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/mock/gomock"
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/anytype/account/mock_account"
|
||||
"github.com/anyproto/anytype-heart/core/block/cache/mock_cache"
|
||||
|
@ -46,6 +48,7 @@ import (
|
|||
"github.com/anyproto/anytype-heart/pkg/lib/threads"
|
||||
"github.com/anyproto/anytype-heart/space/clientspace/mock_clientspace"
|
||||
"github.com/anyproto/anytype-heart/space/mock_space"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage/anystorage/mock_anystorage"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/typeprovider/mock_typeprovider"
|
||||
"github.com/anyproto/anytype-heart/tests/testutil"
|
||||
"github.com/anyproto/anytype-heart/util/pbtypes"
|
||||
|
@ -135,7 +138,7 @@ func TestPublish(t *testing.T) {
|
|||
t.Run("success", func(t *testing.T) {
|
||||
// given
|
||||
isPersonal := true
|
||||
spaceService, err := prepaeSpaceService(t, isPersonal)
|
||||
spaceService, err := prepareSpaceService(t, isPersonal)
|
||||
|
||||
objectTypeId := "customObjectType"
|
||||
expectedUri := "test"
|
||||
|
@ -178,7 +181,7 @@ func TestPublish(t *testing.T) {
|
|||
t.Run("success with space sharing", func(t *testing.T) {
|
||||
// given
|
||||
isPersonal := false
|
||||
spaceService, err := prepaeSpaceService(t, isPersonal)
|
||||
spaceService, err := prepareSpaceService(t, isPersonal)
|
||||
|
||||
objectTypeId := "customObjectType"
|
||||
expectedUri := "test"
|
||||
|
@ -231,7 +234,7 @@ func TestPublish(t *testing.T) {
|
|||
})
|
||||
t.Run("success with space sharing - invite not exists", func(t *testing.T) {
|
||||
isPersonal := false
|
||||
spaceService, err := prepaeSpaceService(t, isPersonal)
|
||||
spaceService, err := prepareSpaceService(t, isPersonal)
|
||||
|
||||
objectTypeId := "customObjectType"
|
||||
expectedUri := "test"
|
||||
|
@ -279,7 +282,7 @@ func TestPublish(t *testing.T) {
|
|||
t.Run("success for member", func(t *testing.T) {
|
||||
// given
|
||||
isPersonal := false
|
||||
spaceService, err := prepaeSpaceService(t, isPersonal)
|
||||
spaceService, err := prepareSpaceService(t, isPersonal)
|
||||
|
||||
objectTypeId := "customObjectType"
|
||||
expectedUri := "test"
|
||||
|
@ -337,7 +340,7 @@ func TestPublish(t *testing.T) {
|
|||
t.Run("internal error", func(t *testing.T) {
|
||||
// given
|
||||
isPersonal := true
|
||||
spaceService, err := prepaeSpaceService(t, isPersonal)
|
||||
spaceService, err := prepareSpaceService(t, isPersonal)
|
||||
|
||||
objectTypeId := "customObjectType"
|
||||
expectedUri := "test"
|
||||
|
@ -660,28 +663,22 @@ func TestService_PublishingList(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func prepaeSpaceService(t *testing.T, isPersonal bool) (*mock_space.MockService, error) {
|
||||
var ctx = context.Background()
|
||||
|
||||
func prepareSpaceService(t *testing.T, isPersonal bool) (*mock_space.MockService, error) {
|
||||
spaceService := mock_space.NewMockService(t)
|
||||
space := mock_clientspace.NewMockSpace(t)
|
||||
ctrl := gomock.NewController(t)
|
||||
space.EXPECT().IsPersonal().Return(isPersonal)
|
||||
space.EXPECT().Id().Return(spaceId)
|
||||
|
||||
storage, err := spacestorage.NewInMemorySpaceStorage(spacestorage.SpaceStorageCreatePayload{
|
||||
AclWithId: &consensusproto.RawRecordWithId{Id: "aclId"},
|
||||
SpaceHeaderWithId: &spacesyncproto.RawSpaceHeaderWithId{Id: spaceId},
|
||||
SpaceSettingsWithId: &treechangeproto.RawTreeChangeWithId{Id: "settingsId"},
|
||||
},
|
||||
)
|
||||
assert.NoError(t, err)
|
||||
objectHeads := []string{"heads"}
|
||||
_, err = storage.CreateTreeStorage(treestorage.TreeStorageCreatePayload{
|
||||
RootRawChange: &treechangeproto.RawTreeChangeWithId{Id: objectId},
|
||||
Heads: objectHeads,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
space.EXPECT().Storage().Return(storage)
|
||||
st := mock_anystorage.NewMockClientSpaceStorage(t)
|
||||
mockSt := mock_objecttree.NewMockStorage(ctrl)
|
||||
st.EXPECT().TreeStorage(mock.Anything, mock.Anything).Return(mockSt, nil)
|
||||
mockSt.EXPECT().Heads(gomock.Any()).Return([]string{"heads"}, nil)
|
||||
space.EXPECT().Storage().Return(st)
|
||||
spaceService.EXPECT().Get(context.Background(), spaceId).Return(space, nil)
|
||||
return spaceService, err
|
||||
return spaceService, nil
|
||||
}
|
||||
|
||||
func prepareExporter(t *testing.T, objectTypeId string, spaceService *mock_space.MockService) export.Export {
|
||||
|
@ -895,3 +892,22 @@ func createTestFile(fileName string, size int64) error {
|
|||
file.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
func createStore(ctx context.Context, t testing.TB) anystore.DB {
|
||||
return createNamedStore(ctx, t, "changes.db")
|
||||
}
|
||||
|
||||
func createNamedStore(ctx context.Context, t testing.TB, name string) anystore.DB {
|
||||
path := filepath.Join(t.TempDir(), name)
|
||||
db, err := anystore.Open(ctx, path, nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
err := db.Close()
|
||||
require.NoError(t, err)
|
||||
unix.Rmdir(path)
|
||||
})
|
||||
return objecttree.TestStore{
|
||||
DB: db,
|
||||
Path: path,
|
||||
}
|
||||
}
|
||||
|
|
|
@ -33,7 +33,7 @@ const CName = "subscription"
|
|||
|
||||
var log = logging.Logger("anytype-mw-subscription")
|
||||
|
||||
var batchTime = 50 * time.Millisecond
|
||||
var batchTime = 250 * time.Millisecond
|
||||
|
||||
func New() Service {
|
||||
return &service{}
|
||||
|
|
|
@ -94,7 +94,7 @@ func (s *syncStatusService) Init(a *app.App) (err error) {
|
|||
s.updateIntervalSecs = syncUpdateInterval
|
||||
s.updateTimeout = syncTimeout
|
||||
s.spaceId = sharedState.SpaceId
|
||||
s.spaceSettingsId = spaceStorage.SpaceSettingsId()
|
||||
s.spaceSettingsId = spaceStorage.StateStorage().SettingsId()
|
||||
s.periodicSync = periodicsync.NewPeriodicSync(
|
||||
s.updateIntervalSecs,
|
||||
s.updateTimeout,
|
||||
|
|
|
@ -5,6 +5,7 @@ import (
|
|||
"testing"
|
||||
|
||||
"github.com/anyproto/any-sync/app"
|
||||
"github.com/anyproto/any-sync/commonspace/headsync/statestorage/mock_statestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestate"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/mock_spacestorage"
|
||||
"github.com/anyproto/any-sync/nodeconf/mock_nodeconf"
|
||||
|
@ -174,7 +175,9 @@ func newFixture(t *testing.T, spaceId string) *fixture {
|
|||
ctrl := gomock.NewController(t)
|
||||
service := mock_nodeconf.NewMockService(ctrl)
|
||||
storage := mock_spacestorage.NewMockSpaceStorage(ctrl)
|
||||
storage.EXPECT().SpaceSettingsId().Return(testSpaceSettingsId)
|
||||
stateStorage := mock_statestorage.NewMockStateStorage(ctrl)
|
||||
storage.EXPECT().StateStorage().AnyTimes().Return(stateStorage)
|
||||
stateStorage.EXPECT().SettingsId().AnyTimes().Return(testSpaceSettingsId)
|
||||
|
||||
spaceState := &spacestate.SpaceState{SpaceId: spaceId}
|
||||
config := &config.Config{}
|
||||
|
|
|
@ -44,7 +44,7 @@ func TestSyncSubscriptions(t *testing.T) {
|
|||
subs.(*syncSubscriptions).service = testSubs
|
||||
err := subs.Run(context.Background())
|
||||
require.NoError(t, err)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
spaceSub, err := subs.GetSubscription("spaceId")
|
||||
require.NoError(t, err)
|
||||
syncCnt := spaceSub.SyncingObjectsCount([]string{"1", "2"})
|
||||
|
@ -59,7 +59,7 @@ func TestSyncSubscriptions(t *testing.T) {
|
|||
objects[i][bundle.RelationKeySyncStatus] = domain.Int64(int64(domain.ObjectSyncStatusSynced))
|
||||
testSubs.AddObjects(t, "spaceId", []objectstore.TestObject{objects[i]})
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
syncCnt = spaceSub.SyncingObjectsCount([]string{"1", "2"})
|
||||
require.Equal(t, 2, syncCnt)
|
||||
err = subs.Close(context.Background())
|
||||
|
|
19
core/util.go
19
core/util.go
|
@ -24,10 +24,16 @@ func (mw *Middleware) getResponseEvent(ctx session.Context) *pb.ResponseEvent {
|
|||
type errToCodeTuple[T ~int32] struct {
|
||||
err error
|
||||
code T
|
||||
|
||||
checkErrorType any
|
||||
}
|
||||
|
||||
func errToCode[T ~int32](err error, code T) errToCodeTuple[T] {
|
||||
return errToCodeTuple[T]{err, code}
|
||||
return errToCodeTuple[T]{err: err, code: code}
|
||||
}
|
||||
|
||||
func errTypeToCode[T ~int32](errTypeProto any, code T) errToCodeTuple[T] {
|
||||
return errToCodeTuple[T]{code: code, checkErrorType: errTypeProto}
|
||||
}
|
||||
|
||||
func mapErrorCode[T ~int32](err error, mappings ...errToCodeTuple[T]) T {
|
||||
|
@ -35,8 +41,15 @@ func mapErrorCode[T ~int32](err error, mappings ...errToCodeTuple[T]) T {
|
|||
return 0
|
||||
}
|
||||
for _, m := range mappings {
|
||||
if errors.Is(err, m.err) {
|
||||
return m.code
|
||||
if m.err != nil {
|
||||
if errors.Is(err, m.err) {
|
||||
return m.code
|
||||
}
|
||||
}
|
||||
if m.checkErrorType != nil {
|
||||
if errors.As(err, m.checkErrorType) {
|
||||
return m.code
|
||||
}
|
||||
}
|
||||
}
|
||||
// Unknown error
|
||||
|
|
|
@ -8,12 +8,20 @@ import (
|
|||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
type testErrorType struct {
|
||||
}
|
||||
|
||||
func (t testErrorType) Error() string {
|
||||
return "error type!"
|
||||
}
|
||||
|
||||
type testCode int32
|
||||
|
||||
func TestErrorCodeMapping(t *testing.T) {
|
||||
err1 := errors.New("err1")
|
||||
err2 := errors.New("err2")
|
||||
err3 := errors.New("err3")
|
||||
err4 := testErrorType{}
|
||||
|
||||
wrapped1 := errors.Join(err1, fmt.Errorf("description of error"))
|
||||
wrapped2 := fmt.Errorf("description of error: %w", err2)
|
||||
|
@ -23,12 +31,14 @@ func TestErrorCodeMapping(t *testing.T) {
|
|||
errToCode(err1, testCode(2)),
|
||||
errToCode(err2, testCode(3)),
|
||||
errToCode(err3, testCode(4)),
|
||||
errTypeToCode(&testErrorType{}, testCode(5)),
|
||||
)
|
||||
}
|
||||
|
||||
assert.True(t, 0 == mapper(nil))
|
||||
assert.True(t, 1 == mapper(errors.New("unknown error")))
|
||||
assert.True(t, 2 == mapper(wrapped1))
|
||||
assert.True(t, 3 == mapper(wrapped2))
|
||||
assert.True(t, 4 == mapper(err3))
|
||||
assert.Equal(t, testCode(0), mapper(nil))
|
||||
assert.Equal(t, testCode(1), mapper(errors.New("unknown error")))
|
||||
assert.Equal(t, testCode(2), mapper(wrapped1))
|
||||
assert.Equal(t, testCode(3), mapper(wrapped2))
|
||||
assert.Equal(t, testCode(4), mapper(err3))
|
||||
assert.Equal(t, testCode(5), mapper(err4))
|
||||
}
|
||||
|
|
23
go.mod
23
go.mod
|
@ -8,7 +8,7 @@ require (
|
|||
github.com/VividCortex/ewma v1.2.0
|
||||
github.com/adrium/goheif v0.0.0-20230113233934-ca402e77a786
|
||||
github.com/anyproto/any-store v0.1.8
|
||||
github.com/anyproto/any-sync v0.5.26
|
||||
github.com/anyproto/any-sync v0.6.1
|
||||
github.com/anyproto/anytype-publish-server/publishclient v0.0.0-20250131145601-de288583ff2a
|
||||
github.com/anyproto/go-chash v0.1.0
|
||||
github.com/anyproto/go-naturaldate/v2 v2.0.2-0.20230524105841-9829cfd13438
|
||||
|
@ -17,7 +17,7 @@ require (
|
|||
github.com/anyproto/protobuf v1.3.3-0.20240814124528-72b8c7e0e0f5
|
||||
github.com/anyproto/tantivy-go v0.3.1
|
||||
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
|
||||
github.com/avast/retry-go/v4 v4.6.1
|
||||
github.com/avast/retry-go/v4 v4.6.0
|
||||
github.com/chai2010/webp v1.1.2-0.20240612091223-aa1b379218b7
|
||||
github.com/cheggaaa/mb/v3 v3.0.2
|
||||
github.com/dave/jennifer v1.7.1
|
||||
|
@ -51,7 +51,7 @@ require (
|
|||
github.com/hbagdi/go-unsplash v0.0.0-20230414214043-474fc02c9119
|
||||
github.com/huandu/skiplist v1.2.1
|
||||
github.com/improbable-eng/grpc-web v0.15.0
|
||||
github.com/ipfs/boxo v0.27.4
|
||||
github.com/ipfs/boxo v0.28.0
|
||||
github.com/ipfs/go-block-format v0.2.0
|
||||
github.com/ipfs/go-cid v0.5.0
|
||||
github.com/ipfs/go-datastore v0.8.1
|
||||
|
@ -107,7 +107,8 @@ require (
|
|||
golang.org/x/image v0.24.0
|
||||
golang.org/x/mobile v0.0.0-20241108191957-fa514ef75a0f
|
||||
golang.org/x/net v0.35.0
|
||||
golang.org/x/oauth2 v0.27.0
|
||||
golang.org/x/oauth2 v0.26.0
|
||||
golang.org/x/sys v0.30.0
|
||||
golang.org/x/text v0.22.0
|
||||
google.golang.org/grpc v1.70.0
|
||||
gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20180125164251-1832d8546a9f
|
||||
|
@ -115,7 +116,6 @@ require (
|
|||
gopkg.in/yaml.v3 v3.0.1
|
||||
storj.io/drpc v0.0.34
|
||||
zombiezen.com/go/sqlite v1.4.0
|
||||
|
||||
)
|
||||
|
||||
require (
|
||||
|
@ -188,7 +188,7 @@ require (
|
|||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/google/flatbuffers v1.12.1 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20250202011525-fc3143867406 // indirect
|
||||
github.com/google/pprof v0.0.0-20250208200701-d0013a598941 // indirect
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect
|
||||
github.com/gorilla/css v1.0.1 // indirect
|
||||
github.com/gosimple/unidecode v1.0.1 // indirect
|
||||
|
@ -215,7 +215,7 @@ require (
|
|||
github.com/klauspost/cpuid/v2 v2.2.9 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
|
||||
github.com/libp2p/go-libp2p v0.39.0 // indirect
|
||||
github.com/libp2p/go-libp2p v0.40.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/mailru/easyjson v0.7.6 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
|
@ -247,7 +247,7 @@ require (
|
|||
github.com/prometheus/common v0.62.0 // indirect
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
github.com/pseudomuto/protokit v0.2.1 // indirect
|
||||
github.com/quic-go/quic-go v0.49.0 // indirect
|
||||
github.com/quic-go/quic-go v0.50.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rs/cors v1.11.0 // indirect
|
||||
github.com/rs/zerolog v1.29.0 // indirect
|
||||
|
@ -279,16 +279,15 @@ require (
|
|||
go.opentelemetry.io/otel/metric v1.34.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.34.0 // indirect
|
||||
golang.org/x/arch v0.8.0 // indirect
|
||||
golang.org/x/crypto v0.33.0 // indirect
|
||||
golang.org/x/crypto v0.35.0 // indirect
|
||||
golang.org/x/mod v0.23.0 // indirect
|
||||
golang.org/x/sync v0.11.0 // indirect
|
||||
golang.org/x/sys v0.30.0 // indirect
|
||||
golang.org/x/term v0.29.0 // indirect
|
||||
golang.org/x/time v0.10.0 // indirect
|
||||
golang.org/x/tools v0.30.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a // indirect
|
||||
google.golang.org/protobuf v1.36.4 // indirect
|
||||
google.golang.org/protobuf v1.36.5 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
lukechampine.com/blake3 v1.3.0 // indirect
|
||||
|
@ -322,3 +321,5 @@ replace google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/
|
|||
replace github.com/btcsuite/btcutil => github.com/btcsuite/btcd/btcutil v1.1.5
|
||||
|
||||
replace github.com/dsoprea/go-jpeg-image-structure/v2 => github.com/dchesterton/go-jpeg-image-structure/v2 v2.0.0-20240318203529-c3eea088bd38
|
||||
|
||||
replace zombiezen.com/go/sqlite => github.com/anyproto/go-sqlite v0.0.0-20250226111550-9b81a8e3cff4
|
||||
|
|
58
go.sum
58
go.sum
|
@ -84,8 +84,8 @@ github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kk
|
|||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/anyproto/any-store v0.1.8 h1:/bxUVq6sBTwYkmPL2g1xUAWNb3axF+zPhP2dvdEBH68=
|
||||
github.com/anyproto/any-store v0.1.8/go.mod h1:GpnVhcGm5aUQtOwCnKeTt4jsWgVXZ773WbQVLFdeCFo=
|
||||
github.com/anyproto/any-sync v0.5.26 h1:JWcR/RFGQ22CYWrdh2Ain6uEWNePtYHCLs+LXsm8hhU=
|
||||
github.com/anyproto/any-sync v0.5.26/go.mod h1:Ljftoz6/mCM/2wP2tK9H1/jtVAxfgqzYplBA4MbiUs0=
|
||||
github.com/anyproto/any-sync v0.6.1 h1:Dasbp7qGQme8diGGpzaDQfSDs5o7PAK3E5rxHHrB/+4=
|
||||
github.com/anyproto/any-sync v0.6.1/go.mod h1:5js8TNBdqe75zwlr9XEQSVDtwhsvEU2qLeC2wTnT/Fo=
|
||||
github.com/anyproto/anytype-publish-server/publishclient v0.0.0-20250131145601-de288583ff2a h1:ZZM+0OUCQMWSLSflpkf0ZMVo3V76qEDDIXPpQOClNs0=
|
||||
github.com/anyproto/anytype-publish-server/publishclient v0.0.0-20250131145601-de288583ff2a/go.mod h1:4fkueCZcGniSMXkrwESO8zzERrh/L7WHimRNWecfGM0=
|
||||
github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580 h1:Ba80IlCCxkZ9H1GF+7vFu/TSpPvbpDCxXJ5ogc4euYc=
|
||||
|
@ -104,6 +104,8 @@ github.com/anyproto/go-slip10 v1.0.0 h1:uAEtSuudR3jJBOfkOXf3bErxVoxbuKwdoJN55M1i
|
|||
github.com/anyproto/go-slip10 v1.0.0/go.mod h1:BCmIlM1KB8wX6K4/8pOvxPl9oVKfEvZ5vsmO5rkK6vg=
|
||||
github.com/anyproto/go-slip21 v1.0.0 h1:CI7lUqTIwmPOEGVAj4jyNLoICvueh++0U2HoAi3m2ZY=
|
||||
github.com/anyproto/go-slip21 v1.0.0/go.mod h1:gbIJt7HAdr5DuT4f2pFTKCBSUWYsm/fysHBNqgsuxT0=
|
||||
github.com/anyproto/go-sqlite v0.0.0-20250226111550-9b81a8e3cff4 h1:HzVjm45VOUVFUrxh2s0cRR4lqfCr/VAee6wNzPLcApI=
|
||||
github.com/anyproto/go-sqlite v0.0.0-20250226111550-9b81a8e3cff4/go.mod h1:0w9F1DN9IZj9AcLS9YDKMboubCACkwYCGkzoy3eG5ik=
|
||||
github.com/anyproto/html-to-markdown v0.0.0-20231025221133-830bf0a6f139 h1:Wp9z0Q2kAstznWUmTZyOb9UgpVmUgYt1LXRvK/cg10E=
|
||||
github.com/anyproto/html-to-markdown v0.0.0-20231025221133-830bf0a6f139/go.mod h1:1zaDDQVWTRwNksmTUTkcVXqgNF28YHiEUIm8FL9Z+II=
|
||||
github.com/anyproto/lexid v0.0.4 h1:2ztI0y5pNdtojd3vChw/YV/P6IO9pB7PccYysImDxWI=
|
||||
|
@ -124,8 +126,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
|
|||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
|
||||
github.com/avast/retry-go/v4 v4.6.1 h1:VkOLRubHdisGrHnTu89g08aQEWEgRU7LVEop3GbIcMk=
|
||||
github.com/avast/retry-go/v4 v4.6.1/go.mod h1:V6oF8njAwxJ5gRo1Q7Cxab24xs5NCWZBeaHHBklR8mA=
|
||||
github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA=
|
||||
github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE=
|
||||
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
|
||||
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
|
||||
|
@ -483,8 +485,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
|
|||
github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20250202011525-fc3143867406 h1:wlQI2cYY0BsWmmPPAnxfQ8SDW0S3Jasn+4B8kXFxprg=
|
||||
github.com/google/pprof v0.0.0-20250202011525-fc3143867406/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
|
||||
github.com/google/pprof v0.0.0-20250208200701-d0013a598941 h1:43XjGa6toxLpeksjcxs1jIoIyr+vUfOqY2c6HB4bpoc=
|
||||
github.com/google/pprof v0.0.0-20250208200701-d0013a598941/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
|
@ -579,8 +581,8 @@ github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf
|
|||
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
|
||||
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
|
||||
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
|
||||
github.com/ipfs/boxo v0.27.4 h1:6nC8lY5GnR6whAbW88hFz6L13wZUj2vr5BRe3iTvYBI=
|
||||
github.com/ipfs/boxo v0.27.4/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM=
|
||||
github.com/ipfs/boxo v0.28.0 h1:io6nXqN8XMOstB7dQGG5GWnMk4WssoMvva9OADErZdI=
|
||||
github.com/ipfs/boxo v0.28.0/go.mod h1:eY9w3iTpmZGKzDfEYjm3oK8f+xjv8KJhhNXJwicmd3I=
|
||||
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
|
||||
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
|
||||
github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
|
||||
|
@ -687,12 +689,12 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
|
|||
github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
|
||||
github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw=
|
||||
github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:st3qqfu8+pMfh+9Mzqb2GTiwrAGjIPszEjZmtksN8Jc=
|
||||
github.com/libp2p/go-libp2p v0.39.0 h1:LmrhDRud4eDkQCSB4l5NfoIFDqvDwAyANmfeYkgnKgs=
|
||||
github.com/libp2p/go-libp2p v0.39.0/go.mod h1:3zicI8Lp7Isun+Afo/JOACUbbJqqR2owK6RQWFsVAbI=
|
||||
github.com/libp2p/go-libp2p v0.40.0 h1:1LOMO3gigxeXFs50HGEc1U79OINewUQB7o4gTKGPC3U=
|
||||
github.com/libp2p/go-libp2p v0.40.0/go.mod h1:hOzj2EAIYsXpVpBnyA1pRHzpUJGF9nbWiDLjgasnbF0=
|
||||
github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94=
|
||||
github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8=
|
||||
github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0=
|
||||
github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk=
|
||||
github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg=
|
||||
github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E=
|
||||
github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA=
|
||||
github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg=
|
||||
github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0=
|
||||
|
@ -701,8 +703,8 @@ github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk=
|
|||
github.com/libp2p/go-nat v0.2.0/go.mod h1:3MJr+GRpRkyT65EpVPBstXLvOlAPzUVlG6Pwg9ohLJk=
|
||||
github.com/libp2p/go-netroute v0.2.2 h1:Dejd8cQ47Qx2kRABg6lPwknU7+nBnFRpko45/fFPuZ8=
|
||||
github.com/libp2p/go-netroute v0.2.2/go.mod h1:Rntq6jUAH0l9Gg17w5bFGhcC9a+vk4KNXs6s7IljKYE=
|
||||
github.com/libp2p/go-yamux/v4 v4.0.2 h1:nrLh89LN/LEiqcFiqdKDRHjGstN300C1269K/EX0CPU=
|
||||
github.com/libp2p/go-yamux/v4 v4.0.2/go.mod h1:C808cCRgOs1iBwY4S71T5oxgMxgLmqUw56qh4AeBW2o=
|
||||
github.com/libp2p/go-yamux/v5 v5.0.0 h1:2djUh96d3Jiac/JpGkKs4TO49YhsfLopAoryfPmf+Po=
|
||||
github.com/libp2p/go-yamux/v5 v5.0.0/go.mod h1:en+3cdX51U0ZslwRdRLrvQsdayFt3TSUKvBGErzpWbU=
|
||||
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
|
||||
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
|
||||
|
@ -867,16 +869,12 @@ github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk=
|
|||
github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE=
|
||||
github.com/pion/dtls/v3 v3.0.4 h1:44CZekewMzfrn9pmGrj5BNnTMDCFwr+6sLH+cCuLM7U=
|
||||
github.com/pion/dtls/v3 v3.0.4/go.mod h1:R373CsjxWqNPf6MEkfdy3aSe9niZvL/JaKlGeFphtMg=
|
||||
github.com/pion/ice/v2 v2.3.37 h1:ObIdaNDu1rCo7hObhs34YSBcO7fjslJMZV0ux+uZWh0=
|
||||
github.com/pion/ice/v2 v2.3.37/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ=
|
||||
github.com/pion/ice/v4 v4.0.6 h1:jmM9HwI9lfetQV/39uD0nY4y++XZNPhvzIPCb8EwxUM=
|
||||
github.com/pion/ice/v4 v4.0.6/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw=
|
||||
github.com/pion/interceptor v0.1.37 h1:aRA8Zpab/wE7/c0O3fh1PqY0AJI3fCSEM5lRWJVorwI=
|
||||
github.com/pion/interceptor v0.1.37/go.mod h1:JzxbJ4umVTlZAf+/utHzNesY8tmRkM2lVmkS82TTj8Y=
|
||||
github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI=
|
||||
github.com/pion/logging v0.2.3/go.mod h1:z8YfknkquMe1csOrxK5kc+5/ZPAzMxbKLX5aXpbpC90=
|
||||
github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8=
|
||||
github.com/pion/mdns v0.0.12/go.mod h1:VExJjv8to/6Wqm1FXK+Ii/Z9tsVk/F5sD/N70cnYFbk=
|
||||
github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM=
|
||||
github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA=
|
||||
github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
|
||||
|
@ -899,12 +897,10 @@ github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQp
|
|||
github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
|
||||
github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0=
|
||||
github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo=
|
||||
github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc=
|
||||
github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
|
||||
github.com/pion/turn/v4 v4.0.0 h1:qxplo3Rxa9Yg1xXDxxH8xaqcyGUtbHYw4QSCvmFWvhM=
|
||||
github.com/pion/turn/v4 v4.0.0/go.mod h1:MuPDkm15nYSklKpN8vWJ9W2M0PlyQZqYt1McGuxG7mA=
|
||||
github.com/pion/webrtc/v4 v4.0.8 h1:T1ZmnT9qxIJIt4d8XoiMOBrTClGHDDXNg9e/fh018Qc=
|
||||
github.com/pion/webrtc/v4 v4.0.8/go.mod h1:HHBeUVBAC+j4ZFnYhovEFStF02Arb1EyD4G7e7HBTJw=
|
||||
github.com/pion/webrtc/v4 v4.0.9 h1:PyOYMRKJgfy0dzPcYtFD/4oW9zaw3Ze3oZzzbj2LV9E=
|
||||
github.com/pion/webrtc/v4 v4.0.9/go.mod h1:ViHLVaNpiuvaH8pdiuQxuA9awuE6KVzAXx3vVWilOck=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
|
@ -954,8 +950,8 @@ github.com/pseudomuto/protokit v0.2.1 h1:kCYpE3thoR6Esm0CUvd5xbrDTOZPvQPTDeyXpZf
|
|||
github.com/pseudomuto/protokit v0.2.1/go.mod h1:gt7N5Rz2flBzYafvaxyIxMZC0TTF5jDZfRnw25hAAyo=
|
||||
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
|
||||
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
|
||||
github.com/quic-go/quic-go v0.49.0 h1:w5iJHXwHxs1QxyBv1EHKuC50GX5to8mJAxvtnttJp94=
|
||||
github.com/quic-go/quic-go v0.49.0/go.mod h1:s2wDnmCdooUQBmQfpUSTCYBl1/D4FcqbULMMkASvR6s=
|
||||
github.com/quic-go/quic-go v0.50.0 h1:3H/ld1pa3CYhkcc20TPIyG1bNsdhn9qZBGN3b9/UyUo=
|
||||
github.com/quic-go/quic-go v0.50.0/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E=
|
||||
github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg=
|
||||
github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
|
@ -1194,8 +1190,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
|
|||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
|
||||
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
|
||||
golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
|
||||
golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
|
@ -1322,8 +1318,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ
|
|||
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M=
|
||||
golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
|
||||
golang.org/x/oauth2 v0.26.0 h1:afQXWNNaeC4nvZ0Ed9XvCCzXM6UHJG7iCg0W4fPqSBE=
|
||||
golang.org/x/oauth2 v0.26.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
|
@ -1644,8 +1640,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
|
|||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM=
|
||||
google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
|
||||
google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
|
||||
google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
@ -1725,5 +1721,3 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
|
|||
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
|
||||
storj.io/drpc v0.0.34 h1:q9zlQKfJ5A7x8NQNFk8x7eKUF78FMhmAbZLnFK+og7I=
|
||||
storj.io/drpc v0.0.34/go.mod h1:Y9LZaa8esL1PW2IDMqJE7CFSNq7d5bQ3RI7mGPtmKMg=
|
||||
zombiezen.com/go/sqlite v1.4.0 h1:N1s3RIljwtp4541Y8rM880qgGIgq3fTD2yks1xftnKU=
|
||||
zombiezen.com/go/sqlite v1.4.0/go.mod h1:0w9F1DN9IZj9AcLS9YDKMboubCACkwYCGkzoy3eG5ik=
|
||||
|
|
|
@ -7,8 +7,8 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/cheggaaa/mb/v3"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/valyala/fastjson"
|
||||
|
||||
"github.com/anyproto/anytype-heart/metrics/anymetry"
|
||||
|
@ -79,19 +79,19 @@ func TestClient_SendEvents(t *testing.T) {
|
|||
go c.startSendingBatchMessages(&testAppInfoProvider{})
|
||||
|
||||
c.send(&testEvent{})
|
||||
time.Sleep(1 * time.Millisecond)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
assert.Equal(t, 1, c.batcher.Len())
|
||||
require.Equal(t, 1, c.batcher.Len())
|
||||
telemetry.AssertNotCalled(t, "SendEvents", mock.Anything, mock.Anything)
|
||||
|
||||
c.send(&testEvent{})
|
||||
time.Sleep(1 * time.Millisecond)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
mutex.Lock()
|
||||
assert.Equal(t, 0, c.batcher.Len())
|
||||
assert.Equal(t, 2, len(events))
|
||||
require.Equal(t, 0, c.batcher.Len())
|
||||
require.Equal(t, 2, len(events))
|
||||
mutex.Unlock()
|
||||
|
||||
assert.True(t, events[0].GetTimestamp() > 0)
|
||||
require.True(t, events[0].GetTimestamp() > 0)
|
||||
telemetry.AssertCalled(t, "SendEvents", mock.Anything, mock.Anything)
|
||||
}
|
||||
|
|
|
@ -732,7 +732,7 @@ message Event {
|
|||
FaviconHash faviconHash = 6;
|
||||
Type type = 7;
|
||||
TargetObjectId targetObjectId = 8;
|
||||
|
||||
|
||||
|
||||
message Url {
|
||||
string value = 1;
|
||||
|
|
|
@ -170,10 +170,13 @@ func (r *clientds) Init(a *app.App) (err error) {
|
|||
r.localstoreDS, err = openBadgerWithRecover(opts)
|
||||
err = anyerror.CleanupError(err)
|
||||
if err != nil && isBadgerCorrupted(err) {
|
||||
log.With("error", err).Error("badger db is corrupted")
|
||||
// because localstore contains mostly recoverable info (with th only exception of objects' lastOpenedDate)
|
||||
// we can just remove and recreate it
|
||||
err2 := os.Rename(opts.Dir, filepath.Join(opts.Dir, "-corrupted"))
|
||||
log.Errorf("failed to rename corrupted localstore: %s", err2)
|
||||
err2 := os.Rename(opts.Dir, opts.Dir+"-corrupted")
|
||||
if err2 != nil {
|
||||
log.Errorf("failed to rename corrupted localstore: %s", err2)
|
||||
}
|
||||
var errAfterRemove error
|
||||
r.localstoreDS, errAfterRemove = openBadgerWithRecover(opts)
|
||||
errAfterRemove = anyerror.CleanupError(errAfterRemove)
|
||||
|
|
|
@ -75,16 +75,14 @@ func (w *walletStub) Name() string { return wallet.CName }
|
|||
func NewStoreFixture(t testing.TB) *StoreFixture {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
walletService := newWalletStub(t)
|
||||
|
||||
fullText := ftsearch.TantivyNew()
|
||||
testApp := &app.App{}
|
||||
|
||||
dataStore, err := datastore.NewInMemory()
|
||||
require.NoError(t, err)
|
||||
|
||||
testApp.Register(newWalletStub(t))
|
||||
testApp.Register(dataStore)
|
||||
testApp.Register(walletService)
|
||||
err = fullText.Init(testApp)
|
||||
require.NoError(t, err)
|
||||
err = fullText.Run(context.Background())
|
||||
|
@ -100,7 +98,7 @@ func NewStoreFixture(t testing.TB) *StoreFixture {
|
|||
fts: fullText,
|
||||
sourceService: &detailsFromId{},
|
||||
arenaPool: &anyenc.ArenaPool{},
|
||||
repoPath: walletService.RepoPath(),
|
||||
objectStorePath: t.TempDir(),
|
||||
oldStore: oldStore,
|
||||
spaceIndexes: map[string]spaceindex.Store{},
|
||||
techSpaceIdProvider: &stubTechSpaceIdProvider{},
|
||||
|
|
|
@ -20,6 +20,7 @@ import (
|
|||
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/anystorehelper"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/oldstore"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceindex"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceresolverstore"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/logging"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
|
||||
)
|
||||
|
@ -52,6 +53,7 @@ type ObjectStore interface {
|
|||
GetCrdtDb(spaceId string) anystore.DB
|
||||
|
||||
SpaceNameGetter
|
||||
spaceresolverstore.Store
|
||||
CrossSpace
|
||||
}
|
||||
|
||||
|
@ -86,9 +88,11 @@ type TechSpaceIdProvider interface {
|
|||
}
|
||||
|
||||
type dsObjectStore struct {
|
||||
repoPath string
|
||||
techSpaceId string
|
||||
anyStoreConfig anystore.Config
|
||||
spaceresolverstore.Store
|
||||
|
||||
objectStorePath string
|
||||
techSpaceId string
|
||||
anyStoreConfig anystore.Config
|
||||
|
||||
anyStore anystore.DB
|
||||
anyStoreLockRemove func() error
|
||||
|
@ -145,6 +149,9 @@ func New() ObjectStore {
|
|||
|
||||
func (s *dsObjectStore) Init(a *app.App) (err error) {
|
||||
s.sourceService = app.MustComponent[spaceindex.SourceDetailsFromID](a)
|
||||
|
||||
repoPath := app.MustComponent[wallet.Wallet](a).RepoPath()
|
||||
|
||||
fts := a.Component(ftsearch.CName)
|
||||
if fts == nil {
|
||||
log.Warnf("init objectstore without fulltext")
|
||||
|
@ -152,8 +159,10 @@ func (s *dsObjectStore) Init(a *app.App) (err error) {
|
|||
s.fts = fts.(ftsearch.FTSearch)
|
||||
}
|
||||
s.arenaPool = &anyenc.ArenaPool{}
|
||||
s.repoPath = app.MustComponent[wallet.Wallet](a).RepoPath()
|
||||
s.anyStoreConfig = *app.MustComponent[configProvider](a).GetAnyStoreConfig()
|
||||
|
||||
cfg := app.MustComponent[configProvider](a)
|
||||
s.objectStorePath = filepath.Join(repoPath, "objectstore")
|
||||
s.anyStoreConfig = *cfg.GetAnyStoreConfig()
|
||||
s.setDefaultConfig()
|
||||
s.oldStore = app.MustComponent[oldstore.Service](a)
|
||||
s.techSpaceIdProvider = app.MustComponent[TechSpaceIdProvider](a)
|
||||
|
@ -168,12 +177,23 @@ func (s *dsObjectStore) Name() (name string) {
|
|||
func (s *dsObjectStore) Run(ctx context.Context) error {
|
||||
s.techSpaceId = s.techSpaceIdProvider.TechSpaceId()
|
||||
|
||||
dbDir := s.storeRootDir()
|
||||
err := ensureDirExists(dbDir)
|
||||
err := ensureDirExists(s.objectStorePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.openDatabase(ctx, filepath.Join(dbDir, "objects.db"))
|
||||
err = s.openDatabase(ctx, filepath.Join(s.objectStorePath, "objects.db"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("open db: %w", err)
|
||||
}
|
||||
|
||||
store, err := spaceresolverstore.New(s.componentCtx, s.anyStore)
|
||||
if err != nil {
|
||||
return fmt.Errorf("new space resolver store: %w", err)
|
||||
}
|
||||
|
||||
s.Store = store
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *dsObjectStore) setDefaultConfig() {
|
||||
|
@ -184,10 +204,6 @@ func (s *dsObjectStore) setDefaultConfig() {
|
|||
s.anyStoreConfig.SQLiteConnectionOptions["synchronous"] = "off"
|
||||
}
|
||||
|
||||
func (s *dsObjectStore) storeRootDir() string {
|
||||
return filepath.Join(s.repoPath, "objectstore")
|
||||
}
|
||||
|
||||
func ensureDirExists(dir string) error {
|
||||
_, err := os.Stat(dir)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
|
@ -238,7 +254,7 @@ func (s *dsObjectStore) preloadExistingObjectStores() error {
|
|||
var err error
|
||||
s.spaceStoreDirsCheck.Do(func() {
|
||||
var entries []os.DirEntry
|
||||
entries, err = os.ReadDir(s.storeRootDir())
|
||||
entries, err = os.ReadDir(s.objectStorePath)
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
for _, entry := range entries {
|
||||
|
@ -300,7 +316,7 @@ func (s *dsObjectStore) SpaceIndex(spaceId string) spaceindex.Store {
|
|||
func (s *dsObjectStore) getOrInitSpaceIndex(spaceId string) spaceindex.Store {
|
||||
store, ok := s.spaceIndexes[spaceId]
|
||||
if !ok {
|
||||
dir := filepath.Join(s.storeRootDir(), spaceId)
|
||||
dir := filepath.Join(s.objectStorePath, spaceId)
|
||||
err := ensureDirExists(dir)
|
||||
if err != nil {
|
||||
return spaceindex.NewInvalidStore(err)
|
||||
|
@ -334,7 +350,7 @@ func (s *dsObjectStore) GetCrdtDb(spaceId string) anystore.DB {
|
|||
|
||||
db, ok := s.crdtDbs[spaceId]
|
||||
if !ok {
|
||||
dir := filepath.Join(s.storeRootDir(), spaceId)
|
||||
dir := filepath.Join(s.objectStorePath, spaceId)
|
||||
err := ensureDirExists(dir)
|
||||
if err != nil {
|
||||
return nil
|
||||
|
|
71
pkg/lib/localstore/objectstore/spaceresolverstore/store.go
Normal file
71
pkg/lib/localstore/objectstore/spaceresolverstore/store.go
Normal file
|
@ -0,0 +1,71 @@
|
|||
package spaceresolverstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
anystore "github.com/anyproto/any-store"
|
||||
"github.com/anyproto/any-store/anyenc"
|
||||
"github.com/anyproto/any-store/query"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/domain"
|
||||
)
|
||||
|
||||
const bindKey = "b"
|
||||
|
||||
type Store interface {
|
||||
BindSpaceId(spaceId, objectId string) error
|
||||
GetSpaceId(objectId string) (spaceId string, err error)
|
||||
}
|
||||
|
||||
type dsObjectStore struct {
|
||||
componentCtx context.Context
|
||||
collection anystore.Collection
|
||||
arenaPool *anyenc.ArenaPool
|
||||
}
|
||||
|
||||
func New(componentCtx context.Context, db anystore.DB) (Store, error) {
|
||||
collection, err := db.Collection(componentCtx, "bindId")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open bindId collection: %w", err)
|
||||
}
|
||||
return &dsObjectStore{
|
||||
componentCtx: componentCtx,
|
||||
arenaPool: &anyenc.ArenaPool{},
|
||||
collection: collection,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *dsObjectStore) BindSpaceId(spaceId, objectId string) error {
|
||||
return d.modifyBind(d.componentCtx, objectId, spaceId)
|
||||
}
|
||||
|
||||
func (d *dsObjectStore) GetSpaceId(objectId string) (spaceId string, err error) {
|
||||
doc, err := d.collection.FindId(d.componentCtx, objectId)
|
||||
if err != nil {
|
||||
if errors.Is(err, anystore.ErrDocNotFound) {
|
||||
return "", domain.ErrObjectNotFound
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
return doc.Value().GetString(bindKey), nil
|
||||
}
|
||||
|
||||
func (d *dsObjectStore) modifyBind(ctx context.Context, objectId, spaceId string) error {
|
||||
tx, err := d.collection.WriteTx(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
arena := d.arenaPool.Get()
|
||||
defer d.arenaPool.Put(arena)
|
||||
mod := query.ModifyFunc(func(a *anyenc.Arena, v *anyenc.Value) (result *anyenc.Value, modified bool, err error) {
|
||||
v.Set(bindKey, arena.NewString(spaceId))
|
||||
return v, true, nil
|
||||
})
|
||||
_, err = d.collection.UpsertId(tx.Context(), objectId, mod)
|
||||
if err != nil {
|
||||
return errors.Join(err, tx.Rollback())
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
|
@ -3,10 +3,12 @@
|
|||
package mock_clientspace
|
||||
|
||||
import (
|
||||
context "context"
|
||||
anystorage "github.com/anyproto/anytype-heart/space/spacecore/storage/anystorage"
|
||||
|
||||
commonspace "github.com/anyproto/any-sync/commonspace"
|
||||
|
||||
context "context"
|
||||
|
||||
domain "github.com/anyproto/anytype-heart/core/domain"
|
||||
|
||||
headsync "github.com/anyproto/any-sync/commonspace/headsync"
|
||||
|
@ -21,8 +23,6 @@ import (
|
|||
|
||||
smartblock "github.com/anyproto/anytype-heart/core/block/editor/smartblock"
|
||||
|
||||
spacestorage "github.com/anyproto/any-sync/commonspace/spacestorage"
|
||||
|
||||
threads "github.com/anyproto/anytype-heart/pkg/lib/threads"
|
||||
|
||||
treestorage "github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
|
@ -1511,19 +1511,19 @@ func (_c *MockSpace_Remove_Call) RunAndReturn(run func(context.Context, string)
|
|||
}
|
||||
|
||||
// Storage provides a mock function with given fields:
|
||||
func (_m *MockSpace) Storage() spacestorage.SpaceStorage {
|
||||
func (_m *MockSpace) Storage() anystorage.ClientSpaceStorage {
|
||||
ret := _m.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Storage")
|
||||
}
|
||||
|
||||
var r0 spacestorage.SpaceStorage
|
||||
if rf, ok := ret.Get(0).(func() spacestorage.SpaceStorage); ok {
|
||||
var r0 anystorage.ClientSpaceStorage
|
||||
if rf, ok := ret.Get(0).(func() anystorage.ClientSpaceStorage); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(spacestorage.SpaceStorage)
|
||||
r0 = ret.Get(0).(anystorage.ClientSpaceStorage)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1547,12 +1547,12 @@ func (_c *MockSpace_Storage_Call) Run(run func()) *MockSpace_Storage_Call {
|
|||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockSpace_Storage_Call) Return(_a0 spacestorage.SpaceStorage) *MockSpace_Storage_Call {
|
||||
func (_c *MockSpace_Storage_Call) Return(_a0 anystorage.ClientSpaceStorage) *MockSpace_Storage_Call {
|
||||
_c.Call.Return(_a0)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockSpace_Storage_Call) RunAndReturn(run func() spacestorage.SpaceStorage) *MockSpace_Storage_Call {
|
||||
func (_c *MockSpace_Storage_Call) RunAndReturn(run func() anystorage.ClientSpaceStorage) *MockSpace_Storage_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
|
|
@ -11,7 +11,6 @@ import (
|
|||
"github.com/anyproto/any-sync/commonspace"
|
||||
"github.com/anyproto/any-sync/commonspace/headsync"
|
||||
"github.com/anyproto/any-sync/commonspace/objecttreebuilder"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage"
|
||||
"github.com/anyproto/any-sync/net/peer"
|
||||
"github.com/anyproto/any-sync/util/crypto"
|
||||
"go.uber.org/zap"
|
||||
|
@ -29,6 +28,7 @@ import (
|
|||
"github.com/anyproto/anytype-heart/space/spacecore"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/peermanager"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage/anystorage"
|
||||
)
|
||||
|
||||
type Space interface {
|
||||
|
@ -40,7 +40,7 @@ type Space interface {
|
|||
DebugAllHeads() []headsync.TreeHeads
|
||||
DeleteTree(ctx context.Context, id string) (err error)
|
||||
StoredIds() []string
|
||||
Storage() spacestorage.SpaceStorage
|
||||
Storage() anystorage.ClientSpaceStorage
|
||||
|
||||
DerivedIDs() threads.DerivedSmartblockIds
|
||||
|
||||
|
@ -124,12 +124,16 @@ func BuildSpace(ctx context.Context, deps SpaceDeps) (Space, error) {
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("derive object ids: %w", err)
|
||||
}
|
||||
if deps.StorageService.IsSpaceCreated(deps.CommonSpace.Id()) {
|
||||
isSpaceCreated, err := sp.Storage().IsSpaceCreated(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("is space created: %w", err)
|
||||
}
|
||||
if isSpaceCreated {
|
||||
err = sp.ObjectProvider.CreateMandatoryObjects(ctx, sp)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create mandatory objects: %w", err)
|
||||
}
|
||||
err = deps.StorageService.UnmarkSpaceCreated(deps.CommonSpace.Id())
|
||||
err = sp.Storage().UnmarkSpaceCreated(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unmark space created: %w", err)
|
||||
}
|
||||
|
@ -207,8 +211,8 @@ func (s *space) StoredIds() []string {
|
|||
return s.common.StoredIds()
|
||||
}
|
||||
|
||||
func (s *space) Storage() spacestorage.SpaceStorage {
|
||||
return s.common.Storage()
|
||||
func (s *space) Storage() anystorage.ClientSpaceStorage {
|
||||
return s.common.Storage().(anystorage.ClientSpaceStorage)
|
||||
}
|
||||
|
||||
func (s *space) DerivedIDs() threads.DerivedSmartblockIds {
|
||||
|
@ -305,10 +309,7 @@ func (s *space) TryLoadBundledObjects(ctx context.Context) (missingSourceIds []s
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
storedIds, err := s.Storage().StoredIds()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
storedIds := s.StoredIds()
|
||||
|
||||
missingIds := bundledObjectIds.Filter(func(bo domain.BundledObjectId) bool {
|
||||
return !slices.Contains(storedIds, bo.DerivedObjectId)
|
||||
|
@ -318,11 +319,7 @@ func (s *space) TryLoadBundledObjects(ctx context.Context) (missingSourceIds []s
|
|||
s.LoadObjectsIgnoreErrs(ctx, missingIds.DerivedObjectIds())
|
||||
// todo: make LoadObjectsIgnoreErrs return list of loaded ids
|
||||
|
||||
storedIds, err = s.Storage().StoredIds()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
storedIds = s.StoredIds()
|
||||
missingIds = bundledObjectIds.Filter(func(bo domain.BundledObjectId) bool {
|
||||
return !slices.Contains(storedIds, bo.DerivedObjectId)
|
||||
})
|
||||
|
@ -348,7 +345,10 @@ func (s *space) migrationProfileObject(ctx context.Context) error {
|
|||
return err
|
||||
}
|
||||
|
||||
extractedProfileExists, _ := s.Storage().HasTree(extractedProfileId)
|
||||
extractedProfileExists, err := s.Storage().HasTree(ctx, extractedProfileId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if extractedProfileExists {
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -77,10 +77,6 @@ func (c *virtualCommonSpace) HandleStreamSyncRequest(ctx context.Context, req *s
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c *virtualCommonSpace) HandleDeprecatedObjectSyncRequest(ctx context.Context, req *spacesyncproto.ObjectSyncMessage) (resp *spacesyncproto.ObjectSyncMessage, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func (c *virtualCommonSpace) HandleStream(stream spacesyncproto.DRPCSpaceSync_ObjectSyncStreamStream) error {
|
||||
return nil
|
||||
}
|
||||
|
@ -113,7 +109,7 @@ func (c *virtualCommonSpace) DebugAllHeads() []headsync.TreeHeads {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c *virtualCommonSpace) Description() (desc commonspace.SpaceDescription, err error) {
|
||||
func (c *virtualCommonSpace) Description(ctx context.Context) (desc commonspace.SpaceDescription, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
@ -238,10 +238,6 @@ func (s *syncAclStub) HandleRequest(ctx context.Context, senderId string, reques
|
|||
return
|
||||
}
|
||||
|
||||
func (s *syncAclStub) SetHeadUpdater(updater headupdater.HeadUpdater) {
|
||||
return
|
||||
}
|
||||
|
||||
func (s *syncAclStub) SetAclUpdater(updater headupdater.AclUpdater) {
|
||||
s.updater = updater
|
||||
return
|
||||
|
|
65
space/spacecore/oldstorage/storage.go
Normal file
65
space/spacecore/oldstorage/storage.go
Normal file
|
@ -0,0 +1,65 @@
|
|||
package oldstorage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/anyproto/any-sync/app"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
|
||||
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage/badgerstorage"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage/sqlitestorage"
|
||||
)
|
||||
|
||||
type SpaceStorageMode int
|
||||
|
||||
const CName = "client.spacecore.oldstorage"
|
||||
|
||||
const (
|
||||
SpaceStorageModeSqlite SpaceStorageMode = iota // used for new account repos
|
||||
SpaceStorageModeBadger // used for existing account repos
|
||||
)
|
||||
|
||||
type ClientStorage interface {
|
||||
oldstorage.SpaceStorageProvider
|
||||
app.ComponentRunnable
|
||||
// GetBoundObjectIds returns list of object ids bound (mapped) to space id
|
||||
GetBoundObjectIds(spaceId string) ([]string, error)
|
||||
AllSpaceIds() (ids []string, err error)
|
||||
DeleteSpaceStorage(ctx context.Context, spaceId string) error
|
||||
GetSpaceID(objectID string) (spaceID string, err error)
|
||||
EstimateSize() (uint64, error)
|
||||
}
|
||||
|
||||
// storageService is a proxy for the actual storage implementation
|
||||
type storageService struct {
|
||||
ClientStorage
|
||||
}
|
||||
|
||||
func New() ClientStorage {
|
||||
return &storageService{}
|
||||
}
|
||||
|
||||
type configGetter interface {
|
||||
GetSpaceStorageMode() storage.SpaceStorageMode
|
||||
}
|
||||
|
||||
func (s *storageService) Name() (name string) {
|
||||
return CName
|
||||
}
|
||||
|
||||
func (s *storageService) Init(a *app.App) (err error) {
|
||||
mode := a.MustComponent("config").(configGetter).GetSpaceStorageMode()
|
||||
if mode == storage.SpaceStorageModeBadger {
|
||||
// for already existing account repos
|
||||
s.ClientStorage = badgerstorage.New()
|
||||
} else if mode == storage.SpaceStorageModeSqlite {
|
||||
// sqlite used for new account repos
|
||||
s.ClientStorage = sqlitestorage.New()
|
||||
} else {
|
||||
return fmt.Errorf("unknown storage mode %d", mode)
|
||||
}
|
||||
|
||||
return s.ClientStorage.Init(a)
|
||||
}
|
|
@ -85,7 +85,7 @@ func (r *rpcHandler) SpacePull(ctx context.Context, request *spacesyncproto.Spac
|
|||
return
|
||||
}
|
||||
|
||||
spaceDesc, err := sp.Description()
|
||||
spaceDesc, err := sp.Description(ctx)
|
||||
if err != nil {
|
||||
err = spacesyncproto.ErrUnexpected
|
||||
return
|
||||
|
|
132
space/spacecore/storage/anystorage/clientstorage.go
Normal file
132
space/spacecore/storage/anystorage/clientstorage.go
Normal file
|
@ -0,0 +1,132 @@
|
|||
package anystorage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
anystore "github.com/anyproto/any-store"
|
||||
"github.com/anyproto/any-store/anyenc"
|
||||
"github.com/anyproto/any-store/query"
|
||||
"github.com/anyproto/any-sync/commonspace/headsync/headstorage"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage"
|
||||
)
|
||||
|
||||
type ClientSpaceStorage interface {
|
||||
spacestorage.SpaceStorage
|
||||
HasTree(ctx context.Context, id string) (has bool, err error)
|
||||
TreeRoot(ctx context.Context, id string) (root *treechangeproto.RawTreeChangeWithId, err error)
|
||||
MarkSpaceCreated(ctx context.Context) error
|
||||
IsSpaceCreated(ctx context.Context) (created bool, err error)
|
||||
UnmarkSpaceCreated(ctx context.Context) error
|
||||
AllDeletedTreeIds(ctx context.Context) (ids []string, err error)
|
||||
}
|
||||
|
||||
var _ ClientSpaceStorage = (*clientStorage)(nil)
|
||||
|
||||
const (
|
||||
clientCollectionKey = "_client"
|
||||
clientDocumentKey = "space"
|
||||
createdKey = "created"
|
||||
rawChangeKey = "r"
|
||||
)
|
||||
|
||||
type clientStorage struct {
|
||||
spacestorage.SpaceStorage
|
||||
clientColl anystore.Collection
|
||||
}
|
||||
|
||||
func (r *clientStorage) AllDeletedTreeIds(ctx context.Context) (ids []string, err error) {
|
||||
err = r.SpaceStorage.HeadStorage().IterateEntries(ctx, headstorage.IterOpts{Deleted: true}, func(entry headstorage.HeadsEntry) (bool, error) {
|
||||
ids = append(ids, entry.Id)
|
||||
return true, nil
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func NewClientStorage(ctx context.Context, spaceStorage spacestorage.SpaceStorage) (*clientStorage, error) {
|
||||
storage := &clientStorage{
|
||||
SpaceStorage: spaceStorage,
|
||||
}
|
||||
anyStore := storage.AnyStore()
|
||||
client, err := anyStore.Collection(ctx, clientCollectionKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
storage.clientColl = client
|
||||
return storage, nil
|
||||
}
|
||||
|
||||
func (r *clientStorage) Close(ctx context.Context) (err error) {
|
||||
spaceStorageErr := r.SpaceStorage.Close(ctx)
|
||||
anyStoreErr := r.SpaceStorage.AnyStore().Close()
|
||||
return errors.Join(spaceStorageErr, anyStoreErr)
|
||||
}
|
||||
|
||||
func (r *clientStorage) HasTree(ctx context.Context, id string) (has bool, err error) {
|
||||
_, err = r.SpaceStorage.HeadStorage().GetEntry(ctx, id)
|
||||
isNotFound := errors.Is(err, anystore.ErrDocNotFound)
|
||||
if err != nil && !isNotFound {
|
||||
return false, err
|
||||
}
|
||||
return !isNotFound, nil
|
||||
}
|
||||
|
||||
func (r *clientStorage) TreeRoot(ctx context.Context, id string) (root *treechangeproto.RawTreeChangeWithId, err error) {
|
||||
// it should be faster to do it that way, instead of calling TreeStorage
|
||||
coll, err := r.SpaceStorage.AnyStore().OpenCollection(ctx, objecttree.CollName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
doc, err := coll.FindId(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &treechangeproto.RawTreeChangeWithId{
|
||||
Id: id,
|
||||
RawChange: doc.Value().GetBytes(rawChangeKey),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *clientStorage) MarkSpaceCreated(ctx context.Context) error {
|
||||
return r.modifyState(ctx, true)
|
||||
}
|
||||
|
||||
func (r *clientStorage) IsSpaceCreated(ctx context.Context) (isCreated bool, err error) {
|
||||
doc, err := r.clientColl.FindId(ctx, clientDocumentKey)
|
||||
isNotFound := errors.Is(err, anystore.ErrDocNotFound)
|
||||
if err != nil && !isNotFound {
|
||||
return false, err
|
||||
}
|
||||
if isNotFound {
|
||||
return false, nil
|
||||
}
|
||||
return doc.Value().GetBool(createdKey), nil
|
||||
}
|
||||
|
||||
func (r *clientStorage) UnmarkSpaceCreated(ctx context.Context) error {
|
||||
return r.modifyState(ctx, false)
|
||||
}
|
||||
|
||||
func (r *clientStorage) modifyState(ctx context.Context, isCreated bool) error {
|
||||
tx, err := r.clientColl.WriteTx(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
arena := &anyenc.Arena{}
|
||||
val := arena.NewTrue()
|
||||
if !isCreated {
|
||||
val = arena.NewFalse()
|
||||
}
|
||||
mod := query.ModifyFunc(func(a *anyenc.Arena, v *anyenc.Value) (result *anyenc.Value, modified bool, err error) {
|
||||
v.Set(createdKey, val)
|
||||
return v, true, nil
|
||||
})
|
||||
_, err = r.clientColl.UpsertId(tx.Context(), clientDocumentKey, mod)
|
||||
if err != nil {
|
||||
rollErr := tx.Rollback()
|
||||
return errors.Join(err, rollErr)
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
|
@ -0,0 +1,917 @@
|
|||
// Code generated by mockery. DO NOT EDIT.
|
||||
|
||||
package mock_anystorage
|
||||
|
||||
import (
|
||||
anystore "github.com/anyproto/any-store"
|
||||
app "github.com/anyproto/any-sync/app"
|
||||
|
||||
context "context"
|
||||
|
||||
headstorage "github.com/anyproto/any-sync/commonspace/headsync/headstorage"
|
||||
|
||||
list "github.com/anyproto/any-sync/commonspace/object/acl/list"
|
||||
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
|
||||
objecttree "github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
|
||||
|
||||
statestorage "github.com/anyproto/any-sync/commonspace/headsync/statestorage"
|
||||
|
||||
treechangeproto "github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||
|
||||
treestorage "github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
)
|
||||
|
||||
// MockClientSpaceStorage is an autogenerated mock type for the ClientSpaceStorage type
|
||||
type MockClientSpaceStorage struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
type MockClientSpaceStorage_Expecter struct {
|
||||
mock *mock.Mock
|
||||
}
|
||||
|
||||
func (_m *MockClientSpaceStorage) EXPECT() *MockClientSpaceStorage_Expecter {
|
||||
return &MockClientSpaceStorage_Expecter{mock: &_m.Mock}
|
||||
}
|
||||
|
||||
// AclStorage provides a mock function with given fields:
|
||||
func (_m *MockClientSpaceStorage) AclStorage() (list.Storage, error) {
|
||||
ret := _m.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for AclStorage")
|
||||
}
|
||||
|
||||
var r0 list.Storage
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func() (list.Storage, error)); ok {
|
||||
return rf()
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func() list.Storage); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(list.Storage)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func() error); ok {
|
||||
r1 = rf()
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_AclStorage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AclStorage'
|
||||
type MockClientSpaceStorage_AclStorage_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// AclStorage is a helper method to define mock.On call
|
||||
func (_e *MockClientSpaceStorage_Expecter) AclStorage() *MockClientSpaceStorage_AclStorage_Call {
|
||||
return &MockClientSpaceStorage_AclStorage_Call{Call: _e.mock.On("AclStorage")}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_AclStorage_Call) Run(run func()) *MockClientSpaceStorage_AclStorage_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_AclStorage_Call) Return(_a0 list.Storage, _a1 error) *MockClientSpaceStorage_AclStorage_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_AclStorage_Call) RunAndReturn(run func() (list.Storage, error)) *MockClientSpaceStorage_AclStorage_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// AllDeletedTreeIds provides a mock function with given fields: ctx
|
||||
func (_m *MockClientSpaceStorage) AllDeletedTreeIds(ctx context.Context) ([]string, error) {
|
||||
ret := _m.Called(ctx)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for AllDeletedTreeIds")
|
||||
}
|
||||
|
||||
var r0 []string
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context) ([]string, error)); ok {
|
||||
return rf(ctx)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context) []string); ok {
|
||||
r0 = rf(ctx)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).([]string)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context) error); ok {
|
||||
r1 = rf(ctx)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_AllDeletedTreeIds_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AllDeletedTreeIds'
|
||||
type MockClientSpaceStorage_AllDeletedTreeIds_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// AllDeletedTreeIds is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
func (_e *MockClientSpaceStorage_Expecter) AllDeletedTreeIds(ctx interface{}) *MockClientSpaceStorage_AllDeletedTreeIds_Call {
|
||||
return &MockClientSpaceStorage_AllDeletedTreeIds_Call{Call: _e.mock.On("AllDeletedTreeIds", ctx)}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_AllDeletedTreeIds_Call) Run(run func(ctx context.Context)) *MockClientSpaceStorage_AllDeletedTreeIds_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_AllDeletedTreeIds_Call) Return(ids []string, err error) *MockClientSpaceStorage_AllDeletedTreeIds_Call {
|
||||
_c.Call.Return(ids, err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_AllDeletedTreeIds_Call) RunAndReturn(run func(context.Context) ([]string, error)) *MockClientSpaceStorage_AllDeletedTreeIds_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// AnyStore provides a mock function with given fields:
|
||||
func (_m *MockClientSpaceStorage) AnyStore() anystore.DB {
|
||||
ret := _m.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for AnyStore")
|
||||
}
|
||||
|
||||
var r0 anystore.DB
|
||||
if rf, ok := ret.Get(0).(func() anystore.DB); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(anystore.DB)
|
||||
}
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_AnyStore_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AnyStore'
|
||||
type MockClientSpaceStorage_AnyStore_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// AnyStore is a helper method to define mock.On call
|
||||
func (_e *MockClientSpaceStorage_Expecter) AnyStore() *MockClientSpaceStorage_AnyStore_Call {
|
||||
return &MockClientSpaceStorage_AnyStore_Call{Call: _e.mock.On("AnyStore")}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_AnyStore_Call) Run(run func()) *MockClientSpaceStorage_AnyStore_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_AnyStore_Call) Return(_a0 anystore.DB) *MockClientSpaceStorage_AnyStore_Call {
|
||||
_c.Call.Return(_a0)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_AnyStore_Call) RunAndReturn(run func() anystore.DB) *MockClientSpaceStorage_AnyStore_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Close provides a mock function with given fields: ctx
|
||||
func (_m *MockClientSpaceStorage) Close(ctx context.Context) error {
|
||||
ret := _m.Called(ctx)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Close")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context) error); ok {
|
||||
r0 = rf(ctx)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close'
|
||||
type MockClientSpaceStorage_Close_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Close is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
func (_e *MockClientSpaceStorage_Expecter) Close(ctx interface{}) *MockClientSpaceStorage_Close_Call {
|
||||
return &MockClientSpaceStorage_Close_Call{Call: _e.mock.On("Close", ctx)}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_Close_Call) Run(run func(ctx context.Context)) *MockClientSpaceStorage_Close_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_Close_Call) Return(err error) *MockClientSpaceStorage_Close_Call {
|
||||
_c.Call.Return(err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_Close_Call) RunAndReturn(run func(context.Context) error) *MockClientSpaceStorage_Close_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// CreateTreeStorage provides a mock function with given fields: ctx, payload
|
||||
func (_m *MockClientSpaceStorage) CreateTreeStorage(ctx context.Context, payload treestorage.TreeStorageCreatePayload) (objecttree.Storage, error) {
|
||||
ret := _m.Called(ctx, payload)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for CreateTreeStorage")
|
||||
}
|
||||
|
||||
var r0 objecttree.Storage
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, treestorage.TreeStorageCreatePayload) (objecttree.Storage, error)); ok {
|
||||
return rf(ctx, payload)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, treestorage.TreeStorageCreatePayload) objecttree.Storage); ok {
|
||||
r0 = rf(ctx, payload)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(objecttree.Storage)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, treestorage.TreeStorageCreatePayload) error); ok {
|
||||
r1 = rf(ctx, payload)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_CreateTreeStorage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateTreeStorage'
|
||||
type MockClientSpaceStorage_CreateTreeStorage_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// CreateTreeStorage is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - payload treestorage.TreeStorageCreatePayload
|
||||
func (_e *MockClientSpaceStorage_Expecter) CreateTreeStorage(ctx interface{}, payload interface{}) *MockClientSpaceStorage_CreateTreeStorage_Call {
|
||||
return &MockClientSpaceStorage_CreateTreeStorage_Call{Call: _e.mock.On("CreateTreeStorage", ctx, payload)}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_CreateTreeStorage_Call) Run(run func(ctx context.Context, payload treestorage.TreeStorageCreatePayload)) *MockClientSpaceStorage_CreateTreeStorage_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(treestorage.TreeStorageCreatePayload))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_CreateTreeStorage_Call) Return(_a0 objecttree.Storage, _a1 error) *MockClientSpaceStorage_CreateTreeStorage_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_CreateTreeStorage_Call) RunAndReturn(run func(context.Context, treestorage.TreeStorageCreatePayload) (objecttree.Storage, error)) *MockClientSpaceStorage_CreateTreeStorage_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// HasTree provides a mock function with given fields: ctx, id
|
||||
func (_m *MockClientSpaceStorage) HasTree(ctx context.Context, id string) (bool, error) {
|
||||
ret := _m.Called(ctx, id)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for HasTree")
|
||||
}
|
||||
|
||||
var r0 bool
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, string) (bool, error)); ok {
|
||||
return rf(ctx, id)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, string) bool); ok {
|
||||
r0 = rf(ctx, id)
|
||||
} else {
|
||||
r0 = ret.Get(0).(bool)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
|
||||
r1 = rf(ctx, id)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_HasTree_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'HasTree'
|
||||
type MockClientSpaceStorage_HasTree_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// HasTree is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - id string
|
||||
func (_e *MockClientSpaceStorage_Expecter) HasTree(ctx interface{}, id interface{}) *MockClientSpaceStorage_HasTree_Call {
|
||||
return &MockClientSpaceStorage_HasTree_Call{Call: _e.mock.On("HasTree", ctx, id)}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_HasTree_Call) Run(run func(ctx context.Context, id string)) *MockClientSpaceStorage_HasTree_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(string))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_HasTree_Call) Return(has bool, err error) *MockClientSpaceStorage_HasTree_Call {
|
||||
_c.Call.Return(has, err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_HasTree_Call) RunAndReturn(run func(context.Context, string) (bool, error)) *MockClientSpaceStorage_HasTree_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// HeadStorage provides a mock function with given fields:
|
||||
func (_m *MockClientSpaceStorage) HeadStorage() headstorage.HeadStorage {
|
||||
ret := _m.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for HeadStorage")
|
||||
}
|
||||
|
||||
var r0 headstorage.HeadStorage
|
||||
if rf, ok := ret.Get(0).(func() headstorage.HeadStorage); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(headstorage.HeadStorage)
|
||||
}
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_HeadStorage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'HeadStorage'
|
||||
type MockClientSpaceStorage_HeadStorage_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// HeadStorage is a helper method to define mock.On call
|
||||
func (_e *MockClientSpaceStorage_Expecter) HeadStorage() *MockClientSpaceStorage_HeadStorage_Call {
|
||||
return &MockClientSpaceStorage_HeadStorage_Call{Call: _e.mock.On("HeadStorage")}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_HeadStorage_Call) Run(run func()) *MockClientSpaceStorage_HeadStorage_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_HeadStorage_Call) Return(_a0 headstorage.HeadStorage) *MockClientSpaceStorage_HeadStorage_Call {
|
||||
_c.Call.Return(_a0)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_HeadStorage_Call) RunAndReturn(run func() headstorage.HeadStorage) *MockClientSpaceStorage_HeadStorage_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Id provides a mock function with given fields:
|
||||
func (_m *MockClientSpaceStorage) Id() string {
|
||||
ret := _m.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Id")
|
||||
}
|
||||
|
||||
var r0 string
|
||||
if rf, ok := ret.Get(0).(func() string); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
r0 = ret.Get(0).(string)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_Id_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Id'
|
||||
type MockClientSpaceStorage_Id_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Id is a helper method to define mock.On call
|
||||
func (_e *MockClientSpaceStorage_Expecter) Id() *MockClientSpaceStorage_Id_Call {
|
||||
return &MockClientSpaceStorage_Id_Call{Call: _e.mock.On("Id")}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_Id_Call) Run(run func()) *MockClientSpaceStorage_Id_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_Id_Call) Return(_a0 string) *MockClientSpaceStorage_Id_Call {
|
||||
_c.Call.Return(_a0)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_Id_Call) RunAndReturn(run func() string) *MockClientSpaceStorage_Id_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Init provides a mock function with given fields: a
|
||||
func (_m *MockClientSpaceStorage) Init(a *app.App) error {
|
||||
ret := _m.Called(a)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Init")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(*app.App) error); ok {
|
||||
r0 = rf(a)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_Init_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Init'
|
||||
type MockClientSpaceStorage_Init_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Init is a helper method to define mock.On call
|
||||
// - a *app.App
|
||||
func (_e *MockClientSpaceStorage_Expecter) Init(a interface{}) *MockClientSpaceStorage_Init_Call {
|
||||
return &MockClientSpaceStorage_Init_Call{Call: _e.mock.On("Init", a)}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_Init_Call) Run(run func(a *app.App)) *MockClientSpaceStorage_Init_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(*app.App))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_Init_Call) Return(err error) *MockClientSpaceStorage_Init_Call {
|
||||
_c.Call.Return(err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_Init_Call) RunAndReturn(run func(*app.App) error) *MockClientSpaceStorage_Init_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// IsSpaceCreated provides a mock function with given fields: ctx
|
||||
func (_m *MockClientSpaceStorage) IsSpaceCreated(ctx context.Context) (bool, error) {
|
||||
ret := _m.Called(ctx)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for IsSpaceCreated")
|
||||
}
|
||||
|
||||
var r0 bool
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context) (bool, error)); ok {
|
||||
return rf(ctx)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context) bool); ok {
|
||||
r0 = rf(ctx)
|
||||
} else {
|
||||
r0 = ret.Get(0).(bool)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context) error); ok {
|
||||
r1 = rf(ctx)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_IsSpaceCreated_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IsSpaceCreated'
|
||||
type MockClientSpaceStorage_IsSpaceCreated_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// IsSpaceCreated is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
func (_e *MockClientSpaceStorage_Expecter) IsSpaceCreated(ctx interface{}) *MockClientSpaceStorage_IsSpaceCreated_Call {
|
||||
return &MockClientSpaceStorage_IsSpaceCreated_Call{Call: _e.mock.On("IsSpaceCreated", ctx)}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_IsSpaceCreated_Call) Run(run func(ctx context.Context)) *MockClientSpaceStorage_IsSpaceCreated_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_IsSpaceCreated_Call) Return(created bool, err error) *MockClientSpaceStorage_IsSpaceCreated_Call {
|
||||
_c.Call.Return(created, err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_IsSpaceCreated_Call) RunAndReturn(run func(context.Context) (bool, error)) *MockClientSpaceStorage_IsSpaceCreated_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// MarkSpaceCreated provides a mock function with given fields: ctx
|
||||
func (_m *MockClientSpaceStorage) MarkSpaceCreated(ctx context.Context) error {
|
||||
ret := _m.Called(ctx)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for MarkSpaceCreated")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context) error); ok {
|
||||
r0 = rf(ctx)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_MarkSpaceCreated_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkSpaceCreated'
|
||||
type MockClientSpaceStorage_MarkSpaceCreated_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// MarkSpaceCreated is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
func (_e *MockClientSpaceStorage_Expecter) MarkSpaceCreated(ctx interface{}) *MockClientSpaceStorage_MarkSpaceCreated_Call {
|
||||
return &MockClientSpaceStorage_MarkSpaceCreated_Call{Call: _e.mock.On("MarkSpaceCreated", ctx)}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_MarkSpaceCreated_Call) Run(run func(ctx context.Context)) *MockClientSpaceStorage_MarkSpaceCreated_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_MarkSpaceCreated_Call) Return(_a0 error) *MockClientSpaceStorage_MarkSpaceCreated_Call {
|
||||
_c.Call.Return(_a0)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_MarkSpaceCreated_Call) RunAndReturn(run func(context.Context) error) *MockClientSpaceStorage_MarkSpaceCreated_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Name provides a mock function with given fields:
|
||||
func (_m *MockClientSpaceStorage) Name() string {
|
||||
ret := _m.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Name")
|
||||
}
|
||||
|
||||
var r0 string
|
||||
if rf, ok := ret.Get(0).(func() string); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
r0 = ret.Get(0).(string)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_Name_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Name'
|
||||
type MockClientSpaceStorage_Name_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Name is a helper method to define mock.On call
|
||||
func (_e *MockClientSpaceStorage_Expecter) Name() *MockClientSpaceStorage_Name_Call {
|
||||
return &MockClientSpaceStorage_Name_Call{Call: _e.mock.On("Name")}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_Name_Call) Run(run func()) *MockClientSpaceStorage_Name_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_Name_Call) Return(name string) *MockClientSpaceStorage_Name_Call {
|
||||
_c.Call.Return(name)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_Name_Call) RunAndReturn(run func() string) *MockClientSpaceStorage_Name_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Run provides a mock function with given fields: ctx
|
||||
func (_m *MockClientSpaceStorage) Run(ctx context.Context) error {
|
||||
ret := _m.Called(ctx)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Run")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context) error); ok {
|
||||
r0 = rf(ctx)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_Run_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Run'
|
||||
type MockClientSpaceStorage_Run_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// Run is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
func (_e *MockClientSpaceStorage_Expecter) Run(ctx interface{}) *MockClientSpaceStorage_Run_Call {
|
||||
return &MockClientSpaceStorage_Run_Call{Call: _e.mock.On("Run", ctx)}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_Run_Call) Run(run func(ctx context.Context)) *MockClientSpaceStorage_Run_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_Run_Call) Return(err error) *MockClientSpaceStorage_Run_Call {
|
||||
_c.Call.Return(err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_Run_Call) RunAndReturn(run func(context.Context) error) *MockClientSpaceStorage_Run_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// StateStorage provides a mock function with given fields:
|
||||
func (_m *MockClientSpaceStorage) StateStorage() statestorage.StateStorage {
|
||||
ret := _m.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for StateStorage")
|
||||
}
|
||||
|
||||
var r0 statestorage.StateStorage
|
||||
if rf, ok := ret.Get(0).(func() statestorage.StateStorage); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(statestorage.StateStorage)
|
||||
}
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_StateStorage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'StateStorage'
|
||||
type MockClientSpaceStorage_StateStorage_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// StateStorage is a helper method to define mock.On call
|
||||
func (_e *MockClientSpaceStorage_Expecter) StateStorage() *MockClientSpaceStorage_StateStorage_Call {
|
||||
return &MockClientSpaceStorage_StateStorage_Call{Call: _e.mock.On("StateStorage")}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_StateStorage_Call) Run(run func()) *MockClientSpaceStorage_StateStorage_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_StateStorage_Call) Return(_a0 statestorage.StateStorage) *MockClientSpaceStorage_StateStorage_Call {
|
||||
_c.Call.Return(_a0)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_StateStorage_Call) RunAndReturn(run func() statestorage.StateStorage) *MockClientSpaceStorage_StateStorage_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// TreeRoot provides a mock function with given fields: ctx, id
|
||||
func (_m *MockClientSpaceStorage) TreeRoot(ctx context.Context, id string) (*treechangeproto.RawTreeChangeWithId, error) {
|
||||
ret := _m.Called(ctx, id)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for TreeRoot")
|
||||
}
|
||||
|
||||
var r0 *treechangeproto.RawTreeChangeWithId
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, string) (*treechangeproto.RawTreeChangeWithId, error)); ok {
|
||||
return rf(ctx, id)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, string) *treechangeproto.RawTreeChangeWithId); ok {
|
||||
r0 = rf(ctx, id)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*treechangeproto.RawTreeChangeWithId)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
|
||||
r1 = rf(ctx, id)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_TreeRoot_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'TreeRoot'
|
||||
type MockClientSpaceStorage_TreeRoot_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// TreeRoot is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - id string
|
||||
func (_e *MockClientSpaceStorage_Expecter) TreeRoot(ctx interface{}, id interface{}) *MockClientSpaceStorage_TreeRoot_Call {
|
||||
return &MockClientSpaceStorage_TreeRoot_Call{Call: _e.mock.On("TreeRoot", ctx, id)}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_TreeRoot_Call) Run(run func(ctx context.Context, id string)) *MockClientSpaceStorage_TreeRoot_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(string))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_TreeRoot_Call) Return(root *treechangeproto.RawTreeChangeWithId, err error) *MockClientSpaceStorage_TreeRoot_Call {
|
||||
_c.Call.Return(root, err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_TreeRoot_Call) RunAndReturn(run func(context.Context, string) (*treechangeproto.RawTreeChangeWithId, error)) *MockClientSpaceStorage_TreeRoot_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// TreeStorage provides a mock function with given fields: ctx, id
|
||||
func (_m *MockClientSpaceStorage) TreeStorage(ctx context.Context, id string) (objecttree.Storage, error) {
|
||||
ret := _m.Called(ctx, id)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for TreeStorage")
|
||||
}
|
||||
|
||||
var r0 objecttree.Storage
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, string) (objecttree.Storage, error)); ok {
|
||||
return rf(ctx, id)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, string) objecttree.Storage); ok {
|
||||
r0 = rf(ctx, id)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(objecttree.Storage)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
|
||||
r1 = rf(ctx, id)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_TreeStorage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'TreeStorage'
|
||||
type MockClientSpaceStorage_TreeStorage_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// TreeStorage is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - id string
|
||||
func (_e *MockClientSpaceStorage_Expecter) TreeStorage(ctx interface{}, id interface{}) *MockClientSpaceStorage_TreeStorage_Call {
|
||||
return &MockClientSpaceStorage_TreeStorage_Call{Call: _e.mock.On("TreeStorage", ctx, id)}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_TreeStorage_Call) Run(run func(ctx context.Context, id string)) *MockClientSpaceStorage_TreeStorage_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(string))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_TreeStorage_Call) Return(_a0 objecttree.Storage, _a1 error) *MockClientSpaceStorage_TreeStorage_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_TreeStorage_Call) RunAndReturn(run func(context.Context, string) (objecttree.Storage, error)) *MockClientSpaceStorage_TreeStorage_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// UnmarkSpaceCreated provides a mock function with given fields: ctx
|
||||
func (_m *MockClientSpaceStorage) UnmarkSpaceCreated(ctx context.Context) error {
|
||||
ret := _m.Called(ctx)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for UnmarkSpaceCreated")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context) error); ok {
|
||||
r0 = rf(ctx)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockClientSpaceStorage_UnmarkSpaceCreated_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UnmarkSpaceCreated'
|
||||
type MockClientSpaceStorage_UnmarkSpaceCreated_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// UnmarkSpaceCreated is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
func (_e *MockClientSpaceStorage_Expecter) UnmarkSpaceCreated(ctx interface{}) *MockClientSpaceStorage_UnmarkSpaceCreated_Call {
|
||||
return &MockClientSpaceStorage_UnmarkSpaceCreated_Call{Call: _e.mock.On("UnmarkSpaceCreated", ctx)}
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_UnmarkSpaceCreated_Call) Run(run func(ctx context.Context)) *MockClientSpaceStorage_UnmarkSpaceCreated_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_UnmarkSpaceCreated_Call) Return(_a0 error) *MockClientSpaceStorage_UnmarkSpaceCreated_Call {
|
||||
_c.Call.Return(_a0)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientSpaceStorage_UnmarkSpaceCreated_Call) RunAndReturn(run func(context.Context) error) *MockClientSpaceStorage_UnmarkSpaceCreated_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// NewMockClientSpaceStorage creates a new instance of MockClientSpaceStorage. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewMockClientSpaceStorage(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *MockClientSpaceStorage {
|
||||
mock := &MockClientSpaceStorage{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
144
space/spacecore/storage/anystorage/storageservice.go
Normal file
144
space/spacecore/storage/anystorage/storageservice.go
Normal file
|
@ -0,0 +1,144 @@
|
|||
package anystorage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
anystore "github.com/anyproto/any-store"
|
||||
"github.com/anyproto/any-sync/app"
|
||||
"github.com/anyproto/any-sync/app/logger"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage"
|
||||
"golang.org/x/exp/maps"
|
||||
)
|
||||
|
||||
// nolint: unused
|
||||
var log = logger.NewNamed(spacestorage.CName)
|
||||
|
||||
func New(rootPath string, anyStoreConfig *anystore.Config) *storageService {
|
||||
return &storageService{
|
||||
rootPath: rootPath,
|
||||
config: anyStoreConfig,
|
||||
}
|
||||
}
|
||||
|
||||
type storageService struct {
|
||||
rootPath string
|
||||
config *anystore.Config
|
||||
sync.Mutex
|
||||
}
|
||||
|
||||
func (s *storageService) AllSpaceIds() (ids []string, err error) {
|
||||
var files []string
|
||||
fileInfo, err := os.ReadDir(s.rootPath)
|
||||
if err != nil {
|
||||
return files, fmt.Errorf("can't read datadir '%v': %w", s.rootPath, err)
|
||||
}
|
||||
for _, file := range fileInfo {
|
||||
if !strings.HasPrefix(file.Name(), ".") {
|
||||
files = append(files, file.Name())
|
||||
}
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func (s *storageService) Run(ctx context.Context) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *storageService) openDb(ctx context.Context, id string) (db anystore.DB, err error) {
|
||||
dbPath := path.Join(s.rootPath, id, "store.db")
|
||||
if _, err := os.Stat(dbPath); err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, spacestorage.ErrSpaceStorageMissing
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return anystore.Open(ctx, dbPath, s.anyStoreConfig())
|
||||
}
|
||||
|
||||
func (s *storageService) createDb(ctx context.Context, id string) (db anystore.DB, err error) {
|
||||
dirPath := path.Join(s.rootPath, id)
|
||||
err = os.MkdirAll(dirPath, 0755)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dbPath := path.Join(dirPath, "store.db")
|
||||
return anystore.Open(ctx, dbPath, s.anyStoreConfig())
|
||||
}
|
||||
|
||||
func (s *storageService) Close(ctx context.Context) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *storageService) Init(a *app.App) (err error) {
|
||||
if _, err = os.Stat(s.rootPath); err != nil {
|
||||
err = os.MkdirAll(s.rootPath, 0755)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *storageService) Name() (name string) {
|
||||
return spacestorage.CName
|
||||
}
|
||||
|
||||
func (s *storageService) WaitSpaceStorage(ctx context.Context, id string) (spacestorage.SpaceStorage, error) {
|
||||
db, err := s.openDb(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
st, err := spacestorage.New(ctx, id, db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewClientStorage(ctx, st)
|
||||
}
|
||||
|
||||
func (s *storageService) SpaceExists(id string) bool {
|
||||
if id == "" {
|
||||
return false
|
||||
}
|
||||
dbPath := path.Join(s.rootPath, id)
|
||||
if _, err := os.Stat(dbPath); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *storageService) CreateSpaceStorage(ctx context.Context, payload spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error) {
|
||||
db, err := s.createDb(ctx, payload.SpaceHeaderWithId.Id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
st, err := spacestorage.Create(ctx, db, payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewClientStorage(ctx, st)
|
||||
}
|
||||
|
||||
func (s *storageService) DeleteSpaceStorage(ctx context.Context, spaceId string) error {
|
||||
dbPath := path.Join(s.rootPath, spaceId)
|
||||
return os.RemoveAll(dbPath)
|
||||
}
|
||||
|
||||
func (s *storageService) anyStoreConfig() *anystore.Config {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
opts := maps.Clone(s.config.SQLiteConnectionOptions)
|
||||
if opts == nil {
|
||||
opts = make(map[string]string)
|
||||
}
|
||||
opts["synchronous"] = "off"
|
||||
return &anystore.Config{
|
||||
ReadConnections: 4,
|
||||
SQLiteConnectionOptions: opts,
|
||||
}
|
||||
}
|
|
@ -60,6 +60,10 @@ func (t treeKeys) RawChangeKey(id string) []byte {
|
|||
return treestorage.JoinStringsToBytes("space", t.spaceId, "t", t.id, id)
|
||||
}
|
||||
|
||||
func (t treeKeys) RawChangesPrefix() []byte {
|
||||
return treestorage.JoinStringsToBytes("space", t.spaceId, "t", t.id)
|
||||
}
|
||||
|
||||
func (t treeKeys) RawChangePrefix() []byte {
|
||||
return t.rawChangePrefix
|
||||
}
|
||||
|
|
|
@ -4,12 +4,15 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
|
||||
"github.com/anyproto/any-sync/consensus/consensusproto"
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
)
|
||||
|
||||
var ErrIncorrectKey = errors.New("key format is incorrect")
|
||||
var (
|
||||
ErrIncorrectKey = errors.New("key format is incorrect")
|
||||
ErrUnknownRecord = errors.New("record does not exist")
|
||||
)
|
||||
|
||||
type listStorage struct {
|
||||
db *badger.DB
|
||||
|
@ -18,7 +21,7 @@ type listStorage struct {
|
|||
root *consensusproto.RawRecordWithId
|
||||
}
|
||||
|
||||
func newListStorage(spaceId string, db *badger.DB, txn *badger.Txn) (ls liststorage.ListStorage, err error) {
|
||||
func newListStorage(spaceId string, db *badger.DB, txn *badger.Txn) (ls oldstorage.ListStorage, err error) {
|
||||
keys := newAclKeys(spaceId)
|
||||
rootId, err := getTxn(txn, keys.RootIdKey())
|
||||
if err != nil {
|
||||
|
@ -45,7 +48,7 @@ func newListStorage(spaceId string, db *badger.DB, txn *badger.Txn) (ls liststor
|
|||
return
|
||||
}
|
||||
|
||||
func createListStorage(spaceID string, db *badger.DB, txn *badger.Txn, root *consensusproto.RawRecordWithId) (ls liststorage.ListStorage, err error) {
|
||||
func createListStorage(spaceID string, db *badger.DB, txn *badger.Txn, root *consensusproto.RawRecordWithId) (ls oldstorage.ListStorage, err error) {
|
||||
keys := newAclKeys(spaceID)
|
||||
_, err = getTxn(txn, keys.RootIdKey())
|
||||
if err != badger.ErrKeyNotFound {
|
||||
|
@ -99,7 +102,7 @@ func (l *listStorage) GetRawRecord(_ context.Context, id string) (raw *consensus
|
|||
res, err := getDB(l.db, l.keys.RawRecordKey(id))
|
||||
if err != nil {
|
||||
if err == badger.ErrKeyNotFound {
|
||||
err = liststorage.ErrUnknownRecord
|
||||
err = ErrUnknownRecord
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
|
@ -4,13 +4,13 @@ import (
|
|||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
|
||||
"github.com/anyproto/any-sync/consensus/consensusproto"
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func testList(t *testing.T, store liststorage.ListStorage, root *consensusproto.RawRecordWithId, head string) {
|
||||
func testList(t *testing.T, store oldstorage.ListStorage, root *consensusproto.RawRecordWithId, head string) {
|
||||
require.Equal(t, store.Id(), root.Id)
|
||||
|
||||
aclRoot, err := store.Root()
|
||||
|
@ -35,7 +35,7 @@ func TestListStorage(t *testing.T) {
|
|||
return nil
|
||||
})
|
||||
|
||||
var listStore liststorage.ListStorage
|
||||
var listStore oldstorage.ListStorage
|
||||
fx.db.View(func(txn *badger.Txn) (err error) {
|
||||
listStore, err = newListStorage(spaceId, fx.db, txn)
|
||||
require.NoError(t, err)
|
||||
|
|
|
@ -7,10 +7,10 @@ import (
|
|||
"fmt"
|
||||
|
||||
"github.com/anyproto/any-sync/app"
|
||||
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
"golang.org/x/exp/slices"
|
||||
|
@ -21,7 +21,7 @@ type spaceStorage struct {
|
|||
spaceSettingsId string
|
||||
objDb *badger.DB
|
||||
keys spaceKeys
|
||||
aclStorage liststorage.ListStorage
|
||||
aclStorage oldstorage.ListStorage
|
||||
header *spacesyncproto.RawSpaceHeaderWithId
|
||||
service *storageService
|
||||
}
|
||||
|
@ -38,7 +38,7 @@ func (s *spaceStorage) Name() (name string) {
|
|||
return spacestorage.CName
|
||||
}
|
||||
|
||||
func newSpaceStorage(objDb *badger.DB, spaceId string, service *storageService) (store spacestorage.SpaceStorage, err error) {
|
||||
func newSpaceStorage(objDb *badger.DB, spaceId string, service *storageService) (store oldstorage.SpaceStorage, err error) {
|
||||
keys := newSpaceKeys(spaceId)
|
||||
err = objDb.View(func(txn *badger.Txn) error {
|
||||
header, err := getTxn(txn, keys.HeaderKey())
|
||||
|
@ -75,7 +75,7 @@ func newSpaceStorage(objDb *badger.DB, spaceId string, service *storageService)
|
|||
return
|
||||
}
|
||||
|
||||
func createSpaceStorage(db *badger.DB, payload spacestorage.SpaceStorageCreatePayload, service *storageService) (store spacestorage.SpaceStorage, err error) {
|
||||
func createSpaceStorage(db *badger.DB, payload spacestorage.SpaceStorageCreatePayload, service *storageService) (store oldstorage.SpaceStorage, err error) {
|
||||
keys := newSpaceKeys(payload.SpaceHeaderWithId.Id)
|
||||
if hasDB(db, keys.HeaderKey()) {
|
||||
err = spacestorage.ErrSpaceStorageExists
|
||||
|
@ -133,15 +133,15 @@ func (s *spaceStorage) HasTree(id string) (bool, error) {
|
|||
return hasDB(s.objDb, keys.RootIdKey()), nil
|
||||
}
|
||||
|
||||
func (s *spaceStorage) TreeStorage(id string) (treestorage.TreeStorage, error) {
|
||||
func (s *spaceStorage) TreeStorage(id string) (oldstorage.TreeStorage, error) {
|
||||
return newTreeStorage(s.objDb, s.spaceId, id)
|
||||
}
|
||||
|
||||
func (s *spaceStorage) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (ts treestorage.TreeStorage, err error) {
|
||||
func (s *spaceStorage) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (ts oldstorage.TreeStorage, err error) {
|
||||
return createTreeStorage(s.objDb, s.spaceId, payload)
|
||||
}
|
||||
|
||||
func (s *spaceStorage) AclStorage() (liststorage.ListStorage, error) {
|
||||
func (s *spaceStorage) AclStorage() (oldstorage.ListStorage, error) {
|
||||
return s.aclStorage, nil
|
||||
}
|
||||
|
||||
|
@ -186,7 +186,7 @@ func (s *spaceStorage) AllDeletedTreeIds() (ids []string, err error) {
|
|||
|
||||
var isDeleted bool
|
||||
err = item.Value(func(val []byte) error {
|
||||
if bytes.Equal(val, []byte(spacestorage.TreeDeletedStatusDeleted)) {
|
||||
if bytes.Equal(val, []byte(oldstorage.TreeDeletedStatusDeleted)) {
|
||||
isDeleted = true
|
||||
}
|
||||
return nil
|
||||
|
|
|
@ -7,7 +7,8 @@ import (
|
|||
"testing"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||
spacestorage "github.com/anyproto/any-sync/commonspace/spacestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
|
||||
"github.com/anyproto/any-sync/consensus/consensusproto"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
@ -36,7 +37,7 @@ func spaceTestPayload() spacestorage.SpaceStorageCreatePayload {
|
|||
}
|
||||
}
|
||||
|
||||
func testSpace(t *testing.T, store spacestorage.SpaceStorage, payload spacestorage.SpaceStorageCreatePayload) {
|
||||
func testSpace(t *testing.T, store oldstorage.SpaceStorage, payload spacestorage.SpaceStorageCreatePayload) {
|
||||
header, err := store.SpaceHeader()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, payload.SpaceHeaderWithId, header)
|
||||
|
@ -139,3 +140,58 @@ func TestSpaceStorage_StoredIds_BigTxn(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
require.Len(t, storedIds, 0)
|
||||
}
|
||||
|
||||
func newServiceFixture(t *testing.T) *storageService {
|
||||
fx := newFixture(t)
|
||||
fx.open(t)
|
||||
|
||||
t.Cleanup(func() {
|
||||
fx.stop(t)
|
||||
})
|
||||
|
||||
s := &storageService{
|
||||
db: fx.db,
|
||||
keys: newStorageServiceKeys(),
|
||||
lockedSpaces: map[string]*lockSpace{},
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestStorageService_BindSpaceID(t *testing.T) {
|
||||
fx := newServiceFixture(t)
|
||||
|
||||
err := fx.BindSpaceID("spaceId1", "objectId1")
|
||||
require.NoError(t, err)
|
||||
|
||||
spaceId, err := fx.GetSpaceID("objectId1")
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, spaceId, "spaceId1")
|
||||
}
|
||||
|
||||
func TestStorageService_GetBoundObjectIds(t *testing.T) {
|
||||
t.Run("with no bindings", func(t *testing.T) {
|
||||
fx := newServiceFixture(t)
|
||||
|
||||
ids, err := fx.GetBoundObjectIds("spaceId")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, ids)
|
||||
})
|
||||
|
||||
t.Run("ok", func(t *testing.T) {
|
||||
fx := newServiceFixture(t)
|
||||
|
||||
spaceId := "spaceId1"
|
||||
err := fx.BindSpaceID(spaceId, "objectId1")
|
||||
require.NoError(t, err)
|
||||
|
||||
err = fx.BindSpaceID(spaceId, "objectId2")
|
||||
require.NoError(t, err)
|
||||
|
||||
ids, err := fx.GetBoundObjectIds(spaceId)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.ElementsMatch(t, []string{"objectId1", "objectId2"}, ids)
|
||||
})
|
||||
|
||||
}
|
||||
|
|
|
@ -1,12 +1,15 @@
|
|||
package badgerstorage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/anyproto/any-sync/app"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/domain"
|
||||
|
@ -46,11 +49,11 @@ func (s *storageService) Name() (name string) {
|
|||
return spacestorage.CName
|
||||
}
|
||||
|
||||
func (s *storageService) SpaceStorage(id string) (spacestorage.SpaceStorage, error) {
|
||||
func (s *storageService) SpaceStorage(id string) (oldstorage.SpaceStorage, error) {
|
||||
return newSpaceStorage(s.db, id, s)
|
||||
}
|
||||
|
||||
func (s *storageService) WaitSpaceStorage(ctx context.Context, id string) (store spacestorage.SpaceStorage, err error) {
|
||||
func (s *storageService) WaitSpaceStorage(ctx context.Context, id string) (store oldstorage.SpaceStorage, err error) {
|
||||
var ls *lockSpace
|
||||
ls, err = s.checkLock(id, func() error {
|
||||
store, err = newSpaceStorage(s.db, id, s)
|
||||
|
@ -152,7 +155,7 @@ func (s *storageService) unlockSpaceStorage(id string) {
|
|||
}
|
||||
}
|
||||
|
||||
func (s *storageService) CreateSpaceStorage(payload spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error) {
|
||||
func (s *storageService) CreateSpaceStorage(payload spacestorage.SpaceStorageCreatePayload) (oldstorage.SpaceStorage, error) {
|
||||
return createSpaceStorage(s.db, payload, s)
|
||||
}
|
||||
|
||||
|
@ -166,6 +169,37 @@ func (s *storageService) GetSpaceID(objectID string) (spaceID string, err error)
|
|||
return spaceID, err
|
||||
}
|
||||
|
||||
func (s *storageService) GetBoundObjectIds(spaceId string) (ids []string, err error) {
|
||||
prefix := []byte("bind/")
|
||||
spaceIdBytes := []byte(spaceId)
|
||||
err = s.db.View(func(txn *badger.Txn) error {
|
||||
opts := badger.DefaultIteratorOptions
|
||||
opts.PrefetchValues = false
|
||||
opts.Prefix = prefix
|
||||
|
||||
it := txn.NewIterator(opts)
|
||||
defer it.Close()
|
||||
|
||||
for it.Rewind(); it.Valid(); it.Next() {
|
||||
item := it.Item()
|
||||
id := item.Key()
|
||||
|
||||
err = item.Value(func(val []byte) error {
|
||||
if bytes.Equal(spaceIdBytes, val) {
|
||||
idStr := string(id)
|
||||
ids = append(ids, idStr[len(prefix):])
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("read value: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func (s *storageService) BindSpaceID(spaceID, objectID string) (err error) {
|
||||
return badgerhelper.SetValue(s.db, s.keys.BindObjectIDKey(objectID), []byte(spaceID))
|
||||
}
|
||||
|
@ -193,6 +227,11 @@ func (s *storageService) AllSpaceIds() (ids []string, err error) {
|
|||
return
|
||||
}
|
||||
|
||||
func (s *storageService) EstimateSize() (uint64, error) {
|
||||
onDiskSize, _ := s.db.EstimateSize(nil)
|
||||
return onDiskSize, nil
|
||||
}
|
||||
|
||||
func (s *storageService) Run(ctx context.Context) (err error) {
|
||||
s.db, err = s.provider.SpaceStorage()
|
||||
if err != nil {
|
||||
|
|
|
@ -1,11 +1,13 @@
|
|||
package badgerstorage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
)
|
||||
|
||||
|
@ -16,7 +18,7 @@ type treeStorage struct {
|
|||
root *treechangeproto.RawTreeChangeWithId
|
||||
}
|
||||
|
||||
func newTreeStorage(db *badger.DB, spaceId, treeId string) (ts treestorage.TreeStorage, err error) {
|
||||
func newTreeStorage(db *badger.DB, spaceId, treeId string) (ts oldstorage.TreeStorage, err error) {
|
||||
keys := newTreeKeys(spaceId, treeId)
|
||||
err = db.View(func(txn *badger.Txn) error {
|
||||
_, err := txn.Get(keys.RootIdKey())
|
||||
|
@ -48,7 +50,7 @@ func newTreeStorage(db *badger.DB, spaceId, treeId string) (ts treestorage.TreeS
|
|||
return
|
||||
}
|
||||
|
||||
func createTreeStorage(db *badger.DB, spaceId string, payload treestorage.TreeStorageCreatePayload) (ts treestorage.TreeStorage, err error) {
|
||||
func createTreeStorage(db *badger.DB, spaceId string, payload treestorage.TreeStorageCreatePayload) (ts oldstorage.TreeStorage, err error) {
|
||||
keys := newTreeKeys(spaceId, payload.RootRawChange.Id)
|
||||
if hasDB(db, keys.RootIdKey()) {
|
||||
err = treestorage.ErrTreeExists
|
||||
|
@ -57,7 +59,7 @@ func createTreeStorage(db *badger.DB, spaceId string, payload treestorage.TreeSt
|
|||
return forceCreateTreeStorage(db, spaceId, payload)
|
||||
}
|
||||
|
||||
func forceCreateTreeStorage(db *badger.DB, spaceId string, payload treestorage.TreeStorageCreatePayload) (ts treestorage.TreeStorage, err error) {
|
||||
func forceCreateTreeStorage(db *badger.DB, spaceId string, payload treestorage.TreeStorageCreatePayload) (ts oldstorage.TreeStorage, err error) {
|
||||
keys := newTreeKeys(spaceId, payload.RootRawChange.Id)
|
||||
err = db.Update(func(txn *badger.Txn) error {
|
||||
err = txn.Set(keys.RawChangeKey(payload.RootRawChange.Id), payload.RootRawChange.GetRawChange())
|
||||
|
@ -108,7 +110,74 @@ func (t *treeStorage) Heads() (heads []string, err error) {
|
|||
}
|
||||
|
||||
func (t *treeStorage) GetAllChangeIds() (chs []string, err error) {
|
||||
return nil, fmt.Errorf("get all change ids should not be called")
|
||||
prefix := t.keys.RawChangesPrefix()
|
||||
err = t.db.View(func(txn *badger.Txn) error {
|
||||
opts := badger.DefaultIteratorOptions
|
||||
opts.PrefetchValues = false
|
||||
opts.Prefix = prefix
|
||||
|
||||
it := txn.NewIterator(opts)
|
||||
defer it.Close()
|
||||
|
||||
for it.Rewind(); it.Valid(); it.Next() {
|
||||
item := it.Item()
|
||||
id := item.Key()
|
||||
|
||||
changeId := string(id[len(prefix)+1:])
|
||||
// Special case
|
||||
if changeId == "heads" {
|
||||
continue
|
||||
}
|
||||
chs = append(chs, changeId)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read value: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return chs, err
|
||||
}
|
||||
|
||||
func (t *treeStorage) GetAllChanges() ([]*treechangeproto.RawTreeChangeWithId, error) {
|
||||
var changes []*treechangeproto.RawTreeChangeWithId
|
||||
err := t.IterateChanges(func(id string, rawChange []byte) error {
|
||||
changes = append(changes, &treechangeproto.RawTreeChangeWithId{
|
||||
Id: id,
|
||||
RawChange: bytes.Clone(rawChange),
|
||||
})
|
||||
return nil
|
||||
})
|
||||
return changes, err
|
||||
}
|
||||
|
||||
func (t *treeStorage) IterateChanges(proc func(id string, rawChange []byte) error) error {
|
||||
prefix := t.keys.RawChangesPrefix()
|
||||
return t.db.View(func(txn *badger.Txn) error {
|
||||
opts := badger.DefaultIteratorOptions
|
||||
opts.PrefetchValues = false
|
||||
opts.Prefix = prefix
|
||||
|
||||
it := txn.NewIterator(opts)
|
||||
defer it.Close()
|
||||
|
||||
for it.Rewind(); it.Valid(); it.Next() {
|
||||
item := it.Item()
|
||||
id := item.Key()
|
||||
|
||||
changeId := string(id[len(prefix)+1:])
|
||||
// Special case
|
||||
if changeId == "heads" {
|
||||
continue
|
||||
}
|
||||
err := item.Value(func(val []byte) error {
|
||||
return proc(changeId, val)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("read value: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (t *treeStorage) SetHeads(heads []string) (err error) {
|
||||
|
|
|
@ -1,16 +1,24 @@
|
|||
package badgerstorage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type oldTreeStorage interface {
|
||||
oldstorage.ChangesIterator
|
||||
oldstorage.TreeStorage
|
||||
}
|
||||
|
||||
func treeTestPayload() treestorage.TreeStorageCreatePayload {
|
||||
rootRawChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("some"), Id: "someRootId"}
|
||||
otherChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("some other"), Id: "otherId"}
|
||||
|
@ -27,7 +35,7 @@ type fixture struct {
|
|||
db *badger.DB
|
||||
}
|
||||
|
||||
func testTreePayload(t *testing.T, store treestorage.TreeStorage, payload treestorage.TreeStorageCreatePayload) {
|
||||
func testTreePayload(t *testing.T, store oldstorage.TreeStorage, payload treestorage.TreeStorageCreatePayload) {
|
||||
require.Equal(t, payload.RootRawChange.Id, store.Id())
|
||||
|
||||
root, err := store.Root()
|
||||
|
@ -126,8 +134,9 @@ func TestTreeStorage_Methods(t *testing.T) {
|
|||
|
||||
fx.open(t)
|
||||
defer fx.stop(t)
|
||||
store, err := newTreeStorage(fx.db, spaceId, payload.RootRawChange.Id)
|
||||
treeStore, err := newTreeStorage(fx.db, spaceId, payload.RootRawChange.Id)
|
||||
require.NoError(t, err)
|
||||
store := treeStore.(oldTreeStorage)
|
||||
testTreePayload(t, store, payload)
|
||||
|
||||
t.Run("update heads", func(t *testing.T) {
|
||||
|
@ -139,7 +148,7 @@ func TestTreeStorage_Methods(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("add raw change, get change and has change", func(t *testing.T) {
|
||||
newChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("ab"), Id: "newId"}
|
||||
newChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("ab"), Id: "id10"}
|
||||
require.NoError(t, store.AddRawChange(newChange))
|
||||
rawCh, err := store.GetRawChange(context.Background(), newChange.Id)
|
||||
require.NoError(t, err)
|
||||
|
@ -157,6 +166,50 @@ func TestTreeStorage_Methods(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
require.False(t, has)
|
||||
})
|
||||
|
||||
t.Run("iterate changes", func(t *testing.T) {
|
||||
newChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("foo"), Id: "id01"}
|
||||
require.NoError(t, store.AddRawChange(newChange))
|
||||
newChange = &treechangeproto.RawTreeChangeWithId{RawChange: []byte("bar"), Id: "id20"}
|
||||
require.NoError(t, store.AddRawChange(newChange))
|
||||
|
||||
var collected []*treechangeproto.RawTreeChangeWithId
|
||||
require.NoError(t, store.IterateChanges(func(id string, rawChange []byte) error {
|
||||
collected = append(collected, &treechangeproto.RawTreeChangeWithId{
|
||||
Id: id,
|
||||
RawChange: bytes.Clone(rawChange),
|
||||
})
|
||||
return nil
|
||||
}))
|
||||
|
||||
want := []*treechangeproto.RawTreeChangeWithId{
|
||||
{Id: "id01", RawChange: []byte("foo")},
|
||||
{Id: "id10", RawChange: []byte("ab")},
|
||||
{Id: "id20", RawChange: []byte("bar")},
|
||||
{Id: "otherId", RawChange: []byte("some other")},
|
||||
{Id: "someRootId", RawChange: []byte("some")},
|
||||
}
|
||||
assert.Equal(t, want, collected)
|
||||
|
||||
got, err := store.GetAllChanges()
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, want, got)
|
||||
})
|
||||
|
||||
t.Run("get all change ids", func(t *testing.T) {
|
||||
got, err := store.GetAllChangeIds()
|
||||
require.NoError(t, err)
|
||||
|
||||
want := []string{"id01",
|
||||
"id10",
|
||||
"id20",
|
||||
"otherId",
|
||||
"someRootId",
|
||||
}
|
||||
|
||||
assert.Equal(t, want, got)
|
||||
})
|
||||
}
|
||||
|
||||
func TestTreeStorage_Delete(t *testing.T) {
|
||||
|
|
247
space/spacecore/storage/migrator/migrator.go
Normal file
247
space/spacecore/storage/migrator/migrator.go
Normal file
|
@ -0,0 +1,247 @@
|
|||
package migrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
anystore "github.com/anyproto/any-store"
|
||||
"github.com/anyproto/any-sync/app"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/migration"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/block/process"
|
||||
"github.com/anyproto/anytype-heart/pb"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/anystorehelper"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceresolverstore"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/oldstorage"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage/migratorfinisher"
|
||||
"github.com/anyproto/anytype-heart/util/freespace"
|
||||
)
|
||||
|
||||
type NotEnoughFreeSpaceError struct {
|
||||
Free uint64
|
||||
Required uint64
|
||||
}
|
||||
|
||||
func (e NotEnoughFreeSpaceError) Error() string {
|
||||
if e.Required == 0 {
|
||||
return fmt.Sprintf("not enough free space: %d", e.Free)
|
||||
}
|
||||
return fmt.Sprintf("Not enough free space: %d, required: %d", e.Free, e.Required)
|
||||
}
|
||||
|
||||
const CName = "client.storage.migration"
|
||||
|
||||
type migrator struct {
|
||||
oldStorage oldstorage.ClientStorage
|
||||
newStorage storage.ClientStorage
|
||||
process process.Service
|
||||
path string
|
||||
objectStorePath string
|
||||
finisher migratorfinisher.Service
|
||||
|
||||
anyStoreConfig *anystore.Config
|
||||
}
|
||||
|
||||
type pathProvider interface {
|
||||
GetNewSpaceStorePath() string
|
||||
GetOldSpaceStorePath() string
|
||||
GetRepoPath() string
|
||||
GetAnyStoreConfig() *anystore.Config
|
||||
}
|
||||
|
||||
func New() app.ComponentRunnable {
|
||||
return &migrator{}
|
||||
}
|
||||
|
||||
func (m *migrator) Init(a *app.App) (err error) {
|
||||
cfg := a.MustComponent("config").(pathProvider)
|
||||
m.path = cfg.GetNewSpaceStorePath()
|
||||
m.objectStorePath = filepath.Join(cfg.GetRepoPath(), "objectstore")
|
||||
m.oldStorage = app.MustComponent[oldstorage.ClientStorage](a)
|
||||
m.newStorage = app.MustComponent[storage.ClientStorage](a)
|
||||
m.process = app.MustComponent[process.Service](a)
|
||||
m.finisher = app.MustComponent[migratorfinisher.Service](a)
|
||||
m.anyStoreConfig = cfg.GetAnyStoreConfig()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *migrator) Name() (name string) {
|
||||
return CName
|
||||
}
|
||||
|
||||
func (m *migrator) Run(ctx context.Context) (err error) {
|
||||
oldSize, err := m.oldStorage.EstimateSize()
|
||||
if err != nil {
|
||||
return fmt.Errorf("estimate size: %w", err)
|
||||
}
|
||||
free, err := freespace.GetFreeDiskSpace(m.path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get free disk space: %w", err)
|
||||
}
|
||||
requiredDiskSpace := oldSize * 15 / 10
|
||||
if requiredDiskSpace > free {
|
||||
return NotEnoughFreeSpaceError{
|
||||
Free: free,
|
||||
Required: requiredDiskSpace,
|
||||
}
|
||||
}
|
||||
|
||||
err = m.run(ctx)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "disk is full") {
|
||||
return NotEnoughFreeSpaceError{
|
||||
Free: free,
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *migrator) run(ctx context.Context) (err error) {
|
||||
progress := process.NewProgress(&pb.ModelProcessMessageOfMigration{Migration: &pb.ModelProcessMigration{}})
|
||||
progress.SetProgressMessage("Migrating spaces")
|
||||
err = m.process.Add(progress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
progress.Finish(err)
|
||||
}()
|
||||
migrator := migration.NewSpaceMigrator(m.oldStorage, m.newStorage, 40, m.path)
|
||||
allIds, err := m.oldStorage.AllSpaceIds()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var (
|
||||
total int64
|
||||
totalMap = make(map[string]int64)
|
||||
)
|
||||
for _, id := range allIds {
|
||||
store, err := m.oldStorage.WaitSpaceStorage(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
storedIds, err := store.StoredIds()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
total += int64(len(storedIds))
|
||||
totalMap[id] = int64(len(storedIds))
|
||||
err = store.Close(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
}
|
||||
progress.SetTotal(total)
|
||||
for _, id := range allIds {
|
||||
err := migrator.MigrateId(ctx, id, progress)
|
||||
if err != nil {
|
||||
if errors.Is(err, migration.ErrAlreadyMigrated) {
|
||||
progress.AddDone(totalMap[id])
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
err = m.doObjectStoreDb(ctx, func(db anystore.DB) error {
|
||||
resolverStore, err := spaceresolverstore.New(ctx, db)
|
||||
if err != nil {
|
||||
return fmt.Errorf("new resolver store: %w", err)
|
||||
}
|
||||
|
||||
for _, spaceId := range allIds {
|
||||
objectIds, err := m.oldStorage.GetBoundObjectIds(spaceId)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get bound object ids: %w", err)
|
||||
}
|
||||
|
||||
for _, objectId := range objectIds {
|
||||
err = resolverStore.BindSpaceId(spaceId, objectId)
|
||||
if err != nil {
|
||||
return fmt.Errorf("bind space id: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("migrate space id bindings: %w", err)
|
||||
}
|
||||
|
||||
// TODO Maybe add some condition?
|
||||
m.finisher.SetMigrationDone()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// nolint:unused
|
||||
func (m *migrator) verify(ctx context.Context, fast bool) ([]*verificationReport, error) {
|
||||
var reports []*verificationReport
|
||||
err := m.doObjectStoreDb(ctx, func(db anystore.DB) error {
|
||||
resolverStore, err := spaceresolverstore.New(ctx, db)
|
||||
if err != nil {
|
||||
return fmt.Errorf("new resolver store: %w", err)
|
||||
}
|
||||
|
||||
v := &verifier{
|
||||
fast: fast,
|
||||
oldStorage: m.oldStorage,
|
||||
newStorage: m.newStorage,
|
||||
resolverStore: resolverStore,
|
||||
}
|
||||
reports, err = v.verify(ctx)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return reports, nil
|
||||
}
|
||||
|
||||
func (m *migrator) doObjectStoreDb(ctx context.Context, proc func(db anystore.DB) error) error {
|
||||
err := ensureDirExists(m.objectStorePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ensure dir exists: %w", err)
|
||||
}
|
||||
|
||||
store, lockRemove, err := anystorehelper.OpenDatabaseWithLockCheck(ctx, filepath.Join(m.objectStorePath, "objects.db"), m.anyStoreConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open database: %w", err)
|
||||
}
|
||||
|
||||
err = proc(store)
|
||||
|
||||
return errors.Join(err, store.Close(), lockRemove())
|
||||
}
|
||||
|
||||
func ensureDirExists(dir string) error {
|
||||
_, err := os.Stat(dir)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
err = os.MkdirAll(dir, 0700)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create db dir: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *migrator) Close(ctx context.Context) (err error) {
|
||||
return nil
|
||||
}
|
203
space/spacecore/storage/migrator/migrator_test.go
Normal file
203
space/spacecore/storage/migrator/migrator_test.go
Normal file
|
@ -0,0 +1,203 @@
|
|||
package migrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/anyproto/any-sync/app"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/anytype/config"
|
||||
"github.com/anyproto/anytype-heart/core/block/process"
|
||||
"github.com/anyproto/anytype-heart/core/event/mock_event"
|
||||
"github.com/anyproto/anytype-heart/core/wallet"
|
||||
"github.com/anyproto/anytype-heart/pb"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/datastore/clientds"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/oldstorage"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage/migratorfinisher"
|
||||
"github.com/anyproto/anytype-heart/tests/testutil"
|
||||
)
|
||||
|
||||
type fixture struct {
|
||||
migrator *migrator
|
||||
app *app.App
|
||||
cfg *config.Config
|
||||
}
|
||||
|
||||
type quicPreferenceSetterStub struct {
|
||||
}
|
||||
|
||||
func (q *quicPreferenceSetterStub) Init(a *app.App) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *quicPreferenceSetterStub) Name() (name string) {
|
||||
return "quicPreferenceSetterStub"
|
||||
}
|
||||
|
||||
func (q *quicPreferenceSetterStub) PreferQuic(b bool) {
|
||||
}
|
||||
|
||||
func newFixture(t *testing.T, mode storage.SpaceStorageMode) *fixture {
|
||||
cfg := config.New()
|
||||
cfg.SpaceStorageMode = mode
|
||||
cfg.RepoPath = t.TempDir()
|
||||
|
||||
fx := &fixture{
|
||||
cfg: cfg,
|
||||
}
|
||||
return fx
|
||||
}
|
||||
|
||||
func (fx *fixture) start(t *testing.T) {
|
||||
walletService := wallet.NewWithRepoDirAndRandomKeys(fx.cfg.RepoPath)
|
||||
oldStorage := oldstorage.New()
|
||||
newStorage := storage.New()
|
||||
processService := process.New()
|
||||
eventSender := mock_event.NewMockSender(t)
|
||||
eventSender.EXPECT().Broadcast(mock.Anything).Run(func(ev *pb.Event) {
|
||||
}).Maybe()
|
||||
eventSender.EXPECT().BroadcastExceptSessions(mock.Anything, mock.Anything).Run(func(ev *pb.Event, exceptSessions []string) {
|
||||
t.Log(ev)
|
||||
}).Maybe()
|
||||
|
||||
migrator := New().(*migrator)
|
||||
|
||||
ctx := context.Background()
|
||||
testApp := &app.App{}
|
||||
testApp.Register(migratorfinisher.New())
|
||||
testApp.Register(clientds.New())
|
||||
testApp.Register(testutil.PrepareMock(ctx, testApp, eventSender))
|
||||
testApp.Register(&quicPreferenceSetterStub{})
|
||||
testApp.Register(walletService)
|
||||
testApp.Register(fx.cfg)
|
||||
testApp.Register(oldStorage)
|
||||
testApp.Register(newStorage)
|
||||
testApp.Register(processService)
|
||||
testApp.Register(migrator)
|
||||
|
||||
fx.app = testApp
|
||||
fx.migrator = migrator
|
||||
|
||||
err := testApp.Start(ctx)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func assertReports(t *testing.T, reports []*verificationReport) {
|
||||
for _, report := range reports {
|
||||
for _, err := range report.errors {
|
||||
assert.NoError(t, err.err, err.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigration(t *testing.T) {
|
||||
t.Run("no old storage", func(t *testing.T) {
|
||||
fx := newFixture(t, storage.SpaceStorageModeSqlite)
|
||||
|
||||
fx.start(t)
|
||||
})
|
||||
|
||||
t.Run("with sqlite, fast verification", func(t *testing.T) {
|
||||
fx := newFixture(t, storage.SpaceStorageModeSqlite)
|
||||
|
||||
err := copyFile("testdata/spaceStore.db", fx.cfg.GetOldSpaceStorePath())
|
||||
require.NoError(t, err)
|
||||
|
||||
// TODO Test object->space bindings were populated
|
||||
|
||||
fx.start(t)
|
||||
|
||||
reports, err := fx.migrator.verify(context.Background(), true)
|
||||
require.NoError(t, err)
|
||||
assertReports(t, reports)
|
||||
})
|
||||
|
||||
t.Run("with sqlite, full verification", func(t *testing.T) {
|
||||
fx := newFixture(t, storage.SpaceStorageModeSqlite)
|
||||
|
||||
err := copyFile("testdata/spaceStore.db", fx.cfg.GetOldSpaceStorePath())
|
||||
require.NoError(t, err)
|
||||
|
||||
// TODO Test object->space bindings were populated
|
||||
|
||||
fx.start(t)
|
||||
|
||||
reports, err := fx.migrator.verify(context.Background(), false)
|
||||
require.NoError(t, err)
|
||||
assertReports(t, reports)
|
||||
})
|
||||
|
||||
t.Run("with badger, fast verification", func(t *testing.T) {
|
||||
fx := newFixture(t, storage.SpaceStorageModeBadger)
|
||||
|
||||
err := copyDir("testdata/badger_spacestore", fx.cfg.GetOldSpaceStorePath())
|
||||
require.NoError(t, err)
|
||||
|
||||
// TODO Test object->space bindings were populated
|
||||
|
||||
fx.start(t)
|
||||
|
||||
reports, err := fx.migrator.verify(context.Background(), true)
|
||||
require.NoError(t, err)
|
||||
assertReports(t, reports)
|
||||
})
|
||||
|
||||
t.Run("with badger, full verification", func(t *testing.T) {
|
||||
fx := newFixture(t, storage.SpaceStorageModeBadger)
|
||||
|
||||
err := copyDir("testdata/badger_spacestore", fx.cfg.GetOldSpaceStorePath())
|
||||
require.NoError(t, err)
|
||||
|
||||
// TODO Test object->space bindings were populated
|
||||
|
||||
fx.start(t)
|
||||
|
||||
reports, err := fx.migrator.verify(context.Background(), false)
|
||||
require.NoError(t, err)
|
||||
assertReports(t, reports)
|
||||
})
|
||||
}
|
||||
|
||||
func copyFile(srcPath string, destPath string) error {
|
||||
src, err := os.Open(srcPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
dest, err := os.Create(destPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dest.Close()
|
||||
_, err = io.Copy(dest, src)
|
||||
return err
|
||||
}
|
||||
|
||||
func copyDir(srcPath string, destPath string) error {
|
||||
dir, err := os.ReadDir(srcPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.MkdirAll(destPath, os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, entry := range dir {
|
||||
src := filepath.Join(srcPath, entry.Name())
|
||||
dst := filepath.Join(destPath, entry.Name())
|
||||
err := copyFile(src, dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
BIN
space/spacecore/storage/migrator/testdata/badger_spacestore/000001.sst
vendored
Normal file
BIN
space/spacecore/storage/migrator/testdata/badger_spacestore/000001.sst
vendored
Normal file
Binary file not shown.
BIN
space/spacecore/storage/migrator/testdata/badger_spacestore/000001.vlog
vendored
Normal file
BIN
space/spacecore/storage/migrator/testdata/badger_spacestore/000001.vlog
vendored
Normal file
Binary file not shown.
BIN
space/spacecore/storage/migrator/testdata/badger_spacestore/DISCARD
vendored
Normal file
BIN
space/spacecore/storage/migrator/testdata/badger_spacestore/DISCARD
vendored
Normal file
Binary file not shown.
1
space/spacecore/storage/migrator/testdata/badger_spacestore/KEYREGISTRY
vendored
Normal file
1
space/spacecore/storage/migrator/testdata/badger_spacestore/KEYREGISTRY
vendored
Normal file
|
@ -0,0 +1 @@
|
|||
/À‰µ 7¤¼¯…püHello Badger
|
BIN
space/spacecore/storage/migrator/testdata/badger_spacestore/MANIFEST
vendored
Normal file
BIN
space/spacecore/storage/migrator/testdata/badger_spacestore/MANIFEST
vendored
Normal file
Binary file not shown.
BIN
space/spacecore/storage/migrator/testdata/spaceStore.db
vendored
Normal file
BIN
space/spacecore/storage/migrator/testdata/spaceStore.db
vendored
Normal file
Binary file not shown.
224
space/spacecore/storage/migrator/verifier.go
Normal file
224
space/spacecore/storage/migrator/verifier.go
Normal file
|
@ -0,0 +1,224 @@
|
|||
// nolint:unused
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
anystore "github.com/anyproto/any-store"
|
||||
"github.com/anyproto/any-store/query"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage"
|
||||
oldstorage2 "github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/domain"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceresolverstore"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/oldstorage"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage"
|
||||
)
|
||||
|
||||
type verifier struct {
|
||||
fast bool
|
||||
oldStorage oldstorage.ClientStorage
|
||||
newStorage storage.ClientStorage
|
||||
resolverStore spaceresolverstore.Store
|
||||
}
|
||||
|
||||
type errorEntry struct {
|
||||
id string
|
||||
err error
|
||||
}
|
||||
|
||||
type verificationReport struct {
|
||||
spaceId string
|
||||
|
||||
treesCompared int
|
||||
errors []errorEntry
|
||||
totalBytesCompared int
|
||||
|
||||
duration time.Duration
|
||||
}
|
||||
|
||||
func (v *verifier) verify(ctx context.Context) ([]*verificationReport, error) {
|
||||
allSpaceIds, err := v.oldStorage.AllSpaceIds()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list all space ids: %w", err)
|
||||
}
|
||||
reports := make([]*verificationReport, 0, len(allSpaceIds))
|
||||
for _, spaceId := range allSpaceIds {
|
||||
report, err := v.verifySpace(ctx, spaceId)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("verify space: %w", err)
|
||||
}
|
||||
report.spaceId = spaceId
|
||||
reports = append(reports, report)
|
||||
}
|
||||
return reports, nil
|
||||
}
|
||||
|
||||
func (v *verifier) verifySpace(ctx context.Context, spaceId string) (*verificationReport, error) {
|
||||
oldStore, err := v.oldStorage.WaitSpaceStorage(ctx, spaceId)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open old store: %w", err)
|
||||
}
|
||||
|
||||
newStore, err := v.newStorage.WaitSpaceStorage(ctx, spaceId)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open new store: %w", err)
|
||||
}
|
||||
|
||||
storedIds, err := oldStore.StoredIds()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
newStoreCollection, err := newStore.AnyStore().Collection(ctx, "changes")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get new store collection: %w", err)
|
||||
}
|
||||
|
||||
report := &verificationReport{}
|
||||
now := time.Now()
|
||||
for _, treeId := range storedIds {
|
||||
bytesCompared, err := v.verifyTree(ctx, oldStore, newStore, newStoreCollection, treeId)
|
||||
if err != nil {
|
||||
report.errors = append(report.errors, errorEntry{id: treeId, err: err})
|
||||
}
|
||||
report.treesCompared++
|
||||
report.totalBytesCompared += bytesCompared
|
||||
}
|
||||
report.duration = time.Since(now)
|
||||
|
||||
err = oldStore.Close(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (v *verifier) verifyTree(ctx context.Context, oldStore oldstorage2.SpaceStorage, newStore spacestorage.SpaceStorage, newStoreCollection anystore.Collection, treeId string) (int, error) {
|
||||
newHeadStorage := newStore.HeadStorage()
|
||||
|
||||
entry, err := newHeadStorage.GetEntry(ctx, treeId)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("get heads entry: %w", err)
|
||||
}
|
||||
|
||||
oldTreeStorage, err := oldStore.TreeStorage(treeId)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("open old tree storage: %w", err)
|
||||
}
|
||||
oldHeads, err := oldTreeStorage.Heads()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("open old heads storage: %w", err)
|
||||
}
|
||||
if !slices.Equal(oldHeads, entry.Heads) {
|
||||
return 0, fmt.Errorf("old heads doesn't match tree storage")
|
||||
}
|
||||
|
||||
err = v.verifySpaceBindings(oldStore, treeId)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("verify space: %w", err)
|
||||
}
|
||||
|
||||
newTreeStorage, err := newStore.TreeStorage(ctx, treeId)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("open new tree storage: %w", err)
|
||||
}
|
||||
|
||||
var bytesCompared int
|
||||
if v.fast {
|
||||
err = v.verifyChangesFast(ctx, oldTreeStorage, newTreeStorage)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("verify tree fast: %w", err)
|
||||
}
|
||||
} else {
|
||||
bytesCompared, err = v.verifyChangesFull(ctx, newStoreCollection, oldTreeStorage)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("verify tree fast: %w", err)
|
||||
}
|
||||
}
|
||||
return bytesCompared, nil
|
||||
}
|
||||
|
||||
func (v *verifier) verifySpaceBindings(oldStore oldstorage2.SpaceStorage, treeId string) error {
|
||||
gotSpaceId, err := v.resolverStore.GetSpaceId(treeId)
|
||||
// If it's not found in new store, check that it's not found in old store either
|
||||
if errors.Is(err, domain.ErrObjectNotFound) {
|
||||
_, err = v.oldStorage.GetSpaceID(treeId)
|
||||
if errors.Is(err, domain.ErrObjectNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err == nil {
|
||||
return fmt.Errorf("binding is not found in new store")
|
||||
}
|
||||
return fmt.Errorf("check binding in old store: %w", err)
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("resolve space id for object: %w", err)
|
||||
}
|
||||
if gotSpaceId != oldStore.Id() {
|
||||
return fmt.Errorf("resolved spaced id mismatch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifyChangesFast checks only existence of changes
|
||||
func (v *verifier) verifyChangesFast(ctx context.Context, oldTreeStorage oldstorage2.TreeStorage, newTreeStorage objecttree.Storage) error {
|
||||
oldChangeIds, err := oldTreeStorage.GetAllChangeIds()
|
||||
if err != nil {
|
||||
return fmt.Errorf("get old change ids: %w", err)
|
||||
}
|
||||
|
||||
if len(oldChangeIds) == 0 {
|
||||
return fmt.Errorf("old change ids is empty")
|
||||
}
|
||||
for _, oldChangeId := range oldChangeIds {
|
||||
ok, err := newTreeStorage.Has(ctx, oldChangeId)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get old change id: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("old change id doesn't exist")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifyChangesFull checks byte contents of changes
|
||||
func (v *verifier) verifyChangesFull(ctx context.Context, newStoreCollection anystore.Collection, oldTreeStorage oldstorage2.TreeStorage) (int, error) {
|
||||
iterator, ok := oldTreeStorage.(oldstorage2.ChangesIterator)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("old tree storage doesn't implement ChangesIterator")
|
||||
}
|
||||
var bytesCompared int
|
||||
iter, err := newStoreCollection.Find(query.Key{Path: []string{"t"}, Filter: query.NewComp(query.CompOpEq, oldTreeStorage.Id())}).Sort("id").Iter(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("new store: changes iterator: %w", err)
|
||||
}
|
||||
defer iter.Close()
|
||||
err = iterator.IterateChanges(func(id string, oldChange []byte) error {
|
||||
if !iter.Next() {
|
||||
return fmt.Errorf("new store iterator: no more changes")
|
||||
}
|
||||
doc, err := iter.Doc()
|
||||
if err != nil {
|
||||
return fmt.Errorf("new store iterator: read doc: %w", err)
|
||||
}
|
||||
|
||||
newId := doc.Value().GetString("id")
|
||||
if newId != id {
|
||||
return fmt.Errorf("new store iterator: id doesn't match")
|
||||
}
|
||||
|
||||
bytesCompared += len(oldChange)
|
||||
if !bytes.Equal(oldChange, doc.Value().GetBytes("r")) {
|
||||
return fmt.Errorf("old tree change doesn't match tree storage")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return bytesCompared, err
|
||||
}
|
96
space/spacecore/storage/migratorfinisher/finisher.go
Normal file
96
space/spacecore/storage/migratorfinisher/finisher.go
Normal file
|
@ -0,0 +1,96 @@
|
|||
package migratorfinisher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/anyproto/any-sync/app"
|
||||
)
|
||||
|
||||
const (
|
||||
migratedName = "space_store_migrated"
|
||||
objectStoreFolder = "objectstore"
|
||||
crdtDb = "crdt"
|
||||
)
|
||||
|
||||
const CName = "space.spacecore.storage.migratorfinisher"
|
||||
|
||||
type finisher struct {
|
||||
isMigrationDone bool
|
||||
|
||||
newStorePath string
|
||||
oldPath string
|
||||
}
|
||||
|
||||
type Service interface {
|
||||
app.ComponentRunnable
|
||||
SetMigrationDone()
|
||||
}
|
||||
|
||||
func New() Service {
|
||||
return &finisher{}
|
||||
}
|
||||
|
||||
type pathProvider interface {
|
||||
GetNewSpaceStorePath() string
|
||||
GetOldSpaceStorePath() string
|
||||
}
|
||||
|
||||
func (f *finisher) Init(a *app.App) (err error) {
|
||||
cfg := a.MustComponent("config").(pathProvider)
|
||||
f.newStorePath = cfg.GetNewSpaceStorePath()
|
||||
f.oldPath = cfg.GetOldSpaceStorePath()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *finisher) Name() (name string) {
|
||||
return CName
|
||||
}
|
||||
|
||||
func (f *finisher) Run(ctx context.Context) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *finisher) SetMigrationDone() {
|
||||
f.isMigrationDone = true
|
||||
}
|
||||
|
||||
func (f *finisher) Close(ctx context.Context) error {
|
||||
if !f.isMigrationDone {
|
||||
return nil
|
||||
}
|
||||
err := f.removeCrdtIndexes()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return f.renameOldStore()
|
||||
}
|
||||
|
||||
func (f *finisher) renameOldStore() error {
|
||||
newName := migratedName
|
||||
newPath := filepath.Join(filepath.Dir(f.oldPath), newName+filepath.Ext(f.oldPath))
|
||||
err := os.Rename(f.oldPath, newPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to rename: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *finisher) removeCrdtIndexes() error {
|
||||
rootDir := filepath.Join(filepath.Dir(f.newStorePath), objectStoreFolder)
|
||||
prefix := crdtDb
|
||||
return filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() && strings.HasPrefix(info.Name(), prefix) {
|
||||
if removeErr := os.Remove(path); removeErr != nil {
|
||||
return removeErr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
|
@ -82,53 +82,6 @@ func (_c *MockClientStorage_AllSpaceIds_Call) RunAndReturn(run func() ([]string,
|
|||
return _c
|
||||
}
|
||||
|
||||
// BindSpaceID provides a mock function with given fields: spaceID, objectID
|
||||
func (_m *MockClientStorage) BindSpaceID(spaceID string, objectID string) error {
|
||||
ret := _m.Called(spaceID, objectID)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for BindSpaceID")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string, string) error); ok {
|
||||
r0 = rf(spaceID, objectID)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockClientStorage_BindSpaceID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BindSpaceID'
|
||||
type MockClientStorage_BindSpaceID_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// BindSpaceID is a helper method to define mock.On call
|
||||
// - spaceID string
|
||||
// - objectID string
|
||||
func (_e *MockClientStorage_Expecter) BindSpaceID(spaceID interface{}, objectID interface{}) *MockClientStorage_BindSpaceID_Call {
|
||||
return &MockClientStorage_BindSpaceID_Call{Call: _e.mock.On("BindSpaceID", spaceID, objectID)}
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_BindSpaceID_Call) Run(run func(spaceID string, objectID string)) *MockClientStorage_BindSpaceID_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(string), args[1].(string))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_BindSpaceID_Call) Return(err error) *MockClientStorage_BindSpaceID_Call {
|
||||
_c.Call.Return(err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_BindSpaceID_Call) RunAndReturn(run func(string, string) error) *MockClientStorage_BindSpaceID_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Close provides a mock function with given fields: ctx
|
||||
func (_m *MockClientStorage) Close(ctx context.Context) error {
|
||||
ret := _m.Called(ctx)
|
||||
|
@ -175,9 +128,9 @@ func (_c *MockClientStorage_Close_Call) RunAndReturn(run func(context.Context) e
|
|||
return _c
|
||||
}
|
||||
|
||||
// CreateSpaceStorage provides a mock function with given fields: payload
|
||||
func (_m *MockClientStorage) CreateSpaceStorage(payload spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error) {
|
||||
ret := _m.Called(payload)
|
||||
// CreateSpaceStorage provides a mock function with given fields: ctx, payload
|
||||
func (_m *MockClientStorage) CreateSpaceStorage(ctx context.Context, payload spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error) {
|
||||
ret := _m.Called(ctx, payload)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for CreateSpaceStorage")
|
||||
|
@ -185,19 +138,19 @@ func (_m *MockClientStorage) CreateSpaceStorage(payload spacestorage.SpaceStorag
|
|||
|
||||
var r0 spacestorage.SpaceStorage
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error)); ok {
|
||||
return rf(payload)
|
||||
if rf, ok := ret.Get(0).(func(context.Context, spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error)); ok {
|
||||
return rf(ctx, payload)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(spacestorage.SpaceStorageCreatePayload) spacestorage.SpaceStorage); ok {
|
||||
r0 = rf(payload)
|
||||
if rf, ok := ret.Get(0).(func(context.Context, spacestorage.SpaceStorageCreatePayload) spacestorage.SpaceStorage); ok {
|
||||
r0 = rf(ctx, payload)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(spacestorage.SpaceStorage)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(spacestorage.SpaceStorageCreatePayload) error); ok {
|
||||
r1 = rf(payload)
|
||||
if rf, ok := ret.Get(1).(func(context.Context, spacestorage.SpaceStorageCreatePayload) error); ok {
|
||||
r1 = rf(ctx, payload)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
@ -211,14 +164,15 @@ type MockClientStorage_CreateSpaceStorage_Call struct {
|
|||
}
|
||||
|
||||
// CreateSpaceStorage is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - payload spacestorage.SpaceStorageCreatePayload
|
||||
func (_e *MockClientStorage_Expecter) CreateSpaceStorage(payload interface{}) *MockClientStorage_CreateSpaceStorage_Call {
|
||||
return &MockClientStorage_CreateSpaceStorage_Call{Call: _e.mock.On("CreateSpaceStorage", payload)}
|
||||
func (_e *MockClientStorage_Expecter) CreateSpaceStorage(ctx interface{}, payload interface{}) *MockClientStorage_CreateSpaceStorage_Call {
|
||||
return &MockClientStorage_CreateSpaceStorage_Call{Call: _e.mock.On("CreateSpaceStorage", ctx, payload)}
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_CreateSpaceStorage_Call) Run(run func(payload spacestorage.SpaceStorageCreatePayload)) *MockClientStorage_CreateSpaceStorage_Call {
|
||||
func (_c *MockClientStorage_CreateSpaceStorage_Call) Run(run func(ctx context.Context, payload spacestorage.SpaceStorageCreatePayload)) *MockClientStorage_CreateSpaceStorage_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(spacestorage.SpaceStorageCreatePayload))
|
||||
run(args[0].(context.Context), args[1].(spacestorage.SpaceStorageCreatePayload))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
@ -228,7 +182,7 @@ func (_c *MockClientStorage_CreateSpaceStorage_Call) Return(_a0 spacestorage.Spa
|
|||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_CreateSpaceStorage_Call) RunAndReturn(run func(spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error)) *MockClientStorage_CreateSpaceStorage_Call {
|
||||
func (_c *MockClientStorage_CreateSpaceStorage_Call) RunAndReturn(run func(context.Context, spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error)) *MockClientStorage_CreateSpaceStorage_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
@ -280,62 +234,6 @@ func (_c *MockClientStorage_DeleteSpaceStorage_Call) RunAndReturn(run func(conte
|
|||
return _c
|
||||
}
|
||||
|
||||
// GetSpaceID provides a mock function with given fields: objectID
|
||||
func (_m *MockClientStorage) GetSpaceID(objectID string) (string, error) {
|
||||
ret := _m.Called(objectID)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for GetSpaceID")
|
||||
}
|
||||
|
||||
var r0 string
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(string) (string, error)); ok {
|
||||
return rf(objectID)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(string) string); ok {
|
||||
r0 = rf(objectID)
|
||||
} else {
|
||||
r0 = ret.Get(0).(string)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(objectID)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockClientStorage_GetSpaceID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSpaceID'
|
||||
type MockClientStorage_GetSpaceID_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// GetSpaceID is a helper method to define mock.On call
|
||||
// - objectID string
|
||||
func (_e *MockClientStorage_Expecter) GetSpaceID(objectID interface{}) *MockClientStorage_GetSpaceID_Call {
|
||||
return &MockClientStorage_GetSpaceID_Call{Call: _e.mock.On("GetSpaceID", objectID)}
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_GetSpaceID_Call) Run(run func(objectID string)) *MockClientStorage_GetSpaceID_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(string))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_GetSpaceID_Call) Return(spaceID string, err error) *MockClientStorage_GetSpaceID_Call {
|
||||
_c.Call.Return(spaceID, err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_GetSpaceID_Call) RunAndReturn(run func(string) (string, error)) *MockClientStorage_GetSpaceID_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Init provides a mock function with given fields: a
|
||||
func (_m *MockClientStorage) Init(a *app.App) error {
|
||||
ret := _m.Called(a)
|
||||
|
@ -382,98 +280,6 @@ func (_c *MockClientStorage_Init_Call) RunAndReturn(run func(*app.App) error) *M
|
|||
return _c
|
||||
}
|
||||
|
||||
// IsSpaceCreated provides a mock function with given fields: id
|
||||
func (_m *MockClientStorage) IsSpaceCreated(id string) bool {
|
||||
ret := _m.Called(id)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for IsSpaceCreated")
|
||||
}
|
||||
|
||||
var r0 bool
|
||||
if rf, ok := ret.Get(0).(func(string) bool); ok {
|
||||
r0 = rf(id)
|
||||
} else {
|
||||
r0 = ret.Get(0).(bool)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockClientStorage_IsSpaceCreated_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IsSpaceCreated'
|
||||
type MockClientStorage_IsSpaceCreated_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// IsSpaceCreated is a helper method to define mock.On call
|
||||
// - id string
|
||||
func (_e *MockClientStorage_Expecter) IsSpaceCreated(id interface{}) *MockClientStorage_IsSpaceCreated_Call {
|
||||
return &MockClientStorage_IsSpaceCreated_Call{Call: _e.mock.On("IsSpaceCreated", id)}
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_IsSpaceCreated_Call) Run(run func(id string)) *MockClientStorage_IsSpaceCreated_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(string))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_IsSpaceCreated_Call) Return(created bool) *MockClientStorage_IsSpaceCreated_Call {
|
||||
_c.Call.Return(created)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_IsSpaceCreated_Call) RunAndReturn(run func(string) bool) *MockClientStorage_IsSpaceCreated_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// MarkSpaceCreated provides a mock function with given fields: id
|
||||
func (_m *MockClientStorage) MarkSpaceCreated(id string) error {
|
||||
ret := _m.Called(id)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for MarkSpaceCreated")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string) error); ok {
|
||||
r0 = rf(id)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockClientStorage_MarkSpaceCreated_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MarkSpaceCreated'
|
||||
type MockClientStorage_MarkSpaceCreated_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// MarkSpaceCreated is a helper method to define mock.On call
|
||||
// - id string
|
||||
func (_e *MockClientStorage_Expecter) MarkSpaceCreated(id interface{}) *MockClientStorage_MarkSpaceCreated_Call {
|
||||
return &MockClientStorage_MarkSpaceCreated_Call{Call: _e.mock.On("MarkSpaceCreated", id)}
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_MarkSpaceCreated_Call) Run(run func(id string)) *MockClientStorage_MarkSpaceCreated_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(string))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_MarkSpaceCreated_Call) Return(err error) *MockClientStorage_MarkSpaceCreated_Call {
|
||||
_c.Call.Return(err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_MarkSpaceCreated_Call) RunAndReturn(run func(string) error) *MockClientStorage_MarkSpaceCreated_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Name provides a mock function with given fields:
|
||||
func (_m *MockClientStorage) Name() string {
|
||||
ret := _m.Called()
|
||||
|
@ -611,52 +417,6 @@ func (_c *MockClientStorage_SpaceExists_Call) RunAndReturn(run func(string) bool
|
|||
return _c
|
||||
}
|
||||
|
||||
// UnmarkSpaceCreated provides a mock function with given fields: id
|
||||
func (_m *MockClientStorage) UnmarkSpaceCreated(id string) error {
|
||||
ret := _m.Called(id)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for UnmarkSpaceCreated")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string) error); ok {
|
||||
r0 = rf(id)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// MockClientStorage_UnmarkSpaceCreated_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UnmarkSpaceCreated'
|
||||
type MockClientStorage_UnmarkSpaceCreated_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// UnmarkSpaceCreated is a helper method to define mock.On call
|
||||
// - id string
|
||||
func (_e *MockClientStorage_Expecter) UnmarkSpaceCreated(id interface{}) *MockClientStorage_UnmarkSpaceCreated_Call {
|
||||
return &MockClientStorage_UnmarkSpaceCreated_Call{Call: _e.mock.On("UnmarkSpaceCreated", id)}
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_UnmarkSpaceCreated_Call) Run(run func(id string)) *MockClientStorage_UnmarkSpaceCreated_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(string))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_UnmarkSpaceCreated_Call) Return(err error) *MockClientStorage_UnmarkSpaceCreated_Call {
|
||||
_c.Call.Return(err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockClientStorage_UnmarkSpaceCreated_Call) RunAndReturn(run func(string) error) *MockClientStorage_UnmarkSpaceCreated_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// WaitSpaceStorage provides a mock function with given fields: ctx, id
|
||||
func (_m *MockClientStorage) WaitSpaceStorage(ctx context.Context, id string) (spacestorage.SpaceStorage, error) {
|
||||
ret := _m.Called(ctx, id)
|
||||
|
|
|
@ -6,12 +6,14 @@ import (
|
|||
"errors"
|
||||
"sync"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
|
||||
"github.com/anyproto/any-sync/consensus/consensusproto"
|
||||
)
|
||||
|
||||
func newListStorage(ss *spaceStorage, treeId string) (liststorage.ListStorage, error) {
|
||||
var ErrUnknownRecord = errors.New("record does not exist")
|
||||
|
||||
func newListStorage(ss *spaceStorage, treeId string) (oldstorage.ListStorage, error) {
|
||||
ts := &listStorage{
|
||||
listId: treeId,
|
||||
spaceStorage: ss,
|
||||
|
@ -38,7 +40,7 @@ type listStorage struct {
|
|||
func (t *listStorage) Root() (*consensusproto.RawRecordWithId, error) {
|
||||
tch, err := t.spaceStorage.TreeRoot(t.listId)
|
||||
if err != nil {
|
||||
return nil, replaceNoRowsErr(err, liststorage.ErrUnknownRecord)
|
||||
return nil, replaceNoRowsErr(err, ErrUnknownRecord)
|
||||
}
|
||||
return &consensusproto.RawRecordWithId{
|
||||
Payload: tch.RawChange,
|
||||
|
@ -66,7 +68,7 @@ func (t *listStorage) SetHead(headId string) error {
|
|||
func (t *listStorage) GetRawRecord(ctx context.Context, id string) (*consensusproto.RawRecordWithId, error) {
|
||||
tch, err := t.spaceStorage.TreeRoot(id)
|
||||
if err != nil {
|
||||
return nil, replaceNoRowsErr(err, liststorage.ErrUnknownRecord)
|
||||
return nil, replaceNoRowsErr(err, ErrUnknownRecord)
|
||||
}
|
||||
return &consensusproto.RawRecordWithId{
|
||||
Payload: tch.RawChange,
|
||||
|
|
|
@ -3,7 +3,7 @@ package sqlitestorage
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
|
||||
"github.com/anyproto/any-sync/consensus/consensusproto"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
@ -56,7 +56,7 @@ func TestListStorage_SetHead(t *testing.T) {
|
|||
|
||||
}
|
||||
|
||||
func testList(t *testing.T, store liststorage.ListStorage, root *consensusproto.RawRecordWithId, head string) {
|
||||
func testList(t *testing.T, store oldstorage.ListStorage, root *consensusproto.RawRecordWithId, head string) {
|
||||
require.Equal(t, store.Id(), root.Id)
|
||||
|
||||
aclRoot, err := store.Root()
|
||||
|
|
|
@ -5,6 +5,7 @@ import (
|
|||
"database/sql"
|
||||
"errors"
|
||||
"net/url"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
|
@ -12,6 +13,7 @@ import (
|
|||
"github.com/anyproto/any-sync/app/logger"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
|
||||
"github.com/globalsign/mgo/bson"
|
||||
"github.com/mattn/go-sqlite3"
|
||||
"go.uber.org/atomic"
|
||||
|
@ -30,7 +32,7 @@ var (
|
|||
)
|
||||
|
||||
type configGetter interface {
|
||||
GetSpaceStorePath() string
|
||||
GetSqliteStorePath() string
|
||||
GetTempDirPath() string
|
||||
}
|
||||
|
||||
|
@ -51,6 +53,8 @@ type storageService struct {
|
|||
allTreeDelStatus,
|
||||
change,
|
||||
hasTree,
|
||||
listChanges,
|
||||
iterateChanges,
|
||||
hasChange,
|
||||
updateTreeHeads,
|
||||
deleteTree,
|
||||
|
@ -60,6 +64,7 @@ type storageService struct {
|
|||
spaceIds,
|
||||
spaceIsCreated,
|
||||
upsertBind,
|
||||
getAllBinds,
|
||||
deleteSpace,
|
||||
deleteTreesBySpace,
|
||||
deleteChangesBySpace,
|
||||
|
@ -91,7 +96,7 @@ func New() *storageService {
|
|||
}
|
||||
|
||||
func (s *storageService) Init(a *app.App) (err error) {
|
||||
s.dbPath = a.MustComponent("config").(configGetter).GetSpaceStorePath()
|
||||
s.dbPath = a.MustComponent("config").(configGetter).GetSqliteStorePath()
|
||||
s.dbTempPath = a.MustComponent("config").(configGetter).GetTempDirPath()
|
||||
s.lockedSpaces = map[string]*lockSpace{}
|
||||
if s.checkpointAfterWrite == 0 {
|
||||
|
@ -159,7 +164,7 @@ func (s *storageService) Name() (name string) {
|
|||
return spacestorage.CName
|
||||
}
|
||||
|
||||
func (s *storageService) WaitSpaceStorage(ctx context.Context, id string) (store spacestorage.SpaceStorage, err error) {
|
||||
func (s *storageService) WaitSpaceStorage(ctx context.Context, id string) (store oldstorage.SpaceStorage, err error) {
|
||||
var ls *lockSpace
|
||||
ls, err = s.checkLock(id, func() error {
|
||||
store, err = newSpaceStorage(s, id)
|
||||
|
@ -277,7 +282,7 @@ func (s *storageService) unlockSpaceStorage(id string) {
|
|||
}
|
||||
}
|
||||
|
||||
func (s *storageService) CreateSpaceStorage(payload spacestorage.SpaceStorageCreatePayload) (ss spacestorage.SpaceStorage, err error) {
|
||||
func (s *storageService) CreateSpaceStorage(payload spacestorage.SpaceStorageCreatePayload) (ss oldstorage.SpaceStorage, err error) {
|
||||
_, err = s.checkLock(payload.SpaceHeaderWithId.Id, func() error {
|
||||
ss, err = createSpaceStorage(s, payload)
|
||||
return err
|
||||
|
@ -291,6 +296,25 @@ func (s *storageService) GetSpaceID(objectID string) (spaceID string, err error)
|
|||
return
|
||||
}
|
||||
|
||||
func (s *storageService) GetBoundObjectIds(spaceId string) ([]string, error) {
|
||||
rows, err := s.stmt.getAllBinds.Query(spaceId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
err = errors.Join(rows.Close())
|
||||
}()
|
||||
var ids []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err = rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (s *storageService) BindSpaceID(spaceID, objectID string) (err error) {
|
||||
_, err = s.stmt.upsertBind.Exec(objectID, spaceID, spaceID)
|
||||
return
|
||||
|
@ -352,6 +376,15 @@ func (s *storageService) checkpoint() (err error) {
|
|||
return err
|
||||
}
|
||||
|
||||
func (s *storageService) EstimateSize() (uint64, error) {
|
||||
stat, err := os.Stat(s.dbPath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// nolint: gosec
|
||||
return uint64(stat.Size()), nil
|
||||
}
|
||||
|
||||
func (s *storageService) Close(ctx context.Context) (err error) {
|
||||
if s.ctxCancel != nil {
|
||||
s.ctxCancel()
|
||||
|
|
|
@ -28,6 +28,34 @@ func TestStorageService_BindSpaceID(t *testing.T) {
|
|||
assert.Equal(t, "spaceId2", spaceId)
|
||||
}
|
||||
|
||||
func TestStorageService_GetBoundObjectIds(t *testing.T) {
|
||||
t.Run("no bindings", func(t *testing.T) {
|
||||
fx := newFixture(t)
|
||||
defer fx.finish(t)
|
||||
|
||||
spaceId := "spaceId"
|
||||
|
||||
ids, err := fx.GetBoundObjectIds(spaceId)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Empty(t, ids)
|
||||
})
|
||||
|
||||
t.Run("ok", func(t *testing.T) {
|
||||
fx := newFixture(t)
|
||||
defer fx.finish(t)
|
||||
|
||||
spaceId := "spaceId"
|
||||
require.NoError(t, fx.BindSpaceID(spaceId, "objectId1"))
|
||||
require.NoError(t, fx.BindSpaceID(spaceId, "objectId2"))
|
||||
|
||||
ids, err := fx.GetBoundObjectIds(spaceId)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.ElementsMatch(t, []string{"objectId1", "objectId2"}, ids)
|
||||
})
|
||||
}
|
||||
|
||||
func TestStorageService_DeleteSpaceStorage(t *testing.T) {
|
||||
fx := newFixture(t)
|
||||
defer fx.finish(t)
|
||||
|
@ -192,7 +220,7 @@ type testConfig struct {
|
|||
tmpDir string
|
||||
}
|
||||
|
||||
func (t *testConfig) GetSpaceStorePath() string {
|
||||
func (t *testConfig) GetSqliteStorePath() string {
|
||||
return filepath.Join(t.tmpDir, "spaceStore.db")
|
||||
}
|
||||
func (t *testConfig) GetTempDirPath() string {
|
||||
|
|
|
@ -7,10 +7,10 @@ import (
|
|||
"sync"
|
||||
|
||||
"github.com/anyproto/any-sync/app"
|
||||
"github.com/anyproto/any-sync/commonspace/object/acl/liststorage"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
@ -34,12 +34,12 @@ type spaceStorage struct {
|
|||
|
||||
isDeleted bool
|
||||
|
||||
aclStorage liststorage.ListStorage
|
||||
aclStorage oldstorage.ListStorage
|
||||
|
||||
header *spacesyncproto.RawSpaceHeaderWithId
|
||||
}
|
||||
|
||||
func newSpaceStorage(s *storageService, spaceId string) (spacestorage.SpaceStorage, error) {
|
||||
func newSpaceStorage(s *storageService, spaceId string) (oldstorage.SpaceStorage, error) {
|
||||
ss := &spaceStorage{
|
||||
spaceId: spaceId,
|
||||
service: s,
|
||||
|
@ -64,7 +64,7 @@ func newSpaceStorage(s *storageService, spaceId string) (spacestorage.SpaceStora
|
|||
return ss, nil
|
||||
}
|
||||
|
||||
func createSpaceStorage(s *storageService, payload spacestorage.SpaceStorageCreatePayload) (spacestorage.SpaceStorage, error) {
|
||||
func createSpaceStorage(s *storageService, payload spacestorage.SpaceStorageCreatePayload) (oldstorage.SpaceStorage, error) {
|
||||
tx, err := s.writeDb.Begin()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -194,7 +194,7 @@ func (s *spaceStorage) TreeDeletedStatus(id string) (status string, err error) {
|
|||
}
|
||||
|
||||
func (s *spaceStorage) AllDeletedTreeIds() (ids []string, err error) {
|
||||
rows, err := s.service.stmt.allTreeDelStatus.Query(s.spaceId, spacestorage.TreeDeletedStatusDeleted)
|
||||
rows, err := s.service.stmt.allTreeDelStatus.Query(s.spaceId, oldstorage.TreeDeletedStatusDeleted)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
|
@ -218,7 +218,7 @@ func (s *spaceStorage) SpaceSettingsId() string {
|
|||
return s.spaceSettingsId
|
||||
}
|
||||
|
||||
func (s *spaceStorage) AclStorage() (liststorage.ListStorage, error) {
|
||||
func (s *spaceStorage) AclStorage() (oldstorage.ListStorage, error) {
|
||||
return s.aclStorage, nil
|
||||
}
|
||||
|
||||
|
@ -258,7 +258,7 @@ func (s *spaceStorage) TreeRoot(id string) (*treechangeproto.RawTreeChangeWithId
|
|||
}, nil
|
||||
}
|
||||
|
||||
func (s *spaceStorage) TreeStorage(id string) (treestorage.TreeStorage, error) {
|
||||
func (s *spaceStorage) TreeStorage(id string) (oldstorage.TreeStorage, error) {
|
||||
return newTreeStorage(s, id)
|
||||
}
|
||||
|
||||
|
@ -271,7 +271,7 @@ func (s *spaceStorage) HasTree(id string) (bool, error) {
|
|||
return res > 0, nil
|
||||
}
|
||||
|
||||
func (s *spaceStorage) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (treestorage.TreeStorage, error) {
|
||||
func (s *spaceStorage) CreateTreeStorage(payload treestorage.TreeStorageCreatePayload) (oldstorage.TreeStorage, error) {
|
||||
return createTreeStorage(s, payload)
|
||||
}
|
||||
|
||||
|
|
|
@ -6,6 +6,7 @@ import (
|
|||
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacesyncproto"
|
||||
"github.com/anyproto/any-sync/consensus/consensusproto"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
@ -90,7 +91,7 @@ func TestSpaceStorage_NewAndCreateTree(t *testing.T) {
|
|||
|
||||
treeIds, err := store.StoredIds()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{payload.SpaceSettingsWithId.Id, otherStore.Id()}, treeIds)
|
||||
assert.Equal(t, []string{payload.SpaceSettingsWithId.Id}, treeIds)
|
||||
|
||||
deletedIds, err := store.(*spaceStorage).AllDeletedTreeIds()
|
||||
require.NoError(t, err)
|
||||
|
@ -106,11 +107,11 @@ func TestSpaceStorage_AllDeletedTreeIds(t *testing.T) {
|
|||
store, err := createSpaceStorage(fx.storageService, payload)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = store.SetTreeDeletedStatus("id1", spacestorage.TreeDeletedStatusDeleted)
|
||||
err = store.SetTreeDeletedStatus("id1", oldstorage.TreeDeletedStatusDeleted)
|
||||
require.NoError(t, err)
|
||||
err = store.SetTreeDeletedStatus("id2", spacestorage.TreeDeletedStatusQueued)
|
||||
err = store.SetTreeDeletedStatus("id2", oldstorage.TreeDeletedStatusQueued)
|
||||
require.NoError(t, err)
|
||||
err = store.SetTreeDeletedStatus("id3", spacestorage.TreeDeletedStatusDeleted)
|
||||
err = store.SetTreeDeletedStatus("id3", oldstorage.TreeDeletedStatusDeleted)
|
||||
require.NoError(t, err)
|
||||
|
||||
deletedIds, err := store.(*spaceStorage).AllDeletedTreeIds()
|
||||
|
@ -127,12 +128,12 @@ func TestSpaceStorage_SetTreeDeletedStatus(t *testing.T) {
|
|||
store, err := createSpaceStorage(fx.storageService, payload)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = store.SetTreeDeletedStatus("treeId", spacestorage.TreeDeletedStatusDeleted)
|
||||
err = store.SetTreeDeletedStatus("treeId", oldstorage.TreeDeletedStatusDeleted)
|
||||
require.NoError(t, err)
|
||||
|
||||
status, err := store.TreeDeletedStatus("treeId")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, spacestorage.TreeDeletedStatusDeleted, status)
|
||||
require.Equal(t, oldstorage.TreeDeletedStatusDeleted, status)
|
||||
|
||||
_, err = store.TreeStorage("treeId")
|
||||
require.ErrorIs(t, err, treestorage.ErrUnknownTreeId)
|
||||
|
@ -202,7 +203,7 @@ func TestSpaceStorage_ReadSpaceHash(t *testing.T) {
|
|||
|
||||
require.NoError(t, ss.WriteSpaceHash("hash"))
|
||||
|
||||
var checkHashes = func(ss spacestorage.SpaceStorage) {
|
||||
var checkHashes = func(ss oldstorage.SpaceStorage) {
|
||||
hash, err = ss.ReadSpaceHash()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "hash", hash)
|
||||
|
@ -238,7 +239,7 @@ func spaceTestPayload() spacestorage.SpaceStorageCreatePayload {
|
|||
}
|
||||
}
|
||||
|
||||
func testSpace(t *testing.T, store spacestorage.SpaceStorage, payload spacestorage.SpaceStorageCreatePayload) {
|
||||
func testSpace(t *testing.T, store oldstorage.SpaceStorage, payload spacestorage.SpaceStorageCreatePayload) {
|
||||
header, err := store.SpaceHeader()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, payload.SpaceHeaderWithId, header)
|
||||
|
|
|
@ -58,7 +58,7 @@ func initStmts(s *storageService) (err error) {
|
|||
if s.stmt.updateSpaceIsDeleted, err = s.writeDb.Prepare(`UPDATE spaces SET isDeleted = ? WHERE id = ?`); err != nil {
|
||||
return
|
||||
}
|
||||
if s.stmt.treeIdsBySpace, err = s.readDb.Prepare(`SELECT id FROM trees WHERE spaceId = ? AND type != 1`); err != nil {
|
||||
if s.stmt.treeIdsBySpace, err = s.readDb.Prepare(`SELECT id FROM trees WHERE spaceId = ? AND type != 1 AND deleteStatus IS NULL`); err != nil {
|
||||
return
|
||||
}
|
||||
if s.stmt.deleteTree, err = s.writeDb.Prepare(`
|
||||
|
@ -88,6 +88,12 @@ func initStmts(s *storageService) (err error) {
|
|||
if s.stmt.hasChange, err = s.readDb.Prepare(`SELECT COUNT(*) FROM changes WHERE id = ? AND treeId = ?`); err != nil {
|
||||
return
|
||||
}
|
||||
if s.stmt.listChanges, err = s.readDb.Prepare(`SELECT id FROM changes WHERE treeId = ? ORDER BY id`); err != nil {
|
||||
return
|
||||
}
|
||||
if s.stmt.iterateChanges, err = s.readDb.Prepare(`SELECT id, data FROM changes WHERE treeId = ? ORDER BY id`); err != nil {
|
||||
return
|
||||
}
|
||||
if s.stmt.updateTreeHeads, err = s.writeDb.Prepare(`UPDATE trees SET heads = ? WHERE id = ?`); err != nil {
|
||||
return
|
||||
}
|
||||
|
@ -112,6 +118,9 @@ func initStmts(s *storageService) (err error) {
|
|||
if s.stmt.upsertBind, err = s.writeDb.Prepare(`INSERT INTO binds (objectId, spaceId) VALUES (?, ?) ON CONFLICT (objectId) DO UPDATE SET spaceId = ?`); err != nil {
|
||||
return
|
||||
}
|
||||
if s.stmt.getAllBinds, err = s.writeDb.Prepare(`SELECT objectId FROM binds WHERE spaceId = ?`); err != nil {
|
||||
return
|
||||
}
|
||||
if s.stmt.deleteSpace, err = s.writeDb.Prepare(`DELETE FROM spaces WHERE id = ?`); err != nil {
|
||||
return
|
||||
}
|
||||
|
|
|
@ -1,15 +1,18 @@
|
|||
package sqlitestorage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
|
||||
)
|
||||
|
||||
func newTreeStorage(ss *spaceStorage, treeId string) (treestorage.TreeStorage, error) {
|
||||
func newTreeStorage(ss *spaceStorage, treeId string) (oldstorage.TreeStorage, error) {
|
||||
ts := &treeStorage{
|
||||
treeId: treeId,
|
||||
spaceStorage: ss,
|
||||
|
@ -23,7 +26,7 @@ func newTreeStorage(ss *spaceStorage, treeId string) (treestorage.TreeStorage, e
|
|||
return ts, nil
|
||||
}
|
||||
|
||||
func createTreeStorage(ss *spaceStorage, payload treestorage.TreeStorageCreatePayload) (ts treestorage.TreeStorage, err error) {
|
||||
func createTreeStorage(ss *spaceStorage, payload treestorage.TreeStorageCreatePayload) (ts oldstorage.TreeStorage, err error) {
|
||||
ts = &treeStorage{
|
||||
treeId: payload.RootRawChange.Id,
|
||||
spaceStorage: ss,
|
||||
|
@ -99,7 +102,54 @@ func (t *treeStorage) Heads() ([]string, error) {
|
|||
}
|
||||
|
||||
func (t *treeStorage) GetAllChangeIds() (chs []string, err error) {
|
||||
return nil, fmt.Errorf("get all change ids should not be called")
|
||||
rows, err := t.service.stmt.listChanges.Query(t.treeId)
|
||||
if err != nil {
|
||||
return nil, replaceNoRowsErr(err, nil)
|
||||
}
|
||||
for rows.Next() {
|
||||
var id string
|
||||
err = rows.Scan(&id)
|
||||
if err != nil {
|
||||
return nil, errors.Join(rows.Close(), err)
|
||||
}
|
||||
chs = append(chs, id)
|
||||
}
|
||||
return chs, rows.Close()
|
||||
}
|
||||
|
||||
func (t *treeStorage) GetAllChanges() ([]*treechangeproto.RawTreeChangeWithId, error) {
|
||||
var changes []*treechangeproto.RawTreeChangeWithId
|
||||
err := t.IterateChanges(func(id string, rawChange []byte) error {
|
||||
changes = append(changes, &treechangeproto.RawTreeChangeWithId{
|
||||
Id: id,
|
||||
RawChange: bytes.Clone(rawChange),
|
||||
})
|
||||
return nil
|
||||
})
|
||||
return changes, err
|
||||
}
|
||||
|
||||
func (t *treeStorage) IterateChanges(proc func(id string, rawChange []byte) error) error {
|
||||
rows, err := t.service.stmt.iterateChanges.Query(t.treeId)
|
||||
if err != nil {
|
||||
return replaceNoRowsErr(err, nil)
|
||||
}
|
||||
|
||||
buf := make([]byte, 0, 1024)
|
||||
for rows.Next() {
|
||||
var id string
|
||||
err = rows.Scan(&id, &buf)
|
||||
if err != nil {
|
||||
return errors.Join(rows.Close(), err)
|
||||
}
|
||||
|
||||
err = proc(id, buf)
|
||||
if err != nil {
|
||||
return errors.Join(rows.Close(), err)
|
||||
}
|
||||
buf = buf[:0]
|
||||
}
|
||||
return rows.Close()
|
||||
}
|
||||
|
||||
func (t *treeStorage) SetHeads(heads []string) error {
|
||||
|
|
|
@ -1,17 +1,26 @@
|
|||
package sqlitestorage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"slices"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto"
|
||||
"github.com/anyproto/any-sync/commonspace/object/tree/treestorage"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage/oldstorage"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type oldTreeStorage interface {
|
||||
oldstorage.ChangesIterator
|
||||
oldstorage.TreeStorage
|
||||
}
|
||||
|
||||
func TestTreeStorage_Create(t *testing.T) {
|
||||
fx := newFixture(t)
|
||||
defer fx.finish(t)
|
||||
|
@ -38,11 +47,13 @@ func TestTreeStorage_Methods(t *testing.T) {
|
|||
ss, err := createSpaceStorage(fx.storageService, spacePayload)
|
||||
require.NoError(t, err)
|
||||
payload := treeTestPayload()
|
||||
store, err := ss.CreateTreeStorage(payload)
|
||||
var store oldTreeStorage
|
||||
treeStore, err := ss.CreateTreeStorage(payload)
|
||||
require.NoError(t, err)
|
||||
|
||||
store, err = ss.TreeStorage(payload.RootRawChange.Id)
|
||||
treeStore, err = ss.TreeStorage(payload.RootRawChange.Id)
|
||||
require.NoError(t, err)
|
||||
store = treeStore.(oldTreeStorage)
|
||||
testTreePayload(t, store, payload)
|
||||
|
||||
t.Run("update heads", func(t *testing.T) {
|
||||
|
@ -54,7 +65,7 @@ func TestTreeStorage_Methods(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("add raw change, get change and has change", func(t *testing.T) {
|
||||
newChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("ab"), Id: "newId"}
|
||||
newChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("ab"), Id: "id10"}
|
||||
require.NoError(t, store.AddRawChange(newChange))
|
||||
rawCh, err := store.GetRawChange(ctx, newChange.Id)
|
||||
require.NoError(t, err)
|
||||
|
@ -72,6 +83,55 @@ func TestTreeStorage_Methods(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
require.False(t, has)
|
||||
})
|
||||
|
||||
t.Run("iterate changes", func(t *testing.T) {
|
||||
newChange := &treechangeproto.RawTreeChangeWithId{RawChange: []byte("foo"), Id: "id01"}
|
||||
require.NoError(t, store.AddRawChange(newChange))
|
||||
newChange = &treechangeproto.RawTreeChangeWithId{RawChange: []byte("bar"), Id: "id20"}
|
||||
require.NoError(t, store.AddRawChange(newChange))
|
||||
|
||||
var collected []*treechangeproto.RawTreeChangeWithId
|
||||
require.NoError(t, store.IterateChanges(func(id string, rawChange []byte) error {
|
||||
collected = append(collected, &treechangeproto.RawTreeChangeWithId{
|
||||
Id: id,
|
||||
RawChange: bytes.Clone(rawChange),
|
||||
})
|
||||
return nil
|
||||
}))
|
||||
|
||||
want := slices.Clone(payload.Changes)
|
||||
want = append(want, []*treechangeproto.RawTreeChangeWithId{
|
||||
{Id: "id01", RawChange: []byte("foo")},
|
||||
{Id: "id10", RawChange: []byte("ab")},
|
||||
{Id: "id20", RawChange: []byte("bar")},
|
||||
}...)
|
||||
|
||||
sort.Slice(want, func(i, j int) bool {
|
||||
return want[i].Id < want[j].Id
|
||||
})
|
||||
assert.Equal(t, want, collected)
|
||||
|
||||
got, err := store.GetAllChanges()
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, want, got)
|
||||
})
|
||||
|
||||
t.Run("get all change ids", func(t *testing.T) {
|
||||
got, err := store.GetAllChangeIds()
|
||||
require.NoError(t, err)
|
||||
|
||||
want := []string{
|
||||
payload.Changes[0].Id,
|
||||
payload.Changes[1].Id,
|
||||
"id01",
|
||||
"id10",
|
||||
"id20",
|
||||
}
|
||||
sort.Strings(want)
|
||||
|
||||
assert.Equal(t, want, got)
|
||||
})
|
||||
}
|
||||
|
||||
func TestTreeStorage_AddRawChangesSetHeads(t *testing.T) {
|
||||
|
@ -190,7 +250,7 @@ func treeTestPayload() treestorage.TreeStorageCreatePayload {
|
|||
}
|
||||
}
|
||||
|
||||
func testTreePayload(t *testing.T, store treestorage.TreeStorage, payload treestorage.TreeStorageCreatePayload) {
|
||||
func testTreePayload(t *testing.T, store oldstorage.TreeStorage, payload treestorage.TreeStorageCreatePayload) {
|
||||
require.Equal(t, payload.RootRawChange.Id, store.Id())
|
||||
|
||||
root, err := store.Root()
|
||||
|
|
|
@ -2,13 +2,12 @@ package storage
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
anystore "github.com/anyproto/any-store"
|
||||
"github.com/anyproto/any-sync/app"
|
||||
"github.com/anyproto/any-sync/commonspace/spacestorage"
|
||||
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage/badgerstorage"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage/sqlitestorage"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage/anystorage"
|
||||
)
|
||||
|
||||
type SpaceStorageMode int
|
||||
|
@ -22,12 +21,7 @@ type ClientStorage interface {
|
|||
spacestorage.SpaceStorageProvider
|
||||
app.ComponentRunnable
|
||||
AllSpaceIds() (ids []string, err error)
|
||||
GetSpaceID(objectID string) (spaceID string, err error)
|
||||
BindSpaceID(spaceID, objectID string) (err error)
|
||||
DeleteSpaceStorage(ctx context.Context, spaceId string) error
|
||||
MarkSpaceCreated(id string) (err error)
|
||||
UnmarkSpaceCreated(id string) (err error)
|
||||
IsSpaceCreated(id string) (created bool)
|
||||
}
|
||||
|
||||
// storageService is a proxy for the actual storage implementation
|
||||
|
@ -40,7 +34,8 @@ func New() ClientStorage {
|
|||
}
|
||||
|
||||
type configGetter interface {
|
||||
GetSpaceStorageMode() SpaceStorageMode
|
||||
GetNewSpaceStorePath() string
|
||||
GetAnyStoreConfig() *anystore.Config
|
||||
}
|
||||
|
||||
func (s *storageService) Name() (name string) {
|
||||
|
@ -48,16 +43,7 @@ func (s *storageService) Name() (name string) {
|
|||
}
|
||||
|
||||
func (s *storageService) Init(a *app.App) (err error) {
|
||||
mode := a.MustComponent("config").(configGetter).GetSpaceStorageMode()
|
||||
if mode == SpaceStorageModeBadger {
|
||||
// for already existing account repos
|
||||
s.ClientStorage = badgerstorage.New()
|
||||
} else if mode == SpaceStorageModeSqlite {
|
||||
// sqlite used for new account repos
|
||||
s.ClientStorage = sqlitestorage.New()
|
||||
} else {
|
||||
return fmt.Errorf("unknown storage mode %d", mode)
|
||||
}
|
||||
|
||||
getter := a.MustComponent("config").(configGetter)
|
||||
s.ClientStorage = anystorage.New(getter.GetNewSpaceStorePath(), getter.GetAnyStoreConfig())
|
||||
return s.ClientStorage.Init(a)
|
||||
}
|
||||
|
|
|
@ -25,6 +25,7 @@ import (
|
|||
"github.com/anyproto/anytype-heart/pkg/lib/logging"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage/anystorage"
|
||||
)
|
||||
|
||||
const CName = "space.typeprovider"
|
||||
|
@ -196,17 +197,15 @@ func (p *provider) objectTypeFromSpace(spaceID string, id string) (tp smartblock
|
|||
return
|
||||
}
|
||||
p.RUnlock()
|
||||
|
||||
sp, err := p.spaceService.Get(context.Background(), spaceID)
|
||||
ctx := context.Background()
|
||||
sp, err := p.spaceService.Get(ctx, spaceID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
store := sp.Storage()
|
||||
rawRoot, err := store.TreeRoot(id)
|
||||
rawRoot, err := sp.Storage().(anystorage.ClientSpaceStorage).TreeRoot(ctx, id)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
tp, _, err = GetTypeAndKeyFromRoot(rawRoot)
|
||||
if err != nil {
|
||||
return
|
||||
|
|
|
@ -17,6 +17,7 @@ import (
|
|||
"github.com/anyproto/anytype-heart/space/internal/spacecontroller"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage"
|
||||
"github.com/anyproto/anytype-heart/space/spacecore/storage/anystorage"
|
||||
"github.com/anyproto/anytype-heart/space/spaceinfo"
|
||||
"github.com/anyproto/anytype-heart/space/techspace"
|
||||
)
|
||||
|
@ -71,7 +72,7 @@ func (s *spaceFactory) CreatePersonalSpace(ctx context.Context, metadata []byte)
|
|||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = s.storageService.MarkSpaceCreated(coreSpace.Id())
|
||||
err = coreSpace.Storage().(anystorage.ClientSpaceStorage).MarkSpaceCreated(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
@ -192,7 +193,11 @@ func (s *spaceFactory) CreateInvitingSpace(ctx context.Context, id, aclHeadId st
|
|||
}
|
||||
|
||||
func (s *spaceFactory) CreateShareableSpace(ctx context.Context, id string) (sp spacecontroller.SpaceController, err error) {
|
||||
err = s.storageService.MarkSpaceCreated(id)
|
||||
coreSpace, err := s.spaceCore.Get(ctx, id)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = coreSpace.Storage().(anystorage.ClientSpaceStorage).MarkSpaceCreated(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue