1
0
Fork 0
mirror of https://github.com/anyproto/anytype-heart.git synced 2025-06-09 09:35:00 +09:00

GO-2875 Merge branch 'main' into GO-2875-add-space-account-status-removing

# Conflicts:
#	core/anytype/bootstrap.go
This commit is contained in:
mcrakhman 2024-02-16 10:30:45 +01:00
commit c486bfb1ba
No known key found for this signature in database
GPG key ID: DED12CFEF5B8396B
19 changed files with 276 additions and 34 deletions

View file

@ -26,6 +26,7 @@ on:
permissions:
actions: 'write'
packages: 'write'
contents: 'write'
name: Build

View file

@ -431,21 +431,25 @@ func (a *aclService) GenerateInvite(ctx context.Context, spaceId string) (result
if err != nil {
return nil, err
}
err = aclClient.AddRecord(ctx, res.InviteRec)
if err != nil {
return nil, fmt.Errorf("%w, %w", ErrAclRequestFailed, err)
}
invite, err := a.buildInvite(ctx, acceptSpace, res.InviteKey)
if err != nil {
return nil, fmt.Errorf("build invite: %w", err)
}
inviteFileCid, inviteFileKey, err := a.inviteStore.StoreInvite(ctx, spaceId, invite)
inviteFileCid, inviteFileKey, err := a.inviteStore.StoreInvite(ctx, invite)
if err != nil {
return nil, fmt.Errorf("store invite in ipfs: %w", err)
}
removeInviteFile := func() {
err := a.inviteStore.RemoveInvite(ctx, inviteFileCid)
if err != nil {
log.Error("remove invite file", zap.Error(err))
}
}
inviteFileKeyRaw, err := EncodeKeyToBase58(inviteFileKey)
if err != nil {
removeInviteFile()
return nil, fmt.Errorf("encode invite file key: %w", err)
}
err = getblock.Do(a.objectGetter, spaceViewId, func(sb smartblock.SmartBlock) error {
@ -456,9 +460,16 @@ func (a *aclService) GenerateInvite(ctx context.Context, spaceId string) (result
return view.SetInviteFileInfo(inviteFileCid.String(), inviteFileKeyRaw)
})
if err != nil {
removeInviteFile()
return nil, fmt.Errorf("set invite file info: %w", err)
}
err = aclClient.AddRecord(ctx, res.InviteRec)
if err != nil {
removeInviteFile()
return nil, fmt.Errorf("%w, %w", ErrAclRequestFailed, err)
}
return &InviteInfo{
InviteFileCid: inviteFileCid.String(),
InviteFileKey: inviteFileKeyRaw,

View file

@ -226,11 +226,11 @@ func Bootstrap(a *app.App, components ...app.Component) {
Register(filestorage.New()).
Register(files.New()).
Register(fileacl.New()).
Register(invitestore.New()).
Register(source.New()).
Register(spacefactory.New()).
Register(space.New()).
Register(deletioncontroller.New()).
Register(invitestore.New()).
Register(fileobject.New()).
Register(acl.New()).
Register(filesync.New()).

View file

@ -97,9 +97,7 @@ func NewTest(t *testing.T, broadcast func(e *pb.Event)) Service {
sender.EXPECT().Name().Return("")
sender.EXPECT().Init(mock.Anything).Return(nil)
if broadcast == nil {
broadcast = func(e *pb.Event) {
t.Log(e)
}
broadcast = func(e *pb.Event) {}
}
sender.EXPECT().Broadcast(mock.Anything).Run(broadcast).Maybe()
a.Register(sender).Register(s)

View file

@ -33,10 +33,12 @@ var errReachedLimit = fmt.Errorf("file upload limit has been reached")
type FileSync interface {
AddFile(spaceId string, fileId domain.FileId, uploadedByUser, imported bool) (err error)
UploadSynchronously(spaceId string, fileId domain.FileId) error
OnUploadStarted(func(fileId domain.FileId) error)
OnUploaded(func(fileId domain.FileId) error)
OnLimited(func(fileId domain.FileId) error)
RemoveFile(spaceId string, fileId domain.FileId) (err error)
RemoveSynchronously(spaceId string, fileId domain.FileId) (err error)
NodeUsage(ctx context.Context) (usage NodeUsage, err error)
SpaceStat(ctx context.Context, spaceId string) (ss SpaceStat, err error)
FileStat(ctx context.Context, spaceId string, fileId domain.FileId) (fs FileStat, err error)

View file

@ -797,6 +797,53 @@ func (_c *MockFileSync_RemoveFile_Call) RunAndReturn(run func(string, domain.Fil
return _c
}
// RemoveSynchronously provides a mock function with given fields: spaceId, fileId
func (_m *MockFileSync) RemoveSynchronously(spaceId string, fileId domain.FileId) error {
ret := _m.Called(spaceId, fileId)
if len(ret) == 0 {
panic("no return value specified for RemoveSynchronously")
}
var r0 error
if rf, ok := ret.Get(0).(func(string, domain.FileId) error); ok {
r0 = rf(spaceId, fileId)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockFileSync_RemoveSynchronously_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RemoveSynchronously'
type MockFileSync_RemoveSynchronously_Call struct {
*mock.Call
}
// RemoveSynchronously is a helper method to define mock.On call
// - spaceId string
// - fileId domain.FileId
func (_e *MockFileSync_Expecter) RemoveSynchronously(spaceId interface{}, fileId interface{}) *MockFileSync_RemoveSynchronously_Call {
return &MockFileSync_RemoveSynchronously_Call{Call: _e.mock.On("RemoveSynchronously", spaceId, fileId)}
}
func (_c *MockFileSync_RemoveSynchronously_Call) Run(run func(spaceId string, fileId domain.FileId)) *MockFileSync_RemoveSynchronously_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(string), args[1].(domain.FileId))
})
return _c
}
func (_c *MockFileSync_RemoveSynchronously_Call) Return(err error) *MockFileSync_RemoveSynchronously_Call {
_c.Call.Return(err)
return _c
}
func (_c *MockFileSync_RemoveSynchronously_Call) RunAndReturn(run func(string, domain.FileId) error) *MockFileSync_RemoveSynchronously_Call {
_c.Call.Return(run)
return _c
}
// Run provides a mock function with given fields: ctx
func (_m *MockFileSync) Run(ctx context.Context) error {
ret := _m.Called(ctx)
@ -987,6 +1034,53 @@ func (_c *MockFileSync_SyncStatus_Call) RunAndReturn(run func() (filesync.SyncSt
return _c
}
// UploadSynchronously provides a mock function with given fields: spaceId, fileId
func (_m *MockFileSync) UploadSynchronously(spaceId string, fileId domain.FileId) error {
ret := _m.Called(spaceId, fileId)
if len(ret) == 0 {
panic("no return value specified for UploadSynchronously")
}
var r0 error
if rf, ok := ret.Get(0).(func(string, domain.FileId) error); ok {
r0 = rf(spaceId, fileId)
} else {
r0 = ret.Error(0)
}
return r0
}
// MockFileSync_UploadSynchronously_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UploadSynchronously'
type MockFileSync_UploadSynchronously_Call struct {
*mock.Call
}
// UploadSynchronously is a helper method to define mock.On call
// - spaceId string
// - fileId domain.FileId
func (_e *MockFileSync_Expecter) UploadSynchronously(spaceId interface{}, fileId interface{}) *MockFileSync_UploadSynchronously_Call {
return &MockFileSync_UploadSynchronously_Call{Call: _e.mock.On("UploadSynchronously", spaceId, fileId)}
}
func (_c *MockFileSync_UploadSynchronously_Call) Run(run func(spaceId string, fileId domain.FileId)) *MockFileSync_UploadSynchronously_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(string), args[1].(domain.FileId))
})
return _c
}
func (_c *MockFileSync_UploadSynchronously_Call) Return(_a0 error) *MockFileSync_UploadSynchronously_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockFileSync_UploadSynchronously_Call) RunAndReturn(run func(string, domain.FileId) error) *MockFileSync_UploadSynchronously_Call {
_c.Call.Return(run)
return _c
}
// NewMockFileSync creates a new instance of MockFileSync. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockFileSync(t interface {

View file

@ -71,6 +71,15 @@ func (f *fileSync) tryToRemove() (domain.FileId, error) {
return fileId, nil
}
func (f *fileSync) RemoveSynchronously(spaceId string, fileId domain.FileId) (err error) {
err = f.removeFile(context.Background(), spaceId, fileId)
if err != nil {
return fmt.Errorf("remove file: %w", err)
}
f.updateSpaceUsageInformation(spaceId)
return
}
func (f *fileSync) removeFile(ctx context.Context, spaceId string, fileId domain.FileId) (err error) {
log.Info("removing file", zap.String("fileId", fileId.String()))
return f.rpcStore.DeleteFiles(ctx, spaceId, fileId)

View file

@ -199,6 +199,7 @@ func (f *fileSync) updateSpaceUsageInformation(spaceID string) {
}
func (f *fileSync) sendSpaceUsageEvent(spaceID string, bytesUsage uint64) {
fmt.Println("USAGE", spaceID, bytesUsage)
f.eventSender.Broadcast(&pb.Event{
Messages: []*pb.EventMessage{
{

View file

@ -114,6 +114,17 @@ func (f *fileSync) tryToUpload() (domain.FileId, error) {
return fileId, f.queue.DoneUpload(spaceId, fileId)
}
func (f *fileSync) UploadSynchronously(spaceId string, fileId domain.FileId) error {
f.runOnUploadStartedHook(fileId, spaceId)
err := f.uploadFile(context.Background(), spaceId, fileId)
if err != nil {
return err
}
f.runOnUploadedHook(fileId, spaceId)
f.updateSpaceUsageInformation(spaceId)
return nil
}
func (f *fileSync) runOnUploadedHook(fileId domain.FileId, spaceId string) {
if f.onUploaded != nil {
err := f.onUploaded(fileId)

View file

@ -153,7 +153,27 @@ func (t *inMemoryStore) AddToFile(ctx context.Context, spaceId string, fileId do
}
func (t *inMemoryStore) DeleteFiles(ctx context.Context, spaceId string, fileIds ...domain.FileId) (err error) {
panic("not implemented")
t.mu.Lock()
defer t.mu.Unlock()
if _, ok := t.spaceFiles[spaceId]; !ok {
return fmt.Errorf("spaceFiles not found: %s", spaceId)
}
if _, ok := t.spaceCids[spaceId]; !ok {
return fmt.Errorf("spaceCids not found: %s", spaceId)
}
for _, fileId := range fileIds {
_, ok := t.spaceFiles[spaceId][fileId]
if ok {
delete(t.spaceFiles[spaceId], fileId)
for cId := range t.files[fileId] {
delete(t.spaceCids[spaceId], cId)
}
delete(t.files, fileId)
}
}
return nil
}
func (t *inMemoryStore) SpaceInfo(ctx context.Context, spaceId string) (*fileproto.SpaceInfoResponse, error) {

View file

@ -13,21 +13,27 @@ import (
"github.com/ipfs/go-cid"
"github.com/anyproto/anytype-heart/core/domain"
"github.com/anyproto/anytype-heart/core/files"
"github.com/anyproto/anytype-heart/core/filestorage/filesync"
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
"github.com/anyproto/anytype-heart/space"
)
const CName = "invitestore"
type Service interface {
app.Component
StoreInvite(ctx context.Context, spaceId string, invite *model.Invite) (id cid.Cid, key crypto.SymKey, err error)
app.ComponentRunnable
StoreInvite(ctx context.Context, invite *model.Invite) (id cid.Cid, key crypto.SymKey, err error)
RemoveInvite(ctx context.Context, id cid.Cid) error
GetInvite(ctx context.Context, id cid.Cid, key crypto.SymKey) (*model.Invite, error)
}
type service struct {
commonFile fileservice.FileService
fileService files.Service
fileSyncService filesync.FileSync
spaceService space.Service
techSpaceId string
}
func New() Service {
@ -35,8 +41,19 @@ func New() Service {
}
func (s *service) Init(a *app.App) error {
s.fileService = app.MustComponent[files.Service](a)
s.commonFile = app.MustComponent[fileservice.FileService](a)
s.fileSyncService = app.MustComponent[filesync.FileSync](a)
s.spaceService = app.MustComponent[space.Service](a)
return nil
}
func (s *service) Run(_ context.Context) error {
s.techSpaceId = s.spaceService.TechSpaceId()
return nil
}
func (s *service) Close(_ context.Context) error {
return nil
}
@ -44,7 +61,7 @@ func (s *service) Name() (name string) {
return CName
}
func (s *service) StoreInvite(ctx context.Context, spaceId string, invite *model.Invite) (cid.Cid, crypto.SymKey, error) {
func (s *service) StoreInvite(ctx context.Context, invite *model.Invite) (cid.Cid, crypto.SymKey, error) {
key, err := crypto.NewRandomAES()
if err != nil {
return cid.Cid{}, nil, fmt.Errorf("generate key: %w", err)
@ -64,13 +81,24 @@ func (s *service) StoreInvite(ctx context.Context, spaceId string, invite *model
if err != nil {
return cid.Cid{}, nil, fmt.Errorf("add data to IPFS: %w", err)
}
err = s.fileSyncService.AddFile(spaceId, domain.FileId(node.Cid().String()), true, false)
err = s.fileSyncService.UploadSynchronously(s.techSpaceId, domain.FileId(node.Cid().String()))
if err != nil {
return cid.Cid{}, nil, fmt.Errorf("add file to sync queue: %w", err)
}
return node.Cid(), key, nil
}
func (s *service) RemoveInvite(ctx context.Context, id cid.Cid) error {
_, err := s.fileService.FileOffload(ctx, domain.FullFileId{
SpaceId: s.techSpaceId,
FileId: domain.FileId(id.String()),
})
if err != nil {
return fmt.Errorf("offload file: %w", err)
}
return s.fileSyncService.RemoveSynchronously(s.techSpaceId, domain.FileId(id.String()))
}
func (s *service) GetInvite(ctx context.Context, id cid.Cid, key crypto.SymKey) (*model.Invite, error) {
rd, err := s.commonFile.GetFile(ctx, id)
if err != nil {

View file

@ -11,10 +11,16 @@ import (
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/anyproto/anytype-heart/core/event/mock_event"
"github.com/anyproto/anytype-heart/core/files"
"github.com/anyproto/anytype-heart/core/filestorage"
"github.com/anyproto/anytype-heart/core/filestorage/filesync/mock_filesync"
"github.com/anyproto/anytype-heart/core/filestorage/filesync"
"github.com/anyproto/anytype-heart/core/filestorage/rpcstore"
"github.com/anyproto/anytype-heart/pkg/lib/datastore"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/filestore"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore"
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
"github.com/anyproto/anytype-heart/space/mock_space"
"github.com/anyproto/anytype-heart/tests/testutil"
)
@ -23,19 +29,24 @@ type fixture struct {
}
func newFixture(t *testing.T) *fixture {
blockStorage := filestorage.NewInMemory()
spaceService := mock_space.NewMockService(t)
spaceService.EXPECT().TechSpaceId().Return("techSpaceId").Maybe()
rpcStore := rpcstore.NewInMemoryStore(1024)
rpcStoreService := rpcstore.NewInMemoryService(rpcStore)
commonFileService := fileservice.New()
fileSyncService := mock_filesync.NewMockFileSync(t)
fileSyncService.EXPECT().AddFile(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe()
eventSender := mock_event.NewMockSender(t)
eventSender.EXPECT().Broadcast(mock.Anything).Return().Maybe()
ctx := context.Background()
a := new(app.App)
a.Register(blockStorage)
a.Register(rpcStoreService)
a.Register(commonFileService)
a.Register(testutil.PrepareMock(ctx, a, fileSyncService))
a.Register(objectstore.NewStoreFixture(t))
a.Register(datastore.NewInMemory())
a.Register(filestore.New())
a.Register(testutil.PrepareMock(ctx, a, eventSender))
a.Register(testutil.PrepareMock(ctx, a, spaceService))
a.Register(filestorage.NewInMemory())
a.Register(rpcstore.NewInMemoryService(rpcStore))
a.Register(fileservice.New())
a.Register(filesync.New())
a.Register(files.New())
err := a.Start(ctx)
require.NoError(t, err)
@ -62,7 +73,8 @@ func TestStore(t *testing.T) {
Payload: payload,
Signature: signature,
}
id, key, err := fx.StoreInvite(ctx, "space1", wantInvite)
id, key, err := fx.StoreInvite(ctx, wantInvite)
require.NoError(t, err)
gotInvite, err := fx.GetInvite(ctx, id, key)
@ -72,4 +84,10 @@ func TestStore(t *testing.T) {
ok, err := pub.Verify(gotInvite.Payload, gotInvite.Signature)
require.NoError(t, err)
assert.True(t, ok)
err = fx.RemoveInvite(ctx, id)
require.NoError(t, err)
_, err = fx.GetInvite(ctx, id, key)
require.Error(t, err)
}

2
go.mod
View file

@ -92,7 +92,7 @@ require (
golang.org/x/net v0.21.0
golang.org/x/oauth2 v0.16.0
golang.org/x/text v0.14.0
google.golang.org/grpc v1.61.0
google.golang.org/grpc v1.61.1
gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20180125164251-1832d8546a9f
gopkg.in/yaml.v3 v3.0.1
storj.io/drpc v0.0.33

4
go.sum
View file

@ -2029,8 +2029,8 @@ google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0=
google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs=
google.golang.org/grpc v1.61.1 h1:kLAiWrZs7YeDM6MumDe7m3y4aM6wacLzM1Y/wiLP9XY=
google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

View file

@ -6,7 +6,7 @@ package bundle
import domain "github.com/anyproto/anytype-heart/core/domain"
const InternalRelationsChecksum = "19cefc1c3cd9e090d6c715ccc819b852798c2b638edb0b162f999b9bf57e98ce"
const InternalRelationsChecksum = "252c59332af390d7006506a90fa7617b48082f796e97ad50a74b8fd3a680bc2c"
// RequiredInternalRelations contains internal relations that will be added to EVERY new or existing object
// if this relation only needs SPECIFIC objects(e.g. of some type) add it to the SystemRelations
@ -37,6 +37,4 @@ var RequiredInternalRelations = []domain.RelationKey{
RelationKeyLinks,
RelationKeyInternalFlags,
RelationKeyRestrictions,
RelationKeyFileId,
RelationKeySizeInBytes,
}

View file

@ -24,7 +24,5 @@
"spaceId",
"links",
"internalFlags",
"restrictions",
"fileId",
"sizeInBytes"
"restrictions"
]

View file

@ -34,6 +34,7 @@ var SystemRelations = append(RequiredInternalRelations, []domain.RelationKey{
RelationKeyRecommendedLayout,
RelationKeyFileExt,
RelationKeyFileMimeType,
RelationKeySizeInBytes,
RelationKeyOldAnytypeID,
RelationKeySpaceDashboardId,
RelationKeyRecommendedRelations,
@ -41,6 +42,7 @@ var SystemRelations = append(RequiredInternalRelations, []domain.RelationKey{
RelationKeyWidthInPixels,
RelationKeyHeightInPixels,
RelationKeyFileExt,
RelationKeySizeInBytes,
RelationKeySourceFilePath,
RelationKeyFileSyncStatus,
RelationKeyDefaultTemplateId,
@ -48,6 +50,7 @@ var SystemRelations = append(RequiredInternalRelations, []domain.RelationKey{
RelationKeyBacklinks,
RelationKeyProfileOwnerIdentity,
RelationKeyFileBackupStatus,
RelationKeyFileId,
RelationKeyFileIndexingStatus,
RelationKeyOrigin,
RelationKeyRevision,

View file

@ -628,6 +628,51 @@ func (_c *MockService_SpaceViewId_Call) RunAndReturn(run func(string) (string, e
return _c
}
// TechSpaceId provides a mock function with given fields:
func (_m *MockService) TechSpaceId() string {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for TechSpaceId")
}
var r0 string
if rf, ok := ret.Get(0).(func() string); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// MockService_TechSpaceId_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'TechSpaceId'
type MockService_TechSpaceId_Call struct {
*mock.Call
}
// TechSpaceId is a helper method to define mock.On call
func (_e *MockService_Expecter) TechSpaceId() *MockService_TechSpaceId_Call {
return &MockService_TechSpaceId_Call{Call: _e.mock.On("TechSpaceId")}
}
func (_c *MockService_TechSpaceId_Call) Run(run func()) *MockService_TechSpaceId_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *MockService_TechSpaceId_Call) Return(_a0 string) *MockService_TechSpaceId_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockService_TechSpaceId_Call) RunAndReturn(run func() string) *MockService_TechSpaceId_Call {
_c.Call.Return(run)
return _c
}
// NewMockService creates a new instance of MockService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockService(t interface {

View file

@ -52,6 +52,7 @@ type Service interface {
Join(ctx context.Context, id string) (err error)
Get(ctx context.Context, id string) (space clientspace.Space, err error)
Delete(ctx context.Context, id string) (err error)
TechSpaceId() string
GetPersonalSpace(ctx context.Context) (space clientspace.Space, err error)
SpaceViewId(spaceId string) (spaceViewId string, err error)
AccountMetadataSymKey() crypto.SymKey
@ -273,3 +274,7 @@ func (s *service) AllSpaceIds() (ids []string) {
}
return
}
func (s *service) TechSpaceId() string {
return s.techSpace.TechSpaceId()
}