anyproto/anytype-heart
1
0
Fork 0
mirror of https://github.com/anyproto/anytype-heart.git synced 2025-06-11 02:13:41 +09:00
Code Activity

GO-4573 Merge branch 'main' of github.com:anyproto/anytype-heart into go-4573-importing-markdown-files-named-in-japanese-or-chinese

Browse source
This commit is contained in:
AnastasiaShemyakinskaya 2024-12-09 10:06:43 +01:00
commit a314557810
No known key found for this signature in database
GPG key ID: CCD60ED83B103281
21 changed files with 356 additions and 346 deletions
Download patch file Download diff file Expand all files Collapse all files

75
core/anytype/account/service.go
View file

@ -2,7 +2,6 @@ package account
import (
"context"
"errors"
"fmt"
"path/filepath"
"time"
@ -25,9 +24,14 @@ import (
"github.com/anyproto/anytype-heart/space"
"github.com/anyproto/anytype-heart/space/spacecore"
"github.com/anyproto/anytype-heart/space/techspace"
"github.com/anyproto/anytype-heart/util/metricsid"
)
const CName = "account"
const (
CName = "account"
analyticsWaitTimeout = time.Second * 30
)
var log = logging.Logger(CName)
@ -173,55 +177,36 @@ func (s *service) GetSpaceInfo(ctx context.Context, spaceId string) (*model.Acco
}
func (s *service) getAnalyticsId(ctx context.Context, techSpace techspace.TechSpace) (analyticsId string, err error) {
if s.config.AnalyticsId != "" {
return s.config.AnalyticsId, nil
}
err = techSpace.DoAccountObject(ctx, func(accountObject techspace.AccountObject) error {
var innerErr error
analyticsId, innerErr = accountObject.GetAnalyticsId()
if innerErr != nil {
log.Debug("failed to get analytics id: %s", innerErr)
}
return nil
})
if err != nil {
return
}
if analyticsId == "" {
for {
// waiting for personal space
ctx, cancel := context.WithTimeout(ctx, time.Second*10)
persErr := s.spaceService.WaitPersonalSpaceMigration(ctx)
cancel()
if persErr != nil && !errors.Is(persErr, ctx.Err()) {
return "", persErr
start := time.Now()
for {
err = techSpace.DoAccountObject(ctx, func(accountObject techspace.AccountObject) error {
var innerErr error
analyticsId, innerErr = accountObject.GetAnalyticsId()
if innerErr != nil {
log.Debug("failed to get analytics id: %s", innerErr)
}
return nil
})
if err != nil {
return
}
if analyticsId != "" {
return analyticsId, nil
}
if time.Since(start) > analyticsWaitTimeout {
metricsId, err := metricsid.DeriveMetricsId(s.keyProvider.Account().SignKey)
if err != nil {
return "", err
}
// there is also this case that account object could be unsynced on start
// because we can't distinguish between accounts without account object (i.e. old accounts)
// and new ones which have the account object
// so it may be the case that it will sync but a little bit later
err = techSpace.DoAccountObject(ctx, func(accountObject techspace.AccountObject) error {
var innerErr error
analyticsId, innerErr = accountObject.GetAnalyticsId()
if innerErr != nil {
log.Debug("failed to get analytics id: %s", err)
}
return nil
return accountObject.SetAnalyticsId(metricsId)
})
if err != nil {
return
return "", err
}
if analyticsId != "" {
return analyticsId, nil
}
if persErr == nil {
return "", fmt.Errorf("failed to get analytics id, but migrated successfully")
}
// adding sleep just in case to avoid infinite loops if we have some unforeseen issues
time.Sleep(time.Second)
return metricsId, nil
}
} else {
return analyticsId, nil
time.Sleep(time.Second)
}
}

6
core/application/account_create_from_export.go
View file

@ -26,6 +26,7 @@ import (
"github.com/anyproto/anytype-heart/util/anyerror"
"github.com/anyproto/anytype-heart/util/builtinobjects"
"github.com/anyproto/anytype-heart/util/constant"
"github.com/anyproto/anytype-heart/util/metricsid"
"github.com/anyproto/anytype-heart/util/pbtypes"
)
@ -104,7 +105,10 @@ func (s *Service) RecoverFromLegacy(req *pb.RpcAccountRecoverFromLegacyExportReq
if profile.AnalyticsId != "" {
cfg.AnalyticsId = profile.AnalyticsId
} else {
cfg.AnalyticsId = metrics.GenerateAnalyticsId()
cfg.AnalyticsId, err = metricsid.DeriveMetricsId(res.Identity)
if err != nil {
return RecoverFromLegacyResponse{}, err
}
}
err = s.startApp(cfg, res)

47
core/block/editor/accountobject/accountobject.go
View file

@ -10,6 +10,7 @@ import (
anystore "github.com/anyproto/any-store"
"github.com/anyproto/any-store/anyenc"
"github.com/anyproto/any-sync/app/logger"
"github.com/anyproto/any-sync/commonspace/object/accountdata"
"github.com/anyproto/any-sync/util/crypto"
"github.com/gogo/protobuf/types"
"go.uber.org/zap"
@ -32,18 +33,18 @@ import (
"github.com/anyproto/anytype-heart/pkg/lib/bundle"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore/spaceindex"
"github.com/anyproto/anytype-heart/pkg/lib/pb/model"
"github.com/anyproto/anytype-heart/util/metricsid"
"github.com/anyproto/anytype-heart/util/pbtypes"
)
var log = logger.NewNamedSugared("common.editor.accountobject")
const (
collectionName = "account"
accountDocumentId = "accountObject"
idKey = "id"
analyticsKey = "analyticsId"
iconMigrationKey = "iconMigration"
privateAnalyticsIdKey = "privateAnalyticsId"
collectionName = "account"
accountDocumentId = "accountObject"
idKey = "id"
analyticsKey = "analyticsId"
iconMigrationKey = "iconMigration"
)
type ProfileDetails struct {
@ -63,7 +64,6 @@ type AccountObject interface {
IsIconMigrated() (bool, error)
SetAnalyticsId(analyticsId string) (err error)
GetAnalyticsId() (string, error)
GetPrivateAnalyticsId() string
}
type StoreDbProvider interface {
@ -75,6 +75,7 @@ var _ AccountObject = (*accountObject)(nil)
type accountObject struct {
anystoredebug.AnystoreDebug
smartblock.SmartBlock
keys *accountdata.AccountKeys
bs basic.DetailsSettable
state *storestate.StoreState
storeSource source.Store
@ -95,6 +96,7 @@ func (a *accountObject) SetDetailsAndUpdateLastUsed(ctx session.Context, details
func New(
sb smartblock.SmartBlock,
keys *accountdata.AccountKeys,
spaceObjects spaceindex.Store,
layoutConverter converter.LayoutConverter,
fileObjectService fileobject.Service,
@ -103,6 +105,7 @@ func New(
cfg *config.Config) AccountObject {
return &accountObject{
crdtDb: crdtDb,
keys: keys,
bs: basic.NewBasic(sb, spaceObjects, layoutConverter, fileObjectService, lastUsedUpdater),
SmartBlock: sb,
cfg: cfg,
@ -127,7 +130,6 @@ func (a *accountObject) Init(ctx *smartblock.InitContext) error {
a.state = stateStore
a.AnystoreDebug = anystoredebug.New(a.SmartBlock, stateStore)
storeSource, ok := ctx.Source.(source.Store)
if !ok {
return fmt.Errorf("source is not a store")
@ -179,27 +181,16 @@ func (a *accountObject) Init(ctx *smartblock.InitContext) error {
return a.SmartBlock.Apply(st, smartblock.NotPushChanges, smartblock.NoHistory, smartblock.SkipIfNoChanges)
}
// GetPrivateAnalyticsId returns the private analytics id of the account object, should not be used directly
// only when hashing it with other data, e.g. hash(privateAnalyticsId + someData)
func (a *accountObject) GetPrivateAnalyticsId() string {
val, err := a.getValue()
if err != nil {
return ""
}
return string(val.GetStringBytes(privateAnalyticsIdKey))
}
func (a *accountObject) genInitialDoc() (builder *storestate.Builder, err error) {
builder = &storestate.Builder{}
privateAnalytics, err := generatePrivateAnalyticsId()
analyticsId, err := metricsid.DeriveMetricsId(a.keys.SignKey)
if err != nil {
err = fmt.Errorf("generate private analytics id: %w", err)
return
}
newDocument := map[string]any{
idKey: accountDocumentId,
analyticsKey: a.cfg.AnalyticsId,
iconMigrationKey: "true",
privateAnalyticsIdKey: privateAnalytics,
idKey: accountDocumentId,
analyticsKey: analyticsId,
iconMigrationKey: "true",
}
for key, val := range newDocument {
if str, ok := val.(string); ok {
@ -277,14 +268,6 @@ func (a *accountObject) SetAnalyticsId(id string) error {
if err != nil {
return nil
}
privateAnalyticsId, err := generatePrivateAnalyticsId()
if err != nil {
return fmt.Errorf("generate private analytics id: %w", err)
}
err = builder.Modify(collectionName, accountDocumentId, []string{privateAnalyticsIdKey}, pb.ModifyOp_Set, fmt.Sprintf(`"%s"`, privateAnalyticsId))
if err != nil {
return nil
}
_, err = a.storeSource.PushStoreChange(a.ctx, source.PushStoreChangeParams{
Changes: builder.ChangeSet,
State: a.state,

27
core/block/editor/accountobject/accountobject_test.go
View file

@ -9,6 +9,7 @@ import (
anystore "github.com/anyproto/any-store"
"github.com/anyproto/any-store/anyenc"
"github.com/anyproto/any-sync/commonspace/object/accountdata"
"github.com/globalsign/mgo/bson"
"github.com/gogo/protobuf/types"
"github.com/stretchr/testify/mock"
@ -24,6 +25,7 @@ import (
"github.com/anyproto/anytype-heart/core/block/source/mock_source"
"github.com/anyproto/anytype-heart/pb"
"github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore"
"github.com/anyproto/anytype-heart/util/metricsid"
"github.com/anyproto/anytype-heart/util/pbtypes"
)
@ -40,7 +42,6 @@ type fixture struct {
func newFixture(t *testing.T, isNewAccount bool, prepareDb func(db anystore.DB)) *fixture {
ctx := context.Background()
cfg := config.New(config.WithNewAccount(isNewAccount))
cfg.AnalyticsId = "analyticsId"
db, err := anystore.Open(ctx, filepath.Join(t.TempDir(), "crdt.db"), nil)
require.NoError(t, err)
if prepareDb != nil {
@ -52,7 +53,9 @@ func newFixture(t *testing.T, isNewAccount bool, prepareDb func(db anystore.DB))
})
sb := smarttest.New("accountId1")
indexStore := objectstore.NewStoreFixture(t).SpaceIndex("spaceId")
object := New(sb, indexStore, nil, nil, nil, db, cfg)
keys, err := accountdata.NewRandom()
require.NoError(t, err)
object := New(sb, keys, indexStore, nil, nil, nil, db, cfg)
fx := &fixture{
storeFx: objectstore.NewStoreFixture(t),
db: db,
@ -136,6 +139,8 @@ func (fx *fixture) assertStateValue(t *testing.T, val any, extract func(str *typ
func TestAccountNew(t *testing.T) {
fx := newFixture(t, true, nil)
expectedId, err := metricsid.DeriveMetricsId(fx.keys.SignKey)
require.NoError(t, err)
st := fx.SmartBlock.NewState()
assertBlock(t, st, "accountId1")
assertBlock(t, st, "title")
@ -146,7 +151,8 @@ func TestAccountNew(t *testing.T) {
require.True(t, res)
id, err := fx.GetAnalyticsId()
require.NoError(t, err)
require.Equal(t, "analyticsId", id)
fmt.Println(id)
require.Equal(t, expectedId, id)
}
func TestAccountOldInitWithData(t *testing.T) {
@ -209,21 +215,6 @@ func TestSetSharedSpacesLimit(t *testing.T) {
require.Equal(t, 10, res)
}
func TestAccountObject_GetPrivateAnalyticsId(t *testing.T) {
t.Run("new account", func(t *testing.T) {
fx := newFixture(t, true, nil)
res := fx.GetPrivateAnalyticsId()
require.NotEmpty(t, res)
})
t.Run("old account", func(t *testing.T) {
fx := newFixture(t, false, nil)
err := fx.SetAnalyticsId("analyticsId")
require.NoError(t, err)
res := fx.GetPrivateAnalyticsId()
require.NotEmpty(t, res)
})
}
func TestAnalyticsId(t *testing.T) {
fx := newFixture(t, true, nil)
err := fx.SetAnalyticsId("analyticsId")

4
core/block/editor/factory.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"github.com/anyproto/any-sync/app"
"github.com/anyproto/any-sync/commonspace/object/accountdata"
"github.com/anyproto/any-sync/commonspace/object/tree/objecttree"
"github.com/anyproto/anytype-heart/core/anytype/config"
@ -44,6 +45,7 @@ type accountService interface {
AccountID() string
PersonalSpaceID() string
MyParticipantId(spaceId string) string
Keys() *accountdata.AccountKeys
}
type deviceService interface {
@ -213,7 +215,7 @@ func (f *ObjectFactory) New(space smartblock.Space, sbType coresb.SmartBlockType
case coresb.SmartBlockTypeChatDerivedObject:
return chatobject.New(sb, f.accountService, f.eventSender, f.objectStore.GetCrdtDb(space.Id())), nil
case coresb.SmartBlockTypeAccountObject:
return accountobject.New(sb, store, f.layoutConverter, f.fileObjectService, f.lastUsedUpdater, f.objectStore.GetCrdtDb(space.Id()), f.config), nil
return accountobject.New(sb, f.accountService.Keys(), store, f.layoutConverter, f.fileObjectService, f.lastUsedUpdater, f.objectStore.GetCrdtDb(space.Id()), f.config), nil
default:
return nil, fmt.Errorf("unexpected smartblock type: %v", sbType)
}

10
core/block/import/markdown/blockconverter.go
View file

@ -127,7 +127,7 @@ func (m *mdConverter) handleSingleMark(block *model.Block, files map[string]*Fil
}
file.HasInboundLinks = true
} else if wholeLineLink {
block.Content = m.convertTextToBookmark(txt.Marks.Marks[0].Param)
m.convertTextToBookmark(txt.Marks.Marks[0].Param, block)
}
}
@ -158,7 +158,7 @@ func (m *mdConverter) handleSingleLinkMark(block *model.Block, files map[string]
return true
}
} else if isWholeLink {
block.Content = m.convertTextToBookmark(mark.Param)
m.convertTextToBookmark(mark.Param, block)
return true
}
return false
@ -242,12 +242,12 @@ func (m *mdConverter) convertTextToPageLink(block *model.Block) {
}
}
func (m *mdConverter) convertTextToBookmark(url string) *model.BlockContentOfBookmark {
func (m *mdConverter) convertTextToBookmark(url string, block *model.Block) {
if err := uri.ValidateURI(url); err != nil {
return nil
return
}
return &model.BlockContentOfBookmark{
block.Content = &model.BlockContentOfBookmark{
Bookmark: &model.BlockContentBookmark{
Url: url,
},

30
core/block/import/markdown/import_test.go
View file

@ -272,6 +272,7 @@ func buildExpectedTree(fileNameToObjectId map[string]string, provider *MockTempD
testMdPath := filepath.Join("testdata", "test.md")
testCsvPath := filepath.Join("testdata", "test.csv")
testTxtPath := filepath.Join("testdata", "test.txt")
url := "http://example.com/%zz"
want := blockbuilder.Root(
blockbuilder.ID(rootId),
blockbuilder.Children(
@ -297,6 +298,13 @@ func buildExpectedTree(fileNameToObjectId map[string]string, provider *MockTempD
Param: fileNameToObjectId[testCsvPath],
},
}})),
blockbuilder.Text("Should not panic test5", blockbuilder.TextMarks(model.BlockContentTextMarks{Marks: []*model.BlockContentTextMark{
{
Range: &model.Range{From: 17, To: 22},
Type: model.BlockContentTextMark_Link,
Param: url,
},
}})),
blockbuilder.Text("File does not exist with bold mark test1", blockbuilder.TextMarks(model.BlockContentTextMarks{Marks: []*model.BlockContentTextMark{
{
Range: &model.Range{From: 35, To: 40},
@ -341,6 +349,17 @@ func buildExpectedTree(fileNameToObjectId map[string]string, provider *MockTempD
Type: model.BlockContentTextMark_Bold,
},
}})),
blockbuilder.Text("Should not panic test5", blockbuilder.TextMarks(model.BlockContentTextMarks{Marks: []*model.BlockContentTextMark{
{
Range: &model.Range{From: 17, To: 22},
Type: model.BlockContentTextMark_Link,
Param: url,
},
{
Range: &model.Range{From: 17, To: 22},
Type: model.BlockContentTextMark_Bold,
},
}})),
blockbuilder.Bookmark(fileMdPath),
blockbuilder.Text("test2", blockbuilder.TextMarks(model.BlockContentTextMarks{Marks: []*model.BlockContentTextMark{
{
@ -365,6 +384,17 @@ func buildExpectedTree(fileNameToObjectId map[string]string, provider *MockTempD
Type: model.BlockContentTextMark_Bold,
},
}})),
blockbuilder.Text("test5", blockbuilder.TextMarks(model.BlockContentTextMarks{Marks: []*model.BlockContentTextMark{
{
Range: &model.Range{From: 0, To: 5},
Type: model.BlockContentTextMark_Link,
Param: url,
},
{
Range: &model.Range{From: 0, To: 5},
Type: model.BlockContentTextMark_Bold,
},
}})),
blockbuilder.Link(rootId),
))
return want

6
core/block/import/markdown/testdata/links.md vendored
View file

@ -6,6 +6,8 @@ Test file block [test3](test.txt)
Test link to csv [test4](test.csv)
Should not panic [test5](http://example.com/%zz)
File does not exist with bold mark **[test1](file.md)**
Test link to page with bold mark **[test2](test.md)**
@ -14,6 +16,8 @@ Test file block with bold mark **[test3](test.txt)**
Test link to csv with bold mark **[test4](test.csv)**
Should not panic **[test5](http://example.com/%zz)**
**[test1](file.md)**
**[test2](test.md)**
@ -21,3 +25,5 @@ Test link to csv with bold mark **[test4](test.csv)**
**[test3](test.txt)**
**[test4](test.csv)**
**[test5](http://example.com/%zz)**

26
go.mod
View file

@ -8,12 +8,13 @@ require (
github.com/VividCortex/ewma v1.2.0
github.com/adrium/goheif v0.0.0-20230113233934-ca402e77a786
github.com/anyproto/any-store v0.1.3
github.com/anyproto/any-sync v0.5.20
github.com/anyproto/any-sync v0.5.21
github.com/anyproto/go-chash v0.1.0
github.com/anyproto/go-naturaldate/v2 v2.0.2-0.20230524105841-9829cfd13438
github.com/anyproto/go-slip10 v1.0.0
github.com/anyproto/lexid v0.0.4
github.com/anyproto/protobuf v1.3.3-0.20240814124528-72b8c7e0e0f5
github.com/anyproto/tantivy-go v0.2.0
github.com/anyproto/tantivy-go v0.3.0
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
github.com/avast/retry-go/v4 v4.6.0
github.com/chai2010/webp v1.1.2-0.20240612091223-aa1b379218b7
@ -71,7 +72,7 @@ require (
github.com/miolini/datacounter v1.0.3
github.com/mr-tron/base58 v1.2.0
github.com/multiformats/go-base32 v0.1.0
github.com/multiformats/go-multiaddr-dns v0.4.0
github.com/multiformats/go-multiaddr-dns v0.4.1
github.com/multiformats/go-multibase v0.2.0
github.com/multiformats/go-multihash v0.2.3
github.com/oov/psd v0.0.0-20220121172623-5db5eafcecbb
@ -99,12 +100,12 @@ require (
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.27.0
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f
golang.org/x/image v0.22.0
golang.org/x/image v0.23.0
golang.org/x/mobile v0.0.0-20241108191957-fa514ef75a0f
golang.org/x/net v0.31.0
golang.org/x/net v0.32.0
golang.org/x/oauth2 v0.24.0
golang.org/x/text v0.20.0
google.golang.org/grpc v1.68.0
golang.org/x/text v0.21.0
google.golang.org/grpc v1.68.1
gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20180125164251-1832d8546a9f
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gopkg.in/yaml.v3 v3.0.1
@ -122,7 +123,6 @@ require (
github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect
github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/anyproto/go-slip10 v1.0.0 // indirect
github.com/anyproto/go-slip21 v1.0.0 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
@ -217,7 +217,7 @@ require (
github.com/prometheus/common v0.60.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/pseudomuto/protokit v0.2.1 // indirect
github.com/quic-go/quic-go v0.48.1 // indirect
github.com/quic-go/quic-go v0.48.2 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/rs/cors v1.11.0 // indirect
@ -246,11 +246,11 @@ require (
go.opentelemetry.io/otel v1.32.0 // indirect
go.opentelemetry.io/otel/metric v1.32.0 // indirect
go.opentelemetry.io/otel/trace v1.32.0 // indirect
golang.org/x/crypto v0.29.0 // indirect
golang.org/x/crypto v0.30.0 // indirect
golang.org/x/mod v0.22.0 // indirect
golang.org/x/sync v0.9.0 // indirect
golang.org/x/sys v0.27.0 // indirect
golang.org/x/term v0.26.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/term v0.27.0 // indirect
golang.org/x/time v0.8.0 // indirect
golang.org/x/tools v0.27.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect

48
go.sum
View file

@ -78,8 +78,8 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/anyproto/any-store v0.1.3 h1:onWLP8tuWiUvYOyV3DoRODscTxAiGEjwQPm+NMxeq3M=
github.com/anyproto/any-store v0.1.3/go.mod h1:6/0OUKgSMWF/vYGoGYzQOl2CII5OdiuXbQlGDXYcNYc=
github.com/anyproto/any-sync v0.5.20 h1:5In7AoMBa8npGz0Zv3wEmd3jhiN6CWGiCaEmgB8rRSM=
github.com/anyproto/any-sync v0.5.20/go.mod h1:K53/whh/9tmGbXrkPS59XyqWU3SaxhsX9g6VIYnBIlU=
github.com/anyproto/any-sync v0.5.21 h1:+Qx69Pij3QbLh2nRZEamD2WaSSy1uWHlXGMCIFEiKW0=
github.com/anyproto/any-sync v0.5.21/go.mod h1:K53/whh/9tmGbXrkPS59XyqWU3SaxhsX9g6VIYnBIlU=
github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580 h1:Ba80IlCCxkZ9H1GF+7vFu/TSpPvbpDCxXJ5ogc4euYc=
github.com/anyproto/badger/v4 v4.2.1-0.20240110160636-80743fa3d580/go.mod h1:T/uWAYxrXdaXw64ihI++9RMbKTCpKd/yE9+saARew7k=
github.com/anyproto/go-chash v0.1.0 h1:I9meTPjXFRfXZHRJzjOHC/XF7Q5vzysKkiT/grsogXY=
@ -106,8 +106,8 @@ github.com/anyproto/protobuf v1.3.3-0.20240814124528-72b8c7e0e0f5 h1:aY7tBzQ+z8H
github.com/anyproto/protobuf v1.3.3-0.20240814124528-72b8c7e0e0f5/go.mod h1:5+PHE01DgsDPkralb8MYmGg2sPQahsqEJ9ue7ciDHKg=
github.com/anyproto/ristretto v0.1.2-0.20240221153107-2b23839cc50c h1:GicoaTUyB2mtCIl3YMrO0OzysqRT5GA4vuvDsqEkhSM=
github.com/anyproto/ristretto v0.1.2-0.20240221153107-2b23839cc50c/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
github.com/anyproto/tantivy-go v0.2.0 h1:+b778sOPy07KpJULL4ztCc106L+zdmwKCIB5UYnWrZo=
github.com/anyproto/tantivy-go v0.2.0/go.mod h1:MMLYW7e5SIzsHS3Q5CYiF1J7kJJaIRT+VVHGArU24IQ=
github.com/anyproto/tantivy-go v0.3.0 h1:VHZ8+EnlndFbjs8pBoqvtYk+zq8l/QXkZwugkDwN48g=
github.com/anyproto/tantivy-go v0.3.0/go.mod h1:7hhkPpyTq7+W1dx9Dcva4bsg4TLHq9xqmmYLCwqDq/k=
github.com/anyproto/zeroconf/v2 v2.2.1-0.20240228113933-f90a5cc4439d h1:5bj7nX/AS8sxGpTIrapE7PC4oPlhkHMwMqXlJbUHBlg=
github.com/anyproto/zeroconf/v2 v2.2.1-0.20240228113933-f90a5cc4439d/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
@ -711,8 +711,8 @@ github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aG
github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI=
github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0=
github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4=
github.com/multiformats/go-multiaddr-dns v0.4.0 h1:P76EJ3qzBXpUXZ3twdCDx/kvagMsNo0LMFXpyms/zgU=
github.com/multiformats/go-multiaddr-dns v0.4.0/go.mod h1:7hfthtB4E4pQwirrz+J0CcDUfbWzTqEzVyYKKIKpgkc=
github.com/multiformats/go-multiaddr-dns v0.4.1 h1:whi/uCLbDS3mSEUMb1MsoT4uzUeZB0N32yzufqS0i5M=
github.com/multiformats/go-multiaddr-dns v0.4.1/go.mod h1:7hfthtB4E4pQwirrz+J0CcDUfbWzTqEzVyYKKIKpgkc=
github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E=
github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo=
github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g=
@ -871,8 +871,8 @@ github.com/pseudomuto/protokit v0.2.1 h1:kCYpE3thoR6Esm0CUvd5xbrDTOZPvQPTDeyXpZf
github.com/pseudomuto/protokit v0.2.1/go.mod h1:gt7N5Rz2flBzYafvaxyIxMZC0TTF5jDZfRnw25hAAyo=
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
github.com/quic-go/quic-go v0.48.1 h1:y/8xmfWI9qmGTc+lBr4jKRUWLGSlSigv847ULJ4hYXA=
github.com/quic-go/quic-go v0.48.1/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs=
github.com/quic-go/quic-go v0.48.2 h1:wsKXZPeGWpMpCGSWqOcqpW2wZYic/8T3aqiOID0/KWE=
github.com/quic-go/quic-go v0.48.2/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs=
github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg=
github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
@ -1091,8 +1091,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ=
golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg=
golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY=
golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -1113,8 +1113,8 @@ golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86h
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.22.0 h1:UtK5yLUzilVrkjMAZAZ34DXGpASN8i8pj8g+O+yd10g=
golang.org/x/image v0.22.0/go.mod h1:9hPFhljd4zZ1GNSIZJ49sqbp45GKK9t6w+iXvGqZUz4=
golang.org/x/image v0.23.0 h1:HseQ7c2OpPKTPVzNjG5fwJsOTCiiwS4QdsYi5XU6H68=
golang.org/x/image v0.23.0/go.mod h1:wJJBTdLfCCf3tiHa1fNxpZmUI4mmoZvwMCPP0ddoNKY=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -1198,8 +1198,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -1224,8 +1224,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -1302,16 +1302,16 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU=
golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -1323,8 +1323,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@ -1496,8 +1496,8 @@ google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0=
google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA=
google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0=
google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

143
pkg/lib/localstore/ftsearch/ftsearchtantivy.go
View file

@ -21,10 +21,10 @@ import (
"strings"
"sync"
"time"
"unicode"
"github.com/anyproto/any-sync/app"
tantivy "github.com/anyproto/tantivy-go"
"github.com/samber/lo"
"github.com/valyala/fastjson"
"github.com/anyproto/anytype-heart/core/wallet"
@ -37,7 +37,7 @@ const (
CName = "fts"
ftsDir = "fts"
ftsDir2 = "fts_tantivy"
ftsVer = "9"
ftsVer = "10"
docLimit = 10000
fieldTitle = "Title"
@ -62,7 +62,7 @@ type FTSearch interface {
NewAutoBatcher() AutoBatcher
BatchIndex(ctx context.Context, docs []SearchDoc, deletedDocs []string) (err error)
BatchDeleteObjects(ids []string) (err error)
Search(spaceIds []string, query string) (results []*DocumentMatch, err error)
Search(spaceIds string, query string) (results []*DocumentMatch, err error)
Iterate(objectId string, fields []string, shouldContinue func(doc *SearchDoc) bool) (err error)
DeleteObject(id string) error
DocCount() (uint64, error)
@ -169,7 +169,7 @@ func (f *ftSearchTantivy) Run(context.Context) error {
}
err = builder.AddTextField(
fieldId,
fieldId, // 0
true,
true,
false,
@ -178,7 +178,7 @@ func (f *ftSearchTantivy) Run(context.Context) error {
)
err = builder.AddTextField(
fieldIdRaw,
fieldIdRaw, // 1
true,
true,
true,
@ -187,7 +187,7 @@ func (f *ftSearchTantivy) Run(context.Context) error {
)
err = builder.AddTextField(
fieldSpace,
fieldSpace, // 2
true,
false,
true,
@ -196,25 +196,7 @@ func (f *ftSearchTantivy) Run(context.Context) error {
)
err = builder.AddTextField(
fieldTitle,
true,
true,
false,
tantivy.IndexRecordOptionWithFreqsAndPositions,
tantivy.TokenizerNgram,
)
err = builder.AddTextField(
fieldTitleZh,
true,
true,
false,
tantivy.IndexRecordOptionWithFreqsAndPositions,
tantivy.TokenizerJieba,
)
err = builder.AddTextField(
fieldText,
fieldTitle, // 3
true,
true,
false,
@ -223,7 +205,25 @@ func (f *ftSearchTantivy) Run(context.Context) error {
)
err = builder.AddTextField(
fieldTextZh,
fieldTitleZh, // 4
true,
true,
false,
tantivy.IndexRecordOptionWithFreqsAndPositions,
tantivy.TokenizerJieba,
)
err = builder.AddTextField(
fieldText, // 5
true,
true,
false,
tantivy.IndexRecordOptionWithFreqsAndPositions,
tantivy.TokenizerSimple,
)
err = builder.AddTextField(
fieldTextZh, // 6
true,
true,
false,
@ -366,28 +366,52 @@ func (f *ftSearchTantivy) BatchIndex(ctx context.Context, docs []SearchDoc, dele
return f.index.AddAndConsumeDocuments(tantivyDocs...)
}
func (f *ftSearchTantivy) Search(spaceIds []string, query string) (results []*DocumentMatch, err error) {
spaceIdsQuery := getSpaceIdsQuery(spaceIds)
func (f *ftSearchTantivy) Search(spaceId string, query string) (results []*DocumentMatch, err error) {
query = prepareQuery(query)
if query == "" {
return nil, nil
}
if spaceIdsQuery != "" {
query = fmt.Sprintf("%s AND %s", spaceIdsQuery, query)
qb := tantivy.NewQueryBuilder()
if len(spaceId) != 0 {
qb.Query(tantivy.Must, fieldSpace, spaceId, tantivy.TermQuery, 1.0)
}
if containsChineseCharacters(query) {
qb.BooleanQuery(tantivy.Must, qb.NestedBuilder().
Query(tantivy.Should, fieldTitleZh, query, tantivy.PhrasePrefixQuery, 5.0).
Query(tantivy.Should, fieldTitleZh, query, tantivy.PhraseQuery, 5.0).
Query(tantivy.Should, fieldTitleZh, query, tantivy.EveryTermQuery, 0.75).
Query(tantivy.Should, fieldTitleZh, query, tantivy.OneOfTermQuery, 0.5).
Query(tantivy.Should, fieldTextZh, query, tantivy.PhrasePrefixQuery, 1.0).
Query(tantivy.Should, fieldTextZh, query, tantivy.PhraseQuery, 1.0).
Query(tantivy.Should, fieldTextZh, query, tantivy.EveryTermQuery, 0.5).
Query(tantivy.Should, fieldTextZh, query, tantivy.OneOfTermQuery, 0.25),
1.0,
)
} else {
qb.BooleanQuery(tantivy.Must, qb.NestedBuilder().
Query(tantivy.Should, fieldTitle, query, tantivy.PhrasePrefixQuery, 10.0).
Query(tantivy.Should, fieldTitle, query, tantivy.PhraseQuery, 10.0).
Query(tantivy.Should, fieldTitle, query, tantivy.EveryTermQuery, 0.75).
Query(tantivy.Should, fieldTitle, query, tantivy.OneOfTermQuery, 0.5).
Query(tantivy.Should, fieldText, query, tantivy.PhrasePrefixQuery, 1.0).
Query(tantivy.Should, fieldText, query, tantivy.PhraseQuery, 1.0).
Query(tantivy.Should, fieldText, query, tantivy.EveryTermQuery, 0.5).
Query(tantivy.Should, fieldText, query, tantivy.OneOfTermQuery, 0.25),
1.0,
)
}
finalQuery := qb.Build()
sCtx := tantivy.NewSearchContextBuilder().
SetQuery(query).
SetQueryFromJson(&finalQuery).
SetDocsLimit(100).
SetWithHighlights(true).
AddFieldDefaultWeight(fieldId).
AddFieldDefaultWeight(fieldSpace).
AddField(fieldTitle, 10.0).
AddField(fieldTitleZh, 10.0).
AddFieldDefaultWeight(fieldText).
AddFieldDefaultWeight(fieldTextZh).
Build()
result, err := f.index.Search(sCtx)
result, err := f.index.SearchJson(sCtx)
if err != nil {
return nil, wrapError(err)
}
@ -432,6 +456,15 @@ func (f *ftSearchTantivy) Search(spaceIds []string, query string) (results []*Do
)
}
func containsChineseCharacters(s string) bool {
for _, r := range s {
if unicode.Is(unicode.Han, r) {
return true
}
}
return false
}
func extractHighlight(object *fastjson.Object, fragments map[string]*Highlight, fieldName string) {
highlightObj := object.Get(fragment)
if highlightObj == nil {
@ -459,26 +492,6 @@ func wrapError(err error) error {
return err
}
func getSpaceIdsQuery(ids []string) string {
ids = lo.Filter(ids, func(item string, index int) bool { return item != "" })
if len(ids) == 0 || lo.EveryBy(ids, func(id string) bool { return id == "" }) {
return ""
}
var builder strings.Builder
var sep string
builder.WriteString("(")
for _, id := range ids {
builder.WriteString(sep)
builder.WriteString(fieldSpace)
builder.WriteString(":")
builder.WriteString(id)
sep = " OR "
}
builder.WriteString(")")
return builder.String()
}
func (f *ftSearchTantivy) Delete(id string) error {
return f.BatchDeleteObjects([]string{id})
}
@ -505,17 +518,5 @@ func prepareQuery(query string) string {
query = text.Truncate(query, 100, "")
query = strings.ToLower(query)
query = strings.TrimSpace(query)
var escapedQuery strings.Builder
for _, char := range query {
if _, found := specialChars[char]; !found {
escapedQuery.WriteRune(char)
}
}
resultQuery := escapedQuery.String()
if resultQuery == "" {
return resultQuery
}
return "(\"" + resultQuery + "\" OR " + resultQuery + ")"
return query
}

110
pkg/lib/localstore/ftsearch/ftsearchtantivy_test.go
View file

@ -4,7 +4,6 @@ import (
"context"
"fmt"
"os"
"strings"
"testing"
"github.com/anyproto/any-sync/app"
@ -76,23 +75,15 @@ func TestDifferentSpaces(t *testing.T) {
SpaceId: "space2",
}))
search, err := ft.Search([]string{"space1"}, "one")
search, err := ft.Search("space1", "one")
require.NoError(t, err)
require.Len(t, search, 1)
search, err = ft.Search([]string{"space2"}, "one")
search, err = ft.Search("space2", "one")
require.NoError(t, err)
require.Len(t, search, 1)
search, err = ft.Search([]string{"space1", "space2"}, "one")
require.NoError(t, err)
require.Len(t, search, 2)
search, err = ft.Search([]string{""}, "one")
require.NoError(t, err)
require.Len(t, search, 2)
search, err = ft.Search(nil, "one")
search, err = ft.Search("", "one")
require.NoError(t, err)
require.Len(t, search, 2)
@ -115,7 +106,12 @@ func TestNewFTSearch(t *testing.T) {
{
name: "assertFoundCaseSensitivePartsOfTheWords",
tester: assertFoundCaseSensitivePartsOfTheWords,
}, {
},
{
name: "assertPrefix",
tester: assertPrefix,
},
{
name: "assertChineseFound",
tester: assertChineseFound,
},
@ -133,6 +129,50 @@ func TestNewFTSearch(t *testing.T) {
}
}
func assertPrefix(t *testing.T, tmpDir string) {
fixture := newFixture(tmpDir, t)
ft := fixture.ft
require.NoError(t, ft.Index(SearchDoc{
Id: "1",
Title: "I love my mum",
Text: "",
}))
require.NoError(t, ft.Index(SearchDoc{
Id: "2",
Title: "",
Text: "Something completely different",
}))
require.NoError(t, ft.Index(SearchDoc{
Id: "4",
Title: "Just random filler",
Text: "",
}))
require.NoError(t, ft.Index(SearchDoc{
Id: "4",
Title: "Another text for fun",
Text: "",
}))
validateSearch(t, ft, "", "I love", 1)
validateSearch(t, ft, "", "I lo", 1)
validateSearch(t, ft, "", "I", 1)
validateSearch(t, ft, "", "lov", 1)
validateSearch(t, ft, "", "Something", 1)
validateSearch(t, ft, "", "Some", 1)
validateSearch(t, ft, "", "comp", 1)
validateSearch(t, ft, "", "diff", 1)
validateSearch(t, ft, "", "Something c", 1)
validateSearch(t, ft, "", "Something different", 1)
validateSearch(t, ft, "", "different something", 1)
_ = ft.Close(nil)
}
func assertFoundCaseSensitivePartsOfTheWords(t *testing.T, tmpDir string) {
fixture := newFixture(tmpDir, t)
ft := fixture.ft
@ -252,7 +292,7 @@ func assertSearch(t *testing.T, tmpDir string) {
}
func validateSearch(t *testing.T, ft FTSearch, spaceID, qry string, times int) {
res, err := ft.Search([]string{spaceID}, qry)
res, err := ft.Search(spaceID, qry)
require.NoError(t, err)
assert.Len(t, res, times)
}
@ -292,51 +332,11 @@ func assertMultiSpace(t *testing.T, tmpDir string) {
validateSearch(t, ft, "", "Advanced", 1)
validateSearch(t, ft, "", "dash", 2)
validateSearch(t, ft, "", "space", 4)
validateSearch(t, ft, "", "of", 5)
validateSearch(t, ft, "", "of", 4)
_ = ft.Close(nil)
}
func TestEscapeQuery(t *testing.T) {
tests := []struct {
input string
expected string
}{
{strings.Repeat("a", 99) + " aa", `("` + strings.Repeat("a", 99) + `" OR ` + strings.Repeat("a", 99) + `)`},
{`""`, ``},
{"simpleQuery", `("simplequery" OR simplequery)`},
{"with+special^chars", `("withspecialchars" OR withspecialchars)`},
{"text`with:brackets{}", `("textwithbrackets" OR textwithbrackets)`},
{"escaped[]symbols()", `("escapedsymbols" OR escapedsymbols)`},
{"multiple!!special~~", `("multiplespecial" OR multiplespecial)`},
}
for _, test := range tests {
actual := prepareQuery(test.input)
if actual != test.expected {
t.Errorf("For input '%s', expected '%s', but got '%s'", test.input, test.expected, actual)
}
}
}
// Tests
func TestGetSpaceIdsQuery(t *testing.T) {
// Test with empty slice of ids
assert.Equal(t, "", getSpaceIdsQuery([]string{}))
// Test with slice containing only empty strings
assert.Equal(t, "", getSpaceIdsQuery([]string{"", "", ""}))
// Test with a single id
assert.Equal(t, "(SpaceID:123)", getSpaceIdsQuery([]string{"123"}))
// Test with multiple ids
assert.Equal(t, "(SpaceID:123 OR SpaceID:456 OR SpaceID:789)", getSpaceIdsQuery([]string{"123", "456", "789"}))
// Test with some empty ids
assert.Equal(t, "(SpaceID:123 OR SpaceID:789)", getSpaceIdsQuery([]string{"123", "", "789"}))
}
func TestFtSearch_Close(t *testing.T) {
// given
fts := new(ftSearchTantivy)

2
pkg/lib/localstore/objectstore/spaceindex/queries.go
View file

@ -281,7 +281,7 @@ func (s *dsObjectStore) performQuery(q database.Query) (records []database.Recor
}
func (s *dsObjectStore) performFulltextSearch(text string, spaceId string) ([]database.FulltextResult, error) {
ftsResults, err := s.fts.Search([]string{spaceId}, text)
ftsResults, err := s.fts.Search(spaceId, text)
if err != nil {
return nil, fmt.Errorf("fullText search: %w", err)
}

32
space/service.go
View file

@ -70,7 +70,6 @@ type Service interface {
SpaceViewId(spaceId string) (spaceViewId string, err error)
AccountMetadataSymKey() crypto.SymKey
AccountMetadataPayload() []byte
WaitPersonalSpaceMigration(ctx context.Context) (err error)
app.ComponentRunnable
}
@ -241,25 +240,6 @@ func (s *service) initAccount(ctx context.Context) (err error) {
if err != nil {
return fmt.Errorf("create tech space for old accounts: %w", err)
}
} else {
var id string
// have we migrated analytics id? we should have it in account object
err = s.techSpace.DoAccountObject(ctx, func(accountObject techspace.AccountObject) error {
id, err = accountObject.GetAnalyticsId()
return err
})
// this error can arise only from database issues
if err != nil {
return fmt.Errorf("get analytics id: %w", err)
}
// we still didn't migrate analytics id, then there is a chance that space view was not created for old accounts
if id == "" {
// creating a space view under the hood
_, err = s.startStatus(ctx, spaceinfo.NewSpacePersistentInfo(s.personalSpaceId))
if err != nil {
return fmt.Errorf("start personal space: %w", err)
}
}
}
s.techSpace.WakeUpViews()
// only persist networkId after successful space init
@ -315,18 +295,6 @@ func (s *service) Wait(ctx context.Context, spaceId string) (sp clientspace.Spac
return waiter.waitSpace(ctx, spaceId)
}
func (s *service) WaitPersonalSpaceMigration(ctx context.Context) (err error) {
waiter := newSpaceWaiter(s, s.ctx, waitSpaceDelay)
_, err = waiter.waitSpace(ctx, s.personalSpaceId)
if err != nil {
return fmt.Errorf("wait personal space: %w", err)
}
s.mu.Lock()
ctrl := s.spaceControllers[s.personalSpaceId]
s.mu.Unlock()
return ctrl.(personalspace.Personal).WaitMigrations(ctx)
}
func (s *service) Get(ctx context.Context, spaceId string) (sp clientspace.Space, err error) {
if spaceId == s.techSpaceId {
return s.getTechSpace(ctx)

34
space/service_test.go
View file

@ -75,28 +75,9 @@ func TestService_Init(t *testing.T) {
t.Run("new account", func(t *testing.T) {
newFixture(t, nil)
})
t.Run("old account, analytics id migrated", func(t *testing.T) {
t.Run("old account", func(t *testing.T) {
newFixture(t, func(t *testing.T, fx *fixture) {
fx.factory.EXPECT().LoadAndSetTechSpace(mock.Anything).Return(&clientspace.TechSpace{TechSpace: fx.techSpace}, nil)
accObject := mock_techspace.NewMockAccountObject(t)
accObject.EXPECT().GetAnalyticsId().Return("analyticsId", nil)
fx.techSpace.EXPECT().DoAccountObject(mock.Anything, mock.Anything).RunAndReturn(func(ctx2 context.Context, f func(techspace.AccountObject) error) error {
return f(accObject)
})
fx.techSpace.EXPECT().WakeUpViews()
})
})
t.Run("old account, analytics id not migrated", func(t *testing.T) {
newFixture(t, func(t *testing.T, fx *fixture) {
fx.factory.EXPECT().LoadAndSetTechSpace(mock.Anything).Return(&clientspace.TechSpace{TechSpace: fx.techSpace}, nil)
accObject := mock_techspace.NewMockAccountObject(t)
accObject.EXPECT().GetAnalyticsId().Return("", nil)
fx.techSpace.EXPECT().DoAccountObject(mock.Anything, mock.Anything).RunAndReturn(func(ctx2 context.Context, f func(techspace.AccountObject) error) error {
return f(accObject)
})
prCtrl := mock_spacecontroller.NewMockSpaceController(t)
fx.factory.EXPECT().NewPersonalSpace(mock.Anything, mock.Anything).Return(prCtrl, nil)
prCtrl.EXPECT().Close(mock.Anything).Return(nil)
fx.techSpace.EXPECT().WakeUpViews()
})
})
@ -105,14 +86,6 @@ func TestService_Init(t *testing.T) {
fx.factory.EXPECT().LoadAndSetTechSpace(mock.Anything).Return(nil, context.DeadlineExceeded).Times(1)
fx.spaceCore.EXPECT().StorageExistsLocally(mock.Anything, fx.spaceId).Return(false, nil)
fx.factory.EXPECT().LoadAndSetTechSpace(mock.Anything).Return(&clientspace.TechSpace{TechSpace: fx.techSpace}, nil)
accObject := mock_techspace.NewMockAccountObject(t)
accObject.EXPECT().GetAnalyticsId().Return("", nil)
fx.techSpace.EXPECT().DoAccountObject(mock.Anything, mock.Anything).RunAndReturn(func(ctx2 context.Context, f func(techspace.AccountObject) error) error {
return f(accObject)
})
prCtrl := mock_spacecontroller.NewMockSpaceController(t)
fx.factory.EXPECT().NewPersonalSpace(mock.Anything, mock.Anything).Return(prCtrl, nil)
prCtrl.EXPECT().Close(mock.Anything).Return(nil)
fx.techSpace.EXPECT().WakeUpViews()
})
})
@ -125,11 +98,6 @@ func TestService_Init(t *testing.T) {
prCtrl := mock_spacecontroller.NewMockSpaceController(t)
fx.factory.EXPECT().NewPersonalSpace(mock.Anything, mock.Anything).Return(prCtrl, nil)
prCtrl.EXPECT().Close(mock.Anything).Return(nil)
accObject := mock_techspace.NewMockAccountObject(t)
accObject.EXPECT().GetAnalyticsId().Return("", nil)
fx.techSpace.EXPECT().DoAccountObject(mock.Anything, mock.Anything).RunAndReturn(func(ctx2 context.Context, f func(techspace.AccountObject) error) error {
return f(accObject)
})
fx.techSpace.EXPECT().WakeUpViews()
})
})

9
space/techspace/techspace.go
View file

@ -3,6 +3,7 @@ package techspace
import (
"context"
"errors"
"fmt"
"sync"
"time"
@ -46,7 +47,6 @@ type AccountObject interface {
IsIconMigrated() (bool, error)
SetAnalyticsId(analyticsId string) (err error)
GetAnalyticsId() (string, error)
GetPrivateAnalyticsId() string
}
type TechSpace interface {
@ -122,9 +122,12 @@ func (s *techSpace) Run(techCoreSpace commonspace.Space, objectCache objectcache
s.objectCache = objectCache
if !create {
exists, err := s.accountObjectExists(s.ctx)
if err != nil || exists {
if err != nil {
return err
}
if exists {
return nil
}
}
return s.accountObjectCreate(s.ctx)
}
@ -360,7 +363,7 @@ func (s *techSpace) DoAccountObject(ctx context.Context, apply func(accountObjec
}
obj, err := s.objectCache.GetObject(ctx, id)
if err != nil {
return ErrAccountObjectNotExists
return fmt.Errorf("account object not exists %w: %w", ErrAccountObjectNotExists, err)
}
accountObject, ok := obj.(AccountObject)
if !ok {

3
space/waiter.go
View file

@ -41,7 +41,8 @@ func (w *spaceWaiter) waitSpace(ctx context.Context, spaceId string) (sp clients
// if there is no such space view then there is no space
exists, err := techSpace.SpaceViewExists(ctx, spaceId)
if err != nil {
return nil, fmt.Errorf("space view exists error: %w", err)
// func returns error only on derive
return nil, fmt.Errorf("space view derive error: %w", err)
}
if !exists {
return nil, ErrSpaceNotExists

22
tantivity_sha256.txt
View file

@ -1,11 +1,11 @@
b9ce3724a4b77c7d08d74843c08c0961ccbff8d446c6186ce0e7fd484bb1cc04 deps/libs/android-386.tar.gz
902dc8730451ad3e0e21d1d59cea3a833bb98bb7dc944b4263e5dbe955835101 deps/libs/android-amd64.tar.gz
9cd93ffd7ec2ab96ef0c3720ef30a1c1f290b785b0764cc9aae1f4eceaf128ca deps/libs/android-arm.tar.gz
220b5876c642080f39e23a0033e4b853d749c38b02b19c8b605571c53f05f833 deps/libs/android-arm64.tar.gz
d691873e86e6677e7df748982f274a0bfea92e6ac6e558c5c5d542263944436b deps/libs/darwin-amd64.tar.gz
83f4360fd53777805ba349924ade7fb1ebce00818bc2e37da87ccf7f8c53f024 deps/libs/darwin-arm64.tar.gz
a22f251a7b12152503d6b175e57c47f9117cccd0e3182e4febedda6615fcbd36 deps/libs/ios-amd64.tar.gz
1875f72dd8d87d833639d1ffef63d6a9e3a6175e442f1cf2954973b81f676125 deps/libs/ios-arm64.tar.gz
e873c9439307e5bacbe48a03bbc112a5550b5d907567950bd9cc33728d4f8cd3 deps/libs/ios-arm64-sim.tar.gz
63bf26424e423230d7f4a267d103ff91db8b5e9ad24916b104190f0ae006d480 deps/libs/linux-amd64-musl.tar.gz
d4269f6bc1fd70e8e15b4f396d9bb1ba046835ba75731d12071a9d079ec0ce4d deps/libs/windows-amd64.tar.gz
8bb5b1d7dc93a95f3b29450531304581ab9201e89238986d69a55df6afa54f65 deps/libs/android-386.tar.gz
0aa7d66c25ce31af1831508f4c1555c5632a6bcae57d5218ddbaf9215ee9c056 deps/libs/android-amd64.tar.gz
7a5f486924256e7ad86f76c85c003f02e181ab46aeb7b25a1c9ab6495b5cdee5 deps/libs/android-arm.tar.gz
266994bd53b14c571a685c9a2a93dca3c131b14d26b0b5493f359093bdb43388 deps/libs/android-arm64.tar.gz
267738043946821174e6682fb887f46fdc8ba2043c23e2b6552242380e945f7b deps/libs/darwin-amd64.tar.gz
40b09e38f1ec50b296df288b2f208f871a5f7420a8c1e797ba184d4303c2b4f2 deps/libs/darwin-arm64.tar.gz
2383e45089229d2deec53fc77bad522d6268581856064ce92980905f3a8ad71c deps/libs/ios-amd64.tar.gz
4e75cfe16e64e86cedf004dc2138c0bc6d6d8465a212ac245b777de666375c3f deps/libs/ios-arm64.tar.gz
aa43f85be9d6b9f8ee791775960814b3a1d9ffc313a6fd0bdad06e67f9b7cf87 deps/libs/ios-arm64-sim.tar.gz
94fecdb1703f7b89dab10b4bcdf802bd210cbaaf0acd53d09a17206f480037fa deps/libs/linux-amd64-musl.tar.gz
6316199ff84a23e4509e81af8e937534318338cb8c50d3726769af179abd0b1a deps/libs/windows-amd64.tar.gz

BIN
util/builtinobjects/data/get_started.zip
View file

Binary file not shown.

48
util/metricsid/metricsid.go Normal file
View file

@ -0,0 +1,48 @@
package metricsid
import (
"bytes"
"github.com/anyproto/any-sync/util/crypto"
"github.com/anyproto/any-sync/util/strkey"
"github.com/anyproto/go-slip10"
)
const (
metricsVersionByte strkey.VersionByte = 0xce
MetricsDerivationPath = "m/99999'/0'"
)
func deriveFromPrivKey(path string, privKey crypto.PrivKey) (key crypto.PrivKey, err error) {
rawBytes, err := privKey.Raw()
if err != nil {
return nil, err
}
node, err := slip10.DeriveForPath(path, rawBytes)
if err != nil {
return nil, err
}
return genKey(node)
}
func genKey(node slip10.Node) (key crypto.PrivKey, err error) {
reader := bytes.NewReader(node.RawSeed())
key, _, err = crypto.GenerateEd25519Key(reader)
return
}
func encodeMetricsId(pubKey crypto.PubKey) (string, error) {
raw, err := pubKey.Raw()
if err != nil {
return "", err
}
return strkey.Encode(metricsVersionByte, raw)
}
func DeriveMetricsId(privKey crypto.PrivKey) (string, error) {
key, err := deriveFromPrivKey(MetricsDerivationPath, privKey)
if err != nil {
return "", err
}
return encodeMetricsId(key.GetPublic())
}

20
util/metricsid/metricsid_test.go Normal file
View file

@ -0,0 +1,20 @@
package metricsid
import (
"testing"
"github.com/anyproto/any-sync/util/crypto"
"github.com/anyproto/any-sync/util/strkey"
"github.com/stretchr/testify/require"
)
func TestMetrics(t *testing.T) {
privKey, _, err := crypto.GenerateRandomEd25519KeyPair()
require.NoError(t, err)
res, err := DeriveMetricsId(privKey)
require.NoError(t, err)
decoded, err := strkey.Decode(metricsVersionByte, res)
require.NoError(t, err)
_, err = crypto.NewSigningEd25519PubKeyFromBytes(decoded)
require.NoError(t, err)
}