diff --git a/core/indexer/fulltext.go b/core/indexer/fulltext.go index ec3cc6913..3ef397fb0 100644 --- a/core/indexer/fulltext.go +++ b/core/indexer/fulltext.go @@ -61,7 +61,7 @@ func (i *indexer) ftLoopRoutine() { } func (i *indexer) runFullTextIndexer(ctx context.Context) { - batcher := i.ftsearch.NewAutoBatcher(ftsearch.AutoBatcherRecommendedMaxDocs, ftsearch.AutoBatcherRecommendedMaxSize) + batcher := i.ftsearch.NewAutoBatcher() err := i.store.BatchProcessFullTextQueue(ctx, ftBatchLimit, func(objectIds []string) error { for _, objectId := range objectIds { objDocs, err := i.prepareSearchDocument(ctx, objectId) @@ -173,7 +173,7 @@ func (i *indexer) prepareSearchDocument(ctx context.Context, id string) (docs [] doc := ftsearch.SearchDoc{ Id: domain.NewObjectPathWithRelation(id, rel.Key).String(), - SpaceID: sb.SpaceID(), + SpaceId: sb.SpaceID(), Text: val, } @@ -202,7 +202,7 @@ func (i *indexer) prepareSearchDocument(ctx context.Context, id string) (docs [] } doc := ftsearch.SearchDoc{ Id: domain.NewObjectPathWithBlock(id, b.Model().Id).String(), - SpaceID: sb.SpaceID(), + SpaceId: sb.SpaceID(), } if len(tb.Text) > ftBlockMaxSize { doc.Text = tb.Text[:ftBlockMaxSize] diff --git a/core/indexer/fulltext_test.go b/core/indexer/fulltext_test.go index a87b77b6e..2e7e5aebb 100644 --- a/core/indexer/fulltext_test.go +++ b/core/indexer/fulltext_test.go @@ -111,7 +111,7 @@ func TestPrepareSearchDocument_Success(t *testing.T) { assert.NoError(t, err) require.Len(t, docs, 1) assert.Equal(t, "objectId1/b/blockId1", docs[0].Id) - assert.Equal(t, "spaceId1", docs[0].SpaceID) + assert.Equal(t, "spaceId1", docs[0].SpaceId) } func TestPrepareSearchDocument_Empty_NotIndexing(t *testing.T) { @@ -366,8 +366,8 @@ func TestRunFullTextIndexer(t *testing.T) { func TestPrepareSearchDocument_Reindex_Removed(t *testing.T) { indexerFx := NewIndexerFixture(t) - indexerFx.ftsearch.Index(ftsearch.SearchDoc{Id: "objectId1/r/blockId1", SpaceID: "spaceId1"}) - indexerFx.ftsearch.Index(ftsearch.SearchDoc{Id: "objectId1/r/blockId2", SpaceID: "spaceId1"}) + indexerFx.ftsearch.Index(ftsearch.SearchDoc{Id: "objectId1/r/blockId1", SpaceId: "spaceId1"}) + indexerFx.ftsearch.Index(ftsearch.SearchDoc{Id: "objectId1/r/blockId2", SpaceId: "spaceId1"}) count, _ := indexerFx.ftsearch.DocCount() assert.Equal(t, uint64(2), count) diff --git a/core/object.go b/core/object.go index b37144617..663728680 100644 --- a/core/object.go +++ b/core/object.go @@ -89,12 +89,12 @@ func (mw *Middleware) ObjectSearch(cctx context.Context, req *pb.RpcObjectSearch ds := mw.applicationService.GetApp().MustComponent(objectstore.CName).(objectstore.ObjectStore) records, err := ds.SpaceIndex(req.SpaceId).Query(database.Query{ - Filters: req.Filters, - SpaceId: req.SpaceId, - Sorts: req.Sorts, - Offset: int(req.Offset), - Limit: int(req.Limit), - FullText: req.FullText, + Filters: req.Filters, + SpaceId: req.SpaceId, + Sorts: req.Sorts, + Offset: int(req.Offset), + Limit: int(req.Limit), + TextQuery: req.FullText, }) if err != nil { return response(pb.RpcObjectSearchResponseError_UNKNOWN_ERROR, nil, err) @@ -136,12 +136,12 @@ func (mw *Middleware) ObjectSearchWithMeta(cctx context.Context, req *pb.RpcObje ds := mw.applicationService.GetApp().MustComponent(objectstore.CName).(objectstore.ObjectStore) results, err := ds.SpaceIndex(req.SpaceId).Query(database.Query{ - Filters: req.Filters, - Sorts: req.Sorts, - Offset: int(req.Offset), - Limit: int(req.Limit), - FullText: req.FullText, - SpaceId: req.SpaceId, + Filters: req.Filters, + Sorts: req.Sorts, + Offset: int(req.Offset), + Limit: int(req.Limit), + TextQuery: req.FullText, + SpaceId: req.SpaceId, }) var resultsModels = make([]*model.SearchResult, 0, len(results)) diff --git a/go.sum b/go.sum index b4fab631e..90fa01a09 100644 --- a/go.sum +++ b/go.sum @@ -1369,8 +1369,6 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= -golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/pkg/lib/database/database.go b/pkg/lib/database/database.go index 398673845..fb3767243 100644 --- a/pkg/lib/database/database.go +++ b/pkg/lib/database/database.go @@ -25,12 +25,12 @@ type Record struct { } type Query struct { - FullText string - SpaceId string - Filters []*model.BlockContentDataviewFilter // filters results. apply sequentially - Sorts []*model.BlockContentDataviewSort // order results. apply hierarchically - Limit int // maximum number of results - Offset int // skip given number of results + TextQuery string + SpaceId string + Filters []*model.BlockContentDataviewFilter // filters results. apply sequentially + Sorts []*model.BlockContentDataviewSort // order results. apply hierarchically + Limit int // maximum number of results + Offset int // skip given number of results } func injectDefaultFilters(filters []*model.BlockContentDataviewFilter) []*model.BlockContentDataviewFilter { @@ -105,7 +105,7 @@ func injectDefaultOrder(qry Query, sorts []*model.BlockContentDataviewSort) []*m var ( hasScoreSort bool ) - if qry.FullText == "" { + if qry.TextQuery == "" { return sorts } diff --git a/pkg/lib/localstore/ftsearch/autobatcher_test.go b/pkg/lib/localstore/ftsearch/autobatcher_test.go index c2679aeb4..255ea13af 100644 --- a/pkg/lib/localstore/ftsearch/autobatcher_test.go +++ b/pkg/lib/localstore/ftsearch/autobatcher_test.go @@ -19,7 +19,7 @@ func Test_AutoBatcher(t *testing.T) { require.NoError(t, err) require.Equal(t, 0, int(docsCount)) - batcher := ft.NewAutoBatcher(30, 100000) + batcher := ft.NewAutoBatcher() for i := 0; i < 32; i++ { err = batcher.UpdateDoc( SearchDoc{ diff --git a/pkg/lib/localstore/ftsearch/autobatchertantivy.go b/pkg/lib/localstore/ftsearch/autobatchertantivy.go index 6745b7caf..7fa4e7190 100644 --- a/pkg/lib/localstore/ftsearch/autobatchertantivy.go +++ b/pkg/lib/localstore/ftsearch/autobatchertantivy.go @@ -17,7 +17,7 @@ type AutoBatcher interface { Finish() error } -func (f *ftSearchTantivy) NewAutoBatcher(maxDocs int, maxSizeBytes uint64) AutoBatcher { +func (f *ftSearchTantivy) NewAutoBatcher() AutoBatcher { return &ftIndexBatcherTantivy{ index: f.index, mu: &f.mu, @@ -88,7 +88,7 @@ func (f *ftSearchTantivy) Iterate(objectId string, fields []string, shouldContin Id: hit.ID, Text: text, Title: title, - SpaceID: spaceId, + SpaceId: spaceId, }) { break } @@ -124,7 +124,7 @@ func (f *ftIndexBatcherTantivy) UpdateDoc(searchDoc SearchDoc) error { return err } - err = doc.AddField(fieldSpace, searchDoc.SpaceID, f.index) + err = doc.AddField(fieldSpace, searchDoc.SpaceId, f.index) if err != nil { return err } diff --git a/pkg/lib/localstore/ftsearch/ftsearchtantivy.go b/pkg/lib/localstore/ftsearch/ftsearchtantivy.go index b851cf995..b91dcd2b5 100644 --- a/pkg/lib/localstore/ftsearch/ftsearchtantivy.go +++ b/pkg/lib/localstore/ftsearch/ftsearchtantivy.go @@ -37,7 +37,7 @@ const ( CName = "fts" ftsDir = "fts" ftsDir2 = "fts_tantivy" - ftsVer = "10" + ftsVer = "8" docLimit = 10000 fieldTitle = "Title" @@ -56,15 +56,10 @@ const ( var log = logging.Logger("ftsearch") -const ( - AutoBatcherRecommendedMaxDocs = 300 - AutoBatcherRecommendedMaxSize = 10 * 1024 * 1024 // 10MB -) - type FTSearch interface { app.ComponentRunnable Index(d SearchDoc) (err error) - NewAutoBatcher(maxDocs int, maxDocsSize uint64) AutoBatcher + NewAutoBatcher() AutoBatcher BatchIndex(ctx context.Context, docs []SearchDoc, deletedDocs []string) (err error) BatchDeleteObjects(ids []string) (err error) Search(spaceIds []string, query string) (results []*DocumentMatch, err error) @@ -74,15 +69,22 @@ type FTSearch interface { } type SearchDoc struct { - //nolint:all Id string - SpaceID string + SpaceId string Title string Text string } -func TantivyNew() FTSearch { - return new(ftSearchTantivy) +type Highlight struct { + Ranges [][]int `json:"r"` + Text string `json:"t"` +} + +type DocumentMatch struct { + Score float64 + ID string + Fragments map[string]*Highlight + Fields map[string]any } var specialChars = map[rune]struct{}{ @@ -101,6 +103,10 @@ type ftSearchTantivy struct { mu sync.Mutex } +func TantivyNew() FTSearch { + return new(ftSearchTantivy) +} + func (f *ftSearchTantivy) BatchDeleteObjects(ids []string) error { if len(ids) == 0 { return nil @@ -303,7 +309,7 @@ func (f *ftSearchTantivy) convertDoc(doc SearchDoc) (*tantivy.Document, error) { if err != nil { return nil, err } - err = document.AddField(fieldSpace, doc.SpaceID, f.index) + err = document.AddField(fieldSpace, doc.SpaceId, f.index) if err != nil { return nil, err } @@ -360,18 +366,6 @@ func (f *ftSearchTantivy) BatchIndex(ctx context.Context, docs []SearchDoc, dele return f.index.AddAndConsumeDocuments(tantivyDocs...) } -type Highlight struct { - Ranges [][]int `json:"r"` - Text string `json:"t"` -} - -type DocumentMatch struct { - Score float64 - ID string - Fragments map[string]*Highlight - Fields map[string]any -} - func (f *ftSearchTantivy) Search(spaceIds []string, query string) (results []*DocumentMatch, err error) { spaceIdsQuery := getSpaceIdsQuery(spaceIds) query = prepareQuery(query) diff --git a/pkg/lib/localstore/ftsearch/ftsearchtantivy_test.go b/pkg/lib/localstore/ftsearch/ftsearchtantivy_test.go index 703589d2c..98c08426d 100644 --- a/pkg/lib/localstore/ftsearch/ftsearchtantivy_test.go +++ b/pkg/lib/localstore/ftsearch/ftsearchtantivy_test.go @@ -68,12 +68,12 @@ func TestDifferentSpaces(t *testing.T) { require.NoError(t, ft.Index(SearchDoc{ Id: "1", Title: "one", - SpaceID: "space1", + SpaceId: "space1", })) require.NoError(t, ft.Index(SearchDoc{ Id: "2", Title: "one", - SpaceID: "space2", + SpaceId: "space2", })) search, err := ft.Search([]string{"space1"}, "one") @@ -219,11 +219,11 @@ func assertProperIds(t *testing.T, tmpDir string) { for i := range 50 { docs = append(docs, SearchDoc{ Id: fmt.Sprintf("randomid%d/r/randomrel%d", i, i+100), - SpaceID: fmt.Sprintf("randomspaceid%d", i), + SpaceId: fmt.Sprintf("randomspaceid%d", i), }) docs = append(docs, SearchDoc{ Id: fmt.Sprintf("randomid%d/r/randomrel%d", i, i+1000), - SpaceID: fmt.Sprintf("randomspaceid%d", i), + SpaceId: fmt.Sprintf("randomspaceid%d", i), }) } assert.NoError(t, ft.BatchIndex(context.Background(), docs, nil)) @@ -262,22 +262,22 @@ func assertMultiSpace(t *testing.T, tmpDir string) { ft := fixture.ft require.NoError(t, ft.Index(SearchDoc{ Id: "1/1", - SpaceID: "first", + SpaceId: "first", Title: "Dashboard of first space", })) require.NoError(t, ft.Index(SearchDoc{ Id: "1/2", - SpaceID: "first", + SpaceId: "first", Title: "Advanced of first space", })) require.NoError(t, ft.Index(SearchDoc{ Id: "2/1", - SpaceID: "second", + SpaceId: "second", Title: "Dashboard of second space", })) require.NoError(t, ft.Index(SearchDoc{ Id: "2/2", - SpaceID: "second", + SpaceId: "second", Title: "Get Started of second space", })) require.NoError(t, ft.Index(SearchDoc{ diff --git a/pkg/lib/localstore/objectstore/spaceindex/queries.go b/pkg/lib/localstore/objectstore/spaceindex/queries.go index 4fb047455..528efa04d 100644 --- a/pkg/lib/localstore/objectstore/spaceindex/queries.go +++ b/pkg/lib/localstore/objectstore/spaceindex/queries.go @@ -264,18 +264,18 @@ func (s *dsObjectStore) performQuery(q database.Query) (records []database.Recor collatorBuffer := s.collatorBufferPool.get() defer s.collatorBufferPool.put(collatorBuffer) - q.FullText = strings.TrimSpace(q.FullText) + q.TextQuery = strings.TrimSpace(q.TextQuery) filters, err := database.NewFilters(q, s, arena, collatorBuffer) if err != nil { return nil, fmt.Errorf("new filters: %w", err) } - if q.FullText != "" { - fulltextResults, err := s.performFulltextSearch(q.FullText, q.SpaceId) + if q.TextQuery != "" { + fulltextResults, err := s.performFulltextSearch(q.TextQuery, q.SpaceId) if err != nil { return nil, fmt.Errorf("perform fulltext search: %w", err) } - return s.QueryFromFulltext(fulltextResults, *filters, q.Limit, q.Offset, q.FullText) + return s.QueryFromFulltext(fulltextResults, *filters, q.Limit, q.Offset, q.TextQuery) } return s.QueryRaw(filters, q.Limit, q.Offset) } diff --git a/pkg/lib/localstore/objectstore/spaceindex/queries_test.go b/pkg/lib/localstore/objectstore/spaceindex/queries_test.go index 91834cd57..72eb01369 100644 --- a/pkg/lib/localstore/objectstore/spaceindex/queries_test.go +++ b/pkg/lib/localstore/objectstore/spaceindex/queries_test.go @@ -197,7 +197,7 @@ func TestQuery(t *testing.T) { t.Run("just full-text", func(t *testing.T) { recs, err := s.Query(database.Query{ - FullText: "important", + TextQuery: "important", }) require.NoError(t, err) @@ -210,7 +210,7 @@ func TestQuery(t *testing.T) { t.Run("fulltext by relation", func(t *testing.T) { recs, err := s.Query(database.Query{ - FullText: "myname1", + TextQuery: "myname1", }) require.NoError(t, err) @@ -222,7 +222,7 @@ func TestQuery(t *testing.T) { t.Run("full-text and filter", func(t *testing.T) { recs, err := s.Query(database.Query{ - FullText: "important", + TextQuery: "important", Filters: []*model.BlockContentDataviewFilter{ { RelationKey: bundle.RelationKeyDescription.String(), @@ -379,7 +379,7 @@ func TestQuery(t *testing.T) { t.Run("full-text relation description", func(t *testing.T) { recs, err := s.Query(database.Query{ - FullText: "first object", + TextQuery: "first object", }) require.NoError(t, err) removeScoreFromRecords(recs) @@ -412,7 +412,7 @@ func TestQuery(t *testing.T) { t.Run("full-text block single match", func(t *testing.T) { recs, err := s.Query(database.Query{ - FullText: "sage", + TextQuery: "sage", }) require.NoError(t, err) removeScoreFromRecords(recs) @@ -432,7 +432,7 @@ func TestQuery(t *testing.T) { t.Run("full-text block multi match", func(t *testing.T) { recs, err := s.Query(database.Query{ - FullText: "block", + TextQuery: "block", Sorts: []*model.BlockContentDataviewSort{ { RelationKey: bundle.RelationKeyId.String(), @@ -479,7 +479,7 @@ func TestQuery(t *testing.T) { t.Run("full-text block single match truncated", func(t *testing.T) { recs, err := s.Query(database.Query{ - FullText: "dog", + TextQuery: "dog", }) require.NoError(t, err) removeScoreFromRecords(recs) @@ -510,7 +510,7 @@ func TestQuery(t *testing.T) { t.Run("full-text block single match truncated cyrillic", func(t *testing.T) { recs, err := s.Query(database.Query{ - FullText: "Сонце", + TextQuery: "Сонце", }) require.NoError(t, err) removeScoreFromRecords(recs) @@ -533,7 +533,7 @@ func TestQuery(t *testing.T) { t.Run("full-text by tag", func(t *testing.T) { recs, err := s.Query(database.Query{ - FullText: "relname", + TextQuery: "relname", Filters: []*model.BlockContentDataviewFilter{ { Operator: 0, @@ -557,7 +557,7 @@ func TestQuery(t *testing.T) { t.Run("full-text by deleted tag", func(t *testing.T) { recs, err := s.Query(database.Query{ - FullText: "deleted", + TextQuery: "deleted", Filters: []*model.BlockContentDataviewFilter{ { Operator: 0, @@ -574,7 +574,7 @@ func TestQuery(t *testing.T) { t.Run("full-text by archived tag", func(t *testing.T) { recs, err := s.Query(database.Query{ - FullText: "archived", + TextQuery: "archived", Filters: []*model.BlockContentDataviewFilter{ { Operator: 0, @@ -591,7 +591,7 @@ func TestQuery(t *testing.T) { t.Run("full-text by type", func(t *testing.T) { recs, err := s.Query(database.Query{ - FullText: "typename", + TextQuery: "typename", Filters: []*model.BlockContentDataviewFilter{ { Operator: 0,