mirror of
https://github.com/anyproto/anytype-heart.git
synced 2025-06-07 21:37:04 +09:00
GO-5635 introduce protections and debug for tantivy
This commit is contained in:
Roman Khafizianov
parent
5c9e764948
commit
0e5a3fa109
9 changed files with 332 additions and 48 deletions
|
|
@ -8,6 +8,7 @@ import (
|
|||
|
||||
"github.com/anyproto/any-sync/app"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/domain"
|
||||
"github.com/anyproto/anytype-heart/core/event"
|
||||
"github.com/anyproto/anytype-heart/core/session"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/logging"
|
||||
|
|
@ -70,6 +71,11 @@ func (s *Service) stop() error {
|
|||
defer task.End()
|
||||
|
||||
if s != nil && s.app != nil {
|
||||
s.app.IterateComponents(func(c app.Component) {
|
||||
if c, ok := c.(app.ComponentStatable); ok {
|
||||
c.StateChange(int(domain.CompStateAppClosingInitiated))
|
||||
}
|
||||
})
|
||||
err := s.app.Close(ctx)
|
||||
if err != nil {
|
||||
log.Warnf("error while stop anytype: %v", err)
|
||||
|
|
|
|||
7
core/domain/app.go
Normal file
7
core/domain/app.go
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
package domain
|
||||
|
||||
type CompState int
|
||||
|
||||
const (
|
||||
CompStateAppClosingInitiated CompState = 1
|
||||
)
|
||||
|
|
@ -45,10 +45,10 @@ func (i *indexer) ForceFTIndex() {
|
|||
|
||||
// ftLoop runs full-text indexer
|
||||
// MUST NOT be called more than once
|
||||
func (i *indexer) ftLoopRoutine() {
|
||||
func (i *indexer) ftLoopRoutine(ctx context.Context) {
|
||||
tickerDuration := ftIndexInterval
|
||||
ticker := time.NewTicker(tickerDuration)
|
||||
ctx := i.runCtx
|
||||
|
||||
prevError := i.runFullTextIndexer(ctx)
|
||||
defer close(i.ftQueueFinished)
|
||||
var lastForceIndex time.Time
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ const (
|
|||
var log = logging.Logger("anytype-doc-indexer")
|
||||
|
||||
func New() Indexer {
|
||||
return &indexer{}
|
||||
return new(indexer)
|
||||
}
|
||||
|
||||
type Indexer interface {
|
||||
|
|
@ -59,6 +59,7 @@ type indexer struct {
|
|||
|
||||
runCtx context.Context
|
||||
runCtxCancel context.CancelFunc
|
||||
ftQueueStop context.CancelFunc
|
||||
ftQueueFinished chan struct{}
|
||||
config *config.Config
|
||||
|
||||
|
|
@ -98,12 +99,20 @@ func (i *indexer) Run(context.Context) (err error) {
|
|||
return i.StartFullTextIndex()
|
||||
}
|
||||
|
||||
func (f *indexer) StateChange(state int) {
|
||||
if state == int(domain.CompStateAppClosingInitiated) && f.ftQueueStop != nil {
|
||||
f.ftQueueStop()
|
||||
}
|
||||
}
|
||||
|
||||
func (i *indexer) StartFullTextIndex() (err error) {
|
||||
if ftErr := i.ftInit(); ftErr != nil {
|
||||
log.Errorf("can't init ft: %v", ftErr)
|
||||
}
|
||||
i.ftQueueFinished = make(chan struct{})
|
||||
go i.ftLoopRoutine()
|
||||
var ftCtx context.Context
|
||||
ftCtx, i.ftQueueStop = context.WithCancel(i.runCtx)
|
||||
go i.ftLoopRoutine(ftCtx)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
5
go.mod
5
go.mod
|
|
@ -16,7 +16,7 @@ require (
|
|||
github.com/anyproto/go-slip10 v1.0.0
|
||||
github.com/anyproto/lexid v0.0.4
|
||||
github.com/anyproto/protobuf v1.3.3-0.20240814124528-72b8c7e0e0f5
|
||||
github.com/anyproto/tantivy-go v1.0.1
|
||||
github.com/anyproto/tantivy-go v1.0.2
|
||||
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
|
||||
github.com/avast/retry-go/v4 v4.6.1
|
||||
github.com/chai2010/webp v1.1.2-0.20240612091223-aa1b379218b7
|
||||
|
|
@ -94,8 +94,6 @@ require (
|
|||
github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c
|
||||
github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef
|
||||
github.com/stretchr/testify v1.10.0
|
||||
github.com/swaggo/files v1.0.1
|
||||
github.com/swaggo/gin-swagger v1.6.0
|
||||
github.com/swaggo/swag/v2 v2.0.0-rc4
|
||||
github.com/uber/jaeger-client-go v2.30.0+incompatible
|
||||
github.com/valyala/fastjson v1.6.4
|
||||
|
|
@ -261,7 +259,6 @@ require (
|
|||
github.com/stretchr/objx v0.5.2 // indirect
|
||||
github.com/subosito/gotenv v1.4.2 // indirect
|
||||
github.com/sv-tools/openapi v0.2.1 // indirect
|
||||
github.com/swaggo/swag v1.16.4 // indirect
|
||||
github.com/tetratelabs/wazero v1.8.1 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.12 // indirect
|
||||
github.com/tklauser/numcpus v0.6.1 // indirect
|
||||
|
|
|
|||
18
go.sum
18
go.sum
|
|
@ -80,12 +80,6 @@ github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kk
|
|||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/anyproto/any-store v0.1.13 h1:1wmm0qQIRShaycBLKwcgkQbRKy3WrNPAShTE5fwzfCY=
|
||||
github.com/anyproto/any-store v0.1.13/go.mod h1:2M0Xf4rmijoKGd+nqqeKG8I1yIokCLEIxrAXEoHjXn4=
|
||||
github.com/anyproto/any-sync v0.6.15 h1:fxZHjiMcZJzJqzBprBNTYmm0MV8Y7NgIGPfLxlsgnWk=
|
||||
github.com/anyproto/any-sync v0.6.15/go.mod h1:TSKgCoTV40Bt8AfCh3RxPUUAfYGrhc8Mzh8/AiVlvX4=
|
||||
github.com/anyproto/any-sync v0.7.2 h1:S1UPzW0iYTLwsMAZ3rN/EJwthTGuadsvXdnGYNiC6cA=
|
||||
github.com/anyproto/any-sync v0.7.2/go.mod h1:TSKgCoTV40Bt8AfCh3RxPUUAfYGrhc8Mzh8/AiVlvX4=
|
||||
github.com/anyproto/any-sync v0.7.4 h1:pEkPn1fxJGvSGlsnAOy0lWVaqRgymyddmNy7T9toUQk=
|
||||
github.com/anyproto/any-sync v0.7.4/go.mod h1:TSKgCoTV40Bt8AfCh3RxPUUAfYGrhc8Mzh8/AiVlvX4=
|
||||
github.com/anyproto/any-sync v0.7.5 h1:VHayuacVpa2eRu5ubxCwrL3l0f/OSN7p45L8TxnaJEw=
|
||||
github.com/anyproto/any-sync v0.7.5/go.mod h1:02tMeQ6s/tneWLhoyzvy/ocGswICtvI48kdwTU8hQf8=
|
||||
github.com/anyproto/anytype-publish-server/publishclient v0.0.0-20250131145601-de288583ff2a h1:ZZM+0OUCQMWSLSflpkf0ZMVo3V76qEDDIXPpQOClNs0=
|
||||
|
|
@ -118,8 +112,8 @@ github.com/anyproto/protobuf v1.3.3-0.20240814124528-72b8c7e0e0f5 h1:aY7tBzQ+z8H
|
|||
github.com/anyproto/protobuf v1.3.3-0.20240814124528-72b8c7e0e0f5/go.mod h1:5+PHE01DgsDPkralb8MYmGg2sPQahsqEJ9ue7ciDHKg=
|
||||
github.com/anyproto/ristretto v0.1.2-0.20240221153107-2b23839cc50c h1:GicoaTUyB2mtCIl3YMrO0OzysqRT5GA4vuvDsqEkhSM=
|
||||
github.com/anyproto/ristretto v0.1.2-0.20240221153107-2b23839cc50c/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
|
||||
github.com/anyproto/tantivy-go v1.0.1 h1:Uc9WqwGEDsVUEwRgSg4nmhoW20GjMUBKRz5FYw4r+ns=
|
||||
github.com/anyproto/tantivy-go v1.0.1/go.mod h1:LtipOpRjGtcYMGcop6gQN7rVl1Pc6BlIs9BTMqeWMsk=
|
||||
github.com/anyproto/tantivy-go v1.0.2 h1:PVr4Tt4QH0JAZDs2Z9T4KJfLL+0mgXizj2Y5LSvEQdU=
|
||||
github.com/anyproto/tantivy-go v1.0.2/go.mod h1:LtipOpRjGtcYMGcop6gQN7rVl1Pc6BlIs9BTMqeWMsk=
|
||||
github.com/anyproto/zeroconf/v2 v2.2.1-0.20240228113933-f90a5cc4439d h1:5bj7nX/AS8sxGpTIrapE7PC4oPlhkHMwMqXlJbUHBlg=
|
||||
github.com/anyproto/zeroconf/v2 v2.2.1-0.20240228113933-f90a5cc4439d/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs=
|
||||
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
|
|
@ -318,8 +312,6 @@ github.com/gammazero/chanqueue v1.1.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+E
|
|||
github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34=
|
||||
github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4=
|
||||
github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
|
||||
|
|
@ -1058,12 +1050,6 @@ github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8
|
|||
github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
|
||||
github.com/sv-tools/openapi v0.2.1 h1:ES1tMQMJFGibWndMagvdoo34T1Vllxr1Nlm5wz6b1aA=
|
||||
github.com/sv-tools/openapi v0.2.1/go.mod h1:k5VuZamTw1HuiS9p2Wl5YIDWzYnHG6/FgPOSFXLAhGg=
|
||||
github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE=
|
||||
github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg=
|
||||
github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+zy8M=
|
||||
github.com/swaggo/gin-swagger v1.6.0/go.mod h1:BG00cCEy294xtVpyIAHG6+e2Qzj/xKlRdOqDkvq0uzo=
|
||||
github.com/swaggo/swag v1.16.4 h1:clWJtd9LStiG3VeijiCfOVODP6VpHtKdQy9ELFG3s1A=
|
||||
github.com/swaggo/swag v1.16.4/go.mod h1:VBsHJRsDvfYvqoiMKnsdwhNV9LEMHgEDZcyVYX0sxPg=
|
||||
github.com/swaggo/swag/v2 v2.0.0-rc4 h1:SZ8cK68gcV6cslwrJMIOqPkJELRwq4gmjvk77MrvHvY=
|
||||
github.com/swaggo/swag/v2 v2.0.0-rc4/go.mod h1:Ow7Y8gF16BTCDn8YxZbyKn8FkMLRUHekv1kROJZpbvE=
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
|
||||
|
|
|
|||
|
|
@ -16,11 +16,13 @@ package ftsearch
|
|||
import "C"
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
|
|
@ -29,9 +31,11 @@ import (
|
|||
tantivy "github.com/anyproto/tantivy-go"
|
||||
"github.com/valyala/fastjson"
|
||||
|
||||
"github.com/anyproto/anytype-heart/core/domain"
|
||||
"github.com/anyproto/anytype-heart/core/wallet"
|
||||
"github.com/anyproto/anytype-heart/metrics"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/bundle"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/localstore/ftsearch/tantivycheck"
|
||||
"github.com/anyproto/anytype-heart/pkg/lib/logging"
|
||||
"github.com/anyproto/anytype-heart/util/text"
|
||||
)
|
||||
|
|
@ -40,7 +44,7 @@ const (
|
|||
CName = "fts"
|
||||
ftsDir = "fts"
|
||||
ftsDir2 = "fts_tantivy"
|
||||
ftsVer = "13"
|
||||
ftsVer = "14"
|
||||
docLimit = 10000
|
||||
|
||||
fieldTitle = "Title"
|
||||
|
|
@ -57,7 +61,10 @@ const (
|
|||
tokenizerId = "SimpleIdTokenizer"
|
||||
)
|
||||
|
||||
var log = logging.Logger("ftsearch")
|
||||
var (
|
||||
log = logging.Logger("ftsearch")
|
||||
ErrAppClosingInitiated = errors.New("app closing initiated")
|
||||
)
|
||||
|
||||
type FTSearch interface {
|
||||
app.ComponentRunnable
|
||||
|
|
@ -93,14 +100,15 @@ type DocumentMatch struct {
|
|||
}
|
||||
|
||||
type ftSearch struct {
|
||||
rootPath string
|
||||
ftsPath string
|
||||
builderId string
|
||||
index *tantivy.TantivyContext
|
||||
parserPool *fastjson.ParserPool
|
||||
mu sync.Mutex
|
||||
blevePath string
|
||||
lang tantivy.Language
|
||||
rootPath string
|
||||
ftsPath string
|
||||
builderId string
|
||||
index *tantivy.TantivyContext
|
||||
parserPool *fastjson.ParserPool
|
||||
mu sync.Mutex
|
||||
blevePath string
|
||||
lang tantivy.Language
|
||||
appClosingInitiated atomic.Bool
|
||||
}
|
||||
|
||||
func (f *ftSearch) ProvideStat() any {
|
||||
|
|
@ -126,6 +134,9 @@ func (f *ftSearch) BatchDeleteObjects(ids []string) error {
|
|||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.appClosingInitiated.Load() {
|
||||
return ErrAppClosingInitiated
|
||||
}
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
spentMs := time.Since(start).Milliseconds()
|
||||
|
|
@ -147,6 +158,9 @@ func (f *ftSearch) BatchDeleteObjects(ids []string) error {
|
|||
func (f *ftSearch) DeleteObject(objectId string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.appClosingInitiated.Load() {
|
||||
return ErrAppClosingInitiated
|
||||
}
|
||||
return f.index.DeleteDocuments(fieldIdRaw, objectId)
|
||||
}
|
||||
|
||||
|
|
@ -182,6 +196,24 @@ func (f *ftSearch) Name() (name string) {
|
|||
}
|
||||
|
||||
func (f *ftSearch) Run(context.Context) error {
|
||||
report, err := tantivycheck.Check(f.ftsPath)
|
||||
if err != nil {
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
log.Warnf("tantivy index checking failed: %v", err)
|
||||
}
|
||||
}
|
||||
if !report.IsOk() {
|
||||
log.With("missingSegments", len(report.MissingSegments)).
|
||||
With("missingDelFiles", len(report.MissingDelFiles)).
|
||||
With("extraSegments", len(report.ExtraSegments)).
|
||||
With("extraDelFiles", len(report.ExtraDelFiles)).
|
||||
With("writerLockPresent", report.WriterLockPresent).
|
||||
With("metaLockPresent", report.MetaLockPresent).
|
||||
With("totalSegmentsInMeta", report.TotalSegmentsInMeta).
|
||||
With("uniqueSegmentPrefixesOnDisk", report.UniqueSegmentPrefixesOnDisk).
|
||||
Warnf("tantivy index is inconsistent state, dry run")
|
||||
}
|
||||
|
||||
builder, err := tantivy.NewSchemaBuilder()
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -328,6 +360,9 @@ func (f *ftSearch) tryToBuildSchema(schema *tantivy.Schema) (*tantivy.TantivyCon
|
|||
func (f *ftSearch) Index(doc SearchDoc) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.appClosingInitiated.Load() {
|
||||
return ErrAppClosingInitiated
|
||||
}
|
||||
metrics.ObjectFTUpdatedCounter.Inc()
|
||||
tantivyDoc, err := f.convertDoc(doc)
|
||||
if err != nil {
|
||||
|
|
@ -375,6 +410,9 @@ func (f *ftSearch) BatchIndex(ctx context.Context, docs []SearchDoc, deletedDocs
|
|||
}
|
||||
}()
|
||||
f.mu.Lock()
|
||||
if f.appClosingInitiated.Load() {
|
||||
return ErrAppClosingInitiated
|
||||
}
|
||||
err = f.index.DeleteDocuments(fieldIdRaw, deletedDocs...)
|
||||
f.mu.Unlock()
|
||||
if err != nil {
|
||||
|
|
@ -390,6 +428,9 @@ func (f *ftSearch) BatchIndex(ctx context.Context, docs []SearchDoc, deletedDocs
|
|||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.appClosingInitiated.Load() {
|
||||
return ErrAppClosingInitiated
|
||||
}
|
||||
return f.index.AddAndConsumeDocuments(tantivyDocs...)
|
||||
}
|
||||
|
||||
|
|
@ -572,8 +613,10 @@ func (f *ftSearch) DocCount() (uint64, error) {
|
|||
|
||||
func (f *ftSearch) Close(ctx context.Context) error {
|
||||
if f.index != nil {
|
||||
f.index.Free()
|
||||
f.index = nil
|
||||
err := f.index.Close()
|
||||
if err != nil {
|
||||
log.Errorf("failed to close tantivy index: %v", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -582,6 +625,12 @@ func (f *ftSearch) cleanupBleve() {
|
|||
_ = os.RemoveAll(f.blevePath)
|
||||
}
|
||||
|
||||
func (f *ftSearch) StateChange(state int) {
|
||||
if state == int(domain.CompStateAppClosingInitiated) {
|
||||
f.appClosingInitiated.Store(true)
|
||||
}
|
||||
}
|
||||
|
||||
func prepareQuery(query string) string {
|
||||
query = text.Truncate(query, 100, "")
|
||||
query = strings.ToLower(query)
|
||||
|
|
|
|||
230
pkg/lib/localstore/ftsearch/tantivycheck/tantivycheck.go
Normal file
230
pkg/lib/localstore/ftsearch/tantivycheck/tantivycheck.go
Normal file
|
|
@ -0,0 +1,230 @@
|
|||
// Package tantivycheck provides a DRY-RUN consistency check for Tantivy index
|
||||
// directories.
|
||||
//
|
||||
// It verifies that
|
||||
//
|
||||
// - every segment listed in meta.json has files on disk
|
||||
// - every expected <segment>.<opstamp>.del file exists
|
||||
// - there are no extra segment prefixes on disk
|
||||
// - there are no extra .del files on disk
|
||||
// - the special lock files INDEX_WRITER_LOCK and META_LOCK are noted
|
||||
//
|
||||
// Nothing is modified on disk; you simply get a ConsistencyReport back.
|
||||
package tantivycheck
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Package-level helpers (compiled once)
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
var (
|
||||
segPrefixRe = regexp.MustCompile(`^([0-9a-f]{32})(?:\..+)?$`)
|
||||
delFileRe = regexp.MustCompile(`^([0-9a-f]{32})\.(\d+)\.del$`)
|
||||
)
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Public API
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
// ConsistencyReport gathers all findings of the dry-run.
|
||||
type ConsistencyReport struct {
|
||||
// Segments present in meta.json but with no files on disk.
|
||||
MissingSegments []string
|
||||
// <segment>.<opstamp>.del files that meta.json expects but are absent.
|
||||
MissingDelFiles []string
|
||||
// Segment prefixes that exist on disk but are not referenced in meta.json.
|
||||
ExtraSegments []string
|
||||
// .del files on disk that are not referenced (wrong opstamp or orphan).
|
||||
ExtraDelFiles []string
|
||||
|
||||
// Lock-file information
|
||||
WriterLockPresent bool // true if INDEX_WRITER_LOCK exists
|
||||
MetaLockPresent bool // true if META_LOCK exists
|
||||
|
||||
// Informational counters
|
||||
TotalSegmentsInMeta int
|
||||
UniqueSegmentPrefixesOnDisk int
|
||||
}
|
||||
|
||||
// Check runs the consistency test against dir and returns a report.
|
||||
//
|
||||
// It fails with an error if meta.json is absent or can’t be decoded.
|
||||
func Check(dir string) (ConsistencyReport, error) {
|
||||
// ---------------------------------------------------------------------
|
||||
// 1) Parse meta.json
|
||||
// ---------------------------------------------------------------------
|
||||
meta, err := readMeta(filepath.Join(dir, "meta.json"))
|
||||
if err != nil {
|
||||
return ConsistencyReport{}, err
|
||||
}
|
||||
|
||||
// Build metaSegments: 32-hex-id (no dashes) → expected opstamp (nil if none)
|
||||
metaSegments := make(map[string]*uint64, len(meta.Segments))
|
||||
for _, s := range meta.Segments {
|
||||
id := stripDashes(s.SegmentID)
|
||||
if s.Deletes != nil {
|
||||
metaSegments[id] = &s.Deletes.Opstamp
|
||||
} else {
|
||||
metaSegments[id] = nil
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 2) Walk directory once
|
||||
// ---------------------------------------------------------------------
|
||||
segmentPrefixesDisk := map[string]struct{}{}
|
||||
delFilesDisk := map[[2]string]struct{}{} // key = [segPrefix, opstamp]
|
||||
|
||||
var writerLockPresent, metaLockPresent bool
|
||||
|
||||
err = filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
name := d.Name()
|
||||
|
||||
switch name {
|
||||
case "INDEX_WRITER_LOCK":
|
||||
writerLockPresent = true
|
||||
case "META_LOCK":
|
||||
metaLockPresent = true
|
||||
}
|
||||
|
||||
if m := segPrefixRe.FindStringSubmatch(name); m != nil {
|
||||
segmentPrefixesDisk[m[1]] = struct{}{}
|
||||
}
|
||||
if m := delFileRe.FindStringSubmatch(name); m != nil {
|
||||
delFilesDisk[[2]string{m[1], m[2]}] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return ConsistencyReport{}, fmt.Errorf("scanning directory: %w", err)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 3) Compare sets
|
||||
// ---------------------------------------------------------------------
|
||||
var (
|
||||
missingSegments []string
|
||||
extraSegments []string
|
||||
missingDelFiles []string
|
||||
extraDelFiles []string
|
||||
)
|
||||
|
||||
// missing segments
|
||||
for id := range metaSegments {
|
||||
if _, ok := segmentPrefixesDisk[id]; !ok {
|
||||
missingSegments = append(missingSegments, id)
|
||||
}
|
||||
}
|
||||
|
||||
// extra segments
|
||||
for id := range segmentPrefixesDisk {
|
||||
if _, ok := metaSegments[id]; !ok {
|
||||
extraSegments = append(extraSegments, id)
|
||||
}
|
||||
}
|
||||
|
||||
// missing del files
|
||||
for id, opPtr := range metaSegments {
|
||||
if opPtr == nil {
|
||||
continue // no deletes expected
|
||||
}
|
||||
opStr := strconv.FormatUint(*opPtr, 10)
|
||||
if _, ok := delFilesDisk[[2]string{id, opStr}]; !ok {
|
||||
missingDelFiles = append(missingDelFiles, fmt.Sprintf("%s.%s.del", id, opStr))
|
||||
}
|
||||
}
|
||||
|
||||
// extra del files
|
||||
for key := range delFilesDisk {
|
||||
id, opStr := key[0], key[1]
|
||||
expectedOpPtr, segKnown := metaSegments[id]
|
||||
if !segKnown || expectedOpPtr == nil || strconv.FormatUint(*expectedOpPtr, 10) != opStr {
|
||||
extraDelFiles = append(extraDelFiles, fmt.Sprintf("%s.%s.del", id, opStr))
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 4) Return aggregated report
|
||||
// ---------------------------------------------------------------------
|
||||
return ConsistencyReport{
|
||||
MissingSegments: missingSegments,
|
||||
MissingDelFiles: missingDelFiles,
|
||||
ExtraSegments: extraSegments,
|
||||
ExtraDelFiles: extraDelFiles,
|
||||
WriterLockPresent: writerLockPresent,
|
||||
MetaLockPresent: metaLockPresent,
|
||||
TotalSegmentsInMeta: len(metaSegments),
|
||||
UniqueSegmentPrefixesOnDisk: len(segmentPrefixesDisk),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// IsOk returns true when the report is free of inconsistencies:
|
||||
//
|
||||
// - no segments are missing
|
||||
// - no .del files are missing
|
||||
// - no extra segments are present
|
||||
// - no extra .del files are present
|
||||
//
|
||||
// The presence of INDEX_WRITER_LOCK or META_LOCK is *not* considered
|
||||
// an inconsistency—these files are expected during normal operation and
|
||||
// merely reported for information.
|
||||
func (r *ConsistencyReport) IsOk() bool {
|
||||
return len(r.MissingSegments) == 0 &&
|
||||
len(r.MissingDelFiles) == 0 &&
|
||||
len(r.ExtraSegments) == 0 &&
|
||||
len(r.ExtraDelFiles) == 0 &&
|
||||
!r.WriterLockPresent &&
|
||||
!r.MetaLockPresent
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Internal helpers
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
// metaFile mirrors only the parts of meta.json we need.
|
||||
type metaFile struct {
|
||||
Segments []struct {
|
||||
SegmentID string `json:"segment_id"`
|
||||
Deletes *struct {
|
||||
Opstamp uint64 `json:"opstamp"`
|
||||
} `json:"deletes"`
|
||||
} `json:"segments"`
|
||||
}
|
||||
|
||||
func readMeta(path string) (*metaFile, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s: %w", path, err)
|
||||
}
|
||||
var m metaFile
|
||||
if err := json.Unmarshal(b, &m); err != nil {
|
||||
return nil, fmt.Errorf("decoding meta.json: %w", err)
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
func stripDashes(s string) string {
|
||||
out := make([]byte, 0, len(s))
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] != '-' {
|
||||
out = append(out, s[i])
|
||||
}
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
565dd299ef61edff535b20ec166f34f2399529b8254eed5d137a771a7b2b7c04 deps/libs/android-386.tar.gz
|
||||
34d9c7d4a400653ddbcf28e64a5c8e055c08ab675df6b921cebedb4c5e667254 deps/libs/android-amd64.tar.gz
|
||||
d90b501033763687f1ae136d69e284033594870c7143e50624799ff474e12f41 deps/libs/android-arm.tar.gz
|
||||
3939f7ad6a1d08a0a559566d0ae5195739317a0023381bcb719603a7e2cac10d deps/libs/android-arm64.tar.gz
|
||||
b7f7c3e82b1054a4c0755e7cd32f98d1dbe2ec55dab49f07e6620c8e2cd3ff83 deps/libs/darwin-amd64.tar.gz
|
||||
40cdb051d2e063eb374e1a911ca7926beb4586c41a0347e430dc18fe5c216790 deps/libs/darwin-arm64.tar.gz
|
||||
2dc93d6cb9356df9b5c330a6c566e1da1a4dfb32a17f1262080f2c1a875a5775 deps/libs/ios-amd64.tar.gz
|
||||
2a949a364f29c5801c3c18607dbc14b5edefdcd002746ce94c89f297bb4f620c deps/libs/ios-arm64.tar.gz
|
||||
b09afc40d37a37377579ec4096511fd88cc22cdd387518225a77f205c226130a deps/libs/ios-arm64-sim.tar.gz
|
||||
122010a6f287a3d3b169a64c26918bd5745a1190c99638e7e91a664172781cf7 deps/libs/linux-amd64-musl.tar.gz
|
||||
2b86930fcd938fe41413f4f915c90a2093d703179c61153ec4a007cdd2ed4a9c deps/libs/linux-arm64-musl.tar.gz
|
||||
2cdefb8975651849f6c5f41c5a934035d959e797c6fdef9a5a946ec9052e3bee deps/libs/windows-amd64.tar.gz
|
||||
bbde99ff75d9c20a7eefeea4ca6c5c77a75c1d1114ee7d53d77b78a49741f2e3 deps/libs/android-386.tar.gz
|
||||
5c4599d29230f689ed2cfb21185548cb2fc3e8e61fd31e856c969fd4bdd7abc3 deps/libs/android-amd64.tar.gz
|
||||
e4d16d8fce08b5de22d475c8e9be02473c07786b298601d6b96768f0fc7ecbd6 deps/libs/android-arm.tar.gz
|
||||
8c67e2b0e8cf701ea6fecbe074f19ee48fb4321d02bc8a8b3fb11055397eb853 deps/libs/android-arm64.tar.gz
|
||||
1278ecd5b7d73e36163e48c6b98b88a6e7c96a663ddba31eb168224a6921fd92 deps/libs/darwin-amd64.tar.gz
|
||||
209429e1ffb2cdd55cedb90c74617d87aced121bc905802430f71b48b0ea6a43 deps/libs/darwin-arm64.tar.gz
|
||||
101f9afde0a06339d81060f1bb77be63055d9674878cea5f7b142fcc3ca9658e deps/libs/ios-amd64.tar.gz
|
||||
8b0f568bbf09e0ada76c9b331eab59ad9ed409aa37dec3b3d0538480babfeb5e deps/libs/ios-arm64.tar.gz
|
||||
f3a90aff6eab8aa0e3e89a2866f0dbbd9c5ffb0d8a47d5aa4ad3d28afbd3416b deps/libs/ios-arm64-sim.tar.gz
|
||||
74c3585022a962794c49f8f15606d3093684cad5543868e90d91d3ca876c45a8 deps/libs/linux-amd64-musl.tar.gz
|
||||
2e1a052e7c12b3a0fa845c53c151228243ab6483c9fa3d83b97f34f40550fa6b deps/libs/linux-arm64-musl.tar.gz
|
||||
5357ae4522bd826a8c53cb39d8c63b77e4cbc1683154c54b6f71542a25b1dcd1 deps/libs/windows-amd64.tar.gz
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue