From 99680e4f345af946eee61e300064c5777b51f124 Mon Sep 17 00:00:00 2001 From: Sergey Date: Thu, 1 Aug 2024 10:51:27 +0200 Subject: [PATCH] Integrate: fixes --- core/block/editor/sub_objects_migration.go | 6 +- core/block/editor/template.go | 1 - core/block/editor/workspaces.go | 4 +- core/indexer/fulltext.go | 2 +- core/kanban/group_status.go | 3 +- core/kanban/group_tag.go | 2 +- core/kanban/service.go | 2 +- core/subscription/collection.go | 1 + pkg/lib/database/order.go | 91 +++++++++---------- pkg/lib/localstore/objectstore/queries.go | 16 ++-- .../systemobjectreviser.go | 4 +- util/internalflag/flag.go | 3 +- util/slice/slice.go | 12 +++ util/time/time.go | 12 --- 14 files changed, 79 insertions(+), 80 deletions(-) diff --git a/core/block/editor/sub_objects_migration.go b/core/block/editor/sub_objects_migration.go index 4b106daed..ace8ee3f1 100644 --- a/core/block/editor/sub_objects_migration.go +++ b/core/block/editor/sub_objects_migration.go @@ -53,7 +53,7 @@ func (m *subObjectsMigration) migrateSubObjects(st *state.State) { return } - if pbtypes.GetBool(info.Details, migratedKey) { + if info.Details.GetBoolOrDefault(migratedKey, false) { return } uniqueKeyRaw := info.Details.GetStringOrDefault(bundle.RelationKeyUniqueKey, "") @@ -114,7 +114,7 @@ func (m *subObjectsMigration) migrateSubObjects(st *state.State) { func (m *subObjectsMigration) migrateSubObject( ctx context.Context, uniqueKeyRaw string, - details *types.Struct, + details *domain.Details, typeKey domain.TypeKey, ) (id string, err error) { uniqueKey, err := domain.UnmarshalUniqueKey(uniqueKeyRaw) @@ -181,7 +181,7 @@ func (m *subObjectsMigration) iterateAllSubObjects(st *state.State, proc func(in continue } - details := v.StructValue + details := domain.NewDetailsFromProto(v.StructValue) details.Set(bundle.RelationKeyUniqueKey, pbtypes.String(uk.Marshal())) proc(smartblock.DocInfo{ diff --git a/core/block/editor/template.go b/core/block/editor/template.go index dacfb8cd5..3ff31c475 100644 --- a/core/block/editor/template.go +++ b/core/block/editor/template.go @@ -9,7 +9,6 @@ import ( "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/pkg/lib/bundle" coresb "github.com/anyproto/anytype-heart/pkg/lib/core/smartblock" - "github.com/anyproto/anytype-heart/util/pbtypes" ) type Template struct { diff --git a/core/block/editor/workspaces.go b/core/block/editor/workspaces.go index 682162985..c12d774d6 100644 --- a/core/block/editor/workspaces.go +++ b/core/block/editor/workspaces.go @@ -113,7 +113,7 @@ func (w *Workspaces) RemoveExistingInviteInfo() (fileCid string, err error) { details := w.Details() fileCid = details.GetStringOrDefault(bundle.RelationKeySpaceInviteFileCid, "") newState := w.NewState() - newState.RemoveDetail(bundle.RelationKeySpaceInviteFileCid.String(), bundle.RelationKeySpaceInviteFileKey.String()) + newState.RemoveDetail(bundle.RelationKeySpaceInviteFileCid, bundle.RelationKeySpaceInviteFileKey) return fileCid, w.Apply(newState) } @@ -127,6 +127,6 @@ func (w *Workspaces) onApply(info smartblock.ApplyInfo) error { } func (w *Workspaces) onWorkspaceChanged(state *state.State) { - details := pbtypes.CopyStruct(state.CombinedDetails(), true) + details := state.CombinedDetails().ShallowCopy() w.spaceService.OnWorkspaceChanged(w.SpaceID(), details) } diff --git a/core/indexer/fulltext.go b/core/indexer/fulltext.go index 626db20f2..380c21e4e 100644 --- a/core/indexer/fulltext.go +++ b/core/indexer/fulltext.go @@ -161,7 +161,7 @@ func (i *indexer) prepareSearchDocument(ctx context.Context, id string) (docs [] if rel.Format != model.RelationFormat_shorttext && rel.Format != model.RelationFormat_longtext { continue } - val := pbtypes.GetString(sb.Details(), rel.Key) + val := sb.Details().GetStringOrDefault(domain.RelationKey(rel.Key), "") if val == "" { continue } diff --git a/core/kanban/group_status.go b/core/kanban/group_status.go index dab33b2a8..c4b582c7c 100644 --- a/core/kanban/group_status.go +++ b/core/kanban/group_status.go @@ -3,13 +3,14 @@ package kanban import ( "sort" + "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/pkg/lib/database" "github.com/anyproto/anytype-heart/pkg/lib/localstore/objectstore" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" ) type GroupStatus struct { - key string + key domain.RelationKey store objectstore.ObjectStore Options []*model.RelationOption } diff --git a/core/kanban/group_tag.go b/core/kanban/group_tag.go index 199f672fa..d4913bf1e 100644 --- a/core/kanban/group_tag.go +++ b/core/kanban/group_tag.go @@ -28,7 +28,7 @@ func (t *GroupTag) InitGroups(spaceID string, f *database.Filters) error { } filterTag := database.FiltersAnd{ - database.FilterNot{Filter: database.FilterEmpty{Key: string(t.Key)}}, + database.FilterNot{Filter: database.FilterEmpty{Key: t.Key}}, } if f == nil { diff --git a/core/kanban/service.go b/core/kanban/service.go index a26ddfbd4..251be7271 100644 --- a/core/kanban/service.go +++ b/core/kanban/service.go @@ -43,7 +43,7 @@ func (s *service) Init(a *app.App) (err error) { s.objectStore = a.MustComponent(objectstore.CName).(objectstore.ObjectStore) s.groupColumns[model.RelationFormat_status] = func(key string) Grouper { - return &GroupStatus{key: key, store: s.objectStore} + return &GroupStatus{key: domain.RelationKey(key), store: s.objectStore} } s.groupColumns[model.RelationFormat_tag] = func(key string) Grouper { return &GroupTag{Key: domain.RelationKey(key), store: s.objectStore} diff --git a/core/subscription/collection.go b/core/subscription/collection.go index 90f5c7888..09e433239 100644 --- a/core/subscription/collection.go +++ b/core/subscription/collection.go @@ -126,6 +126,7 @@ func (c *collectionObserver) FilterObject(g *types.Struct) bool { } // AnystoreSort called only once when subscription is created +// TODO make collectionObserver to satify query.Filter interface func (c *collectionObserver) AnystoreFilter() query.Filter { c.lock.RLock() defer c.lock.RUnlock() diff --git a/pkg/lib/database/order.go b/pkg/lib/database/order.go index a33448e1e..c502fddea 100644 --- a/pkg/lib/database/order.go +++ b/pkg/lib/database/order.go @@ -1,21 +1,22 @@ package database import ( + "time" + "github.com/anyproto/any-store/encoding" "github.com/anyproto/any-store/query" - "github.com/gogo/protobuf/types" "github.com/valyala/fastjson" "golang.org/x/text/collate" "golang.org/x/text/language" + "github.com/anyproto/anytype-heart/core/domain" "github.com/anyproto/anytype-heart/pkg/lib/bundle" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" - "github.com/anyproto/anytype-heart/util/pbtypes" time_util "github.com/anyproto/anytype-heart/util/time" ) type Order interface { - Compare(a, b *types.Struct) int + Compare(a, b *domain.Details) int AnystoreSort() query.Sort } @@ -27,7 +28,7 @@ type ObjectStore interface { type SetOrder []Order -func (so SetOrder) Compare(a, b *types.Struct) int { +func (so SetOrder) Compare(a, b *domain.Details) int { for _, o := range so { if comp := o.Compare(a, b); comp != 0 { return comp @@ -68,9 +69,9 @@ func (ko *KeyOrder) ensureCollator() { } } -func (ko *KeyOrder) Compare(a, b *types.Struct) int { - av := pbtypes.Get(a, ko.Key) - bv := pbtypes.Get(b, ko.Key) +func (ko *KeyOrder) Compare(a, b *domain.Details) int { + av := a.Get(domain.RelationKey(ko.Key)) + bv := b.Get(domain.RelationKey(ko.Key)) av, bv = ko.tryExtractSnippet(a, b, av, bv) av, bv = ko.tryExtractDateTime(av, bv) @@ -117,9 +118,9 @@ func (ko *KeyOrder) basicSort(valType fastjson.Type) query.Sort { return ko.emptyPlacementSort(valType) } else { return &query.SortField{ - Path: []string{ko.Key}, + Path: []string{string(ko.Key)}, Reverse: ko.Type == model.BlockContentDataviewSort_Desc, - Field: ko.Key, + Field: string(ko.Key), } } } @@ -129,7 +130,7 @@ func (ko *KeyOrder) tagStatusSort() query.Sort { ko.Options = make(map[string]string) } if len(ko.Options) == 0 && ko.Store != nil { - ko.Options = optionsToMap(ko.SpaceID, ko.Key, ko.Store) + ko.Options = optionsToMap(ko.SpaceID, domain.RelationKey(ko.Key), ko.Store) } return tagStatusSort{ arena: ko.arena, @@ -171,18 +172,12 @@ func (ko *KeyOrder) textSort() query.Sort { } } -func (ko *KeyOrder) tryAdjustEmptyPositions(av *types.Value, bv *types.Value, comp int) int { +func (ko *KeyOrder) tryAdjustEmptyPositions(av domain.Value, bv domain.Value, comp int) int { if ko.EmptyPlacement == model.BlockContentDataviewSort_NotSpecified { return comp } - _, aNull := av.GetKind().(*types.Value_NullValue) - _, bNull := bv.GetKind().(*types.Value_NullValue) - if av == nil { - aNull = true - } - if bv == nil { - bNull = true - } + aNull := !av.Ok() + bNull := !bv.Ok() if aNull && bNull { comp = 0 } else if aNull { @@ -197,22 +192,22 @@ func (ko *KeyOrder) tryAdjustEmptyPositions(av *types.Value, bv *types.Value, co return comp } -func (ko *KeyOrder) tryCompareStrings(av *types.Value, bv *types.Value) int { +func (ko *KeyOrder) tryCompareStrings(av domain.Value, bv domain.Value) int { comp := 0 - _, aString := av.GetKind().(*types.Value_StringValue) - _, bString := bv.GetKind().(*types.Value_StringValue) + aStringVal, aString := av.String() + bStringVal, bString := bv.String() if ko.isSpecialSortOfEmptyValuesNeed(av, bv, aString, bString) { - if av.GetStringValue() == "" && bv.GetStringValue() != "" { + if aStringVal == "" && bStringVal != "" { comp = 1 - } else if av.GetStringValue() != "" && bv.GetStringValue() == "" { + } else if aStringVal != "" && bStringVal == "" { comp = -1 } } if aString && bString && comp == 0 { ko.ensureCollator() - comp = ko.collator.CompareString(av.GetStringValue(), bv.GetStringValue()) + comp = ko.collator.CompareString(aStringVal, bStringVal) } - if av.GetStringValue() == "" || bv.GetStringValue() == "" { + if aStringVal == "" || bStringVal == "" { comp = ko.tryFlipComp(comp) } return comp @@ -226,12 +221,12 @@ func (ko *KeyOrder) tryFlipComp(comp int) int { return comp } -func (ko *KeyOrder) isSpecialSortOfEmptyValuesNeed(av *types.Value, bv *types.Value, aString bool, bString bool) bool { +func (ko *KeyOrder) isSpecialSortOfEmptyValuesNeed(av domain.Value, bv domain.Value, aString bool, bString bool) bool { return (ko.EmptyPlacement != model.BlockContentDataviewSort_NotSpecified) && - (aString || av == nil) && (bString || bv == nil) + (aString || !av.Ok()) && (bString || !bv.Ok()) } -func (ko *KeyOrder) tryExtractTag(av *types.Value, bv *types.Value) (*types.Value, *types.Value) { +func (ko *KeyOrder) tryExtractTag(av domain.Value, bv domain.Value) (domain.Value, domain.Value) { if ko.RelationFormat == model.RelationFormat_tag || ko.RelationFormat == model.RelationFormat_status { av = ko.GetOptionValue(av) bv = ko.GetOptionValue(bv) @@ -239,50 +234,54 @@ func (ko *KeyOrder) tryExtractTag(av *types.Value, bv *types.Value) (*types.Valu return av, bv } -func (ko *KeyOrder) tryExtractDateTime(av *types.Value, bv *types.Value) (*types.Value, *types.Value) { +func (ko *KeyOrder) tryExtractDateTime(av domain.Value, bv domain.Value) (domain.Value, domain.Value) { if ko.RelationFormat == model.RelationFormat_date && !ko.IncludeTime { - av = time_util.CutValueToDay(av) - bv = time_util.CutValueToDay(bv) + if v, ok := av.Float(); ok { + av = domain.SomeValue(time_util.CutToDay(time.Unix(int64(v), 0))) + } + if v, ok := bv.Float(); ok { + bv = domain.SomeValue(time_util.CutToDay(time.Unix(int64(v), 0))) + } } return av, bv } -func (ko *KeyOrder) tryExtractSnippet(a *types.Struct, b *types.Struct, av *types.Value, bv *types.Value) (*types.Value, *types.Value) { +func (ko *KeyOrder) tryExtractSnippet(a *domain.Details, b *domain.Details, av domain.Value, bv domain.Value) (domain.Value, domain.Value) { av = ko.trySubstituteSnippet(a, av) bv = ko.trySubstituteSnippet(b, bv) return av, bv } -func (ko *KeyOrder) trySubstituteSnippet(getter *types.Struct, value *types.Value) *types.Value { +func (ko *KeyOrder) trySubstituteSnippet(getter *domain.Details, value domain.Value) domain.Value { if ko.Key == bundle.RelationKeyName.String() && getLayout(getter) == model.ObjectType_note { - value = pbtypes.Get(getter, bundle.RelationKeyName.String()) - if value == nil { - value = pbtypes.Get(getter, bundle.RelationKeySnippet.String()) + _, ok := getter.GetString(bundle.RelationKeyName) + if !ok { + return getter.Get(bundle.RelationKeySnippet) } } return value } -func getLayout(getter *types.Struct) model.ObjectTypeLayout { - rawLayout := pbtypes.Get(getter, bundle.RelationKeyLayout.String()).GetNumberValue() +func getLayout(getter *domain.Details) model.ObjectTypeLayout { + rawLayout := getter.GetInt64OrDefault(bundle.RelationKeyLayout, 0) return model.ObjectTypeLayout(int32(rawLayout)) } -func (ko *KeyOrder) GetOptionValue(value *types.Value) *types.Value { +func (ko *KeyOrder) GetOptionValue(value domain.Value) domain.Value { if ko.Options == nil { ko.Options = make(map[string]string) } if len(ko.Options) == 0 && ko.Store != nil { - ko.Options = optionsToMap(ko.SpaceID, ko.Key, ko.Store) + ko.Options = optionsToMap(ko.SpaceID, domain.RelationKey(ko.Key), ko.Store) } res := "" - for _, optID := range pbtypes.GetStringListValue(value) { + for _, optID := range value.StringListOrDefault(nil) { res += ko.Options[optID] } - return pbtypes.String(res) + return domain.SomeValue(res) } func newCustomOrder(arena *fastjson.Arena, key string, idsIndices map[string]int, keyOrd *KeyOrder) customOrder { @@ -330,9 +329,9 @@ func (co customOrder) AnystoreSort() query.Sort { return co } -func (co customOrder) Compare(a, b *types.Struct) int { - aID, okA := co.NeedOrderMap[pbtypes.Get(a, co.Key).GetStringValue()] - bID, okB := co.NeedOrderMap[pbtypes.Get(b, co.Key).GetStringValue()] +func (co customOrder) Compare(a, b *domain.Details) int { + aID, okA := co.NeedOrderMap[a.GetStringOrDefault(domain.RelationKey(co.Key), "")] + bID, okB := co.NeedOrderMap[b.GetStringOrDefault(domain.RelationKey(co.Key), "")] if okA && okB { if aID == bID { diff --git a/pkg/lib/localstore/objectstore/queries.go b/pkg/lib/localstore/objectstore/queries.go index 9a9ee88fa..4d7aac691 100644 --- a/pkg/lib/localstore/objectstore/queries.go +++ b/pkg/lib/localstore/objectstore/queries.go @@ -34,10 +34,9 @@ func (s *dsObjectStore) Query(q database.Query) ([]database.Record, error) { } // getObjectsWithObjectInRelation returns objects that have a relation with the given object in the value, while also matching the given filters -func (s *dsObjectStore) getObjectsWithObjectInRelation(relationKey, objectId string, limit int, params database.Filters) ([]database.Record, error) { - listValue := pbtypes.StringList([]string{objectId}) +func (s *dsObjectStore) getObjectsWithObjectInRelation(relationKey domain.RelationKey, objectId string, limit int, params database.Filters) ([]database.Record, error) { f := database.FiltersAnd{ - database.FilterAllIn{Key: relationKey, Value: listValue.GetListValue()}, + database.FilterAllIn{Key: relationKey, Strings: []string{objectId}}, params.FilterObj, } return s.queryAnyStore(f, params.Order, uint(limit), 0) @@ -70,7 +69,7 @@ func (s *dsObjectStore) getInjectedResults(details *domain.Details, score float6 default: return nil } - recs, err := s.getObjectsWithObjectInRelation(relationKey, id, maxLength, params) + recs, err := s.getObjectsWithObjectInRelation(domain.RelationKey(relationKey), id, maxLength, params) if err != nil { log.Errorf("getInjectedResults failed to get objects with object in relation: %v", err) return nil @@ -187,7 +186,7 @@ func (s *dsObjectStore) QueryFromFulltext(results []database.FulltextResult, par details.Set(bundle.RelationKeyId, res.Path.ObjectId) details.Set(database.RecordScoreField, res.Score) rec := database.Record{Details: details} - if params.FilterObj == nil || params.FilterObj.FilterObject(rec.Details.ToProto()) { + if params.FilterObj == nil || params.FilterObj.FilterObject(rec.Details) { resultObjectMap[res.Path.ObjectId] = struct{}{} records = append(records, rec) } @@ -207,7 +206,7 @@ func (s *dsObjectStore) QueryFromFulltext(results []database.FulltextResult, par details.Set(database.RecordScoreField, res.Score) rec := database.Record{Details: details} - if params.FilterObj == nil || params.FilterObj.FilterObject(rec.Details.ToProto()) { + if params.FilterObj == nil || params.FilterObj.FilterObject(rec.Details) { rec.Meta = res.Model() if rec.Meta.Highlight == "" { title := details.GetStringOrDefault(bundle.RelationKeyName, "") @@ -251,8 +250,7 @@ func (s *dsObjectStore) QueryFromFulltext(results []database.FulltextResult, par } if params.Order != nil { sort.Slice(records, func(i, j int) bool { - // TODO TEMP ToProto - return params.Order.Compare(records[i].Details.ToProto(), records[j].Details.ToProto()) == -1 + return params.Order.Compare(records[i].Details, records[j].Details) == -1 }) } if limit > 0 { @@ -425,7 +423,7 @@ func getSpaceIdsFromFilter(fltr database.Filter) []string { switch f := fltr.(type) { case database.FilterEq: if f.Key == bundle.RelationKeySpaceId.String() { - return []string{f.Value.GetStringValue()} + return []string{domain.SomeValue(f.Value).StringOrDefault("")} } case database.FilterIn: if f.Key == bundle.RelationKeySpaceId.String() { diff --git a/space/internal/components/migration/systemobjectreviser/systemobjectreviser.go b/space/internal/components/migration/systemobjectreviser/systemobjectreviser.go index 88a33107d..ebf159f7d 100644 --- a/space/internal/components/migration/systemobjectreviser/systemobjectreviser.go +++ b/space/internal/components/migration/systemobjectreviser/systemobjectreviser.go @@ -132,12 +132,12 @@ func isSystemObject(details *domain.Details) bool { func buildDiffDetails(origin, current *domain.Details) *domain.Details { diff := domain.StructDiff(current, origin) - diff = diff.CopyOnlyWithKeys([]domain.RelationKey{ + diff = diff.CopyOnlyWithKeys( bundle.RelationKeyName, bundle.RelationKeyDescription, bundle.RelationKeyIsReadonly, bundle.RelationKeyIsHidden, bundle.RelationKeyRevision, bundle.RelationKeyRelationReadonlyValue, bundle.RelationKeyRelationMaxCount, bundle.RelationKeyTargetObjectType, - }) + ) details := domain.NewDetails() diff.Iterate(func(key domain.RelationKey, value any) bool { diff --git a/util/internalflag/flag.go b/util/internalflag/flag.go index 4c407d5f5..a2e0e46cb 100644 --- a/util/internalflag/flag.go +++ b/util/internalflag/flag.go @@ -6,6 +6,7 @@ import ( "github.com/anyproto/anytype-heart/pkg/lib/bundle" "github.com/anyproto/anytype-heart/pkg/lib/pb/model" "github.com/anyproto/anytype-heart/util/pbtypes" + "github.com/anyproto/anytype-heart/util/slice" ) const relationKey = bundle.RelationKeyInternalFlags @@ -18,7 +19,7 @@ func NewFromState(st *state.State) *Set { flags := st.Details().GetFloatListOrDefault(relationKey, nil) return &Set{ - flags: flags, + flags: slice.FloatsInto[int](flags), } } diff --git a/util/slice/slice.go b/util/slice/slice.go index 10e144a66..b5e6a3a20 100644 --- a/util/slice/slice.go +++ b/util/slice/slice.go @@ -253,3 +253,15 @@ func IntoStrings[T ~string](from []T) []string { } return to } + +type numeric interface { + constraints.Integer | constraints.Float +} + +func FloatsInto[T numeric](from []float64) []T { + to := make([]T, len(from)) + for i, v := range from { + to[i] = T(v) + } + return to +} diff --git a/util/time/time.go b/util/time/time.go index 6d46b4ed5..f70042f52 100644 --- a/util/time/time.go +++ b/util/time/time.go @@ -2,20 +2,8 @@ package time import ( "time" - - "github.com/gogo/protobuf/types" - - "github.com/anyproto/anytype-heart/util/pbtypes" ) -func CutValueToDay(val *types.Value) *types.Value { - if val == nil { - return val - } - t := time.Unix(int64(val.GetNumberValue()), 0) - return pbtypes.Int64(CutToDay(t).Unix()) -} - func CutToDay(t time.Time) time.Time { roundTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) return roundTime