Merge pull request #1532 from anyproto/go-3853-fix-creationdate-for-get-started

GO-3853 Fix creationDate for get_started

README.md

@@ -9,6 +9,7 @@ Middleware library for Anytype, distributed as part of the Anytype clients.
 - [Project architecture](docs/Architecture.md)
 - [Style guide](docs/Codestyle.md)
 - [Project workflows](docs/Flow.md)
+- [Complex filters](docs/ComplexFilters.md)
 
 ### CLI tools
 - [Usecase validator](docs/UsecaseValidator.md)

docs/ComplexFilters.md

@@ -0,0 +1,62 @@
+## Complex search filters
+
+The majority of search filters for ObjectSearch-requests have simple format `key-condition-value`:
+```json
+{
+     "RelationKey": "type",
+     "condition": 2, // equals
+     "value": {
+        "stringValue": "ot-page"
+     }
+}
+```
+
+However, this format does not allow to build complex filters, that involve multiple relation keys analysis.
+So we introduce complex filters that uses struct-values to build up more sophisticated requests.
+
+**type** field defined in the struct stands for complex filters types.
+
+### Two relation values comparison
+
+**type** = **valueFromRelation** allows to filter out objects by comparing values of two different relations.
+First relation is set to root _RelationKey_ field, second - to _relationKey_ field of the struct value.
+
+For example, if we want to get all objects that were added to space before last modification, we can build up following filter:
+```json
+{
+  "RelationKey": "addedDate",
+  "condition": 4, // less
+  "value": {
+    "structValue" : {
+      "fields": {
+        "type": {
+          "stringValue": "valueFromRelation" 
+        },
+        "relationKey": {
+          "stringValue": "lastModifiedDate" 
+        }
+      }
+    }
+  }
+}
+```
+
+All objects that has similar name and description:
+```json
+{
+  "RelationKey": "name",
+  "condition": 1,
+  "value": {
+    "structValue": {
+      "fields": {
+        "type": {
+          "stringValue": "valueFromRelation"
+        },
+        "relationKey": {
+          "stringValue": "description"
+        }
+      }
+    }
+  }
+}
+```

pkg/lib/database/filter.go

@@ -1,6 +1,7 @@
 package database
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"regexp"
@@ -116,6 +117,15 @@ func makeFilterByCondition(spaceID string, rawFilter *model.BlockContentDataview
 			Value:            pbtypes.Bool(true),
 		}
 	}
+
+	if str := rawFilter.Value.GetStructValue(); str != nil {
+		filter, err := makeComplexFilter(rawFilter, str)
+		if err == nil {
+			return filter, nil
+		}
+		log.Errorf("failed to build complex filter: %v", err)
+	}
+
 	switch rawFilter.Condition {
 	case model.BlockContentDataviewFilter_Equal,
 		model.BlockContentDataviewFilter_Greater,
@@ -203,6 +213,19 @@ func makeFilterByCondition(spaceID string, rawFilter *model.BlockContentDataview
 	}
 }
 
+func makeComplexFilter(rawFilter *model.BlockContentDataviewFilter, s *types.Struct) (Filter, error) {
+	filterType := pbtypes.GetString(s, bundle.RelationKeyType.String())
+	// TODO: rewrite to switch statement once we have more filter types
+	if filterType == "valueFromRelation" {
+		return Filter2ValuesComp{
+			Key1: rawFilter.RelationKey,
+			Key2: pbtypes.GetString(s, bundle.RelationKeyRelationKey.String()),
+			Cond: rawFilter.Condition,
+		}, nil
+	}
+	return nil, fmt.Errorf("unsupported type of complex filter: %s", filterType)
+}
+
 type WithNestedFilter interface {
 	IterateNestedFilters(func(nestedFilter Filter) error) error
 }
@@ -845,3 +868,93 @@ func (i *FilterNestedNotIn) AnystoreFilter() query.Filter {
 func (i *FilterNestedNotIn) IterateNestedFilters(fn func(nestedFilter Filter) error) error {
 	return fn(i)
 }
+
+type Filter2ValuesComp struct {
+	Key1, Key2 string
+	Cond       model.BlockContentDataviewFilterCondition
+}
+
+func (i Filter2ValuesComp) FilterObject(g *types.Struct) bool {
+	val1 := pbtypes.Get(g, i.Key1)
+	val2 := pbtypes.Get(g, i.Key2)
+	eq := FilterEq{Value: val2, Cond: i.Cond}
+	return eq.filterObject(val1)
+}
+
+func (i Filter2ValuesComp) AnystoreFilter() query.Filter {
+	var op query.CompOp
+	switch i.Cond {
+	case model.BlockContentDataviewFilter_Equal:
+		op = query.CompOpEq
+	case model.BlockContentDataviewFilter_Greater:
+		op = query.CompOpGt
+	case model.BlockContentDataviewFilter_GreaterOrEqual:
+		op = query.CompOpGte
+	case model.BlockContentDataviewFilter_Less:
+		op = query.CompOpLt
+	case model.BlockContentDataviewFilter_LessOrEqual:
+		op = query.CompOpLte
+	case model.BlockContentDataviewFilter_NotEqual:
+		op = query.CompOpNe
+	}
+	return &Anystore2ValuesComp{
+		RelationKey1: i.Key1,
+		RelationKey2: i.Key2,
+		CompOp:       op,
+	}
+}
+
+type Anystore2ValuesComp struct {
+	RelationKey1, RelationKey2 string
+	CompOp                     query.CompOp
+	buf1, buf2                 []byte
+}
+
+func (e *Anystore2ValuesComp) Ok(v *fastjson.Value) bool {
+	value1 := v.Get(e.RelationKey1)
+	value2 := v.Get(e.RelationKey2)
+	e.buf1 = encoding.AppendJSONValue(e.buf1[:0], value1)
+	e.buf2 = encoding.AppendJSONValue(e.buf2[:0], value2)
+	comp := bytes.Compare(e.buf1, e.buf2)
+	switch e.CompOp {
+	case query.CompOpEq:
+		return comp == 0
+	case query.CompOpGt:
+		return comp > 0
+	case query.CompOpGte:
+		return comp >= 0
+	case query.CompOpLt:
+		return comp < 0
+	case query.CompOpLte:
+		return comp <= 0
+	case query.CompOpNe:
+		return comp != 0
+	default:
+		panic(fmt.Errorf("unexpected comp op: %v", e.CompOp))
+	}
+}
+
+func (e *Anystore2ValuesComp) IndexBounds(_ string, bs query.Bounds) (bounds query.Bounds) {
+	return bs
+}
+
+func (e *Anystore2ValuesComp) String() string {
+	var comp string
+	switch e.CompOp {
+	case query.CompOpEq:
+		comp = "$eq"
+	case query.CompOpGt:
+		comp = "$gt"
+	case query.CompOpGte:
+		comp = "$gte"
+	case query.CompOpLt:
+		comp = "$lt"
+	case query.CompOpLte:
+		comp = "$lte"
+	case query.CompOpNe:
+		comp = "$ne"
+	default:
+		panic(fmt.Errorf("unexpected comp op: %v", e.CompOp))
+	}
+	return fmt.Sprintf(`{"$comp_values": {"%s": ["%s", "%s"]}}`, comp, e.RelationKey1, e.RelationKey2)
+}

pkg/lib/database/filter_test.go

@@ -4,7 +4,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/anyproto/any-store/query"
 	"github.com/gogo/protobuf/types"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
@@ -19,11 +18,10 @@ import (
 func assertFilter(t *testing.T, f Filter, obj *types.Struct, expected bool) {
 	assert.Equal(t, expected, f.FilterObject(obj))
 	anystoreFilter := f.AnystoreFilter()
-	_, err := query.ParseCondition(anystoreFilter.String())
-	require.NoError(t, err, anystoreFilter.String())
 	arena := &fastjson.Arena{}
 	val := pbtypes.ProtoToJson(arena, obj)
-	assert.Equal(t, expected, anystoreFilter.Ok(val))
+	result := anystoreFilter.Ok(val)
+	assert.Equal(t, expected, result)
 }
 
 func TestEq_FilterObject(t *testing.T) {
@@ -886,3 +884,50 @@ func TestMakeFilters(t *testing.T) {
 		assert.NotNil(t, filters.(FiltersOr)[1].(FilterEq))
 	})
 }
+
+func TestFilter2ValuesComp_FilterObject(t *testing.T) {
+	t.Run("equal", func(t *testing.T) {
+		eq := Filter2ValuesComp{
+			Key1: "a",
+			Key2: "b",
+			Cond: model.BlockContentDataviewFilter_Equal,
+		}
+		obj1 := &types.Struct{Fields: map[string]*types.Value{
+			"a": pbtypes.String("x"),
+			"b": pbtypes.String("x"),
+		}}
+		obj2 := &types.Struct{Fields: map[string]*types.Value{
+			"a": pbtypes.String("x"),
+			"b": pbtypes.String("y"),
+		}}
+		obj3 := &types.Struct{Fields: map[string]*types.Value{
+			"b": pbtypes.String("x"),
+		}}
+		assertFilter(t, eq, obj1, true)
+		assertFilter(t, eq, obj2, false)
+		assertFilter(t, eq, obj3, false)
+	})
+
+	t.Run("greater", func(t *testing.T) {
+		eq := Filter2ValuesComp{
+			Key1: "a",
+			Key2: "b",
+			Cond: model.BlockContentDataviewFilter_Greater,
+		}
+		obj1 := &types.Struct{Fields: map[string]*types.Value{
+			"a": pbtypes.Int64(100),
+			"b": pbtypes.Int64(200),
+		}}
+		obj2 := &types.Struct{Fields: map[string]*types.Value{
+			"a": pbtypes.Int64(300),
+			"b": pbtypes.Int64(-500),
+		}}
+		obj3 := &types.Struct{Fields: map[string]*types.Value{
+			"a": pbtypes.String("xxx"),
+			"b": pbtypes.String("ddd"),
+		}}
+		assertFilter(t, eq, obj1, false)
+		assertFilter(t, eq, obj2, true)
+		assertFilter(t, eq, obj3, true)
+	})
+}