mirror of
https://github.com/anyproto/any-sync.git
synced 2025-06-10 18:10:54 +09:00
Merge branch 'GO-4709-change-differ' into GO-4146-new-spacestore
This commit is contained in:
mcrakhman
commit
e1468c6539
3 changed files with 306 additions and 3 deletions
208
commonspace/object/tree/objecttree/changediffer.go
Normal file
208
commonspace/object/tree/objecttree/changediffer.go
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
package objecttree
|
||||
|
||||
import (
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/anyproto/any-sync/util/slice"
|
||||
)
|
||||
|
||||
type (
|
||||
hasChangesFunc func(ids ...string) bool
|
||||
treeBuilderFunc func(heads []string) (ReadableObjectTree, error)
|
||||
onRemoveFunc func(ids []string)
|
||||
)
|
||||
|
||||
type ChangeDiffer struct {
|
||||
hasChanges hasChangesFunc
|
||||
attached map[string]*Change
|
||||
waitList map[string][]*Change
|
||||
visitedBuf []*Change
|
||||
}
|
||||
|
||||
func NewChangeDiffer(tree ReadableObjectTree, hasChanges hasChangesFunc) (*ChangeDiffer, error) {
|
||||
diff := &ChangeDiffer{
|
||||
hasChanges: hasChanges,
|
||||
attached: make(map[string]*Change),
|
||||
waitList: make(map[string][]*Change),
|
||||
}
|
||||
if tree == nil {
|
||||
return diff, nil
|
||||
}
|
||||
err := tree.IterateRoot(nil, func(c *Change) (isContinue bool) {
|
||||
diff.add(&Change{
|
||||
Id: c.Id,
|
||||
PreviousIds: c.PreviousIds,
|
||||
})
|
||||
return true
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return diff, nil
|
||||
}
|
||||
|
||||
func (d *ChangeDiffer) RemoveBefore(ids []string) (removed []string, notFound []string) {
|
||||
var attached []*Change
|
||||
for _, id := range ids {
|
||||
if ch, ok := d.attached[id]; ok {
|
||||
attached = append(attached, ch)
|
||||
continue
|
||||
}
|
||||
// check if we have it at the bottom
|
||||
if !d.hasChanges(id) {
|
||||
notFound = append(notFound, id)
|
||||
}
|
||||
}
|
||||
d.dfsPrev(attached, func(ch *Change) (isContinue bool) {
|
||||
removed = append(removed, ch.Id)
|
||||
return true
|
||||
}, func(changes []*Change) {
|
||||
for _, ch := range removed {
|
||||
delete(d.attached, ch)
|
||||
}
|
||||
for _, ch := range d.attached {
|
||||
ch.Previous = slice.DiscardFromSlice(ch.Previous, func(change *Change) bool {
|
||||
return change.visited
|
||||
})
|
||||
}
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func (d *ChangeDiffer) dfsPrev(stack []*Change, visit func(ch *Change) (isContinue bool), afterVisit func([]*Change)) {
|
||||
d.visitedBuf = d.visitedBuf[:0]
|
||||
defer func() {
|
||||
if afterVisit != nil {
|
||||
afterVisit(d.visitedBuf)
|
||||
}
|
||||
for _, ch := range d.visitedBuf {
|
||||
ch.visited = false
|
||||
}
|
||||
}()
|
||||
for len(stack) > 0 {
|
||||
ch := stack[len(stack)-1]
|
||||
stack = stack[:len(stack)-1]
|
||||
if ch.visited {
|
||||
continue
|
||||
}
|
||||
ch.visited = true
|
||||
d.visitedBuf = append(d.visitedBuf, ch)
|
||||
for _, prevCh := range ch.Previous {
|
||||
if !prevCh.visited {
|
||||
stack = append(stack, prevCh)
|
||||
}
|
||||
}
|
||||
if !visit(ch) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *ChangeDiffer) Add(changes ...*Change) {
|
||||
for _, ch := range changes {
|
||||
d.add(ch)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (d *ChangeDiffer) add(change *Change) {
|
||||
_, exists := d.attached[change.Id]
|
||||
if exists {
|
||||
return
|
||||
}
|
||||
d.attached[change.Id] = change
|
||||
wl, exists := d.waitList[change.Id]
|
||||
if exists {
|
||||
for _, ch := range wl {
|
||||
ch.Previous = append(ch.Previous, change)
|
||||
}
|
||||
delete(d.waitList, change.Id)
|
||||
}
|
||||
for _, id := range change.PreviousIds {
|
||||
prev, exists := d.attached[id]
|
||||
if exists {
|
||||
change.Previous = append(change.Previous, prev)
|
||||
continue
|
||||
}
|
||||
wl := d.waitList[id]
|
||||
wl = append(wl, change)
|
||||
d.waitList[id] = wl
|
||||
}
|
||||
}
|
||||
|
||||
type DiffManager struct {
|
||||
differ *ChangeDiffer
|
||||
readableTree ReadableObjectTree
|
||||
notFound map[string]struct{}
|
||||
onRemove func(ids []string)
|
||||
heads []string
|
||||
seenHeads []string
|
||||
}
|
||||
|
||||
func NewDiffManager(initHeads, curHeads []string, treeBuilder treeBuilderFunc, onRemove onRemoveFunc) (*DiffManager, error) {
|
||||
allHeads := make([]string, 0, len(initHeads)+len(curHeads))
|
||||
allHeads = append(allHeads, initHeads...)
|
||||
allHeads = append(allHeads, curHeads...)
|
||||
readableTree, err := treeBuilder(allHeads)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
differ, err := NewChangeDiffer(readableTree, readableTree.HasChanges)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &DiffManager{
|
||||
differ: differ,
|
||||
heads: curHeads,
|
||||
seenHeads: initHeads,
|
||||
onRemove: onRemove,
|
||||
readableTree: readableTree,
|
||||
notFound: make(map[string]struct{}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *DiffManager) Init() {
|
||||
removed, _ := d.differ.RemoveBefore(d.heads)
|
||||
d.onRemove(removed)
|
||||
}
|
||||
|
||||
func (d *DiffManager) Remove(ids []string) {
|
||||
removed, notFound := d.differ.RemoveBefore(ids)
|
||||
for _, id := range notFound {
|
||||
d.notFound[id] = struct{}{}
|
||||
}
|
||||
d.onRemove(removed)
|
||||
}
|
||||
|
||||
func (d *DiffManager) Add(change *Change) {
|
||||
d.differ.Add(change)
|
||||
}
|
||||
|
||||
func (d *DiffManager) Update(objTree ObjectTree) {
|
||||
var (
|
||||
toAdd = make([]*Change, 0, objTree.Len())
|
||||
toRemove []string
|
||||
)
|
||||
err := objTree.IterateRoot(nil, func(ch *Change) bool {
|
||||
if ch.IsNew {
|
||||
toAdd = append(toAdd, &Change{
|
||||
Id: ch.Id,
|
||||
PreviousIds: ch.PreviousIds,
|
||||
})
|
||||
}
|
||||
if _, ok := d.notFound[ch.Id]; ok {
|
||||
toRemove = append(toRemove, ch.Id)
|
||||
delete(d.notFound, ch.Id)
|
||||
}
|
||||
return true
|
||||
})
|
||||
if err != nil {
|
||||
log.Warn("error while iterating over object tree", zap.Error(err))
|
||||
return
|
||||
}
|
||||
d.differ.Add(toAdd...)
|
||||
d.heads = make([]string, 0, len(d.heads))
|
||||
d.heads = append(d.heads, objTree.Heads()...)
|
||||
removed, _ := d.differ.RemoveBefore(toRemove)
|
||||
d.onRemove(removed)
|
||||
}
|
||||
98
commonspace/object/tree/objecttree/changediffer_test.go
Normal file
98
commonspace/object/tree/objecttree/changediffer_test.go
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
package objecttree
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestChangeDiffer_Add(t *testing.T) {
|
||||
t.Run("remove all", func(t *testing.T) {
|
||||
changes := []*Change{
|
||||
newChange("0", ""),
|
||||
newChange("1", "0", "0"),
|
||||
newChange("2", "0", "0"),
|
||||
newChange("3", "0", "1", "2"),
|
||||
newChange("4", "0", "0"),
|
||||
newChange("5", "0", "4"),
|
||||
newChange("6", "0", "5"),
|
||||
newChange("7", "0", "3", "6"),
|
||||
}
|
||||
differ, _ := NewChangeDiffer(nil, func(ids ...string) bool {
|
||||
return false
|
||||
})
|
||||
differ.Add(changes...)
|
||||
res, notFound := differ.RemoveBefore([]string{"7"})
|
||||
require.Len(t, notFound, 0)
|
||||
require.Equal(t, len(changes), len(res))
|
||||
})
|
||||
t.Run("remove in two parts", func(t *testing.T) {
|
||||
changes := []*Change{
|
||||
newChange("0", ""),
|
||||
newChange("1", "0", "0"),
|
||||
newChange("2", "0", "0"),
|
||||
newChange("3", "0", "1", "2"),
|
||||
newChange("4", "0", "0"),
|
||||
newChange("5", "0", "4"),
|
||||
newChange("6", "0", "5"),
|
||||
newChange("7", "0", "3", "6"),
|
||||
}
|
||||
differ, _ := NewChangeDiffer(nil, func(ids ...string) bool {
|
||||
return false
|
||||
})
|
||||
differ.Add(changes...)
|
||||
res, notFound := differ.RemoveBefore([]string{"4"})
|
||||
require.Len(t, notFound, 0)
|
||||
require.Equal(t, 2, len(res))
|
||||
res, notFound = differ.RemoveBefore([]string{"7"})
|
||||
require.Len(t, notFound, 0)
|
||||
require.Equal(t, 6, len(res))
|
||||
})
|
||||
t.Run("add and remove", func(t *testing.T) {
|
||||
changes := []*Change{
|
||||
newChange("0", ""),
|
||||
newChange("1", "0", "0"),
|
||||
newChange("2", "0", "0"),
|
||||
newChange("3", "0", "1", "2"),
|
||||
}
|
||||
differ, _ := NewChangeDiffer(nil, func(ids ...string) bool {
|
||||
return false
|
||||
})
|
||||
differ.Add(changes...)
|
||||
res, notFound := differ.RemoveBefore([]string{"3"})
|
||||
require.Len(t, notFound, 0)
|
||||
require.Equal(t, len(changes), len(res))
|
||||
changes = []*Change{
|
||||
newChange("4", "0", "0"),
|
||||
newChange("5", "0", "4"),
|
||||
newChange("6", "0", "5"),
|
||||
newChange("7", "0", "3", "6"),
|
||||
}
|
||||
differ.Add(changes...)
|
||||
res, notFound = differ.RemoveBefore([]string{"7"})
|
||||
require.Len(t, notFound, 0)
|
||||
require.Equal(t, len(changes), len(res))
|
||||
})
|
||||
t.Run("remove not found", func(t *testing.T) {
|
||||
differ, _ := NewChangeDiffer(nil, func(ids ...string) bool {
|
||||
return false
|
||||
})
|
||||
_, notFound := differ.RemoveBefore([]string{"3", "4", "5"})
|
||||
require.Len(t, notFound, 3)
|
||||
})
|
||||
t.Run("exists in storage", func(t *testing.T) {
|
||||
storedIds := []string{"3", "4", "5"}
|
||||
differ, _ := NewChangeDiffer(nil, func(ids ...string) bool {
|
||||
for _, id := range ids {
|
||||
if !slices.Contains(storedIds, id) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
res, notFound := differ.RemoveBefore([]string{"3", "4", "5"})
|
||||
require.Len(t, res, 0)
|
||||
require.Len(t, notFound, 0)
|
||||
})
|
||||
}
|
||||
|
|
@ -205,9 +205,6 @@ func (t *Tree) add(c *Change) bool {
|
|||
if c == nil {
|
||||
return false
|
||||
}
|
||||
if _, exists := t.invalidChanges[c.Id]; exists {
|
||||
return false
|
||||
}
|
||||
|
||||
if t.root == nil { // first element
|
||||
t.root = c
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue