mirror of
https://github.com/anyproto/any-sync.git
synced 2025-06-11 10:18:08 +09:00
diff container and mod tidy
This commit is contained in:
Sergey Cherepanov
Mikhail Iudin
parent
84d1f0df06
commit
b30038011f
4 changed files with 419 additions and 8 deletions
11
go.mod
11
go.mod
|
|
@ -4,22 +4,26 @@ go 1.18
|
|||
|
||||
require (
|
||||
github.com/awalterschulze/gographviz v0.0.0-20190522210029-fa59802746ab
|
||||
github.com/cheggaaa/mb v1.0.3
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
github.com/goccy/go-graphviz v0.0.9
|
||||
github.com/gogo/protobuf v1.3.2
|
||||
github.com/huandu/skiplist v1.2.0
|
||||
github.com/ipfs/go-cid v0.1.0
|
||||
github.com/libp2p/go-libp2p v0.20.3
|
||||
github.com/libp2p/go-libp2p-core v0.16.1
|
||||
github.com/mr-tron/base58 v1.2.0
|
||||
github.com/multiformats/go-multibase v0.0.3
|
||||
github.com/multiformats/go-multihash v0.1.0
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/stretchr/testify v1.8.0
|
||||
github.com/zeebo/blake3 v0.2.3
|
||||
go.uber.org/zap v1.21.0
|
||||
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
storj.io/drpc v0.0.32
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/OneOfOne/xxhash v1.2.8 // indirect
|
||||
github.com/btcsuite/btcd v0.22.1 // indirect
|
||||
github.com/btcsuite/btcd/btcec/v2 v2.1.3 // indirect
|
||||
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
|
||||
|
|
@ -27,7 +31,7 @@ require (
|
|||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
|
||||
github.com/fogleman/gg v1.3.0 // indirect
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.0.12 // indirect
|
||||
github.com/libp2p/go-buffer-pool v0.0.2 // indirect
|
||||
github.com/libp2p/go-openssl v0.0.7 // indirect
|
||||
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
|
||||
|
|
@ -48,7 +52,6 @@ require (
|
|||
golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
|
||||
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
|
||||
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
lukechampine.com/blake3 v1.1.6 // indirect
|
||||
)
|
||||
|
|
|
|||
27
go.sum
27
go.sum
|
|
@ -1,3 +1,6 @@
|
|||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
|
||||
github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
|
||||
github.com/awalterschulze/gographviz v0.0.0-20190522210029-fa59802746ab h1:+cdNqtOJWjvepyhxy23G7z7vmpYCoC65AP0nqi1f53s=
|
||||
github.com/awalterschulze/gographviz v0.0.0-20190522210029-fa59802746ab/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs=
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
|
|
@ -8,8 +11,8 @@ github.com/btcsuite/btcd/btcec/v2 v2.1.3 h1:xM/n3yIhHAhHy04z4i43C8p4ehixJZMsnrVJ
|
|||
github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE=
|
||||
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
|
||||
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
|
||||
github.com/cheggaaa/mb v1.0.3 h1:03ksWum+6kHclB+kjwKMaBtgl5gtNYUwNpxsHQciKe8=
|
||||
github.com/cheggaaa/mb v1.0.3/go.mod h1:NUl0GBtFLlfg2o6iZwxzcG7Lslc2wV/ADTFbLXtVPE4=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4rUs6kA=
|
||||
github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
|
@ -27,6 +30,10 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
|||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
|
||||
github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c=
|
||||
github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U=
|
||||
github.com/huandu/skiplist v1.2.0 h1:gox56QD77HzSC0w+Ws3MH3iie755GBJU1OER3h5VsYw=
|
||||
github.com/huandu/skiplist v1.2.0/go.mod h1:7v3iFjLcSAzO4fN5B8dvebvo/qsfumiLiDXMrPiHF9w=
|
||||
github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
|
||||
github.com/ipfs/go-cid v0.1.0 h1:YN33LQulcRHjfom/i25yoOZR4Telp1Hr/2RU3d0PnC0=
|
||||
github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o=
|
||||
|
|
@ -34,8 +41,9 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS
|
|||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
|
|
@ -85,18 +93,28 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
|
|||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU=
|
||||
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=
|
||||
github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ=
|
||||
github.com/zeebo/errs v1.2.2 h1:5NFypMTuSdoySVTqlNs1dEoU21QVamMQJxW/Fii5O7g=
|
||||
github.com/zeebo/errs v1.2.2/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
|
||||
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
|
||||
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
|
||||
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
|
|
@ -164,6 +182,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR
|
|||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
|
||||
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
|
|
|
|||
288
util/ldiff/diff.go
Normal file
288
util/ldiff/diff.go
Normal file
|
|
@ -0,0 +1,288 @@
|
|||
// Package ldiff provides a container of elements with fixed id and changeable content.
|
||||
// Diff can calculate the difference with another diff container (you can make it remote) with minimum hops and traffic.
|
||||
package ldiff
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/cespare/xxhash"
|
||||
"github.com/huandu/skiplist"
|
||||
"github.com/zeebo/blake3"
|
||||
"math"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// New creates new Diff container
|
||||
//
|
||||
// divideFactor - means how many hashes you want to ask for once
|
||||
//
|
||||
// it must be 2 or greater
|
||||
// normal value usually between 4 and 64
|
||||
//
|
||||
// compareThreshold - means the maximum count of elements remote diff will send directly
|
||||
//
|
||||
// if elements under range will be more - remote diff will send only hash
|
||||
// it must be 1 or greater
|
||||
// normal value between 8 and 64
|
||||
//
|
||||
// Less threshold and divideFactor - less traffic but more requests
|
||||
func New(divideFactor, compareThreshold int) Diff {
|
||||
if divideFactor < 2 {
|
||||
divideFactor = 2
|
||||
}
|
||||
if compareThreshold < 1 {
|
||||
compareThreshold = 1
|
||||
}
|
||||
d := &diff{
|
||||
divideFactor: divideFactor,
|
||||
compareThreshold: compareThreshold,
|
||||
}
|
||||
d.sl = skiplist.New(d)
|
||||
return d
|
||||
}
|
||||
|
||||
var hashersPool = &sync.Pool{
|
||||
New: func() any {
|
||||
return blake3.New()
|
||||
},
|
||||
}
|
||||
|
||||
var ErrElementNotFound = errors.New("element not found")
|
||||
|
||||
// Element of data
|
||||
type Element struct {
|
||||
Id string
|
||||
Head string
|
||||
}
|
||||
|
||||
// Range request to get RangeResult
|
||||
type Range struct {
|
||||
From, To uint64
|
||||
Limit int
|
||||
}
|
||||
|
||||
// RangeResult response for Range
|
||||
type RangeResult struct {
|
||||
Hash []byte
|
||||
Elements []Element
|
||||
Count int
|
||||
}
|
||||
|
||||
type element struct {
|
||||
Element
|
||||
hash uint64
|
||||
}
|
||||
|
||||
// Diff contains elements and can compare it with Remote diff
|
||||
type Diff interface {
|
||||
Remote
|
||||
// Set adds or update element in container
|
||||
Set(e Element)
|
||||
// RemoveId removes element by id
|
||||
RemoveId(id string) error
|
||||
// Diff makes diff with remote container
|
||||
Diff(ctx context.Context, dl Remote) (newIds, changedIds, removedIds []string, err error)
|
||||
}
|
||||
|
||||
// Remote interface for using in the Diff
|
||||
type Remote interface {
|
||||
// Ranges calculates given ranges and return results
|
||||
Ranges(ctx context.Context, ranges []Range, resBuf []RangeResult) (results []RangeResult, err error)
|
||||
}
|
||||
|
||||
type diff struct {
|
||||
sl *skiplist.SkipList
|
||||
divideFactor int
|
||||
compareThreshold int
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// Compare implements skiplist interface
|
||||
func (d *diff) Compare(lhs, rhs interface{}) int {
|
||||
lhe := lhs.(*element)
|
||||
rhe := rhs.(*element)
|
||||
if lhe.Id == rhe.Id {
|
||||
return 0
|
||||
}
|
||||
if lhe.hash > rhe.hash {
|
||||
return 1
|
||||
} else if lhe.hash < rhe.hash {
|
||||
return -1
|
||||
}
|
||||
if lhe.Id > rhe.Id {
|
||||
return 1
|
||||
} else {
|
||||
return -1
|
||||
}
|
||||
}
|
||||
|
||||
// CalcScore implements skiplist interface
|
||||
func (d *diff) CalcScore(key interface{}) float64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Set adds or update element in container
|
||||
func (d *diff) Set(e Element) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
el := &element{Element: e, hash: xxhash.Sum64([]byte(e.Id))}
|
||||
d.sl.Remove(el)
|
||||
d.sl.Set(el, nil)
|
||||
}
|
||||
|
||||
// RemoveId removes element by id
|
||||
func (d *diff) RemoveId(id string) error {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
el := &element{Element: Element{
|
||||
Id: id,
|
||||
}, hash: xxhash.Sum64([]byte(id))}
|
||||
if d.sl.Remove(el) == nil {
|
||||
return ErrElementNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *diff) getRange(r Range) (rr RangeResult) {
|
||||
hasher := hashersPool.Get().(*blake3.Hasher)
|
||||
defer hashersPool.Put(hasher)
|
||||
hasher.Reset()
|
||||
|
||||
el := d.sl.Find(&element{hash: r.From})
|
||||
rr.Elements = make([]Element, 0, r.Limit)
|
||||
var overfill bool
|
||||
for el != nil && el.Key().(*element).hash <= r.To {
|
||||
elem := el.Key().(*element).Element
|
||||
el = el.Next()
|
||||
|
||||
hasher.WriteString(elem.Id)
|
||||
hasher.WriteString(elem.Head)
|
||||
rr.Count++
|
||||
if !overfill {
|
||||
if len(rr.Elements) < r.Limit {
|
||||
rr.Elements = append(rr.Elements, elem)
|
||||
}
|
||||
if len(rr.Elements) == r.Limit && el != nil {
|
||||
overfill = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if overfill {
|
||||
rr.Elements = nil
|
||||
}
|
||||
rr.Hash = hasher.Sum(nil)
|
||||
return
|
||||
}
|
||||
|
||||
// Ranges calculates given ranges and return results
|
||||
func (d *diff) Ranges(ctx context.Context, ranges []Range, resBuf []RangeResult) (results []RangeResult, err error) {
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
results = resBuf[:0]
|
||||
for _, r := range ranges {
|
||||
results = append(results, d.getRange(r))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type diffCtx struct {
|
||||
newIds, changedIds, removedIds []string
|
||||
|
||||
toSend, prepare []Range
|
||||
myRes, otherRes []RangeResult
|
||||
}
|
||||
|
||||
var errMismatched = errors.New("query and results mismatched")
|
||||
|
||||
// Diff makes diff with remote container
|
||||
func (d *diff) Diff(ctx context.Context, dl Remote) (newIds, changedIds, removedIds []string, err error) {
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
|
||||
dctx := &diffCtx{}
|
||||
dctx.toSend = append(dctx.toSend, Range{
|
||||
From: 0,
|
||||
To: math.MaxUint64,
|
||||
Limit: d.compareThreshold,
|
||||
})
|
||||
for len(dctx.toSend) > 0 {
|
||||
fmt.Println("fill ranges:", len(dctx.toSend))
|
||||
if dctx.myRes, err = d.Ranges(ctx, dctx.toSend, dctx.myRes); err != nil {
|
||||
return
|
||||
}
|
||||
if dctx.otherRes, err = dl.Ranges(ctx, dctx.toSend, dctx.otherRes); err != nil {
|
||||
return
|
||||
}
|
||||
if len(dctx.otherRes) != len(dctx.toSend) || len(dctx.myRes) != len(dctx.toSend) {
|
||||
err = errMismatched
|
||||
return
|
||||
}
|
||||
for i, r := range dctx.toSend {
|
||||
d.compareResults(dctx, r, dctx.myRes[i], dctx.otherRes[i])
|
||||
}
|
||||
dctx.toSend, dctx.prepare = dctx.prepare, dctx.toSend
|
||||
dctx.prepare = dctx.prepare[:0]
|
||||
}
|
||||
return dctx.newIds, dctx.changedIds, dctx.removedIds, nil
|
||||
}
|
||||
|
||||
func (d *diff) compareResults(dctx *diffCtx, r Range, myRes, otherRes RangeResult) {
|
||||
// both hash equals - do nothing
|
||||
if bytes.Equal(myRes.Hash, otherRes.Hash) {
|
||||
return
|
||||
}
|
||||
|
||||
// both has elements
|
||||
if len(myRes.Elements) == myRes.Count && len(otherRes.Elements) == otherRes.Count {
|
||||
d.compareElements(dctx, myRes.Elements, otherRes.Elements)
|
||||
return
|
||||
}
|
||||
|
||||
// make more queries
|
||||
divideFactor := uint64(d.divideFactor)
|
||||
perRange := (r.To - r.From) / divideFactor
|
||||
align := ((r.To-r.From)%divideFactor + 1) % divideFactor
|
||||
if align == 0 {
|
||||
perRange += 1
|
||||
}
|
||||
var j = r.From
|
||||
for i := 0; i < d.divideFactor; i++ {
|
||||
if i == d.divideFactor-1 {
|
||||
perRange += align
|
||||
}
|
||||
dctx.prepare = append(dctx.prepare, Range{From: j, To: j + perRange - 1, Limit: r.Limit})
|
||||
j += perRange
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (d *diff) compareElements(dctx *diffCtx, my, other []Element) {
|
||||
find := func(list []Element, targetEl Element) (has, eq bool) {
|
||||
for _, el := range list {
|
||||
if el.Id == targetEl.Id {
|
||||
return true, el.Head == targetEl.Head
|
||||
}
|
||||
}
|
||||
return false, false
|
||||
}
|
||||
|
||||
for _, el := range my {
|
||||
has, eq := find(other, el)
|
||||
if !has {
|
||||
dctx.removedIds = append(dctx.removedIds, el.Id)
|
||||
continue
|
||||
} else {
|
||||
if !eq {
|
||||
dctx.changedIds = append(dctx.changedIds, el.Id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, el := range other {
|
||||
if has, _ := find(my, el); !has {
|
||||
dctx.newIds = append(dctx.newIds, el.Id)
|
||||
}
|
||||
}
|
||||
}
|
||||
101
util/ldiff/diff_test.go
Normal file
101
util/ldiff/diff_test.go
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
package ldiff
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gopkg.in/mgo.v2/bson"
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDiff_fillRange(t *testing.T) {
|
||||
d := New(4, 4).(*diff)
|
||||
for i := 0; i < 10; i++ {
|
||||
el := Element{
|
||||
Id: fmt.Sprint(i),
|
||||
Head: fmt.Sprint("h", i),
|
||||
}
|
||||
d.Set(el)
|
||||
}
|
||||
t.Log(d.sl.Len())
|
||||
|
||||
t.Run("elements", func(t *testing.T) {
|
||||
r := Range{From: 0, To: math.MaxUint64, Limit: 10}
|
||||
res := d.getRange(r)
|
||||
assert.NotNil(t, res.Hash)
|
||||
assert.Len(t, res.Elements, 10)
|
||||
})
|
||||
t.Run("hash", func(t *testing.T) {
|
||||
r := Range{From: 0, To: math.MaxUint64, Limit: 9}
|
||||
res := d.getRange(r)
|
||||
t.Log(len(res.Elements))
|
||||
assert.NotNil(t, res.Hash)
|
||||
assert.Nil(t, res.Elements)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDiff_Diff(t *testing.T) {
|
||||
t.Run("basic", func(t *testing.T) {
|
||||
d1 := New(16, 16)
|
||||
d2 := New(16, 16)
|
||||
for i := 0; i < 1000; i++ {
|
||||
id := fmt.Sprint(i)
|
||||
head := bson.NewObjectId().Hex()
|
||||
d1.Set(Element{
|
||||
Id: id,
|
||||
Head: head,
|
||||
})
|
||||
d2.Set(Element{
|
||||
Id: id,
|
||||
Head: head,
|
||||
})
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
newIds, changedIds, removedIds, err := d1.Diff(ctx, d2)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, newIds, 0)
|
||||
assert.Len(t, changedIds, 0)
|
||||
assert.Len(t, removedIds, 0)
|
||||
|
||||
d2.Set(Element{
|
||||
Id: "newD1",
|
||||
Head: "newD1",
|
||||
})
|
||||
d2.Set(Element{
|
||||
Id: "1",
|
||||
Head: "changed",
|
||||
})
|
||||
require.NoError(t, d2.RemoveId("0"))
|
||||
|
||||
newIds, changedIds, removedIds, err = d1.Diff(ctx, d2)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, newIds, 1)
|
||||
assert.Len(t, changedIds, 1)
|
||||
assert.Len(t, removedIds, 1)
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkDiff_Ranges(b *testing.B) {
|
||||
d := New(16, 16)
|
||||
for i := 0; i < 10000; i++ {
|
||||
id := fmt.Sprint(i)
|
||||
head := bson.NewObjectId().Hex()
|
||||
d.Set(Element{
|
||||
Id: id,
|
||||
Head: head,
|
||||
})
|
||||
}
|
||||
ctx := context.Background()
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
var resBuf []RangeResult
|
||||
var ranges = []Range{{From: 0, To: math.MaxUint64, Limit: 10}}
|
||||
for i := 0; i < b.N; i++ {
|
||||
d.Ranges(ctx, ranges, resBuf)
|
||||
resBuf = resBuf[:0]
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue