From 92ea74913510f78742768f0e4f52c1973203f279 Mon Sep 17 00:00:00 2001 From: requilence Date: Thu, 4 Mar 2021 17:42:17 +0300 Subject: [PATCH] add cli tool to manually reindex objects --- .gitignore | 1 + Makefile | 11 ++- cmd/cli/cli.go | 24 +++++ cmd/cli/migrate.go | 53 ++++++++++++ dist/.gitignore | 3 +- go.mod | 1 + go.sum | 20 ++--- pkg/lib/core/migration.go | 149 +++++++++++++++++--------------- pkg/lib/core/threads/derived.go | 2 +- 9 files changed, 176 insertions(+), 88 deletions(-) create mode 100644 cmd/cli/cli.go create mode 100644 cmd/cli/migrate.go diff --git a/.gitignore b/.gitignore index 2a0ac0d47..5d631a77f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ package-lock.json /vendor/ *_mock.go lib-server/service.pb.go +dist/cli diff --git a/Makefile b/Makefile index dce058aad..c50842d63 100644 --- a/Makefile +++ b/Makefile @@ -86,13 +86,13 @@ build-js-addon: build-ios: setup-go @echo 'Building library for iOS...' @$(eval FLAGS := $$(shell govvv -flags | sed 's/main/github.com\/anytypeio\/go-anytype-middleware\/core/g')) - @GOPRIVATE=github.com/anytypeio gomobile bind -tags nogrpcserver -ldflags "$(FLAGS)" -v -target=ios github.com/anytypeio/go-anytype-middleware/clientlibrary/service github.com/anytypeio/go-anytype-middleware/core + @GOPRIVATE=github.com/anytypeio gomobile bind -tags nogrpcserver -ldflags "$(FLAGS)" -v -target=ios -o Lib.framework github.com/anytypeio/go-anytype-middleware/clientlibrary/service github.com/anytypeio/go-anytype-middleware/core @mkdir -p dist/ios/ && mv Lib.framework dist/ios/ build-android: setup-go @echo 'Building library for Android...' @$(eval FLAGS := $$(shell govvv -flags | sed 's/main/github.com\/anytypeio\/go-anytype-middleware\/core/g')) - @GOPRIVATE=github.com/anytypeio gomobile bind -tags nogrpcserver -ldflags "$(FLAGS)" -v -target=android -o mobile.aar github.com/anytypeio/go-anytype-middleware/clientlibrary/service github.com/anytypeio/go-anytype-middleware/core + @GOPRIVATE=github.com/anytypeio gomobile bind -tags nogrpcserver -ldflags "$(FLAGS)" -v -target=android -o lib.aar github.com/anytypeio/go-anytype-middleware/clientlibrary/service github.com/anytypeio/go-anytype-middleware/core @mkdir -p dist/android/ && mv lib.aar dist/android/ setup-protoc-go: @@ -176,10 +176,15 @@ protos-java: @echo 'Generating protobuf packages (Java)...' @protoc -I ./ --java_out=./dist/android/pb pb/protos/*.proto pkg/lib/pb/model/protos/*.proto pkg/lib/pb/relation/protos/*.proto +build-cli: + @echo 'Building middleware cli...' + @$(eval FLAGS := $$(shell govvv -flags -pkg github.com/anytypeio/go-anytype-middleware/core)) + @go build -v -o dist/cli -ldflags "$(FLAGS)" ./cmd/cli + build-server: protos-server @echo 'Building middleware server...' @$(eval FLAGS := $$(shell govvv -flags -pkg github.com/anytypeio/go-anytype-middleware/core)) - @go build -i -v -o dist/server -ldflags "$(FLAGS)" ./cmd/grpcserver/grpc.go + @go build -v -o dist/server -ldflags "$(FLAGS)" ./cmd/grpcserver/grpc.go build-server-debug: protos-server @echo 'Building middleware server with debug symbols...' diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go new file mode 100644 index 000000000..ab615ea58 --- /dev/null +++ b/cmd/cli/cli.go @@ -0,0 +1,24 @@ +package main + +import ( + "fmt" + "github.com/spf13/cobra" +) + +var CliCmd = &cobra.Command{ + Use: "cli", + Short: "CLI utilities", + Long: `Various CLI utilities for cafe operations.`, +} + +func init() { + // subcommands + CliCmd.AddCommand(migrateCmd) + // local flags +} + +func main() { + if err := CliCmd.Execute(); err != nil { + fmt.Printf("failed to execute: %s", err.Error()) + } +} diff --git a/cmd/cli/migrate.go b/cmd/cli/migrate.go new file mode 100644 index 000000000..cb1dda624 --- /dev/null +++ b/cmd/cli/migrate.go @@ -0,0 +1,53 @@ +package main + +import ( + "github.com/anytypeio/go-anytype-middleware/core" + "github.com/anytypeio/go-anytype-middleware/core/event" + "github.com/anytypeio/go-anytype-middleware/pb" + core2 "github.com/anytypeio/go-anytype-middleware/pkg/lib/core" + "github.com/spf13/cobra" + "os" +) + +var migrateCmd = &cobra.Command{ + Use: "migrate", + Short: "Migrations commands", +} + +var ( + migrateRepoPath string + migrateAccount string +) + +var reindex = &cobra.Command{ + Use: "reindex", + Short: "Reindex all existing objects in the local repo", + Run: func(c *cobra.Command, args []string) { + var mw = core.New() + mw.EventSender = event.NewCallbackSender(func(event *pb.Event) { + // nothing to do + }) + + resp := mw.AccountSelect(&pb.RpcAccountSelectRequest{Id: migrateAccount, RootPath: migrateRepoPath}) + if resp.Error.Code != 0 { + c.PrintErrf("failed to open account repo: %s\n", resp.Error.Description) + return + } + + migrated, err := core2.ReindexAll(mw.Anytype.(*core2.Anytype)) + if err != nil { + c.PrintErrf("failed to run reindex migration: %s\n", resp.Error.Description) + } + c.Printf("reindexed %d objects\n", migrated) + c.Printf("Shutting down account...\n", migrated) + mw.Shutdown(&pb.RpcShutdownRequest{}) + }, +} + +func init() { + // subcommands + homeDir, _ := os.UserHomeDir() + migrateCmd.AddCommand(reindex) + migrateCmd.PersistentFlags().StringVarP(&migrateRepoPath, "repo", "r", homeDir+"/Library/Application Support/anytype2/dev/data", "path to dir with accounts folder") + migrateCmd.PersistentFlags().StringVarP(&migrateAccount, "account", "a", "", "id of account in the repo folder") +} diff --git a/dist/.gitignore b/dist/.gitignore index 1b82a079d..93c74da2d 100644 --- a/dist/.gitignore +++ b/dist/.gitignore @@ -2,4 +2,5 @@ server lib.a lib.h ios/Lib.framework -android/mobile.aar +android/lib.aar +cli \ No newline at end of file diff --git a/go.mod b/go.mod index 9aac96bbc..0befe6bdb 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,7 @@ require ( github.com/otiai10/opengraph v1.1.3 github.com/prometheus/client_golang v1.9.0 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd + github.com/spf13/cobra v0.0.5 github.com/stretchr/testify v1.7.0 github.com/textileio/go-threads v1.0.2-0.20210304072541-d0f91da84404 github.com/tyler-smith/go-bip39 v1.0.1-0.20190808214741-c55f737395bc diff --git a/go.sum b/go.sum index 8fcc99914..a26313268 100644 --- a/go.sum +++ b/go.sum @@ -375,8 +375,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.1.0 h1:+eIkrewn5q6b30y+g/BJINVVdi2xH7je5MPJ3ZPK3JA= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= @@ -439,8 +439,7 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.4 h1:0ecGp3skIrHWPNGPJDaBIghfA6Sp7Ruo2Io8eLKzWm0= -github.com/google/uuid v1.1.4/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= @@ -476,8 +475,7 @@ github.com/gxed/hashland/keccakpg v0.0.1 h1:wrk3uMNaMxbXiHibbPO4S0ymqJMm41WiudyF github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1 h1:SheiaIt0sda5K+8FLz952/1iWS9zrnKsEJaOJu4ZbSc= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= -github.com/h2non/filetype v1.1.0 h1:Or/gjocJrJRNK/Cri/TDEKFjAR+cfG6eK65NGYB6gBA= -github.com/h2non/filetype v1.1.0/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= +github.com/h2non/filetype v1.1.1 h1:xvOwnXKAckvtLWsN398qS9QhlxlnVXBjXBydK2/UFB4= github.com/h2non/filetype v1.1.1/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/consul/api v1.3.0 h1:HXNYlRkkM/t+Y/Yhxtwcy02dlYwIaoxzvxPnS+cqy78= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -726,8 +724,8 @@ github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0Lh github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0 h1:reN85Pxc5larApoH1keMBiu2GWtPqXQ1nc9gx+jOU+E= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -1653,9 +1651,9 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b h1:GgiSbuUyC0BlbUmHQBgFqu32eiRR/CEYdjOjOd4zE6Y= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1689,8 +1687,8 @@ golang.org/x/net v0.0.0-20200320220750-118fecf932d8/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1706,8 +1704,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1798,14 +1796,14 @@ golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa h1:5E4dL8+NgFOgjwbTKz+OOEGGhP+ectTmF842l6KjupQ= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= diff --git a/pkg/lib/core/migration.go b/pkg/lib/core/migration.go index 830e715c9..da28c835e 100644 --- a/pkg/lib/core/migration.go +++ b/pkg/lib/core/migration.go @@ -435,80 +435,85 @@ func removeBundleRelationsFromDs(a *Anytype, lastMigration bool) error { }) } +func ReindexAll(a *Anytype) (int, error) { + ids, err := a.localStore.Objects.ListIds() + if err != nil { + return 0, err + } + total := len(ids) + var migrated int + for _, id := range ids { + sbt, err := smartblock.SmartBlockTypeFromID(id) + if err != nil { + return 0, fmt.Errorf("migration reindexAll:failed to extract smartblock type: %w", err) + } + if sbt == smartblock.SmartBlockTypeArchive { + // remove archive we have accidentally indexed + err = a.localStore.Objects.DeleteObject(id) + if err != nil { + log.Errorf("migration reindexAll: failed to delete archive from index: %s", err.Error()) + } + total-- + continue + } + for _, idx := range a.localStore.Objects.Indexes() { + //if idx.Name == "objtype_relkey_setid" { + // skip it because we can't reindex relations in sets for now + // continue + //} + + err = localstore.EraseIndex(idx, a.t.Datastore().(ds.TxnDatastore)) + if err != nil { + log.Errorf("migration reindexAll: failed to delete archive from index: %s", err.Error()) + } + } + oi, err := a.localStore.Objects.GetByIDs(id) + if err != nil { + log.Errorf("migration reindexAll: failed to get objects by id: %s", err.Error()) + continue + } + + if len(oi) < 1 { + log.Errorf("migration reindexAll: failed to get objects: not found") + continue + } + o := oi[0] + var objType string + if pbtypes.HasField(o.Details, bundle.RelationKeyType.String()) { + objTypes := pbtypes.GetStringList(o.Details, bundle.RelationKeyType.String()) + if len(objTypes) > 0 { + objType = objTypes[0] + } + } + + if strings.HasPrefix(objType, localstore.OldCustomObjectTypeURLPrefix) { + objType = strings.TrimPrefix(objType, localstore.OldCustomObjectTypeURLPrefix) + } else if strings.HasPrefix(objType, localstore.OldBundledObjectTypeURLPrefix) { + objType = localstore.BundledObjectTypeURLPrefix + strings.TrimPrefix(objType, localstore.OldBundledObjectTypeURLPrefix) + } else if bundle.HasObjectType(objType) { + objType = localstore.BundledObjectTypeURLPrefix + objType + } + + o.Details.Fields[bundle.RelationKeyType.String()] = pbtypes.String(objType) + err = a.localStore.Objects.CreateObject(id, o.Details, o.Relations, nil, o.Snippet) + if err != nil { + log.Errorf("migration reindexAll: failed to get objects by id: %s", err.Error()) + continue + } + migrated++ + } + if migrated != total { + log.Errorf("migration reindexAll: %d/%d completed", migrated, len(ids)) + } else { + log.Debugf("migration reindexAll completed for %d objects", migrated) + } + return migrated, nil +} + func reindexAll(a *Anytype, lastMigration bool) error { return doWithRunningNode(a, true, !lastMigration, func() error { - ids, err := a.localStore.Objects.ListIds() - if err != nil { - return err - } - total := len(ids) - var migrated int - for _, id := range ids { - sbt, err := smartblock.SmartBlockTypeFromID(id) - if err != nil { - return fmt.Errorf("migration reindexAll:failed to extract smartblock type: %w", err) - } - if sbt == smartblock.SmartBlockTypeArchive { - // remove archive we have accidentally indexed - err = a.localStore.Objects.DeleteObject(id) - if err != nil { - log.Errorf("migration reindexAll: failed to delete archive from index: %s", err.Error()) - } - total-- - continue - } - for _, idx := range a.localStore.Objects.Indexes() { - //if idx.Name == "objtype_relkey_setid" { - // skip it because we can't reindex relations in sets for now - // continue - //} - - err = localstore.EraseIndex(idx, a.t.Datastore().(ds.TxnDatastore)) - if err != nil { - log.Errorf("migration reindexAll: failed to delete archive from index: %s", err.Error()) - } - } - oi, err := a.localStore.Objects.GetByIDs(id) - if err != nil { - log.Errorf("migration reindexAll: failed to get objects by id: %s", err.Error()) - continue - } - - if len(oi) < 1 { - log.Errorf("migration reindexAll: failed to get objects: not found") - continue - } - o := oi[0] - var objType string - if pbtypes.HasField(o.Details, bundle.RelationKeyType.String()) { - objTypes := pbtypes.GetStringList(o.Details, bundle.RelationKeyType.String()) - if len(objTypes) > 0 { - objType = objTypes[0] - } - } - - if strings.HasPrefix(objType, localstore.OldCustomObjectTypeURLPrefix) { - objType = strings.TrimPrefix(objType, localstore.OldCustomObjectTypeURLPrefix) - } else if strings.HasPrefix(objType, localstore.OldBundledObjectTypeURLPrefix) { - objType = localstore.BundledObjectTypeURLPrefix + strings.TrimPrefix(objType, localstore.OldBundledObjectTypeURLPrefix) - } else if bundle.HasObjectType(objType) { - objType = localstore.BundledObjectTypeURLPrefix + objType - } - - o.Details.Fields[bundle.RelationKeyType.String()] = pbtypes.String(objType) - err = a.localStore.Objects.CreateObject(id, o.Details, o.Relations, nil, o.Snippet) - if err != nil { - log.Errorf("migration reindexAll: failed to get objects by id: %s", err.Error()) - continue - } - migrated++ - } - if migrated != total { - log.Errorf("migration reindexAll: %d/%d completed", migrated, len(ids)) - } else { - log.Debugf("migration reindexAll completed for %d objects", migrated) - } - return nil + _, err := ReindexAll(a) + return err }) } diff --git a/pkg/lib/core/threads/derived.go b/pkg/lib/core/threads/derived.go index b61a61cf5..54ac3582e 100644 --- a/pkg/lib/core/threads/derived.go +++ b/pkg/lib/core/threads/derived.go @@ -108,7 +108,7 @@ func (s *service) EnsurePredefinedThreads(ctx context.Context, newAccount bool) // pull only after threadsDbInit to handle all events _, err = s.pullThread(ctxPull, account.ID) if err != nil { - log.Errorf("account pull failed") + log.Errorf("account pull failed: %s", err.Error()) return } else { log.Infof("account pull done")