From 20ff0b5bbd66a6b4e51e2a0b752b504f6890f0ce Mon Sep 17 00:00:00 2001 From: Mikhail Iudin Date: Thu, 22 Aug 2024 17:45:00 +0200 Subject: [PATCH] GO-3906 Fix tantivy build for ios --- .github/workflows/build.yml | 3 +- Makefile | 7 +- cmd/iosrepack/main.go | 214 ++++++++++++++++++ go.mod | 6 +- go.sum | 12 +- .../localstore/ftsearch/ftsearchtantivy.go | 4 +- tantivity_sha256.txt | 21 +- 7 files changed, 244 insertions(+), 23 deletions(-) create mode 100644 cmd/iosrepack/main.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ccdd8788d..9098acd2b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -82,7 +82,7 @@ jobs: ${{ runner.os }}-go-${{ matrix.go-version }}- - name: Install old MacOS SDK (for backward compatibility of CGO) run: source .github/install_macos_sdk.sh 10.15 - if: github.event.inputs.run-on-runner != 'ARM64' + if: github.event.inputs.run-on-runner != 'ARM64' && github.event_name != 'schedule' - name: Set env vars env: UNSPLASH_KEY: ${{ secrets.UNSPLASH_KEY }} @@ -233,6 +233,7 @@ jobs: go install go.uber.org/mock/mockgen@v0.3.0 make test-deps gomobile bind -tags "envproduction nogrpcserver gomobile nowatchdog nosigar nomutexdeadlockdetector timetzdata" -ldflags "$FLAGS" -v -target=ios -o Lib.xcframework github.com/anyproto/anytype-heart/clientlibrary/service github.com/anyproto/anytype-heart/core || true + go run cmd/iosrepack/main.go gtar --exclude ".*" -czvf ios_framework.tar.gz Lib.xcframework protobuf json gradle publish mv ios_framework.tar.gz .release/ios_framework_${VERSION}.tar.gz diff --git a/Makefile b/Makefile index cc06e1438..702e9d945 100644 --- a/Makefile +++ b/Makefile @@ -135,6 +135,8 @@ endif @cp pkg/lib/bundle/relations.json dist/ios/json/ @cp pkg/lib/bundle/internal*.json dist/ios/json/ @go mod tidy + @echo 'Repacking iOS framework...' + @go run cmd/iosrepack/main.go install-dev-ios: setup-go build-ios protos-swift @echo 'Installing iOS framework locally at $(CLIENT_IOS_PATH)...' @@ -337,7 +339,7 @@ endif ### Tantivy Section REPO := anyproto/tantivy-go -VERSION := v0.0.7 +VERSION := v0.0.8 OUTPUT_DIR := deps/libs SHA_FILE = tantivity_sha256.txt @@ -349,6 +351,7 @@ TANTIVY_LIBS := android-386.tar.gz \ darwin-arm64.tar.gz \ ios-amd64.tar.gz \ ios-arm64.tar.gz \ + ios-arm64-sim.tar.gz \ linux-amd64-musl.tar.gz \ windows-amd64.tar.gz @@ -376,10 +379,12 @@ download-tantivy-all-force: download-tantivy @echo "SHA256 checksums generated." download-tantivy-all: download-tantivy + @rm -rf /deps/libs/* @echo "Validating SHA256 checksums..." @shasum -a 256 -c $(SHA_FILE) --status || { echo "Hash mismatch detected."; exit 1; } @echo "All files are valid." download-tantivy-local: + @rm -rf /deps/libs/* @mkdir -p $(OUTPUT_DIR) @cp -r $(TANTIVY_GO_PATH)/libs/ $(OUTPUT_DIR) \ No newline at end of file diff --git a/cmd/iosrepack/main.go b/cmd/iosrepack/main.go new file mode 100644 index 000000000..d2d94287d --- /dev/null +++ b/cmd/iosrepack/main.go @@ -0,0 +1,214 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" +) + +const ( + repack = "repack" + iosArm64 = "ios_arm64.a" + iosArm64Sim = "ios_arm64_sim.a" + iosX86Sim = "ios_x86_sim.a" + iosArm64Repack = "iosArm64" + iosArm64SimRepack = "iosArm64Sim" + iosX86SimRepack = "iosX86Sim" + combined = "libcombined.a" +) + +type iosContext struct { + tantivyArm64 string + tantivyArm64Sim string + tantivyX86Sim string + iosLib string +} + +func main() { + err := delegateMain() + if err != nil { + fmt.Println("Error:", err) + os.Exit(-1) + } +} + +func delegateMain() error { + ctx := new(iosContext) + ctx.tantivyArm64 = "deps/libs/ios-arm64/libtantivy_go.a" + ctx.tantivyArm64Sim = "deps/libs/ios-arm64-sim/libtantivy_go.a" + ctx.tantivyX86Sim = "deps/libs/ios-amd64/libtantivy_go.a" + ctx.iosLib = "dist/ios/Lib.xcframework" + + iosArm64Lib, iosSimLib, err := thinIosLibs(ctx) + if err != nil { + return err + } + + err = repackIosLibsWithTantivy(ctx) + if err != nil { + return err + } + + err = removeOldIosLibs(iosArm64Lib, iosSimLib) + if err != nil { + return err + } + + err = createNewIosLibs(iosArm64Lib, iosSimLib) + if err != nil { + return err + } + + return nil +} + +func createNewIosLibs(iosArm64Lib string, iosSimLib string) error { + defer func() { + _ = execute("Cleanup:", "rm", "-rf", repack) + _ = execute("Cleanup:", "rm", iosArm64) + _ = execute("Cleanup:", "rm", iosArm64Sim) + _ = execute("Cleanup:", "rm", iosX86Sim) + }() + + err := execute("Error creating lib:", "lipo", "-create", + filepath.Join(repack, iosArm64Repack, combined), "-output", "Lib") + if err != nil { + return err + } + + err = execute("Move created lib:", "mv", "Lib", iosArm64Lib) + if err != nil { + return err + } + + err = execute("Error creating lib:", "lipo", "-create", + filepath.Join(repack, iosArm64SimRepack, combined), + filepath.Join(repack, iosX86SimRepack, combined), "-output", "Lib") + if err != nil { + return err + } + + err = execute("Move created lib:", "mv", "Lib", iosSimLib) + if err != nil { + return err + } + + return nil +} + +func removeOldIosLibs(iosArm64Lib string, iosSimLib string) error { + err := execute("Error removing lib:", "rm", iosArm64Lib) + if err != nil { + return err + } + + err = execute("Error removing lib:", "rm", iosSimLib) + if err != nil { + return err + } + return nil +} + +func thinIosLibs(ctx *iosContext) (string, string, error) { + iosArm64Lib := filepath.Join(ctx.iosLib, "ios-arm64", "Lib.framework", "Lib") + err := execute("Error extracting lib:", "lipo", iosArm64Lib, "-thin", "arm64", "-output", iosArm64) + if err != nil { + return "", "", err + } + + iosSimLib := filepath.Join(ctx.iosLib, "ios-arm64_x86_64-simulator", "Lib.framework", "Lib") + err = execute("Error extracting lib:", "lipo", iosSimLib, "-thin", "arm64", "-output", iosArm64Sim) + if err != nil { + return "", "", err + } + + err = execute("Error extracting lib:", "lipo", iosSimLib, "-thin", "x86_64", "-output", iosX86Sim) + if err != nil { + return "", "", err + } + return iosArm64Lib, iosSimLib, nil +} + +func repackIosLibsWithTantivy(ctx *iosContext) error { + err := os.MkdirAll(repack, os.ModePerm) + if err != nil { + return err + } + + err = os.Chdir(repack) + if err != nil { + return err + } + + err = repackLib(iosArm64Repack, ctx.tantivyArm64, iosArm64) + if err != nil { + return err + } + + err = repackLib(iosArm64SimRepack, ctx.tantivyArm64Sim, iosArm64Sim) + if err != nil { + return err + } + + err = repackLib(iosX86SimRepack, ctx.tantivyX86Sim, iosX86Sim) + if err != nil { + return err + } + + err = os.Chdir("..") + if err != nil { + return err + } + return nil +} + +func repackLib(dir string, tantivyLib string, iosLib string) error { + err := os.MkdirAll(dir, os.ModePerm) + if err != nil { + return err + } + + err = os.Chdir(dir) + if err != nil { + return err + } + + err = execute("Error extracting lib:", "ar", "-x", filepath.Join("..", "..", tantivyLib)) + if err != nil { + return err + } + + err = execute("Error extracting lib:", "ar", "-x", filepath.Join("..", "..", iosLib)) + if err != nil { + return err + } + + oFiles, err := filepath.Glob("*.o") + if err != nil { + return err + } + + err = execute("Error combine lib:", "ar", append([]string{"-qc", combined}, oFiles...)...) + if err != nil { + return err + } + + err = os.Chdir("..") + if err != nil { + return err + } + return nil +} + +func execute(errText string, name string, arg ...string) error { + cmd := exec.Command(name, arg...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + fmt.Println(errText, err) + return err + } + return nil +} diff --git a/go.mod b/go.mod index 10a979350..a1b9c0ed0 100644 --- a/go.mod +++ b/go.mod @@ -93,7 +93,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 golang.org/x/image v0.19.0 - golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b + golang.org/x/mobile v0.0.0-20240806205939-81131f6468ab golang.org/x/net v0.28.0 golang.org/x/oauth2 v0.22.0 golang.org/x/text v0.17.0 @@ -257,12 +257,12 @@ require ( go.opentelemetry.io/otel v1.14.0 // indirect go.opentelemetry.io/otel/trace v1.14.0 // indirect golang.org/x/crypto v0.26.0 // indirect - golang.org/x/mod v0.19.0 // indirect + golang.org/x/mod v0.20.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.23.0 // indirect golang.org/x/term v0.23.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.23.0 // indirect + golang.org/x/tools v0.24.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect google.golang.org/protobuf v1.34.2 // indirect diff --git a/go.sum b/go.sum index 81e566c8e..71768e643 100644 --- a/go.sum +++ b/go.sum @@ -1550,8 +1550,8 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b h1:WX7nnnLfCEXg+FmdYZPai2XuP3VqCP1HZVMST0n9DF0= -golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b/go.mod h1:EiXZlVfUTaAyySFVJb9rsODuiO+WXu8HrUuySb7nYFw= +golang.org/x/mobile v0.0.0-20240806205939-81131f6468ab h1:KONOFF8Uy3b60HEzOsGnNghORNhY4ImyOx0PGm73K9k= +golang.org/x/mobile v0.0.0-20240806205939-81131f6468ab/go.mod h1:udWezQGYjqrCxz5nV321pXQTx5oGbZx+khZvFjZNOPM= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -1563,8 +1563,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1841,8 +1841,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/lib/localstore/ftsearch/ftsearchtantivy.go b/pkg/lib/localstore/ftsearch/ftsearchtantivy.go index 044be8194..bc5d4ba09 100644 --- a/pkg/lib/localstore/ftsearch/ftsearchtantivy.go +++ b/pkg/lib/localstore/ftsearch/ftsearchtantivy.go @@ -1,8 +1,8 @@ package ftsearch /* -#cgo windows,amd64 LDFLAGS:-L${SRCDIR}/../../../../deps/libs/windows-amd64 -ltantivy_go -lm -pthread -lws2_32 -lbcrypt -lwsock32 -lntdll -luserenv -lsynchronization -#cgo darwin,amd64 LDFLAGS:-L${SRCDIR}/../../../../deps/libs/darwin-amd64 -ltantivy_go -lm -pthread -framework CoreFoundation -framework Security -ldl +#cgo windows,amd64 LDFLAGS:-L${SRCDIR}/../../../../deps/libs/windows-amd64 -ltantivy_go -lm -pthread -lws2_32 -lbcrypt -lntdll -luserenv +#cgo darwin,amd64 LDFLAGS:-L${SRCDIR}/../../../../deps/libs/darwin-amd64 -ltantivy_go -lm -pthread -ldl #cgo darwin,arm64 LDFLAGS:-L${SRCDIR}/../../../../deps/libs/darwin-arm64 -ltantivy_go -lm -pthread -ldl #cgo ios,arm64 LDFLAGS:-L${SRCDIR}/../../../../deps/libs/ios-arm64 -ltantivy_go -lm -pthread -ldl #cgo ios,amd64 LDFLAGS:-L${SRCDIR}/../../../../deps/libs/ios-amd64 -ltantivy_go -lm -pthread -ldl diff --git a/tantivity_sha256.txt b/tantivity_sha256.txt index f1b29b7ca..ac044d150 100644 --- a/tantivity_sha256.txt +++ b/tantivity_sha256.txt @@ -1,10 +1,11 @@ -31eaddb2ce286ff3c8b9fe01781adaa2f8fd7db759d01c5fab2f655cb85202bb deps/libs/android-386.tar.gz -011be437ffd1498aa295191f0c5f44d30de5ec7dc293f28f57d3acc311a59168 deps/libs/android-amd64.tar.gz -93864215777aba8451ee6ba09a3b2fc2b0582273b166a7f43ecc54e00b37615b deps/libs/android-arm.tar.gz -3a7a5ed4b205d4dafe12dc2cac592cf1f37f5a0f6c99a471365db70eda19effb deps/libs/android-arm64.tar.gz -d9fcba7ec18aaf6e59d06e0ef8011de45df578b7617a566ce38fbe764ebfa0ed deps/libs/darwin-amd64.tar.gz -5d8a1600029c1544bebc6ee0d35098ff8d1242056a23b041da739e5197185557 deps/libs/darwin-arm64.tar.gz -0222fa5e850ddc1641ea9138107138ba754254c577f77b43c1b5bfb11c264610 deps/libs/ios-amd64.tar.gz -e3adc97753f1b8fa95e4d2e37385da2c710dd40ba7706af34303af5bce69506c deps/libs/ios-arm64.tar.gz -f88cff214cc0bb01f2cf7b6e54ca617cce1eac6393decf2d86c6d24f0b903df0 deps/libs/linux-amd64-musl.tar.gz -cb29b5328e9b09d99342b936812903410a1864196486d2edd5cc9028bba9c0a2 deps/libs/windows-amd64.tar.gz +435a2d6cc44cd85c6dfe4a051534ae2ef26d8abb99f2a998da207340350e48fa deps/libs/android-386.tar.gz +b8949b51356b1abb4cf04ac1a5e507c840023a70bd9b82a75e63bc3b3c8070c9 deps/libs/android-amd64.tar.gz +9fc54c7b58941d15f3f1ed8c573592b3ee582dffd29b968ae5db8d6e0d2e8fb8 deps/libs/android-arm.tar.gz +0192d6188ca85b81ba3d2a89e2132b34a50d0f188f201fa51685af6668a4d236 deps/libs/android-arm64.tar.gz +0cc77c0024020c53c8424381649c6477b387a511f3e5f4406fdc2436f40419da deps/libs/darwin-amd64.tar.gz +1a75157778da0afe2d06e91cc8a644848dced50fd49bd3b54ff7256b084728cf deps/libs/darwin-arm64.tar.gz +a5b9c9e573db75e8662b24231517d32c0174661ab1bfb911a66ce74de7c71c71 deps/libs/ios-amd64.tar.gz +d8d0eba9682831862d4925182a066ce28b3bae1539bc338b6200176cba2209d0 deps/libs/ios-arm64.tar.gz +429d68cb69f880ee475422c4856e230bb49cbb554d4d2f61f8e5b23c7597b814 deps/libs/ios-arm64-sim.tar.gz +e83caf65dcaad0fd8fb4e8374c35033841ddb1485381ddff1d0b70b5ee380748 deps/libs/linux-amd64-musl.tar.gz +89fc84bc28461422a08f1662352eb96e5511b0dd679c3e75be80d44fece1296b deps/libs/windows-amd64.tar.gz