diff --git a/.env b/.env index 488839c..6c0cac0 100644 --- a/.env +++ b/.env @@ -1,7 +1,39 @@ AWS_ACCESS_KEY_ID=testid AWS_SECRET_ACCESS_KEY=testkey +AWS_PORT=4569 ANY_SYNC_NODE_VERSION=latest ANY_SYNC_FILENODE_VERSION=latest ANY_SYNC_COORDINATOR_VERSION=latest ANY_SYNC_CONSENSUSNODE_VERSION=latest + +ANY_SYNC_NODE_1_HOST=any-sync-node-1 +ANY_SYNC_NODE_1_PORT=1001 +ANY_SYNC_NODE_1_ADDRESSES=${ANY_SYNC_NODE_1_HOST}:${ANY_SYNC_NODE_1_PORT} + +ANY_SYNC_NODE_2_HOST=any-sync-node-2 +ANY_SYNC_NODE_2_PORT=1002 +ANY_SYNC_NODE_2_ADDRESSES=${ANY_SYNC_NODE_2_HOST}:${ANY_SYNC_NODE_2_PORT} + +ANY_SYNC_NODE_3_HOST=any-sync-node-3 +ANY_SYNC_NODE_3_PORT=1003 +ANY_SYNC_NODE_3_ADDRESSES=${ANY_SYNC_NODE_3_HOST}:${ANY_SYNC_NODE_3_PORT} + +ANY_SYNC_COORDINATOR_HOST=any-sync-coordinator +ANY_SYNC_COORDINATOR_PORT=1004 +ANY_SYNC_COORDINATOR_ADDRESSES=${ANY_SYNC_COORDINATOR_HOST}:${ANY_SYNC_COORDINATOR_PORT} + +ANY_SYNC_FILENODE_HOST=any-sync-filenode +ANY_SYNC_FILENODE_PORT=1005 +ANY_SYNC_FILENODE_ADDRESSES=${ANY_SYNC_FILENODE_HOST}:${ANY_SYNC_FILENODE_PORT} + +ANY_SYNC_CONSENSUSNODE_HOST=any-sync-consensusnode +ANY_SYNC_CONSENSUSNODE_PORT=1006 +ANY_SYNC_CONSENSUSNODE_ADDRESSES=${ANY_SYNC_CONSENSUSNODE_HOST}:${ANY_SYNC_CONSENSUSNODE_PORT} + +MONGO_1_PORT=27001 +MONGO_2_PORT=27002 +MONGO_3_PORT=27003 +MONGO_CONNECT=mongodb://mongo-1:${MONGO_1_PORT},mongo-2:${MONGO_2_PORT},mongo-3:${MONGO_3_PORT} +REDIS_PORT=6379 +REDIS_URL=redis://redis:${REDIS_PORT}?dial_timeout=3&read_timeout=6s diff --git a/Dockerfile-generateconfig b/Dockerfile-generateconfig new file mode 100644 index 0000000..f975d76 --- /dev/null +++ b/Dockerfile-generateconfig @@ -0,0 +1,16 @@ +FROM golang:1.19 + +# git+ssh {{ +RUN apt-get update && apt-get install -y ca-certificates git-core ssh rsync +RUN mkdir -p -m 0700 ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts +RUN git config --global url.ssh://git@github.com/.insteadOf https://github.com/ +# }} + +RUN apt-get install -y yq + +WORKDIR /generateconfig +COPY .env /.env +COPY gen.sh /gen.sh + +ENV GOPRIVATE="github.com/anyproto" +RUN --mount=type=ssh go install github.com/anyproto/any-sync-tools/anyconf@latest diff --git a/Makefile b/Makefile index d8fd7b1..092f050 100644 --- a/Makefile +++ b/Makefile @@ -1,25 +1,40 @@ .DEFAULT_GOAL := start -SHELL=/bin/bash +SHELL := /bin/bash +include .env generate_etc: + docker compose --file docker-compose-generateconfig.yml up --build --remove-orphans --detach + sleep 1 install -d \ tmp/etc/any-sync-node-{1..3}/ \ tmp/etc/any-sync-filenode/ \ tmp/etc/any-sync-coordinator/ \ tmp/etc/any-sync-consensusnode/ - cat etc/{network,common,node-1}.yml > tmp/etc/any-sync-node-1/config.yml - cat etc/{network,common,node-2}.yml > tmp/etc/any-sync-node-2/config.yml - cat etc/{network,common,node-3}.yml > tmp/etc/any-sync-node-3/config.yml - cat etc/{network,common,filenode}.yml > tmp/etc/any-sync-filenode/config.yml - cat etc/{network,common,coordinator}.yml > tmp/etc/any-sync-coordinator/config.yml - cat etc/{network,common,consensusnode}.yml > tmp/etc/any-sync-consensusnode/config.yml - cat etc/network.yml | grep -v '^network:' > tmp/etc/any-sync-coordinator/network.yml + sed 's|^| |; 1s|^|network:\n|' tmp/generateconfig/nodes.yml > tmp/etc/network.yaml + cat tmp/etc/network.yaml etc/common.yml tmp/generateconfig/account0.yml etc/node-1.yml > tmp/etc/any-sync-node-1/config.yml + cat tmp/etc/network.yaml etc/common.yml tmp/generateconfig/account1.yml etc/node-2.yml > tmp/etc/any-sync-node-2/config.yml + cat tmp/etc/network.yaml etc/common.yml tmp/generateconfig/account2.yml etc/node-3.yml > tmp/etc/any-sync-node-3/config.yml + cat tmp/etc/network.yaml etc/common.yml tmp/generateconfig/account3.yml etc/coordinator.yml > tmp/etc/any-sync-coordinator/config.yml + cat tmp/etc/network.yaml etc/common.yml tmp/generateconfig/account4.yml etc/filenode.yml > tmp/etc/any-sync-filenode/config.yml + cat tmp/etc/network.yaml etc/common.yml tmp/generateconfig/account5.yml etc/consensusnode.yml > tmp/etc/any-sync-consensusnode/config.yml + cp tmp/generateconfig/nodes.yml tmp/etc/any-sync-coordinator/network.yml + perl -i -pe's|%ANY_SYNC_NODE_1_ADDRESSES%|${ANY_SYNC_NODE_1_ADDRESSES}|g' tmp/etc/network.yaml tmp/etc/*/*.yml + perl -i -pe's|%ANY_SYNC_NODE_2_ADDRESSES%|${ANY_SYNC_NODE_2_ADDRESSES}|g' tmp/etc/network.yaml tmp/etc/*/*.yml + perl -i -pe's|%ANY_SYNC_NODE_3_ADDRESSES%|${ANY_SYNC_NODE_3_ADDRESSES}|g' tmp/etc/network.yaml tmp/etc/*/*.yml + perl -i -pe's|%ANY_SYNC_COORDINATOR_ADDRESSES%|${ANY_SYNC_COORDINATOR_ADDRESSES}|g' tmp/etc/network.yaml tmp/etc/*/*.yml + perl -i -pe's|%ANY_SYNC_FILENODE_ADDRESSES%|${ANY_SYNC_FILENODE_ADDRESSES}|g' tmp/etc/network.yaml tmp/etc/*/*.yml + perl -i -pe's|%ANY_SYNC_CONSENSUSNODE_ADDRESSES%|${ANY_SYNC_CONSENSUSNODE_ADDRESSES}|g' tmp/etc/network.yaml tmp/etc/*/*.yml + perl -i -pe's|%MONGO_CONNECT%|${MONGO_CONNECT}|g' tmp/etc/network.yaml tmp/etc/*/*.yml + perl -i -pe's|%REDIS_URL%|${REDIS_URL}|g' tmp/etc/network.yaml tmp/etc/*/*.yml + perl -i -pe's|%AWS_PORT%|${AWS_PORT}|g' tmp/etc/network.yaml tmp/etc/*/*.yml + docker compose --file docker-compose-generateconfig.yml stop start: generate_etc docker compose up --force-recreate --build --remove-orphans --detach --pull always stop: docker compose stop + docker compose --file docker-compose-generateconfig.yml stop clean: docker system prune --all @@ -29,6 +44,7 @@ pull: down: docker compose down + docker compose --file docker-compose-generateconfig.yml down # build with "plain" log for debug build: diff --git a/docker-compose-generateconfig.yml b/docker-compose-generateconfig.yml new file mode 100644 index 0000000..30e1b89 --- /dev/null +++ b/docker-compose-generateconfig.yml @@ -0,0 +1,12 @@ +version: "3.9" +services: + generateconfig: + image: generateconfig + build: + context: . + dockerfile: Dockerfile-generateconfig + ssh: + - default + volumes: + - ./tmp/generateconfig/:/generateconfig/ + command: /gen.sh diff --git a/docker-compose.any-sync-tools.yml b/docker-compose.any-sync-tools.yml deleted file mode 100644 index 4e10b9c..0000000 --- a/docker-compose.any-sync-tools.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: "3.9" -services: - any-sync-tools: - image: localbuild-tools - build: - context: . - dockerfile: Dockerfile - ssh: - - default - args: - REPO_DIR: repos/any-sync-tools/ - volumes: - - ./tmp/tools:/data - command: bash -c "sleep 600" diff --git a/docker-compose.yml b/docker-compose.yml index c271f46..f618bc4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,27 +2,27 @@ version: "3.9" services: mongo-1: image: mongo - command: --replSet rs0 + command: --replSet rs0 --port ${MONGO_1_PORT} ports: - - 27010:27017 + - "${MONGO_1_PORT}:${MONGO_1_PORT}" volumes: - ./tmp/mongo-1/:/data/db healthcheck: - test: test $$(echo "rs.initiate({_id:'rs0',members:[{_id:0,host:\"mongo-1:27017\"},{_id:1,host:\"mongo-2:27017\"},{_id:2,host:\"mongo-3:27017\"}]}).ok || rs.status().ok" | mongosh --port 27017 --quiet) -eq 1 + test: test $$(echo "rs.initiate({_id:'rs0',members:[{_id:0,host:\"mongo-1:${MONGO_1_PORT}\"},{_id:1,host:\"mongo-2:${MONGO_2_PORT}\"},{_id:2,host:\"mongo-3:${MONGO_3_PORT}\"}]}).ok || rs.status().ok" | mongosh --port ${MONGO_1_PORT} --quiet) -eq 1 interval: 10s start_period: 30s mongo-2: image: mongo - command: --replSet rs0 + command: --replSet rs0 --port ${MONGO_2_PORT} ports: - - 27011:27017 + - "${MONGO_2_PORT}:${MONGO_2_PORT}" volumes: - ./tmp/mongo-2/:/data/db mongo-3: image: mongo - command: --replSet rs0 + command: --replSet rs0 --port ${MONGO_3_PORT} ports: - - 27012:27017 + - "${MONGO_3_PORT}:${MONGO_3_PORT}" volumes: - ./tmp/mongo-3/:/data/db redis: @@ -31,13 +31,13 @@ services: restart: always command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy noeviction ports: - - "6379:6379" + - "${REDIS_PORT}:${REDIS_PORT}" volumes: - ./tmp/redis/:/data/ s3-emulator: image: stanislavt/s3-emulator ports: - - 4569:4569 + - "${AWS_PORT}:${AWS_PORT}" volumes: - ./tmp/s3_root:/s3_root any-sync-coordinator_bootstrap: @@ -53,7 +53,7 @@ services: - mongo-1 - any-sync-coordinator_bootstrap ports: - - 4435:443 + - "${ANY_SYNC_COORDINATOR_PORT}:${ANY_SYNC_COORDINATOR_PORT}" - 8005:8000 volumes: - ./tmp/etc/any-sync-coordinator/:/etc/any-sync-coordinator/ @@ -70,7 +70,7 @@ services: - s3-emulator - any-sync-coordinator ports: - - 4434:443 + - "${ANY_SYNC_FILENODE_PORT}:${ANY_SYNC_FILENODE_PORT}" - 8004:8000 volumes: - ./tmp/etc/any-sync-filenode/:/etc/any-sync-filenode/ @@ -83,9 +83,10 @@ services: command: bash -c "sleep 15; /bin/any-sync-filenode -c /etc/any-sync-filenode/config.yml" any-sync-node-1: image: "ghcr.io/anyproto/any-sync-node:${ANY_SYNC_NODE_VERSION}" - depends_on: [any-sync-coordinator] + depends_on: + - any-sync-coordinator ports: - - 4431:443 + - "${ANY_SYNC_NODE_1_PORT}:${ANY_SYNC_NODE_1_PORT}" - 8081:8080 - 8001:8000 volumes: @@ -99,9 +100,10 @@ services: command: bash -c "sleep 15; /bin/any-sync-node -c /etc/any-sync-node/config.yml" any-sync-node-2: image: "ghcr.io/anyproto/any-sync-node:${ANY_SYNC_NODE_VERSION}" - depends_on: [any-sync-coordinator] + depends_on: + - any-sync-coordinator ports: - - 4432:443 + - "${ANY_SYNC_NODE_2_PORT}:${ANY_SYNC_NODE_2_PORT}" - 8082:8080 - 8002:8000 volumes: @@ -115,9 +117,10 @@ services: command: bash -c "sleep 15; /bin/any-sync-node -c /etc/any-sync-node/config.yml" any-sync-node-3: image: "ghcr.io/anyproto/any-sync-node:${ANY_SYNC_NODE_VERSION}" - depends_on: [any-sync-coordinator] + depends_on: + - any-sync-coordinator ports: - - 4433:443 + - "${ANY_SYNC_NODE_3_PORT}:${ANY_SYNC_NODE_3_PORT}" - 8083:8080 - 8003:8000 volumes: @@ -131,9 +134,10 @@ services: command: bash -c "sleep 15; /bin/any-sync-node -c /etc/any-sync-node/config.yml" any-sync-consensusnode: image: "ghcr.io/anyproto/any-sync-consensusnode:${ANY_SYNC_CONSENSUSNODE_VERSION}" - depends_on: [any-sync-coordinator] + depends_on: + - any-sync-coordinator ports: - - 4436:443 + - "${ANY_SYNC_CONSENSUSNODE_PORT}:${ANY_SYNC_CONSENSUSNODE_PORT}" - 8006:8000 volumes: - ./tmp/etc/any-sync-consensusnode/:/etc/any-sync-consensusnode/ diff --git a/etc/consensusnode.yml b/etc/consensusnode.yml index 30c2ba3..4fbd2a0 100644 --- a/etc/consensusnode.yml +++ b/etc/consensusnode.yml @@ -1,11 +1,5 @@ - -account: - peerId: 12D3KooWNT4hbNKXeFuDtdQ13ZbhQ1oJJ2AVWgAQVRrCj6Zb5wxk - peerKey: 5Btn70kUmMokNhSDhu3GzuFcbLxK9aRnoyb/zPWkLvu7svtWJZxQDJ5AWYZ4TxGhgb/GbZB1nPjp7Kz+V+J4OQ== - signingKey: 0ojibPdVZUL9hJmaQHpRBJYjH0Y0MuoLL+Lc2hFchorGS26RaLuUVs36ikGDDRi39BlzafEDNomB+IrBt8HCdA== - mongo: - connect: mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/?w=majority + connect: %MONGO_CONNECT%/?w=majority database: consensus log: log @@ -16,7 +10,7 @@ drpc: yamux: listenAddrs: - - any-sync-consensusnode:443 + - %ANY_SYNC_CONSENSUSNODE_ADDRESSES% writeTimeoutSec: 10 dialTimeoutSec: 10 diff --git a/etc/coordinator.yml b/etc/coordinator.yml index 4299073..2546a01 100644 --- a/etc/coordinator.yml +++ b/etc/coordinator.yml @@ -1,11 +1,5 @@ - -account: - peerId: 12D3KooWPUMFQFeiyZDwS2aGZJfcYZamwq3NLQJT6RskEzPfP2Xd - peerKey: dQ+IwS15rKyNfjeUdgmg1AFhvl69JzkI/FpkYnfl9ITK4vX8BsI+O9xyxPhQfJbsgixRJLIQ6fKceXL6GRiTZA== - signingKey: 0ojibPdVZUL9hJmaQHpRBJYjH0Y0MuoLL+Lc2hFchorGS26RaLuUVs36ikGDDRi39BlzafEDNomB+IrBt8HCdA== - mongo: - connect: mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/ + connect: %MONGO_CONNECT% database: coordinator log: log spaces: spaces @@ -21,7 +15,7 @@ drpc: yamux: listenAddrs: - - any-sync-coordinator:443 + - %ANY_SYNC_COORDINATOR_ADDRESSES% writeTimeoutSec: 10 dialTimeoutSec: 10 diff --git a/etc/filenode.yml b/etc/filenode.yml index 251cb24..a152e79 100644 --- a/etc/filenode.yml +++ b/etc/filenode.yml @@ -1,18 +1,13 @@ -account: - peerId: 12D3KooWMwE15iEG51Sewt2zYqwgbrBz6QMuFVdWLBMes1SADtCh - peerKey: kTZYumPO7MoEsU7ZyeNTLFgYaO3rGFEHk2qXJdqnV9O0DhMk/wdbgkwrbqtIpVH4Bkx7LPEHesTg3DfH4G524g== - signingKey: kTZYumPO7MoEsU7ZyeNTLFgYaO3rGFEHk2qXJdqnV9O0DhMk/wdbgkwrbqtIpVH4Bkx7LPEHesTg3DfH4G524g== - s3Store: - bucket: s3-emulator:4569 + bucket: s3-emulator:%AWS_PORT% maxThreads: 16 profile: default region: eu-central-1 - endpoint: http://s3-emulator:4569 + endpoint: http://s3-emulator:%AWS_PORT% redis: isCluster: false - url: redis://redis:6379?dial_timeout=3&read_timeout=6s + url: %REDIS_URL% drpc: stream: @@ -21,7 +16,7 @@ drpc: yamux: listenAddrs: - - any-sync-filenode:443 + - %ANY_SYNC_FILENODE_ADDRESSES% writeTimeoutSec: 10 dialTimeoutSec: 10 diff --git a/etc/network.yml b/etc/network.yml deleted file mode 100644 index a4367a1..0000000 --- a/etc/network.yml +++ /dev/null @@ -1,21 +0,0 @@ -network: - networkId: N8bBRuGei9idgvJeP1yzcjUtxZEXYFgxRwU81vFBpZwzPVXh - nodes: - - peerId: 12D3KooWEPvwLVBZaE8adsDoLh7QT9wrsw4PC82AdcDGYPgssNH7 - addresses: ["any-sync-node-1:443"] - types: [tree] - - peerId: 12D3KooWFMZGP4Jp4uNgDKAjzFpoyYaHbpAVL2jMv9NdYJMFgaqp - addresses: ["any-sync-node-2:443"] - types: [tree] - - peerId: 12D3KooWPQSfv8JbmwXv3QvcPQfSEuM2PujnMVXfcMojVgc5zGFD - addresses: ["any-sync-node-3:443"] - types: [tree] - - peerId: 12D3KooWPUMFQFeiyZDwS2aGZJfcYZamwq3NLQJT6RskEzPfP2Xd - addresses: ["any-sync-coordinator:443"] - types: [coordinator] - - peerId: 12D3KooWMwE15iEG51Sewt2zYqwgbrBz6QMuFVdWLBMes1SADtCh - addresses: [ "any-sync-filenode:443" ] - types: [file] - - peerId: 12D3KooWNT4hbNKXeFuDtdQ13ZbhQ1oJJ2AVWgAQVRrCj6Zb5wxk - addresses: [ "any-sync-consensusnode:443" ] - types: [consensus] diff --git a/etc/node-1.yml b/etc/node-1.yml index 252bc71..1f13f9d 100644 --- a/etc/node-1.yml +++ b/etc/node-1.yml @@ -1,8 +1,3 @@ -account: - peerId: 12D3KooWEPvwLVBZaE8adsDoLh7QT9wrsw4PC82AdcDGYPgssNH7 - peerKey: /UxND22fiH7QgOaMee30x9ZHLFDMY8wWIqWSqOWCvtBEB/dbsFViVokAalOqgRT23hcZLGB8gkLinOc3+u996g== - signingKey: /UxND22fiH7QgOaMee30x9ZHLFDMY8wWIqWSqOWCvtBEB/dbsFViVokAalOqgRT23hcZLGB8gkLinOc3+u996g== - apiServer: listenAddr: 0.0.0.0:8080 @@ -13,7 +8,7 @@ drpc: yamux: listenAddrs: - - any-sync-node-1:443 + - %ANY_SYNC_NODE_1_ADDRESSES% writeTimeoutSec: 10 dialTimeoutSec: 10 diff --git a/etc/node-2.yml b/etc/node-2.yml index 1152e59..1a7e62b 100644 --- a/etc/node-2.yml +++ b/etc/node-2.yml @@ -1,8 +1,3 @@ -account: - peerId: 12D3KooWFMZGP4Jp4uNgDKAjzFpoyYaHbpAVL2jMv9NdYJMFgaqp - peerKey: jYCOxScFFgBVpwc3x/hABTWOm8mA3Pgmtz+kZNf0XL5SR/zCPJCkXiWuYUlJHEXckqWQRgEwWwQNkYhWO+r96w== - signingKey: jYCOxScFFgBVpwc3x/hABTWOm8mA3Pgmtz+kZNf0XL5SR/zCPJCkXiWuYUlJHEXckqWQRgEwWwQNkYhWO+r96w== - apiServer: listenAddr: 0.0.0.0:8080 @@ -13,7 +8,7 @@ drpc: yamux: listenAddrs: - - any-sync-node-2:443 + - %ANY_SYNC_NODE_2_ADDRESSES% writeTimeoutSec: 10 dialTimeoutSec: 10 diff --git a/etc/node-3.yml b/etc/node-3.yml index 6bd0d04..0eba368 100644 --- a/etc/node-3.yml +++ b/etc/node-3.yml @@ -1,8 +1,3 @@ -account: - peerId: 12D3KooWPQSfv8JbmwXv3QvcPQfSEuM2PujnMVXfcMojVgc5zGFD - peerKey: of4HQe2AK6+0vsppJUw+MEuvcv1BhH2S/8f7+sOiN6fJ4sTiOc4qiSRgAFYE8yJR2ZLmOtSti0l8iwDkaGq8HA== - signingKey: of4HQe2AK6+0vsppJUw+MEuvcv1BhH2S/8f7+sOiN6fJ4sTiOc4qiSRgAFYE8yJR2ZLmOtSti0l8iwDkaGq8HA== - apiServer: listenAddr: 0.0.0.0:8080 @@ -13,7 +8,7 @@ drpc: yamux: listenAddrs: - - any-sync-node-3:443 + - %ANY_SYNC_NODE_3_ADDRESSES% writeTimeoutSec: 10 dialTimeoutSec: 10 diff --git a/gen.sh b/gen.sh new file mode 100755 index 0000000..95c4995 --- /dev/null +++ b/gen.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +source /.env + +# generate networkId +if ! [[ -s .networkId ]]; then + anyconf create-network + yq -r .networkId nodes.yml > .networkId + yq -r .account.signingKey account.yml > .networkSigningKey +fi +NETWORK_ID=$( cat .networkId) +NETWORK_SIGNING_KEY=$( cat .networkSigningKey ) + +if ! [[ -s account0.yml ]]; then + anyconf generate-nodes \ + --t tree \ + --t tree \ + --t tree \ + --t coordinator \ + --t file \ + --t consensus \ + --addresses ${ANY_SYNC_NODE_1_ADDRESSES} \ + --addresses ${ANY_SYNC_NODE_2_ADDRESSES} \ + --addresses ${ANY_SYNC_NODE_3_ADDRESSES} \ + --addresses ${ANY_SYNC_COORDINATOR_ADDRESSES} \ + --addresses ${ANY_SYNC_FILENODE_ADDRESSES} \ + --addresses ${ANY_SYNC_CONSENSUSNODE_ADDRESSES} \ + +fi + +yq -yi ".networkId |= \"${NETWORK_ID}\"" nodes.yml +yq -yi ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account3.yml +yq -yi ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account5.yml