diff --git a/.env b/.env.common similarity index 93% rename from .env rename to .env.common index add49aa..5d5040d 100644 --- a/.env +++ b/.env.common @@ -4,17 +4,17 @@ MINIO_BUCKET=minio-bucket MINIO_PORT=9000 MINIO_WEB_PORT=9001 -ANY_SYNC_NODE_VERSION=v0.3.25 -ANY_SYNC_FILENODE_VERSION=v0.6.1 -ANY_SYNC_COORDINATOR_VERSION=v0.3.17 -ANY_SYNC_CONSENSUSNODE_VERSION=v0.1.5 +ANY_SYNC_NODE_VERSION=prod +ANY_SYNC_FILENODE_VERSION=prod +ANY_SYNC_COORDINATOR_VERSION=prod +ANY_SYNC_CONSENSUSNODE_VERSION=prod ANY_SYNC_ADMIN_VERSION=latest MONGO_VERSION=7.0.2 REDIS_VERSION=7.2.0-v6 MINIO_VERSION=RELEASE.2024-01-16T16-07-38Z -EXTERNAL_LISTEN_HOST="127.0.0.1" +EXTERNAL_LISTEN_HOSTS="127.0.0.1" ANY_SYNC_NODE_1_HOST=any-sync-node-1 ANY_SYNC_NODE_1_PORT=1001 diff --git a/.env.override.latest b/.env.override.latest new file mode 100644 index 0000000..e8b7f00 --- /dev/null +++ b/.env.override.latest @@ -0,0 +1,4 @@ +ANY_SYNC_NODE_VERSION=latest +ANY_SYNC_FILENODE_VERSION=latest +ANY_SYNC_COORDINATOR_VERSION=latest +ANY_SYNC_CONSENSUSNODE_VERSION=latest diff --git a/.env.override.prod b/.env.override.prod new file mode 100644 index 0000000..f722eb2 --- /dev/null +++ b/.env.override.prod @@ -0,0 +1,4 @@ +ANY_SYNC_NODE_VERSION=prod +ANY_SYNC_FILENODE_VERSION=prod +ANY_SYNC_COORDINATOR_VERSION=prod +ANY_SYNC_CONSENSUSNODE_VERSION=prod diff --git a/.env.override.stage1 b/.env.override.stage1 new file mode 100644 index 0000000..5d82b19 --- /dev/null +++ b/.env.override.stage1 @@ -0,0 +1,4 @@ +ANY_SYNC_NODE_VERSION=stage1 +ANY_SYNC_FILENODE_VERSION=stage1 +ANY_SYNC_COORDINATOR_VERSION=stage1 +ANY_SYNC_CONSENSUSNODE_VERSION=stage1 diff --git a/.gitignore b/.gitignore index bd9a4e0..da8f2b4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ +.DS_Store /etc/ /storage/ /docker-compose.override.yml /repos/ +/.env +/.env.override diff --git a/Dockerfile-generateconfig b/Dockerfile-generateconfig deleted file mode 100644 index 79d0e51..0000000 --- a/Dockerfile-generateconfig +++ /dev/null @@ -1,17 +0,0 @@ -FROM golang:1.21.6-alpine3.19 as generator -RUN apk add --no-cache bash yq -RUN go install github.com/anyproto/any-sync-tools/anyconf@latest -WORKDIR /opt/generateconfig -COPY ./storage/docker-generateconfig/ . -COPY --chmod=777 docker-generateconfig/generate_config.sh .env . -RUN ./generate_config.sh - -FROM alpine:3.18.4 -RUN apk add --no-cache bash yq perl python3 py3-yaml rsync -WORKDIR /opt/processing -COPY docker-generateconfig/etc/ tmp-etc/ -COPY --chmod=777 docker-generateconfig/processing.sh . -COPY --chmod=777 docker-generateconfig/setListenIp.py . -COPY --from=generator /opt/generateconfig/ generateconfig/ - -CMD ./processing.sh diff --git a/Dockerfile-generateconfig-anyconf b/Dockerfile-generateconfig-anyconf new file mode 100644 index 0000000..8268ef9 --- /dev/null +++ b/Dockerfile-generateconfig-anyconf @@ -0,0 +1,12 @@ +# syntax=docker/dockerfile:1 +FROM golang:1.21.6-alpine3.19 +RUN apk add --no-cache bash yq +RUN go install github.com/anyproto/any-sync-tools/anyconf@latest +WORKDIR /code +HEALTHCHECK \ + --start-period=3s \ + --retries=20 \ + --interval=10s \ + --timeout=3s \ + CMD echo 'HEALTHCHECK' | nc 127.0.0.1 8000 | grep -q HEALTHCHECK +ENTRYPOINT ./docker-generateconfig/anyconf.sh diff --git a/Dockerfile-generateconfig-env b/Dockerfile-generateconfig-env new file mode 100644 index 0000000..2ad7233 --- /dev/null +++ b/Dockerfile-generateconfig-env @@ -0,0 +1,6 @@ +# syntax=docker/dockerfile:1 +FROM python:3.11-alpine +WORKDIR /code +COPY docker-generateconfig/env-requirements.txt requirements.txt +RUN pip install -r requirements.txt +ENTRYPOINT ./docker-generateconfig/env.py diff --git a/Dockerfile-generateconfig-processing b/Dockerfile-generateconfig-processing new file mode 100644 index 0000000..9e9e57d --- /dev/null +++ b/Dockerfile-generateconfig-processing @@ -0,0 +1,11 @@ +# syntax=docker/dockerfile:1 +FROM alpine:3.18.4 +RUN apk add --no-cache bash yq perl python3 py3-yaml +WORKDIR /code +HEALTHCHECK \ + --start-period=3s \ + --retries=10 \ + --interval=5s \ + --timeout=3s \ + CMD echo 'HEALTHCHECK' | nc 127.0.0.1 8000 | grep -q HEALTHCHECK +ENTRYPOINT ./docker-generateconfig/processing.sh diff --git a/Makefile b/Makefile index 5f8de9d..f55e818 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,12 @@ .DEFAULT_GOAL := start -include .env -generate_config: - mkdir -p ./storage/docker-generateconfig/ - docker build -t generateconfig -f Dockerfile-generateconfig . +generate_env: + docker buildx build --tag generateconfig-env --file Dockerfile-generateconfig-env . docker run --rm \ - --volume ${CURDIR}/etc:/opt/processing/etc \ - --volume ${CURDIR}/storage/docker-generateconfig:/opt/processing/docker-generateconfig \ - --name any-sync-generator generateconfig + --volume ${CURDIR}/:/code/ \ + generateconfig-env -start: generate_config +start: generate_env docker compose up -d @echo "Done! Upload your self-hosted network configuration file ${CURDIR}/etc/client.yml into the client app" @echo "See: https://doc.anytype.io/anytype-docs/data-and-security/self-hosting#switching-between-networks" diff --git a/README.md b/README.md index bed5cd9..4edb90c 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Self-host for any-sync, designed for personal usage or for review and testing pu - [Prepare](#prepare) - [Usage](#usage) - [Configuration](#configuration) +- [Troubleshooting](#troubleshooting) - [Compatible versions](#compatible-versions) - [Local build](#local-build) - [Limits web admin](#limits-web-admin) @@ -36,8 +37,8 @@ Self-host for any-sync, designed for personal usage or for review and testing pu git config --global core.autocrlf false # Generate config - docker build -t generateconfig -f Dockerfile-generateconfig . - docker run --rm -v ${PWD}/etc:/opt/processing/etc --name any-sync-generator generateconfig + docker buildx build --tag generateconfig-env --file Dockerfile-generateconfig-env . + docker run --rm --volume ${PWD}/:/code/ generateconfig-env # Run containers docker compose up -d ``` @@ -96,10 +97,38 @@ Self-host for any-sync, designed for personal usage or for review and testing pu ANYTYPE_LOG_LEVEL="*=DEBUG" ANYPROF=:6060 ANY_SYNC_NETWORK=$(pwd)/etc/client.yml /Applications/Anytype.app/Contents/MacOS/Anytype ``` -## configuration -Use file .env -* Set specific versions: find and edit variables with suffix "_VERSION" -* Set external listen host: default 127.0.0.1, for change you need edit variable "EXTERNAL_LISTEN_HOST" +## Configuration +> [!WARNING] +> The .env file is generated automatically. +It is based on the .env.common file, which is overridden or extended by variables from the .env.override file. +### Version control +By default, we use "prod" image version for any-sync-* daemons. +Also you can use "stage1" or "latest" verions: +``` +# for use stage1 version +ln -F -s .env.override.stage1 .env.override +# for use latest version +ln -F -s .env.override.latest .env.override +``` +### external listen host +By default, we use only the listen address 127.0.0.1, which is sufficient for running tests and a local client. +If you need to connect external clients, please add "EXTERNAL_LISTEN_HOSTS" in .env.override file. +Use spaces separation, multiline is not supported. For example: +``` +EXTERNAL_LISTEN_HOSTS= rs.status() +MongoServerError: Our replica set config is invalid or we are not a member of it +``` +please run command: +``` +docker compose exec mongo-1 mongosh --port 27001 --eval 'rs.reconfig({_id: rs.conf()._id, members: [{ _id: 0, host: "mongo-1:27001" }]}, {force: true});' +``` ## Compatible versions You can find compatible versions on these pages: @@ -122,6 +151,8 @@ If you need to create local build binaries for any-sync-*, you can do so by usin make restart ``` +ln -F -s .env.override.stage1 .env.override + ## Limits web admin open link in browser: http://127.0.0.1:80 diff --git a/UPGRADE.md b/UPGRADE.md index 43b1f8d..a08ffc0 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -37,3 +37,8 @@ After Upgrade please run: ``` docker compose exec mongo-1 mongosh --port 27001 --eval 'rs.reconfig({_id: rs.conf()._id, members: [{ _id: 0, host: "mongo-1:27001" }]}, {force: true});' ``` + +## Upgrading from v3.x.x to v4.x.x +1. You need to transfer your modifications from the .env file to the .env.override file. +2. The variable EXTERNAL_LISTEN_HOST is deprecated. + Please, use the variable EXTERNAL_LISTEN_HOSTS instead. diff --git a/docker-compose.yml b/docker-compose.yml index dd5abbe..eeb397c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,31 @@ version: "3.9" services: + # generate configs using the anyconf utility + generateconfig-anyconf: + image: generateconfig-anyconf + build: + context: . + dockerfile: Dockerfile-generateconfig-anyconf + volumes: + - ./:/code + + # processing any-sync-* configs + generateconfig-processing: + depends_on: + generateconfig-anyconf: + condition: service_healthy + image: generateconfig-processing + build: + context: . + dockerfile: Dockerfile-generateconfig-processing + volumes: + - ./:/code + mongo-1: + depends_on: + generateconfig-processing: + condition: service_healthy image: "mongo:${MONGO_VERSION}" command: --replSet ${MONGO_REPLICA_SET} --port ${MONGO_1_PORT} ports: @@ -9,11 +33,14 @@ services: volumes: - ./storage/mongo-1/:/data/db healthcheck: - test: echo "try { rs.status() } catch (err) { rs.initiate() }" | mongosh --port ${MONGO_1_PORT} --quiet + test: test $$(mongosh --port ${MONGO_1_PORT} --quiet --eval "try {rs.initiate({_id:'${MONGO_REPLICA_SET}',members:[{_id:0,host:\"mongo-1:${MONGO_1_PORT}\"}]})} catch(e) {rs.status().ok}") -eq 1 interval: 10s start_period: 30s redis: + depends_on: + generateconfig-processing: + condition: service_healthy image: "redis/redis-stack-server:${REDIS_VERSION}" restart: always command: redis-server --dir /data/ --appendonly yes --maxmemory 256mb --maxmemory-policy noeviction --protected-mode no --loadmodule /opt/redis-stack/lib/redisbloom.so @@ -64,16 +91,24 @@ services: any-sync-coordinator_bootstrap: image: "ghcr.io/anyproto/any-sync-coordinator:${ANY_SYNC_COORDINATOR_VERSION}" depends_on: - - mongo-1 + generateconfig-processing: + condition: service_healthy + mongo-1: + condition: service_healthy volumes: - ./etc/any-sync-coordinator/:/etc/any-sync-coordinator/ restart: no - command: bash -c "sleep 10; /bin/any-sync-confapply -c /etc/any-sync-coordinator/config.yml -n /etc/any-sync-coordinator/network.yml -e" + command: bash -c "/bin/any-sync-confapply -c /etc/any-sync-coordinator/config.yml -n /etc/any-sync-coordinator/network.yml -e" any-sync-coordinator: image: "ghcr.io/anyproto/any-sync-coordinator:${ANY_SYNC_COORDINATOR_VERSION}" depends_on: - - any-sync-coordinator_bootstrap + generateconfig-processing: + condition: service_healthy + mongo-1: + condition: service_healthy + any-sync-coordinator_bootstrap: + condition: service_completed_successfully ports: - "${ANY_SYNC_COORDINATOR_PORT}:${ANY_SYNC_COORDINATOR_PORT}" - "${ANY_SYNC_COORDINATOR_QUIC_PORT}:${ANY_SYNC_COORDINATOR_QUIC_PORT}" @@ -86,14 +121,16 @@ services: limits: memory: 500M restart: on-failure - command: bash -c "sleep 10; /bin/any-sync-coordinator -c /etc/any-sync-coordinator/config.yml" any-sync-filenode: image: "ghcr.io/anyproto/any-sync-filenode:${ANY_SYNC_FILENODE_VERSION}" depends_on: - - redis - - minio - - any-sync-coordinator + redis: + condition: service_healthy + minio: + condition: service_healthy + any-sync-coordinator: + condition: service_started ports: - "${ANY_SYNC_FILENODE_PORT}:${ANY_SYNC_FILENODE_PORT}" - "${ANY_SYNC_FILENODE_QUIC_PORT}:${ANY_SYNC_FILENODE_QUIC_PORT}" @@ -107,12 +144,12 @@ services: limits: memory: 500M restart: on-failure - command: bash -c "sleep 15; /bin/any-sync-filenode -c /etc/any-sync-filenode/config.yml" any-sync-node-1: image: "ghcr.io/anyproto/any-sync-node:${ANY_SYNC_NODE_VERSION}" depends_on: - - any-sync-coordinator + any-sync-coordinator: + condition: service_started ports: - "${ANY_SYNC_NODE_1_PORT}:${ANY_SYNC_NODE_1_PORT}" - "${ANY_SYNC_NODE_1_QUIC_PORT}:${ANY_SYNC_NODE_1_QUIC_PORT}" @@ -127,12 +164,12 @@ services: limits: memory: 500M restart: on-failure - command: bash -c "sleep 15; /bin/any-sync-node -c /etc/any-sync-node/config.yml" any-sync-node-2: image: "ghcr.io/anyproto/any-sync-node:${ANY_SYNC_NODE_VERSION}" depends_on: - - any-sync-coordinator + any-sync-coordinator: + condition: service_started ports: - "${ANY_SYNC_NODE_2_PORT}:${ANY_SYNC_NODE_2_PORT}" - "${ANY_SYNC_NODE_2_QUIC_PORT}:${ANY_SYNC_NODE_2_QUIC_PORT}" @@ -147,12 +184,12 @@ services: limits: memory: 500M restart: on-failure - command: bash -c "sleep 15; /bin/any-sync-node -c /etc/any-sync-node/config.yml" any-sync-node-3: image: "ghcr.io/anyproto/any-sync-node:${ANY_SYNC_NODE_VERSION}" depends_on: - - any-sync-coordinator + any-sync-coordinator: + condition: service_started ports: - "${ANY_SYNC_NODE_3_PORT}:${ANY_SYNC_NODE_3_PORT}" - "${ANY_SYNC_NODE_3_QUIC_PORT}:${ANY_SYNC_NODE_3_QUIC_PORT}" @@ -167,12 +204,12 @@ services: limits: memory: 500M restart: on-failure - command: bash -c "sleep 15; /bin/any-sync-node -c /etc/any-sync-node/config.yml" any-sync-consensusnode: image: "ghcr.io/anyproto/any-sync-consensusnode:${ANY_SYNC_CONSENSUSNODE_VERSION}" depends_on: - - any-sync-coordinator + any-sync-coordinator: + condition: service_started ports: - "${ANY_SYNC_CONSENSUSNODE_PORT}:${ANY_SYNC_CONSENSUSNODE_PORT}" - "${ANY_SYNC_CONSENSUSNODE_QUIC_PORT}:${ANY_SYNC_CONSENSUSNODE_QUIC_PORT}" @@ -185,7 +222,6 @@ services: limits: memory: 500M restart: on-failure - command: bash -c "sleep 15; /bin/any-sync-consensusnode -c /etc/any-sync-consensusnode/config.yml" any-sync-admin: image: "ghcr.io/anyproto/any-sync-admin:${ANY_SYNC_ADMIN_VERSION}" diff --git a/docker-generateconfig/anyconf.sh b/docker-generateconfig/anyconf.sh new file mode 100755 index 0000000..56f83cd --- /dev/null +++ b/docker-generateconfig/anyconf.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +echo "INFO: $0 start" +echo "INFO: loading .env file" +source .env + +echo "INFO: create persistent config dir='./storage/docker-generateconfig'" +install -d ./storage/docker-generateconfig +cd ./storage/docker-generateconfig + +# generate networkId +if [[ -s .networkId ]]; then + echo "INFO: saved networkId found, skipping" +else + echo "INFO: saved networkId not found, creating" + anyconf create-network + cat nodes.yml | grep '^networkId:' | awk '{print $NF}' > .networkId + cat account.yml | yq '.account.signingKey' > .networkSigningKey + + if [ $? -ne 0 ]; then + echo "ERROR: Failed network creations!" + exit 1 + fi +fi +NETWORK_ID=$( cat .networkId) +NETWORK_SIGNING_KEY=$( cat .networkSigningKey ) + +if [[ -s account0.yml ]]; then + echo "INFO: saved nodes and accounts configuration found, skipping" +else + echo "INFO: save nodes and accounts not found, createing" + anyconf generate-nodes \ + --t tree \ + --t tree \ + --t tree \ + --t coordinator \ + --t file \ + --t consensus \ + --addresses ${ANY_SYNC_NODE_1_ADDRESSES} \ + --addresses ${ANY_SYNC_NODE_2_ADDRESSES} \ + --addresses ${ANY_SYNC_NODE_3_ADDRESSES} \ + --addresses ${ANY_SYNC_COORDINATOR_ADDRESSES} \ + --addresses ${ANY_SYNC_FILENODE_ADDRESSES} \ + --addresses ${ANY_SYNC_CONSENSUSNODE_ADDRESSES} \ + + if [ $? -ne 0 ]; then + echo "ERROR: Failed to generate nodes and accounts!" + exit 1 + fi +fi + +echo "INFO: yq processing yml files" +yq --indent 2 --inplace 'del(.creationTime)' nodes.yml +yq --indent 2 --inplace ".networkId |= \"${NETWORK_ID}\"" nodes.yml +yq --indent 2 --inplace ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account3.yml +yq --indent 2 --inplace ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account5.yml + +echo "INFO: $0 done" +echo "INFO: starting nc as status service" +nc -lk -p 8000 -e /bin/cat diff --git a/docker-generateconfig/env-requirements.txt b/docker-generateconfig/env-requirements.txt new file mode 100644 index 0000000..2c24336 --- /dev/null +++ b/docker-generateconfig/env-requirements.txt @@ -0,0 +1 @@ +requests==2.31.0 diff --git a/docker-generateconfig/env.py b/docker-generateconfig/env.py new file mode 100755 index 0000000..e7fbbf7 --- /dev/null +++ b/docker-generateconfig/env.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import os +import json +import requests +import re + +cfg = { + 'inputFile': '.env.common', + 'overrideFile': '.env.override', + 'outputFile': '.env', + 'overrideVarMap': { + 'ANY_SYNC_NODE_VERSION': 'pkg::any-sync-node', + 'ANY_SYNC_FILENODE_VERSION': 'pkg::any-sync-filenode', + 'ANY_SYNC_COORDINATOR_VERSION': 'pkg::any-sync-coordinator', + 'ANY_SYNC_CONSENSUSNODE_VERSION': 'pkg::any-sync-consensusnode', + }, + 'versionsUrlMap': { + 'prod': 'https://puppetdoc.anytype.io/api/v1/prod-any-sync-compatible-versions/', + 'stage1': 'https://puppetdoc.anytype.io/api/v1/stage1-any-sync-compatible-versions/', + }, +} + +# load variables from inputFile +envVars = dict() +if os.path.exists(cfg['inputFile']) and os.path.getsize(cfg['inputFile']) > 0: + with open(cfg['inputFile']) as file: + for line in file: + if line.startswith('#') or not line.strip(): + continue + key, value = line.strip().split('=', 1) + if key in envVars: + print(f"WARNING: dublicate key={key} in env file={cfg['inputFile']}") + envVars[key] = value +else: + print(f"ERROR: file={cfg['inputFile']} not found or size=0") + exit(1) + +# override variables from overrideFile +if os.path.exists(cfg['overrideFile']) and os.path.getsize(cfg['overrideFile']) > 0: + with open(cfg['overrideFile']) as file: + for line in file: + if line.startswith('#') or not line.strip(): + continue + key, value = line.strip().split('=', 1) + envVars[key] = value + +# api request +def apiRequest(url): + try: + response = requests.get(url, timeout=(3.05, 5)) + jsonResponse = response.json() + except Exception as e: + print(f"failed response url={url}, error={str(e)}") + exit(1) + if response.status_code != 200: + print(f"failed response url={url}, status_code={response.status_code}, text={response.text}") + exit(1) + return jsonResponse + +# get latest version +def getLatestVersions(role): + versions = apiRequest(cfg['versionsUrlMap'][role]) + sortedVersions = dict(sorted(versions.items(), key=lambda x: int(x[0]))) + lastVersionsTimestamp, lastVersions = next(reversed(sortedVersions.items())) + return lastVersions + +# process variables +for key,value in envVars.items(): + if key in cfg['overrideVarMap'].keys(): + if value in cfg['versionsUrlMap'].keys(): + latestVersions = getLatestVersions(value) + lastVersionKey = cfg['overrideVarMap'].get(key) + lastVersionValue = latestVersions.get(lastVersionKey) + if lastVersionKey and lastVersionValue: + envVars[key] = 'v'+str(lastVersionValue) + +# save in output file +with open(cfg['outputFile'], 'w') as file: + for key, value in envVars.items(): + file.write(f"{key}={value}\n") diff --git a/docker-generateconfig/generate_config.sh b/docker-generateconfig/generate_config.sh deleted file mode 100755 index ce35464..0000000 --- a/docker-generateconfig/generate_config.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -source .env - -# generate networkId -if ! [[ -s .networkId ]]; then - anyconf create-network - echo "Create network" - cat nodes.yml | grep '^networkId:' | awk '{print $NF}' > .networkId - cat account.yml | yq '.account.signingKey' > .networkSigningKey - - if [ $? -ne 0 ]; then - echo "Failed network creations!" - exit 1 - fi -fi -NETWORK_ID=$( cat .networkId) -NETWORK_SIGNING_KEY=$( cat .networkSigningKey ) - -if ! [[ -s account0.yml ]]; then - echo "Generate nodes and accounts" - anyconf generate-nodes \ - --t tree \ - --t tree \ - --t tree \ - --t coordinator \ - --t file \ - --t consensus \ - --addresses ${ANY_SYNC_NODE_1_ADDRESSES} \ - --addresses ${ANY_SYNC_NODE_2_ADDRESSES} \ - --addresses ${ANY_SYNC_NODE_3_ADDRESSES} \ - --addresses ${ANY_SYNC_COORDINATOR_ADDRESSES} \ - --addresses ${ANY_SYNC_FILENODE_ADDRESSES} \ - --addresses ${ANY_SYNC_CONSENSUSNODE_ADDRESSES} \ - - if [ $? -ne 0 ]; then - echo "Failed to generate nodes and accounts!" - exit 1 - fi -fi - -yq --indent 4 --inplace 'del(.creationTime)' nodes.yml -yq --indent 4 --inplace ".networkId |= \"${NETWORK_ID}\"" nodes.yml -yq --indent 4 --inplace ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account3.yml -yq --indent 4 --inplace ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account5.yml diff --git a/docker-generateconfig/processing.sh b/docker-generateconfig/processing.sh index 2d23b97..665b340 100755 --- a/docker-generateconfig/processing.sh +++ b/docker-generateconfig/processing.sh @@ -1,54 +1,73 @@ #!/bin/bash -source generateconfig/.env +echo "INFO: $0 start" +echo "INFO: loading .env file" +source .env # Set file paths -DEST_PATH="etc" +DEST_PATH="./etc" NETWORK_FILE="${DEST_PATH}/network.yml" -# Create directories for all node types +echo "INFO: Create directories for all node types" for NODE_TYPE in node-1 node-2 node-3 filenode coordinator consensusnode admin; do mkdir -p "${DEST_PATH}/any-sync-${NODE_TYPE}" done -# Create directory for aws credentials +echo "INFO: Create directory for aws credentials" mkdir -p "${DEST_PATH}/.aws" -# add external listen host -./setListenIp.py "${EXTERNAL_LISTEN_HOST}" "generateconfig/nodes.yml" +echo "INFO: Configure external listen host" +./docker-generateconfig/setListenIp.py "./storage/docker-generateconfig/nodes.yml" ${EXTERNAL_LISTEN_HOST} ${EXTERNAL_LISTEN_HOSTS} -# create config for clients -cp "generateconfig/nodes.yml" "${DEST_PATH}/client.yml" +echo "INFO: Create config for clients" +cp "./storage/docker-generateconfig/nodes.yml" "${DEST_PATH}/client.yml" -# Generate network file -sed 's|^| |; 1s|^|network:\n|' "generateconfig/nodes.yml" > "${NETWORK_FILE}" +echo "INFO: Generate network file" +#sed 's|^| |; 1s|^|network:\n|' "generateconfig/nodes.yml" > "${NETWORK_FILE}" +yq eval '. as $item | {"network": $item}' --indent 2 ./storage/docker-generateconfig/nodes.yml > "${NETWORK_FILE}" -# Generate config files for 3 nodes +echo "INFO: Generate config files for 3 nodes" for i in {0..2}; do - cat "${NETWORK_FILE}" tmp-etc/common.yml generateconfig/account${i}.yml tmp-etc/node-$((i+1)).yml > "${DEST_PATH}/any-sync-node-$((i+1))/config.yml" + cat \ + "${NETWORK_FILE}" \ + docker-generateconfig/etc/common.yml \ + storage/docker-generateconfig/account${i}.yml \ + docker-generateconfig/etc/node-$((i+1)).yml \ + > "${DEST_PATH}/any-sync-node-$((i+1))/config.yml" done -# Generate config files for coordinator, filenode, consensusnode -cat "${NETWORK_FILE}" tmp-etc/common.yml generateconfig/account3.yml tmp-etc/coordinator.yml > ${DEST_PATH}/any-sync-coordinator/config.yml -cat "${NETWORK_FILE}" tmp-etc/common.yml generateconfig/account4.yml tmp-etc/filenode.yml > ${DEST_PATH}/any-sync-filenode/config.yml -cat "${NETWORK_FILE}" tmp-etc/common.yml generateconfig/account5.yml tmp-etc/consensusnode.yml > ${DEST_PATH}/any-sync-consensusnode/config.yml +echo "INFO: Generate config files for coordinator" +cat "${NETWORK_FILE}" docker-generateconfig/etc/common.yml storage/docker-generateconfig/account3.yml docker-generateconfig/etc/coordinator.yml \ + > ${DEST_PATH}/any-sync-coordinator/config.yml +echo "INFO: Generate config files for filenode" +cat "${NETWORK_FILE}" docker-generateconfig/etc/common.yml storage/docker-generateconfig/account4.yml docker-generateconfig/etc/filenode.yml \ + > ${DEST_PATH}/any-sync-filenode/config.yml +echo "INFO: Generate config files for consensusnode" +cat "${NETWORK_FILE}" docker-generateconfig/etc/common.yml storage/docker-generateconfig/account5.yml docker-generateconfig/etc/consensusnode.yml \ + > ${DEST_PATH}/any-sync-consensusnode/config.yml -# Copy network file to coordinator directory -cp "generateconfig/nodes.yml" "${DEST_PATH}/any-sync-coordinator/network.yml" +echo "INFO: Copy network file to coordinator directory" +cp "storage/docker-generateconfig/nodes.yml" "${DEST_PATH}/any-sync-coordinator/network.yml" -# Generate any-sync-admin config -cp "tmp-etc/admin.yml" ${DEST_PATH}/any-sync-admin/config.yml +echo "INFO: Copy any-sync-admin config" +cp "docker-generateconfig/etc/admin.yml" "${DEST_PATH}/any-sync-admin/config.yml" -# Generate aws credentials -cp "tmp-etc/aws-credentials" ${DEST_PATH}/.aws/credentials +echo "INFO: Copy aws credentials config" +cp "docker-generateconfig/etc/aws-credentials" "${DEST_PATH}/.aws/credentials" -# Replace variables from .env file -for PLACEHOLDER in $( perl -ne 'print "$1\n" if /^([A-z0-9_-]+)=/' generateconfig/.env ); do +echo "INFO: Replace variables from .env file" +for PLACEHOLDER in $( perl -ne 'print "$1\n" if /^([A-z0-9_-]+)=/' .env ); do perl -i -pe "s|%${PLACEHOLDER}%|${!PLACEHOLDER}|g" \ "${DEST_PATH}/"/.aws/credentials \ "${NETWORK_FILE}" \ "${DEST_PATH}/"/*/*.yml done -# save generated configs -rsync -a --delete generateconfig/ /opt/processing/docker-generateconfig +echo "INFO: fix indent in yml files" +for FILE in $( find ${DEST_PATH}/ -name "*.yml" ); do + yq --inplace --indent=2 $FILE +done + +echo "INFO: $0 done" +echo "INFO: starting nc as status service" +nc -lk -p 8000 -e /bin/cat diff --git a/docker-generateconfig/setListenIp.py b/docker-generateconfig/setListenIp.py index 6ce6502..f94b4ac 100755 --- a/docker-generateconfig/setListenIp.py +++ b/docker-generateconfig/setListenIp.py @@ -4,8 +4,9 @@ import sys import yaml -listenHost = sys.argv[1] -yamlFile = sys.argv[2] +arguments = sys.argv[1:] +yamlFile = arguments[0] +listenHosts = arguments[1:] with open(yamlFile, 'r') as file: config = yaml.load(file,Loader=yaml.Loader) @@ -13,9 +14,10 @@ with open(yamlFile, 'r') as file: for index, nodes in enumerate(config['nodes']): addresses = nodes['addresses'] port = addresses[0].split(':')[1] - listenAddress = listenHost +':'+ port - if listenAddress not in addresses: - addresses.append(listenAddress) + for listenHost in listenHosts: + listenAddress = listenHost +':'+ port + if listenAddress not in addresses: + addresses.append(listenAddress) with open(yamlFile, 'w') as file: yaml.dump(config, file)