added support prod,stage1 versions for ghcr docker images

.env.common

@@ -4,17 +4,17 @@ MINIO_BUCKET=minio-bucket
 MINIO_PORT=9000
 MINIO_WEB_PORT=9001
 
-ANY_SYNC_NODE_VERSION=v0.3.25
+ANY_SYNC_NODE_VERSION=prod
-ANY_SYNC_FILENODE_VERSION=v0.6.1
+ANY_SYNC_FILENODE_VERSION=prod
-ANY_SYNC_COORDINATOR_VERSION=v0.3.17
+ANY_SYNC_COORDINATOR_VERSION=prod
-ANY_SYNC_CONSENSUSNODE_VERSION=v0.1.5
+ANY_SYNC_CONSENSUSNODE_VERSION=prod
 ANY_SYNC_ADMIN_VERSION=latest
 
 MONGO_VERSION=7.0.2
 REDIS_VERSION=7.2.0-v6
 MINIO_VERSION=RELEASE.2024-01-16T16-07-38Z
 
-EXTERNAL_LISTEN_HOST="127.0.0.1"
+EXTERNAL_LISTEN_HOSTS="127.0.0.1"
 
 ANY_SYNC_NODE_1_HOST=any-sync-node-1
 ANY_SYNC_NODE_1_PORT=1001

.env.override.latest

@@ -0,0 +1,4 @@
+ANY_SYNC_NODE_VERSION=latest
+ANY_SYNC_FILENODE_VERSION=latest
+ANY_SYNC_COORDINATOR_VERSION=latest
+ANY_SYNC_CONSENSUSNODE_VERSION=latest

.env.override.prod

@@ -0,0 +1,4 @@
+ANY_SYNC_NODE_VERSION=prod
+ANY_SYNC_FILENODE_VERSION=prod
+ANY_SYNC_COORDINATOR_VERSION=prod
+ANY_SYNC_CONSENSUSNODE_VERSION=prod

.env.override.stage1

@@ -0,0 +1,4 @@
+ANY_SYNC_NODE_VERSION=stage1
+ANY_SYNC_FILENODE_VERSION=stage1
+ANY_SYNC_COORDINATOR_VERSION=stage1
+ANY_SYNC_CONSENSUSNODE_VERSION=stage1

.gitignore

@@ -1,4 +1,7 @@
+.DS_Store
 /etc/
 /storage/
 /docker-compose.override.yml
 /repos/
+/.env
+/.env.override

Dockerfile-generateconfig

@@ -1,17 +0,0 @@
-FROM golang:1.21.6-alpine3.19 as generator
-RUN apk add --no-cache bash yq
-RUN go install github.com/anyproto/any-sync-tools/anyconf@latest
-WORKDIR /opt/generateconfig
-COPY ./storage/docker-generateconfig/ .
-COPY --chmod=777 docker-generateconfig/generate_config.sh .env .
-RUN ./generate_config.sh
-
-FROM alpine:3.18.4
-RUN apk add --no-cache bash yq perl python3 py3-yaml rsync
-WORKDIR /opt/processing
-COPY docker-generateconfig/etc/ tmp-etc/
-COPY --chmod=777 docker-generateconfig/processing.sh .
-COPY --chmod=777 docker-generateconfig/setListenIp.py .
-COPY --from=generator /opt/generateconfig/ generateconfig/
-
-CMD ./processing.sh

Dockerfile-generateconfig-anyconf

@@ -0,0 +1,12 @@
+# syntax=docker/dockerfile:1
+FROM golang:1.21.6-alpine3.19
+RUN apk add --no-cache bash yq
+RUN go install github.com/anyproto/any-sync-tools/anyconf@latest
+WORKDIR /code
+HEALTHCHECK \
+    --start-period=3s \
+    --retries=20 \
+    --interval=10s \
+    --timeout=3s \
+    CMD echo 'HEALTHCHECK' | nc 127.0.0.1 8000 | grep -q HEALTHCHECK
+ENTRYPOINT ./docker-generateconfig/anyconf.sh

Dockerfile-generateconfig-env

@@ -0,0 +1,6 @@
+# syntax=docker/dockerfile:1
+FROM python:3.11-alpine
+WORKDIR /code
+COPY docker-generateconfig/env-requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+ENTRYPOINT ./docker-generateconfig/env.py

Dockerfile-generateconfig-processing

@@ -0,0 +1,11 @@
+# syntax=docker/dockerfile:1
+FROM alpine:3.18.4
+RUN apk add --no-cache bash yq perl python3 py3-yaml
+WORKDIR /code
+HEALTHCHECK \
+    --start-period=3s \
+    --retries=10 \
+    --interval=5s \
+    --timeout=3s \
+    CMD echo 'HEALTHCHECK' | nc 127.0.0.1 8000 | grep -q HEALTHCHECK
+ENTRYPOINT ./docker-generateconfig/processing.sh

Makefile

@@ -1,15 +1,12 @@
 .DEFAULT_GOAL := start
-include .env
 
-generate_config:
+generate_env:
-	mkdir -p ./storage/docker-generateconfig/
+	docker buildx build --tag generateconfig-env --file Dockerfile-generateconfig-env .
-	docker build -t generateconfig -f Dockerfile-generateconfig .
 	docker run --rm \
-		--volume ${CURDIR}/etc:/opt/processing/etc \
+		--volume ${CURDIR}/:/code/ \
-		--volume ${CURDIR}/storage/docker-generateconfig:/opt/processing/docker-generateconfig \
+		generateconfig-env
-		--name any-sync-generator generateconfig
 
-start: generate_config
+start: generate_env
 	docker compose up -d
 	@echo "Done! Upload your self-hosted network configuration file ${CURDIR}/etc/client.yml into the client app"
 	@echo "See: https://doc.anytype.io/anytype-docs/data-and-security/self-hosting#switching-between-networks"

README.md

@@ -12,6 +12,7 @@ Self-host for any-sync, designed for personal usage or for review and testing pu
 - [Prepare](#prepare)
 - [Usage](#usage)
 - [Configuration](#configuration)
+- [Troubleshooting](#troubleshooting)
 - [Compatible versions](#compatible-versions)
 - [Local build](#local-build)
 - [Limits web admin](#limits-web-admin)
@@ -36,8 +37,8 @@ Self-host for any-sync, designed for personal usage or for review and testing pu
   git config --global core.autocrlf false
 
   # Generate config
-  docker build -t generateconfig -f Dockerfile-generateconfig .
+  docker buildx build --tag generateconfig-env --file Dockerfile-generateconfig-env .
-  docker run --rm -v ${PWD}/etc:/opt/processing/etc --name any-sync-generator generateconfig
+  docker run --rm --volume ${PWD}/:/code/ generateconfig-env
   # Run containers
   docker compose up -d
   ```
@@ -96,10 +97,38 @@ Self-host for any-sync, designed for personal usage or for review and testing pu
   ANYTYPE_LOG_LEVEL="*=DEBUG" ANYPROF=:6060 ANY_SYNC_NETWORK=$(pwd)/etc/client.yml /Applications/Anytype.app/Contents/MacOS/Anytype
   ```
 
-## configuration
+## Configuration
-Use file .env
+> [!WARNING]
-* Set specific versions: find and edit variables with suffix "_VERSION"
+> The .env file is generated automatically.
-* Set external listen host: default 127.0.0.1, for change you need edit variable "EXTERNAL_LISTEN_HOST"
+It is based on the .env.common file, which is overridden or extended by variables from the .env.override file.
+### Version control
+By default, we use "prod" image version for any-sync-* daemons.  
+Also you can use "stage1" or "latest" verions:
+```
+# for use stage1 version
+ln -F -s .env.override.stage1 .env.override
+# for use latest version
+ln -F -s .env.override.latest .env.override
+```
+### external listen host
+By default, we use only the listen address 127.0.0.1, which is sufficient for running tests and a local client.  
+If you need to connect external clients, please add "EXTERNAL_LISTEN_HOSTS" in .env.override file.  
+Use spaces separation, multiline is not supported. For example:
+```
+EXTERNAL_LISTEN_HOSTS=<yourExternalIp1> <yourExternalIp2 ...
+```
+
+## Troubleshooting & FAQ
+### mongo replica set error
+if you have mongo replica set error like this:
+```
+test> rs.status()
+MongoServerError: Our replica set config is invalid or we are not a member of it
+```
+please run command:
+```
+docker compose exec mongo-1 mongosh --port 27001 --eval 'rs.reconfig({_id: rs.conf()._id, members: [{ _id: 0, host: "mongo-1:27001" }]}, {force: true});'
+```
 
 ## Compatible versions
 You can find compatible versions on these pages:
@@ -122,6 +151,8 @@ If you need to create local build binaries for any-sync-*, you can do so by usin
   make restart
   ```
 
+ln -F -s .env.override.stage1 .env.override
+
 ## Limits web admin
 open link in browser: http://127.0.0.1:80
 

UPGRADE.md

@@ -37,3 +37,8 @@ After Upgrade please run:
 ```
 docker compose exec mongo-1 mongosh --port 27001 --eval 'rs.reconfig({_id: rs.conf()._id, members: [{ _id: 0, host: "mongo-1:27001" }]}, {force: true});'
 ```
+
+## Upgrading from v3.x.x to v4.x.x
+1. You need to transfer your modifications from the .env file to the .env.override file.
+2. The variable EXTERNAL_LISTEN_HOST is deprecated.
+   Please, use the variable EXTERNAL_LISTEN_HOSTS instead.

docker-compose.yml

@@ -1,7 +1,31 @@
 version: "3.9"
 
 services:
+  # generate configs using the anyconf utility
+  generateconfig-anyconf:
+    image: generateconfig-anyconf
+    build:
+      context: .
+      dockerfile: Dockerfile-generateconfig-anyconf
+    volumes:
+      - ./:/code
+
+  # processing any-sync-* configs
+  generateconfig-processing:
+    depends_on:
+      generateconfig-anyconf:
+        condition: service_healthy
+    image: generateconfig-processing
+    build:
+      context: .
+      dockerfile: Dockerfile-generateconfig-processing
+    volumes:
+      - ./:/code
+
   mongo-1:
+    depends_on:
+      generateconfig-processing:
+        condition: service_healthy
     image: "mongo:${MONGO_VERSION}"
     command: --replSet ${MONGO_REPLICA_SET} --port ${MONGO_1_PORT}
     ports:
@@ -9,11 +33,14 @@ services:
     volumes:
       - ./storage/mongo-1/:/data/db
     healthcheck:
-      test: echo "try { rs.status() } catch (err) { rs.initiate() }" | mongosh --port ${MONGO_1_PORT} --quiet
+      test: test $$(mongosh --port ${MONGO_1_PORT} --quiet --eval "try {rs.initiate({_id:'${MONGO_REPLICA_SET}',members:[{_id:0,host:\"mongo-1:${MONGO_1_PORT}\"}]})} catch(e) {rs.status().ok}") -eq 1
       interval: 10s
       start_period: 30s
 
   redis:
+    depends_on:
+      generateconfig-processing:
+        condition: service_healthy
     image: "redis/redis-stack-server:${REDIS_VERSION}"
     restart: always
     command: redis-server --dir /data/ --appendonly yes --maxmemory 256mb --maxmemory-policy noeviction --protected-mode no --loadmodule /opt/redis-stack/lib/redisbloom.so
@@ -64,16 +91,24 @@ services:
   any-sync-coordinator_bootstrap:
     image: "ghcr.io/anyproto/any-sync-coordinator:${ANY_SYNC_COORDINATOR_VERSION}"
     depends_on:
-      - mongo-1
+      generateconfig-processing:
+        condition: service_healthy
+      mongo-1:
+        condition: service_healthy
     volumes:
       - ./etc/any-sync-coordinator/:/etc/any-sync-coordinator/
     restart: no
-    command: bash -c "sleep 10; /bin/any-sync-confapply -c /etc/any-sync-coordinator/config.yml -n /etc/any-sync-coordinator/network.yml -e"
+    command: bash -c "/bin/any-sync-confapply -c /etc/any-sync-coordinator/config.yml -n /etc/any-sync-coordinator/network.yml -e"
 
   any-sync-coordinator:
     image: "ghcr.io/anyproto/any-sync-coordinator:${ANY_SYNC_COORDINATOR_VERSION}"
     depends_on:
-      - any-sync-coordinator_bootstrap
+      generateconfig-processing:
+        condition: service_healthy
+      mongo-1:
+        condition: service_healthy
+      any-sync-coordinator_bootstrap:
+        condition: service_completed_successfully
     ports:
       - "${ANY_SYNC_COORDINATOR_PORT}:${ANY_SYNC_COORDINATOR_PORT}"
       - "${ANY_SYNC_COORDINATOR_QUIC_PORT}:${ANY_SYNC_COORDINATOR_QUIC_PORT}"
@@ -86,14 +121,16 @@ services:
         limits:
           memory: 500M
     restart: on-failure
-    command: bash -c "sleep 10; /bin/any-sync-coordinator -c /etc/any-sync-coordinator/config.yml"
 
   any-sync-filenode:
     image: "ghcr.io/anyproto/any-sync-filenode:${ANY_SYNC_FILENODE_VERSION}"
     depends_on:
-      - redis
+      redis:
-      - minio
+        condition: service_healthy
-      - any-sync-coordinator
+      minio:
+        condition: service_healthy
+      any-sync-coordinator:
+        condition: service_started
     ports:
       - "${ANY_SYNC_FILENODE_PORT}:${ANY_SYNC_FILENODE_PORT}"
       - "${ANY_SYNC_FILENODE_QUIC_PORT}:${ANY_SYNC_FILENODE_QUIC_PORT}"
@@ -107,12 +144,12 @@ services:
         limits:
           memory: 500M
     restart: on-failure
-    command: bash -c "sleep 15; /bin/any-sync-filenode -c /etc/any-sync-filenode/config.yml"
 
   any-sync-node-1:
     image: "ghcr.io/anyproto/any-sync-node:${ANY_SYNC_NODE_VERSION}"
     depends_on:
-      - any-sync-coordinator
+      any-sync-coordinator:
+        condition: service_started
     ports:
       - "${ANY_SYNC_NODE_1_PORT}:${ANY_SYNC_NODE_1_PORT}"
       - "${ANY_SYNC_NODE_1_QUIC_PORT}:${ANY_SYNC_NODE_1_QUIC_PORT}"
@@ -127,12 +164,12 @@ services:
         limits:
           memory: 500M
     restart: on-failure
-    command: bash -c "sleep 15; /bin/any-sync-node -c /etc/any-sync-node/config.yml"
 
   any-sync-node-2:
     image: "ghcr.io/anyproto/any-sync-node:${ANY_SYNC_NODE_VERSION}"
     depends_on:
-      - any-sync-coordinator
+      any-sync-coordinator:
+        condition: service_started
     ports:
       - "${ANY_SYNC_NODE_2_PORT}:${ANY_SYNC_NODE_2_PORT}"
       - "${ANY_SYNC_NODE_2_QUIC_PORT}:${ANY_SYNC_NODE_2_QUIC_PORT}"
@@ -147,12 +184,12 @@ services:
         limits:
           memory: 500M
     restart: on-failure
-    command: bash -c "sleep 15; /bin/any-sync-node -c /etc/any-sync-node/config.yml"
 
   any-sync-node-3:
     image: "ghcr.io/anyproto/any-sync-node:${ANY_SYNC_NODE_VERSION}"
     depends_on:
-      - any-sync-coordinator
+      any-sync-coordinator:
+        condition: service_started
     ports:
       - "${ANY_SYNC_NODE_3_PORT}:${ANY_SYNC_NODE_3_PORT}"
       - "${ANY_SYNC_NODE_3_QUIC_PORT}:${ANY_SYNC_NODE_3_QUIC_PORT}"
@@ -167,12 +204,12 @@ services:
         limits:
           memory: 500M
     restart: on-failure
-    command: bash -c "sleep 15; /bin/any-sync-node -c /etc/any-sync-node/config.yml"
 
   any-sync-consensusnode:
     image: "ghcr.io/anyproto/any-sync-consensusnode:${ANY_SYNC_CONSENSUSNODE_VERSION}"
     depends_on:
-      - any-sync-coordinator
+      any-sync-coordinator:
+        condition: service_started
     ports:
       - "${ANY_SYNC_CONSENSUSNODE_PORT}:${ANY_SYNC_CONSENSUSNODE_PORT}"
       - "${ANY_SYNC_CONSENSUSNODE_QUIC_PORT}:${ANY_SYNC_CONSENSUSNODE_QUIC_PORT}"
@@ -185,7 +222,6 @@ services:
         limits:
           memory: 500M
     restart: on-failure
-    command: bash -c "sleep 15; /bin/any-sync-consensusnode -c /etc/any-sync-consensusnode/config.yml"
 
   any-sync-admin:
     image: "ghcr.io/anyproto/any-sync-admin:${ANY_SYNC_ADMIN_VERSION}"

docker-generateconfig/anyconf.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+echo "INFO: $0 start"
+echo "INFO: loading .env file"
+source .env
+
+echo "INFO: create persistent config dir='./storage/docker-generateconfig'"
+install -d ./storage/docker-generateconfig
+cd ./storage/docker-generateconfig
+
+# generate networkId
+if [[ -s .networkId ]]; then
+    echo "INFO: saved networkId found, skipping"
+else
+    echo "INFO: saved networkId not found, creating"
+    anyconf create-network
+    cat nodes.yml | grep '^networkId:' | awk '{print $NF}' > .networkId
+    cat account.yml | yq '.account.signingKey' > .networkSigningKey
+
+    if [ $? -ne 0 ]; then
+        echo "ERROR: Failed network creations!"
+        exit 1
+    fi
+fi
+NETWORK_ID=$( cat .networkId)
+NETWORK_SIGNING_KEY=$( cat .networkSigningKey )
+
+if [[ -s account0.yml ]]; then
+    echo "INFO: saved nodes and accounts configuration found, skipping"
+else
+    echo "INFO: save nodes and accounts not found, createing"
+    anyconf generate-nodes \
+        --t tree \
+        --t tree \
+        --t tree \
+        --t coordinator \
+        --t file \
+        --t consensus \
+        --addresses ${ANY_SYNC_NODE_1_ADDRESSES} \
+        --addresses ${ANY_SYNC_NODE_2_ADDRESSES} \
+        --addresses ${ANY_SYNC_NODE_3_ADDRESSES} \
+        --addresses ${ANY_SYNC_COORDINATOR_ADDRESSES} \
+        --addresses ${ANY_SYNC_FILENODE_ADDRESSES} \
+        --addresses ${ANY_SYNC_CONSENSUSNODE_ADDRESSES} \
+
+    if [ $? -ne 0 ]; then
+        echo "ERROR: Failed to generate nodes and accounts!"
+        exit 1
+    fi
+fi
+
+echo "INFO: yq processing yml files"
+yq --indent 2 --inplace 'del(.creationTime)' nodes.yml
+yq --indent 2 --inplace ".networkId |= \"${NETWORK_ID}\"" nodes.yml
+yq --indent 2 --inplace ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account3.yml
+yq --indent 2 --inplace ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account5.yml
+
+echo "INFO: $0 done"
+echo "INFO: starting nc as status service"
+nc -lk -p 8000 -e /bin/cat

docker-generateconfig/env-requirements.txt

@@ -0,0 +1 @@
+requests==2.31.0

docker-generateconfig/env.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import os
+import json
+import requests
+import re
+
+cfg = {
+    'inputFile': '.env.common',
+    'overrideFile': '.env.override',
+    'outputFile': '.env',
+    'overrideVarMap': {
+        'ANY_SYNC_NODE_VERSION': 'pkg::any-sync-node',
+        'ANY_SYNC_FILENODE_VERSION': 'pkg::any-sync-filenode',
+        'ANY_SYNC_COORDINATOR_VERSION': 'pkg::any-sync-coordinator',
+        'ANY_SYNC_CONSENSUSNODE_VERSION': 'pkg::any-sync-consensusnode',
+    },
+    'versionsUrlMap': {
+        'prod': 'https://puppetdoc.anytype.io/api/v1/prod-any-sync-compatible-versions/',
+        'stage1': 'https://puppetdoc.anytype.io/api/v1/stage1-any-sync-compatible-versions/',
+    },
+}
+
+# load variables from inputFile
+envVars = dict()
+if os.path.exists(cfg['inputFile']) and os.path.getsize(cfg['inputFile']) > 0:
+    with open(cfg['inputFile']) as file:
+        for line in file:
+            if line.startswith('#') or not line.strip():
+                continue
+            key, value = line.strip().split('=', 1)
+            if key in envVars:
+                print(f"WARNING: dublicate key={key} in env file={cfg['inputFile']}")
+            envVars[key] = value
+else:
+    print(f"ERROR: file={cfg['inputFile']} not found or size=0")
+    exit(1)
+
+# override variables from overrideFile
+if os.path.exists(cfg['overrideFile']) and os.path.getsize(cfg['overrideFile']) > 0:
+    with open(cfg['overrideFile']) as file:
+        for line in file:
+            if line.startswith('#') or not line.strip():
+                continue
+            key, value = line.strip().split('=', 1)
+            envVars[key] = value
+
+# api request
+def apiRequest(url):
+    try:
+        response = requests.get(url, timeout=(3.05, 5))
+        jsonResponse = response.json()
+    except Exception as e:
+        print(f"failed response url={url}, error={str(e)}")
+        exit(1)
+    if response.status_code != 200:
+        print(f"failed response url={url}, status_code={response.status_code}, text={response.text}")
+        exit(1)
+    return jsonResponse
+
+# get latest version
+def getLatestVersions(role):
+    versions = apiRequest(cfg['versionsUrlMap'][role])
+    sortedVersions = dict(sorted(versions.items(), key=lambda x: int(x[0])))
+    lastVersionsTimestamp, lastVersions = next(reversed(sortedVersions.items()))
+    return lastVersions
+
+# process variables
+for key,value in envVars.items():
+    if key in cfg['overrideVarMap'].keys():
+        if value in cfg['versionsUrlMap'].keys():
+            latestVersions = getLatestVersions(value)
+            lastVersionKey = cfg['overrideVarMap'].get(key)
+            lastVersionValue = latestVersions.get(lastVersionKey)
+            if lastVersionKey and lastVersionValue:
+                envVars[key] = 'v'+str(lastVersionValue)
+
+# save in output file
+with open(cfg['outputFile'], 'w') as file:
+    for key, value in envVars.items():
+        file.write(f"{key}={value}\n")

docker-generateconfig/generate_config.sh

@@ -1,45 +0,0 @@
-#!/bin/bash
-
-source .env
-
-# generate networkId
-if ! [[ -s .networkId ]]; then
-    anyconf create-network
-    echo "Create network"
-    cat nodes.yml | grep '^networkId:' | awk '{print $NF}' > .networkId
-    cat account.yml | yq '.account.signingKey' > .networkSigningKey
-
-    if [ $? -ne 0 ]; then
-        echo "Failed network creations!"
-        exit 1
-    fi
-fi
-NETWORK_ID=$( cat .networkId)
-NETWORK_SIGNING_KEY=$( cat .networkSigningKey )
-
-if ! [[ -s account0.yml ]]; then
-    echo "Generate nodes and accounts"
-    anyconf generate-nodes \
-        --t tree \
-        --t tree \
-        --t tree \
-        --t coordinator \
-        --t file \
-        --t consensus \
-        --addresses ${ANY_SYNC_NODE_1_ADDRESSES} \
-        --addresses ${ANY_SYNC_NODE_2_ADDRESSES} \
-        --addresses ${ANY_SYNC_NODE_3_ADDRESSES} \
-        --addresses ${ANY_SYNC_COORDINATOR_ADDRESSES} \
-        --addresses ${ANY_SYNC_FILENODE_ADDRESSES} \
-        --addresses ${ANY_SYNC_CONSENSUSNODE_ADDRESSES} \
-
-    if [ $? -ne 0 ]; then
-        echo "Failed to generate nodes and accounts!"
-        exit 1
-    fi
-fi
-
-yq --indent 4 --inplace 'del(.creationTime)' nodes.yml
-yq --indent 4 --inplace ".networkId |= \"${NETWORK_ID}\"" nodes.yml
-yq --indent 4 --inplace ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account3.yml
-yq --indent 4 --inplace ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account5.yml

docker-generateconfig/processing.sh

@@ -1,54 +1,73 @@
 #!/bin/bash
 
-source generateconfig/.env
+echo "INFO: $0 start"
+echo "INFO: loading .env file"
+source .env
 
 # Set file paths
-DEST_PATH="etc"
+DEST_PATH="./etc"
 NETWORK_FILE="${DEST_PATH}/network.yml"
 
-# Create directories for all node types
+echo "INFO: Create directories for all node types"
 for NODE_TYPE in node-1 node-2 node-3 filenode coordinator consensusnode admin; do
     mkdir -p "${DEST_PATH}/any-sync-${NODE_TYPE}"
 done
 
-# Create directory for aws credentials
+echo "INFO: Create directory for aws credentials"
 mkdir -p "${DEST_PATH}/.aws"
 
-# add external listen host
+echo "INFO: Configure external listen host"
-./setListenIp.py "${EXTERNAL_LISTEN_HOST}" "generateconfig/nodes.yml"
+./docker-generateconfig/setListenIp.py "./storage/docker-generateconfig/nodes.yml" ${EXTERNAL_LISTEN_HOST} ${EXTERNAL_LISTEN_HOSTS}
 
-# create config for clients
+echo "INFO: Create config for clients"
-cp "generateconfig/nodes.yml" "${DEST_PATH}/client.yml"
+cp "./storage/docker-generateconfig/nodes.yml" "${DEST_PATH}/client.yml"
 
-# Generate network file
+echo "INFO: Generate network file"
-sed 's|^|    |; 1s|^|network:\n|' "generateconfig/nodes.yml" > "${NETWORK_FILE}"
+#sed 's|^|    |; 1s|^|network:\n|' "generateconfig/nodes.yml" > "${NETWORK_FILE}"
+yq eval '. as $item | {"network": $item}' --indent 2 ./storage/docker-generateconfig/nodes.yml > "${NETWORK_FILE}"
 
-# Generate config files for 3 nodes
+echo "INFO: Generate config files for 3 nodes"
 for i in {0..2}; do
-    cat "${NETWORK_FILE}" tmp-etc/common.yml generateconfig/account${i}.yml tmp-etc/node-$((i+1)).yml > "${DEST_PATH}/any-sync-node-$((i+1))/config.yml"
+    cat \
+        "${NETWORK_FILE}" \
+        docker-generateconfig/etc/common.yml \
+        storage/docker-generateconfig/account${i}.yml \
+        docker-generateconfig/etc/node-$((i+1)).yml \
+        > "${DEST_PATH}/any-sync-node-$((i+1))/config.yml"
 done
 
-# Generate config files for coordinator, filenode, consensusnode
+echo "INFO: Generate config files for coordinator"
-cat "${NETWORK_FILE}" tmp-etc/common.yml generateconfig/account3.yml tmp-etc/coordinator.yml > ${DEST_PATH}/any-sync-coordinator/config.yml
+cat "${NETWORK_FILE}" docker-generateconfig/etc/common.yml storage/docker-generateconfig/account3.yml docker-generateconfig/etc/coordinator.yml \
-cat "${NETWORK_FILE}" tmp-etc/common.yml generateconfig/account4.yml tmp-etc/filenode.yml > ${DEST_PATH}/any-sync-filenode/config.yml
+    > ${DEST_PATH}/any-sync-coordinator/config.yml
-cat "${NETWORK_FILE}" tmp-etc/common.yml generateconfig/account5.yml tmp-etc/consensusnode.yml > ${DEST_PATH}/any-sync-consensusnode/config.yml
+echo "INFO: Generate config files for filenode"
+cat "${NETWORK_FILE}" docker-generateconfig/etc/common.yml storage/docker-generateconfig/account4.yml docker-generateconfig/etc/filenode.yml \
+    > ${DEST_PATH}/any-sync-filenode/config.yml
+echo "INFO: Generate config files for consensusnode"
+cat "${NETWORK_FILE}" docker-generateconfig/etc/common.yml storage/docker-generateconfig/account5.yml docker-generateconfig/etc/consensusnode.yml \
+    > ${DEST_PATH}/any-sync-consensusnode/config.yml
 
-# Copy network file to coordinator directory
+echo "INFO: Copy network file to coordinator directory"
-cp "generateconfig/nodes.yml" "${DEST_PATH}/any-sync-coordinator/network.yml"
+cp "storage/docker-generateconfig/nodes.yml" "${DEST_PATH}/any-sync-coordinator/network.yml"
 
-# Generate any-sync-admin config
+echo "INFO: Copy any-sync-admin config"
-cp "tmp-etc/admin.yml" ${DEST_PATH}/any-sync-admin/config.yml
+cp "docker-generateconfig/etc/admin.yml" "${DEST_PATH}/any-sync-admin/config.yml"
 
-# Generate aws credentials
+echo "INFO: Copy aws credentials config"
-cp "tmp-etc/aws-credentials" ${DEST_PATH}/.aws/credentials
+cp "docker-generateconfig/etc/aws-credentials" "${DEST_PATH}/.aws/credentials"
 
-# Replace variables from .env file
+echo "INFO: Replace variables from .env file"
-for PLACEHOLDER in $( perl -ne 'print "$1\n" if /^([A-z0-9_-]+)=/' generateconfig/.env ); do
+for PLACEHOLDER in $( perl -ne 'print "$1\n" if /^([A-z0-9_-]+)=/' .env ); do
     perl -i -pe "s|%${PLACEHOLDER}%|${!PLACEHOLDER}|g" \
         "${DEST_PATH}/"/.aws/credentials \
         "${NETWORK_FILE}" \
         "${DEST_PATH}/"/*/*.yml
 done
 
-# save generated configs
+echo "INFO: fix indent in yml files"
-rsync -a --delete generateconfig/ /opt/processing/docker-generateconfig
+for FILE in $( find ${DEST_PATH}/ -name "*.yml" ); do
+    yq --inplace --indent=2 $FILE
+done
+
+echo "INFO: $0 done"
+echo "INFO: starting nc as status service"
+nc -lk -p 8000 -e /bin/cat

docker-generateconfig/setListenIp.py

@@ -4,8 +4,9 @@
 import sys
 import yaml
 
-listenHost = sys.argv[1]
+arguments = sys.argv[1:]
-yamlFile = sys.argv[2]
+yamlFile = arguments[0]
+listenHosts = arguments[1:]
 
 with open(yamlFile, 'r') as file:
     config = yaml.load(file,Loader=yaml.Loader)
@@ -13,6 +14,7 @@ with open(yamlFile, 'r') as file:
 for index, nodes in enumerate(config['nodes']):
     addresses = nodes['addresses']
     port = addresses[0].split(':')[1]
+    for listenHost in listenHosts:
         listenAddress = listenHost +':'+ port
         if listenAddress not in addresses:
             addresses.append(listenAddress)