Merge pull request #7 from anyproto/OPS-325

Compatibility support, improvements and fixes

.env

@@ -7,6 +7,10 @@ ANY_SYNC_FILENODE_VERSION=latest
 ANY_SYNC_COORDINATOR_VERSION=latest
 ANY_SYNC_CONSENSUSNODE_VERSION=latest
 
+MONGO_VERSION=7.0.2
+REDIS_VERSION=7.2.0-v6
+S3_EMULATOR_VERSION=0.0.1
+
 ANY_SYNC_NODE_1_HOST=any-sync-node-1
 ANY_SYNC_NODE_1_PORT=1001
 ANY_SYNC_NODE_1_ADDRESSES=${ANY_SYNC_NODE_1_HOST}:${ANY_SYNC_NODE_1_PORT}
@@ -48,4 +52,4 @@ MONGO_2_PORT=27002
 MONGO_3_PORT=27003
 MONGO_CONNECT=mongodb://mongo-1:${MONGO_1_PORT},mongo-2:${MONGO_2_PORT},mongo-3:${MONGO_3_PORT}
 REDIS_PORT=6379
-REDIS_URL=redis://redis:${REDIS_PORT}?dial_timeout=3&read_timeout=6s
+REDIS_URL="redis://redis:${REDIS_PORT}?dial_timeout=3&read_timeout=6s"

.gitignore

@@ -1,3 +1,4 @@
-tmp/
+etc/
+storage/
 repos/
 docker-compose.override.yml

Dockerfile-generateconfig

@@ -1,13 +1,18 @@
-FROM golang:1.19
+FROM golang:1.19 as generator
+RUN wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && \
+    chmod +x /usr/bin/yq && \
+    go install github.com/anyproto/any-sync-tools/anyconf@latest
 
-# git {{
+WORKDIR /opt/generateconfig
-RUN apt-get update && apt-get install -y ca-certificates git-core
+COPY --chmod=777 docker-generateconfig/generate_config.sh .env .
-# }}
+RUN ./generate_config.sh
 
-RUN wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq
+FROM alpine:3.18.4
+RUN apk add --no-cache bash perl
+WORKDIR /opt/processing
+COPY docker-generateconfig/etc/ tmp-etc/
+COPY --chmod=777 docker-generateconfig/processing.sh .
+COPY --from=generator /opt/generateconfig/ generateconfig/
 
-WORKDIR /generateconfig
+CMD ./processing.sh
-COPY .env /.env
-COPY gen.sh /gen.sh
 
-RUN go install github.com/anyproto/any-sync-tools/anyconf@latest

Makefile

@@ -1,46 +1,15 @@
 .DEFAULT_GOAL := start
-SHELL := /bin/bash
 include .env
 
-generate_etc:
+generate_config:
-	install -d \
+	docker build -t generateconfig -f Dockerfile-generateconfig .
-		tmp/etc/any-sync-node-{1..3}/ \
+	docker run --rm -v ${CURDIR}/etc:/opt/processing/etc --name any-sync-generator generateconfig
-		tmp/etc/any-sync-filenode/ \
-		tmp/etc/any-sync-coordinator/ \
-		tmp/etc/any-sync-consensusnode/
-	docker compose --file docker-compose-generateconfig.yml up --build --remove-orphans --detach
-	sleep 2
-	sed 's|^|    |; 1s|^|network:\n|' tmp/generateconfig/nodes.yml > tmp/etc/network.yml
-	cat tmp/etc/network.yml etc/common.yml tmp/generateconfig/account0.yml etc/node-1.yml > tmp/etc/any-sync-node-1/config.yml
-	cat tmp/etc/network.yml etc/common.yml tmp/generateconfig/account1.yml etc/node-2.yml > tmp/etc/any-sync-node-2/config.yml
-	cat tmp/etc/network.yml etc/common.yml tmp/generateconfig/account2.yml etc/node-3.yml > tmp/etc/any-sync-node-3/config.yml
-	cat tmp/etc/network.yml etc/common.yml tmp/generateconfig/account3.yml etc/coordinator.yml > tmp/etc/any-sync-coordinator/config.yml
-	cat tmp/etc/network.yml etc/common.yml tmp/generateconfig/account4.yml etc/filenode.yml > tmp/etc/any-sync-filenode/config.yml
-	cat tmp/etc/network.yml etc/common.yml tmp/generateconfig/account5.yml etc/consensusnode.yml > tmp/etc/any-sync-consensusnode/config.yml
-	cp tmp/generateconfig/nodes.yml tmp/etc/any-sync-coordinator/network.yml
-	perl -i -pe's|%ANY_SYNC_NODE_1_ADDRESSES%|${ANY_SYNC_NODE_1_ADDRESSES}|g' tmp/etc/network.yml tmp/etc/*/*.yml
-	perl -i -pe's|%ANY_SYNC_NODE_2_ADDRESSES%|${ANY_SYNC_NODE_2_ADDRESSES}|g' tmp/etc/network.yml tmp/etc/*/*.yml
-	perl -i -pe's|%ANY_SYNC_NODE_3_ADDRESSES%|${ANY_SYNC_NODE_3_ADDRESSES}|g' tmp/etc/network.yml tmp/etc/*/*.yml
-	perl -i -pe's|%ANY_SYNC_COORDINATOR_ADDRESSES%|${ANY_SYNC_COORDINATOR_ADDRESSES}|g' tmp/etc/network.yml tmp/etc/*/*.yml
-	perl -i -pe's|%ANY_SYNC_FILENODE_ADDRESSES%|${ANY_SYNC_FILENODE_ADDRESSES}|g' tmp/etc/network.yml tmp/etc/*/*.yml
-	perl -i -pe's|%ANY_SYNC_CONSENSUSNODE_ADDRESSES%|${ANY_SYNC_CONSENSUSNODE_ADDRESSES}|g' tmp/etc/network.yml tmp/etc/*/*.yml
-	perl -i -pe's|%ANY_SYNC_NODE_1_QUIC_ADDRESSES%|${ANY_SYNC_NODE_1_QUIC_ADDRESSES}|g' tmp/etc/network.yml tmp/etc/*/*.yml
-	perl -i -pe's|%ANY_SYNC_NODE_2_QUIC_ADDRESSES%|${ANY_SYNC_NODE_2_QUIC_ADDRESSES}|g' tmp/etc/network.yml tmp/etc/*/*.yml
-	perl -i -pe's|%ANY_SYNC_NODE_3_QUIC_ADDRESSES%|${ANY_SYNC_NODE_3_QUIC_ADDRESSES}|g' tmp/etc/network.yml tmp/etc/*/*.yml
-	perl -i -pe's|%ANY_SYNC_COORDINATOR_QUIC_ADDRESSES%|${ANY_SYNC_COORDINATOR_QUIC_ADDRESSES}|g' tmp/etc/network.yml tmp/etc/*/*.yml
-	perl -i -pe's|%ANY_SYNC_FILENODE_QUIC_ADDRESSES%|${ANY_SYNC_FILENODE_QUIC_ADDRESSES}|g' tmp/etc/network.yml tmp/etc/*/*.yml
-	perl -i -pe's|%ANY_SYNC_CONSENSUSNODE_QUIC_ADDRESSES%|${ANY_SYNC_CONSENSUSNODE_QUIC_ADDRESSES}|g' tmp/etc/network.yml tmp/etc/*/*.yml
-	perl -i -pe's|%MONGO_CONNECT%|${MONGO_CONNECT}|g' tmp/etc/network.yml tmp/etc/*/*.yml
-	perl -i -pe's|%REDIS_URL%|${REDIS_URL}|g' tmp/etc/network.yml tmp/etc/*/*.yml
-	perl -i -pe's|%AWS_PORT%|${AWS_PORT}|g' tmp/etc/network.yml tmp/etc/*/*.yml
-	docker compose --file docker-compose-generateconfig.yml stop
 
-start: generate_etc
+start: generate_config
-	docker compose up --force-recreate --build --remove-orphans --detach --pull always
+	docker compose up -d
 
 stop:
 	docker compose stop
-	docker compose --file docker-compose-generateconfig.yml stop
 
 clean:
 	docker system prune --all
@@ -50,15 +19,15 @@ pull:
 
 down:
 	docker compose down
-	docker compose --file docker-compose-generateconfig.yml down
 
 # build with "plain" log for debug
 build:
 	docker compose build --no-cache --progress plain
 
-restart: stop start
+restart: down start
-update: stop pull start
+update: down pull start
-upgrade: stop clean start
+upgrade: down clean start
 
-cleanTmp:
+cleanEtcStorage:
-	rm -rf tmp/
+	rm -rf etc/
+	rm -rf storage/

README.md

@@ -7,7 +7,7 @@ Self-host for any-sync, designed for review and testing purposes.
 * install docker and docker-compose https://docs.docker.com/compose/install/linux/
 
 ## Usage
-* start stand:
+* start stand - at the first run the directories `etc/` of configuration files and `storage/` for data storage will be generated:
   ```
   make start
   ```
@@ -27,9 +27,9 @@ Self-host for any-sync, designed for review and testing purposes.
   ```
   make clean
   ```
-* clean tmp files - deleting data for redis, mongo, s3, any-sync-*:
+* clean config and storage files - deleting data for redis, mongo, s3, any-sync-*:
   ```
-  make cleanTmp
+  make cleanEtcStorage
   ```
 * show logs:
   ```
@@ -37,13 +37,18 @@ Self-host for any-sync, designed for review and testing purposes.
   docker-compose logs -f any-sync-filenode
   docker-compose logs -f
   ```
-* attach to vm:
+* attach to container:
   ```
   docker compose exec mongo-1 bash
   docker compose exec any-sync-node-1 bash
   docker compose exec any-sync-coordinator bash
   ```
 
+* restart certain container:
+  ```
+  docker compose restart any-sync-node-1
+  ```
+
 * get current network config
   ```
   docker compose exec mongo-1 mongosh coordinator

docker-compose.yml

@@ -1,69 +1,78 @@
 version: "3.9"
+
 services:
   mongo-1:
-    image: mongo
+    image: "mongo:${MONGO_VERSION}"
     command: --replSet rs0 --port ${MONGO_1_PORT}
     ports:
       - "${MONGO_1_PORT}:${MONGO_1_PORT}"
     volumes:
-      - ./tmp/mongo-1/:/data/db
+      - ./storage/mongo-1/:/data/db
     healthcheck:
       test: test $$(echo "rs.initiate({_id:'rs0',members:[{_id:0,host:\"mongo-1:${MONGO_1_PORT}\"},{_id:1,host:\"mongo-2:${MONGO_2_PORT}\"},{_id:2,host:\"mongo-3:${MONGO_3_PORT}\"}]}).ok || rs.status().ok" | mongosh --port ${MONGO_1_PORT} --quiet) -eq 1
       interval: 10s
       start_period: 30s
+
   mongo-2:
-    image: mongo
+    image: "mongo:${MONGO_VERSION}"
     command: --replSet rs0 --port ${MONGO_2_PORT}
     ports:
       - "${MONGO_2_PORT}:${MONGO_2_PORT}"
     volumes:
-      - ./tmp/mongo-2/:/data/db
+      - ./storage/mongo-2/:/data/db
+
   mongo-3:
-    image: mongo
+    image: "mongo:${MONGO_VERSION}"
     command: --replSet rs0 --port ${MONGO_3_PORT}
     ports:
       - "${MONGO_3_PORT}:${MONGO_3_PORT}"
     volumes:
-      - ./tmp/mongo-3/:/data/db
+      - ./storage/mongo-3/:/data/db
+
   redis:
     container_name: redis
-    image: redis/redis-stack-server:latest
+    image: "redis/redis-stack-server:${REDIS_VERSION}"
     restart: always
     command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy noeviction --protected-mode no --loadmodule /opt/redis-stack/lib/redisbloom.so
     ports:
       - "${REDIS_PORT}:${REDIS_PORT}"
     volumes:
-      - ./tmp/redis/:/data/
+      - ./storage/redis/:/data/
+
   s3-emulator:
-    image: stanislavt/s3-emulator
+    image: "stanislavt/s3-emulator:${S3_EMULATOR_VERSION}"
     ports:
-      - "${AWS_PORT}:${AWS_PORT}"
+      - "${AWS_PORT}:4569"
     volumes:
-      - ./tmp/s3_root:/s3_root
+      - ./storage/s3_root:/s3_root
+
   any-sync-coordinator_bootstrap:
-    image: "ghcr.io/anyproto/any-sync-coordinator:${ANY_SYNC_COORDINATOR_VERSION}"
-    depends_on: [mongo-1]
-    restart: "no"
-    volumes:
-      - ./tmp/etc/any-sync-coordinator/:/etc/any-sync-coordinator/
-    command: bash -c "sleep 10; /bin/any-sync-confapply -c /etc/any-sync-coordinator/config.yml -n /etc/any-sync-coordinator/network.yml -e"
-  any-sync-coordinator:
     image: "ghcr.io/anyproto/any-sync-coordinator:${ANY_SYNC_COORDINATOR_VERSION}"
     depends_on:
       - mongo-1
+    volumes:
+      - ./etc/any-sync-coordinator/:/etc/any-sync-coordinator/
+    restart: no
+    command: bash -c "sleep 10; /bin/any-sync-confapply -c /etc/any-sync-coordinator/config.yml -n /etc/any-sync-coordinator/network.yml -e"
+
+  any-sync-coordinator:
+    image: "ghcr.io/anyproto/any-sync-coordinator:${ANY_SYNC_COORDINATOR_VERSION}"
+    depends_on:
       - any-sync-coordinator_bootstrap
     ports:
       - "${ANY_SYNC_COORDINATOR_PORT}:${ANY_SYNC_COORDINATOR_PORT}"
       - "${ANY_SYNC_COORDINATOR_QUIC_PORT}:${ANY_SYNC_COORDINATOR_QUIC_PORT}"
       - 8005:8000
     volumes:
-      - ./tmp/etc/any-sync-coordinator/:/etc/any-sync-coordinator/
+      - ./etc/any-sync-coordinator/:/etc/any-sync-coordinator/
-      - ./tmp/networkStore/any-sync-coordinator/:/networkStore/
+      - ./storage/networkStore/any-sync-coordinator/:/networkStore/
     deploy:
       resources:
         limits:
           memory: 500M
+    restart: on-failure
     command: bash -c "sleep 10; /bin/any-sync-coordinator -c /etc/any-sync-coordinator/config.yml"
+
   any-sync-filenode:
     image: "ghcr.io/anyproto/any-sync-filenode:${ANY_SYNC_FILENODE_VERSION}"
     depends_on:
@@ -75,14 +84,16 @@ services:
       - "${ANY_SYNC_FILENODE_QUIC_PORT}:${ANY_SYNC_FILENODE_QUIC_PORT}"
       - 8004:8000
     volumes:
-      - ./tmp/etc/any-sync-filenode/:/etc/any-sync-filenode/
+      - ./etc/any-sync-filenode/:/etc/any-sync-filenode/
       - ./.aws:/root/.aws:ro
-      - ./tmp/networkStore/any-sync-filenode/:/networkStore/
+      - ./storage/networkStore/any-sync-filenode/:/networkStore/
     deploy:
       resources:
         limits:
           memory: 500M
+    restart: on-failure
     command: bash -c "sleep 15; /bin/any-sync-filenode -c /etc/any-sync-filenode/config.yml"
+
   any-sync-node-1:
     image: "ghcr.io/anyproto/any-sync-node:${ANY_SYNC_NODE_VERSION}"
     depends_on:
@@ -93,14 +104,16 @@ services:
       - 8081:8080
       - 8001:8000
     volumes:
-      - ./tmp/etc/any-sync-node-1/:/etc/any-sync-node/
+      - ./etc/any-sync-node-1/:/etc/any-sync-node/
-      - ./tmp/storage/any-sync-node-1/:/storage/
+      - ./storage/any-sync-node-1/:/storage/
-      - ./tmp/networkStore/any-sync-node-1/:/networkStore/
+      - ./storage/networkStore/any-sync-node-1/:/networkStore/
     deploy:
       resources:
         limits:
           memory: 500M
+    restart: on-failure
     command: bash -c "sleep 15; /bin/any-sync-node -c /etc/any-sync-node/config.yml"
+
   any-sync-node-2:
     image: "ghcr.io/anyproto/any-sync-node:${ANY_SYNC_NODE_VERSION}"
     depends_on:
@@ -111,14 +124,16 @@ services:
       - 8082:8080
       - 8002:8000
     volumes:
-      - ./tmp/etc/any-sync-node-2/:/etc/any-sync-node/
+      - ./etc/any-sync-node-2/:/etc/any-sync-node/
-      - ./tmp/storage/any-sync-node-2/:/storage/
+      - ./storage/any-sync-node-2/:/storage/
-      - ./tmp/networkStore/any-sync-node-2/:/networkStore/
+      - ./storage/networkStore/any-sync-node-2/:/networkStore/
     deploy:
       resources:
         limits:
           memory: 500M
+    restart: on-failure
     command: bash -c "sleep 15; /bin/any-sync-node -c /etc/any-sync-node/config.yml"
+
   any-sync-node-3:
     image: "ghcr.io/anyproto/any-sync-node:${ANY_SYNC_NODE_VERSION}"
     depends_on:
@@ -129,14 +144,16 @@ services:
       - 8083:8080
       - 8003:8000
     volumes:
-      - ./tmp/etc/any-sync-node-3/:/etc/any-sync-node/
+      - ./etc/any-sync-node-3/:/etc/any-sync-node/
-      - ./tmp/storage/any-sync-node-3/:/storage/
+      - ./storage/any-sync-node-3/:/storage/
-      - ./tmp/networkStore/any-sync-node-3/:/networkStore/
+      - ./storage/networkStore/any-sync-node-3/:/networkStore/
     deploy:
       resources:
         limits:
           memory: 500M
+    restart: on-failure
     command: bash -c "sleep 15; /bin/any-sync-node -c /etc/any-sync-node/config.yml"
+
   any-sync-consensusnode:
     image: "ghcr.io/anyproto/any-sync-consensusnode:${ANY_SYNC_CONSENSUSNODE_VERSION}"
     depends_on:
@@ -146,10 +163,11 @@ services:
       - "${ANY_SYNC_CONSENSUSNODE_QUIC_PORT}:${ANY_SYNC_CONSENSUSNODE_QUIC_PORT}"
       - 8006:8000
     volumes:
-      - ./tmp/etc/any-sync-consensusnode/:/etc/any-sync-consensusnode/
+      - ./etc/any-sync-consensusnode/:/etc/any-sync-consensusnode/
-      - ./tmp/networkStore/any-sync-consensusnode/:/networkStore/
+      - ./storage/networkStore/any-sync-consensusnode/:/networkStore/
     deploy:
       resources:
         limits:
           memory: 500M
+    restart: on-failure
     command: bash -c "sleep 15; /bin/any-sync-consensusnode -c /etc/any-sync-consensusnode/config.yml"

docker-generateconfig/generate_config.sh

@@ -1,17 +1,24 @@
 #!/bin/bash
 
-source /.env
+source .env
 
 # generate networkId
 if ! [[ -s .networkId ]]; then
     anyconf create-network
+    echo "Create network"
     cat nodes.yml | grep '^networkId:' | awk '{print $NF}' > .networkId
     cat account.yml | yq '.account.signingKey' > .networkSigningKey
+
+    if [ $? -ne 0 ]; then
+        echo "Failed network creations!"
+        exit 1
+    fi
 fi
 NETWORK_ID=$( cat .networkId)
 NETWORK_SIGNING_KEY=$( cat .networkSigningKey )
 
 if ! [[ -s account0.yml ]]; then
+    echo "Generate nodes and accounts"
     anyconf generate-nodes \
         --t tree \
         --t tree \
@@ -26,6 +33,10 @@ if ! [[ -s account0.yml ]]; then
         --addresses ${ANY_SYNC_FILENODE_ADDRESSES} \
         --addresses ${ANY_SYNC_CONSENSUSNODE_ADDRESSES} \
 
+    if [ $? -ne 0 ]; then
+        echo "Failed to generate nodes and accounts!"
+        exit 1
+    fi
 fi
 
 yq --indent 4 --inplace ".networkId |= \"${NETWORK_ID}\"" nodes.yml

docker-generateconfig/processing.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+source generateconfig/.env
+
+# Set file paths
+dest_path="etc"
+network_file="${dest_path}/network.yml"
+
+# Create directories for all node
+for i in {1..3}; do
+    mkdir -p "${dest_path}/any-sync-node-${i}"
+done
+
+# Create directories for other node types
+for node_type in filenode coordinator consensusnode; do
+    mkdir -p "${dest_path}/any-sync-${node_type}"
+done
+
+# Generate network file
+sed 's|^|    |; 1s|^|network:\n|' "generateconfig/nodes.yml" > "${network_file}"
+
+# Generate config files for 3 nodes
+for i in {0..2}; do
+    node_file="${dest_path}/any-sync-node-$((i+1))/config.yml"
+    cat "${network_file}" tmp-etc/common.yml generateconfig/account${i}.yml tmp-etc/node-$((i+1)).yml > "${node_file}"
+done
+
+# Generate config files for coordinator, filenode, consensusnode
+cat "${network_file}" tmp-etc/common.yml generateconfig/account3.yml tmp-etc/coordinator.yml > ${dest_path}/any-sync-coordinator/config.yml
+cat "${network_file}" tmp-etc/common.yml generateconfig/account4.yml tmp-etc/filenode.yml > ${dest_path}/any-sync-filenode/config.yml
+cat "${network_file}" tmp-etc/common.yml generateconfig/account5.yml tmp-etc/consensusnode.yml > ${dest_path}/any-sync-consensusnode/config.yml
+
+# Copy network file to coordinator directory
+cp "generateconfig/nodes.yml" "${dest_path}/any-sync-coordinator/network.yml"
+
+# Replace placeholders in config files
+for node_type in node_1 node_2 node_3 coordinator filenode consensusnode; do
+    addresses="ANY_SYNC_${node_type^^}_ADDRESSES"
+    quic_addresses="ANY_SYNC_${node_type^^}_QUIC_ADDRESSES"
+    perl -i -pe "s|%${addresses}%|${!addresses}|g" "${network_file}" "${dest_path}/"/*/*.yml
+    perl -i -pe "s|%${quic_addresses}%|${!quic_addresses}|g" "${network_file}" "${dest_path}/"/*/*.yml
+done
+
+# Replace other placeholders
+placeholders=( "MONGO_CONNECT" "REDIS_URL" "AWS_PORT" )
+for placeholder in "${placeholders[@]}"; do
+    perl -i -pe "s|%${placeholder}%|${!placeholder}|g" "${network_file}" "${dest_path}/"/*/*.yml
+done