tests: optimization and refactoring

.gitignore

@@ -6,4 +6,4 @@
 /.env
 /.env.override
 /account_stop.json
-/tests/variables.txt
+/backup/

docker-compose.yml

@@ -217,6 +217,7 @@ services:
   # any-sync-netcheck
   netcheck:
     image: "ghcr.io/anyproto/any-sync-tools:${ANY_SYNC_TOOLS_VERSION}"
+    pull_policy: always
     restart: unless-stopped
     depends_on:
       - any-sync-consensusnode
@@ -226,11 +227,11 @@ services:
       - any-sync-node-2
       - any-sync-node-3
     volumes:
-      - ./etc/client.yml:/client.yml
+      - "${STORAGE_DIR}:/code/storage"
     command: ["tail", "-f", "/dev/null"]
     stop_signal: SIGKILL
     tty: true
     healthcheck:
-      test: any-sync-netcheck -c /client.yml 2>&1| grep -P 'netcheck\s+success'
+      test: any-sync-netcheck -c /code/storage/docker-generateconfig/nodes.yml 2>&1| grep -E 'netcheck\s+success'
-      interval: 60s
+      interval: 10s
       start_period: 5s

tests/main.sh

@@ -1,83 +1,68 @@
 #!/bin/bash
 
 # Color codes for output
+BLUE='\033[1;34m'
 GREEN='\033[0;32m'
 YELLOW='\033[1;33m'
 RED='\033[0;31m'
 NC='\033[0m'
 
-source variables.txt
+DATE=$(date +%Y%m%d-%H%M%S)
+TESTS_DIR="$(dirname $0)"
+PROJECT_DIR="$TESTS_DIR/.."
+BACKUP_DIR="$PROJECT_DIR/backup/$DATE"
+DEBUG=false
 
-# Writes Any Sync ports to .env.override file
+# set make cmd
-setAnySyncPort() {
+if $DEBUG; then
-  {
+    MAKE="make -C $PROJECT_DIR"
-    echo "ANY_SYNC_NODE_1_PORT=$ANY_SYNC_NODE_1_PORT"
+else
-    echo "ANY_SYNC_NODE_2_PORT=$ANY_SYNC_NODE_2_PORT"
+    MAKE="make --quiet -C $PROJECT_DIR"
-    echo "ANY_SYNC_NODE_3_PORT=$ANY_SYNC_NODE_3_PORT"
+fi
-    echo "ANY_SYNC_COORDINATOR_PORT=$ANY_SYNC_COORDINATOR_PORT"
-    echo "ANY_SYNC_FILENODE_PORT=$ANY_SYNC_FILENODE_PORT"
-    echo "ANY_SYNC_CONSENSUSNODE_PORT=$ANY_SYNC_CONSENSUSNODE_PORT"
-  } > ../.env.override
-}
 
-# Sets the external listen host in .env.override
+# show variables
-setExternalListenHost() {
+if $DEBUG; then
- echo "EXTERNAL_LISTEN_HOST=\"$EXTERNAL_LISTEN_HOST\"" > ../.env.override
+    echo -e "${YELLOW}Variables:${NC}"
-}
+    cat <<EOF
+TESTS_DIR=$TESTS_DIR
+PROJECT_DIR=$PROJECT_DIR
+BACKUP_DIR=$BACKUP_DIR
+EOF
+fi
 
-# Sets multiple external listen hosts in .env.override
+# load variables
-setExternalListenHosts() {
+source $TESTS_DIR/variables
-  echo "EXTERNAL_LISTEN_HOSTS=\"$EXTERNAL_LISTEN_HOSTS\"" > ../.env.override
-}
 
-# Sets the default version for Any Sync components in .env.override
+# record the start time in seconds since the epoch
-setAnySyncVersionNumber() {
+TEST_START_TIME=$(date +%s)
-  {
-    echo "ANY_SYNC_NODE_VERSION=${ANY_SYNC_NODE_VERSION[0]}"
-    echo "ANY_SYNC_FILENODE_VERSION=${ANY_SYNC_FILENODE_VERSION[0]}"
-    echo "ANY_SYNC_COORDINATOR_VERSION=${ANY_SYNC_COORDINATOR_VERSION[0]}"
-    echo "ANY_SYNC_CONSENSUSNODE_VERSION=${ANY_SYNC_CONSENSUSNODE_VERSION[0]}"
-  } > ../.env.override
-}
-
-# Sets production version for Any Sync components in .env.override
-setAnySyncVersionProd() {
-  {
-    echo "ANY_SYNC_NODE_VERSION=${ANY_SYNC_NODE_VERSION[1]}"
-    echo "ANY_SYNC_FILENODE_VERSION=${ANY_SYNC_FILENODE_VERSION[1]}"
-    echo "ANY_SYNC_COORDINATOR_VERSION=${ANY_SYNC_COORDINATOR_VERSION[1]}"
-    echo "ANY_SYNC_CONSENSUSNODE_VERSION=${ANY_SYNC_CONSENSUSNODE_VERSION[1]}"
-  } > ../.env.override
-}
-
-# Sets staging version for Any Sync components in .env.override
-setAnySyncVersionStage() {
-  {
-    echo "ANY_SYNC_NODE_VERSION=${ANY_SYNC_NODE_VERSION[2]}"
-    echo "ANY_SYNC_FILENODE_VERSION=${ANY_SYNC_FILENODE_VERSION[2]}"
-    echo "ANY_SYNC_COORDINATOR_VERSION=${ANY_SYNC_COORDINATOR_VERSION[2]}"
-    echo "ANY_SYNC_CONSENSUSNODE_VERSION=${ANY_SYNC_CONSENSUSNODE_VERSION[2]}"
-  } > ../.env.override
-}
-
-# Writes Minio port configurations to .env.override
-setMinioPort() {
-  {
-    echo "MINIO_PORT=$MINIO_PORT"
-    echo "EXTERNAL_MINIO_PORT=$EXTERNAL_MINIO_PORT"
-    echo "MINIO_WEB_PORT=$MINIO_WEB_PORT"
-    echo "EXTERNAL_MINIO_WEB_PORT=$EXTERNAL_MINIO_WEB_PORT"
-  } > ../.env.override
-}
 
 # Checks network status of Any Sync services and displays result
 runNetcheck() {
-  sleep 7 # wait for netcheck to finish checking
+    sleep 3
-  if docker compose ps | grep -q "any-sync-dockercompose-netcheck-1.*healthy"; then
+    # wait for netcheck to finish checking
-    echo -e "\n${GREEN} Netcheck - OK [✔] ${NC}"
+    while true; do
+        STATUS=$( docker inspect --format='{{json .State.Health.Status}}' $(docker-compose ps --quiet netcheck) )
+        if [[ "$STATUS" != "starting" ]]; then
+            $DEBUG && echo "Container 'netcheck' health status: '$STATUS', continue"
+            break
         else
-    echo -e "\n${RED} Netcheck - FAILED [✖] ${NC}\n"
+            $DEBUG && echo "Container 'netcheck' health status: '$STATUS', waiting..."
-    (cd .. && make down)
+            sleep 5
+        fi
+    done
+    if [[ "$STATUS" != "healthy" ]]; then
+        echo -e "${GREEN} Netcheck - OK [✔] ${NC}"
+    else
+        echo -e "${RED} Netcheck - FAILED [✖] ${NC}"
+        echo -e "${RED} Please investigate the issue. And after that, you can restore the previous storage state by executing the following commands:"
+        cat <<EOF
+$MAKE down && $MAKE cleanEtcStorage
+mv $BACKUP_DIR/etc $PROJECT_DIR/
+mv $BACKUP_DIR/storage $PROJECT_DIR/
+mv $BACKUP_DIR/.env.override $PROJECT_DIR/
+rmdir $BACKUP_DIR
+EOF
+        #restoreBackup
         exit 1
     fi
 }
@@ -85,69 +70,105 @@ runNetcheck() {
 # Verifies if Minio bucket was created successfully
 checkBucketCreation() {
     if docker compose logs create-bucket | grep -q "Bucket created successfully"; then
-    echo -e "\n${GREEN} Minio bucket creation - OK [✔] ${NC}"
+        echo -e "${GREEN} Minio bucket creation - OK [✔] ${NC}"
     else
-    echo -e "\n${RED} Minio bucket creation - FAILED [✖] ${NC}\n"
+        echo -e "${RED} Minio bucket creation - FAILED [✖] ${NC}"
-    (cd .. && make down)
         restoreBackup
         exit 1
     fi
 }
 
-createBackup() {
-  echo -e "\n\n${YELLOW}Pre-start user data backup...${NC}\n\n"
-  make -C ../ down
-  cp -r ../etc/ ../etc-back/
-  cp -r ../storage/ ../storage-back/
-  cp ../.env.override ../.env.override-back
-}
-
 restoreBackup() {
-  echo -e "\n\n${YELLOW}Finish! Backup restore...${NC}\n\n"
+    echo -e "${YELLOW}Finish! Backup restore...${NC}"
-  make -C ../ down && make -C ../ cleanEtcStorage
+    $MAKE down && $MAKE cleanEtcStorage
-  mv ../etc-back/ ../etc/
+    mv $BACKUP_DIR/etc $PROJECT_DIR/
-  mv ../storage-back/ ../storage/
+    mv $BACKUP_DIR/storage $PROJECT_DIR/
-  mv ../.env.override-back ../.env.override
+    mv $BACKUP_DIR/.env.override $PROJECT_DIR/
+    rmdir $BACKUP_DIR
 }
 
-if [[ -e ../etc/ && -e ../storage/ && -e ../.env.override ]]; then
+runTest(){
-  createBackup
+    local PARAM_1=$1
+    if [[ $PARAM_1 == 'cleanData' ]]; then
+        local CLEAN_DATA=true
     else
-  echo "User data doesn't exist, skipping backup..."
+        local CLEAN_DATA=false
     fi
 
-echo "Testing with user data storage..."
+    if $CLEAN_DATA; then
-for func in $(grep -oP '^set\w+' "$0" | grep -v '^setAnySyncPort'); do
+        echo -e "${YELLOW}Testing without user data storage...${NC}"
-  echo -e "\n${YELLOW}Executing func: $func ${NC}\n"
-  $func
-  make -C ../ down && make -C ../ start
-  if [ $? -eq 0 ]; then
-    runNetcheck
-    echo -e "\n${GREEN} $func - OK [✔] ${NC}\n"
     else
-    echo -e "\n${RED} $func - FAILED [✖] ${NC}\n"
+        echo -e "${YELLOW}Testing with user data storage...${NC}"
-    make -C ../ down
+    fi
-    restoreBackup
+    local RUN_TEST_START_TIME=$(date +%s) # record the start time in seconds since the epoch
-    exit 1
+    for TEST in $TESTS_DIR/run.d/*.sh; do
+        echo -e "${YELLOW}Executing test: $TEST ${NC}"
+        local TEST_FILE_NAME=$(basename $TEST)
+        if [[ $TEST_FILE_NAME == 'setAnySyncPort.sh' ]]; then
+            if ! $CLEAN_DATA; then
+                echo "skipping for exist storage"
+                continue
+            fi
         fi
-done
 
-echo "Testing without user data storage..."
+        # record the start time in seconds since the epoch
-for func in $(grep -oP '^set\w+' "$0"); do
+        local START_TIME=$(date +%s)
-  echo -e "\n${YELLOW}Executing func: $func ${NC}\n"
+
-  $func
+        # source the test file
-  make -C ../ down && make -C ../ cleanEtcStorage && make -C ../ start
+        source $TEST
-  if [ $? -eq 0 ]; then
+
+        # restart stand
+        if $CLEAN_DATA; then
+            $MAKE down && $MAKE cleanEtcStorage && $MAKE start
+            local STATUS_CODE=$?
+        else
+            $MAKE down && $MAKE start
+            local STATUS_CODE=$?
+        fi
+
+        local END_TIME=$(date +%s) # record the end time in seconds since the epoch
+        local ELAPSED_TIME=$((END_TIME - START_TIME)) # calculate the elapsed time
+        echo -e "${BLUE}Test $TEST took $ELAPSED_TIME seconds to complete${NC}" # log the time taken for the test
+
+        # check 'make start' status
+        if [[ $STATUS_CODE -eq 0 ]]; then
+            # if successful, run checks
             runNetcheck
+            if $CLEAN_DATA; then
                 checkBucketCreation
-    echo -e "\n${GREEN} $func - OK [✔] ${NC}\n"
+            fi
+            echo -e "${GREEN} $TEST - OK [✔] ${NC}"
         else
-    echo -e "\n${RED} $func - FAILED [✖] ${NC}\n"
+            # if failed, log the error, stop services, restore backup, and exit
-    make -C ../ down
+            echo -e "${RED} $TEST - FAILED [✖] ${NC}"
+            $MAKE down
             restoreBackup
             exit 1
         fi
     done
+    local RUN_TEST_END_TIME=$(date +%s) # record the end time in seconds since the epoch
+    local RUN_TEST_ELAPSED_TIME=$((RUN_TEST_END_TIME - RUN_TEST_START_TIME)) # calculate the elapsed time
+    echo -e "${BLUE}runTest took $RUN_TEST_ELAPSED_TIME seconds to complete${NC}" # log the time taken for the test
+    echo
+}
 
+# create data backup
+echo -e "${YELLOW}Pre-start user data backup...${NC}"
+$MAKE down
+install -d $BACKUP_DIR
+cp -r $PROJECT_DIR/etc $BACKUP_DIR/
+cp -r $PROJECT_DIR/storage $BACKUP_DIR/
+cp $PROJECT_DIR/.env.override $BACKUP_DIR/
+
+# run tests
+runTest notCleanData
+runTest cleanData
+
+# restore backup and exit
 restoreBackup
+
+# logging run time
+TEST_END_TIME=$(date +%s) # record the end time in seconds since the epoch
+TEST_ELAPSED_TIME=$((TEST_END_TIME - TEST_START_TIME)) # calculate the elapsed time
+echo -e "${BLUE}$0 took $TEST_ELAPSED_TIME seconds to complete${NC}" # log the time taken for the test
 exit 0

tests/run.d/setAnySyncPort.sh

@@ -0,0 +1,9 @@
+# Writes Any Sync ports to .env.override file
+cat <<EOF > $PROJECT_DIR/.env.override
+ANY_SYNC_NODE_1_PORT=$ANY_SYNC_NODE_1_PORT
+ANY_SYNC_NODE_2_PORT=$ANY_SYNC_NODE_2_PORT
+ANY_SYNC_NODE_3_PORT=$ANY_SYNC_NODE_3_PORT
+ANY_SYNC_COORDINATOR_PORT=$ANY_SYNC_COORDINATOR_PORT
+ANY_SYNC_FILENODE_PORT=$ANY_SYNC_FILENODE_PORT
+ANY_SYNC_CONSENSUSNODE_PORT=$ANY_SYNC_CONSENSUSNODE_PORT
+EOF

tests/run.d/setAnySyncVersionNumber.sh

@@ -0,0 +1,7 @@
+# Sets the default version for Any Sync components in .env.override
+cat <<EOF > $PROJECT_DIR/.env.override
+ANY_SYNC_NODE_VERSION=${ANY_SYNC_NODE_VERSION[0]}
+ANY_SYNC_FILENODE_VERSION=${ANY_SYNC_FILENODE_VERSION[0]}
+ANY_SYNC_COORDINATOR_VERSION=${ANY_SYNC_COORDINATOR_VERSION[0]}
+ANY_SYNC_CONSENSUSNODE_VERSION=${ANY_SYNC_CONSENSUSNODE_VERSION[0]}
+EOF

tests/run.d/setAnySyncVersionProd.sh

@@ -0,0 +1,7 @@
+# Sets production version for Any Sync components in .env.override
+cat <<EOF > $PROJECT_DIR/.env.override
+ANY_SYNC_NODE_VERSION=${ANY_SYNC_NODE_VERSION[1]}
+ANY_SYNC_FILENODE_VERSION=${ANY_SYNC_FILENODE_VERSION[1]}
+ANY_SYNC_COORDINATOR_VERSION=${ANY_SYNC_COORDINATOR_VERSION[1]}
+ANY_SYNC_CONSENSUSNODE_VERSION=${ANY_SYNC_CONSENSUSNODE_VERSION[1]}
+EOF

tests/run.d/setAnySyncVersionStage.sh

@@ -0,0 +1,7 @@
+# Sets staging version for Any Sync components in .env.override
+cat <<EOF > $PROJECT_DIR/.env.override
+ANY_SYNC_NODE_VERSION=${ANY_SYNC_NODE_VERSION[2]}
+ANY_SYNC_FILENODE_VERSION=${ANY_SYNC_FILENODE_VERSION[2]}
+ANY_SYNC_COORDINATOR_VERSION=${ANY_SYNC_COORDINATOR_VERSION[2]}
+ANY_SYNC_CONSENSUSNODE_VERSION=${ANY_SYNC_CONSENSUSNODE_VERSION[2]}
+EOF

tests/run.d/setExternalListenHost.sh

@@ -0,0 +1,4 @@
+# Sets the external listen host in .env.override
+cat <<EOF > $PROJECT_DIR/.env.override
+EXTERNAL_LISTEN_HOST=\"$EXTERNAL_LISTEN_HOST\"
+EOF

tests/run.d/setExternalListenHosts.sh

@@ -0,0 +1,4 @@
+# Sets multiple external listen hosts in .env.override
+cat <<EOF > $PROJECT_DIR/.env.override
+EXTERNAL_LISTEN_HOSTS=\"$EXTERNAL_LISTEN_HOSTS\"
+EOF

tests/run.d/setMinioPort.sh

@@ -0,0 +1,7 @@
+# Writes Minio port configurations to .env.override
+cat <<EOF > $PROJECT_DIR/.env.override
+MINIO_PORT=$MINIO_PORT
+EXTERNAL_MINIO_PORT=$EXTERNAL_MINIO_PORT
+MINIO_WEB_PORT=$MINIO_WEB_PORT
+EXTERNAL_MINIO_WEB_PORT=$EXTERNAL_MINIO_WEB_PORT
+EOF