Merge branch 'main' of https://git.format.hu/safebox/framework-scheduler

scripts/scheduler/entrypoint.sh

@@ -91,21 +91,28 @@ debug() {
     fi
 }
 
-## Start prevously deployed firewall rules depend on framework scheduler startup at first time
+add_json_target(){
 
-if [ -d /etc/user/config/services ]; then
-    cd /etc/user/config/services
-    for FIREWALL in $(ls firewall*.json); do
-        $service_exec $FIREWALL start &
-    done
-fi
+        local TASK_NAME=$1
+
+        if [ -n "$TASK_NAME" ]; then
+            TASK="upgrade-$TASK_NAME"
+        fi
+        
+        install -m 664 -g 65534 /dev/null $SHARED/output/$TASK.json
+        echo $JSON_TARGET | base64 -d >$SHARED/output/$TASK.json
+}
 
 backup_query_state() {
 
+    echo "backup_query_state"
+
 }
 
 backup_set_service() {
 
+    echo "backup_set_service"
+
 }
 
 backup_set_client() {
@@ -149,65 +156,70 @@ backup_set_client() {
         ENVS='"ENVS": [{"SSH_USER":"'$SSH_USER'"},{"SSH_PORT":"'$SSH_PORT'"},{"SSH_PASSWORD":"'$SSH_PASSWORD'"},{"VPN_CLIENT_KEY":"'$VPN_KEY'"}],'
 
         echo '{
-        "main": {
-        "SERVICE_NAME": "'$NAME'"
-        },
-        "containers": [
+    "main": {
+    "SERVICE_NAME": "'$NAME'"
+    },
+    "containers": [
+    {
+    "IMAGE": "alpine:latest",
+    "NAME": "'$NAME'-init",
+    "UPDATE": "true",
+    "MEMORY": "64M",
+    "EXTRA": "--rm",
+    "VOLUMES":[
         {
-        "IMAGE": "alpine:latest",
-        "NAME": "'$NAME'-init",
-        "UPDATE": "true",
-        "MEMORY": "64M",
-        "EXTRA": "--rm",
-        "VOLUMES":[
-            {
-            "SOURCE": "USER_DATA",
-            "DEST": "/etc/user/data/",
-            "TYPE": "rw"
-            }
-                ],
-        "ENTRYPOINT": "sh -c",
-        "CMD": "mkdir -p /etc/user/data/backup/clients/'$NAME'/backup && /etc/user/data/backup/clients/'$NAME'/ssh",
-        "POST_START": []
-        },
-        {
-        "IMAGE": "safebox/backup-client:latest",
-        "NAME": "'$NAME'",
-        "UPDATE": "true",
-        "MEMORY": "64M",
-        "NETWORK": "'$NETWORK'",
-        '$ADDITIONAL',
-        '$ENVS'
-        '$PORT'
-        "VOLUMES":[
-            { 
-            "SOURCE": "/etc/user/data/backup/clients/'$NAME'/backup",
-            "DEST": "/backup",
-            "TYPE": "rw"
-            },
-            { 
-            "SOURCE": "/etc/user/data/backup/clients/'$NAME'/ssh",
-            "DEST": "/home/'$SSH_USER'/",
-            "TYPE": "rw"
-            }
-                ],
-        "POST_START": []
+        "SOURCE": "USER_DATA",
+        "DEST": "/etc/user/data/",
+        "TYPE": "rw"
         }
-      ]
-    }' | jq -r . >/etc/user/config/services/service-backup-client-$NAME.json
+            ],
+    "ENTRYPOINT": "sh -c",
+    "CMD": "mkdir -p /etc/user/data/backup/clients/'$NAME'/backup && /etc/user/data/backup/clients/'$NAME'/ssh",
+    "POST_START": []
+    },
+    {
+    "IMAGE": "safebox/backup-client:latest",
+    "NAME": "'$NAME'",
+    "UPDATE": "true",
+    "MEMORY": "64M",
+    "NETWORK": "'$NETWORK'",
+    '$ADDITIONAL',
+    '$ENVS'
+    '$PORT'
+    "VOLUMES":[
+        { 
+        "SOURCE": "/etc/user/data/backup/clients/'$NAME'/backup",
+        "DEST": "/backup",
+        "TYPE": "rw"
+        },
+        { 
+        "SOURCE": "/etc/user/data/backup/clients/'$NAME'/ssh",
+        "DEST": "/home/'$SSH_USER'/",
+        "TYPE": "rw"
+        }
+            ],
+    "POST_START": []
+        }
+    ]
+ }' | jq -r . >/etc/user/config/services/service-backup-client-$NAME.json
 
-    debug "service-backup-client-$NAME.json stop force dns-remove"
-    $service_exec service-backup-client-$NAME.json start &
+        debug "service-backup-client-$NAME.json stop force dns-remove"
+        $service_exec service-backup-client-$NAME.json start &
 
     fi
 
 }
 
 backup_challenge_clients() {
+
+    echo "backup_challenge_clients"
+
 }
 
 restore_from_backup() {
 
+    echo "restore_from_backup"
+
 }
 
 create_htpasswd_file() {
@@ -685,7 +697,7 @@ check_update() {
         debug "$REMOTE_URL not accessible, http error code: $CURL_CHECK_CODE"
 
         echo "Force image pull has started without digest check..."
-        DOCKER_PULL="docker pull $IMAGE"
+        DOCKER_PULL="/usr/bin/docker pull $IMAGE"
         eval $DOCKER_PULL
         STATUS=$?
         debug "PULL STATUS: $STATUS"
@@ -699,7 +711,19 @@ check_update() {
 
 upgrade_scheduler() {
 
-    DOCKER_START="--entrypoint=sh $DOCKER_REGISTRY_URL/$FRAMEWORK_SCHEDULER_IMAGE:$FRAMEWORK_SCHEDULER_VERSION -c '/scripts/upgrade.sh'"
+    # Upgrading framework scheduler
+    debug "Upgrading framework scheduler..."
+    /usr/bin/docker pull "$DOCKER_REGISTRY_URL/$FRAMEWORK_SCHEDULER_IMAGE:$FRAMEWORK_SCHEDULER_VERSION"
+
+    if [ "$DEBUG_MODE" == "true" ]; then
+        DOCKER_START="--entrypoint=sh $DOCKER_REGISTRY_URL/$FRAMEWORK_SCHEDULER_IMAGE:$FRAMEWORK_SCHEDULER_VERSION -c 'sleep 86400'"
+        SET_DEBUG_MODE="--env DEBUG_MODE=true"
+    else
+        DOCKER_START="$DOCKER_REGISTRY_URL/$FRAMEWORK_SCHEDULER_IMAGE:$FRAMEWORK_SCHEDULER_VERSION"
+        SET_DEBUG_MODE=""
+    fi
+
+    FRAMEWORK_SCHEDULER_NAME="$FRAMEWORK_SCHEDULER_NAME-$(head /dev/urandom | tr -dc '0-9' | head -c 6)"
 
     DOCKER_RUN="/usr/bin/docker run -d \
         -v SHARED:/var/tmp/shared \
@@ -711,6 +735,8 @@ upgrade_scheduler() {
 		-v USER_CONFIG:/etc/user/config \
 		-v USER_SECRET:/etc/user/secret \
 		--restart=always \
+        --name $FRAMEWORK_SCHEDULER_NAME \
+        $SET_DEBUG_MODE \
 	  	--env WEBSERVER_PORT=$WEBSERVER_PORT \
 	  	--network $FRAMEWORK_SCHEDULER_NETWORK \
 		--env RUN_FORCE=$RUN_FORCE \
@@ -722,20 +748,20 @@ upgrade_scheduler() {
 upgrade() {
     local NAME=$1
 
-    if [ "$NAME" == "web-installer" ]; then
-
-        debug "$service_exec service-framework.containers.webserver stop force"
-        $service_exec service-framework.containers.webserver stop force
-        debug "$service_exec service-framework.containers.webserver start info"
-        $service_exec service-framework.containers.webserver start info &
-
+    if [ "$NAME" == "webserver" ]; then
+        debug "$service_exec service-framework.containers.$NAME stop force"
+        $service_exec service-framework.containers.$NAME stop force
+        debug "$service_exec service-framework.containers.$NAME start info"
+        $service_exec service-framework.containers.$NAME start info &
     else
 
         debug "$service_exec $NAME.json stop force"
         $service_exec $NAME.json stop force
         debug "$service_exec $NAME.json start info"
         $service_exec $NAME.json start info &
+
     fi
+
     PID=$!
 }
 
@@ -778,7 +804,13 @@ execute_task() {
 
             if [ "$REQUEST" == "requested" ]; then
                 echo "New certificate for $DOMAIN is requested."
-                touch /etc/system/data/ssl/keys/$DOMAIN/new_certificate
+                echo "Modifying $DOMAIN_FILE.json for $DOMAIN"
+                jq '.containers[0].ENVS |= map(if has("OPERATION") then .OPERATION = "MODIFY" else . end) | \
+                .containers[0].ENVS |= map(if has("DOMAIN") then .DOMAIN = "'$DOMAIN'" else . end)' \
+                /etc/user/config/services/$DOMAIN_FILE.json > /tmp/$DOMAIN_FILE.json && \
+                mv /tmp/$DOMAIN_FILE.json /etc/user/config/services/$DOMAIN_FILE.json
+                debug "$service_exec $DOMAIN_FILE.json start info"
+                $service_exec $DOMAIN_FILE.json start info &
             fi
         done
         JSON_TARGET=$B64_JSON
@@ -921,8 +953,7 @@ execute_task() {
                 else
                     SEP=""
                 fi
-                DEPLOYMENTS="$DEPLOYMENTS"$SEP'"'$APP_NAME'": {"subtitle": "'"$APP_SUBTITLE"'", "version": "'"$APP_VERSION"'", "icon":
- "'"$APP_ICON"'"}'
+                DEPLOYMENTS="$DEPLOYMENTS"$SEP'"'$APP_NAME'":{"subtitle":"'"$APP_SUBTITLE"'","version":"'"$APP_VERSION"'","icon":"'"$APP_ICON"'"}'
             done < <(echo "$APPS") # preserve DEPLOYMENTS variable
         done
         if [ "$DEPLOYMENTS" == "" ]; then
@@ -957,21 +988,22 @@ execute_task() {
 
         for TREE in $TREES; do
             APPS=$(jq -rc '.apps[]' $TREE)
-            for APP in $APPS; do
+            #for APP in $APPS; do
+            while IFS= read -r APP; do
                 APP_NAME=$(echo "$APP" | jq -r '.name' | awk '{print tolower($0)}')
                 APP_SUBTITLE=$(echo "$APP" | jq -r '.subtitle')
                 APP_VERSION=$(echo "$APP" | jq -r '.version')
                 APP_DIR=$(dirname $TREE)"/"$APP_NAME
-                debug "$APP_TEMPLATE"
                 if [ "$APP_NAME" == "$DEPLOY_NAME" ]; then
                     if [ "$DEPLOY_ACTION" == "ask" ]; then
                         APP_TEMPLATE=$APP_DIR"/template.json"
                         TEMPLATE=$(cat $APP_TEMPLATE | base64 -w0)
-                        JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "STATUS": "0", "TEMPLATE": "'$TEMPLATE'" }' | jq -r . | base64 -w0)
+                        debug "$APP_TEMPLATE"
+                        JSON_TARGET=$(echo '{"DATE":"'$DATE'","STATUS": "0","TEMPLATE":"'$TEMPLATE'"}' | jq -r . | base64 -w0)
                     elif [ "$DEPLOY_ACTION" == "reinstall" ]; then
                         APP_TEMPLATE=$APP_DIR"/template.json"
                         TEMPLATE=$(cat $APP_TEMPLATE)
-                        for LINE in $(cat $SERVICE_DIR/service-$DEPLOY_NAME.json | jq -rc '.containers[].ENVS[] | to_entries[]'); do
+                        for LINE in $(cat $SERVICE_DIR/service-$DEPLOY_NAME.json | jq -rc '.containers[].ENVS[] | to_entries[]' 2>/dev/null); do
                             KEY=$(echo $LINE | jq -r .key)
                             VALUE=$(echo $LINE | jq -r .value)
                             debug "$KEY: $VALUE"
@@ -980,14 +1012,14 @@ execute_task() {
                             TEMPLATE=$(echo "$TEMPLATE" | jq -r '.fields |= map(if .key == "'$KEY'" then .value = "'$VALUE'" else . end)')
                         done
                         # write ENV value from domain file to template value by key name
-                        for LINE in $(cat $SERVICE_DIR/domain-$DEPLOY_NAME.json | jq -rc '.containers[].ENVS[] | to_entries[]'); do
+                        for LINE in $(cat $SERVICE_DIR/domain-$DEPLOY_NAME.json | jq -rc '.containers[].ENVS[] | to_entries[]' 2>/dev/null); do
                             KEY=$(echo $LINE | jq -r .key)
                             VALUE=$(echo $LINE | jq -r .value)
                             debug "$KEY: $VALUE"
                             TEMPLATE=$(echo "$TEMPLATE" | jq -r '.fields |= map(if .key == "'$KEY'" then .value = "'$VALUE'" else . end)')
                         done
                         # write ENV value from secret file to template value by key name
-                        for LINE in $(cat $SECRET_DIR/$DEPLOY_NAME/$DEPLOY_NAME.json | jq -rc '.[] | to_entries[]'); do
+                        for LINE in $(cat $SECRET_DIR/$DEPLOY_NAME/$DEPLOY_NAME.json | jq -rc '.[] | to_entries[]' 2>/dev/null); do
                             KEY=$(echo $LINE | jq -r .key)
                             VALUE=$(echo $LINE | jq -r .value)
                             debug "$KEY: $VALUE"
@@ -1034,7 +1066,7 @@ execute_task() {
                         JSON_TARGET=""
                     fi
                 fi
-            done
+            done < <(echo "$APPS") # preserve variables
         done
 
     elif [ "$TASK_NAME" == "repositories" ]; then
@@ -1075,6 +1107,9 @@ execute_task() {
 
     elif [ "$TASK_NAME" == "save_vpn" ]; then
 
+        JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "STATUS": "0", "RESULT": "" }' | jq -r . | base64 -w0)
+        add_json_target
+
         VPN_PROXY_REPO="wireguard-proxy-client"
         if [ ! -d "/tmp/$VPN_PROXY_REPO" ]; then
             git clone https://git.format.hu/safebox/$VPN_PROXY_REPO.git /tmp/$VPN_PROXY_REPO >/dev/null
@@ -1092,7 +1127,7 @@ execute_task() {
         # install vpn only
         sh /scripts/install.sh "$B64_JSON" "$service_exec" "vpn" "$GLOBAL_VERSION"
 
-        JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "STATUS": "'$VPN_STATUS'", "RESULT": "'$VPN_RESULT'" }' | jq -r . | base64 -w0)
+        JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "STATUS": "1", "RESULT": "'$VPN_RESULT'" }' | jq -r . | base64 -w0)
 
     elif [ "$TASK_NAME" == "containers" ]; then # not in use
         CONTAINERS=$(docker ps -a --format '{{.Names}} {{.Status}}' | grep -v framework-scheduler)
@@ -1145,10 +1180,27 @@ execute_task() {
         JSON="$(echo $B64_JSON | base64 -d)"
         NAME=$(echo "$JSON" | jq -r .NAME | awk '{print tolower($0)}')
         if [ "$NAME" == "framework" ]; then
-            upgrade "web-installer"
+            JSON_TARGET=$(echo '{"DATE":"'$DATE'","INSTALL_STATUS":0}' | jq -r . | base64 -w0)
+            add_json_target $NAME
+            echo "Upgrading service: webserver"
+            upgrade webserver
+
+            echo "Upgrading framework scheduler..."
+            echo "Cleaning temporary files..."
+            
+            rm -rf /var/tmp/shared/input/*
+            rm -rf /var/tmp/shared/output/*
+
             upgrade_scheduler
+            echo "Removing old framework scheduler container..."
+            JSON_TARGET=$(echo '{"DATE":"'$DATE'","INSTALL_STATUS":1}' | jq -r . | base64 -w0)
+            add_json_target $NAME
+            sleep 1
+            /usr/bin/docker rm -f $HOSTNAME
+
             #CONTAINERS=$(docker ps -a --format '{{.Names}} {{.Status}}' | grep -E 'framework-scheduler|webserver')
         else
+            echo "Upgrading service: $NAME"
             upgrade "$NAME"
             #CONTAINERS=$(docker ps -a --format '{{.Names}} {{.Status}}' | grep -w "$NAME")
         fi
@@ -1161,9 +1213,7 @@ execute_task() {
     fi
 
     if [ "$JSON_TARGET" != "" ]; then
-        #redis-cli -h $REDIS_SERVER -p $REDIS_PORT SET $TASK "$JSON_TARGET"
-        install -m 664 -g 65534 /dev/null $SHARED/output/$TASK.json
-        echo $JSON_TARGET | base64 -d >$SHARED/output/$TASK.json
+        add_json_target
     fi
 
 }
@@ -1233,39 +1283,15 @@ check_redis_availability() {
     done
 }
 
-start_framework_scheduler() {
-
-    if [ "$DEBUG_MODE" == "true" ]; then
-        DOCKER_START="--entrypoint=sh $DOCKER_REGISTRY_URL/$FRAMEWORK_SCHEDULER_IMAGE:$FRAMEWORK_SCHEDULER_VERSION -c 'sleep 86400'"
-    else
-        DOCKER_START="$DOCKER_REGISTRY_URL/$FRAMEWORK_SCHEDULER_IMAGE:$FRAMEWORK_SCHEDULER_VERSION"
-    fi
-    DOCKER_RUN="/usr/bin/docker run -d \
-        -v SHARED:/var/tmp/shared \
-	  	-v /var/run/docker.sock:/var/run/docker.sock \
-		-v SYSTEM_DATA:/etc/system/data \
-		-v SYSTEM_CONFIG:/etc/system/config \
-		-v SYSTEM_LOG:/etc/system/log \
-		-v USER_DATA:/etc/user/data \
-		-v USER_CONFIG:/etc/user/config \
-		-v USER_SECRET:/etc/user/secret \
-		--restart=always \
-		--name $FRAMEWORK_SCHEDULER_NAME \
-	  	--env WEBSERVER_PORT=$WEBSERVER_PORT \
-	  	--network $FRAMEWORK_SCHEDULER_NETWORK \
-		--env RUN_FORCE=$RUN_FORCE \
-		--env DOCKER_REGISTRY_URL=$DOCKER_REGISTRY_URL \
-	  $DOCKER_START"
-    eval "$DOCKER_RUN"
-
-}
-
 ### SYSTEM INITIALIZATION ###
+## Start prevously deployed firewall rules depend on framework scheduler startup at first time
 
-## DOCKER NETWORK VARIABLES
-## FILESYSTEM VARIABLES
-## PORTS VARIABLES
-### RESTART SCHEDULER IF NEEDED
+if [ -d /etc/user/config/services ]; then
+    cd /etc/user/config/services
+    for FIREWALL in $(ls firewall*.json); do
+        $service_exec $FIREWALL start &
+    done
+fi
 
 SN=$(check_subnets)
 if [ "$SN" != "1" ]; then
@@ -1282,7 +1308,7 @@ fi
 
 VOL=$(check_volumes)
 if [ "$VOL" != "1" ]; then
-    start_framework_scheduler
+    upgrade_scheduler
     /usr/bin/docker rm -f $HOSTNAME
 fi
 
@@ -1294,15 +1320,12 @@ if [ "$DF" != "1" ]; then
 fi
 
 #RS=$(docker ps | grep redis-server)
-WS=$(docker ps | grep webserver)
+WS=$(/usr/bin/docker ps | grep -o webserver)
 
-#if [[ "$WS" == "" && "$RS" == "" ]]; then
 if [ "$WS" == "" ]; then
-
     # START SERVICES
-    #$service_exec service-framework.containers.redis-server start &
+    echo "Starting webserver"
     $service_exec service-framework.containers.webserver start &
-    sleep 5
 
 fi
 

scripts/scheduler/install.sh

@@ -200,14 +200,18 @@ elif [ "$FIRST_INSTALL" == "vpn" ]; then
 
     get_vpn_key
 
-    edit_user_json $LETSENCRYPT_MAIL $LETSENCRYPT_SERVERNAME
+    if [ "$VPN_PROXY" != "no" ]; then
 
-    $SERVICE_EXEC vpn-proxy stop force
-    $SERVICE_EXEC vpn-proxy start
-    echo "$INIT_SERVICE_PATH/vpn-proxy.json" >>$AUTO_START_SERVICES/.init_services
-    echo "$INIT_SERVICE_PATH/firewall-vpn-smarthost-loadbalancer" >>$AUTO_START_SERVICES/.init_services
-    echo "$INIT_SERVICE_PATH/firewall-vpn-proxy-postrouting" >>$AUTO_START_SERVICES/.init_services
-    echo "$INIT_SERVICE_PATH/firewall-vpn-proxy-prerouting" >>$AUTO_START_SERVICES/.init_services
+	    edit_user_json $LETSENCRYPT_MAIL $LETSENCRYPT_SERVERNAME
+
+	    $SERVICE_EXEC vpn-proxy stop force
+	    $SERVICE_EXEC vpn-proxy start
+	    echo "$INIT_SERVICE_PATH/vpn-proxy.json" >>$AUTO_START_SERVICES/.init_services
+	    echo "$INIT_SERVICE_PATH/firewall-vpn-smarthost-loadbalancer" >>$AUTO_START_SERVICES/.init_services
+	    echo "$INIT_SERVICE_PATH/firewall-vpn-proxy-postrouting" >>$AUTO_START_SERVICES/.init_services
+	    echo "$INIT_SERVICE_PATH/firewall-vpn-proxy-prerouting" >>$AUTO_START_SERVICES/.init_services
+
+    fi;
 
     exit