Update scripts/scheduler/entrypoint.sh

.drone.yml

@@ -3,7 +3,7 @@ type: kubernetes
 name: default
 
 node_selector:
-  physical-node: dev2
+  physical-node: dev1
 
 trigger:
   event:
@@ -14,26 +14,6 @@ workspace:
   path: /drone/src
 
 steps:
-  - name: build multiarch from dev
-    image: docker.io/owncloudci/drone-docker-buildx:4
-    privileged: true
-    settings:
-      cache-from: [ "registry.dev.format.hu/framework-scheduler" ]
-      registry: registry.dev.format.hu
-      repo: registry.dev.format.hu/framework-scheduler
-      tags: latest
-      dockerfile: Dockerfile
-      username:
-        from_secret: dev-hu-registry-username
-      password: 
-        from_secret: dev-hu-registry-password
-      platforms:
-        - linux/amd64
-        - linux/arm64
-    when:  
-      event:
-        - push
-
   - name: pull image to dockerhub
     image: docker.io/owncloudci/drone-docker-buildx:4
     privileged: true
@@ -51,3 +31,25 @@ steps:
     when:  
       event:
         - tag
+
+  - name: build multiarch from dev
+    image: docker.io/owncloudci/drone-docker-buildx:4
+    privileged: true
+    #environment:
+    #  DOCKER_PLUGIN_MIRROR: "https://mirror.dev.format.hu"
+    settings:
+      cache-from: [ "registry.dev.format.hu/framework-scheduler" ]
+      registry: registry.dev.format.hu
+      repo: registry.dev.format.hu/framework-scheduler
+      tags: latest
+      dockerfile: Dockerfile
+      username:
+        from_secret: dev-hu-registry-username
+      password: 
+        from_secret: dev-hu-registry-password
+      platforms:
+        - linux/amd64
+        - linux/arm64
+    when:  
+      event:
+        - push

Dockerfile

@@ -14,10 +14,10 @@ FROM alpine:latest
 # COPY --from=redis-source /usr/src/redis/src/redis-cli /usr/bin/redis-cli
 # RUN chmod +x /usr/bin/redis-cli
 
-RUN apk add --update --no-cache docker-cli wget curl dos2unix jq openssl git coreutils inotify-tools acl
+RUN apk add --update --no-cache docker-cli wget curl dos2unix jq openssl git coreutils inotify-tools acl apache2-utils
 
 COPY scripts/scheduler/*.sh /scripts/
 RUN find ./scripts -name "*.sh" | xargs dos2unix
 RUN ["chmod", "+x", "-R", "/scripts/"]
 
-ENTRYPOINT ["/scripts/entrypoint.sh"]
+CMD /scripts/entrypoint.sh

scripts/scheduler/backup_challenge_clients.sh

@@ -0,0 +1,89 @@
+# Get MY IP
+
+# Get VPN network if exists
+
+# Define port
+
+# Define local IP range
+
+# Define VPN IP range
+
+# Store results
+
+
+scan_network(){
+        MyIP=$(ifconfig ${Interface}|grep inet |awk '{ print $2 }');
+        TargetIP=$(echo $MyIP|cut -d . -f1-3);
+        X=0
+	OpenIP=""
+        for i in $(seq 1 255); do
+                nc -w 1 -z $TargetIP.$i 60022;
+                if [ $? -eq 0 ]
+                then
+			if [ $MyIP != $TargetIP.$i ]
+                        then
+                                if [ $X = 1 ]
+                                then
+                                        # tobb nyitott IP
+					echo "Found more than one IP addresses"
+                                        echo "MAILKULDES"
+					echo "">OpenIP.txt;
+					# TODO mailkuldes ahova kell
+					exit 1;
+				else
+					OpenIP=$TargetIP.$i;
+                                fi
+				X=1;
+                        fi
+                fi
+	done
+	if [ $X = 1  ]
+	then
+		echo $OpenIP>OpenIP.txt;
+		echo "start LVM SYNC";
+		echo "OpenIP mukodik = "$OpenIP;
+		lvm_sync_create $OpenIP;
+	else
+		echo "No available local IP address found!"
+		try_target_VPN;
+	fi
+
+
+}
+
+try_target_IP(){
+	MyIP=$(ifconfig ${Interface}|grep inet |awk '{ print $2 }');
+ 	nc -w 1 -z $OpenIP 60022;
+        if [ $? -eq 0 ] 
+                then
+			if [ $MyIP = $OpenIP ]
+				then	
+				echo "Only own IP address found = "$OpenIP
+				scan_network;
+			fi
+	else
+	scan_network;
+	fi
+}
+
+try_target_VPN(){
+	nc -w 1 -z $VPN 60022;
+        if [ $? -eq 0 ]
+		then
+		for i in {0..99}; do
+		MyVPN=$(ifconfig tun$i 2>/dev/null |grep inet |awk '{ print $2 }');
+			echo "My VPN="$MyVPN;
+			echo "Found VPN="$VPN;
+			if [ $VPN != $MyVPN ]
+				then
+				echo "VPN accessible="$VPN;
+				lvm_sync_create $VPN;
+					else
+					echo "Only own VPN accessible="$VPN;
+					exit 1;
+			fi
+		done
+		else
+		echo "No available server"
+	fi
+}

scripts/scheduler/entrypoint.sh

@@ -2,6 +2,7 @@
 
 cd /scripts
 DEBUG_MODE=${DEBUG_MODE:-false}
+VERSION=1.1.1
 
 #DOCKER_REGISTRY_URL=${DOCKER_REGISTRY_URL:-registry.format.hu}
 DOCKER_REGISTRY_URL=${DOCKER_REGISTRY_URL:-safebox}
@@ -30,6 +31,7 @@ REDIS_VERSION=${REDIS_VERSION:-latest}
 
 SOURCE=${SOURCE:-user-config}
 SMARTHOST_PROXY_PATH=$SMARTHOST_PROXY_PATH
+HTPASSWD_FILE=${HTPASSWD_FILE:-/etc/system/config/smarthost-proxy/nginx/htpasswd}
 
 GIT_URL=${GIT_URL:-git.format.hu}
 REPO=$REPO
@@ -90,15 +92,294 @@ debug() {
     fi
 }
 
-## Start prevously deployed firewall rules depend on framework scheduler startup at first time
+add_json_target(){
 
-if [ -d /etc/user/config/services ]; then
+        local TASK_NAME=$1
-    cd /etc/user/config/services
+
-    for FIREWALL in $(ls firewall*.json); do
+        if [ -n "$TASK_NAME" ]; then
-        $service_exec $FIREWALL start &
+            TASK="upgrade-$TASK_NAME"
-    done
         fi
         
+        install -m 664 -g 65534 /dev/null $SHARED/output/$TASK.json
+        echo $JSON_TARGET | base64 -d >$SHARED/output/$TASK.json
+}
+
+backup_query_state() {
+
+    echo "backup_query_state"
+
+}
+
+generate_backup_server_secrets () {
+
+        mkdir -p $SECRET_DIR/backup/server
+
+        echo '{ 
+            "backupserver":{
+                "SSH_USER":"'$SSH_USER'",
+                "SSH_PORT":"'$SSH_PORT'",
+                "SSH_PASSWORD":"'$SSH_PASSWORD'",
+                "PASSWORD":"'$PASSWORD'",
+                "PERIOD":"'$PERIOD'",
+                "COMPRESSION":"'$COMPRESSION'",
+                "DIRECTORIES":"'$DIRECTORIES'",
+                "SERVICES":"'$SERVICES'",
+                "BACKUP_LOCAL_CLIENTS":"'$BACKUP_LOCAL_CLIENTS'",
+                "BACKUP_VPN_CLIENTS":"'$BACKUP_VPN_CLIENTS'"
+            }
+        }' | jq -r . > $SECRET_DIR/backup/server/backup.json
+}
+
+create_backup_service () {
+
+    ADDITIONAL=""
+        ADDITIONAL='"EXTRA":"--rm","PRE_START":[],"DEPEND": [],"CMD":""'
+
+        BACKUP_SERVER='{
+    "main": {
+    "SERVICE_NAME": "backup-server"
+    },
+    "containers": [
+    {
+    "IMAGE": "alpine:latest",
+    "NAME": "backup-init",
+    "NETWORK": "host",
+    "UPDATE": "true",
+    "MEMORY": "64M",
+    "EXTRA": "--rm",
+    "VOLUMES":[
+        {
+        "SOURCE": "USER_DATA",
+        "DEST": "/etc/user/data/",
+        "TYPE": "rw"
+        }
+            ],
+    "ENTRYPOINT": "sh -c",
+    "CMD": "mkdir -p /etc/user/data/backup/server/ssh",
+    "POST_START": []
+    },
+    {
+    "IMAGE": "safebox/backup-server:latest",
+    "NAME": "backupserver",
+    "NETWORK": "'$NETWORK'",
+    "UPDATE": "true",
+    "MEMORY": "64M",
+    "VOLUMES":[
+        {
+        "SOURCE": "USER_DATA",
+        "DEST": "/etc/user/data/",
+        "TYPE": "ro"
+        },
+        {
+        "SOURCE": "USER_CONFIG",
+        "DEST": "/etc/user/config/",
+        "TYPE": "ro"
+        },
+        {
+        "SOURCE": "USER_SECRET",
+        "DEST": "/etc/user/secret/",
+        "TYPE": "ro"
+        },
+        {
+        "SOURCE": "/etc/user/data/backup/server/ssh",
+        "DEST": "/home/'$SSH_USER'/",
+        "TYPE": "rw"
+        }
+            ],
+    "ENV_FILES":["/etc/user/secret/backup/server/backup.json"],
+    '$ADDITIONAL',
+    "POST_START": []
+        }
+    ]
+ }' 
+    # create backup server secrets from variables
+    generate_backup_server_secrets
+}
+
+backup_set_service() {
+
+
+    local PASSWORD="$1"
+    local PERIOD="$2"
+    local COMPRESSION="$3"
+
+    local PLANNED_TIME="$(echo "$4" | base64 -d)"
+    local DIRECTRIES="$5"
+    local SERVICES="$6"
+    local BACKUP_LOCAL_CLIENTS="$7"
+    local BACKUP_VPN_CLIENTS="$8"
+
+    local VPN="$9"
+    local SSH_PORT="${10:-20022}"
+    local SSH_USER="${11:-"backup"}"
+    local SSH_PASSWORD="${12:-"backup"}"
+    local OPERATION="${13}"
+
+    if [ "$OPERATION" == "DELETE" ]; then
+
+        sed -i '/service-backup/d' /etc/user/data/cron/crontab.txt
+        # delete service
+        rm -f /etc/user/config/services/service-backup-server*
+        rm -rf /etc/user/data/backup/server
+        rm -rf /etc/user/secret/backup/server
+        debug "Service backup server service deleted."
+
+    elif [ "$OPERATION" == "MODIFY" ]; then
+
+        # modify only secrets for backup server, it will be affected at the next cron job
+        generate_backup_server_secrets
+
+    else
+
+        if [ -z "$SSH_PORT" ] ; then
+            SSH_PORT=20022
+        fi
+
+        if [ "$VPN" == "true" ]; then
+            NETWORK=$VPN_NETWORK
+            create_backup_service
+        else
+            NETWORK="host"
+            create_backup_service
+        fi
+    
+    fi
+
+    if [ -n "$PLANNED_TIME" ]; then
+        if [ "$VPN" == "true" ]; then
+            if [ -n "$BACKUP_SERVER" ] ; then
+                echo "$BACKUP_SERVER" | jq -r . >/etc/user/config/services/service-backup-server-vpn.json
+            fi
+            echo "'$PLANNED_TIME' service service-backup-server-vpn" >> /etc/user/data/cron/crontab.txt 
+        else
+            if [ -n "$BACKUP_SERVER" ] ; then
+             echo "$BACKUP_SERVER" | jq -r . >/etc/user/config/services/service-backup-server-local.json
+            fi
+            echo "'$PLANNED_TIME' service service-backup-server-local" >> /etc/user/data/cron/crontab.txt 
+        fi
+    fi    
+
+
+}
+
+backup_set_client() {
+
+    local NAME="$1"
+    local SIZE="$2"
+    local VPN="$3"
+    local SSH_PORT="${4:-20022}"
+    local SSH_USER="${5:-"backup"}"
+    local SSH_PASSWORD="${6:-"backup"}"
+    local OPERATION="$7"
+    local VPN_KEY="$8"
+
+    if [ "$OPERATION" == "DELETE" ]; then
+        # delete service
+        if [ -f "/etc/user/config/services/service-backup-client-$NAME.json" ]; then
+
+            debug "service-backup-client-$NAME.json stop force dns-remove"
+            $service_exec service-backup-client-$NAME.json stop force dns-remove
+            rm -f /etc/user/config/services/service-backup-client-$NAME.json
+            debug "Service backup client $NAME deleted."
+
+        fi
+
+    else
+
+        if [ -z "$SSH_PORT" ] ; then
+            SSH_PORT=20022
+        fi
+
+        if [ "$VPN" == "true" ]; then
+            NETWORK=$NAME
+            PORT='"PORTS": [{"SOURCE":"null","DEST":"'$SSH_PORT'","TYPE":"tcp"}],'
+        else
+            NETWORK="host"
+            PORT='"PORTS": [{"SOURCE":"'$SSH_PORT'","DEST":"'$SSH_PORT'","TYPE":"tcp"}],'
+        fi
+
+        ADDITIONAL=""
+        ADDITIONAL='"EXTRA":"--restart=always","PRE_START":[],"DEPEND":[],"CMD": ""'
+        ENVS='"ENVS": [{"SSH_USER":"'$SSH_USER'"},{"SSH_PORT":"'$SSH_PORT'"},{"SSH_PASSWORD":"'$SSH_PASSWORD'"},{"VPN_CLIENT_KEY":"'$VPN_KEY'"}],'
+
+        echo '{
+    "main": {
+    "SERVICE_NAME": "'$NAME'"
+    },
+    "containers": [
+    {
+    "IMAGE": "alpine:latest",
+    "NAME": "'$NAME'-init",
+    "NETWORK": "host",
+    "UPDATE": "true",
+    "MEMORY": "64M",
+    "EXTRA": "--rm",
+    "VOLUMES":[
+        {
+        "SOURCE": "USER_DATA",
+        "DEST": "/etc/user/data/",
+        "TYPE": "rw"
+        }
+            ],
+    "ENTRYPOINT": "sh -c",
+    "CMD": "mkdir -p /etc/user/data/backup/clients/'$NAME'/backup && mkdir -p /etc/user/data/backup/clients/'$NAME'/ssh && chmod -R '$SSH_USER':'$SSH_USER' /etc/user/data/backup/clients/'$NAME'",
+    "POST_START": []
+    },
+    {
+    "IMAGE": "safebox/backup-client:latest",
+    "NAME": "'$NAME'",
+    "UPDATE": "true",
+    "MEMORY": "64M",
+    "NETWORK": "'$NETWORK'",
+    '$ADDITIONAL',
+    '$ENVS'
+    '$PORT'
+    "VOLUMES":[
+        { 
+        "SOURCE": "/etc/user/data/backup/clients/'$NAME'/backup",
+        "DEST": "/home/'$SSH_USER'/backup",
+        "TYPE": "rw"                   
+        },                       
+        {
+        "SOURCE": "/etc/user/data/backup/clients/'$NAME'/ssh",                   
+        "DEST": "/home/'$SSH_USER'/.ssh",
+        }
+            ],
+    "POST_START": []
+        }
+    ]
+ }' | jq -r . >/etc/user/config/services/service-backup-client-$NAME.json
+
+        debug "service-backup-client-$NAME.json start info"
+        $service_exec service-backup-client-$NAME.json start info &
+
+    fi
+
+}
+
+backup_challenge_clients() {
+
+    echo "backup_challenge_clients"
+
+}
+
+restore_from_backup() {
+
+    echo "restore_from_backup"
+
+}
+
+create_htpasswd_file() {
+
+    local USER="$1"
+    local PASSWD="$2"
+
+    if [ ! -f "$HTPASSWD_FILE" ]; then
+        install -m 664 -g 65534 /dev/null $HTPASSWD_FILE
+        htpasswd -cb $HTPASSWD_FILE $USER $PASSWD
+    fi
+}
+
 deploy_additionals() {
 
     local DIR="$1"
@@ -146,14 +427,80 @@ remove_additionals() {
 
     debug "UNINSTALL: $NAME"
 
+    # delete firewall rules
+    FIREWALLS=""
+    FIREWALLS="$(ls $SERVICE_DIR/firewall-*.json | grep $NAME)"
+    for FIREWALL in $(echo $FIREWALLS); do
+        cat $FIREWALL | jq '.containers[] |= (
+        if (.ENVS | map(has("OPERATION")) | any) then
+            # If any entry has OPERATION key, update it
+            .ENVS = [.ENVS[] | if has("OPERATION") then {"OPERATION": "DELETE"} else . end]
+        else
+            # If no entry has OPERATION key, add new entry
+            .ENVS += [{"OPERATION": "DELETE"}]
+        end
+        )' >$FIREWALL.tmp
+        debug "$service_exec $FIREWALL.tmp start info"
+        $service_exec $FIREWALL.tmp start info
+        rm $FIREWALL.tmp
+    done
+
+    # delete domains
+    DOMMAINS=""
+    DOMAINS="$(ls $SERVICE_DIR/domain-*.json | grep $NAME)"
+    for DOMAIN in $(echo $DOMAINS); do
+        cat $DOMAIN | jq '.containers[] |= (
+        if (.ENVS | map(has("OPERATION")) | any) then
+            # If any entry has OPERATION key, update it
+            .ENVS = [.ENVS[] | if has("OPERATION") then {"OPERATION": "DELETE"} else . end]
+        else
+            # If no entry has OPERATION key, add new entry
+            .ENVS += [{"OPERATION": "DELETE"}]
+        end
+        )' >$DOMAIN.tmp
+        debug "$service_exec $DOMAIN.tmp start info"
+        $service_exec $DOMAIN.tmp start info
+        rm $DOMAIN.tmp
+    done
+
+    # remove related directories and files
+    # get volume destinations
+    DESTINATIONS=""
+    VOLUMES=""
+    DESTINATIONS=$(cat $SERVICE_DIR/service-$NAME.json | jq -r '[.containers[] | select(has("VOLUMES")) | .VOLUMES[] | select(.SHARED != "true") | .SOURCE] | unique[]' | grep $NAME)
+    for DESTINATION in $(echo $DESTINATIONS); do
+        if [ -d "$DESTINATION" ] || [ -f "$DESTINATION" ]; then
+            rm -rf $DESTINATION
+            debug "deleted directory or file: $DESTINATION"
+        fi
+    done
+
+    ENV_FILES=$(cat $SERVICE_DIR/service-$NAME.json | jq -r '[.containers[] | select(has("ENV_FILES")) | .ENV_FILES[]] | unique[]')
+    for ENV_FILE in $(echo $ENV_FILES); do
+        if [ -f "$ENV_FILE" ]; then
+            rm -rf $ENV_FILE
+            debug "deleted enviroment file: $ENV_FILE"
+        fi
+    done
+
+    VOLUMES=$(cat $SERVICE_DIR/service-$NAME.json | jq -r '[.containers[] | select(has("VOLUMES")) | .VOLUMES[] | select(.SHARED != "true") | .SOURCE] | unique[]' | grep -vE 'USER|SYSTEM')
+
     # stop service
     # force - remove stopped container, docker rm
     debug "$service_exec service-$NAME.json stop force dns-remove"
     $service_exec service-$NAME.json stop force dns-remove
 
+    for VOLUME in $(echo $VOLUMES | grep -vE 'USER|SYSTEM|SHARED'); do
+        if [ "$(echo $VOLUME | cut -d '/' -f1)" ]; then
+            docker volume rm $VOLUME
+            debug "deleted volume: $VOLUME"
+
+        fi
+    done
+
     # remove service files
     rm $SERVICE_DIR/*"-"$NAME.json # service, domain, etc.
-    rm $SECRET_DIR/$NAME/$NAME.json
+
 }
 
 get_repositories() {
@@ -163,15 +510,19 @@ get_repositories() {
     local TREES=""
     local REPO
 
+    if [ ! -f "/etc/user/config/repositories.json" ]; then
+        create_repositories_json
+    fi
+
     REPOS=$(jq -r .repositories[] /etc/user/config/repositories.json) # list of repos, delimiter by space
     for REPO in $REPOS; do
 
         BASE=$(basename $REPO | cut -d '.' -f1)
         if [ ! -d "/tmp/$BASE" ]; then
-            git clone $REPO /tmp/$BASE >/dev/null
+            GIT_HTTP_CONNECT_TIMEOUT=10 GIT_HTTP_TIMEOUT=30 git clone $REPO /tmp/$BASE >/dev/null
         else
             cd /tmp/$BASE
-            git pull >/dev/null
+            GIT_HTTP_CONNECT_TIMEOUT=10 GIT_HTTP_TIMEOUT=30 git pull >/dev/null
         fi
         if [ -f "/tmp/$BASE/applications-tree.json" ]; then
             TREES=$TREES" /tmp/$BASE/applications-tree.json"
@@ -497,7 +848,7 @@ check_update() {
         debug "$REMOTE_URL not accessible, http error code: $CURL_CHECK_CODE"
 
         echo "Force image pull has started without digest check..."
-        DOCKER_PULL="docker pull $IMAGE"
+        DOCKER_PULL="/usr/bin/docker pull $IMAGE"
         eval $DOCKER_PULL
         STATUS=$?
         debug "PULL STATUS: $STATUS"
@@ -511,7 +862,19 @@ check_update() {
 
 upgrade_scheduler() {
 
-    DOCKER_START="--entrypoint=sh $DOCKER_REGISTRY_URL/$FRAMEWORK_SCHEDULER_IMAGE:$FRAMEWORK_SCHEDULER_VERSION -c '/scripts/upgrade.sh'"
+    # Upgrading framework scheduler
+    debug "Upgrading framework scheduler..."
+    /usr/bin/docker pull "$DOCKER_REGISTRY_URL/$FRAMEWORK_SCHEDULER_IMAGE:$FRAMEWORK_SCHEDULER_VERSION"
+
+    if [ "$DEBUG_MODE" == "true" ]; then
+        DOCKER_START="--entrypoint=sh $DOCKER_REGISTRY_URL/$FRAMEWORK_SCHEDULER_IMAGE:$FRAMEWORK_SCHEDULER_VERSION -c 'sleep 86400'"
+        SET_DEBUG_MODE="--env DEBUG_MODE=true"
+    else
+        DOCKER_START="$DOCKER_REGISTRY_URL/$FRAMEWORK_SCHEDULER_IMAGE:$FRAMEWORK_SCHEDULER_VERSION"
+        SET_DEBUG_MODE=""
+    fi
+
+    FRAMEWORK_SCHEDULER_NAME="$FRAMEWORK_SCHEDULER_NAME-$(head /dev/urandom | tr -dc '0-9' | head -c 6)"
 
     DOCKER_RUN="/usr/bin/docker run -d \
         -v SHARED:/var/tmp/shared \
@@ -523,6 +886,8 @@ upgrade_scheduler() {
 		-v USER_CONFIG:/etc/user/config \
 		-v USER_SECRET:/etc/user/secret \
 		--restart=always \
+        --name $FRAMEWORK_SCHEDULER_NAME \
+        $SET_DEBUG_MODE \
 	  	--env WEBSERVER_PORT=$WEBSERVER_PORT \
 	  	--network $FRAMEWORK_SCHEDULER_NETWORK \
 		--env RUN_FORCE=$RUN_FORCE \
@@ -534,20 +899,22 @@ upgrade_scheduler() {
 upgrade() {
     local NAME=$1
 
-	if [ "$NAME" == "web-installer" ]; then
+    if [ "$NAME" == "webserver" ]; then
-		
+        debug "$service_exec service-framework.containers.$NAME stop force"
-		debug "$service_exec service-framework-scheduler.containers.webserver start info"
+        $service_exec service-framework.containers.$NAME stop force
-		$service_exec service-framework-scheduler.containers.webserver stop force
+        debug "$service_exec service-framework.containers.$NAME start info"
-		$service_exec service-framework-scheduler.containers.webserver start info &
+        $service_exec service-framework.containers.$NAME start info &
-	
     else
 
-		debug "$service_exec service-$NAME.json start info"
+        debug "$service_exec $NAME.json stop force"
-		$service_exec service-$NAME.json stop force
+        $service_exec $NAME.json stop force
-		$service_exec service-$NAME.json start info &
+        debug "$service_exec $NAME.json start info"
-	fi
+        $service_exec $NAME.json start info &
-}
 
+    fi
+
+    PID=$!
+}
 
 execute_task() {
     TASK="$1"
@@ -555,8 +922,10 @@ execute_task() {
     DATE=$(date +"%Y%m%d%H%M")
 
     # Executing task
-    debug "TASK: $(echo $TASK | cut -d ':' -f1)"
     TASK_NAME=$(echo $TASK | cut -d ':' -f1)
+    if [ "$TASK_NAME" != "check_vpn" ]; then
+        debug "TASK: $(echo $TASK_NAME | cut -d ':' -f1)"
+    fi
 
     # checking sytem status
     SYSTEM_STATUS=$(ls /etc/user/config/services/*.json | grep -v service-framework.json)
@@ -579,6 +948,24 @@ execute_task() {
         #fi;
         JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "INSTALL_STATUS": "'$INSTALL_STATUS'" }' | jq -r . | base64 -w0)
 
+    elif [ "$TASK_NAME" == "request_letsencrypt" ]; then
+        DOMAINS=$(echo $B64_JSON | base64 -d | jq -r 'keys[]')
+        for DOMAIN in $(echo $DOMAINS); do
+            REQUEST=$(echo $B64_JSON | base64 -d | jq -r ".[\"$DOMAIN\"].status")
+
+            if [ "$REQUEST" == "requested" ]; then
+                echo "New certificate for $DOMAIN is requested."
+                echo "Modifying $DOMAIN_FILE.json for $DOMAIN"
+                jq '.containers[0].ENVS |= map(if has("OPERATION") then .OPERATION = "MODIFY" else . end) | \
+                .containers[0].ENVS |= map(if has("DOMAIN") then .DOMAIN = "'$DOMAIN'" else . end)' \
+                /etc/user/config/services/$DOMAIN_FILE.json > /tmp/$DOMAIN_FILE.json && \
+                mv /tmp/$DOMAIN_FILE.json /etc/user/config/services/$DOMAIN_FILE.json
+                debug "$service_exec $DOMAIN_FILE.json start info"
+                $service_exec $DOMAIN_FILE.json start info &
+            fi
+        done
+        JSON_TARGET=$B64_JSON
+
     elif [ "$TASK_NAME" == "system" ]; then
         #SYSTEM_LIST="core-dns.json cron.json domain-local-backend.json firewall-letsencrypt.json firewall-local-backend.json firewall-localloadbalancer-dns.json firewall-localloadbalancer-to-smarthostbackend.json firewall-smarthost-backend-dns.json firewall-smarthost-loadbalancer-dns.json firewall-smarthost-to-backend.json firewall-smarthostloadbalancer-from-publicbackend.json letsencrypt.json local-backend.json local-proxy.json service-framework.json smarthost-proxy-scheduler.json smarthost-proxy.json"
         SYSTEM_LIST="core-dns.json cron.json letsencrypt.json local-proxy.json service-framework.json smarthost-proxy-scheduler.json smarthost-proxy.json"
@@ -704,18 +1091,21 @@ execute_task() {
     elif [ "$TASK_NAME" == "deployments" ]; then
         DEPLOYMENTS=""
         TREES=$(get_repositories)
-        for TREE in $TREES; do
+        for TREE in "$TREES"; do
             APPS=$(jq -rc '.apps[]' $TREE)
-            for APP in $APPS; do
+            #for APP in "$APPS"; do #space problem
+            while IFS= read -r APP; do
                 APP_NAME=$(echo "$APP" | jq -r '.name')
+                APP_SUBTITLE="$(echo "$APP" | jq -r '.subtitle')"
                 APP_VERSION=$(echo "$APP" | jq -r '.version')
+                APP_ICON=$(echo "$APP" | jq -r '.icon')
                 if [ "$DEPLOYMENTS" != "" ]; then
                     SEP=","
                 else
                     SEP=""
                 fi
-                DEPLOYMENTS=$DEPLOYMENTS$SEP'"'$APP_NAME'": "'$APP_VERSION'"'
+                DEPLOYMENTS="$DEPLOYMENTS"$SEP'"'$APP_NAME'":{"subtitle":"'"$APP_SUBTITLE"'","version":"'"$APP_VERSION"'","icon":"'"$APP_ICON"'"}'
-            done
+            done < <(echo "$APPS") # preserve DEPLOYMENTS variable
         done
         if [ "$DEPLOYMENTS" == "" ]; then
             DEPLOYMENTS='"deployments": "NONE"'
@@ -749,20 +1139,22 @@ execute_task() {
 
         for TREE in $TREES; do
             APPS=$(jq -rc '.apps[]' $TREE)
-            for APP in $APPS; do
+            #for APP in $APPS; do
+            while IFS= read -r APP; do
                 APP_NAME=$(echo "$APP" | jq -r '.name' | awk '{print tolower($0)}')
+                APP_SUBTITLE=$(echo "$APP" | jq -r '.subtitle')
                 APP_VERSION=$(echo "$APP" | jq -r '.version')
                 APP_DIR=$(dirname $TREE)"/"$APP_NAME
-                debug "$APP_TEMPLATE"
                 if [ "$APP_NAME" == "$DEPLOY_NAME" ]; then
                     if [ "$DEPLOY_ACTION" == "ask" ]; then
                         APP_TEMPLATE=$APP_DIR"/template.json"
                         TEMPLATE=$(cat $APP_TEMPLATE | base64 -w0)
+                        debug "$APP_TEMPLATE"
                         JSON_TARGET=$(echo '{"DATE":"'$DATE'","STATUS": "0","TEMPLATE":"'$TEMPLATE'"}' | jq -r . | base64 -w0)
                     elif [ "$DEPLOY_ACTION" == "reinstall" ]; then
                         APP_TEMPLATE=$APP_DIR"/template.json"
                         TEMPLATE=$(cat $APP_TEMPLATE)
-                        for LINE in $(cat $SERVICE_DIR/service-$DEPLOY_NAME.json | jq -rc '.containers[].ENVS[] | to_entries[]'); do
+                        for LINE in $(cat $SERVICE_DIR/service-$DEPLOY_NAME.json | jq -rc '.containers[].ENVS[] | to_entries[]' 2>/dev/null); do
                             KEY=$(echo $LINE | jq -r .key)
                             VALUE=$(echo $LINE | jq -r .value)
                             debug "$KEY: $VALUE"
@@ -771,14 +1163,14 @@ execute_task() {
                             TEMPLATE=$(echo "$TEMPLATE" | jq -r '.fields |= map(if .key == "'$KEY'" then .value = "'$VALUE'" else . end)')
                         done
                         # write ENV value from domain file to template value by key name
-                        for LINE in $(cat $SERVICE_DIR/domain-$DEPLOY_NAME.json | jq -rc '.containers[].ENVS[] | to_entries[]'); do
+                        for LINE in $(cat $SERVICE_DIR/domain-$DEPLOY_NAME.json | jq -rc '.containers[].ENVS[] | to_entries[]' 2>/dev/null); do
                             KEY=$(echo $LINE | jq -r .key)
                             VALUE=$(echo $LINE | jq -r .value)
                             debug "$KEY: $VALUE"
                             TEMPLATE=$(echo "$TEMPLATE" | jq -r '.fields |= map(if .key == "'$KEY'" then .value = "'$VALUE'" else . end)')
                         done
                         # write ENV value from secret file to template value by key name
-                        for LINE in $(cat $SECRET_DIR/$DEPLOY_NAME/$DEPLOY_NAME.json | jq -rc '.[] | to_entries[]'); do
+                        for LINE in $(cat $SECRET_DIR/$DEPLOY_NAME/$DEPLOY_NAME.json | jq -rc '.[] | to_entries[]' 2>/dev/null); do
                             KEY=$(echo $LINE | jq -r .key)
                             VALUE=$(echo $LINE | jq -r .value)
                             debug "$KEY: $VALUE"
@@ -788,6 +1180,7 @@ execute_task() {
 
                         TEMPLATE=$(echo "$TEMPLATE" | base64 -w0)
                         JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "STATUS": "0", "TEMPLATE": "'$TEMPLATE'" }' | jq -r . | base64 -w0)
+
                     elif [ "$DEPLOY_ACTION" == "deploy" ]; then
                         JSON_TARGET=""
                         #JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "STATUS": "1" }' | jq -r . | base64 -w0) # deployment has started
@@ -803,6 +1196,18 @@ execute_task() {
                         DEPLOY_PAYLOAD=$(echo "$JSON" | jq -r .PAYLOAD) # base64 list of key-value pairs in JSON
                         deploy_additionals "$APP_DIR" "$DEPLOY_NAME" "$DEPLOY_PAYLOAD"
                         sh /scripts/check_pid.sh "$PID" "$SHARED" "deploy-$DEPLOY_NAME" "$DATE" "$DEBUG" &
+
+                    elif [ "$DEPLOY_ACTION" == "edit" ]; then
+                        JSON_TARGET=""
+                        DEPLOY_PAYLOAD=$(echo "$JSON" | jq -r .PAYLOAD) # base64 list of key-value pairs in JSON
+
+                        # stop service before edit
+                        debug "$service_exec service-$DEPLOY_NAME.json stop force"
+                        $service_exec service-$DEPLOY_NAME.json stop force
+
+                        deploy_additionals "$APP_DIR" "$DEPLOY_NAME" "$DEPLOY_PAYLOAD"
+                        sh /scripts/check_pid.sh "$PID" "$SHARED" "deploy-$DEPLOY_NAME" "$DATE" "$DEBUG" &
+
                     elif [ "$DEPLOY_ACTION" == "uninstall" ]; then
                         remove_additionals "$APP_DIR" "$DEPLOY_NAME"
                         # uninstall has finished
@@ -812,7 +1217,7 @@ execute_task() {
                         JSON_TARGET=""
                     fi
                 fi
-            done
+            done < <(echo "$APPS") # preserve variables
         done
 
     elif [ "$TASK_NAME" == "repositories" ]; then
@@ -839,20 +1244,25 @@ execute_task() {
 
         VPN_STATUS="0"
         VPN_RESULT=""
+        if [ -f $SECRET_DIR/vpn-proxy/wg0.conf ]; then
             CONTAINERS=$(docker ps -a --format '{{.Names}} {{.Status}}' | grep -w wireguardproxy)
             if [ "$CONTAINERS" != "" ]; then
                 UP=$(echo $CONTAINERS | grep -w 'Up')
                 if [ "$UP" != "" ]; then
-                VPN_STATUS="2"
-            else
                     VPN_STATUS="1"
+                else
+                    VPN_STATUS="2"
                 fi
                 VPN_RESULT=$(echo "$CONTAINERS" | base64 -w0)
             fi
+        fi
         JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "STATUS": "'$VPN_STATUS'", "RESULT": "'$VPN_RESULT'" }' | jq -r . | base64 -w0)
 
     elif [ "$TASK_NAME" == "save_vpn" ]; then
 
+        JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "STATUS": "0", "RESULT": "" }' | jq -r . | base64 -w0)
+        add_json_target
+
         VPN_PROXY_REPO="wireguard-proxy-client"
         if [ ! -d "/tmp/$VPN_PROXY_REPO" ]; then
             git clone https://git.format.hu/safebox/$VPN_PROXY_REPO.git /tmp/$VPN_PROXY_REPO >/dev/null
@@ -870,29 +1280,103 @@ execute_task() {
         # install vpn only
         sh /scripts/install.sh "$B64_JSON" "$service_exec" "vpn" "$GLOBAL_VERSION"
 
-        JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "STATUS": "'$VPN_STATUS'", "RESULT": "'$VPN_RESULT'" }' | jq -r . | base64 -w0)
+        JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "STATUS": "1", "RESULT": "'$VPN_RESULT'" }' | jq -r . | base64 -w0)
 
     elif [ "$TASK_NAME" == "containers" ]; then # not in use
         CONTAINERS=$(docker ps -a --format '{{.Names}} {{.Status}}' | grep -v framework-scheduler)
         RESULT=$(echo "$CONTAINERS" | base64 -w0)
         JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "RESULT": "'$RESULT'" }' | jq -r . | base64 -w0)
+
+    elif [ "$TASK_NAME" == "backup" ]; then
+
+        TASK_TYPE=$(echo $B64_JSON | base64 -d | jq -r '.TASK_TYPE')
+
+        if [ "$TASK_TYPE" == "backup_query_state" ]; then
+            echo "task type is backup_query_state"
+
+        elif [ "$TASK_TYPE" == "backup_set_service" ]; then
+
+            PASSWORD="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_SERVER_PASSWORD')"
+            PERIOD="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_PERIOD')"
+            COMPRESSION="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_COMPRESSION')"
+            PLANNED_TIME="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_PLANNED_TIME')"
+            DIRECTRIES="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_DIRECTORIES')"
+            SERVICES="$(echo $B64_JSON | base64 -d | jq -r '.SERVICES')"
+            BACKUP_LOCAL_CLIENTS="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_LOCAL_CLIENTS')"
+            BACKUP_VPN_CLIENTS="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_VPN_CLIENTS')"
+            VPN="$(echo $B64_JSON | base64 -d | jq -r '.VPN')"
+            SSH_PORT="$(echo $B64_JSON | base64 -d | jq -r '.SSH_PORT')"
+            SSH_USER="$(echo $B64_JSON | base64 -d | jq -r '.SSH_USER')"
+            SSH_PASSWORD="$(echo $B64_JSON | base64 -d | jq -r '.SSH_PASSWORD')"
+            OPERATION="$(echo $B64_JSON | base64 -d | jq -r '.OPERATION')"
+
+            echo "task type is backup_set_service"
+            backup_set_service "$PASSWORD" "$PERIOD" "$COMPRESSION" "$PLANNED_TIME" "$DIRECTRIES" "$SERVICES" "$BACKUP_LOCAL_CLIENTS" "$BACKUP_VPN_CLIENTS" "$VPN" "$SSH_PORT" "$SSH_USER" "$SSH_PASSWORD" "$OPERATION"
+
+        elif [ "$TASK_TYPE" == "backup_set_client" ]; then
+            
+            NAME="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_CLIENT_NAME')"
+            SIZE="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_CLIENT_SIZE')"
+            VPN="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_CLIENT_VPN')"
+            SSH_PORT="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_CLIENT_SSH_PORT')"
+            SSH_USER="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_CLIENT_SSH_USER')"
+            SSH_PASSWORD="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_CLIENT_SSH_PASSWORD')"
+            OPERATION="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_CLIENT_OPERATION')"
+            
+            debug "task type is backup_set_client for $NAME"
+            backup_set_client "$NAME" "$SIZE" "$VPN" "$SSH_PORT" "$SSH_USER" "$SSH_PASSWORD" "$OPERATION"
+
+        elif [ "$TASK_TYPE" == "backup_challenge_clients" ]; then
+            echo "task type is backup_challenge_clients"
+
+        elif [ "$TASK_TYPE" == "restore_from_backup" ]; then
+            echo "task type is restore_from_backup"
+
+        else
+            echo "Unknown task type: $TASK_TYPE"
+        fi
+
+        RESULT=$(echo "$CONTAINERS" | base64 -w0)
+        JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "RESULT": "'$RESULT'" }' | jq -r . | base64 -w0)
+
     elif [ "$TASK_NAME" == "upgrade" ]; then
         JSON="$(echo $B64_JSON | base64 -d)"
         NAME=$(echo "$JSON" | jq -r .NAME | awk '{print tolower($0)}')
         if [ "$NAME" == "framework" ]; then
+            JSON_TARGET=$(echo '{"DATE":"'$DATE'","INSTALL_STATUS":0}' | jq -r . | base64 -w0)
+            add_json_target $NAME
+            echo "Upgrading service: webserver"
+            upgrade webserver
+
+            echo "Upgrading framework scheduler..."
+            echo "Cleaning temporary files..."
+            
+            rm -rf /var/tmp/shared/input/*
+            rm -rf /var/tmp/shared/output/*
+
             upgrade_scheduler
-		upgrade "web-installer"
+            echo "Removing old framework scheduler container..."
+            JSON_TARGET=$(echo '{"DATE":"'$DATE'","INSTALL_STATUS":1,"VERSION":'$VERSION'}' | jq -r . | base64 -w0)
+            add_json_target $NAME
+            sleep 1
+            /usr/bin/docker rm -f $HOSTNAME
+
+            #CONTAINERS=$(docker ps -a --format '{{.Names}} {{.Status}}' | grep -E 'framework-scheduler|webserver')
         else
+            echo "Upgrading service: $NAME"
             upgrade "$NAME"
+            #CONTAINERS=$(docker ps -a --format '{{.Names}} {{.Status}}' | grep -w "$NAME")
         fi
+        #RESULT=$(echo "$CONTAINERS" | base64 -w0)
+        sh /scripts/check_pid.sh "$PID" "$SHARED" "$TASK_NAME-$NAME" "$DATE" "$DEBUG" &
     fi
 
+    if [ "$TASK_NAME" != "check_vpn" ]; then
         debug "JSON_TARGET: $JSON_TARGET"
+    fi
 
     if [ "$JSON_TARGET" != "" ]; then
-        #redis-cli -h $REDIS_SERVER -p $REDIS_PORT SET $TASK "$JSON_TARGET"
+        add_json_target
-        install -m 664 -g 65534 /dev/null $SHARED/output/$TASK.json
-        echo $JSON_TARGET | base64 -d >$SHARED/output/$TASK.json
     fi
 
 }
@@ -962,39 +1446,15 @@ check_redis_availability() {
     done
 }
 
-start_framework_scheduler() {
-
-    if [ "$DEBUG_MODE" == "true" ]; then
-        DOCKER_START="--entrypoint=sh $DOCKER_REGISTRY_URL/$FRAMEWORK_SCHEDULER_IMAGE:$FRAMEWORK_SCHEDULER_VERSION -c 'sleep 86400'"
-    else
-        DOCKER_START="$DOCKER_REGISTRY_URL/$FRAMEWORK_SCHEDULER_IMAGE:$FRAMEWORK_SCHEDULER_VERSION"
-    fi
-    DOCKER_RUN="/usr/bin/docker run -d \
-        -v SHARED:/var/tmp/shared \
-	  	-v /var/run/docker.sock:/var/run/docker.sock \
-		-v SYSTEM_DATA:/etc/system/data \
-		-v SYSTEM_CONFIG:/etc/system/config \
-		-v SYSTEM_LOG:/etc/system/log \
-		-v USER_DATA:/etc/user/data \
-		-v USER_CONFIG:/etc/user/config \
-		-v USER_SECRET:/etc/user/secret \
-		--restart=always \
-		--name $FRAMEWORK_SCHEDULER_NAME \
-	  	--env WEBSERVER_PORT=$WEBSERVER_PORT \
-	  	--network $FRAMEWORK_SCHEDULER_NETWORK \
-		--env RUN_FORCE=$RUN_FORCE \
-		--env DOCKER_REGISTRY_URL=$DOCKER_REGISTRY_URL \
-	  $DOCKER_START"
-    eval "$DOCKER_RUN"
-
-}
-
 ### SYSTEM INITIALIZATION ###
+## Start prevously deployed firewall rules depend on framework scheduler startup at first time
 
-## DOCKER NETWORK VARIABLES
+if [ -d /etc/user/config/services ]; then
-## FILESYSTEM VARIABLES
+    cd /etc/user/config/services
-## PORTS VARIABLES
+    for FIREWALL in $(ls firewall*.json); do
-### RESTART SCHEDULER IF NEEDED
+        $service_exec $FIREWALL start &
+    done
+fi
 
 SN=$(check_subnets)
 if [ "$SN" != "1" ]; then
@@ -1011,7 +1471,7 @@ fi
 
 VOL=$(check_volumes)
 if [ "$VOL" != "1" ]; then
-    start_framework_scheduler
+    upgrade_scheduler
     /usr/bin/docker rm -f $HOSTNAME
 fi
 
@@ -1023,15 +1483,12 @@ if [ "$DF" != "1" ]; then
 fi
 
 #RS=$(docker ps | grep redis-server)
-WS=$(docker ps | grep webserver)
+WS=$(/usr/bin/docker ps | grep -o webserver)
 
-#if [[ "$WS" == "" && "$RS" == "" ]]; then
 if [ "$WS" == "" ]; then
-
     # START SERVICES
-    #$service_exec service-framework.containers.redis-server start &
+    echo "Starting webserver"
     $service_exec service-framework.containers.webserver start &
-    sleep 5
 
 fi
 
@@ -1043,13 +1500,21 @@ DATE=$(date +%F-%H-%M-%S)
 DIR=$SHARED/input
 
 # Triggers by certificate or domain config changes
+# Set installed version number
+echo '{}' | jq --arg VERSION "$VERSION" '.VERSION = $VERSION' > $SHARED/output/version.json
+############################
 
+if [ "$DEBUG_MODE" == "true" ]; then
+    rm $DIR/*
+fi
 unset IFS
 
 inotifywait --exclude "\.(swp|tmp)" -m -e CREATE,CLOSE_WRITE,DELETE,MOVED_TO -r $DIR |
     while read dir op file; do
         if [ "${op}" == "CLOSE_WRITE,CLOSE" ]; then
+            if [ "$file" != "check_vpn.json" ]; then
                 echo "new file created: $file"
+            fi
             B64_JSON=$(cat $DIR/$file | base64 -w0)
             TASK=$(echo $file | cut -d '.' -f1)
             execute_task "$TASK" "$B64_JSON"

scripts/scheduler/install.sh

@@ -200,6 +200,8 @@ elif [ "$FIRST_INSTALL" == "vpn" ]; then
 
     get_vpn_key
 
+    if [ "$VPN_PROXY" != "no" ]; then
+
 	    edit_user_json $LETSENCRYPT_MAIL $LETSENCRYPT_SERVERNAME
 
 	    $SERVICE_EXEC vpn-proxy stop force
@@ -209,6 +211,8 @@ elif [ "$FIRST_INSTALL" == "vpn" ]; then
 	    echo "$INIT_SERVICE_PATH/firewall-vpn-proxy-postrouting" >>$AUTO_START_SERVICES/.init_services
 	    echo "$INIT_SERVICE_PATH/firewall-vpn-proxy-prerouting" >>$AUTO_START_SERVICES/.init_services
 
+    fi;
+
     exit
 
 else