Merge branch 'main' of https://git.format.hu/safebox/framework-scheduler

Dockerfile

@@ -20,4 +20,4 @@ COPY scripts/scheduler/*.sh /scripts/
 RUN find ./scripts -name "*.sh" | xargs dos2unix
 RUN ["chmod", "+x", "-R", "/scripts/"]
 
-ENTRYPOINT ["/scripts/entrypoint.sh"]
+CMD /scripts/entrypoint.sh

scripts/scheduler/backup_challenge_clients.sh

@@ -0,0 +1,89 @@
+# Get MY IP
+
+# Get VPN network if exists
+
+# Define port
+
+# Define local IP range
+
+# Define VPN IP range
+
+# Store results
+
+
+scan_network(){
+        MyIP=$(ifconfig ${Interface}|grep inet |awk '{ print $2 }');
+        TargetIP=$(echo $MyIP|cut -d . -f1-3);
+        X=0
+	OpenIP=""
+        for i in $(seq 1 255); do
+                nc -w 1 -z $TargetIP.$i 60022;
+                if [ $? -eq 0 ]
+                then
+			if [ $MyIP != $TargetIP.$i ]
+                        then
+                                if [ $X = 1 ]
+                                then
+                                        # tobb nyitott IP
+					echo "Found more than one IP addresses"
+                                        echo "MAILKULDES"
+					echo "">OpenIP.txt;
+					# TODO mailkuldes ahova kell
+					exit 1;
+				else
+					OpenIP=$TargetIP.$i;
+                                fi
+				X=1;
+                        fi
+                fi
+	done
+	if [ $X = 1  ]
+	then
+		echo $OpenIP>OpenIP.txt;
+		echo "start LVM SYNC";
+		echo "OpenIP mukodik = "$OpenIP;
+		lvm_sync_create $OpenIP;
+	else
+		echo "No available local IP address found!"
+		try_target_VPN;
+	fi
+
+
+}
+
+try_target_IP(){
+	MyIP=$(ifconfig ${Interface}|grep inet |awk '{ print $2 }');
+ 	nc -w 1 -z $OpenIP 60022;
+        if [ $? -eq 0 ] 
+                then
+			if [ $MyIP = $OpenIP ]
+				then	
+				echo "Only own IP address found = "$OpenIP
+				scan_network;
+			fi
+	else
+	scan_network;
+	fi
+}
+
+try_target_VPN(){
+	nc -w 1 -z $VPN 60022;
+        if [ $? -eq 0 ]
+		then
+		for i in {0..99}; do
+		MyVPN=$(ifconfig tun$i 2>/dev/null |grep inet |awk '{ print $2 }');
+			echo "My VPN="$MyVPN;
+			echo "Found VPN="$VPN;
+			if [ $VPN != $MyVPN ]
+				then
+				echo "VPN accessible="$VPN;
+				lvm_sync_create $VPN;
+					else
+					echo "Only own VPN accessible="$VPN;
+					exit 1;
+			fi
+		done
+		else
+		echo "No available server"
+	fi
+}

scripts/scheduler/entrypoint.sh

@@ -100,6 +100,116 @@ if [ -d /etc/user/config/services ]; then
     done
 fi
 
+backup_query_state() {
+
+}
+
+backup_set_service() {
+
+}
+
+backup_set_client() {
+
+    NAME="$1"
+    SIZE="$2"
+    VPN="$3"
+    SSH_PORT="$4"
+    SSH_USER="$5"
+    SSH_PASSWORD="$6"
+    OPERATION="$7"
+    VPN_KEY="$8"
+
+    if [ "$OPERATION" == "DELETE" ]; then
+        # delete service
+        if [ -f "/etc/user/config/services/service-backup-client-$NAME.json" ]; then
+
+            debug "service-backup-client-$NAME.json stop force dns-remove"
+            $service_exec service-backup-client-$NAME.json stop force dns-remove
+            rm -f /etc/user/config/services/service-backup-client-$NAME.json
+            debug "Service backup client $NAME deleted."
+
+        fi
+
+    else
+
+        if [ -z "$SSH_PORT" ] ; then
+            SSH_PORT=20022
+        fi
+
+        if [ "$VPN" == "true" ]; then
+            NETWORK=$NAME
+            PORT='"PORTS": [{"SOURCE":"null","DEST":"'$SSH_PORT'","TYPE":"tcp"}],'
+        else
+            NETWORK="host"
+            PORT='"PORTS": [{"SOURCE":"'$SSH_PORT'","DEST":"'$SSH_PORT'","TYPE":"tcp"}],'
+        fi
+
+        ADDITIONAL=""
+        ADDITIONAL='"EXTRA": "--label logging=promtail_user --label logging_jobname=containers --restart=always", "PRE_START": [], "DEPEND": [], "CMD": ""'
+        ENVS='"ENVS": [{"SSH_USER":"'$SSH_USER'"},{"SSH_PORT":"'$SSH_PORT'"},{"SSH_PASSWORD":"'$SSH_PASSWORD'"},{"VPN_CLIENT_KEY":"'$VPN_KEY'"}],'
+
+        echo '{
+        "main": {
+        "SERVICE_NAME": "'$NAME'"
+        },
+        "containers": [
+        {
+        "IMAGE": "alpine:latest",
+        "NAME": "'$NAME'-init",
+        "UPDATE": "true",
+        "MEMORY": "64M",
+        "EXTRA": "--rm",
+        "VOLUMES":[
+            {
+            "SOURCE": "USER_DATA",
+            "DEST": "/etc/user/data/",
+            "TYPE": "rw"
+            }
+                ],
+        "ENTRYPOINT": "sh -c",
+        "CMD": "mkdir -p /etc/user/data/backup/clients/'$NAME'/backup && /etc/user/data/backup/clients/'$NAME'/ssh",
+        "POST_START": []
+        },
+        {
+        "IMAGE": "safebox/backup-client:latest",
+        "NAME": "'$NAME'",
+        "UPDATE": "true",
+        "MEMORY": "64M",
+        "NETWORK": "'$NETWORK'",
+        '$ADDITIONAL',
+        '$ENVS'
+        '$PORT'
+        "VOLUMES":[
+            { 
+            "SOURCE": "/etc/user/data/backup/clients/'$NAME'/backup",
+            "DEST": "/backup",
+            "TYPE": "rw"
+            },
+            { 
+            "SOURCE": "/etc/user/data/backup/clients/'$NAME'/ssh",
+            "DEST": "/home/'$SSH_USER'/",
+            "TYPE": "rw"
+            }
+                ],
+        "POST_START": []
+        }
+      ]
+    }' | jq -r . >/etc/user/config/services/service-backup-client-$NAME.json
+
+    debug "service-backup-client-$NAME.json stop force dns-remove"
+    $service_exec service-backup-client-$NAME.json start &
+
+    fi
+
+}
+
+backup_challenge_clients() {
+}
+
+restore_from_backup() {
+
+}
+
 create_htpasswd_file() {
 
     local USER="$1"
@@ -111,8 +221,6 @@ create_htpasswd_file() {
     fi
 }
 
-install -m 664 -g 65534 /dev/null
-
 deploy_additionals() {
 
     local DIR="$1"
@@ -160,14 +268,80 @@ remove_additionals() {
 
     debug "UNINSTALL: $NAME"
 
+    # delete firewall rules
+    FIREWALLS=""
+    FIREWALLS="$(ls $SERVICE_DIR/firewall-*.json | grep $NAME)"
+    for FIREWALL in $(echo $FIREWALLS); do
+        cat $FIREWALL | jq '.containers[] |= (
+        if (.ENVS | map(has("OPERATION")) | any) then
+            # If any entry has OPERATION key, update it
+            .ENVS = [.ENVS[] | if has("OPERATION") then {"OPERATION": "DELETE"} else . end]
+        else
+            # If no entry has OPERATION key, add new entry
+            .ENVS += [{"OPERATION": "DELETE"}]
+        end
+        )' >$FIREWALL.tmp
+        debug "$service_exec $FIREWALL.tmp start info"
+        $service_exec $FIREWALL.tmp start info
+        rm $FIREWALL.tmp
+    done
+
+    # delete domains
+    DOMMAINS=""
+    DOMAINS="$(ls $SERVICE_DIR/domain-*.json | grep $NAME)"
+    for DOMAIN in $(echo $DOMAINS); do
+        cat $DOMAIN | jq '.containers[] |= (
+        if (.ENVS | map(has("OPERATION")) | any) then
+            # If any entry has OPERATION key, update it
+            .ENVS = [.ENVS[] | if has("OPERATION") then {"OPERATION": "DELETE"} else . end]
+        else
+            # If no entry has OPERATION key, add new entry
+            .ENVS += [{"OPERATION": "DELETE"}]
+        end
+        )' >$DOMAIN.tmp
+        debug "$service_exec $DOMAIN.tmp start info"
+        $service_exec $DOMAIN.tmp start info
+        rm $DOMAIN.tmp
+    done
+
+    # remove related directories and files
+    # get volume destinations
+    DESTINATIONS=""
+    VOLUMES=""
+    DESTINATIONS=$(cat $SERVICE_DIR/service-$NAME.json | jq -r '[.containers[] | select(has("VOLUMES")) | .VOLUMES[] | select(.SHARED != "true") | .SOURCE] | unique[]' | grep $NAME)
+    for DESTINATION in $(echo $DESTINATIONS); do
+        if [ -d "$DESTINATION" ] || [ -f "$DESTINATION" ]; then
+            rm -rf $DESTINATION
+            debug "deleted directory or file: $DESTINATION"
+        fi
+    done
+
+    ENV_FILES=$(cat $SERVICE_DIR/service-$NAME.json | jq -r '[.containers[] | select(has("ENV_FILES")) | .ENV_FILES[]] | unique[]')
+    for ENV_FILE in $(echo $ENV_FILES); do
+        if [ -f "$ENV_FILE" ]; then
+            rm -rf $ENV_FILE
+            debug "deleted enviroment file: $ENV_FILE"
+        fi
+    done
+
+    VOLUMES=$(cat $SERVICE_DIR/service-$NAME.json | jq -r '[.containers[] | select(has("VOLUMES")) | .VOLUMES[] | select(.SHARED != "true") | .SOURCE] | unique[]' | grep -vE 'USER|SYSTEM')
+
     # stop service
     # force - remove stopped container, docker rm
     debug "$service_exec service-$NAME.json stop force dns-remove"
     $service_exec service-$NAME.json stop force dns-remove
 
+    for VOLUME in $(echo $VOLUMES | grep -vE 'USER|SYSTEM|SHARED'); do
+        if [ "$(echo $VOLUME | cut -d '/' -f1)" ]; then
+            docker volume rm $VOLUME
+            debug "deleted volume: $VOLUME"
+
+        fi
+    done
+
     # remove service files
     rm $SERVICE_DIR/*"-"$NAME.json # service, domain, etc.
-    rm $SECRET_DIR/$NAME/$NAME.json
+
 }
 
 get_repositories() {
@@ -182,10 +356,10 @@ get_repositories() {
 
         BASE=$(basename $REPO | cut -d '.' -f1)
         if [ ! -d "/tmp/$BASE" ]; then
-            git clone $REPO /tmp/$BASE >/dev/null
+            GIT_HTTP_CONNECT_TIMEOUT=10 GIT_HTTP_TIMEOUT=30 git clone $REPO /tmp/$BASE >/dev/null
         else
             cd /tmp/$BASE
-            git pull >/dev/null
+            GIT_HTTP_CONNECT_TIMEOUT=10 GIT_HTTP_TIMEOUT=30 git pull >/dev/null
         fi
         if [ -f "/tmp/$BASE/applications-tree.json" ]; then
             TREES=$TREES" /tmp/$BASE/applications-tree.json"
@@ -550,16 +724,19 @@ upgrade() {
 
     if [ "$NAME" == "web-installer" ]; then
 
-        debug "$service_exec service-framework.containers.webserver start info"
+        debug "$service_exec service-framework.containers.webserver stop force"
         $service_exec service-framework.containers.webserver stop force
+        debug "$service_exec service-framework.containers.webserver start info"
         $service_exec service-framework.containers.webserver start info &
 
     else
 
-        debug "$service_exec $NAME.json start info"
+        debug "$service_exec $NAME.json stop force"
         $service_exec $NAME.json stop force
+        debug "$service_exec $NAME.json start info"
         $service_exec $NAME.json start info &
     fi
+    PID=$!
 }
 
 execute_task() {
@@ -594,7 +771,7 @@ execute_task() {
         #fi;
         JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "INSTALL_STATUS": "'$INSTALL_STATUS'" }' | jq -r . | base64 -w0)
 
-    elif [ "$TASK_NAME" == "letsencrypt" ]; then
+    elif [ "$TASK_NAME" == "request_letsencrypt" ]; then
         DOMAINS=$(echo $B64_JSON | base64 -d | jq -r 'keys[]')
         for DOMAIN in $(echo $DOMAINS); do
             REQUEST=$(echo $B64_JSON | base64 -d | jq -r ".[\"$DOMAIN\"].status")
@@ -604,6 +781,7 @@ execute_task() {
                 touch /etc/system/data/ssl/keys/$DOMAIN/new_certificate
             fi
         done
+        JSON_TARGET=$B64_JSON
 
     elif [ "$TASK_NAME" == "system" ]; then
         #SYSTEM_LIST="core-dns.json cron.json domain-local-backend.json firewall-letsencrypt.json firewall-local-backend.json firewall-localloadbalancer-dns.json firewall-localloadbalancer-to-smarthostbackend.json firewall-smarthost-backend-dns.json firewall-smarthost-loadbalancer-dns.json firewall-smarthost-to-backend.json firewall-smarthostloadbalancer-from-publicbackend.json letsencrypt.json local-backend.json local-proxy.json service-framework.json smarthost-proxy-scheduler.json smarthost-proxy.json"
@@ -730,18 +908,22 @@ execute_task() {
     elif [ "$TASK_NAME" == "deployments" ]; then
         DEPLOYMENTS=""
         TREES=$(get_repositories)
-        for TREE in $TREES; do
+        for TREE in "$TREES"; do
             APPS=$(jq -rc '.apps[]' $TREE)
-            for APP in $APPS; do
+            #for APP in "$APPS"; do #space problem
+            while IFS= read -r APP; do
                 APP_NAME=$(echo "$APP" | jq -r '.name')
+                APP_SUBTITLE="$(echo "$APP" | jq -r '.subtitle')"
                 APP_VERSION=$(echo "$APP" | jq -r '.version')
+                APP_ICON=$(echo "$APP" | jq -r '.icon')
                 if [ "$DEPLOYMENTS" != "" ]; then
                     SEP=","
                 else
                     SEP=""
                 fi
-                DEPLOYMENTS=$DEPLOYMENTS$SEP'"'$APP_NAME'": "'$APP_VERSION'"'
-            done
+                DEPLOYMENTS="$DEPLOYMENTS"$SEP'"'$APP_NAME'": {"subtitle": "'"$APP_SUBTITLE"'", "version": "'"$APP_VERSION"'", "icon":
+ "'"$APP_ICON"'"}'
+            done < <(echo "$APPS") # preserve DEPLOYMENTS variable
         done
         if [ "$DEPLOYMENTS" == "" ]; then
             DEPLOYMENTS='"deployments": "NONE"'
@@ -777,6 +959,7 @@ execute_task() {
             APPS=$(jq -rc '.apps[]' $TREE)
             for APP in $APPS; do
                 APP_NAME=$(echo "$APP" | jq -r '.name' | awk '{print tolower($0)}')
+                APP_SUBTITLE=$(echo "$APP" | jq -r '.subtitle')
                 APP_VERSION=$(echo "$APP" | jq -r '.version')
                 APP_DIR=$(dirname $TREE)"/"$APP_NAME
                 debug "$APP_TEMPLATE"
@@ -830,6 +1013,18 @@ execute_task() {
                         DEPLOY_PAYLOAD=$(echo "$JSON" | jq -r .PAYLOAD) # base64 list of key-value pairs in JSON
                         deploy_additionals "$APP_DIR" "$DEPLOY_NAME" "$DEPLOY_PAYLOAD"
                         sh /scripts/check_pid.sh "$PID" "$SHARED" "deploy-$DEPLOY_NAME" "$DATE" "$DEBUG" &
+
+                    elif [ "$DEPLOY_ACTION" == "edit" ]; then
+                        JSON_TARGET=""
+                        DEPLOY_PAYLOAD=$(echo "$JSON" | jq -r .PAYLOAD) # base64 list of key-value pairs in JSON
+
+                        # stop service before edit
+                        debug "$service_exec service-$DEPLOY_NAME.json stop force"
+                        $service_exec service-$DEPLOY_NAME.json stop force
+
+                        deploy_additionals "$APP_DIR" "$DEPLOY_NAME" "$DEPLOY_PAYLOAD"
+                        sh /scripts/check_pid.sh "$PID" "$SHARED" "deploy-$DEPLOY_NAME" "$DATE" "$DEBUG" &
+
                     elif [ "$DEPLOY_ACTION" == "uninstall" ]; then
                         remove_additionals "$APP_DIR" "$DEPLOY_NAME"
                         # uninstall has finished
@@ -903,15 +1098,62 @@ execute_task() {
         CONTAINERS=$(docker ps -a --format '{{.Names}} {{.Status}}' | grep -v framework-scheduler)
         RESULT=$(echo "$CONTAINERS" | base64 -w0)
         JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "RESULT": "'$RESULT'" }' | jq -r . | base64 -w0)
+
+    elif [ "$TASK_NAME" == "backup" ]; then
+
+        TASK_TYPE=$(echo $B64_JSON | base64 -d | jq -r '.TASK_TYPE)')
+
+        if [ "$TASK_TYPE" == "backup_query_state" ]; then
+            echo "task type is backup_query_state"
+
+        elif [ "$TASK_TYPE" == "backup_set_service" ]; then
+            echo "task type is backup_set_service"
+
+        elif [ "$TASK_TYPE" == "backup_set_client" ]; then
+            
+            NAME="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_CLIENT_NAME')"
+            SIZE="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_CLIENT_SIZE')"
+            VPN="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_CLIENT_VPN')"
+            SSH_PORT="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_CLIENT_SSH_PORT')"
+            SSH_USER="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_CLIENT_SSH_USER')"
+            SSH_PASSWORD="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_CLIENT_SSH_PASSWORD')"
+            OPERATION="$(echo $B64_JSON | base64 -d | jq -r '.BACKUP_CLIENT_OPERATION')"
+            debug "task type is backup_set_client for $NAME"
+            debug "   size: $SIZE"
+            debug "   vpn: $VPN"
+            debug "   ssh_port: $SSH_PORT"
+            debug "   ssh_user: $SSH_USER"
+            debug "   ssh_password: $SSH_PASSWORD"
+            debug "   operation: $OPERATION"
+
+            backup_set_client "$NAME" "$SIZE" "$VPN" "$SSH_PORT" "$SSH_USER" "$SSH_PASSWORD" "$OPERATION"
+
+        elif [ "$TASK_TYPE" == "backup_challenge_clients" ]; then
+            echo "task type is backup_challenge_clients"
+
+        elif [ "$TASK_TYPE" == "restore_from_backup" ]; then
+            echo "task type is restore_from_backup"
+
+        else
+            echo "Unknown task type: $TASK_TYPE"
+        fi
+
+        RESULT=$(echo "$CONTAINERS" | base64 -w0)
+        JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "RESULT": "'$RESULT'" }' | jq -r . | base64 -w0)
+
     elif [ "$TASK_NAME" == "upgrade" ]; then
         JSON="$(echo $B64_JSON | base64 -d)"
         NAME=$(echo "$JSON" | jq -r .NAME | awk '{print tolower($0)}')
         if [ "$NAME" == "framework" ]; then
-            upgrade_scheduler
             upgrade "web-installer"
+            upgrade_scheduler
+            #CONTAINERS=$(docker ps -a --format '{{.Names}} {{.Status}}' | grep -E 'framework-scheduler|webserver')
         else
             upgrade "$NAME"
+            #CONTAINERS=$(docker ps -a --format '{{.Names}} {{.Status}}' | grep -w "$NAME")
         fi
+        #RESULT=$(echo "$CONTAINERS" | base64 -w0)
+        sh /scripts/check_pid.sh "$PID" "$SHARED" "$TASK_NAME-$NAME" "$DATE" "$DEBUG" &
     fi
 
     if [ "$TASK_NAME" != "check_vpn" ]; then