Merge branch 'main' of https://git.format.hu/safebox/framework-scheduler

.drone.yml

@@ -3,7 +3,7 @@ type: kubernetes
 name: default
 
 node_selector:
-  physical-node: dev2
+  physical-node: dev1
 
 trigger:
   event:
@@ -14,26 +14,6 @@ workspace:
   path: /drone/src
 
 steps:
-  - name: build multiarch from dev
-    image: docker.io/owncloudci/drone-docker-buildx:4
-    privileged: true
-    settings:
-      cache-from: [ "registry.dev.format.hu/framework-scheduler" ]
-      registry: registry.dev.format.hu
-      repo: registry.dev.format.hu/framework-scheduler
-      tags: latest
-      dockerfile: Dockerfile
-      username:
-        from_secret: dev-hu-registry-username
-      password: 
-        from_secret: dev-hu-registry-password
-      platforms:
-        - linux/amd64
-        - linux/arm64
-    when:  
-      event:
-        - push
-
   - name: pull image to dockerhub
     image: docker.io/owncloudci/drone-docker-buildx:4
     privileged: true
@@ -51,3 +31,25 @@ steps:
     when:  
       event:
         - tag
+
+  - name: build multiarch from dev
+    image: docker.io/owncloudci/drone-docker-buildx:4
+    privileged: true
+    #environment:
+    #  DOCKER_PLUGIN_MIRROR: "https://mirror.dev.format.hu"
+    settings:
+      cache-from: [ "registry.dev.format.hu/framework-scheduler" ]
+      registry: registry.dev.format.hu
+      repo: registry.dev.format.hu/framework-scheduler
+      tags: latest
+      dockerfile: Dockerfile
+      username:
+        from_secret: dev-hu-registry-username
+      password: 
+        from_secret: dev-hu-registry-password
+      platforms:
+        - linux/amd64
+        - linux/arm64
+    when:  
+      event:
+        - push

Dockerfile

@@ -14,7 +14,7 @@ FROM alpine:latest
 # COPY --from=redis-source /usr/src/redis/src/redis-cli /usr/bin/redis-cli
 # RUN chmod +x /usr/bin/redis-cli
 
-RUN apk add --update --no-cache docker-cli wget curl dos2unix jq openssl git coreutils inotify-tools acl
+RUN apk add --update --no-cache docker-cli wget curl dos2unix jq openssl git coreutils inotify-tools acl apache2-utils
 
 COPY scripts/scheduler/*.sh /scripts/
 RUN find ./scripts -name "*.sh" | xargs dos2unix

scripts/scheduler/entrypoint.sh

@@ -30,6 +30,7 @@ REDIS_VERSION=${REDIS_VERSION:-latest}
 
 SOURCE=${SOURCE:-user-config}
 SMARTHOST_PROXY_PATH=$SMARTHOST_PROXY_PATH
+HTPASSWD_FILE=${HTPASSWD_FILE:-/etc/system/config/smarthost-proxy/nginx/htpasswd}
 
 GIT_URL=${GIT_URL:-git.format.hu}
 REPO=$REPO
@@ -99,6 +100,19 @@ if [ -d /etc/user/config/services ]; then
     done
 fi
 
+create_htpasswd_file() {
+
+    local USER="$1"
+    local PASSWD="$2"
+
+    if [ ! -f "$HTPASSWD_FILE" ]; then
+        install -m 664 -g 65534 /dev/null $HTPASSWD_FILE
+        htpasswd -cb $HTPASSWD_FILE $USER $PASSWD
+    fi
+}
+
+install -m 664 -g 65534 /dev/null
+
 deploy_additionals() {
 
     local DIR="$1"
@@ -146,6 +160,63 @@ remove_additionals() {
 
     debug "UNINSTALL: $NAME"
 
+    # delete firewall rules
+    FIREWALLS=""
+    FIREWALLS="$(ls $SERVICE_DIR/firewall-*.json | grep $NAME)"
+    for FIREWALL in $(echo $FIREWALLS); do
+        cat $FIREWALL | jq '.containers[] |= (
+        if (.ENVS | map(has("OPERATION")) | any) then
+            # If any entry has OPERATION key, update it
+            .ENVS = [.ENVS[] | if has("OPERATION") then {"OPERATION": "DELETE"} else . end]
+        else
+            # If no entry has OPERATION key, add new entry
+            .ENVS += [{"OPERATION": "DELETE"}]
+        end
+        )' >$FIREWALL.tmp
+        debug "$service_exec $FIREWALL.tmp start info"
+        $service_exec $FIREWALL.tmp start info
+        rm $FIREWALL.tmp
+    done
+
+    # delete domains
+    DOMMAINS=""
+    DOMAINS="$(ls $SERVICE_DIR/domain-*.json | grep $NAME)"
+    for DOMAIN in $(echo $DOMAINS); do
+        cat $DOMAIN | jq '.containers[] |= (
+        if (.ENVS | map(has("OPERATION")) | any) then
+            # If any entry has OPERATION key, update it
+            .ENVS = [.ENVS[] | if has("OPERATION") then {"OPERATION": "DELETE"} else . end]
+        else
+            # If no entry has OPERATION key, add new entry
+            .ENVS += [{"OPERATION": "DELETE"}]
+        end
+        )' >$DOMAIN.tmp
+        debug "$service_exec $DOMAIN.tmp start info"
+        $service_exec $DOMAIN.tmp start info
+        rm $DOMAIN.tmp
+    done
+
+    # remove related directories and files
+    # get volume destinations
+    DESTINATIONS=""
+    VOLUMES=""
+    DESTINATIONS=$(cat $SERVICE_DIR/service-$NAME.json | jq -r '[.containers[] | select(has("VOLUMES")) | .VOLUMES[] | select(.SHARED != "true") | .SOURCE] | unique[]' | grep $NAME)
+    for DESTINATION in $(echo $DESTINATIONS); do
+        if [ -d "$DESTINATION" ] || [ -f "$DESTINATION" ]; then
+            rm -rf $DESTINATION
+            debug "deleted directory or file: $DESTINATION"
+        fi
+    done
+
+    VOLUMES=$(cat $SERVICE_DIR/service-$NAME.json | jq -r '[.containers[] | select(has("VOLUMES")) | .VOLUMES[] | select(.SHARED != "true") | .SOURCE] | unique[]' | grep -vE 'USER|SYSTEM')
+    for VOLUME in $(echo $VOLUMES); do
+        if [ "$(echo $VOLUME | cut -d '/' -f1)" ]; then
+            docker volume rm $VOLUME
+            debug "deleted volume: $VOLUME"
+
+        fi
+    done
+
     # stop service
     # force - remove stopped container, docker rm
     debug "$service_exec service-$NAME.json stop force dns-remove"
@@ -153,7 +224,7 @@ remove_additionals() {
 
     # remove service files
     rm $SERVICE_DIR/*"-"$NAME.json # service, domain, etc.
-    rm $SECRET_DIR/$NAME/$NAME.json
+
 }
 
 get_repositories() {
@@ -168,10 +239,10 @@ get_repositories() {
 
         BASE=$(basename $REPO | cut -d '.' -f1)
         if [ ! -d "/tmp/$BASE" ]; then
-            git clone $REPO /tmp/$BASE >/dev/null
+            GIT_HTTP_CONNECT_TIMEOUT=10 GIT_HTTP_TIMEOUT=30 git clone $REPO /tmp/$BASE >/dev/null
         else
             cd /tmp/$BASE
-            git pull >/dev/null
+            GIT_HTTP_CONNECT_TIMEOUT=10 GIT_HTTP_TIMEOUT=30 git pull >/dev/null
         fi
         if [ -f "/tmp/$BASE/applications-tree.json" ]; then
             TREES=$TREES" /tmp/$BASE/applications-tree.json"
@@ -429,30 +500,30 @@ check_update() {
 
     REPOSITORY_URL=$(echo $IMAGE | cut -d '/' -f1)
 
-        # if image repository url doesn't contain dot (safebox)
+    # if image repository url doesn't contain dot (safebox)
-        if [[ "$(echo "$REPOSITORY_URL" | grep '\.')" == "" ]]; then
+    if [[ "$(echo "$REPOSITORY_URL" | grep '\.')" == "" ]]; then
-            REMOTE_URL="registry.hub.docker.com"
+        REMOTE_URL="registry.hub.docker.com"
-            TEMP_PATH=$IMAGE
+        TEMP_PATH=$IMAGE
-	    TEMP_IMAGE=$(echo $TEMP_PATH | cut -d ':' -f1)
-	    TOKEN=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:{$TEMP_IMAGE}:pull" | jq -r .token)
-	    TOKEN_HEADER='-H "Authorization: Bearer '$TOKEN'"'
-        else
-            REMOTE_URL=""
-            # -f2- IMAGE can contain subdirectories
-            TEMP_PATH=$(echo $IMAGE | cut -d '/' -f2-)
-	    TOKEN=""
-	    TOKEN_HEADER=""
-        fi
-
-        debug "TEMP PATH: $TEMP_PATH"
         TEMP_IMAGE=$(echo $TEMP_PATH | cut -d ':' -f1)
-        TEMP_VERSION=$(echo $TEMP_PATH | cut -d ':' -f2)
+        TOKEN=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:{$TEMP_IMAGE}:pull" | jq -r .token)
-        if [ "$TEMP_VERSION" == "$TEMP_IMAGE" ]; then # version is not set
+        TOKEN_HEADER='-H "Authorization: Bearer '$TOKEN'"'
-            TEMP_VERSION="latest"
+    else
-        fi
+        REMOTE_URL=""
+        # -f2- IMAGE can contain subdirectories
+        TEMP_PATH=$(echo $IMAGE | cut -d '/' -f2-)
+        TOKEN=""
+        TOKEN_HEADER=""
+    fi
 
-        REMOTE_URL="https://$REMOTE_URL/v2/$TEMP_IMAGE/manifests/$TEMP_VERSION"
+    debug "TEMP PATH: $TEMP_PATH"
-        debug "REMOTE_URL: $REMOTE_URL"
+    TEMP_IMAGE=$(echo $TEMP_PATH | cut -d ':' -f1)
+    TEMP_VERSION=$(echo $TEMP_PATH | cut -d ':' -f2)
+    if [ "$TEMP_VERSION" == "$TEMP_IMAGE" ]; then # version is not set
+        TEMP_VERSION="latest"
+    fi
+
+    REMOTE_URL="https://$REMOTE_URL/v2/$TEMP_IMAGE/manifests/$TEMP_VERSION"
+    debug "REMOTE_URL: $REMOTE_URL"
 
     # Check whether repository url is available
     #CURL_CHECK="curl -m 5 -s -o /dev/null -w "%{http_code}" https://$REPOSITORY_URL/v2/"
@@ -460,12 +531,12 @@ check_update() {
     CURL_CHECK_CODE=$(eval $CURL_CHECK)
 
     # if valid accessible url
-    if [[ "$CURL_CHECK_CODE" == "200" ]] ; then
+    if [[ "$CURL_CHECK_CODE" == "200" ]]; then
         debug "$REMOTE_URL repository accessed successfully"
 
         #digest=$(curl --silent -H "Accept: application/vnd.docker.distribution.manifest.v2+json" "$REMOTE_URL" | jq -r '.config.digest');
         # Digest for the whole manifest, which includes all architectures.
-	CURL_DIGEST='curl -s -I '"$TOKEN_HEADER"' -H "Accept: application/vnd.oci.image.index.v1+json" '"$REMOTE_URL"' | grep -i Docker-Content-Digest | cut -d " " -f2 | tr -d "\r\n"'
+        CURL_DIGEST='curl -s -I '"$TOKEN_HEADER"' -H "Accept: application/vnd.oci.image.index.v1+json" '"$REMOTE_URL"' | grep -i Docker-Content-Digest | cut -d " " -f2 | tr -d "\r\n"'
         digest=$(eval $CURL_DIGEST)
 
         #debug "docker images -q --no-trunc $REPOSITORY_URL/$TEMP_IMAGE:$TEMP_VERSION";
@@ -532,31 +603,35 @@ upgrade_scheduler() {
 }
 
 upgrade() {
-	local NAME=$1
+    local NAME=$1
 
-	if [ "$NAME" == "web-installer" ]; then
+    if [ "$NAME" == "web-installer" ]; then
 
-		debug "$service_exec service-framework-scheduler.containers.webserver start info"
+        debug "$service_exec service-framework.containers.webserver stop force"
-		$service_exec service-framework-scheduler.containers.webserver stop force
+        $service_exec service-framework.containers.webserver stop force
-		$service_exec service-framework-scheduler.containers.webserver start info &
+        debug "$service_exec service-framework.containers.webserver start info"
+        $service_exec service-framework.containers.webserver start info &
 
-	else	
+    else
 
-		debug "$service_exec service-$NAME.json start info"
+        debug "$service_exec $NAME.json stop force"
-		$service_exec service-$NAME.json stop force
+        $service_exec $NAME.json stop force
-		$service_exec service-$NAME.json start info &
+        debug "$service_exec $NAME.json start info"
-	fi
+        $service_exec $NAME.json start info &
+    fi
+    PID=$!
 }
 
-
 execute_task() {
     TASK="$1"
     B64_JSON="$2"
     DATE=$(date +"%Y%m%d%H%M")
 
     # Executing task
-    debug "TASK: $(echo $TASK | cut -d ':' -f1)"
     TASK_NAME=$(echo $TASK | cut -d ':' -f1)
+    if [ "$TASK_NAME" != "check_vpn" ]; then
+        debug "TASK: $(echo $TASK_NAME | cut -d ':' -f1)"
+    fi
 
     # checking sytem status
     SYSTEM_STATUS=$(ls /etc/user/config/services/*.json | grep -v service-framework.json)
@@ -579,6 +654,18 @@ execute_task() {
         #fi;
         JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "INSTALL_STATUS": "'$INSTALL_STATUS'" }' | jq -r . | base64 -w0)
 
+    elif [ "$TASK_NAME" == "request_letsencrypt" ]; then
+        DOMAINS=$(echo $B64_JSON | base64 -d | jq -r 'keys[]')
+        for DOMAIN in $(echo $DOMAINS); do
+            REQUEST=$(echo $B64_JSON | base64 -d | jq -r ".[\"$DOMAIN\"].status")
+
+            if [ "$REQUEST" == "requested" ]; then
+                echo "New certificate for $DOMAIN is requested."
+                touch /etc/system/data/ssl/keys/$DOMAIN/new_certificate
+            fi
+        done
+        JSON_TARGET=$B64_JSON
+
     elif [ "$TASK_NAME" == "system" ]; then
         #SYSTEM_LIST="core-dns.json cron.json domain-local-backend.json firewall-letsencrypt.json firewall-local-backend.json firewall-localloadbalancer-dns.json firewall-localloadbalancer-to-smarthostbackend.json firewall-smarthost-backend-dns.json firewall-smarthost-loadbalancer-dns.json firewall-smarthost-to-backend.json firewall-smarthostloadbalancer-from-publicbackend.json letsencrypt.json local-backend.json local-proxy.json service-framework.json smarthost-proxy-scheduler.json smarthost-proxy.json"
         SYSTEM_LIST="core-dns.json cron.json letsencrypt.json local-proxy.json service-framework.json smarthost-proxy-scheduler.json smarthost-proxy.json"
@@ -788,6 +875,7 @@ execute_task() {
 
                         TEMPLATE=$(echo "$TEMPLATE" | base64 -w0)
                         JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "STATUS": "0", "TEMPLATE": "'$TEMPLATE'" }' | jq -r . | base64 -w0)
+
                     elif [ "$DEPLOY_ACTION" == "deploy" ]; then
                         JSON_TARGET=""
                         #JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "STATUS": "1" }' | jq -r . | base64 -w0) # deployment has started
@@ -876,18 +964,25 @@ execute_task() {
         CONTAINERS=$(docker ps -a --format '{{.Names}} {{.Status}}' | grep -v framework-scheduler)
         RESULT=$(echo "$CONTAINERS" | base64 -w0)
         JSON_TARGET=$(echo '{ "DATE": "'$DATE'", "RESULT": "'$RESULT'" }' | jq -r . | base64 -w0)
+
     elif [ "$TASK_NAME" == "upgrade" ]; then
         JSON="$(echo $B64_JSON | base64 -d)"
         NAME=$(echo "$JSON" | jq -r .NAME | awk '{print tolower($0)}')
         if [ "$NAME" == "framework" ]; then
-		upgrade_scheduler
+            upgrade_scheduler
-		upgrade "web-installer"
+            upgrade "web-installer"
-	else
+            #CONTAINERS=$(docker ps -a --format '{{.Names}} {{.Status}}' | grep -E 'framework-scheduler|webserver')
-		upgrade "$NAME"
+        else
-	fi
+            upgrade "$NAME"
+            #CONTAINERS=$(docker ps -a --format '{{.Names}} {{.Status}}' | grep -w "$NAME")
+        fi
+        #RESULT=$(echo "$CONTAINERS" | base64 -w0)
+        sh /scripts/check_pid.sh "$PID" "$SHARED" "$TASK_NAME-$NAME" "$DATE" "$DEBUG" &
     fi
 
-    debug "JSON_TARGET: $JSON_TARGET"
+    if [ "$TASK_NAME" != "check_vpn" ]; then
+        debug "JSON_TARGET: $JSON_TARGET"
+    fi
 
     if [ "$JSON_TARGET" != "" ]; then
         #redis-cli -h $REDIS_SERVER -p $REDIS_PORT SET $TASK "$JSON_TARGET"
@@ -1049,7 +1144,9 @@ unset IFS
 inotifywait --exclude "\.(swp|tmp)" -m -e CREATE,CLOSE_WRITE,DELETE,MOVED_TO -r $DIR |
     while read dir op file; do
         if [ "${op}" == "CLOSE_WRITE,CLOSE" ]; then
-            echo "new file created: $file"
+            if [ "$file" != "check_vpn.json" ]; then
+                echo "new file created: $file"
+            fi
             B64_JSON=$(cat $DIR/$file | base64 -w0)
             TASK=$(echo $file | cut -d '.' -f1)
             execute_task "$TASK" "$B64_JSON"