Merge pull request 'removed uneccessary pipelines relevant parts' (#116) from dev into main

Reviewed-on: #116

openwebui/openwebui-secret.json

@@ -5,12 +5,22 @@
         "POSTGRES_PASSWORD": "#POSTGRES_PASSWORD",
         "POSTGRES_ROOT_PASSWORD": "#POSTGRES_ROOT_PASSWORD"
     },
+    "openwebuisearxng": {
+        "SEARXNG_SECRET": "#SEARXNG_SECRET"
+    },
+    "openwebuicrawl4ai": {
+        "CRAWL4AI_API_TOKEN": "#CRAWL4AI_API_TOKEN",
+        "SECRET_KEY": "#CRAWL4AI_SECRET"
+    },
     "openwebuiapp": {
         "WEBUI_ADMIN_NAME": "#WEBUI_ADMIN_NAME",
         "WEBUI_ADMIN_EMAIL": "#WEBUI_ADMIN_EMAIL",
         "WEBUI_ADMIN_PASSWORD": "#WEBUI_ADMIN_PASSWORD",
         "WEBUI_SECRET_KEY": "#WEBUI_SECRET_KEY",
         "OPENAI_API_KEY": "#OPENAI_API_KEY",
-        "DATABASE_URL": "postgres://#POSTGRES_USER:#POSTGRES_PASSWORD@openwebuipostgres-db:5432/#POSTGRES_DB"
+        "HF_TOKEN": "#HF_TOKEN",
+        "DATABASE_URL": "postgres://#POSTGRES_USER:#POSTGRES_PASSWORD@openwebuipostgres-db:5432/#POSTGRES_DB",
+        "EXTERNAL_WEB_SEARCH_API_KEY": "#SEARXNG_SECRET",
+        "EXTERNAL_WEB_LOADER_API_KEY": "#CRAWL4AI_API_TOKEN"
     }
 }

openwebui/service-openwebui.json

@@ -24,7 +24,7 @@
             ],
             "START_ON_BOOT": "false",
             "ENTRYPOINT": "sh -c",
-            "CMD": "mkdir -p /etc/user/data/openwebui/data && mkdir -p /etc/user/data/openwebui/db && mkdir -p /etc/user/data/openwebui/pipelines && mkdir -p /etc/user/data/openwebui/qdrant",
+            "CMD": "mkdir -p /etc/user/data/openwebui/data && mkdir -p /etc/user/data/openwebui/db && mkdir -p /etc/user/data/openwebui/qdrant && mkdir -p /etc/user/data/openwebui/searxng && mkdir -p /etc/user/data/openwebui/crawl4ai && printf %s c2VydmVyOgogIGJpbmRfYWRkcmVzczogMC4wLjAuMAogIHBvcnQ6IDgwODAKICBsaW1pdGVyOiBmYWxzZQogIHB1YmxpY19pbnN0YW5jZTogZmFsc2UKc2VhcmNoOgogIGZvcm1hdHM6CiAgICAtIGh0bWwKICAgIC0ganNvbgp1aToKICBzdGF0aWNfdXNlX2hhc2g6IHRydWUKcmVkaXM6CiAgdXJsOiBmYWxzZQpvdXRnb2luZzoKICB1c2VyYWdlbnQ6IE1vemlsbGEvNS4wIChXaW5kb3dzIE5UIDEwLjA7IFdpbjY0OyB4NjQpIEFwcGxlV2ViS2l0LzUzNy4zNiAoS0hUTUwsIGxpa2UgR2Vja28pIENocm9tZS8xMzEuMC4wLjAgU2FmYXJpLzUzNy4zNgp1c2VfZGVmYXVsdF9zZXR0aW5nczoKICBlbmdpbmVzOgogICAgcmVtb3ZlOgogICAgICAtIGFobWlhCiAgICAgIC0gdG9yY2gKICAgICAgLSB3aWtpZGF0YQpib3RkZXRlY3Rpb246CiAgaXBfbGlzdHM6CiAgICBwYXNzX2lwOgogICAgICAtIDAuMC4wLjAvMAogICAgICAtIDo6LzA= | base64 -d > /etc/user/data/openwebui/searxng/settings.yml && printf %s W2JvdGRldGVjdGlvbi5pcF9saW1pdF0KbGlua190b2tlbiA9IGZhbHNlCg== | base64 -d > /etc/user/data/openwebui/searxng/limiter.toml",
             "PRE_START": "null",
             "POST_START": "null"
         },
@@ -62,10 +62,10 @@
         {
             "IMAGE": "qdrant/qdrant:latest",
             "UPDATE": "true",
-            "NAME": "openwebui-qdrant",
+            "NAME": "openwebuiqdrant-app",
             "MEMORY": "1024M",
             "NETWORK": "openwebui-net",
-            "SELECTOR": "openwebui-qdrant",
+            "SELECTOR": "openwebuiqdrant-app",
             "VOLUMES": [
                 {
                     "SOURCE": "/etc/user/data/openwebui/qdrant",
@@ -93,43 +93,60 @@
             "POST_START": "null"
         },
         {
-            "IMAGE": "ghcr.io/open-webui/pipelines:main",
+            "IMAGE": "searxng/searxng:latest",
             "UPDATE": "true",
-            "NAME": "openwebui-pipelines",
+            "NAME": "openwebuisearxng-app",
-            "MEMORY": "1024M",
+            "MEMORY": "512M",
             "NETWORK": "openwebui-net",
-            "SELECTOR": "openwebui-pipelines",
+            "SELECTOR": "openwebuisearxng-app",
             "VOLUMES": [
                 {
-                    "SOURCE": "/etc/user/data/openwebui/pipelines",
+                    "SOURCE": "/etc/user/data/openwebui/searxng",
-                    "DEST": "/app/pipelines",
+                    "DEST": "/etc/searxng",
                     "TYPE": "rw"
                 }
             ],
             "PORTS": [
                 {
                     "SOURCE": "null",
-                    "DEST": "9099",
+                    "DEST": "8080",
                     "TYPE": "tcp"
                 }
             ],
-            "ENVS": [
+            "ENV_FILES": [
-                {
+                "/etc/user/secret/openwebui/openwebui.json"
-                    "OPENAI_API_BASE_URL": "#OPENAI_API_BASE_URL"
+            ],
+            "EXTRA": "--restart always",
+            "DEPEND": [],
+            "START_ON_BOOT": "false",
+            "CMD": "null",
+            "PRE_START": "null",
+            "POST_START": "null"
         },
         {
-                    "VECTOR_DB": "qdrant"
+            "IMAGE": "unclecode/crawl4ai:latest",
-                },
+            "UPDATE": "true",
+            "NAME": "openwebuicrawl4ai-app",
+            "MEMORY": "1024M",
+            "NETWORK": "openwebui-net",
+            "SELECTOR": "openwebuicrawl4ai-app",
+            "VOLUMES": [
                 {
-                    "QDRANT_HOST": "openwebui-qdrant"
+                    "SOURCE": "/etc/user/data/openwebui/crawl4ai",
-                },
+                    "DEST": "/app/data",
-                {
+                    "TYPE": "rw"
-                    "QDRANT_PORT": "6333"
-                },
-                {
-                    "QDRANT_URI": "http://openwebui-qdrant:6333"
                 }
             ],
+            "PORTS": [
+                {
+                    "SOURCE": "null",
+                    "DEST": "11235",
+                    "TYPE": "tcp"
+                }
+            ],
+            "ENV_FILES": [
+                "/etc/user/secret/openwebui/openwebui.json"
+            ],
             "EXTRA": "--restart always",
             "DEPEND": [],
             "START_ON_BOOT": "false",
@@ -159,6 +176,12 @@
                 }
             ],
             "ENVS": [
+                {
+                    "ENABLE_PERSISTENT_CONFIG": "false"
+                },
+                {
+                    "HF_HUB_OFFLINE": "#DISABLE_HUGGINGFACE"
+                },
                 {
                     "OPENAI_API_BASE_URL": "#OPENAI_API_BASE_URL"
                 },
@@ -166,19 +189,22 @@
                     "VECTOR_DB": "qdrant"
                 },
                 {
-                    "QDRANT_HOST": "openwebui-qdrant"
+                    "QDRANT_URI": "http://openwebuiqdrant-app:6333"
                 },
                 {
-                    "QDRANT_PORT": "6333"
+                    "ENABLE_WEB_SEARCH": "#ENABLE_WEB_SEARCH"
                 },
                 {
-                    "QDRANT_URI": "http://openwebui-qdrant:6333"
+                    "WEB_SEARCH_ENGINE": "#WEB_SEARCH_ENGINE"
                 },
                 {
-                    "ENABLE_RAG_PIPELINE": "True"
+                    "WEB_LOADER_ENGINE": "#WEB_LOADER_ENGINE"
                 },
                 {
-                    "RAG_PIPELINE_URL": "http://openwebui-pipelines:9099/v1"
+                    "SEARXNG_QUERY_URL": "#SEARXNG_QUERY_URL"
+                },
+                {
+                    "EXTERNAL_WEB_LOADER_URL": "#EXTERNAL_WEB_LOADER_URL"
                 }
             ],
             "ENV_FILES": [

openwebui/template.json

@@ -55,6 +55,42 @@
             "advanced": "true",
             "type": "password"
         },
+        {
+            "description": "Hugging Face token",
+            "key": "HF_TOKEN",
+            "value": "",
+            "info": "Optional Hugging Face API token for TTS and model access features.",
+            "advanced": "true",
+            "type": "password"
+        },
+        {
+            "description": "Disable Hugging Face requests",
+            "key": "DISABLE_HUGGINGFACE",
+            "value": "1",
+            "info": "Set to 1 to disable all outgoing requests to huggingface.co at startup. Set to 0 or leave empty to allow Hugging Face access.",
+            "advanced": "true"
+        },
+        {
+            "description": "SearXNG secret key",
+            "key": "SEARXNG_SECRET",
+            "value": "",
+            "required": "true",
+            "generated": "openssl|hex|32"
+        },
+        {
+            "description": "Crawl4AI API token",
+            "key": "CRAWL4AI_API_TOKEN",
+            "value": "",
+            "required": "true",
+            "generated": "random|md5|24"
+        },
+        {
+            "description": "Crawl4AI secret key",
+            "key": "CRAWL4AI_SECRET",
+            "value": "",
+            "required": "true",
+            "generated": "openssl|hex|32"
+        },
         {
             "description": "Postgres database name",
             "key": "POSTGRES_DB",
@@ -82,6 +118,41 @@
             "value": "",
             "required": "true",
             "generated": "random|sha256|20"
+        },
+        {
+            "description": "Enable web search",
+            "key": "ENABLE_WEB_SEARCH",
+            "value": "true",
+            "info": "Optional web search enabling, keep it empty if you don't want it",
+            "advanced": "true"
+        },
+        {
+            "description": "Web search engine",
+            "key": "WEB_SEARCH_ENGINE",
+            "value": "searxng",
+            "info": "Optional web search engine Searxng, keep it empty if you don't want it or define any other if you have",
+            "advanced": "true"
+        },
+        {
+            "description": "External web search engine",
+            "key": "WEB_LOADER_ENGINE",
+            "value": "external",
+            "info": "If optional web search engine defined, keep it as is, otherwise keep it empty",
+            "advanced": "true"
+        },
+        {
+            "description": "Web",
+            "key": "SEARXNG_QUERY_URL",
+            "value": "http://openwebuisearxng-app:8080/search",
+            "info": "Optional web search engine Searxng, keep it empty if you don't want it",
+            "advanced": "true"
+        },
+        {
+            "description": "External web search engine",
+            "key": "EXTERNAL_WEB_LOADER_URL",
+            "value": "http://openwebuicrawl4ai-app:11235/crawl",
+            "info": "Optional crawl4ai application url, keep it as is, otherwise keep it empty",
+            "advanced": "true"
         }    
     ]
 }