add searxng (#413)

* add searxng * fix * fix versions * update readme
truenas · Sep 13, 2024 · e0cfcf4 · e0cfcf4
1 parent 16e2925
commit e0cfcf4
Show file tree

Hide file tree

Showing 30 changed files with 2,060 additions and 1 deletion.
diff --git a/.github/renovate-config.js b/.github/renovate-config.js
@@ -113,6 +113,11 @@ module.exports = {
       "^(?<major>\\d+)\\.(?<minor>\\d+)\\.(?<patch>[1-9]\\d*)\\.(?<build>\\d+)$",
       ["emby/embyserver"]
     ),
+    customVersioning(
+      // 2023.12.31-3535377c9
+      "^(?<major>\\d+)\\.(?<minor>\\d+)\\.(?<patch>\\d+)-.+$",
+      ["searxng/searxng"]
+    ),
   ],
 };
 

diff --git a/README.md b/README.md
@@ -101,7 +101,7 @@ Thank you for your understanding.
 | rsyncd               | community  | ✅    | ✅       |
 | rust-desk            | community  | ✅    | ✅       |
 | sabnzbd              | community  | ✅    | ✅       |
-| searxng              | community  | -     | -        |
+| searxng              | community  | ✅    | ✅       |
 | sftpgo               | community  | -     | -        |
 | sonarr               | community  | ✅    | ✅       |
 | tailscale            | community  | ✅    | ✅       |

diff --git a/ix-dev/community/searxng/README.md b/ix-dev/community/searxng/README.md
@@ -0,0 +1,3 @@
+# SearXNG
+
+[SearXNG](https://github.com/searxng/searxng) is a privacy-respecting, hackable metasearch engine
diff --git a/ix-dev/community/searxng/app.yaml b/ix-dev/community/searxng/app.yaml
@@ -0,0 +1,34 @@
+app_version: 2023.12.31
+capabilities:
+- description: SearXNG requires this ability to switch user for sub-processes.
+  name: SETUID
+- description: SearXNG requires this ability to switch group for sub-processes.
+  name: SETGID
+categories:
+- productivity
+description: SearXNG is a privacy-respecting, hackable metasearch engine
+home: https://github.com/searxng/searxng
+host_mounts: []
+icon: https://media.sys.truenas.net/apps/searxng/icons/icon.svg
+keywords:
+- search
+lib_version: 1.0.2
+lib_version_hash: 4d8ed8593eb4333a91c4e64cfb5fd4fc790c30a512581315c34bc9aca08f44ff
+maintainers:
+- email: [email protected]
+  name: truenas
+  url: https://www.truenas.com/
+name: searxng
+run_as_context:
+- description: SearXNG runs as root user.
+  gid: 0
+  group_name: root
+  uid: 0
+  user_name: root
+screenshots: []
+sources:
+- https://hub.docker.com/r/searxng/searxng
+- https://github.com/searxng/searxng
+title: SearXNG
+train: community
+version: 1.0.0
diff --git a/ix-dev/community/searxng/item.yaml b/ix-dev/community/searxng/item.yaml
@@ -0,0 +1,6 @@
+categories:
+- productivity
+icon_url: https://media.sys.truenas.net/apps/searxng/icons/icon.svg
+screenshots: []
+tags:
+- search
diff --git a/ix-dev/community/searxng/ix_values.yaml b/ix-dev/community/searxng/ix_values.yaml
@@ -0,0 +1,8 @@
+images:
+  image:
+    repository: searxng/searxng
+    tag: 2023.12.31-3535377c9
+
+consts:
+  searxng_container_name: searxng
+  perms_container_name: permissions
diff --git a/ix-dev/community/searxng/migrations/migrate_from_kubernetes b/ix-dev/community/searxng/migrations/migrate_from_kubernetes
@@ -0,0 +1,44 @@
+#!/usr/bin/python3
+
+import os
+import sys
+import yaml
+
+from migration_helpers.resources import migrate_resources
+from migration_helpers.storage import migrate_storage_item
+
+
+def migrate(values):
+    config = values.get("helm_secret", {}).get("config", {})
+    if not config:
+        raise ValueError("No config found in values")
+
+    new_values = {
+        "searxng": {
+            "instance_name": config["searxngConfig"].get("instanceName", "searxng"),
+            "additional_envs": config["searxngConfig"].get("additionalEnvs", []),
+        },
+        "network": {
+            "host_network": config["searxngNetwork"].get("hostNetwork", False),
+            "web_port": config["searxngNetwork"].get("webPort", 20910),
+        },
+        "storage": {
+            "config": migrate_storage_item(config["searxngStorage"]["config"]),
+            "additional_storage": [
+                migrate_storage_item(item, include_read_only=True)
+                for item in config["searxngStorage"]["additionalStorages"]
+            ],
+        },
+        "resources": migrate_resources(config["resources"]),
+    }
+
+    return new_values
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        exit(1)
+
+    if os.path.exists(sys.argv[1]):
+        with open(sys.argv[1], "r") as f:
+            print(yaml.dump(migrate(yaml.safe_load(f.read()))))
diff --git a/ix-dev/community/searxng/migrations/migration_helpers/__init__.py b/ix-dev/community/searxng/migrations/migration_helpers/__init__.py
diff --git a/ix-dev/community/searxng/migrations/migration_helpers/cpu.py b/ix-dev/community/searxng/migrations/migration_helpers/cpu.py
@@ -0,0 +1,30 @@
+import math
+import re
+import os
+
+CPU_COUNT = os.cpu_count()
+
+NUMBER_REGEX = re.compile(r"^[1-9][0-9]$")
+FLOAT_REGEX = re.compile(r"^[0-9]+\.[0-9]+$")
+MILI_CPU_REGEX = re.compile(r"^[0-9]+m$")
+
+
+def transform_cpu(cpu) -> int:
+    result = 2
+    if NUMBER_REGEX.match(cpu):
+        result = int(cpu)
+    elif FLOAT_REGEX.match(cpu):
+        result = int(math.ceil(float(cpu)))
+    elif MILI_CPU_REGEX.match(cpu):
+        num = int(cpu[:-1])
+        num = num / 1000
+        result = int(math.ceil(num))
+
+    if CPU_COUNT is not None:
+        # Do not exceed the actual CPU count
+        result = min(result, CPU_COUNT)
+
+    if int(result) == 0:
+        result = CPU_COUNT if CPU_COUNT else 2
+
+    return int(result)
diff --git a/ix-dev/community/searxng/migrations/migration_helpers/dns_config.py b/ix-dev/community/searxng/migrations/migration_helpers/dns_config.py
@@ -0,0 +1,9 @@
+def migrate_dns_config(dns_config):
+    if not dns_config:
+        return []
+
+    dns_opts = []
+    for opt in dns_config.get("options", []):
+        dns_opts.append(f"{opt['name']}:{opt['value']}")
+
+    return dns_opts
diff --git a/ix-dev/community/searxng/migrations/migration_helpers/kubernetes_secrets.py b/ix-dev/community/searxng/migrations/migration_helpers/kubernetes_secrets.py
@@ -0,0 +1,16 @@
+def get_value_from_secret(secrets=None, secret_name=None, key=None):
+    secrets = secrets if secrets else dict()
+    secret_name = secret_name if secret_name else ""
+    key = key if key else ""
+
+    if not secrets or not secret_name or not key:
+        raise ValueError("Expected [secrets], [secret_name] and [key] to be set")
+    for curr_secret_name, curr_data in secrets.items():
+        if curr_secret_name.endswith(secret_name):
+            if not curr_data.get(key, None):
+                raise ValueError(
+                    f"Expected [{key}] to be set in secret [{curr_secret_name}]"
+                )
+            return curr_data[key]
+
+    raise ValueError(f"Secret [{secret_name}] not found")
diff --git a/ix-dev/community/searxng/migrations/migration_helpers/memory.py b/ix-dev/community/searxng/migrations/migration_helpers/memory.py
@@ -0,0 +1,53 @@
+import re
+import math
+import psutil
+
+TOTAL_MEM = psutil.virtual_memory().total
+
+SINGLE_SUFFIX_REGEX = re.compile(r"^[1-9][0-9]*([EPTGMK])$")
+DOUBLE_SUFFIX_REGEX = re.compile(r"^[1-9][0-9]*([EPTGMK])i$")
+BYTES_INTEGER_REGEX = re.compile(r"^[1-9][0-9]*$")
+EXPONENT_REGEX = re.compile(r"^[1-9][0-9]*e[0-9]+$")
+
+SUFFIX_MULTIPLIERS = {
+    "K": 10**3,
+    "M": 10**6,
+    "G": 10**9,
+    "T": 10**12,
+    "P": 10**15,
+    "E": 10**18,
+}
+
+DOUBLE_SUFFIX_MULTIPLIERS = {
+    "Ki": 2**10,
+    "Mi": 2**20,
+    "Gi": 2**30,
+    "Ti": 2**40,
+    "Pi": 2**50,
+    "Ei": 2**60,
+}
+
+
+def transform_memory(memory):
+    result = 4096  # Default to 4GB
+
+    if re.match(SINGLE_SUFFIX_REGEX, memory):
+        suffix = memory[-1]
+        result = int(memory[:-1]) * SUFFIX_MULTIPLIERS[suffix]
+    elif re.match(DOUBLE_SUFFIX_REGEX, memory):
+        suffix = memory[-2:]
+        result = int(memory[:-2]) * DOUBLE_SUFFIX_MULTIPLIERS[suffix]
+    elif re.match(BYTES_INTEGER_REGEX, memory):
+        result = int(memory)
+    elif re.match(EXPONENT_REGEX, memory):
+        result = int(float(memory))
+
+    result = math.ceil(result)
+    result = min(result, TOTAL_MEM)
+    # Convert to Megabytes
+    result = result / 1024 / 1024
+
+    if int(result) == 0:
+        result = TOTAL_MEM if TOTAL_MEM else 4096
+
+    return int(result)
diff --git a/ix-dev/community/searxng/migrations/migration_helpers/resources.py b/ix-dev/community/searxng/migrations/migration_helpers/resources.py
@@ -0,0 +1,59 @@
+from .memory import transform_memory, TOTAL_MEM
+from .cpu import transform_cpu, CPU_COUNT
+
+
+def migrate_resources(resources, gpus=None, system_gpus=None):
+    gpus = gpus or {}
+    system_gpus = system_gpus or []
+
+    result = {
+        "limits": {
+            "cpus": int((CPU_COUNT or 2) / 2),
+            "memory": int(TOTAL_MEM / 1024 / 1024),
+        }
+    }
+
+    if resources.get("limits", {}).get("cpu", ""):
+        result["limits"].update(
+            {"cpus": transform_cpu(resources.get("limits", {}).get("cpu", ""))}
+        )
+    if resources.get("limits", {}).get("memory", ""):
+        result["limits"].update(
+            {"memory": transform_memory(resources.get("limits", {}).get("memory", ""))}
+        )
+
+    gpus_result = {}
+    for gpu in gpus.items() if gpus else []:
+        kind = gpu[0].lower()  # Kind of gpu (amd, nvidia, intel)
+        count = gpu[1]  # Number of gpus user requested
+
+        if count == 0:
+            continue
+
+        if "amd" in kind or "intel" in kind:
+            gpus_result.update({"use_all_gpus": True})
+        elif "nvidia" in kind:
+            sys_gpus = [
+                gpu_item
+                for gpu_item in system_gpus
+                if gpu_item.get("error") is None
+                and gpu_item.get("vendor", None) is not None
+                and gpu_item.get("vendor", "").upper() == "NVIDIA"
+            ]
+            for sys_gpu in sys_gpus:
+                if count == 0:  # We passed # of gpus that user previously requested
+                    break
+                guid = sys_gpu.get("vendor_specific_config", {}).get("uuid", "")
+                pci_slot = sys_gpu.get("pci_slot", "")
+                if not guid or not pci_slot:
+                    continue
+
+                gpus_result.update(
+                    {"nvidia_gpu_selection": {pci_slot: {"uuid": guid, "use_gpu": True}}}
+                )
+                count -= 1
+
+    if gpus_result:
+        result.update({"gpus": gpus_result})
+
+    return result
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# SearXNG

		[SearXNG](https://github.com/searxng/searxng) is a privacy-respecting, hackable metasearch engine