v2 lib (#629)

* initial commit for library v2 * initial commit for library v2 * rm * git ignore * git ignore * more container stuff * update test-lib app * rm * rm * partial devices * split * device * resources * environment * update test app * Refactor error messages for better readability and consistency in device and resource validation checks * some extra tests * add cgroup for devices * add dns * hashes * add todos * add unittest workflow * update hasesh * install packages * fix resources * some renames and add depends_on * fix test * fix rendering * update hashes * mark internal as internal * fix usage * mark some more * sort imports * health + restart * add healthchecks * remove * netmode auto * validate paths * update comment * ports * update hash * portals * add notes * update hashes * cleaner * clean * have a helper to clear out cpu and memory for ix-app * add profile * better checks on ports * update hashes * add fucns * change how validation works * update hashes * add todo * better path validations * update hashes * init volumes * dont fail * install bcrypt * add a way to remove devices from resources * init volumes/volume mounts for host path * ix volumes support * python's bcrypt * hashes * more validation * some improvements * cleanup hostpath parser * more improvements * try typecheck * type hints * dedupe * better message * use more properties * docstring * another prop * untouch * try dir * split files a bit * cifs volume * nfs vol * add label support * validate * fix check * tmpfs * hashes * better storage implementation for v2-lib (#660) * better storage implementation * add the rest * cleanup * update hashes * update hashes * update hasehs * fix healthcheck * volume * anonymous * cleaner * split * shorter usage * shorter usage * add configs * mode fix * init permissions container * add some todo * extend container permission impl * add some tests * no lint * make sure host is not empty * better namign * remove pass * add comment * perm script * perm script * cleaner * fix test * hashes * remove import * add build image * add some deps * some types * fix * be explicit when adding a perms container * better name * update hashes * update text * fix test and hashes
truenas · Oct 23, 2024 · 3ad10f6 · 3ad10f6
1 parent 0724df2
commit 3ad10f6
Show file tree

Hide file tree

Showing 52 changed files with 5,063 additions and 0 deletions.
diff --git a/.github/workflows/library-tests.yaml b/.github/workflows/library-tests.yaml
@@ -0,0 +1,28 @@
+name: Unit Tests
+
+on:
+  pull_request:
+    paths:
+      - "library/**"
+
+jobs:
+  run-unit-test:
+    name: Run Unit Tests
+    runs-on: ubuntu-latest
+    container: python:3.11-slim
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          apt update
+          apt install -y \
+            python3-bcrypt \
+            python3-pytest \
+            python3-pytest-mock
+
+      - name: Run Tests
+        run: |
+          pytest library
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,7 @@
 __pycache__
 ix-dev/**/rendered
 .DS_Store
+
+ix-dev/test/test-lib/migrations
+out.yaml
+.coverage
diff --git a/cspell.config.yaml b/cspell.config.yaml
@@ -53,6 +53,7 @@ words:
   - freshrss
   - fscrawler
   - ftpd
+  - funcs
   - gandi
   - gdrive
   - gensalt
@@ -113,6 +114,7 @@ words:
   - navidrome
   - netboot
   - netbootxyz
+  - netcat
   - netcup
   - netdata
   - nextauth

diff --git a/library/2.0.0/__init__.py b/library/2.0.0/__init__.py
diff --git a/library/2.0.0/configs.py b/library/2.0.0/configs.py
@@ -0,0 +1,81 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from render import Render
+
+try:
+    from .error import RenderError
+    from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise
+except ImportError:
+    from error import RenderError
+    from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise
+
+
+class Configs:
+    def __init__(self, render_instance: "Render"):
+        self._render_instance = render_instance
+        self._configs: dict[str, dict] = {}
+
+    def add(self, name: str, data: str):
+        if not isinstance(data, str):
+            raise RenderError(f"Expected [data] to be a string, got [{type(data)}]")
+
+        if name not in self._configs:
+            self._configs[name] = {"name": name, "data": data}
+            return
+
+        if data == self._configs[name]["data"]:
+            return
+
+        raise RenderError(f"Config [{name}] already added with different data")
+
+    def has_configs(self):
+        return bool(self._configs)
+
+    def render(self):
+        return {c["name"]: {"content": c["data"]} for c in sorted(self._configs.values(), key=lambda c: c["name"])}
+
+
+class ContainerConfigs:
+    def __init__(self, render_instance: "Render", configs: Configs):
+        self._render_instance = render_instance
+        self.top_level_configs: Configs = configs
+        self.container_configs: set[ContainerConfig] = set()
+
+    def add(self, name: str, data: str, target: str, mode: str = ""):
+        self.top_level_configs.add(name, data)
+
+        if target == "":
+            raise RenderError(f"Expected [target] to be set for config [{name}]")
+        if mode != "":
+            mode = valid_octal_mode_or_raise(mode)
+
+        if target in [c.target for c in self.container_configs]:
+            raise RenderError(f"Target [{target}] already used for another config")
+        target = valid_fs_path_or_raise(target)
+        self.container_configs.add(ContainerConfig(self._render_instance, name, target, mode))
+
+    def has_configs(self):
+        return bool(self.container_configs)
+
+    def render(self):
+        return [c.render() for c in sorted(self.container_configs, key=lambda c: c.source)]
+
+
+class ContainerConfig:
+    def __init__(self, render_instance: "Render", source: str, target: str, mode: str):
+        self._render_instance = render_instance
+        self.source = source
+        self.target = target
+        self.mode = mode
+
+    def render(self):
+        result: dict[str, str | int] = {
+            "source": self.source,
+            "target": self.target,
+        }
+
+        if self.mode:
+            result["mode"] = int(self.mode, 8)
+
+        return result
diff --git a/library/2.0.0/container.py b/library/2.0.0/container.py
@@ -0,0 +1,222 @@
+from typing import Any, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from render import Render
+    from storage import IxStorage
+
+try:
+    from .configs import ContainerConfigs
+    from .depends import Depends
+    from .deploy import Deploy
+    from .devices import Devices
+    from .dns import Dns
+    from .environment import Environment
+    from .error import RenderError
+    from .formatter import escape_dollar, get_image_with_hashed_data
+    from .healthcheck import Healthcheck
+    from .labels import Labels
+    from .ports import Ports
+    from .restart import RestartPolicy
+    from .validations import valid_network_mode_or_raise, valid_cap_or_raise
+    from .storage import Storage
+except ImportError:
+    from configs import ContainerConfigs
+    from depends import Depends
+    from deploy import Deploy
+    from devices import Devices
+    from dns import Dns
+    from environment import Environment
+    from error import RenderError
+    from formatter import escape_dollar, get_image_with_hashed_data
+    from healthcheck import Healthcheck
+    from labels import Labels
+    from ports import Ports
+    from restart import RestartPolicy
+    from validations import valid_network_mode_or_raise, valid_cap_or_raise
+    from storage import Storage
+
+
+class Container:
+    def __init__(self, render_instance: "Render", name: str, image: str):
+        self._render_instance = render_instance
+
+        self._name: str = name
+        self._image: str = self._resolve_image(image)  # TODO: account for inline dockerfile
+        self._build_image: str = ""
+        self._user: str = ""
+        self._tty: bool = False
+        self._stdin_open: bool = False
+        self._cap_drop: set[str] = set(["ALL"])  # Drop all capabilities by default and add caps granularly
+        self._cap_add: set[str] = set()
+        self._security_opt: set[str] = set(["no-new-privileges"])
+        self._network_mode: str = ""
+        self._entrypoint: list[str] = []
+        self._command: list[str] = []
+        self._grace_period: int | None = None
+        self._storage: Storage = Storage(self._render_instance)
+        self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs)
+        self.deploy: Deploy = Deploy(self._render_instance)
+        self.networks: set[str] = set()
+        self.devices: Devices = Devices(self._render_instance)
+        self.environment: Environment = Environment(self._render_instance, self.deploy.resources)
+        self.dns: Dns = Dns(self._render_instance)
+        self.depends: Depends = Depends(self._render_instance)
+        self.healthcheck: Healthcheck = Healthcheck(self._render_instance)
+        # TODO: have a known labels dict in the config and parse it automatically
+        # at render time so all the containers are defined
+        self.labels: Labels = Labels(self._render_instance)
+        self.restart: RestartPolicy = RestartPolicy(self._render_instance)
+        self.ports: Ports = Ports(self._render_instance)
+
+        self._auto_set_network_mode()
+
+    def _auto_set_network_mode(self):
+        if self._render_instance.values.get("network", {}).get("host_network", False):
+            self.set_network_mode("host")
+
+    def _resolve_image(self, image: str):
+        images = self._render_instance.values["images"]
+        if image not in images:
+            raise RenderError(
+                f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]"
+            )
+        repo = images[image].get("repository", "")
+        tag = images[image].get("tag", "")
+
+        if not repo:
+            raise RenderError(f"Repository not found for image [{image}]")
+        if not tag:
+            raise RenderError(f"Tag not found for image [{image}]")
+
+        return f"{repo}:{tag}"
+
+    def build_image(self, content: list[str | None]):
+        dockerfile = f"FROM {self._image}\n"
+        for line in content:
+            if not line:
+                continue
+            if line.startswith("FROM"):
+                # TODO: This will also block multi-stage builds
+                # We can revisit this later if we need it
+                raise RenderError(
+                    "FROM cannot be used in build image. Define the base image when creating the container."
+                )
+            dockerfile += line + "\n"
+
+        self._build_image = dockerfile
+        self._image = get_image_with_hashed_data(self._image, dockerfile)
+
+    def set_user(self, user: int, group: int):
+        for i in (user, group):
+            if not isinstance(i, int) or i < 0:
+                raise RenderError(f"User/Group [{i}] is not valid")
+        self._user = f"{user}:{group}"
+
+    def set_tty(self, enabled: bool = False):
+        self._tty = enabled
+
+    def set_stdin(self, enabled: bool = False):
+        self._stdin_open = enabled
+
+    def set_grace_period(self, grace_period: int):
+        if grace_period < 0:
+            raise RenderError(f"Grace period [{grace_period}] cannot be negative")
+        self._grace_period = grace_period
+
+    def add_caps(self, caps: list[str]):
+        for c in caps:
+            if c in self._cap_add:
+                raise RenderError(f"Capability [{c}] already added")
+            self._cap_add.add(valid_cap_or_raise(c))
+
+    def add_security_opt(self, opt: str):
+        if opt in self._security_opt:
+            raise RenderError(f"Security Option [{opt}] already added")
+        self._security_opt.add(opt)
+
+    def remove_security_opt(self, opt: str):
+        self._security_opt.remove(opt)
+
+    def set_network_mode(self, mode: str):
+        self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names())
+
+    def set_entrypoint(self, entrypoint: list[str]):
+        self._entrypoint = [escape_dollar(e) for e in entrypoint]
+
+    def set_command(self, command: list[str]):
+        self._command = [escape_dollar(e) for e in command]
+
+    def add_storage(self, mount_path: str, config: "IxStorage"):
+        self._storage.add(mount_path, config)
+
+    def render(self) -> dict[str, Any]:
+        if self._network_mode and self.networks:
+            raise RenderError("Cannot set both [network_mode] and [networks]")
+
+        result = {
+            "image": self._image,
+            "tty": self._tty,
+            "stdin_open": self._stdin_open,
+            "restart": self.restart.render(),
+            "cap_drop": sorted(self._cap_drop),
+            "healthcheck": self.healthcheck.render(),
+        }
+        if self._build_image:
+            result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image}
+
+        if self.configs.has_configs():
+            result["configs"] = self.configs.render()
+
+        if self._grace_period is not None:
+            result["stop_grace_period"] = self._grace_period
+
+        if self._user:
+            result["user"] = self._user
+
+        if self._cap_add:
+            result["cap_add"] = sorted(self._cap_add)
+
+        if self._security_opt:
+            result["security_opt"] = sorted(self._security_opt)
+
+        if self._network_mode:
+            result["network_mode"] = self._network_mode
+
+        if self._network_mode != "host":
+            if self.ports.has_ports():
+                result["ports"] = self.ports.render()
+
+        if self._entrypoint:
+            result["entrypoint"] = self._entrypoint
+
+        if self._command:
+            result["command"] = self._command
+
+        if self.devices.has_devices():
+            result["devices"] = self.devices.render()
+
+        if self.deploy.has_deploy():
+            result["deploy"] = self.deploy.render()
+
+        if self.environment.has_variables():
+            result["environment"] = self.environment.render()
+
+        if self.labels.has_labels():
+            result["labels"] = self.labels.render()
+
+        if self.dns.has_dns_nameservers():
+            result["dns"] = self.dns.render_dns_nameservers()
+
+        if self.dns.has_dns_searches():
+            result["dns_search"] = self.dns.render_dns_searches()
+
+        if self.dns.has_dns_opts():
+            result["dns_opt"] = self.dns.render_dns_opts()
+
+        if self.depends.has_dependencies():
+            result["depends_on"] = self.depends.render()
+
+        if self._storage.has_mounts():
+            result["volumes"] = self._storage.render()
+
+        return result
diff --git a/library/2.0.0/depends.py b/library/2.0.0/depends.py
@@ -0,0 +1,34 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from render import Render
+
+try:
+    from .error import RenderError
+    from .validations import valid_depend_condition_or_raise
+except ImportError:
+    from error import RenderError
+    from validations import valid_depend_condition_or_raise
+
+
+class Depends:
+    def __init__(self, render_instance: "Render"):
+        self._render_instance = render_instance
+        self._dependencies: dict[str, str] = {}
+
+    def add_dependency(self, name: str, condition: str):
+        condition = valid_depend_condition_or_raise(condition)
+        if name in self._dependencies.keys():
+            raise RenderError(f"Dependency [{name}] already added")
+        if name not in self._render_instance.container_names():
+            raise RenderError(
+                f"Dependency [{name}] not found in defined containers. "
+                f"Available containers: [{', '.join(self._render_instance.container_names())}]"
+            )
+        self._dependencies[name] = condition
+
+    def has_dependencies(self):
+        return len(self._dependencies) > 0
+
+    def render(self):
+        return {d: {"condition": c} for d, c in self._dependencies.items()}