Merge pull request #40 from getindata/release-1.10.0

Release 1.10.0
getindata · Mar 21, 2023 · 4b67a53 · 4b67a53
2 parents 9d486bf + da5744d
commit 4b67a53
Show file tree

Hide file tree

Showing 20 changed files with 362 additions and 23 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,17 @@
 
 ## [Unreleased]
 
+## [1.10.0] - 2023-03-21
+
+### Added
+
+-   Deploy to vvp with given deployment descriptor
+-   Documentation on adding certificates needed by requests library
+
+### Changed
+
+-   Ververica job deployment now uses PUT http method instead of POST
+
 ## [1.9.1] - 2023-03-02
 
 ### Added
@@ -101,18 +112,16 @@
 
 -   First release
 
-&lt;&lt;&lt;&lt;&lt;&lt;&lt; HEAD
-
 # [Unreleased]&#x3A; <https://github.com/getindata/streaming-cli/compare/1.8.0...HEAD>
 
-[Unreleased]: https://github.com/getindata/streaming-cli/compare/1.9.1...HEAD
+[Unreleased]: https://github.com/getindata/streaming-cli/compare/1.10.0...HEAD
+
+[1.10.0]: https://github.com/getindata/streaming-cli/compare/1.9.1...1.10.0
 
 [1.9.1]: https://github.com/getindata/streaming-cli/compare/1.9.0...1.9.1
 
 [1.9.0]: https://github.com/getindata/streaming-cli/compare/1.8.0...1.9.0
 
-> > > > > > > master
-
 [1.8.0]: https://github.com/getindata/streaming-cli/compare/1.7.1...1.8.0
 
 [1.7.0]: https://github.com/getindata/streaming-cli/compare/1.6.0...1.7.0

diff --git a/README.md b/README.md
@@ -124,6 +124,17 @@ scli docker login \
 
 > Parameters`--docker-image-tag` is optional and has default value `latest`.
 
+### Providing certificates
+Scli uses `requests` library to deploy jobs to Ververica Platform via REST Api. Currently `requests` does not support
+automatic downloading of intermediate certificates so entire chain of certificates should be present before making
+a http call
+
+example:
+```sh
+sh -c 'CA_CERT_PATH=`python3 -c "import requests; print(requests.certs.where())"`;for CERT in certs/*; do cat ${CERT}; done >> ${CA_CERT_PATH}';
+
+```
+
 ### Project operations
 
 ## SCLI Development

diff --git a/release/Dockerfile b/release/Dockerfile
@@ -8,3 +8,5 @@ RUN apk add --no-cache build-base python3>=${PYTHON_VERSION} python3-dev>=${PYTH
 RUN python3 -m pip install --no-cache wheel
 RUN python3 -m pip install --no-cache streamingcli==${STREAMING_CLI_VERSION} --ignore-installed six
 
+WORKDIR /root
+ENTRYPOINT ["sh", "-c", "scli"]
diff --git a/setup.cfg b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.9.1
+current_version = 1.10.0
 
 [flake8]
 exclude = .git,__pycache__,build,dist,*.yml

diff --git a/setup.py b/setup.py
@@ -5,7 +5,7 @@
 from setuptools import find_packages, setup
 from setuptools.command.install import install
 
-__version__ = "1.9.1"
+__version__ = "1.10.0"
 
 with open("README.md", "r") as fh:
     long_description = fh.read()

diff --git a/streamingcli/main.py b/streamingcli/main.py
@@ -110,6 +110,11 @@ def project_init(
     "ververica_deployment_target_name",
     help="Ververica deployment target name",
 )
+@click.option(
+    "--vvp-deployment-template-path",
+    "ververica_deployment_template_path",
+    help="Optional custom Ververica deployment descriptor file absolute path",
+)
 @click.option(
     "--vvp-api-token",
     "vvp_api_token",
@@ -129,6 +134,7 @@ def project_deploy(
     ververica_url: Optional[str] = None,
     ververica_namespace: Optional[str] = None,
     ververica_deployment_target_name: Optional[str] = None,
+    ververica_deployment_template_path: Optional[str] = None,
     vvp_api_token: Optional[str] = None,
     k8s_namespace: Optional[str] = None,
     overrides_from_yaml: Optional[str] = None,
@@ -142,6 +148,7 @@ def project_deploy(
         ververica_url=ververica_url,
         ververica_namespace=ververica_namespace,
         ververica_deployment_target_name=ververica_deployment_target_name,
+        ververica_deployment_template_path=ververica_deployment_template_path,
         ververica_webtoken_secret=vvp_api_token,
         k8s_namespace=k8s_namespace,
         overrides_from_yaml=overrides_from_yaml,
@@ -347,6 +354,11 @@ def profile() -> None:
 @click.option(
     "--vvp-deployment-target", required=False, help="Ververica deployment target name"
 )
+@click.option(
+    "--vvp-deployment-template-path",
+    required=False,
+    help="Ververica deployment template absolute path",
+)
 @click.option("--vvp-api-token", required=False, help="Ververica API Token")
 @click.option(
     "--docker-registry-url",
@@ -360,6 +372,7 @@ def add_profile(
     vvp_url: Optional[str],
     vvp_namespace: Optional[str],
     vvp_deployment_target: Optional[str],
+    vvp_deployment_template_path: Optional[str],
     vvp_api_token: Optional[str],
     docker_registry_url: Optional[str],
     k8s_namespace: Optional[str],
@@ -370,6 +383,7 @@ def add_profile(
         ververica_url=vvp_url,
         ververica_namespace=vvp_namespace,
         ververica_deployment_target=vvp_deployment_target,
+        ververica_deployment_template_path=vvp_deployment_template_path,
         ververica_api_token=vvp_api_token,
         docker_registry_url=docker_registry_url,
         k8s_namespace=k8s_namespace,

diff --git a/streamingcli/platform/deployment_adapter.py b/streamingcli/platform/deployment_adapter.py
@@ -25,9 +25,8 @@ def deploy(self, deployment_yml: str) -> Optional[str]:
     def validate_profile_data(self) -> None:
         pass
 
-    @staticmethod
     @abstractmethod
-    def get_template_name() -> str:
+    def get_template_name(self) -> str:
         pass
 
     def generate_project_template(self, dependencies: List[str]) -> str:

diff --git a/streamingcli/platform/k8s/deployment_adapter.py b/streamingcli/platform/k8s/deployment_adapter.py
@@ -27,8 +27,7 @@ def validate_profile_data(self) -> None:
         if self.docker_image_tag is None or len(self.docker_image_tag) == 0:
             raise click.ClickException("Missing Docker image tag attribute")
 
-    @staticmethod
-    def get_template_name() -> str:
+    def get_template_name(self) -> str:
         return "k8s_flink_deployment.yml"
 
     @staticmethod

diff --git a/streamingcli/platform/ververica/deployment_adapter.py b/streamingcli/platform/ververica/deployment_adapter.py
@@ -9,10 +9,10 @@
 
 class VervericaDeploymentAdapter(DeploymentAdapter):
     def deploy(self, deployment_yml: str) -> Optional[str]:
-        response = self.post_deployment_file(deployment_yml)
+        response = self.put_deployment_file(deployment_yml)
 
         if response.status_code != 201:
-            raise click.ClickException("Failed to POST deployment.yaml file")
+            raise click.ClickException("Failed to PUT deployment.yaml file")
         else:
             deployment_name = response.json()["metadata"]["name"]
             return (
@@ -43,17 +43,19 @@ def validate_profile_data(self) -> None:
         if self.docker_image_tag is None or len(self.docker_image_tag) == 0:
             raise click.ClickException("Missing Docker image tag attribute")
 
-    @staticmethod
-    def get_template_name() -> str:
-        return "vvp_flink_deployment.yml"
+    def get_template_name(self) -> str:
+        return (
+            self.profile_data.ververica_deployment_template_path
+            or "vvp_flink_deployment.yml"
+        )
 
-    def post_deployment_file(self, deployment_file: str) -> Response:
+    def put_deployment_file(self, deployment_file: str) -> Response:
         deployments_url = (
             f"{self.profile_data.ververica_url}/api/v1/namespaces/"
-            + f"{self.profile_data.ververica_namespace}/deployments"
+            + f"{self.profile_data.ververica_namespace}/deployments/{self.project_name}"
         )
 
-        response = requests.post(
+        response = requests.put(
             url=deployments_url,
             data=deployment_file,
             headers={

diff --git a/streamingcli/profile/profile_adapter.py b/streamingcli/profile/profile_adapter.py
@@ -40,6 +40,7 @@ class ScliProfile:
     ververica_url: Optional[str] = field(default=None)
     ververica_namespace: Optional[str] = field(default=None)
     ververica_deployment_target: Optional[str] = field(default=None)
+    ververica_deployment_template_path: Optional[str] = field(default=None)
     ververica_api_token: Optional[str] = field(default=None)
     docker_registry_url: Optional[str] = field(default=None)
     k8s_namespace: Optional[str] = field(default=None)
@@ -108,6 +109,7 @@ def update_profile_data(
         ververica_namespace: Optional[str] = None,
         ververica_deployment_target_name: Optional[str] = None,
         ververica_webtoken_secret: Optional[str] = None,
+        ververica_deployment_template_path: Optional[str] = None,
         docker_registry_url: Optional[str] = None,
         k8s_namespace: Optional[str] = None,
     ) -> ScliProfile:
@@ -120,6 +122,10 @@ def update_profile_data(
             params["ververica_namespace"] = ververica_namespace
         if ververica_deployment_target_name is not None:
             params["ververica_deployment_target"] = ververica_deployment_target_name
+        if ververica_deployment_template_path is not None:
+            params[
+                "ververica_deployment_template_path"
+            ] = ververica_deployment_template_path
         if ververica_webtoken_secret is not None:
             params["ververica_api_token"] = ververica_webtoken_secret
         if docker_registry_url is not None:

diff --git a/streamingcli/profile/profile_command.py b/streamingcli/profile/profile_command.py
@@ -17,6 +17,7 @@ def create_profile(
         ververica_url: Optional[str] = None,
         ververica_namespace: Optional[str] = None,
         ververica_deployment_target: Optional[str] = None,
+        ververica_deployment_template_path: Optional[str] = None,
         ververica_api_token: Optional[str] = None,
         docker_registry_url: Optional[str] = None,
         k8s_namespace: Optional[str] = None,
@@ -27,6 +28,7 @@ def create_profile(
             ververica_url=ververica_url,
             ververica_namespace=ververica_namespace,
             ververica_deployment_target=ververica_deployment_target,
+            ververica_deployment_template_path=ververica_deployment_template_path,
             docker_registry_url=docker_registry_url,
             ververica_api_token=ververica_api_token,
             k8s_namespace=k8s_namespace,

diff --git a/streamingcli/project/deploy_command.py b/streamingcli/project/deploy_command.py
@@ -36,6 +36,7 @@ def deploy_project(
         ververica_url: Optional[str] = None,
         ververica_namespace: Optional[str] = None,
         ververica_deployment_target_name: Optional[str] = None,
+        ververica_deployment_template_path: Optional[str] = None,
         ververica_webtoken_secret: Optional[str] = None,
         k8s_namespace: Optional[str] = None,
         overrides_from_yaml: Optional[str] = None,
@@ -57,6 +58,7 @@ def deploy_project(
             ververica_url=ververica_url,
             ververica_namespace=ververica_namespace,
             ververica_deployment_target_name=ververica_deployment_target_name,
+            ververica_deployment_template_path=ververica_deployment_template_path,
             ververica_webtoken_secret=ververica_webtoken_secret,
             docker_registry_url=docker_registry_url,
             k8s_namespace=k8s_namespace,

diff --git a/streamingcli/project/template_loader.py b/streamingcli/project/template_loader.py
@@ -1,16 +1,36 @@
 import importlib.resources as pkg_resources
+from pathlib import Path
+from typing import Optional
 
 from streamingcli.project import templates
 
 
 class TemplateLoader:
     @staticmethod
     def load_project_template(template_name: str) -> str:
-        template = pkg_resources.read_text(templates, template_name)
+        template = TemplateLoader.try_to_load_text_from_packages(
+            template_name
+        ) or TemplateLoader.try_to_load_text_from_path(template_name)
+        if template is None:
+            raise ValueError(f"Could not read file for name: {template_name}")
         return template
 
     @staticmethod
     def copy_binary(file_name: str, target_path: str) -> None:
         file_content = pkg_resources.read_binary(templates, file_name)
         with open(target_path, "wb") as target_file:
             target_file.write(file_content)
+
+    @staticmethod
+    def try_to_load_text_from_packages(name: str) -> Optional[str]:
+        try:
+            return pkg_resources.read_text(templates, name)
+        except ValueError:
+            return None
+
+    @staticmethod
+    def try_to_load_text_from_path(name: str) -> Optional[str]:
+        try:
+            return Path(name).absolute().read_text()
+        except OSError:
+            return None
diff --git a/streamingcli/project/templates/vvp_flink_deployment.yml b/streamingcli/project/templates/vvp_flink_deployment.yml
@@ -1,4 +1,5 @@
 metadata:
+  name: {{project_name}}
   displayName: {{project_name}}
 spec:
   deploymentTargetName: {{ververica_deployment_target}}
@@ -22,9 +23,9 @@ spec:
         flinkVersion: '1.14'
         additionalDependencies:
           - file:///app/lib/flink-python_2.12-1.14.0.jar
-          {% for dependency in dependencies -%}
+          {%- for dependency in dependencies %}
           - file://{{dependency}}
-          {% endfor %}
+          {%- endfor %}
         jarUri: file:///app/lib/flink-python-loader-1.1.0.jar
         entryClass: com.getindata.streaming.FlinkPythonLoader
         kind: JAR

diff --git a/tests/streamingcli/resources/platform/vvp/custom_vvp_flink_deployment_template.yml b/tests/streamingcli/resources/platform/vvp/custom_vvp_flink_deployment_template.yml
@@ -0,0 +1,50 @@
+metadata:
+  displayName: {{project_name}}
+  name: {{project_name}}
+spec:
+  deploymentTargetName: {{ververica_deployment_target}}
+  maxJobCreationAttempts: 4
+  maxSavepointCreationAttempts: 4
+  restoreStrategy:
+    allowNonRestoredState: false
+    kind: LATEST_STATE
+  state: RUNNING
+  template:
+    metadata:
+      annotations:
+        flink.queryable-state.enabled: 'false'
+        flink.security.ssl.enabled: 'false'
+    spec:
+      artifact:
+        flinkImageRegistry: {{docker_registry_url}}
+        flinkImageRepository: {{project_name}}
+        flinkImageTag: {{docker_image_tag}}
+        flinkVersion: '1.16'
+        kind: JAR
+        jarUri: 'file:///opt/flink/opt/flink-python-1.16.0.jar'
+        entryClass: org.apache.flink.client.python.PythonDriver
+        mainArgs: '--python /app/src/flink_app.py'
+      flinkConfiguration:
+        execution.checkpointing.externalized-checkpoint-retention: RETAIN_ON_CANCELLATION
+        execution.checkpointing.interval: 10s
+        execution.checkpointing.min-pause: 10s
+        high-availability: vvp-kubernetes
+        metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+        state.backend: filesystem
+        taskmanager.memory.managed.fraction: '0.0'
+        taskmanager.numberOfTaskSlots: '1'
+        web.cancel.enable: 'false'
+      logging:
+        log4jLoggers:
+          '': INFO
+        loggingProfile: default
+      parallelism: 1
+      resources:
+        jobmanager:
+          cpu: 1
+          memory: 1G
+        taskmanager:
+          cpu: 1
+          memory: 1G
+  upgradeStrategy:
+    kind: STATEFUL