Merge branch 'stream-217.add_profile' into 'master'

[STREAM-218] CLI command for creating profile See merge request getindata/streaming-labs/streaming-cli!23
getindata · Oct 22, 2021 · 0ae75a6 · 0ae75a6
2 parents 43370af + e1ebb27
commit 0ae75a6
Show file tree

Hide file tree

Showing 10 changed files with 190 additions and 62 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,4 +6,5 @@ build/
 streamingcli.egg-info/
 tmp_project/
 requirements.txt
-.env
+.env
+__pycache__
diff --git a/Makefile b/Makefile
@@ -56,3 +56,6 @@ platform/apitoken/create:
 
 platform/apitoken/remove:
 	export PIPENV_VERBOSITY=-1; pipenv run scli platform api-token remove --vvp-url "http://localhost:8080" --vvp-namespace default --name "test-token"
+
+profile/add:
+	pipenv run scli profile add new_profile --ververica-url "http://localhost:8080" --ververica-namespace default --ververica-deployment-target default --docker-registry-url localhost:5000
diff --git a/Pipfile b/Pipfile
@@ -13,6 +13,8 @@ Jinja2 = "*"
 setuptools = "*"
 docker = "*"
 HiYaPyCo = "*"
+marshmallow = "*" 
+marshmallow-dataclass = "*"
 
 [dev-packages]
 pytest = "*"

diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/streamingcli/Config.py b/streamingcli/Config.py
@@ -1,3 +1,5 @@
+from pathlib import Path
+
 INITIAL_PROJECT_REPO = "[email protected]:getindata/streaming-labs/flink-sandbox-python.git"
 PROJECT_LOCAL_CONFIG_FILE_NAME = ".streaming_config.yml"
 PROJECT_LOCAL_TEMPLATE_DIR_NAME = ".vvp"
@@ -6,5 +8,6 @@
 PLATFORM_K8S_CONFIGMAP_KEY = "platform_config.json"
 PLATFORM_K8S_SECRET_NAME = "streaming-platform-secret"
 PLATFORM_DEFAULT_DEPLOYMENT_TARGET_NAME = "default"
-
 PROFILE_ENV_VARIABLE_NAME = "SCLI_PROFILE"
+SCLI_CONFIG_DIR_NAME = ".scli"
+DEFAULT_PROFILE_DIR = f"{str(Path.home())}/{SCLI_CONFIG_DIR_NAME}/profiles.yml"
diff --git a/streamingcli/main.py b/streamingcli/main.py
@@ -4,6 +4,7 @@
 from streamingcli.platform.setup_command import PlatformSetupCommand
 from streamingcli.platform.apitoken_create_command import VervericaApiTokenCreateCommand
 from streamingcli.platform.apitoken_remove_command import VervericaApiTokenRemoveCommand
+from streamingcli.profile.profile_command import ProfileCommand
 from streamingcli.project.build_command import ProjectBuilder
 from streamingcli.project.cicd_command import CICDInitializer
 from streamingcli.project.deploy_command import ProjectDeployer
@@ -34,7 +35,6 @@ def project_init(project_name: str):
     NewProjectInitializer.createProject(project_name)
     click.echo(f"Project: {project_name} initialized")
 
-
 @project.command()
 @click.option('--profile',
               help='Profile name to use')
@@ -158,6 +158,36 @@ def cicd():
 def cicd_setup(provider: str):
     CICDInitializer.setup_cicd(provider)
 
+@cli.group()
+def profile():
+    pass
+
+@profile.command()
+@click.argument('profile_name')
+@click.option('--ververica-url', prompt='Ververica URL', required=False,
+                help='URL for Ververica cluster, i.e: "https://vvp.streaming-platform.example.com"')
+@click.option('--ververica-namespace', prompt='Ververica namespace', required=False,
+                help='Ververica namespace')
+@click.option('--ververica-deployment-target', prompt='Ververica deployment target name',
+                required=False, help='Ververica deployment target name')
+@click.option('--vvp-api-token', prompt='Ververica API Token',
+                required=False, help='Ververica API Token')
+@click.option('--docker-registry-url', prompt='Docker registry URL',
+                required=False, help='URL for Docker registry, i.e: "https://hub.docker.com/"')
+def add_profile(profile_name: str, 
+                ververica_url: str,
+                ververica_namespace: str,
+                ververica_deployment_target: str,
+                vvp_api_token: str,
+                docker_registry_url: str):
+    ProfileCommand.create_profile(profile_name=profile_name,
+                                    ververica_url=ververica_url,
+                                    ververica_namespace=ververica_namespace,
+                                    ververica_deployment_target=ververica_deployment_target,
+                                    ververica_api_token= vvp_api_token,
+                                    docker_registry_url=docker_registry_url
+                                    )
+
 
 project.add_command(project_init, "init")
 project.add_command(project_deploy, "deploy")
@@ -167,6 +197,7 @@ def cicd_setup(provider: str):
 api_token.add_command(platform_apitoken_create, "create")
 api_token.add_command(platform_apitoken_remove, "remove")
 cicd.add_command(cicd_setup, "setup")
+profile.add_command(add_profile, "add")
 
 if __name__ == '__main__':
     cli()
diff --git a/streamingcli/profile/__init__.py b/streamingcli/profile/__init__.py
diff --git a/streamingcli/profile/profile_command.py b/streamingcli/profile/profile_command.py
@@ -0,0 +1,74 @@
+from dataclasses import asdict, dataclass, field, replace
+from pathlib import Path
+from typing import Any, Dict, Optional, Type
+
+import click
+from marshmallow import Schema
+from marshmallow_dataclass import class_schema
+from streamingcli.Config import DEFAULT_PROFILE_DIR
+from yaml import SafeLoader, load, safe_dump
+
+
+@dataclass(repr=True)
+class ScliProfile:
+    profile_name: str
+    ververica_url: Optional[str] = field(default=None)
+    ververica_namespace: Optional[str] = field(default=None)
+    ververica_deployment_target: Optional[str] = field(default=None)
+    ververica_api_token: Optional[str] = field(default=None)
+    docker_registry_url: Optional[str] = field(default=None)
+
+@dataclass
+class ScliProfiles:    
+    profiles: Dict[str, ScliProfile] = field(default_factory=dict)
+
+
+class ProfileCommand:
+
+    @staticmethod
+    def create_profile(profile_name: str,
+                        ververica_url: str=None,
+                        ververica_namespace: str=None,
+                        ververica_deployment_target: str=None,
+                        ververica_api_token: str=None,
+                        docker_registry_url: str=None):
+        scli_profile = ScliProfile(
+            profile_name=profile_name,
+            ververica_url=ververica_url,
+            ververica_namespace=ververica_namespace,
+            ververica_deployment_target=ververica_deployment_target,
+            docker_registry_url=docker_registry_url,
+            ververica_api_token=ververica_api_token
+        )
+        ProfileCommand.save_profile(scli_profile=scli_profile)
+        click.echo(f"Scli profile {profile_name} saved")
+
+    @staticmethod
+    def save_profile(scli_profile: ScliProfile):
+        profiles = ProfileCommand.load_profiles()
+        profiles_dict = profiles.profiles
+        profiles_dict[scli_profile.profile_name] = scli_profile
+        content = safe_dump(asdict(replace(profiles, profiles = profiles_dict)))
+        with open(DEFAULT_PROFILE_DIR, "w") as file:
+            file.write(content)
+
+    @staticmethod
+    def get_profile(profile_name: str) -> Optional[ScliProfile]: 
+        profiles_dict = ProfileCommand.load_profiles().profiles
+        return profiles_dict.get(profile_name)
+
+    @staticmethod
+    def load_profiles() -> ScliProfiles:
+        profiles_path = Path(DEFAULT_PROFILE_DIR)
+
+        if (profiles_path.is_file() is False):
+            return ScliProfiles
+
+        with open(profiles_path, "r") as file:
+            content = file.read()
+            return ProfileCommand.strict_load_yaml(content, ScliProfiles)
+
+    @staticmethod
+    def strict_load_yaml(yaml: str, loaded_type: Type[Any]):
+        schema = class_schema(loaded_type)
+        return schema().load(load(yaml, Loader=SafeLoader))
diff --git a/streamingcli/project/deploy_command.py b/streamingcli/project/deploy_command.py
@@ -1,14 +1,14 @@
+import os
 from typing import Optional
 
 import click
 from jinja2 import Environment
+from streamingcli.Config import PROFILE_ENV_VARIABLE_NAME
 from streamingcli.platform.ververica.deployment_adapter import \
     DeploymentAdapter
 from streamingcli.project.local_project_config import LocalProjectConfigIO
 from streamingcli.project.template_loader import TemplateLoader
 from streamingcli.project.yaml_merger import YamlMerger
-from streamingcli.Config import PROFILE_ENV_VARIABLE_NAME
-import os
 
 
 class ProjectDeployer:
@@ -21,19 +21,21 @@ def get_profile_name(profile_name: Optional[str]) -> Optional[str]:
 
     @staticmethod
     def deploy_project(profile: Optional[str] = None,
-                       ververica_url: Optional[str] = None,
-                       ververica_namespace: Optional[str] = None,
-                       ververica_deployment_target_name: Optional[str] = None,
-                       ververica_webtoken_secret: Optional[str] = None,
-                       docker_registry_url: Optional[str] = None,
-                       docker_image_tag: Optional[str] = None,
-                       overrides_from_yaml: Optional[str] = None):
+                        ververica_url: Optional[str] = None,
+                        ververica_namespace: Optional[str] = None,
+                        ververica_deployment_target_name: Optional[str] = None,
+                        ververica_webtoken_secret: Optional[str] = None,
+                        docker_registry_url: Optional[str] = None,
+                        docker_image_tag: Optional[str] = None,
+                        overrides_from_yaml: Optional[str] = None):
         profile_name = ProjectDeployer.get_profile_name(profile_name=profile)
 
         # TODO Use ScliProfile dataclass instead
         profile_data = {}
 
         # TODO Load profile data for {profile_name}
+        # Load platform ConfigMap
+
 
         # Explicitly defined parameters will override profile ones
         if ververica_url is not None:
@@ -70,8 +72,8 @@ def deploy_project(profile: Optional[str] = None,
             auth_token=profile_data["ververica_webtoken_secret"]
         )
         click.echo(f"Created deployment: "
-                   f"{profile_data['ververica_url']}/app/#/namespaces/"
-                   f"{profile_data['ververica_namespace']}/deployments/{deployment_name}")
+                    f"{profile_data['ververica_url']}/app/#/namespaces/"
+                    f"{profile_data['ververica_namespace']}/deployments/{deployment_name}")
 
     @staticmethod
     def generate_project_template(

diff --git a/streamingcli/project/publish_command.py b/streamingcli/project/publish_command.py
@@ -57,7 +57,7 @@ def show_progressbar(data: Generator, image_tag: str):
                 elif(status == 'Pushing' and progress_detail != None):
                     item_total = progress_detail.get('total')
                     item_current = progress_detail.get('current')
-                    if(item_current > item_total):
+                    if(item_total == None or item_current == None or item_current > item_total):
                         continue  
                     elif id in items:
                         diff = (item_current - items[id])