fix breaking change - arg renamed, bettered debug logs

google38438 · Jan 23, 2019 · 8460077 · 8460077
1 parent 5bd7f6e
commit 8460077
Show file tree

Hide file tree

Showing 5 changed files with 65 additions and 19 deletions.
diff --git a/acceptancetests/assess_caas_bootstrap.py b/acceptancetests/assess_caas_bootstrap.py
@@ -40,14 +40,14 @@ def assess_caas_bootstrap(client):
         juju_ver=client.version
     )
 
-    caas_client = deploy_caas_stack(bundle_path=bundle, client=client, timeout=3600)
+    caas_client = deploy_caas_stack(path=bundle, client=client, timeout=3600)
     if not caas_client.is_cluster_healthy:
         raise JujuAssertionError('k8s cluster is not healthy coz kubectl is not accessible')
 
 
 def parse_args(argv):
     """Parse all arguments."""
-    parser = argparse.ArgumentParser(description="Cass Bootstrap CI test")
+    parser = argparse.ArgumentParser(description="CAAS Bootstrap CI test")
 
     add_basic_testing_arguments(parser, existing=False)
     return parser.parse_args(argv)

diff --git a/acceptancetests/assess_caas_deploy_charms.py b/acceptancetests/assess_caas_deploy_charms.py
@@ -158,18 +158,33 @@
 """
 
 
-def check_app_healthy(url, timeout=300):
+def run_hook(func, is_raise=False):
+    if func is not None and callable(func):
+        try:
+            return func()
+        except Exception as e:
+            log.error(e)
+            if is_raise:
+                raise e
+    return None
+
+
+def check_app_healthy(url, timeout=300, success_hook=None, fail_hook=None):
     status_code = None
-    for _ in until_timeout(timeout):
+    for remaining in until_timeout(timeout):
         try:
             r = requests.get(url)
             if r.ok and r.status_code < 400:
-                return
+                return run_hook(success_hook)
             status_code = r.status_code
         except IOError as e:
             log.error(e)
-        sleep(3)
+        finally:
+            sleep(3)
+            if remaining % 60 == 0:
+                log.info('timeout in %ss', remaining)
     log.error('HTTP health check failed -> %s, status_code -> %s !', url, status_code)
+    run_hook(fail_hook)
     raise JujuAssertionError('gitlab is not healthy')
 
 
@@ -181,19 +196,24 @@ def assess_caas_charm_deployment(client):
         juju_ver=client.version
     )
 
-    caas_client = deploy_caas_stack(bundle_path=bundle, client=client)
+    caas_client = deploy_caas_stack(path=bundle, client=client, timeout=4000)
     external_hostname = caas_client.get_external_hostname()
 
     if not caas_client.is_cluster_healthy:
         raise JujuAssertionError('k8s cluster is not healthy because kubectl is not accessible')
 
     # tmp fix kubernetes core ingress issue
-    caas_client.kubectl(
-        'patch', 'daemonset.apps/nginx-ingress-kubernetes-worker-controller', '--patch',
+    ingress_controller_daemonset_name = 'daemonset.apps/nginx-ingress-kubernetes-worker-controller'
+    o = caas_client.kubectl(
+        'patch', ingress_controller_daemonset_name, '--patch',
         '''
         {"spec": {"template": {"spec": {"containers": [{"name": "nginx-ingress-kubernetes-worker","args": ["/nginx-ingress-controller", "--default-backend-service=$(POD_NAMESPACE)/default-http-backend", "--configmap=$(POD_NAMESPACE)/nginx-load-balancer-conf", "--enable-ssl-chain-completion=False", "--publish-status-address=%s"]}]}}}}
         ''' % caas_client.get_first_worker_ip()
     )
+    log.info(o)
+
+    o = caas_client.kubectl('get', ingress_controller_daemonset_name, '-o', 'yaml')
+    log.info(o)
 
     # add caas model for deploying caas charms on top of it
     model_name = 'testcaas'
@@ -235,11 +255,24 @@ def assess_caas_charm_deployment(client):
     k8s_model.juju('expose', ('gitlab-k8s',))
     k8s_model.wait_for_workloads(timeout=3600)
 
+    def success_hook():
+        log.info(caas_client.kubectl('get', 'all', '--all-namespaces'))
+
+    def fail_hook():
+        success_hook()
+        log.info(caas_client.kubectl('get', ingress_controller_daemonset_name, '-o', 'yaml'))
+        log.info(caas_client.kubectl('get', 'pv,pvc', '-n', model_name))
+
     url = '{}://{}/{}'.format('http', external_hostname, 'gitlab-k8s')
-    check_app_healthy(url, timeout=1200)
+    check_app_healthy(
+        url, timeout=1800,
+        success_hook=success_hook,
+        fail_hook=fail_hook,
+    )
 
-    log.info(caas_client.kubectl('get', 'all', '--all-namespaces'))
     k8s_model.juju(k8s_model._show_status, ('--format', 'tabular'))
+    # current destroy controller does not handle storage, so destroy current caas model with --destroy-storage.
+    k8s_model.destroy_model()
 
 
 def parse_args(argv):

diff --git a/acceptancetests/deploy_stack.py b/acceptancetests/deploy_stack.py
@@ -113,7 +113,7 @@ class NoExistingController(Exception):
 
 def deploy_dummy_stack(client, charm_series, use_charmstore=False):
     """"Deploy a dummy stack in the specified environment."""
-    # Centos requires specific machine configuration (i.e. network device
+    # CentOS requires specific machine configuration (i.e. network device
     # order).
     if charm_series.startswith("centos") and client.env.maas:
         client.set_model_constraints({'tags': 'MAAS_NIC_1'})
@@ -146,6 +146,7 @@ def deploy_dummy_stack(client, charm_series, use_charmstore=False):
     else:
         client.wait_for_started(3600)
 
+
 def _deploy_stack(path, client, timeout=3600, charm=False, lxd_profile=None):
     # workaround to ensure lxd profile
     if lxd_profile is not None:
@@ -171,6 +172,7 @@ def _deploy_stack(path, client, timeout=3600, charm=False, lxd_profile=None):
     client.juju(client._show_status, ('--format', 'tabular'))
     return client
 
+
 def deploy_iaas_stack(path, client, timeout=3600, charm=False):
     """deploy a IAAS stack, it assumes that the path to a bundle/charm can be
        reached.
@@ -185,13 +187,15 @@ def deploy_iaas_stack(path, client, timeout=3600, charm=False):
     :param charm: deploy a charm or bundle
     """
 
-    client = _deploy_stack(path, client,
+    client = _deploy_stack(
+        path, client,
         timeout=timeout,
         charm=charm,
         lxd_profile=IAAS_LXD_PROFILE,
     )
     return IaasClient(client)
 
+
 def deploy_caas_stack(path, client, timeout=3600, charm=False):
     """deploy a CAAS stack, it assumes that the path to a bundle/charm can be
        reached.
@@ -206,13 +210,15 @@ def deploy_caas_stack(path, client, timeout=3600, charm=False):
     :param charm: deploy a charm or bundle
     """
 
-    client = _deploy_stack(path, client,
+    client = _deploy_stack(
+        path, client,
         timeout=timeout,
         charm=charm,
         lxd_profile=CAAS_LXD_PROFILE,
     )
     return CaasClient(client)
 
+
 def assess_juju_relations(client):
     token = get_random_string()
     client.set_config('dummy-source', {'token': token})

diff --git a/acceptancetests/jujupy/client.py b/acceptancetests/jujupy/client.py
@@ -2129,12 +2129,13 @@ def switch(self, model=None, controller=None):
 # so parse it via env var ->  https://github.com/kubernetes/client-go/blob/master/tools/clientcmd/loader.go#L44
 KUBE_CONFIG_PATH_ENV_VAR = 'KUBECONFIG'
 
+
 class CaasClient:
     """CaasClient defines a client that can interact with CAAS setup directly.
        Methods and properties that solely interact with a kubernetes
        infrastructure can then be added to the following class.
     """
- 
+
     cloud_name = 'k8cloud'
 
     def __init__(self, client):

diff --git a/acceptancetests/utility.py b/acceptancetests/utility.py
@@ -2,7 +2,7 @@
 from datetime import (
     datetime,
     timedelta,
-    )
+)
 import errno
 import json
 import logging
@@ -14,7 +14,7 @@
 from time import (
     sleep,
     time,
-    )
+)
 from jujupy.utility import (
     ensure_deleted,
     ensure_dir,
@@ -29,7 +29,7 @@
     temp_dir,
     temp_yaml_file,
     until_timeout
-    )
+)
 
 # Imported for other call sites to use.
 __all__ = [
@@ -43,7 +43,7 @@
     'skip_on_missing_file',
     'temp_dir',
     'temp_yaml_file',
-    ]
+]
 
 
 # Equivalent of socket.EAI_NODATA when using windows sockets
@@ -54,6 +54,7 @@
 
 log = logging.getLogger("utility")
 
+
 class PortTimeoutError(Exception):
     pass
 
@@ -64,6 +65,7 @@ class LoggedException(BaseException):
     This is a wrapper to avoid double-printing real Exceptions while still
     unwinding the stack appropriately.
     """
+
     def __init__(self, exception):
         self.exception = exception
 
@@ -224,6 +226,7 @@ def add_arg_juju_bin(parser):
                         ' will use $PATH/juju',
                         default=None)
 
+
 def add_basic_testing_arguments(
         parser, using_jes=False, deadline=True, env=True, existing=True):
     """Returns the parser loaded with basic testing arguments.
@@ -426,6 +429,7 @@ def assert_dict_is_subset(sub_dict, super_dict):
             'Found: {} \nExpected: {}'.format(super_dict, sub_dict))
     return True
 
+
 def add_model(client):
     """Adds a model to the current juju environment then destroys it.
 
@@ -443,6 +447,7 @@ def add_model(client):
         raise JujuAssertionError(error)
     return new_client
 
+
 def get_current_model(client):
     """Gets the current model from Juju's list-models command.
 
@@ -456,6 +461,7 @@ def get_current_model(client):
         log.warning('No model is currently selected.')
         return None
 
+
 def list_models(client):
     """List models.
     :param client: Jujupy ModelClient object