Merge branch '2.9' into mergs-2.9-20210326

jangocheng · Mar 26, 2021 · 0462c2a · 0462c2a
2 parents 70d9cfc + 0581aa3
commit 0462c2a
Show file tree

Hide file tree

Showing 190 changed files with 7,226 additions and 4,976 deletions.
diff --git a/acceptancetests/assess_upgrade.py b/acceptancetests/assess_upgrade.py
@@ -72,7 +72,7 @@ def assess_upgrade_from_stable_to_develop(args, stable_bsm, devel_client):
             stable_client, base_dir, 'released')
         setup_agent_metadata(
             stream_server, args.devel_juju_agent,
-            devel_client, base_dir, 'released')
+            devel_client, base_dir, 'devel')
         with stream_server.server() as url:
             stable_client.env.update_config({
                 'agent-metadata-url': url,
@@ -177,12 +177,19 @@ def setup_agent_metadata(
         version_parts.version,
         version_parts.arch,
         agent_details)
-    # Trusty needed for wikimedia charm.
-    stream_server.add_product(
-        stream,
-        version_parts.version,
-        version_parts.arch,
-        agent_details)
+
+    try:
+        major, minor, patch = version_parts.version.split('.')
+    except ValueError:
+        major, minor_patch = version_parts.version.split('.')
+        minor, patch = minor_patch.split('-')
+    if int(major) == 2 and int(minor) <= 9:
+        stream_server.add_product(
+            stream,
+            version_parts.version,
+            version_parts.arch,
+            agent_details,
+            client.env.get_option('default-series'))
 
 
 def get_version_parts(version_string):

diff --git a/acceptancetests/jujupy/k8s_provider/aks.py b/acceptancetests/jujupy/k8s_provider/aks.py
@@ -21,8 +21,8 @@
 
 import logging
 import os
-from pprint import pformat
 from datetime import datetime, timezone
+from pprint import pformat
 
 import yaml
 from azure.identity import ClientSecretCredential
@@ -46,8 +46,8 @@ class AKS(Base):
     driver = None
     parameters = None
 
-    def __init__(self, bs_manager, cluster_name=None, timeout=1800):
-        super().__init__(bs_manager, cluster_name, timeout)
+    def __init__(self, bs_manager, cluster_name=None, enable_rbac=False, timeout=1800):
+        super().__init__(bs_manager, cluster_name, enable_rbac, timeout)
 
         self.default_storage_class_name = ''
         self.__init_client(bs_manager.client.env)
@@ -137,7 +137,7 @@ def _get_parameters(
             service_principal_profile=service_principal_profile,
             agent_pool_profiles=[agentpool_default],
             linux_profile=linux_profile,
-            enable_rbac=True,
+            enable_rbac=self.enable_rbac,
             tags={'createdAt': datetime.now(tz=timezone.utc).isoformat()},
         )
 

diff --git a/acceptancetests/jujupy/k8s_provider/base.py b/acceptancetests/jujupy/k8s_provider/base.py
@@ -33,6 +33,7 @@
 from jujupy.client import temp_bootstrap_env
 from jujupy.utility import ensure_dir, until_timeout
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -157,6 +158,7 @@ def substrate_context(self):
             self._ensure_kube_dir()
             self.check_cluster_healthy(300)
             self._ensure_cluster_config()
+            self.assert_rbac_config()
 
             yield self
         finally:
@@ -202,6 +204,21 @@ def check_cluster_healthy(self, timeout=60):
         logger.error(err)
         return False
 
+    def assert_rbac_config(self):
+        rbac_enabled_in_cluster = self.check_rbac_enable()
+        if self.enable_rbac and not rbac_enabled_in_cluster:
+            raise Exception("RBAC is required but it's NOT enabled in the cluster")
+        if not self.enable_rbac and rbac_enabled_in_cluster:
+            raise Exception("RBAC is unexpectedly enabled in the cluster")
+
+    def check_rbac_enable(self):
+        timeout = 180
+        cmd = ['/bin/sh', '-c', f'{" ".join(self._kubectl_bin)} run --timeout={timeout}s tmp-shell --restart=Never --rm -i --tty --image bitnami/kubectl:latest -- auth can-i create pods; exit 0']
+        o = self.sh(*cmd, timeout=timeout)
+        logger.info('checking RBAC by run "%s" -> %s', ' '.join(cmd), o)
+        # The default SA in the default namespace does NOT have permission to create pods when RBAC is enabled.
+        return 'no' in o.split()
+
     def kubectl(self, *args):
         return self.sh(*(self._kubectl_bin + args))
 
@@ -217,14 +234,15 @@ def patch_configmap(self, namespace, cm_name, key, value):
         cm['data'] = data
         self.kubectl_apply(json.dumps(cm))
 
-    def sh(self, *args, shell=False, ignore_quote=False):
+    def sh(self, *args, shell=False, ignore_quote=False, timeout=None):
         args = [quote(str(arg)) if shell and not ignore_quote else str(arg) for arg in args]
         logger.debug('sh -> %s', ' '.join(args))
         return subprocess.check_output(
             # cmd should be a list of str.
             args,
             stderr=subprocess.STDOUT,
             shell=shell,
+            timeout=timeout,
         ).decode('UTF-8').strip()
 
     def _ensure_kubectl_bin(self):

diff --git a/acceptancetests/jujupy/k8s_provider/eks.py b/acceptancetests/jujupy/k8s_provider/eks.py
@@ -43,8 +43,8 @@ class EKS(Base):
     location = None
     parameters = None
 
-    def __init__(self, bs_manager, cluster_name=None, timeout=1800):
-        super().__init__(bs_manager, cluster_name, timeout)
+    def __init__(self, bs_manager, cluster_name=None, enable_rbac=False, timeout=1800):
+        super().__init__(bs_manager, cluster_name, enable_rbac, timeout)
 
         self._eksctl_bin = os.path.join(self.juju_home, 'eksctl')
         self._ensure_eksctl_bin()

diff --git a/acceptancetests/jujupy/k8s_provider/gke.py b/acceptancetests/jujupy/k8s_provider/gke.py
@@ -48,8 +48,8 @@ class GKE(Base):
     driver = None
     default_params = None
 
-    def __init__(self, bs_manager, cluster_name=None, timeout=1800):
-        super().__init__(bs_manager, cluster_name, timeout)
+    def __init__(self, bs_manager, cluster_name=None, enable_rbac=False, timeout=1800):
+        super().__init__(bs_manager, cluster_name, enable_rbac, timeout)
 
         self.default_storage_class_name = ''
         self.__init_driver(bs_manager.client.env)

diff --git a/acceptancetests/jujupy/k8s_provider/kubernetes_core.py b/acceptancetests/jujupy/k8s_provider/kubernetes_core.py
@@ -20,16 +20,12 @@
 from __future__ import print_function
 
 import logging
-import tempfile
 import subprocess
+import tempfile
 
-from .base import (
-    Base,
-    K8sProviderType,
-)
+from .base import Base, K8sProviderType
 from .factory import register_provider
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -182,8 +178,8 @@ class KubernetesCore(Base):
 
     name = K8sProviderType.K8S_CORE
 
-    def __init__(self, bs_manager, cluster_name=None, timeout=1800):
-        super().__init__(bs_manager, cluster_name, timeout)
+    def __init__(self, bs_manager, cluster_name=None, enable_rbac=False, timeout=1800):
+        super().__init__(bs_manager, cluster_name, enable_rbac, timeout)
         self.default_storage_class_name = "juju-storageclass"
 
     def _ensure_kube_dir(self):

diff --git a/acceptancetests/jujupy/k8s_provider/microk8s.py b/acceptancetests/jujupy/k8s_provider/microk8s.py
@@ -43,8 +43,8 @@ class MicroK8s(Base):
     name = K8sProviderType.MICROK8S
     cloud_name = 'microk8s'  # built-in cloud name
 
-    def __init__(self, bs_manager, cluster_name=None, timeout=1800):
-        super().__init__(bs_manager, cluster_name, timeout)
+    def __init__(self, bs_manager, cluster_name=None, enable_rbac=False, timeout=1800):
+        super().__init__(bs_manager, cluster_name, enable_rbac, timeout)
         self.default_storage_class_name = 'microk8s-hostpath'
 
     def _ensure_cluster_stack(self):
@@ -95,7 +95,7 @@ def enable_microk8s_addons(self, addons=None):
                 addons.append('rbac')
         else:
             addons = [addon for addon in addons if addon != 'rbac']
-            logger.info('disabling rbac -> ', self.sh('microk8s.disable', 'rbac'))
+            logger.info('disabling rbac -> %s', self.sh('microk8s.disable', 'rbac'))
 
         def wait_until_ready(timeout, checker):
             for _ in until_timeout(timeout):

diff --git a/acceptancetests/jujupy/stream_server.py b/acceptancetests/jujupy/stream_server.py
@@ -65,7 +65,7 @@ def __init__(self, working_dir):
 
         os.makedirs(self._agent_path)
 
-    def add_product(self, content_id, version, arch, agent_tgz_path):
+    def add_product(self, content_id, version, arch, agent_tgz_path, series=None):
         """Add a new product to generate stream data for.
 
         :param content_id: String ID (e.g.'proposed', 'release')
@@ -74,10 +74,12 @@ def add_product(self, content_id, version, arch, agent_tgz_path):
         :param agent_tgz_path: String full path to agent tarball file to use.
           This file is copied into the JujuStreamData working dir to be served
           up at a later date.
+        :param series: Series string that appears in item_name
+          (e.g. 'bionic', 'xenial', 'centos')
         """
         shutil.copy(agent_tgz_path, self._agent_path)
         product_dict = _generate_product_json(
-            content_id, version, arch, agent_tgz_path)
+            content_id, version, arch, agent_tgz_path, series)
         self.products.append(product_dict)
 
     def generate_stream_data(self):
@@ -106,7 +108,7 @@ def __init__(self, base_dir, stream_data_type=_JujuStreamData):
         self.base_dir = base_dir
         self.stream_data = stream_data_type(base_dir)
 
-    def add_product(self, content_id, version, arch, agent_tgz_path):
+    def add_product(self, content_id, version, arch, agent_tgz_path, series=None):
         """Add a new product to generate stream data for.
 
         :param content_id: String ID (e.g.'proposed', 'released')
@@ -115,9 +117,11 @@ def add_product(self, content_id, version, arch, agent_tgz_path):
         :param agent_tgz_path: String full path to agent tarball file to use.
           This file is copied into the JujuStreamData working dir to be served
           up at a later date.
+        :param series: Series string that appears in item_name
+          (e.g. 'bionic', 'xenial', 'centos')
         """
         self.stream_data.add_product(
-            content_id, version, arch, agent_tgz_path)
+            content_id, version, arch, agent_tgz_path, series)
         # Re-generate when adding a product allows updating the server while
         # running.
         # Can be noisey in the logs, if a lot of products need to be added can
@@ -208,44 +212,54 @@ def agent_tgz_from_juju_binary(
     return tgz_path
 
 
-def _generate_product_json(content_id, version, arch, series, agent_tgz_path):
+def _generate_product_json(content_id, version, arch, agent_tgz_path, series=None):
     """Return dict containing product metadata from provided args."""
     tgz_name = os.path.basename(agent_tgz_path)
     file_details = _get_tgz_file_details(agent_tgz_path)
-    item_name = '{version}-ubuntu-{arch}'.format(
+    index_part = 'agents'
+    product_part = 'ubuntu'
+    release = 'ubuntu'
+    # If series is supplied we need to generate old style metadata.
+    if series is not None:
+        release = series
+        product_part = _get_series_code(series)
+        index_part = 'tools'
+    item_name = '{version}-{release}-{arch}'.format(
         version=version,
+        release=release,
         arch=arch)
     return dict(
         arch=arch,
-        content_id='com.ubuntu.juju:{}:agents'.format(content_id),
+        content_id='com.ubuntu.juju:{}:{}'.format(content_id, index_part),
         format='products:1.0',
         ftype='tar.gz',
         item_name=item_name,
         md5=file_details['md5'],
         path=os.path.join('agent', tgz_name),
-        product_name='com.ubuntu.juju:ubuntu:{arch}'.format(
+        product_name='com.ubuntu.juju:{product_part}:{arch}'.format(
+            product_part=product_part,
             arch=arch),
-        release='ubuntu',
+        release=release,
         sha256=file_details['sha256'],
         size=file_details['size'],
         version=version,
         version_name=datetime.utcnow().strftime('%Y%m%d')
     )
 
 
-def _get_series_details(series):
+def _get_series_code(series):
     # Ubuntu agents use series and a code (i.e. trusty:14.04), others don't.
     _series_lookup = dict(
         trusty=14.04,
         xenial=16.04,
-        artful=17.10,
         bionic=18.04,
+        focal=20.04,
     )
     try:
         series_code = _series_lookup[series]
     except KeyError:
-        return series, series
-    return series, series_code
+        return series
+    return series_code
 
 
 def _get_tgz_file_details(agent_tgz_path):

diff --git a/api/agent/machine_test.go b/api/agent/machine_test.go
@@ -167,7 +167,7 @@ func (s *machineSuite) TestEntitySetPassword(c *gc.C) {
 	// Check that we cannot log in to mongo with the correct password.
 	// This is because there's no mongo password set for s.machine,
 	// which has JobHostUnits
-	info := s.MongoInfo(c)
+	info := s.MongoInfo()
 	// TODO(dfc) this entity.Tag should return a Tag
 	tag, err := names.ParseTag(entity.Tag())
 	c.Assert(err, jc.ErrorIsNil)

diff --git a/api/migrationtarget/client.go b/api/migrationtarget/client.go
@@ -76,6 +76,7 @@ func (c *Client) Activate(modelUUID string) error {
 func (c *Client) UploadCharm(modelUUID string, curl *charm.URL, content io.ReadSeeker) (*charm.URL, error) {
 	args := url.Values{}
 	args.Add("schema", curl.Schema)
+	args.Add("arch", curl.Architecture)
 	args.Add("user", curl.User)
 	args.Add("series", curl.Series)
 	args.Add("revision", strconv.Itoa(curl.Revision))
@@ -87,11 +88,11 @@ func (c *Client) UploadCharm(modelUUID string, curl *charm.URL, content io.ReadS
 		return nil, errors.Trace(err)
 	}
 
-	curl, err := charm.ParseURL(resp.CharmURL)
+	respCurl, err := charm.ParseURL(resp.CharmURL)
 	if err != nil {
 		return nil, errors.Annotatef(err, "bad charm URL in response")
 	}
-	return curl, nil
+	return respCurl, nil
 }
 
 // UploadTools uploads tools at the specified location to the API server over HTTPS

diff --git a/api/migrationtarget/client_test.go b/api/migrationtarget/client_test.go
@@ -189,7 +189,25 @@ func (s *ClientSuite) TestUploadCharm(c *gc.C) {
 	c.Assert(err, jc.ErrorIsNil)
 	c.Assert(outCurl, gc.DeepEquals, curl)
 	c.Assert(doer.method, gc.Equals, "POST")
-	c.Assert(doer.url, gc.Equals, "/migrate/charms?revision=2&schema=cs&series=&user=user")
+	c.Assert(doer.url, gc.Equals, "/migrate/charms?arch=&revision=2&schema=cs&series=&user=user")
+	c.Assert(doer.body, gc.Equals, charmBody)
+}
+
+func (s *ClientSuite) TestUploadCharmHubCharm(c *gc.C) {
+	const charmBody = "charming"
+	curl := charm.MustParseURL("ch:s390x/bionic/juju-qa-test-15")
+	doer := newFakeDoer(c, params.CharmsResponse{
+		CharmURL: curl.String(),
+	})
+	caller := &fakeHTTPCaller{
+		httpClient: &httprequest.Client{Doer: doer},
+	}
+	client := migrationtarget.NewClient(caller)
+	outCurl, err := client.UploadCharm("uuid", curl, strings.NewReader(charmBody))
+	c.Assert(err, jc.ErrorIsNil)
+	c.Assert(outCurl, gc.DeepEquals, curl)
+	c.Assert(doer.method, gc.Equals, "POST")
+	c.Assert(doer.url, gc.Equals, "/migrate/charms?arch=s390x&revision=15&schema=ch&series=bionic&user=")
 	c.Assert(doer.body, gc.Equals, charmBody)
 }
 

diff --git a/apiserver/charms.go b/apiserver/charms.go
@@ -247,10 +247,11 @@ func (h *charmsHandler) processPost(r *http.Request, st *state.State) (*charm.UR
 
 	// We got it, now let's reserve a charm URL for it in state.
 	curl := &charm.URL{
-		Schema:   schema,
-		Name:     archive.Meta().Name,
-		Revision: archive.Revision(),
-		Series:   series,
+		Schema:       schema,
+		Architecture: query.Get("arch"),
+		Name:         archive.Meta().Name,
+		Revision:     archive.Revision(),
+		Series:       series,
 	}
 	switch charm.Schema(schema) {
 	case charm.Local: