Fix CI upgrade tests and non amd64 unit tests

sdg0919 · Mar 26, 2021 · 6e58d7a · 6e58d7a
1 parent 85c0e7a
commit 6e58d7a
Show file tree

Hide file tree

Showing 7 changed files with 95 additions and 41 deletions.
diff --git a/acceptancetests/assess_upgrade.py b/acceptancetests/assess_upgrade.py
@@ -72,7 +72,7 @@ def assess_upgrade_from_stable_to_develop(args, stable_bsm, devel_client):
             stable_client, base_dir, 'released')
         setup_agent_metadata(
             stream_server, args.devel_juju_agent,
-            devel_client, base_dir, 'develop')
+            devel_client, base_dir, 'devel')
         with stream_server.server() as url:
             stable_client.env.update_config({
                 'agent-metadata-url': url,
@@ -178,6 +178,19 @@ def setup_agent_metadata(
         version_parts.arch,
         agent_details)
 
+    try:
+        major, minor, patch = version_parts.version.split('.')
+    except ValueError:
+        major, minor_patch = version_parts.version.split('.')
+        minor, patch = minor_patch.split('-')
+    if int(major) == 2 and int(minor) <= 9:
+        stream_server.add_product(
+            stream,
+            version_parts.version,
+            version_parts.arch,
+            agent_details,
+            client.env.get_option('default-series'))
+
 
 def get_version_parts(version_string):
     parts = get_version_string_parts(version_string)

diff --git a/acceptancetests/jujupy/stream_server.py b/acceptancetests/jujupy/stream_server.py
@@ -65,7 +65,7 @@ def __init__(self, working_dir):
 
         os.makedirs(self._agent_path)
 
-    def add_product(self, content_id, version, arch, agent_tgz_path):
+    def add_product(self, content_id, version, arch, agent_tgz_path, series=None):
         """Add a new product to generate stream data for.
 
         :param content_id: String ID (e.g.'proposed', 'release')
@@ -74,10 +74,12 @@ def add_product(self, content_id, version, arch, agent_tgz_path):
         :param agent_tgz_path: String full path to agent tarball file to use.
           This file is copied into the JujuStreamData working dir to be served
           up at a later date.
+        :param series: Series string that appears in item_name
+          (e.g. 'bionic', 'xenial', 'centos')
         """
         shutil.copy(agent_tgz_path, self._agent_path)
         product_dict = _generate_product_json(
-            content_id, version, arch, agent_tgz_path)
+            content_id, version, arch, agent_tgz_path, series)
         self.products.append(product_dict)
 
     def generate_stream_data(self):
@@ -106,7 +108,7 @@ def __init__(self, base_dir, stream_data_type=_JujuStreamData):
         self.base_dir = base_dir
         self.stream_data = stream_data_type(base_dir)
 
-    def add_product(self, content_id, version, arch, agent_tgz_path):
+    def add_product(self, content_id, version, arch, agent_tgz_path, series=None):
         """Add a new product to generate stream data for.
 
         :param content_id: String ID (e.g.'proposed', 'released')
@@ -115,9 +117,11 @@ def add_product(self, content_id, version, arch, agent_tgz_path):
         :param agent_tgz_path: String full path to agent tarball file to use.
           This file is copied into the JujuStreamData working dir to be served
           up at a later date.
+        :param series: Series string that appears in item_name
+          (e.g. 'bionic', 'xenial', 'centos')
         """
         self.stream_data.add_product(
-            content_id, version, arch, agent_tgz_path)
+            content_id, version, arch, agent_tgz_path, series)
         # Re-generate when adding a product allows updating the server while
         # running.
         # Can be noisey in the logs, if a lot of products need to be added can
@@ -208,31 +212,56 @@ def agent_tgz_from_juju_binary(
     return tgz_path
 
 
-def _generate_product_json(content_id, version, arch, agent_tgz_path):
+def _generate_product_json(content_id, version, arch, agent_tgz_path, series=None):
     """Return dict containing product metadata from provided args."""
     tgz_name = os.path.basename(agent_tgz_path)
     file_details = _get_tgz_file_details(agent_tgz_path)
-    item_name = '{version}-ubuntu-{arch}'.format(
+    index_part = 'agents'
+    product_part = 'ubuntu'
+    release = 'ubuntu'
+    # If series is supplied we need to generate old style metadata.
+    if series is not None:
+        release = series
+        product_part = _get_series_code(series)
+        index_part = 'tools'
+    item_name = '{version}-{release}-{arch}'.format(
         version=version,
+        release=release,
         arch=arch)
     return dict(
         arch=arch,
-        content_id='com.ubuntu.juju:{}:agents'.format(content_id),
+        content_id='com.ubuntu.juju:{}:{}'.format(content_id, index_part),
         format='products:1.0',
         ftype='tar.gz',
         item_name=item_name,
         md5=file_details['md5'],
         path=os.path.join('agent', tgz_name),
-        product_name='com.ubuntu.juju:ubuntu:{arch}'.format(
+        product_name='com.ubuntu.juju:{product_part}:{arch}'.format(
+            product_part=product_part,
             arch=arch),
-        release='ubuntu',
+        release=release,
         sha256=file_details['sha256'],
         size=file_details['size'],
         version=version,
         version_name=datetime.utcnow().strftime('%Y%m%d')
     )
 
 
+def _get_series_code(series):
+    # Ubuntu agents use series and a code (i.e. trusty:14.04), others don't.
+    _series_lookup = dict(
+        trusty=14.04,
+        xenial=16.04,
+        bionic=18.04,
+        focal=20.04,
+    )
+    try:
+        series_code = _series_lookup[series]
+    except KeyError:
+        return series
+    return series_code
+
+
 def _get_tgz_file_details(agent_tgz_path):
     file_details = dict(size=os.path.getsize(agent_tgz_path))
     with open(agent_tgz_path, 'rb') as f:

diff --git a/api/agent/machine_test.go b/api/agent/machine_test.go
@@ -167,7 +167,7 @@ func (s *machineSuite) TestEntitySetPassword(c *gc.C) {
 	// Check that we cannot log in to mongo with the correct password.
 	// This is because there's no mongo password set for s.machine,
 	// which has JobHostUnits
-	info := s.MongoInfo(c)
+	info := s.MongoInfo()
 	// TODO(dfc) this entity.Tag should return a Tag
 	tag, err := names.ParseTag(entity.Tag())
 	c.Assert(err, jc.ErrorIsNil)

diff --git a/apiserver/facades/client/application/application.go b/apiserver/facades/client/application/application.go
@@ -1106,7 +1106,7 @@ func (api *APIBase) setConfig(app Application, generation, settingsYAML string,
 	return nil
 }
 
-// UpdateApplicationSeries updates the application series. Release for
+// UpdateApplicationSeries updates the application series. Series for
 // subordinates updated too.
 func (api *APIBase) UpdateApplicationSeries(args params.UpdateSeriesArgs) (params.ErrorResults, error) {
 	if err := api.checkCanWrite(); err != nil {

diff --git a/apiserver/facades/schema.json b/apiserver/facades/schema.json
@@ -2388,7 +2388,7 @@
                             "$ref": "#/definitions/ErrorResults"
                         }
                     },
-                    "description": "UpdateApplicationSeries updates the application series. Release for\nsubordinates updated too."
+                    "description": "UpdateApplicationSeries updates the application series. Series for\nsubordinates updated too."
                 }
             },
             "definitions": {

diff --git a/cmd/jujud/agent/agenttest/agent.go b/cmd/jujud/agent/agenttest/agent.go
@@ -135,7 +135,7 @@ func (s *AgentSuite) PrimeAgentVersion(c *gc.C, tag names.Tag, password string,
 	c.Assert(err, jc.ErrorIsNil)
 	c.Assert(tools1, gc.DeepEquals, agentTools)
 
-	stateInfo := s.MongoInfo(c)
+	stateInfo := s.MongoInfo()
 	apiInfo := s.APIInfo(c)
 	paths := agent.DefaultPaths
 	paths.DataDir = s.DataDir()
@@ -198,7 +198,7 @@ func (s *AgentSuite) WriteStateAgentConfig(
 	vers version.Binary,
 	modelTag names.ModelTag,
 ) agent.ConfigSetterWriter {
-	stateInfo := s.MongoInfo(c)
+	stateInfo := s.MongoInfo()
 	apiPort := gitjujutesting.FindTCPPort()
 	s.SetControllerConfigAPIPort(c, apiPort)
 	apiAddr := []string{fmt.Sprintf("localhost:%d", apiPort)}

diff --git a/juju/testing/conn.go b/juju/testing/conn.go
@@ -15,11 +15,14 @@ import (
 
 	"github.com/juju/charm/v8"
 	"github.com/juju/cmd/cmdtesting"
+	"github.com/juju/collections/set"
 	"github.com/juju/errors"
+	coreseries "github.com/juju/juju/core/series"
 	"github.com/juju/loggo"
 	"github.com/juju/names/v4"
+	"github.com/juju/os/v2/series"
 	"github.com/juju/pubsub"
-	gitjujutesting "github.com/juju/testing"
+	jujutesting "github.com/juju/testing"
 	jc "github.com/juju/testing/checkers"
 	"github.com/juju/utils/v2"
 	"github.com/juju/utils/v2/arch"
@@ -98,7 +101,7 @@ type JujuConnSuite struct {
 	// /var/lib/juju: the use cases are completely non-overlapping, and any tests that
 	// really do need both to exist ought to be embedding distinct fixtures for the
 	// distinct environments.
-	gitjujutesting.MgoSuite
+	jujutesting.MgoSuite
 	testing.FakeJujuXDGDataHomeSuite
 	envtesting.ToolsFixture
 
@@ -239,7 +242,7 @@ func (s *JujuConnSuite) controllerIdleFunc() {
 func (s *JujuConnSuite) WaitForNextSync(c *gc.C) {
 	select {
 	case <-s.txnSyncNotify:
-	case <-time.After(gitjujutesting.LongWait):
+	case <-time.After(jujutesting.LongWait):
 		c.Fatal("no sync event sent, is the watcher dead?")
 	}
 	// It is possible that the previous sync was in progress
@@ -248,7 +251,7 @@ func (s *JujuConnSuite) WaitForNextSync(c *gc.C) {
 	// the txnwatcher.
 	select {
 	case <-s.txnSyncNotify:
-	case <-time.After(gitjujutesting.LongWait):
+	case <-time.After(jujutesting.LongWait):
 		c.Fatal("no sync event sent, is the watcher dead?")
 	}
 }
@@ -290,7 +293,7 @@ func (s *JujuConnSuite) WaitForModelWatchersIdle(c *gc.C, modelUUID string) {
 		}
 	}()
 
-	timeout := time.After(gitjujutesting.LongWait)
+	timeout := time.After(jujutesting.LongWait)
 watcher:
 	for {
 		select {
@@ -306,7 +309,7 @@ watcher:
 	select {
 	case <-controllerIdleChan:
 		// done
-	case <-time.After(gitjujutesting.LongWait):
+	case <-time.After(jujutesting.LongWait):
 		c.Fatal("no controller idle event sent, is the controller dead?")
 	}
 }
@@ -341,7 +344,7 @@ func (s *JujuConnSuite) AdminUserTag(c *gc.C) names.UserTag {
 	return owner
 }
 
-func (s *JujuConnSuite) MongoInfo(c *gc.C) *mongo.MongoInfo {
+func (s *JujuConnSuite) MongoInfo() *mongo.MongoInfo {
 	info := statetesting.NewMongoInfo()
 	info.Password = AdminSecret
 	return info
@@ -421,23 +424,30 @@ func (s *JujuConnSuite) OpenAPIAsNewMachine(c *gc.C, jobs ...state.MachineJob) (
 // environment's host architecture. Additionally, it ensures that 'versions'
 // for amd64 are returned if that is not the current host's architecture.
 func DefaultVersions(conf *config.Config) []version.Binary {
-	agentVersion, set := conf.AgentVersion()
-	if !set {
+	agentVersion, isSet := conf.AgentVersion()
+	if !isSet {
 		agentVersion = jujuversion.Current
 	}
+	osTypes := set.NewStrings("ubuntu")
+	hostSeries, err := series.HostSeries()
+	if err != nil {
+		osTypes.Add(coreseries.DefaultOSTypeNameFromSeries(hostSeries))
+	}
 	var versions []version.Binary
-	versions = append(versions, version.Binary{
-		Number:  agentVersion,
-		Arch:    arch.HostArch(),
-		Release: "ubuntu",
-	})
-	if arch.HostArch() != "amd64" {
+	for _, osType := range osTypes.Values() {
 		versions = append(versions, version.Binary{
 			Number:  agentVersion,
-			Arch:    "amd64",
-			Release: "ubuntu",
+			Arch:    arch.HostArch(),
+			Release: osType,
 		})
+		if arch.HostArch() != "amd64" {
+			versions = append(versions, version.Binary{
+				Number:  agentVersion,
+				Arch:    "amd64",
+				Release: osType,
+			})
 
+		}
 	}
 	return versions
 }
@@ -457,7 +467,7 @@ type UserHomeParams struct {
 	SetOldHome bool
 }
 
-// Create a home directory and Juju data home for user username.
+// CreateUserHome creates a home directory and Juju data home for user username.
 // This is used by setUpConn to create the 'ubuntu' user home, after RootDir,
 // and may be used again later for other users.
 func (s *JujuConnSuite) CreateUserHome(c *gc.C, params *UserHomeParams) {
@@ -645,7 +655,9 @@ func (s *JujuConnSuite) setUpConn(c *gc.C) {
 func (s *JujuConnSuite) AddToolsToState(c *gc.C, versions ...version.Binary) {
 	stor, err := s.State.ToolsStorage()
 	c.Assert(err, jc.ErrorIsNil)
-	defer stor.Close()
+	defer func() {
+		_ = stor.Close()
+	}()
 	for _, v := range versions {
 		content := v.String()
 		hash := fmt.Sprintf("sha256(%s)", content)
@@ -658,7 +670,7 @@ func (s *JujuConnSuite) AddToolsToState(c *gc.C, versions ...version.Binary) {
 	}
 }
 
-// AddDefaultTools adds tools to tools storage for default juju
+// AddDefaultToolsToState adds tools to tools storage for default juju
 // series and architectures.
 func (s *JujuConnSuite) AddDefaultToolsToState(c *gc.C) {
 	versions := DefaultVersions(s.Environ.Config())
@@ -764,7 +776,7 @@ type GetStater interface {
 }
 
 func (s *JujuConnSuite) tearDownConn(c *gc.C) {
-	testServer := gitjujutesting.MgoServer.Addr()
+	testServer := jujutesting.MgoServer.Addr()
 	serverAlive := testServer != ""
 
 	// Close any api connections we know about first.
@@ -873,13 +885,13 @@ func (s *JujuConnSuite) AddTestingApplicationWithBindings(c *gc.C, name string,
 func (s *JujuConnSuite) AgentConfigForTag(c *gc.C, tag names.Tag) agent.ConfigSetterWriter {
 	password, err := utils.RandomPassword()
 	c.Assert(err, jc.ErrorIsNil)
-	paths := agent.DefaultPaths
-	paths.DataDir = s.DataDir()
+	defaultPaths := agent.DefaultPaths
+	defaultPaths.DataDir = s.DataDir()
 	model, err := s.State.Model()
 	c.Assert(err, jc.ErrorIsNil)
-	config, err := agent.NewAgentConfig(
+	agentConfig, err := agent.NewAgentConfig(
 		agent.AgentConfigParams{
-			Paths:             paths,
+			Paths:             defaultPaths,
 			Tag:               tag,
 			UpgradedToVersion: jujuversion.Current,
 			Password:          password,
@@ -890,7 +902,7 @@ func (s *JujuConnSuite) AgentConfigForTag(c *gc.C, tag names.Tag) agent.ConfigSe
 			Model:             model.ModelTag(),
 		})
 	c.Assert(err, jc.ErrorIsNil)
-	return config
+	return agentConfig
 }
 
 // AssertConfigParameterUpdated updates environment parameter and