Merge branch '2.8' into merge-2.8-20210315

sdg0919 · Mar 16, 2021 · da0a1fd · da0a1fd
2 parents 82b88d1 + 1581034
commit da0a1fd
Show file tree

Hide file tree

Showing 157 changed files with 2,383 additions and 823 deletions.
diff --git a/acceptancetests/assess_caas_deploy_kubeflow.py b/acceptancetests/assess_caas_deploy_kubeflow.py
@@ -340,10 +340,13 @@ def prepare(caas_client, caas_provider, build):
         )
 
 
-def run_test(caas_provider, k8s_model, bundle):
+def run_test(caas_provider, k8s_model, bundle, build):
     if caas_provider != K8sProviderType.MICROK8S.name:
         # tests/run.sh only works for microk8s.
-        log.info("%s/tests/run.sh skipped for %s k8s provider", KUBEFLOW_DIR, caas_provider)
+        log.info("%s/tests/run.sh is skipped for %s k8s provider", KUBEFLOW_DIR, caas_provider)
+        return
+    if not build:
+        log.info("%s/tests/run.sh is skipped for released bundle", KUBEFLOW_DIR)
         return
     # inject `JUJU_DATA` for running tests.
     os.environ['JUJU_DATA'] = k8s_model.env.juju_home
@@ -352,25 +355,58 @@ def run_test(caas_provider, k8s_model, bundle):
         run("sg", "microk8s", "-c", f"{KUBEFLOW_DIR}/tests/run.sh -m {bundle}")
 
 
-def assess_caas_kubeflow_deployment(caas_client, caas_provider, bundle, build=False):
+def dump_k8s_log(artifacts_dir, file_name, content):
+    if not os.path.isdir(artifacts_dir):
+        os.mkdir(artifacts_dir)
+
+    path = os.path.join(artifacts_dir, file_name)
+    if os.path.isfile(path):
+        raise Exception(f'{path} already exists')
+    with open(path, 'w') as f:
+        f.write(content)
+
+
+def dump_containers_log(model_name, kubectl, dump_log):
+    for pod in json.loads(kubectl('get', 'pods', '-n', model_name, '-o', 'json'))['items']:
+        for container in pod['spec']['containers']:
+            pod_name = pod['metadata']['name']
+            container_name = container['name']
+            log = kubectl('-n', model_name, 'logs', '--timestamps', pod_name, '-c', container_name)
+            dump_log(f'{pod_name}-{container_name}.log', log)
+
+
+def assess_caas_kubeflow_deployment(caas_client, caas_provider, bundle, build=False, log_dir=None):
     if not caas_client.check_cluster_healthy(timeout=60):
         raise JujuAssertionError('k8s cluster is not healthy because kubectl is not accessible')
 
     model_name = 'kubeflow'
     k8s_model = caas_client.add_model(model_name)
 
+    def dump_log(file_name, content): dump_k8s_log(os.path.join(log_dir, model_name), file_name, content)
+
     def success_hook():
-        log.info(caas_client.kubectl('get', 'all,pv,pvc,ing', '--all-namespaces', '-o', 'wide'))
-        log.info(caas_client.kubectl('get', 'sa,roles,clusterroles,rolebindings,clusterrolebindings', '-oyaml', '-A'))
+        dump_log(
+            'all_pv_pvc_ing.txt',
+            caas_client.kubectl('get', 'all,pv,pvc,ing', '--all-namespaces', '-o', 'wide'),
+        )
+        dump_log(
+            'sa_roles_clusterroles_rolebindings_clusterrolebindings.yaml',
+            caas_client.kubectl('get', 'sa,roles,clusterroles,rolebindings,clusterrolebindings', '-oyaml', '-A'),
+        )
 
     def fail_hook():
         success_hook()
-        ns_dumps = caas_client.kubectl('get', 'all,pv,pvc,ing', '-n', model_name, '-o', 'json')
-        log.info('all resources in namespace %s -> %s', model_name, pformat(json.loads(ns_dumps)))
-
-        log.info(
-            'describing pods in %s ->\n%s',
-            model_name, caas_client.kubectl('describe', 'pods', f'-n{model_name}'),
+        dump_log(
+            f'all_pv_pvc_ing_{model_name}.yaml',
+            caas_client.kubectl('get', 'all,pv,pvc,ing', '-n', model_name, '-o', 'yaml'),
+        )
+        dump_log(
+            f'describe_pods_{model_name}.txt',
+            caas_client.kubectl('describe', 'pods', f'-n{model_name}'),
+        )
+        dump_containers_log(
+            model_name,
+            caas_client.kubectl, dump_log,
         )
         caas_client.ensure_cleanup()
 
@@ -380,7 +416,7 @@ def fail_hook():
         log.info("sleeping for 30 seconds to let everything start up")
         sleep(30)
 
-        run_test(caas_provider, k8s_model, bundle)
+        run_test(caas_provider, k8s_model, bundle, build)
         k8s_model.juju(k8s_model._show_status, ('--format', 'tabular'))
         success_hook()
     except:  # noqa: E722
@@ -445,7 +481,11 @@ def main(argv=None):
             if not args.k8s_controller:
                 # add-k8s to controller
                 caas_client.add_k8s(False)
-            assess_caas_kubeflow_deployment(caas_client, args.caas_provider, args.bundle, args.build)
+            assess_caas_kubeflow_deployment(
+                caas_client,
+                args.caas_provider, args.bundle, args.build,
+                bs_manager.log_dir,
+            )
         return 0
 
 

diff --git a/api/facadeversions.go b/api/facadeversions.go
@@ -70,7 +70,7 @@ var facadeVersions = map[string]int{
 	"MachineActions":               1,
 	"MachineManager":               6,
 	"MachineUndertaker":            1,
-	"Machiner":                     4,
+	"Machiner":                     5,
 	"MeterStatus":                  2,
 	"MetricsAdder":                 2,
 	"MetricsDebug":                 2,

diff --git a/api/imagemetadatamanager/client_test.go b/api/imagemetadatamanager/client_test.go
@@ -7,13 +7,13 @@ import (
 	"regexp"
 
 	"github.com/juju/errors"
-	"github.com/juju/os/v2/series"
 	jc "github.com/juju/testing/checkers"
 	gc "gopkg.in/check.v1"
 
 	"github.com/juju/juju/api/base/testing"
 	"github.com/juju/juju/api/imagemetadatamanager"
 	"github.com/juju/juju/apiserver/params"
+	"github.com/juju/juju/core/series"
 	coretesting "github.com/juju/juju/testing"
 )
 

diff --git a/api/machiner/machine.go b/api/machiner/machine.go
@@ -125,19 +125,35 @@ func (m *Machine) SetObservedNetworkConfig(netConfig []params.NetworkConfig) err
 	return nil
 }
 
-// RecordAgentStartTime updates the start time for the agent running on the
-// machine.
-func (m *Machine) RecordAgentStartTime() error {
+// RecordAgentStartInformation reports the host name of the machine and updates
+// the start time for the agent.
+func (m *Machine) RecordAgentStartInformation(hostname string) error {
+	var (
+		result params.ErrorResults
+		err    error
+	)
+
+	switch {
 	// Ignore if connecting to an older, not upgraded controller
-	if m.st.facade.BestAPIVersion() < 2 {
+	case m.st.facade.BestAPIVersion() < 2:
 		return nil
+	case m.st.facade.BestAPIVersion() < 5: // Only supports RecordAgentStartTime
+		args := params.Entities{
+			Entities: []params.Entity{{Tag: m.tag.String()}},
+		}
+		err = m.st.facade.FacadeCall("RecordAgentStartTime", args, &result)
+	default: // Supports RecordAgentStartInformation
+		args := params.RecordAgentStartInformationArgs{
+			Args: []params.RecordAgentStartInformationArg{
+				{
+					Tag:      m.tag.String(),
+					Hostname: hostname,
+				},
+			},
+		}
+		err = m.st.facade.FacadeCall("RecordAgentStartInformation", args, &result)
 	}
 
-	var result params.ErrorResults
-	args := params.Entities{
-		Entities: []params.Entity{{Tag: m.tag.String()}},
-	}
-	err := m.st.facade.FacadeCall("RecordAgentStartTime", args, &result)
 	if err != nil {
 		return err
 	}

diff --git a/api/machiner/machiner_test.go b/api/machiner/machiner_test.go
@@ -208,7 +208,7 @@ func (s *machinerSuite) TestWatch(c *gc.C) {
 	wc.AssertOneChange()
 }
 
-func (s *machinerSuite) TestRecordAgentStartTime(c *gc.C) {
+func (s *machinerSuite) TestRecordAgentStartInformation(c *gc.C) {
 	mTag := names.NewMachineTag("1")
 	stMachine, err := s.State.Machine(mTag.Id())
 	c.Assert(err, jc.ErrorIsNil)
@@ -217,11 +217,12 @@ func (s *machinerSuite) TestRecordAgentStartTime(c *gc.C) {
 	machine, err := s.machiner.Machine(mTag)
 	c.Assert(err, jc.ErrorIsNil)
 
-	err = machine.RecordAgentStartTime()
+	err = machine.RecordAgentStartInformation("thundering-herds")
 	c.Assert(err, jc.ErrorIsNil)
 
 	err = stMachine.Refresh()
 	c.Assert(err, jc.ErrorIsNil)
 
 	c.Assert(stMachine.AgentStartTime(), gc.Not(gc.Equals), oldStartedAt, gc.Commentf("expected the agent start time to be updated"))
+	c.Assert(stMachine.Hostname(), gc.Equals, "thundering-herds", gc.Commentf("expected for the recorded machine hostname to be updated"))
 }
diff --git a/apiserver/allfacades.go b/apiserver/allfacades.go
@@ -257,7 +257,8 @@ func AllFacades() *facade.Registry {
 	reg("Machiner", 1, machine.NewMachinerAPIV1)
 	reg("Machiner", 2, machine.NewMachinerAPIV2) // Adds RecordAgentStartTime.
 	reg("Machiner", 3, machine.NewMachinerAPIV3) // Relies on agent-set origin in SetObservedNetworkConfig.
-	reg("Machiner", 4, machine.NewMachinerAPI)   // Removes SetProviderNetworkConfig.
+	reg("Machiner", 4, machine.NewMachinerAPIV4) // Removes SetProviderNetworkConfig.
+	reg("Machiner", 5, machine.NewMachinerAPI)   // Adds RecordAgentHostAndStartTime.
 
 	reg("MeterStatus", 1, meterstatus.NewMeterStatusFacadeV1)
 	reg("MeterStatus", 2, meterstatus.NewMeterStatusFacade)

diff --git a/apiserver/authentication/agent.go b/apiserver/authentication/agent.go
@@ -14,7 +14,7 @@ import (
 	"github.com/juju/juju/state"
 )
 
-// AgentIdentityProvider performs authentication for machine and unit agents.
+// AgentAuthenticator performs authentication for machine and unit agents.
 type AgentAuthenticator struct{}
 
 var _ EntityAuthenticator = (*AgentAuthenticator)(nil)

diff --git a/apiserver/facades/agent/machine/machiner.go b/apiserver/facades/agent/machine/machiner.go
@@ -159,7 +159,31 @@ func (api *MachinerAPI) RecordAgentStartTime(args params.Entities) (params.Error
 			results.Results[i].Error = apiservererrors.ServerError(err)
 			continue
 		}
-		if err := m.RecordAgentStartTime(); err != nil {
+		if err := m.RecordAgentStartInformation(""); err != nil {
+			results.Results[i].Error = apiservererrors.ServerError(err)
+		}
+	}
+	return results, nil
+}
+
+// RecordAgentStartInformation syncs the machine model with information
+// reported by a machine agent when it starts.
+func (api *MachinerAPI) RecordAgentStartInformation(args params.RecordAgentStartInformationArgs) (params.ErrorResults, error) {
+	results := params.ErrorResults{
+		Results: make([]params.ErrorResult, len(args.Args)),
+	}
+	canModify, err := api.getCanModify()
+	if err != nil {
+		return results, err
+	}
+
+	for i, arg := range args.Args {
+		m, err := api.getMachine(arg.Tag, canModify)
+		if err != nil {
+			results.Results[i].Error = apiservererrors.ServerError(err)
+			continue
+		}
+		if err := m.RecordAgentStartInformation(arg.Hostname); err != nil {
 			results.Results[i].Error = apiservererrors.ServerError(err)
 		}
 	}
@@ -179,6 +203,9 @@ type MachinerAPIV1 struct {
 	*MachinerAPIV2
 }
 
+// RecordAgentStartTime is not available in V1.
+func (api *MachinerAPIV1) RecordAgentStartTime(_, _ struct{}) {}
+
 // MachinerAPIV2 implements the V2 API used by the machiner worker.
 // It adds RecordAgentStartTime and back-fills the missing origin in
 // NetworkConfig.
@@ -187,8 +214,14 @@ type MachinerAPIV2 struct {
 }
 
 // MachinerAPIV3 implements the V3 API used by the machiner worker.
-// It removes SetProviderNetworkConfig.
+// It Relies on agent-set origin in SetObservedNetworkConfig.
 type MachinerAPIV3 struct {
+	*MachinerAPIV4
+}
+
+// MachinerAPIV4 implements the V4 API used by the machiner worker.
+// It removes SetProviderNetworkConfig.
+type MachinerAPIV4 struct {
 	*MachinerAPI
 }
 
@@ -228,7 +261,7 @@ func (api *MachinerAPIV2) SetObservedNetworkConfig(args params.SetMachineNetwork
 func NewMachinerAPIV3(
 	ctx facade.Context,
 ) (*MachinerAPIV3, error) {
-	api, err := NewMachinerAPI(ctx)
+	api, err := NewMachinerAPIV4(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -247,5 +280,17 @@ func (api *MachinerAPIV3) SetProviderNetworkConfig(args params.Entities) (params
 	}, nil
 }
 
-// RecordAgentStartTime is not available in V1.
-func (api *MachinerAPIV1) RecordAgentStartTime(_, _ struct{}) {}
+// NewMachinerAPIV4 creates a new instance of the V4 Machiner API.
+func NewMachinerAPIV4(
+	ctx facade.Context,
+) (*MachinerAPIV4, error) {
+	api, err := NewMachinerAPI(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return &MachinerAPIV4{api}, nil
+}
+
+// RecordAgentStartInformation is not available in V4.
+func (api *MachinerAPIV4) RecordAgentStartInformation(_, _ struct{}) {}
diff --git a/apiserver/facades/agent/machine/machiner_test.go b/apiserver/facades/agent/machine/machiner_test.go
@@ -291,3 +291,25 @@ func (s *machinerSuite) TestWatch(c *gc.C) {
 	wc := statetesting.NewNotifyWatcherC(c, s.State, resource.(state.NotifyWatcher))
 	wc.AssertNoChange()
 }
+
+func (s *machinerSuite) TestRecordAgentStartInformation(c *gc.C) {
+	args := params.RecordAgentStartInformationArgs{Args: []params.RecordAgentStartInformationArg{
+		{Tag: "machine-1", Hostname: "thundering-herds"},
+		{Tag: "machine-0", Hostname: "eldritch-octopii"},
+		{Tag: "machine-42", Hostname: "missing-gem"},
+	}}
+
+	result, err := s.machiner.RecordAgentStartInformation(args)
+	c.Assert(err, jc.ErrorIsNil)
+	c.Assert(result, gc.DeepEquals, params.ErrorResults{
+		Results: []params.ErrorResult{
+			{nil},
+			{apiservertesting.ErrUnauthorized},
+			{apiservertesting.ErrUnauthorized},
+		},
+	})
+
+	err = s.machine1.Refresh()
+	c.Assert(err, jc.ErrorIsNil)
+	c.Assert(s.machine1.Hostname(), gc.Equals, "thundering-herds", gc.Commentf("expected the machine hostname to be updated"))
+}
diff --git a/apiserver/facades/agent/provisioner/provisioninginfo.go b/apiserver/facades/agent/provisioner/provisioninginfo.go
@@ -11,7 +11,6 @@ import (
 	"github.com/juju/collections/set"
 	"github.com/juju/errors"
 	"github.com/juju/names/v4"
-	"github.com/juju/os/v2/series"
 
 	"github.com/juju/juju/apiserver/common/storagecommon"
 	apiservererrors "github.com/juju/juju/apiserver/errors"
@@ -20,6 +19,7 @@ import (
 	"github.com/juju/juju/core/lxdprofile"
 	"github.com/juju/juju/core/model"
 	"github.com/juju/juju/core/network"
+	"github.com/juju/juju/core/series"
 	"github.com/juju/juju/environs"
 	"github.com/juju/juju/environs/imagemetadata"
 	"github.com/juju/juju/environs/simplestreams"
@@ -675,7 +675,7 @@ func (api *ProvisionerAPI) constructImageConstraint(m *state.Machine, env enviro
 		lookup.CloudSpec = spec
 	}
 
-	return imagemetadata.NewImageConstraint(lookup), nil
+	return imagemetadata.NewImageConstraint(lookup)
 }
 
 // findImageMetadata returns all image metadata or an error fetching them.

diff --git a/apiserver/facades/client/bundle/bundle.go b/apiserver/facades/client/bundle/bundle.go
@@ -18,7 +18,6 @@ import (
 	"github.com/juju/errors"
 	"github.com/juju/loggo"
 	"github.com/juju/names/v4"
-	"github.com/juju/os/v2/series"
 	"gopkg.in/yaml.v2"
 
 	"github.com/juju/juju/apiserver/common"
@@ -31,6 +30,7 @@ import (
 	"github.com/juju/juju/core/network"
 	"github.com/juju/juju/core/network/firewall"
 	"github.com/juju/juju/core/permission"
+	"github.com/juju/juju/core/series"
 	"github.com/juju/juju/state"
 	"github.com/juju/juju/storage"
 )

diff --git a/apiserver/facades/client/client/client.go b/apiserver/facades/client/client/client.go
@@ -11,8 +11,6 @@ import (
 	"github.com/juju/errors"
 	"github.com/juju/loggo"
 	"github.com/juju/names/v4"
-	"github.com/juju/os/v2"
-	"github.com/juju/os/v2/series"
 	"github.com/juju/replicaset"
 	"github.com/juju/version"
 
@@ -29,7 +27,9 @@ import (
 	"github.com/juju/juju/core/life"
 	"github.com/juju/juju/core/multiwatcher"
 	"github.com/juju/juju/core/network"
+	"github.com/juju/juju/core/os"
 	"github.com/juju/juju/core/permission"
+	"github.com/juju/juju/core/series"
 	"github.com/juju/juju/environs"
 	"github.com/juju/juju/environs/config"
 	"github.com/juju/juju/environs/context"

diff --git a/apiserver/facades/client/client/status.go b/apiserver/facades/client/client/status.go
@@ -1011,6 +1011,7 @@ func (c *statusContext) makeMachineStatus(machine *state.Machine, appStatusInfo
 			logger.Debugf("error fetching public address: %q", err)
 		}
 		status.DNSName = addr.Value
+		status.Hostname = machine.Hostname()
 		mAddrs := machine.Addresses()
 		if len(mAddrs) == 0 {
 			logger.Debugf("no IP addresses fetched for machine %q", instid)