Allow add-k8s to work with no undercloud specified

orbital-transfer-survey · Nov 1, 2019 · a36206b · a36206b
1 parent 8b58d4d
commit a36206b
Show file tree

Hide file tree

Showing 16 changed files with 291 additions and 72 deletions.
diff --git a/caas/kubernetes/provider/bootstrap.go b/caas/kubernetes/provider/bootstrap.go
@@ -52,11 +52,13 @@ const (
 type controllerServiceSpec struct {
 	// ServiceType is required.
 	ServiceType core.ServiceType
+	// ExternalIPs is optional.
+	ExternalIPs []string
 	// Annotations is optional.
 	Annotations k8sannotations.Annotation
 }
 
-var controllerServiceSpecs = map[string]*controllerServiceSpec{
+var controllerServiceSpecs = map[string]controllerServiceSpec{
 	caas.K8sCloudAzure: {
 		ServiceType: core.ServiceTypeLoadBalancer,
 	},
@@ -81,7 +83,7 @@ var controllerServiceSpecs = map[string]*controllerServiceSpec{
 		ServiceType: core.ServiceTypeClusterIP, // TODO(caas): test and verify this.
 	},
 	caas.K8sCloudOther: {
-		ServiceType: core.ServiceTypeLoadBalancer, // Default svc spec for any other cloud is not listed above.
+		ServiceType: core.ServiceTypeClusterIP, // Default svc spec for any other cloud is not listed above.
 	},
 }
 
@@ -343,27 +345,44 @@ func (c *controllerStack) Deploy() (err error) {
 	return nil
 }
 
-func (c *controllerStack) getControllerSvcSpec(cloudType string) (*controllerServiceSpec, error) {
+func (c *controllerStack) getControllerSvcSpec(cloudType string, cfg *podcfg.BootstrapConfig) (*controllerServiceSpec, error) {
 	spec, ok := controllerServiceSpecs[cloudType]
 	if !ok {
 		logger.Debugf("fallback to default svc spec for %q", cloudType)
 		spec = controllerServiceSpecs[caas.K8sCloudOther]
 	}
-	if spec == nil || spec.ServiceType == "" {
+	if cfg != nil && cfg.ControllerServiceType != "" {
+		var err error
+		if spec.ServiceType, err = caasServiceToK8s(caas.ServiceType(cfg.ControllerServiceType)); err != nil {
+			return nil, errors.Trace(err)
+		}
+	}
+	if spec.ServiceType == "" {
 		// ServiceType is required.
 		return nil, errors.NotValidf("service type is empty for %q", cloudType)
 	}
-	return spec, nil
+	if cfg != nil {
+		spec.ExternalIPs = append([]string(nil), cfg.ControllerExternalIPs...)
+	}
+	return &spec, nil
 }
 
 func (c *controllerStack) createControllerService() error {
 	svcName := c.resourceNameService
 
 	cloudType, _, _ := cloud.SplitHostCloudRegion(c.pcfg.Bootstrap.ControllerCloud.HostCloudRegion)
-	controllerSvcSpec, err := c.getControllerSvcSpec(cloudType)
+	controllerSvcSpec, err := c.getControllerSvcSpec(cloudType, c.pcfg.Bootstrap)
 	if err != nil {
 		return errors.Trace(err)
 	}
+	externalName := ""
+	if controllerSvcSpec.ServiceType == core.ServiceTypeExternalName {
+		externalName = c.pcfg.Bootstrap.ControllerExternalName
+	}
+	loadBalancerIP := ""
+	if controllerSvcSpec.ServiceType == core.ServiceTypeLoadBalancer && len(c.pcfg.Bootstrap.ControllerExternalIPs) > 0 {
+		loadBalancerIP = c.pcfg.Bootstrap.ControllerExternalIPs[0]
+	}
 	spec := &core.Service{
 		ObjectMeta: v1.ObjectMeta{
 			Name:        svcName,
@@ -381,6 +400,9 @@ func (c *controllerStack) createControllerService() error {
 					Port:       int32(c.portAPIServer),
 				},
 			},
+			ExternalName:   externalName,
+			ExternalIPs:    controllerSvcSpec.ExternalIPs,
+			LoadBalancerIP: loadBalancerIP,
 		},
 	}
 	if controllerSvcSpec.Annotations != nil {

diff --git a/caas/kubernetes/provider/bootstrap_test.go b/caas/kubernetes/provider/bootstrap_test.go
@@ -171,10 +171,10 @@ func (s *bootstrapSuite) TestGetControllerSvcSpec(c *gc.C) {
 			ServiceType: core.ServiceTypeClusterIP,
 		},
 		"unknown-cloud": {
-			ServiceType: core.ServiceTypeLoadBalancer,
+			ServiceType: core.ServiceTypeClusterIP,
 		},
 	} {
-		spec, _ := s.controllerStackerGetter().GetControllerSvcSpec(cloudType)
+		spec, _ := s.controllerStackerGetter().GetControllerSvcSpec(cloudType, nil)
 		c.Check(spec, jc.DeepEquals, out)
 	}
 }
@@ -211,6 +211,7 @@ func (s *bootstrapSuite) TestBootstrap(c *gc.C) {
 	s.broker.GetAnnotations().Add("juju.io/is-controller", "true")
 
 	s.pcfg.Bootstrap.Timeout = 10 * time.Minute
+	s.pcfg.Bootstrap.ControllerExternalIPs = []string{"10.0.0.1"}
 
 	controllerStacker := s.controllerStackerGetter()
 	// Broker's namespace should be set to controller name now.
@@ -247,14 +248,15 @@ func (s *bootstrapSuite) TestBootstrap(c *gc.C) {
 		},
 		Spec: core.ServiceSpec{
 			Selector: map[string]string{"juju-app": "juju-controller-test"},
-			Type:     core.ServiceType("LoadBalancer"),
+			Type:     core.ServiceType("ClusterIP"),
 			Ports: []core.ServicePort{
 				{
 					Name:       "api-server",
 					TargetPort: intstr.FromInt(APIPort),
 					Port:       int32(APIPort),
 				},
 			},
+			ExternalIPs: []string{"10.0.0.1"},
 		},
 	}
 

diff --git a/caas/kubernetes/provider/cloud.go b/caas/kubernetes/provider/cloud.go
@@ -157,16 +157,19 @@ func UpdateKubeCloudWithStorage(k8sCloud *cloud.Cloud, storageParams KubeCloudSt
 	}
 	k8sCloud.HostCloudRegion = storageParams.HostCloudRegion
 
-	cloudType, region, err := cloud.SplitHostCloudRegion(k8sCloud.HostCloudRegion)
-	if err != nil {
-		// Region is optional, but cloudType is required for next step.
-		return "", ClusterQueryError{}
-	}
-	if region != "" {
-		k8sCloud.Regions = []cloud.Region{{
-			Name:     region,
-			Endpoint: k8sCloud.Endpoint,
-		}}
+	var cloudType, region string
+	if k8sCloud.HostCloudRegion != "" {
+		cloudType, region, err = cloud.SplitHostCloudRegion(k8sCloud.HostCloudRegion)
+		if err != nil {
+			// Shouldn't happen as HostCloudRegion is validated earlier.
+			return "", errors.Trace(err)
+		}
+		if region != "" {
+			k8sCloud.Regions = []cloud.Region{{
+				Name:     region,
+				Endpoint: k8sCloud.Endpoint,
+			}}
+		}
 	}
 
 	// If the user has not specified storage and cloudType is usable, check Juju's opinionated defaults.
@@ -210,8 +213,10 @@ func UpdateKubeCloudWithStorage(k8sCloud *cloud.Cloud, storageParams KubeCloudSt
 		volumeBindingMode = nonPreferredStorageErr.VolumeBindingMode
 		params = nonPreferredStorageErr.Parameters
 	} else if clusterMetadata.NominatedStorageClass != nil {
-		// no preferred storage class config but nominated storage found.
-		scName = clusterMetadata.NominatedStorageClass.Name
+		if scName == "" {
+			// no preferred storage class config but nominated storage found.
+			scName = clusterMetadata.NominatedStorageClass.Name
+		}
 	}
 	sp, existing, err := storageParams.MetadataChecker.EnsureStorageProvisioner(caas.StorageProvisioner{
 		Name:              scName,

diff --git a/caas/kubernetes/provider/export_test.go b/caas/kubernetes/provider/export_test.go
@@ -51,7 +51,7 @@ type ControllerStackerForTest interface {
 	GetAgentConfigContent(*gc.C) string
 	GetSharedSecretAndSSLKey(*gc.C) (string, string)
 	GetStorageSize() resource.Quantity
-	GetControllerSvcSpec(string) (*controllerServiceSpec, error)
+	GetControllerSvcSpec(string, *podcfg.BootstrapConfig) (*controllerServiceSpec, error)
 }
 
 func (cs *controllerStack) GetAgentConfigContent(c *gc.C) string {
@@ -70,8 +70,8 @@ func (cs *controllerStack) GetStorageSize() resource.Quantity {
 	return cs.storageSize
 }
 
-func (cs *controllerStack) GetControllerSvcSpec(cloudType string) (*controllerServiceSpec, error) {
-	return cs.getControllerSvcSpec(cloudType)
+func (cs *controllerStack) GetControllerSvcSpec(cloudType string, cfg *podcfg.BootstrapConfig) (*controllerServiceSpec, error) {
+	return cs.getControllerSvcSpec(cloudType, cfg)
 }
 
 func NewcontrollerStackForTest(

diff --git a/caas/kubernetes/provider/k8s.go b/caas/kubernetes/provider/k8s.go
@@ -1621,6 +1621,26 @@ func (k *kubernetesClient) deleteVolumeClaims(appName string, p *core.Pod) ([]st
 	return deletedClaimVolumes, nil
 }
 
+func caasServiceToK8s(in caas.ServiceType) (core.ServiceType, error) {
+	serviceType := defaultServiceType
+	if in != "" {
+		switch in {
+		case caas.ServiceCluster:
+			serviceType = core.ServiceTypeClusterIP
+		case caas.ServiceLoadBalancer:
+			serviceType = core.ServiceTypeLoadBalancer
+		case caas.ServiceExternal:
+			serviceType = core.ServiceTypeExternalName
+		case caas.ServiceOmit:
+			logger.Debugf("no service to be created because service type is %q", in)
+			return "", nil
+		default:
+			return "", errors.NotSupportedf("service type %q", in)
+		}
+	}
+	return serviceType, nil
+}
+
 func (k *kubernetesClient) configureService(
 	appName, deploymentName string,
 	containerPorts []core.ContainerPort,
@@ -1647,21 +1667,9 @@ func (k *kubernetesClient) configureService(
 		})
 	}
 
-	serviceType := defaultServiceType
-	if params.Deployment.ServiceType != "" {
-		switch params.Deployment.ServiceType {
-		case caas.ServiceCluster:
-			serviceType = core.ServiceTypeClusterIP
-		case caas.ServiceLoadBalancer:
-			serviceType = core.ServiceTypeLoadBalancer
-		case caas.ServiceExternal:
-			serviceType = core.ServiceTypeExternalName
-		case caas.ServiceOmit:
-			logger.Debugf("no service to be created because service type is %q", params.Deployment.ServiceType)
-			return nil
-		default:
-			return errors.NotSupportedf("service type %q", params.Deployment.ServiceType)
-		}
+	serviceType, err := caasServiceToK8s(params.Deployment.ServiceType)
+	if err != nil {
+		return errors.Trace(err)
 	}
 	serviceType = core.ServiceType(config.GetString(ServiceTypeConfigKey, string(serviceType)))
 	annotations, err := config.GetStringMap(serviceAnnotationsKey, nil)

diff --git a/cloudconfig/instancecfg/instancecfg.go b/cloudconfig/instancecfg/instancecfg.go
@@ -243,6 +243,15 @@ type BootstrapConfig struct {
 	// JujuDbSnapAssertions is a path to a .assert file that will be used
 	// to verify the .snap at JujuDbSnapPath
 	JujuDbSnapAssertionsPath string
+
+	// ControllerServiceType is the service type of a k8s controller.
+	ControllerServiceType string
+
+	// ControllerExternalName is the external name of a k8s controller.
+	ControllerExternalName string
+
+	// ControllerExternalIPs is the list of external ips for a k8s controller.
+	ControllerExternalIPs []string
 }
 
 // SSHHostKeys contains the SSH host keys to configure for a bootstrap host.

diff --git a/cloudconfig/podcfg/podcfg.go b/cloudconfig/podcfg/podcfg.go
@@ -57,11 +57,6 @@ type ControllerPodConfig struct {
 	// ControllerName is the controller name.
 	ControllerName string
 
-	// TODO(bootstrap): remove me.
-	// PodNonce is set at provisioning/bootstrap time and used to
-	// ensure the agent is running on the correct instance.
-	PodNonce string
-
 	// JujuVersion is the juju version.
 	JujuVersion version.Number
 
@@ -207,7 +202,7 @@ func (cfg *ControllerPodConfig) verifyBootstrapConfig() (err error) {
 		return errors.New(`
 host cloud region is missing.
 The k8s cloud definition might be stale, please try to re-import the k8s cloud using
-    juju add-k8s <cloud-name> --cluster-name <cluster-name> --local
+    juju add-k8s <cloud-name> --cluster-name <cluster-name> --client
 
 See juju help add-k8s for more information.
 `[1:])

diff --git a/cmd/juju/caas/add.go b/cmd/juju/caas/add.go
@@ -391,6 +391,11 @@ var unknownClusterErrMsg = `
 	Run add-k8s again, using --storage=<name> to specify the storage class to use.
 `[1:]
 
+var noClusterSpecifiedErrMsg = `
+	Juju needs to know what storage class to use to provision workload and operator storage.
+	Run add-k8s again, using --storage=<name> to specify the storage class to use.
+`[1:]
+
 var noRecommendedStorageErrMsg = `
 	No recommended storage configuration is defined on this cluster.
 	Run add-k8s again with --storage=<name> and Juju will use the
@@ -470,20 +475,29 @@ func (c *AddCAASCommand) Run(ctx *cmd.Context) (err error) {
 			return errors.Errorf(noRecommendedStorageErrMsg, err.(provider.NoRecommendedStorageError).StorageProvider())
 		}
 		if provider.IsUnknownClusterError(err) {
-			return errors.Errorf(unknownClusterErrMsg, err.(provider.UnknownClusterError).CloudName)
+			cloudName := err.(provider.UnknownClusterError).CloudName
+			if cloudName == "" {
+				return errors.New(noClusterSpecifiedErrMsg)
+			}
+			return errors.Errorf(unknownClusterErrMsg, cloudName)
 		}
 		return errors.Trace(err)
 	}
 
-	newCloud.HostCloudRegion, err = c.validateCloudRegion(ctx, newCloud.HostCloudRegion)
-	if err != nil {
-		return errors.Trace(err)
+	if newCloud.HostCloudRegion != "" {
+		newCloud.HostCloudRegion, err = c.validateCloudRegion(ctx, newCloud.HostCloudRegion)
+		if err != nil {
+			return errors.Trace(err)
+		}
 	}
 	// By this stage, we know if cloud name/type and/or region input is needed from the user.
 	// If we could not detect it, check what was provided.
 	if err := checkCloudRegion(c.givenHostCloudRegion, newCloud.HostCloudRegion); err != nil {
 		return errors.Trace(err)
 	}
+	if newCloud.HostCloudRegion == "" {
+		newCloud.HostCloudRegion = caas.K8sCloudOther
+	}
 	if c.Client {
 		if err := addCloudToLocal(c.cloudMetadataStore, newCloud); err != nil {
 			return errors.Trace(err)
@@ -493,10 +507,13 @@ func (c *AddCAASCommand) Run(ctx *cmd.Context) (err error) {
 		}
 	}
 
-	if clusterName == "" {
+	if clusterName == "" && newCloud.HostCloudRegion != caas.K8sCloudOther {
 		clusterName = newCloud.HostCloudRegion
 	}
-	successMsg := fmt.Sprintf("k8s substrate %q added as cloud %q%s", clusterName, c.caasName, storageMsg)
+	if clusterName != "" {
+		clusterName = fmt.Sprintf("%q ", clusterName)
+	}
+	successMsg := fmt.Sprintf("k8s substrate %sadded as cloud %q%s", clusterName, c.caasName, storageMsg)
 	var msgDisplayed bool
 	if c.Client {
 		msgDisplayed = true