Update apt mirrors on all nodes when model value changes

sdg0919 · Jun 2, 2021 · 1851d85 · 1851d85
1 parent 15284a5
commit 1851d85
Show file tree

Hide file tree

Showing 19 changed files with 210 additions and 92 deletions.
diff --git a/api/proxyupdater/proxyupdater.go b/api/proxyupdater/proxyupdater.go
@@ -80,6 +80,8 @@ type ProxyConfiguration struct {
 	APTProxy    proxy.Settings
 	SnapProxy   proxy.Settings
 
+	AptMirror string
+
 	SnapStoreProxyId         string
 	SnapStoreProxyAssertions string
 	SnapStoreProxyURL        string
@@ -117,6 +119,7 @@ func (api *API) ProxyConfig() (ProxyConfiguration, error) {
 		APTProxy:    proxySettingsParamToProxySettings(result.APTProxySettings),
 		SnapProxy:   proxySettingsParamToProxySettings(result.SnapProxySettings),
 
+		AptMirror:                result.AptMirror,
 		SnapStoreProxyId:         result.SnapStoreProxyId,
 		SnapStoreProxyAssertions: result.SnapStoreProxyAssertions,
 		SnapStoreProxyURL:        result.SnapStoreProxyURL,

diff --git a/api/proxyupdater/proxyupdater_test.go b/api/proxyupdater/proxyupdater_test.go
@@ -104,6 +104,7 @@ func (s *ProxyUpdaterSuite) TestProxyConfig(c *gc.C) {
 			HTTP:  "http-snap",
 			HTTPS: "https-snap",
 		},
+		AptMirror: "http://mirror",
 	}
 
 	called, api := newAPI(c, 2, apitesting.APICall{
@@ -138,6 +139,7 @@ func (s *ProxyUpdaterSuite) TestProxyConfig(c *gc.C) {
 		Http:  "http-snap",
 		Https: "https-snap",
 	})
+	c.Check(config.AptMirror, gc.Equals, "http://mirror")
 }
 
 func (s *ProxyUpdaterSuite) TestProxyConfigV1(c *gc.C) {

diff --git a/apiserver/facades/agent/proxyupdater/proxyupdater.go b/apiserver/facades/agent/proxyupdater/proxyupdater.go
@@ -129,7 +129,7 @@ func (api *APIBase) oneWatch() params.NotifyWatchResult {
 	return result
 }
 
-// WatchForProxyConfigAndAPIHostPortChanges watches for cleanups to be perfomed in state
+// WatchForProxyConfigAndAPIHostPortChanges watches for changes to the proxy and api host port settings.
 func (api *APIBase) WatchForProxyConfigAndAPIHostPortChanges(args params.Entities) params.NotifyWatchResults {
 	results := params.NotifyWatchResults{
 		Results: make([]params.NotifyWatchResult, len(args.Entities)),
@@ -205,6 +205,7 @@ func (api *APIBase) proxyConfig() params.ProxyConfigResult {
 	result.LegacyProxySettings = toParams(legacyProxySettings)
 
 	result.APTProxySettings = toParams(config.AptProxySettings())
+	result.AptMirror = config.AptMirror()
 
 	result.SnapProxySettings = toParams(config.SnapProxySettings())
 	result.SnapStoreProxyId = config.SnapStoreProxy()

diff --git a/apiserver/facades/agent/proxyupdater/proxyupdater_test.go b/apiserver/facades/agent/proxyupdater/proxyupdater_test.go
@@ -114,6 +114,21 @@ func (s *ProxyUpdaterSuite) TestProxyConfigV1(c *gc.C) {
 	}
 	c.Assert(cfg.Results[0], jc.DeepEquals, r)
 }
+func (s *ProxyUpdaterSuite) TestMirrorConfig(c *gc.C) {
+	s.state.SetModelConfig(coretesting.Attrs{
+		"apt-mirror": "http://mirror",
+	})
+	// Check that the ProxyConfig combines data from ModelConfig and APIHostPorts
+	cfg := s.facade.ProxyConfig(s.oneEntity())
+
+	s.state.Stub.CheckCallNames(c,
+		"ModelConfig",
+		"APIHostPortsForAgents",
+	)
+
+	c.Assert(cfg.Results, gc.HasLen, 1)
+	c.Assert(cfg.Results[0].AptMirror, gc.Equals, "http://mirror")
+}
 
 func (s *ProxyUpdaterSuite) TestProxyConfig(c *gc.C) {
 	// Check that the ProxyConfig combines data from ModelConfig and APIHostPorts

diff --git a/apiserver/facades/schema.json b/apiserver/facades/schema.json
@@ -33447,7 +33447,6 @@
                         "snap-store-assertions",
                         "snap-store-proxy-id",
                         "snap-store-proxy-url",
-                        "apt-mirror",
                         "UpdateBehavior"
                     ]
                 },
@@ -35211,7 +35210,7 @@
                             "$ref": "#/definitions/NotifyWatchResults"
                         }
                     },
-                    "description": "WatchForProxyConfigAndAPIHostPortChanges watches for cleanups to be perfomed in state"
+                    "description": "WatchForProxyConfigAndAPIHostPortChanges watches for changes to the proxy and api host port settings."
                 }
             },
             "definitions": {
@@ -35324,6 +35323,9 @@
                 "ProxyConfigResult": {
                     "type": "object",
                     "properties": {
+                        "apt-mirror": {
+                            "type": "string"
+                        },
                         "apt-proxy-settings": {
                             "$ref": "#/definitions/ProxyConfig"
                         },

diff --git a/apiserver/params/network.go b/apiserver/params/network.go
@@ -1175,6 +1175,7 @@ type ProxyConfigResult struct {
 	SnapStoreProxyId         string      `json:"snap-store-id,omitempty"`
 	SnapStoreProxyAssertions string      `json:"snap-store-assertions,omitempty"`
 	SnapStoreProxyURL        string      `json:"snap-store-proxy-url,omitempty"`
+	AptMirror                string      `json:"apt-mirror,omitempty"`
 	Error                    *Error      `json:"error,omitempty"`
 }
 

diff --git a/apiserver/params/params.go b/apiserver/params/params.go
@@ -767,7 +767,7 @@ type ContainerConfig struct {
 	SnapStoreAssertions        string                 `json:"snap-store-assertions"`
 	SnapStoreProxyID           string                 `json:"snap-store-proxy-id"`
 	SnapStoreProxyURL          string                 `json:"snap-store-proxy-url"`
-	AptMirror                  string                 `json:"apt-mirror"`
+	AptMirror                  string                 `json:"apt-mirror,omitempty"`
 	CloudInitUserData          map[string]interface{} `json:"cloudinit-userdata,omitempty"`
 	ContainerInheritProperties string                 `json:"container-inherit-properties,omitempty"`
 	*UpdateBehavior

diff --git a/cloudconfig/cloudinit/cloudinit_ubuntu.go b/cloudconfig/cloudinit/cloudinit_ubuntu.go
@@ -38,8 +38,8 @@ func (cfg *ubuntuCloudConfig) UnsetPackageProxy() {
 
 // PackageProxy is defined on the PackageProxyConfig interface.
 func (cfg *ubuntuCloudConfig) PackageProxy() string {
-	proxy, _ := cfg.attrs["apt_proxy"].(string)
-	return proxy
+	p, _ := cfg.attrs["apt_proxy"].(string)
+	return p
 }
 
 // SetPackageMirror is defined on the PackageMirrorConfig interface.
@@ -149,17 +149,14 @@ func (cfg *ubuntuCloudConfig) getCommandsForAddingPackages() ([]string, error) {
 	}
 
 	var cmds []string
+	pkgCmder := cfg.paccmder[jujupackaging.AptPackageManager]
 
 	// If a mirror is specified, rewrite sources.list and rename cached index files.
 	if newMirror := cfg.PackageMirror(); newMirror != "" {
-		cmds = append(cmds, LogProgressCmd("Changing apt mirror to "+newMirror))
-		cmds = append(cmds, "old_mirror=$("+config.ExtractAptSource+")")
-		cmds = append(cmds, "new_mirror="+newMirror)
-		cmds = append(cmds, `sed -i s,$old_mirror,$new_mirror, `+config.AptSourcesFile)
-		cmds = append(cmds, renameAptListFilesCommands("$new_mirror", "$old_mirror")...)
+		cmds = append(cmds, LogProgressCmd(fmt.Sprintf("Changing apt mirror to %q", newMirror)))
+		cmds = append(cmds, pkgCmder.SetMirrorCommands(newMirror, newMirror)...)
 	}
 
-	pkgCmder := cfg.paccmder[jujupackaging.AptPackageManager]
 	if len(cfg.PackageSources()) > 0 {
 		// Ensure add-apt-repository is available.
 		cmds = append(cmds, LogProgressCmd("Installing add-apt-repository"))
@@ -243,26 +240,6 @@ func (cfg *ubuntuCloudConfig) getCommandsForAddingPackages() ([]string, error) {
 
 }
 
-// renameAptListFilesCommands takes a new and old mirror string,
-// and returns a sequence of commands that will rename the files
-// in aptListsDirectory.
-func renameAptListFilesCommands(newMirror, oldMirror string) []string {
-	oldPrefix := "old_prefix=" + config.AptListsDirectory + "/$(echo " + oldMirror + " | " + config.AptSourceListPrefix + ")"
-	newPrefix := "new_prefix=" + config.AptListsDirectory + "/$(echo " + newMirror + " | " + config.AptSourceListPrefix + ")"
-	renameFiles := `
-for old in ${old_prefix}_*; do
-    new=$(echo $old | sed s,^$old_prefix,$new_prefix,)
-    mv $old $new
-done`
-
-	return []string{
-		oldPrefix,
-		newPrefix,
-		// Don't do anything unless the mirror/source has changed.
-		`[ "$old_prefix" != "$new_prefix" ] &&` + renameFiles,
-	}
-}
-
 // addRequiredPackages is defined on the AdvancedPackagingConfig interface.
 func (cfg *ubuntuCloudConfig) addRequiredPackages() {
 	packages := []string{

diff --git a/cloudconfig/cloudinit/interface.go b/cloudconfig/cloudinit/interface.go
@@ -124,7 +124,7 @@ type PackageMirrorConfig interface {
 	PackageMirror() string
 }
 
-// PackageSourceConfig is the interface for package source settings on a cloudconfig.
+// PackageSourcesConfig is the interface for package source settings on a cloudconfig.
 type PackageSourcesConfig interface {
 	// AddPackageSource adds a new repository and optional key to be
 	// used as a package source by the system's specific package manager.

diff --git a/cloudconfig/cloudinit/renderscript_test.go b/cloudconfig/cloudinit/renderscript_test.go
@@ -219,16 +219,30 @@ func (s *configureSuite) TestAptUpgrade(c *gc.C) {
 
 func (s *configureSuite) TestAptMirrorWrapper(c *gc.C) {
 	expectedCommands := regexp.QuoteMeta(`
-echo 'Changing apt mirror to http://woat.com' >&$JUJU_PROGRESS_FD
-old_mirror=$(awk "/^deb .* $(lsb_release -sc) .*main.*\$/{print \$2;exit}" /etc/apt/sources.list)
-new_mirror=http://woat.com
-sed -i s,$old_mirror,$new_mirror, /etc/apt/sources.list
-old_prefix=/var/lib/apt/lists/$(echo $old_mirror | sed 's,.*://,,' | sed 's,/$,,' | tr / _)
-new_prefix=/var/lib/apt/lists/$(echo $new_mirror | sed 's,.*://,,' | sed 's,/$,,' | tr / _)
+echo 'Changing apt mirror to "http://woat.com"' >&$JUJU_PROGRESS_FD
+old_archive_mirror=$(awk "/^deb .* $(awk -F= '/DISTRIB_CODENAME=/ {gsub(/"/,""); print $2}' /etc/lsb-release) .*main.*\$/{print \$2;exit}" /etc/apt/sources.list)
+new_archive_mirror=http://woat.com
+sed -i s,$old_archive_mirror,$new_archive_mirror, /etc/apt/sources.list
+old_prefix=/var/lib/apt/lists/$(echo $old_archive_mirror | sed 's,.*://,,' | sed 's,/$,,' | tr / _)
+new_prefix=/var/lib/apt/lists/$(echo $new_archive_mirror | sed 's,.*://,,' | sed 's,/$,,' | tr / _)
 [ "$old_prefix" != "$new_prefix" ] &&
 for old in ${old_prefix}_*; do
     new=$(echo $old | sed s,^$old_prefix,$new_prefix,)
-    mv $old $new
+    if [ -f $old ]; then
+      mv $old $new
+    fi
+done
+old_security_mirror=$(awk "/^deb .* $(awk -F= '/DISTRIB_CODENAME=/ {gsub(/"/,""); print $2}' /etc/lsb-release)-security .*main.*\$/{print \$2;exit}" /etc/apt/sources.list)
+new_security_mirror=http://woat.com
+sed -i s,$old_security_mirror,$new_security_mirror, /etc/apt/sources.list
+old_prefix=/var/lib/apt/lists/$(echo $old_security_mirror | sed 's,.*://,,' | sed 's,/$,,' | tr / _)
+new_prefix=/var/lib/apt/lists/$(echo $new_security_mirror | sed 's,.*://,,' | sed 's,/$,,' | tr / _)
+[ "$old_prefix" != "$new_prefix" ] &&
+for old in ${old_prefix}_*; do
+    new=$(echo $old | sed s,^$old_prefix,$new_prefix,)
+    if [ -f $old ]; then
+      mv $old $new
+    fi
 done`)
 	aptMirrorRegexp := "(.|\n)*" + expectedCommands + "(.|\n)*"
 	cfg, err := cloudinit.New("quantal")

diff --git a/cloudconfig/userdatacfg.go b/cloudconfig/userdatacfg.go
@@ -203,7 +203,7 @@ func shquote(p string) string {
 	return utils.ShQuote(p)
 }
 
-// packageManagerProxySettings implements cloudinit.PackageManagerConfig.
+// packageManagerProxySettings implements cloudinit.PackageManagerProxyConfig.
 type packageManagerProxySettings struct {
 	aptProxy            proxy.Settings
 	aptMirror           string
@@ -213,20 +213,20 @@ type packageManagerProxySettings struct {
 	snapStoreProxyURL   string
 }
 
-// AptProxy implements cloudinit.PackageManagerConfig.
+// AptProxy implements cloudinit.PackageManagerProxyConfig.
 func (p packageManagerProxySettings) AptProxy() proxy.Settings { return p.aptProxy }
 
 // AptMirror implements cloudinit.PackageManagerConfig.
 func (p packageManagerProxySettings) AptMirror() string { return p.aptMirror }
 
-// SnapProxy implements cloudinit.PackageManagerConfig.
+// SnapProxy implements cloudinit.PackageManagerProxyConfig.
 func (p packageManagerProxySettings) SnapProxy() proxy.Settings { return p.snapProxy }
 
-// SnapStoreAssertions implements cloudinit.PackageManagerConfig.
+// SnapStoreAssertions implements cloudinit.PackageManagerProxyConfig.
 func (p packageManagerProxySettings) SnapStoreAssertions() string { return p.snapStoreAssertions }
 
-// SnapStoreProxyID implements cloudinit.PackageManagerConfig.
+// SnapStoreProxyID implements cloudinit.PackageManagerProxyConfig.
 func (p packageManagerProxySettings) SnapStoreProxyID() string { return p.snapStoreProxyID }
 
-// SnapStoreProxyURL implements cloudinit.PackageManagerConfig.
+// SnapStoreProxyURL implements cloudinit.PackageManagerProxyConfig.
 func (p packageManagerProxySettings) SnapStoreProxyURL() string { return p.snapStoreProxyURL }
diff --git a/cmd/containeragent/unit/manifolds.go b/cmd/containeragent/unit/manifolds.go
@@ -217,6 +217,7 @@ func Manifolds(config manifoldsConfig) dependency.Manifolds {
 			WorkerFunc:          proxyupdater.NewWorker,
 			InProcessUpdate:     proxy.DefaultConfig.Set,
 			SupportLegacyValues: false,
+			RunFunc:             proxyupdater.RunWithStdIn,
 		})),
 
 		// The logging config updater is a leaf worker that indirectly

diff --git a/cmd/jujud/agent/caasoperator/manifolds.go b/cmd/jujud/agent/caasoperator/manifolds.go
@@ -221,6 +221,7 @@ func Manifolds(config ManifoldsConfig) dependency.Manifolds {
 			WorkerFunc:          proxyupdater.NewWorker,
 			InProcessUpdate:     proxy.DefaultConfig.Set,
 			SupportLegacyValues: false,
+			RunFunc:             proxyupdater.RunWithStdIn,
 		})),
 
 		// The logging config updater is a leaf worker that indirectly