Merge pull request juju#608 from axw/cloudinit-download-tools-apiserver

Download tools from API server We will be eliminating the use of provider storage for tools. This is a step in that direction: hiding the use of provider storage behind an HTTP API provided by the API server for fetching tools. Later we will change the backend to fetch the tools from GridFS, when that work has been implemented. Now that we are serving tools in the API server, we must consider the availability of the API server at the time a machine is provisioned. The aria2 package is an alternative to curl that is capable of failing over to additional sources and optionally downloading chunks in parallel. Note: curl can be built with support for Metalink, which would have been a more conservative option. Unfortunately the version of curl packaged in Ubuntu does not have Metalink support enabled. The local provider always has tools available locally, and user-data for lxc and kvm does not have the size limitations as in cloud providers. Thus, we can safely serialise the tools tarball into user-data in the local provider.
zhatin · Aug 28, 2014 · f8ff456 · f8ff456
2 parents 0c6ec68 + d657c30
commit f8ff456
Show file tree

Hide file tree

Showing 12 changed files with 261 additions and 39 deletions.
diff --git a/environs/cloudinit/cloudinit.go b/environs/cloudinit/cloudinit.go
@@ -192,7 +192,7 @@ func AddAptCommands(
 	// If we're not doing an update, adding these packages is
 	// meaningless.
 	if addUpdateScripts {
-		c.AddPackage("curl")
+		c.AddPackage("aria2")
 		c.AddPackage("cpu-checker")
 		// TODO(axw) 2014-07-02 #1277359
 		// Don't install bridge-utils in cloud-init;

diff --git a/environs/cloudinit/cloudinit_test.go b/environs/cloudinit/cloudinit_test.go
@@ -209,7 +209,7 @@ chown syslog:adm /var/log/juju
 bin='/var/lib/juju/tools/1\.2\.3-precise-amd64'
 mkdir -p \$bin
 echo 'Fetching tools.*
-curl -sSfw 'tools from %{url_effective} downloaded: HTTP %{http_code}; time %{time_total}s; size %{size_download} bytes; speed %{speed_download} bytes/s ' --retry 10 -o \$bin/tools\.tar\.gz 'http://foo\.com/tools/releases/juju1\.2\.3-precise-amd64\.tgz'
+aria2c --max-tries=0 --retry-wait=3 -d \$bin -o tools\.tar\.gz 'http://foo\.com/tools/releases/juju1\.2\.3-precise-amd64\.tgz'
 sha256sum \$bin/tools\.tar\.gz > \$bin/juju1\.2\.3-precise-amd64\.sha256
 grep '1234' \$bin/juju1\.2\.3-precise-amd64.sha256 \|\| \(echo "Tools checksum mismatch"; exit 1\)
 tar zxf \$bin/tools.tar.gz -C \$bin
@@ -262,7 +262,7 @@ rm \$bin/tools\.tar\.gz && rm \$bin/juju1\.2\.3-precise-amd64\.sha256
 		inexactMatch: true,
 		expectScripts: `
 bin='/var/lib/juju/tools/1\.2\.3-raring-amd64'
-curl -sSfw 'tools from %{url_effective} downloaded: HTTP %{http_code}; time %{time_total}s; size %{size_download} bytes; speed %{speed_download} bytes/s ' --retry 10 -o \$bin/tools\.tar\.gz 'http://foo\.com/tools/releases/juju1\.2\.3-raring-amd64\.tgz'
+aria2c --max-tries=0 --retry-wait=3 -d \$bin -o tools\.tar\.gz 'http://foo\.com/tools/releases/juju1\.2\.3-raring-amd64\.tgz'
 sha256sum \$bin/tools\.tar\.gz > \$bin/juju1\.2\.3-raring-amd64\.sha256
 grep '1234' \$bin/juju1\.2\.3-raring-amd64.sha256 \|\| \(echo "Tools checksum mismatch"; exit 1\)
 printf %s '{"version":"1\.2\.3-raring-amd64","url":"http://foo\.com/tools/releases/juju1\.2\.3-raring-amd64\.tgz","sha256":"1234","size":10}' > \$bin/downloaded-tools\.txt
@@ -315,7 +315,7 @@ chown syslog:adm /var/log/juju
 bin='/var/lib/juju/tools/1\.2\.3-quantal-amd64'
 mkdir -p \$bin
 echo 'Fetching tools.*
-curl -sSfw 'tools from %{url_effective} downloaded: HTTP %{http_code}; time %{time_total}s; size %{size_download} bytes; speed %{speed_download} bytes/s ' --retry 10 -o \$bin/tools\.tar\.gz 'http://foo\.com/tools/releases/juju1\.2\.3-quantal-amd64\.tgz'
+aria2c --max-tries=0 --retry-wait=3 --check-certificate=false -d \$bin -o tools\.tar\.gz 'https://state-addr\.testing\.invalid:54321/tools/1\.2\.3-quantal-amd64'
 sha256sum \$bin/tools\.tar\.gz > \$bin/juju1\.2\.3-quantal-amd64\.sha256
 grep '1234' \$bin/juju1\.2\.3-quantal-amd64.sha256 \|\| \(echo "Tools checksum mismatch"; exit 1\)
 tar zxf \$bin/tools.tar.gz -C \$bin
@@ -404,7 +404,7 @@ start jujud-machine-2-lxc-1
 		},
 		inexactMatch: true,
 		expectScripts: `
-curl -sSfw 'tools from %{url_effective} downloaded: HTTP %{http_code}; time %{time_total}s; size %{size_download} bytes; speed %{speed_download} bytes/s ' --retry 10 --insecure -o \$bin/tools\.tar\.gz 'http://foo\.com/tools/releases/juju1\.2\.3-quantal-amd64\.tgz'
+aria2c --max-tries=0 --retry-wait=3 --check-certificate=false -d \$bin -o tools\.tar\.gz 'https://state-addr\.testing\.invalid:54321/tools/1\.2\.3-quantal-amd64'
 `,
 	}, {
 		// empty contraints.
@@ -543,7 +543,7 @@ func (*cloudinitSuite) TestCloudInit(c *gc.C) {
 		if test.cfg.Config != nil {
 			checkEnvConfig(c, test.cfg.Config, configKeyValues, scripts)
 		}
-		checkPackage(c, configKeyValues, "curl", test.cfg.EnableOSRefreshUpdate)
+		checkPackage(c, configKeyValues, "aria2", test.cfg.EnableOSRefreshUpdate)
 
 		tag := names.NewMachineTag(test.cfg.MachineId).String()
 		acfg := getAgentConfig(c, tag, scripts)

diff --git a/environs/cloudinit/cloudinit_ubuntu.go b/environs/cloudinit/cloudinit_ubuntu.go
@@ -22,6 +22,8 @@ import (
 	"github.com/juju/juju/service/upstart"
 )
 
+const aria2Command = "aria2c"
+
 type ubuntuConfigure struct {
 	mcfg     *MachineConfig
 	conf     *cloudinit.Config
@@ -151,12 +153,33 @@ func (w *ubuntuConfigure) ConfigureJuju() error {
 		}
 		w.conf.AddBinaryFile(path.Join(w.mcfg.jujuTools(), "tools.tar.gz"), []byte(toolsData), 0644)
 	} else {
-		curlCommand := "curl -sSfw 'tools from %{url_effective} downloaded: HTTP %{http_code}; time %{time_total}s; size %{size_download} bytes; speed %{speed_download} bytes/s '"
-		curlCommand += " --retry 10"
-		if w.mcfg.DisableSSLHostnameVerification {
-			curlCommand += " --insecure"
+		var copyCmd string
+		// Retry indefinitely.
+		aria2Command := aria2Command + " --max-tries=0 --retry-wait=3"
+		if w.mcfg.Bootstrap {
+			if w.mcfg.DisableSSLHostnameVerification {
+				aria2Command += " --check-certificate=false"
+			}
+			copyCmd = fmt.Sprintf("%s -d $bin -o tools.tar.gz %s", aria2Command, shquote(w.mcfg.Tools.URL))
+		} else {
+			var urls []string
+			for _, addr := range w.mcfg.apiHostAddrs() {
+				// TODO(axw) encode env UUID in URL when EnvironTag
+				// is guaranteed to be available in APIInfo.
+				url := fmt.Sprintf("https://%s/tools/%s", addr, w.mcfg.Tools.Version)
+				urls = append(urls, shquote(url))
+			}
+
+			// Our certificates are unusable by aria2c (invalid subject name),
+			// so we must disable certificate validation. It doesn't actually
+			// matter, because there is no sensitive information being transmitted
+			// and we verify the tools' hash after.
+			copyCmd = fmt.Sprintf(
+				"%s --check-certificate=false -d $bin -o tools.tar.gz %s",
+				aria2Command,
+				strings.Join(urls, " "),
+			)
 		}
-		copyCmd := fmt.Sprintf("%s -o $bin/tools.tar.gz %s", curlCommand, shquote(w.mcfg.Tools.URL))
 		w.conf.AddRunCmd(cloudinit.LogProgressCmd("Fetching tools: %s", copyCmd))
 		w.conf.AddRunCmd(toolsDownloadCommandWithRetry(copyCmd))
 	}

diff --git a/provider/ec2/local_test.go b/provider/ec2/local_test.go
@@ -287,7 +287,7 @@ func (t *localServerSuite) TestBootstrapInstanceUserDataAndState(c *gc.C) {
 	userDataMap = nil
 	err = goyaml.Unmarshal(userData, &userDataMap)
 	c.Assert(err, gc.IsNil)
-	CheckPackage(c, userDataMap, "curl", true)
+	CheckPackage(c, userDataMap, "aria2", true)
 	CheckPackage(c, userDataMap, "mongodb-server", false)
 	CheckScripts(c, userDataMap, "jujud bootstrap-state", false)
 	CheckScripts(c, userDataMap, "/var/lib/juju/agents/machine-1/agent.conf", true)

diff --git a/provider/local/environ.go b/provider/local/environ.go
@@ -366,6 +366,14 @@ func (env *localEnviron) StartInstance(args environs.StartInstanceParams) (insta
 	logger.Debugf("StartInstance: %q, %s", args.MachineConfig.MachineId, series)
 	args.MachineConfig.Tools = args.Tools[0]
 
+	// The local provider's user-data size is only limited by the
+	// host's disk, so it's safe to serialise tools in cloud-config.
+	toolsPath := filepath.Join(
+		env.config.storageDir(),
+		envtools.StorageName(args.Tools[0].Version),
+	)
+	args.MachineConfig.Tools.URL = fmt.Sprintf("file://%s", toolsPath)
+
 	args.MachineConfig.MachineContainerType = env.config.container()
 	logger.Debugf("tools: %#v", args.MachineConfig.Tools)
 	if err := environs.FinishMachineConfig(args.MachineConfig, env.config.Config); err != nil {

diff --git a/provider/local/environ_test.go b/provider/local/environ_test.go
@@ -28,6 +28,7 @@ import (
 	"github.com/juju/juju/instance"
 	"github.com/juju/juju/juju/arch"
 	"github.com/juju/juju/juju/osenv"
+	jujutesting "github.com/juju/juju/juju/testing"
 	"github.com/juju/juju/mongo"
 	"github.com/juju/juju/provider/local"
 	"github.com/juju/juju/service/common"
@@ -398,3 +399,43 @@ func (s *localJujuTestSuite) TestStateServerInstances(c *gc.C) {
 	c.Assert(err, gc.IsNil)
 	c.Assert(instances, gc.DeepEquals, []instance.Id{"localhost"})
 }
+
+func (s *localJujuTestSuite) TestToolsInCloudConfigForLXC(c *gc.C) {
+	s.testToolsInCloudConfig(c, "lxc")
+}
+
+func (s *localJujuTestSuite) TestToolsInCloudConfigForKVM(c *gc.C) {
+	s.testToolsInCloudConfig(c, "kvm")
+}
+
+func (s *localJujuTestSuite) testToolsInCloudConfig(c *gc.C, containerType string) {
+	dir := c.MkDir()
+	toolsDir := filepath.Join(dir, "storage", "tools", "releases")
+	err := os.MkdirAll(toolsDir, 0755)
+	c.Assert(err, gc.IsNil)
+	config := localConfig(c, map[string]interface{}{
+		"root-dir":  dir,
+		"container": containerType,
+	})
+	ctx := coretesting.Context(c)
+	env, err := local.Provider.Prepare(ctx, config)
+	c.Assert(err, gc.IsNil)
+
+	machineId := "1"
+	stateInfo := jujutesting.FakeStateInfo(machineId)
+	apiInfo := jujutesting.FakeAPIInfo(machineId)
+	machineConfig, err := environs.NewMachineConfig(machineId, "", "", "precise", nil, stateInfo, apiInfo)
+	c.Assert(err, gc.IsNil)
+	params := environs.StartInstanceParams{
+		MachineConfig: machineConfig,
+		Tools: coretools.List{{
+			Version: version.MustParseBinary("5.4.5-precise-amd64"),
+			URL:     "whatevers",
+		}},
+	}
+
+	local.PatchCreateContainer(&s.CleanupSuite, c, "file://"+filepath.Join(toolsDir, "juju-5.4.5-precise-amd64.tgz"))
+	inst, _, _, err := env.StartInstance(params)
+	c.Assert(err, gc.IsNil)
+	c.Assert(inst.Id(), gc.Equals, instance.Id("mock"))
+}
diff --git a/provider/local/environprovider_test.go b/provider/local/environprovider_test.go
@@ -16,6 +16,7 @@ import (
 	lxctesting "github.com/juju/juju/container/lxc/testing"
 	"github.com/juju/juju/environs"
 	"github.com/juju/juju/environs/config"
+	"github.com/juju/juju/instance"
 	"github.com/juju/juju/provider"
 	"github.com/juju/juju/provider/local"
 	coretesting "github.com/juju/juju/testing"
@@ -30,6 +31,9 @@ func (s *baseProviderSuite) SetUpTest(c *gc.C) {
 	s.TestSuite.SetUpTest(c)
 	loggo.GetLogger("juju.provider.local").SetLogLevel(loggo.TRACE)
 	s.restore = local.MockAddressForInterface()
+	s.PatchValue(&local.VerifyPrerequisites, func(containerType instance.ContainerType) error {
+		return nil
+	})
 }
 
 func (s *baseProviderSuite) TearDownTest(c *gc.C) {

diff --git a/state/apiserver/apiserver.go b/state/apiserver/apiserver.go
@@ -216,7 +216,14 @@ func (srv *Server) run(lis net.Listener) {
 	// tests currently assert that errors come back as application/json and
 	// pat only does "text/plain" responses.
 	handleAll(mux, "/environment/:envuuid/tools",
-		&toolsHandler{httpHandler{state: srv.state}},
+		&toolsUploadHandler{toolsHandler{
+			httpHandler{state: srv.state},
+		}},
+	)
+	handleAll(mux, "/environment/:envuuid/tools/:version",
+		&toolsDownloadHandler{toolsHandler{
+			httpHandler{state: srv.state},
+		}},
 	)
 	handleAll(mux, "/environment/:envuuid/api", http.HandlerFunc(srv.apiHandler))
 	// For backwards compatibility we register all the old paths
@@ -231,7 +238,14 @@ func (srv *Server) run(lis net.Listener) {
 			dataDir:     srv.dataDir},
 	)
 	handleAll(mux, "/tools",
-		&toolsHandler{httpHandler{state: srv.state}},
+		&toolsUploadHandler{toolsHandler{
+			httpHandler{state: srv.state},
+		}},
+	)
+	handleAll(mux, "/tools/:version",
+		&toolsDownloadHandler{toolsHandler{
+			httpHandler{state: srv.state},
+		}},
 	)
 	handleAll(mux, "/", http.HandlerFunc(srv.apiHandler))
 	// The error from http.Serve is not interesting.

diff --git a/state/apiserver/charms.go b/state/apiserver/charms.go
@@ -166,8 +166,10 @@ func (h *charmsHandler) fileSender(filePath string) bundleContentSenderFunc {
 }
 
 // sendError sends a JSON-encoded error response.
-func (h *charmsHandler) sendError(w http.ResponseWriter, statusCode int, message string) error {
-	return h.sendJSON(w, statusCode, &params.CharmsResponse{Error: message})
+func (h *charmsHandler) sendError(w http.ResponseWriter, statusCode int, message string) {
+	if err := h.sendJSON(w, statusCode, &params.CharmsResponse{Error: message}); err != nil {
+		logger.Errorf("failed to send error: %v", err)
+	}
 }
 
 // processPost handles a charm upload POST request after authentication.

diff --git a/state/apiserver/httphandler.go b/state/apiserver/httphandler.go
@@ -18,7 +18,7 @@ import (
 
 // errorSender implementations send errors back to the caller.
 type errorSender interface {
-	sendError(w http.ResponseWriter, statusCode int, message string) error
+	sendError(w http.ResponseWriter, statusCode int, message string)
 }
 
 // httpHandler handles http requests through HTTPS in the API server.