Skip to content

Commit 0c385db

Browse files
axwaxw
committed
environs/cloudinit: use aria2 to download tools from API servers
Now that we are serving tools in the API server, we must consider the availability of the API server at the time a machine is provisioned. The aria2 package is an alternative to curl that is capable of failing over to additional sources and optionally downloading chunks in parallel. Note: curl can be built with support for Metalink, which would have been a more conservative option. Unfortunately the version of curl packaged in Ubuntu does not have Metalink support enabled.
1 parent 9929ccd commit 0c385db

File tree

5 files changed

+47
-50
lines changed

5 files changed

+47
-50
lines changed

‎environs/cloudinit/cloudinit.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,7 @@ func AddAptCommands(
192192
// If we're not doing an update, adding these packages is
193193
// meaningless.
194194
if addUpdateScripts {
195-
c.AddPackage("curl")
195+
c.AddPackage("aria2")
196196
c.AddPackage("cpu-checker")
197197
// TODO(axw) 2014-07-02 #1277359
198198
// Don't install bridge-utils in cloud-init;

‎environs/cloudinit/cloudinit_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ chown syslog:adm /var/log/juju
209209
bin='/var/lib/juju/tools/1\.2\.3-precise-amd64'
210210
mkdir -p \$bin
211211
echo 'Fetching tools.*
212-
curl -sSfw 'tools from %{url_effective} downloaded: HTTP %{http_code}; time %{time_total}s; size %{size_download} bytes; speed %{speed_download} bytes/s ' --retry 10 -o \$bin/tools\.tar\.gz 'http://foo\.com/tools/releases/juju1\.2\.3-precise-amd64\.tgz'
212+
aria2c --max-tries=10 --retry-wait=3 -d \$bin -o tools\.tar\.gz 'http://foo\.com/tools/releases/juju1\.2\.3-precise-amd64\.tgz'
213213
sha256sum \$bin/tools\.tar\.gz > \$bin/juju1\.2\.3-precise-amd64\.sha256
214214
grep '1234' \$bin/juju1\.2\.3-precise-amd64.sha256 \|\| \(echo "Tools checksum mismatch"; exit 1\)
215215
tar zxf \$bin/tools.tar.gz -C \$bin
@@ -262,7 +262,7 @@ rm \$bin/tools\.tar\.gz && rm \$bin/juju1\.2\.3-precise-amd64\.sha256
262262
inexactMatch: true,
263263
expectScripts: `
264264
bin='/var/lib/juju/tools/1\.2\.3-raring-amd64'
265-
curl -sSfw 'tools from %{url_effective} downloaded: HTTP %{http_code}; time %{time_total}s; size %{size_download} bytes; speed %{speed_download} bytes/s ' --retry 10 -o \$bin/tools\.tar\.gz 'http://foo\.com/tools/releases/juju1\.2\.3-raring-amd64\.tgz'
265+
aria2c --max-tries=10 --retry-wait=3 -d \$bin -o tools\.tar\.gz 'http://foo\.com/tools/releases/juju1\.2\.3-raring-amd64\.tgz'
266266
sha256sum \$bin/tools\.tar\.gz > \$bin/juju1\.2\.3-raring-amd64\.sha256
267267
grep '1234' \$bin/juju1\.2\.3-raring-amd64.sha256 \|\| \(echo "Tools checksum mismatch"; exit 1\)
268268
printf %s '{"version":"1\.2\.3-raring-amd64","url":"http://foo\.com/tools/releases/juju1\.2\.3-raring-amd64\.tgz","sha256":"1234","size":10}' > \$bin/downloaded-tools\.txt
@@ -315,7 +315,7 @@ chown syslog:adm /var/log/juju
315315
bin='/var/lib/juju/tools/1\.2\.3-quantal-amd64'
316316
mkdir -p \$bin
317317
echo 'Fetching tools.*
318-
curl -sSfw 'tools from %{url_effective} downloaded: HTTP %{http_code}; time %{time_total}s; size %{size_download} bytes; speed %{speed_download} bytes/s ' --retry 10 --insecure -o \$bin/tools\.tar\.gz 'https://state-addr\.testing\.invalid:54321/tools/1\.2\.3-quantal-amd64'
318+
aria2c --max-tries=10 --retry-wait=3 --check-certificate=false -d \$bin -o tools\.tar\.gz 'https://state-addr\.testing\.invalid:54321/tools/1\.2\.3-quantal-amd64'
319319
sha256sum \$bin/tools\.tar\.gz > \$bin/juju1\.2\.3-quantal-amd64\.sha256
320320
grep '1234' \$bin/juju1\.2\.3-quantal-amd64.sha256 \|\| \(echo "Tools checksum mismatch"; exit 1\)
321321
tar zxf \$bin/tools.tar.gz -C \$bin
@@ -404,7 +404,7 @@ start jujud-machine-2-lxc-1
404404
},
405405
inexactMatch: true,
406406
expectScripts: `
407-
curl -sSfw 'tools from %{url_effective} downloaded: HTTP %{http_code}; time %{time_total}s; size %{size_download} bytes; speed %{speed_download} bytes/s ' --retry 10 --insecure -o \$bin/tools\.tar\.gz 'https://state-addr\.testing\.invalid:54321/tools/1\.2\.3-quantal-amd64'
407+
aria2c --max-tries=10 --retry-wait=3 --check-certificate=false -d \$bin -o tools\.tar\.gz 'https://state-addr\.testing\.invalid:54321/tools/1\.2\.3-quantal-amd64'
408408
`,
409409
}, {
410410
// empty contraints.
@@ -543,7 +543,7 @@ func (*cloudinitSuite) TestCloudInit(c *gc.C) {
543543
if test.cfg.Config != nil {
544544
checkEnvConfig(c, test.cfg.Config, configKeyValues, scripts)
545545
}
546-
checkPackage(c, configKeyValues, "curl", test.cfg.EnableOSRefreshUpdate)
546+
checkPackage(c, configKeyValues, "aria2", test.cfg.EnableOSRefreshUpdate)
547547

548548
tag := names.NewMachineTag(test.cfg.MachineId).String()
549549
acfg := getAgentConfig(c, tag, scripts)

‎environs/cloudinit/cloudinit_ubuntu.go

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,10 @@ import (
2222
"github.com/juju/juju/service/upstart"
2323
)
2424

25-
const curlCommand = "curl -sSfw " +
26-
"'tools from %{url_effective} downloaded: HTTP %{http_code}; time %{time_total}s; " +
27-
"size %{size_download} bytes; speed %{speed_download} bytes/s '"
25+
//const curlCommand = "curl -sSfw " +
26+
// "'tools from %{url_effective} downloaded: HTTP %{http_code}; time %{time_total}s; " +
27+
// "size %{size_download} bytes; speed %{speed_download} bytes/s '"
28+
const aria2Command = "aria2c"
2829

2930
type ubuntuConfigure struct {
3031
mcfg *MachineConfig
@@ -156,30 +157,29 @@ func (w *ubuntuConfigure) ConfigureJuju() error {
156157
w.conf.AddBinaryFile(path.Join(w.mcfg.jujuTools(), "tools.tar.gz"), []byte(toolsData), 0644)
157158
} else {
158159
var copyCmd string
159-
curlCommand := curlCommand + " --retry 10"
160+
aria2Command := aria2Command + " --max-tries=10 --retry-wait=3"
160161
if w.mcfg.Bootstrap {
161162
if w.mcfg.DisableSSLHostnameVerification {
162-
curlCommand += " --insecure"
163+
aria2Command += " --check-certificate=false"
163164
}
164-
copyCmd = fmt.Sprintf("%s -o $bin/tools.tar.gz %s", curlCommand, shquote(w.mcfg.Tools.URL))
165+
copyCmd = fmt.Sprintf("%s -d $bin -o tools.tar.gz %s", aria2Command, shquote(w.mcfg.Tools.URL))
165166
} else {
166-
// Our CA certificates are unusable by curl (invalid subject name),
167-
// so we must use --insecure. It doesn't actually matter, because
168-
// there is no sensitive information being transmitted and we verify
169-
// the tools' hash.
170-
171-
// TODO(axw) multi-source download. For now we're
172-
// just picking the first API server address.
173-
apiHostAddrs := w.mcfg.apiHostAddrs()
174-
175-
// TODO(axw) encode env UUID in URL when EnvironTag
176-
// is guaranteed to be available in APIInfo.
177-
urlbase := fmt.Sprintf("https://%s", apiHostAddrs[0])
167+
var urls []string
168+
for _, addr := range w.mcfg.apiHostAddrs() {
169+
// TODO(axw) encode env UUID in URL when EnvironTag
170+
// is guaranteed to be available in APIInfo.
171+
url := fmt.Sprintf("https://%s/tools/%s", addr, w.mcfg.Tools.Version)
172+
urls = append(urls, shquote(url))
173+
}
178174

175+
// Our certificates are unusable by aria2c (invalid subject name),
176+
// so we must disable certificate validation. It doesn't actually
177+
// matter, because there is no sensitive information being transmitted
178+
// and we verify the tools' hash after.
179179
copyCmd = fmt.Sprintf(
180-
"%s --insecure -o $bin/tools.tar.gz %s",
181-
curlCommand,
182-
shquote(fmt.Sprintf("%s/tools/%s", urlbase, w.mcfg.Tools.Version)),
180+
"%s --check-certificate=false -d $bin -o tools.tar.gz %s",
181+
aria2Command,
182+
strings.Join(urls, " "),
183183
)
184184
}
185185
w.conf.AddRunCmd(cloudinit.LogProgressCmd("Fetching tools: %s", copyCmd))

‎provider/ec2/local_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ func (t *localServerSuite) TestBootstrapInstanceUserDataAndState(c *gc.C) {
287287
userDataMap = nil
288288
err = goyaml.Unmarshal(userData, &userDataMap)
289289
c.Assert(err, gc.IsNil)
290-
CheckPackage(c, userDataMap, "curl", true)
290+
CheckPackage(c, userDataMap, "aria2", true)
291291
CheckPackage(c, userDataMap, "mongodb-server", false)
292292
CheckScripts(c, userDataMap, "jujud bootstrap-state", false)
293293
CheckScripts(c, userDataMap, "/var/lib/juju/agents/machine-1/agent.conf", true)

‎state/apiserver/tools_test.go

Lines changed: 19 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -210,16 +210,28 @@ func (s *toolsSuite) TestUploadSeriesExpanded(c *gc.C) {
210210
}
211211
}
212212

213-
func (s *toolsSuite) TestDownload(c *gc.C) {
213+
func (s *toolsSuite) TestDownloadEnvUUIDPath(c *gc.C) {
214214
environ, err := s.State.Environment()
215215
c.Assert(err, gc.IsNil)
216+
s.testDownload(c, environ.UUID())
217+
}
218+
219+
func (s *toolsSuite) TestDownloadTopLevelPath(c *gc.C) {
220+
s.testDownload(c, "")
221+
}
216222

223+
func (s *toolsSuite) testDownload(c *gc.C, uuid string) {
217224
stor := s.Environ.Storage()
218225
envtesting.RemoveTools(c, stor)
219226
tools := envtesting.AssertUploadFakeToolsVersions(c, stor, version.Current)[0]
220227

221-
data := s.testDownload(c, tools.Version, environ.UUID())
228+
resp, err := s.downloadRequest(c, tools.Version, uuid)
229+
c.Assert(err, gc.IsNil)
230+
defer resp.Body.Close()
231+
data, err := ioutil.ReadAll(resp.Body)
232+
c.Assert(err, gc.IsNil)
222233
c.Assert(data, gc.HasLen, int(tools.Size))
234+
223235
hash := sha256.New()
224236
hash.Write(data)
225237
c.Assert(fmt.Sprintf("%x", hash.Sum(nil)), gc.Equals, tools.SHA256)
@@ -231,16 +243,6 @@ func (s *toolsSuite) TestDownloadRejectsWrongEnvUUIDPath(c *gc.C) {
231243
s.assertErrorResponse(c, resp, http.StatusNotFound, `unknown environment: "dead-beef-123456"`)
232244
}
233245

234-
func (s *toolsSuite) TestDownloadRejectsTopLevelPath(c *gc.C) {
235-
url := s.toolsURL(c, "")
236-
url.Path = fmt.Sprintf("/tools/%s", version.Current)
237-
resp, err := s.sendRequest(c, "", "", "GET", url.String(), "", nil)
238-
if resp != nil && resp.Body != nil {
239-
resp.Body.Close()
240-
}
241-
c.Assert(err, gc.NotNil)
242-
}
243-
244246
func (s *toolsSuite) toolsURL(c *gc.C, query string) *url.URL {
245247
uri := s.baseURL(c)
246248
uri.Path += "/tools"
@@ -255,18 +257,13 @@ func (s *toolsSuite) toolsURI(c *gc.C, query string) string {
255257
return s.toolsURL(c, query).String()
256258
}
257259

258-
func (s *toolsSuite) testDownload(c *gc.C, version version.Binary, uuid string) []byte {
259-
resp, err := s.downloadRequest(c, version, uuid)
260-
c.Assert(err, gc.IsNil)
261-
defer resp.Body.Close()
262-
data, err := ioutil.ReadAll(resp.Body)
263-
c.Assert(err, gc.IsNil)
264-
return data
265-
}
266-
267260
func (s *toolsSuite) downloadRequest(c *gc.C, version version.Binary, uuid string) (*http.Response, error) {
268261
url := s.toolsURL(c, "")
269-
url.Path = fmt.Sprintf("/environment/%s/tools/%s", uuid, version)
262+
if uuid == "" {
263+
url.Path = fmt.Sprintf("/tools/%s", version)
264+
} else {
265+
url.Path = fmt.Sprintf("/environment/%s/tools/%s", uuid, version)
266+
}
270267
return s.sendRequest(c, "", "", "GET", url.String(), "", nil)
271268
}
272269

0 commit comments

Comments
 (0)