Skip to content

Commit

Permalink
Make environs.tools.FindTools accept multiple streams
Browse files Browse the repository at this point in the history 
It will query the metadata for each stream in turn, combining the
binaries found.

Update callers that already use PreferredStreams to pass them.
  • Loading branch information
@babbageclunk
babbageclunk committed Nov 16, 2017
1 parent a865679 commit 4861280
Show file tree
Hide file tree
Showing 14 changed files with 235 additions and 123 deletions.
4 changes: 2 additions & 2 deletions apiserver/common/tools.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,9 +259,9 @@ func (f *ToolsFinder) findMatchingTools(args params.FindToolsParams) (coretools.
}
filter := toolsFilter(args)
cfg := env.Config()
stream := envtools.PreferredStreams(&args.Number, cfg.Development(), cfg.AgentStream())[0]
streams := envtools.PreferredStreams(&args.Number, cfg.Development(), cfg.AgentStream())
simplestreamsList, err := envtoolsFindTools(
env, args.MajorVersion, args.MinorVersion, stream, filter,
env, args.MajorVersion, args.MinorVersion, streams, filter,
)
if len(storageList) == 0 && err != nil {
return nil, err
Expand Down
14 changes: 7 additions & 7 deletions apiserver/common/tools_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,10 +172,10 @@ func (s *toolsSuite) TestFindTools(c *gc.C) {
SHA256: "feedface",
}}

s.PatchValue(common.EnvtoolsFindTools, func(e environs.Environ, major, minor int, stream string, filter coretools.Filter) (coretools.List, error) {
s.PatchValue(common.EnvtoolsFindTools, func(e environs.Environ, major, minor int, streams []string, filter coretools.Filter) (coretools.List, error) {
c.Assert(major, gc.Equals, 123)
c.Assert(minor, gc.Equals, 456)
c.Assert(stream, gc.Equals, "released")
c.Assert(streams, gc.DeepEquals, []string{"released"})
c.Assert(filter.Series, gc.Equals, "win81")
c.Assert(filter.Arch, gc.Equals, "alpha")
return envtoolsList, nil
Expand Down Expand Up @@ -206,7 +206,7 @@ func (s *toolsSuite) TestFindTools(c *gc.C) {
}

func (s *toolsSuite) TestFindToolsNotFound(c *gc.C) {
s.PatchValue(common.EnvtoolsFindTools, func(e environs.Environ, major, minor int, stream string, filter coretools.Filter) (list coretools.List, err error) {
s.PatchValue(common.EnvtoolsFindTools, func(e environs.Environ, major, minor int, stream []string, filter coretools.Filter) (list coretools.List, err error) {
return nil, errors.NotFoundf("tools")
})
toolsFinder := common.NewToolsFinder(stateenvirons.EnvironConfigGetter{s.State, s.IAASModel.Model}, s.State, sprintfURLGetter("%s"))
Expand Down Expand Up @@ -241,15 +241,15 @@ func (s *toolsSuite) TestFindToolsExactNotInStorage(c *gc.C) {

func (s *toolsSuite) testFindToolsExact(c *gc.C, t common.ToolsStorageGetter, inStorage bool, develVersion bool) {
var called bool
s.PatchValue(common.EnvtoolsFindTools, func(e environs.Environ, major, minor int, stream string, filter coretools.Filter) (list coretools.List, err error) {
s.PatchValue(common.EnvtoolsFindTools, func(e environs.Environ, major, minor int, stream []string, filter coretools.Filter) (list coretools.List, err error) {
called = true
c.Assert(filter.Number, gc.Equals, jujuversion.Current)
c.Assert(filter.Series, gc.Equals, series.MustHostSeries())
c.Assert(filter.Arch, gc.Equals, arch.HostArch())
if develVersion {
c.Assert(stream, gc.Equals, "devel")
c.Assert(stream, gc.DeepEquals, []string{"devel", "proposed", "released"})
} else {
c.Assert(stream, gc.Equals, "released")
c.Assert(stream, gc.DeepEquals, []string{"released"})
}
return nil, errors.NotFoundf("tools")
})
Expand All @@ -273,7 +273,7 @@ func (s *toolsSuite) testFindToolsExact(c *gc.C, t common.ToolsStorageGetter, in

func (s *toolsSuite) TestFindToolsToolsStorageError(c *gc.C) {
var called bool
s.PatchValue(common.EnvtoolsFindTools, func(e environs.Environ, major, minor int, stream string, filter coretools.Filter) (list coretools.List, err error) {
s.PatchValue(common.EnvtoolsFindTools, func(e environs.Environ, major, minor int, stream []string, filter coretools.Filter) (list coretools.List, err error) {
called = true
return nil, errors.NotFoundf("tools")
})
Expand Down
8 changes: 4 additions & 4 deletions apiserver/facades/controller/agenttools/agenttools.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func NewFacade(st *state.State, resources facade.Resources, authorizer facade.Au
func NewAgentToolsAPI(
modelGetter ModelGetter,
newEnviron func() (environs.Environ, error),
findTools func(environs.Environ, int, int, string, coretools.Filter) (coretools.List, error),
findTools func(environs.Environ, int, int, []string, coretools.Filter) (coretools.List, error),
envVersionUpdate func(*state.Model, version.Number) error,
authorizer facade.Authorizer,
) (*AgentToolsAPI, error) {
Expand All @@ -66,7 +66,7 @@ type ModelGetter interface {
}

type newEnvironFunc func() (environs.Environ, error)
type toolsFinder func(environs.Environ, int, int, string, coretools.Filter) (coretools.List, error)
type toolsFinder func(environs.Environ, int, int, []string, coretools.Filter) (coretools.List, error)
type envVersionUpdater func(*state.Model, version.Number) error

func checkToolsAvailability(newEnviron newEnvironFunc, modelCfg *config.Config, finder toolsFinder) (version.Number, error) {
Expand All @@ -84,10 +84,10 @@ func checkToolsAvailability(newEnviron newEnvironFunc, modelCfg *config.Config,
// only return patches for the passed major.minor (from major.minor.patch).
// We'll try the released stream first, then fall back to the current configured stream
// if no released tools are found.
vers, err := finder(env, currentVersion.Major, currentVersion.Minor, tools.ReleasedStream, coretools.Filter{})
vers, err := finder(env, currentVersion.Major, currentVersion.Minor, []string{tools.ReleasedStream}, coretools.Filter{})
preferredStream := tools.PreferredStreams(&currentVersion, modelCfg.Development(), modelCfg.AgentStream())[0]
if preferredStream != tools.ReleasedStream && errors.Cause(err) == coretools.ErrNoMatches {
vers, err = finder(env, currentVersion.Major, currentVersion.Minor, preferredStream, coretools.Filter{})
vers, err = finder(env, currentVersion.Major, currentVersion.Minor, []string{preferredStream}, coretools.Filter{})
}
if err != nil {
return version.Zero, errors.Annotatef(err, "cannot find available agent binaries")
Expand Down
18 changes: 9 additions & 9 deletions apiserver/facades/controller/agenttools/agenttools_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,14 @@ func (s *AgentToolsSuite) TestCheckTools(c *gc.C) {
var (
calledWithMajor, calledWithMinor int
)
fakeToolFinder := func(e environs.Environ, maj int, min int, stream string, filter coretools.Filter) (coretools.List, error) {
fakeToolFinder := func(e environs.Environ, maj int, min int, streams []string, filter coretools.Filter) (coretools.List, error) {
calledWithMajor = maj
calledWithMinor = min
ver := version.Binary{Number: version.Number{Major: maj, Minor: min}}
t := coretools.Tools{Version: ver, URL: "http://example.com", Size: 1}
c.Assert(calledWithMajor, gc.Equals, 2)
c.Assert(calledWithMinor, gc.Equals, 5)
c.Assert(stream, gc.Equals, "released")
c.Assert(streams, gc.DeepEquals, []string{"released"})
return coretools.List{&t}, nil
}

Expand All @@ -71,13 +71,13 @@ func (s *AgentToolsSuite) TestCheckToolsNonReleasedStream(c *gc.C) {
c.Assert(err, jc.ErrorIsNil)
var (
calledWithMajor, calledWithMinor int
calledWithStreams []string
calledWithStreams [][]string
)
fakeToolFinder := func(e environs.Environ, maj int, min int, stream string, filter coretools.Filter) (coretools.List, error) {
fakeToolFinder := func(e environs.Environ, maj int, min int, streams []string, filter coretools.Filter) (coretools.List, error) {
calledWithMajor = maj
calledWithMinor = min
calledWithStreams = append(calledWithStreams, stream)
if stream == "released" {
calledWithStreams = append(calledWithStreams, streams)
if len(streams) == 1 && streams[0] == "released" {
return nil, coretools.ErrNoMatches
}
ver := version.Binary{Number: version.Number{Major: maj, Minor: min}}
Expand All @@ -88,7 +88,7 @@ func (s *AgentToolsSuite) TestCheckToolsNonReleasedStream(c *gc.C) {
}
ver, err := checkToolsAvailability(getDummyEnviron, cfg, fakeToolFinder)
c.Assert(err, jc.ErrorIsNil)
c.Assert(calledWithStreams, gc.DeepEquals, []string{"released", "proposed"})
c.Assert(calledWithStreams, gc.DeepEquals, [][]string{{"released"}, {"proposed"}})
c.Assert(ver, gc.Not(gc.Equals), version.Zero)
c.Assert(ver, gc.Equals, version.Number{Major: 2, Minor: 5, Patch: 0})
}
Expand All @@ -111,7 +111,7 @@ func (s *AgentToolsSuite) TestUpdateToolsAvailability(c *gc.C) {
}
s.PatchValue(&modelConfig, fakeModelConfig)

fakeToolFinder := func(_ environs.Environ, _ int, _ int, _ string, _ coretools.Filter) (coretools.List, error) {
fakeToolFinder := func(_ environs.Environ, _ int, _ int, _ []string, _ coretools.Filter) (coretools.List, error) {
ver := version.Binary{Number: version.Number{Major: 2, Minor: 5, Patch: 2}}
olderVer := version.Binary{Number: version.Number{Major: 2, Minor: 5, Patch: 1}}
t := coretools.Tools{Version: ver, URL: "http://example.com", Size: 1}
Expand Down Expand Up @@ -145,7 +145,7 @@ func (s *AgentToolsSuite) TestUpdateToolsAvailabilityNoMatches(c *gc.C) {
s.PatchValue(&modelConfig, fakeModelConfig)

// No new tools available.
fakeToolFinder := func(_ environs.Environ, _ int, _ int, _ string, _ coretools.Filter) (coretools.List, error) {
fakeToolFinder := func(_ environs.Environ, _ int, _ int, _ []string, _ coretools.Filter) (coretools.List, error) {
return nil, errors.NotFoundf("tools")
}

Expand Down
6 changes: 3 additions & 3 deletions cmd/jujud/bootstrap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,8 @@ func (c *BootstrapCommand) Run(_ *cmd.Context) error {
if ok && desiredVersion != jujuversion.Current {
// If we have been asked for a newer version, ensure the newer
// tools can actually be found, or else bootstrap won't complete.
stream := envtools.PreferredStreams(&desiredVersion, args.ControllerModelConfig.Development(), args.ControllerModelConfig.AgentStream())[0]
logger.Infof("newer agent binaries requested, looking for %v in stream %v", desiredVersion, stream)
streams := envtools.PreferredStreams(&desiredVersion, args.ControllerModelConfig.Development(), args.ControllerModelConfig.AgentStream())
logger.Infof("newer agent binaries requested, looking for %v in streams: %v", desiredVersion, strings.Join(streams, ","))
hostSeries, err := series.HostSeries()
if err != nil {
return errors.Trace(err)
Expand All @@ -161,7 +161,7 @@ func (c *BootstrapCommand) Run(_ *cmd.Context) error {
Arch: arch.HostArch(),
Series: hostSeries,
}
_, toolsErr := envtools.FindTools(env, -1, -1, stream, filter)
_, toolsErr := envtools.FindTools(env, -1, -1, streams, filter)
if toolsErr == nil {
logger.Infof("agent binaries are available, upgrade will occur after bootstrap")
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/plugins/juju-metadata/toolsmetadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ func (c *toolsMetadataCommand) Run(context *cmd.Context) error {
if err != nil {
return errors.Trace(err)
}
toolsList, err = envtools.FindToolsForCloud(toolsDataSources(source), simplestreams.CloudSpec{}, c.stream, -1, -1, coretools.Filter{})
toolsList, err = envtools.FindToolsForCloud(toolsDataSources(source), simplestreams.CloudSpec{}, []string{c.stream}, -1, -1, coretools.Filter{})
}
if err != nil {
return errors.Trace(err)
Expand Down
16 changes: 8 additions & 8 deletions environs/bootstrap/bootstrap_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,7 @@ func (s *bootstrapSuite) TestBootstrapLocalTools(c *gc.C) {

s.PatchValue(&jujuos.HostOS, func() jujuos.OSType { return jujuos.CentOS })
s.PatchValue(&arch.HostArch, func() string { return arch.AMD64 })
s.PatchValue(bootstrap.FindTools, func(environs.Environ, int, int, string, tools.Filter) (tools.List, error) {
s.PatchValue(bootstrap.FindTools, func(environs.Environ, int, int, []string, tools.Filter) (tools.List, error) {
return nil, errors.NotFoundf("tools")
})
env := newEnviron("foo", useDefaultKeys, nil)
Expand Down Expand Up @@ -454,7 +454,7 @@ func (s *bootstrapSuite) TestBootstrapLocalToolsMismatchingOS(c *gc.C) {

s.PatchValue(&jujuos.HostOS, func() jujuos.OSType { return jujuos.Windows })
s.PatchValue(&arch.HostArch, func() string { return arch.AMD64 })
s.PatchValue(bootstrap.FindTools, func(environs.Environ, int, int, string, tools.Filter) (tools.List, error) {
s.PatchValue(bootstrap.FindTools, func(environs.Environ, int, int, []string, tools.Filter) (tools.List, error) {
return nil, errors.NotFoundf("tools")
})
env := newEnviron("foo", useDefaultKeys, nil)
Expand All @@ -481,7 +481,7 @@ func (s *bootstrapSuite) TestBootstrapLocalToolsDifferentLinuxes(c *gc.C) {

s.PatchValue(&jujuos.HostOS, func() jujuos.OSType { return jujuos.GenericLinux })
s.PatchValue(&arch.HostArch, func() string { return arch.AMD64 })
s.PatchValue(bootstrap.FindTools, func(environs.Environ, int, int, string, tools.Filter) (tools.List, error) {
s.PatchValue(bootstrap.FindTools, func(environs.Environ, int, int, []string, tools.Filter) (tools.List, error) {
return nil, errors.NotFoundf("tools")
})
env := newEnviron("foo", useDefaultKeys, nil)
Expand Down Expand Up @@ -509,7 +509,7 @@ func (s *bootstrapSuite) TestBootstrapBuildAgent(c *gc.C) {
// Patch out HostArch and FindTools to allow the test to pass on other architectures,
// such as s390.
s.PatchValue(&arch.HostArch, func() string { return arch.ARM64 })
s.PatchValue(bootstrap.FindTools, func(environs.Environ, int, int, string, tools.Filter) (tools.List, error) {
s.PatchValue(bootstrap.FindTools, func(environs.Environ, int, int, []string, tools.Filter) (tools.List, error) {
c.Fatal("should not call FindTools if BuildAgent is specified")
return nil, errors.NotFoundf("tools")
})
Expand Down Expand Up @@ -555,7 +555,7 @@ func (s *bootstrapSuite) assertBootstrapPackagedToolsAvailable(c *gc.C, clientAr
toolsArch = "amd64"
}
findToolsOk := false
s.PatchValue(bootstrap.FindTools, func(_ environs.Environ, _ int, _ int, _ string, filter tools.Filter) (tools.List, error) {
s.PatchValue(bootstrap.FindTools, func(_ environs.Environ, _ int, _ int, _ []string, filter tools.Filter) (tools.List, error) {
c.Assert(filter.Arch, gc.Equals, toolsArch)
c.Assert(filter.Series, gc.Equals, "quantal")
findToolsOk = true
Expand Down Expand Up @@ -601,7 +601,7 @@ func (s *bootstrapSuite) TestBootstrapNoToolsNonReleaseStream(c *gc.C) {
// Patch out HostArch and FindTools to allow the test to pass on other architectures,
// such as s390.
s.PatchValue(&arch.HostArch, func() string { return arch.ARM64 })
s.PatchValue(bootstrap.FindTools, func(environs.Environ, int, int, string, tools.Filter) (tools.List, error) {
s.PatchValue(bootstrap.FindTools, func(environs.Environ, int, int, []string, tools.Filter) (tools.List, error) {
return nil, errors.NotFoundf("tools")
})
env := newEnviron("foo", useDefaultKeys, map[string]interface{}{
Expand All @@ -625,7 +625,7 @@ func (s *bootstrapSuite) TestBootstrapNoToolsDevelopmentConfig(c *gc.C) {
}

s.PatchValue(&arch.HostArch, func() string { return arch.ARM64 })
s.PatchValue(bootstrap.FindTools, func(environs.Environ, int, int, string, tools.Filter) (tools.List, error) {
s.PatchValue(bootstrap.FindTools, func(environs.Environ, int, int, []string, tools.Filter) (tools.List, error) {
return nil, errors.NotFoundf("tools")
})
env := newEnviron("foo", useDefaultKeys, map[string]interface{}{
Expand Down Expand Up @@ -1260,7 +1260,7 @@ func (s *bootstrapSuite) TestAvailableToolsInvalidArch(c *gc.C) {
s.PatchValue(&arch.HostArch, func() string {
return arch.S390X
})
s.PatchValue(bootstrap.FindTools, func(environs.Environ, int, int, string, tools.Filter) (tools.List, error) {
s.PatchValue(bootstrap.FindTools, func(environs.Environ, int, int, []string, tools.Filter) (tools.List, error) {
c.Fatal("find packaged tools should not be called")
return nil, errors.New("unexpected")
})
Expand Down
2 changes: 1 addition & 1 deletion environs/bootstrap/tools.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,5 +119,5 @@ func findBootstrapTools(env environs.Environ, vers *version.Number, arch, series
filter.Number = *vers
}
streams := envtools.PreferredStreams(vers, env.Config().Development(), env.Config().AgentStream())
return findTools(env, cliVersion.Major, cliVersion.Minor, streams[0], filter)
return findTools(env, cliVersion.Major, cliVersion.Minor, streams, filter)
}
32 changes: 16 additions & 16 deletions environs/bootstrap/tools_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,12 +85,12 @@ func (s *toolsSuite) TestValidateUploadAllowed(c *gc.C) {
func (s *toolsSuite) TestFindBootstrapTools(c *gc.C) {
var called int
var filter tools.Filter
var findStream string
s.PatchValue(bootstrap.FindTools, func(_ environs.Environ, major, minor int, stream string, f tools.Filter) (tools.List, error) {
var findStreams []string
s.PatchValue(bootstrap.FindTools, func(_ environs.Environ, major, minor int, streams []string, f tools.Filter) (tools.List, error) {
called++
c.Check(major, gc.Equals, jujuversion.Current.Major)
c.Check(minor, gc.Equals, jujuversion.Current.Minor)
findStream = stream
findStreams = streams
filter = f
return nil, nil
})
Expand All @@ -105,7 +105,7 @@ func (s *toolsSuite) TestFindBootstrapTools(c *gc.C) {
series *string
dev bool
filter tools.Filter
stream string
streams []string
}
tests := []test{{
version: nil,
Expand Down Expand Up @@ -140,33 +140,33 @@ func (s *toolsSuite) TestFindBootstrapTools(c *gc.C) {
arch: &arm64,
series: nil,
filter: tools.Filter{Arch: arm64, Number: devVers},
stream: "devel",
streams: []string{"devel", "proposed", "released"},
}}

for i, test := range tests {
c.Logf("test %d: %#v", i, test)
extra := map[string]interface{}{"development": test.dev}
if test.stream != "" {
extra["agent-stream"] = test.stream
if test.streams != nil {
extra["agent-stream"] = test.streams[0]
}
env := newEnviron("foo", useDefaultKeys, extra)
bootstrap.FindBootstrapTools(env, test.version, test.arch, test.series)
c.Assert(called, gc.Equals, i+1)
c.Assert(filter, gc.Equals, test.filter)
if test.stream != "" {
c.Check(findStream, gc.Equals, test.stream)
if test.streams != nil {
c.Check(findStreams, gc.DeepEquals, test.streams)
} else {
if test.dev || jujuversion.IsDev(*test.version) {
c.Check(findStream, gc.Equals, "devel")
c.Check(findStreams, gc.DeepEquals, []string{"devel", "proposed", "released"})
} else {
c.Check(findStream, gc.Equals, "released")
c.Check(findStreams, gc.DeepEquals, []string{"released"})
}
}
}
}

func (s *toolsSuite) TestFindAvailableToolsError(c *gc.C) {
s.PatchValue(bootstrap.FindTools, func(_ environs.Environ, major, minor int, stream string, f tools.Filter) (tools.List, error) {
s.PatchValue(bootstrap.FindTools, func(_ environs.Environ, major, minor int, streams []string, f tools.Filter) (tools.List, error) {
return nil, errors.New("splat")
})
env := newEnviron("foo", useDefaultKeys, nil)
Expand All @@ -175,7 +175,7 @@ func (s *toolsSuite) TestFindAvailableToolsError(c *gc.C) {
}

func (s *toolsSuite) TestFindAvailableToolsNoUpload(c *gc.C) {
s.PatchValue(bootstrap.FindTools, func(_ environs.Environ, major, minor int, stream string, f tools.Filter) (tools.List, error) {
s.PatchValue(bootstrap.FindTools, func(_ environs.Environ, major, minor int, streams []string, f tools.Filter) (tools.List, error) {
return nil, errors.NotFoundf("tools")
})
env := newEnviron("foo", useDefaultKeys, map[string]interface{}{
Expand All @@ -195,11 +195,11 @@ func (s *toolsSuite) TestFindAvailableToolsSpecificVersion(c *gc.C) {
currentVersion.Minor = 3
s.PatchValue(&jujuversion.Current, currentVersion.Number)
var findToolsCalled int
s.PatchValue(bootstrap.FindTools, func(_ environs.Environ, major, minor int, stream string, f tools.Filter) (tools.List, error) {
s.PatchValue(bootstrap.FindTools, func(_ environs.Environ, major, minor int, streams []string, f tools.Filter) (tools.List, error) {
c.Assert(f.Number.Major, gc.Equals, 10)
c.Assert(f.Number.Minor, gc.Equals, 11)
c.Assert(f.Number.Patch, gc.Equals, 12)
c.Assert(stream, gc.Equals, "released")
c.Assert(streams, gc.DeepEquals, []string{"released"})
findToolsCalled++
return []*tools.Tools{
&tools.Tools{
Expand Down Expand Up @@ -237,7 +237,7 @@ func (s *toolsSuite) TestFindAvailableToolsCompleteNoValidate(c *gc.C) {
})
}

s.PatchValue(bootstrap.FindTools, func(_ environs.Environ, major, minor int, stream string, f tools.Filter) (tools.List, error) {
s.PatchValue(bootstrap.FindTools, func(_ environs.Environ, major, minor int, streams []string, f tools.Filter) (tools.List, error) {
return allTools, nil
})
env := newEnviron("foo", useDefaultKeys, nil)
Expand Down
Loading

0 comments on commit 4861280

Please sign in to comment.