Skip to content

Commit

Permalink
Allow forcing a desired model version.
Browse files Browse the repository at this point in the history
At the State level, allow a force flag that ignores the agent-version
check.  Expose this with a direct client that can trigger an upgrade to
any version you want.
  • Loading branch information
jameinel committed Oct 24, 2017
1 parent d19dc5b commit c8cfee0
Show file tree
Hide file tree
Showing 10 changed files with 162 additions and 31 deletions.
2 changes: 1 addition & 1 deletion apiserver/client/backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ type Backend interface {
ModelUUID() string
RemoveUserAccess(names.UserTag, names.Tag) error
SetAnnotations(state.GlobalEntity, map[string]string) error
SetModelAgentVersion(version.Number) error
SetModelAgentVersion(version.Number, bool) error
SetModelConstraints(constraints.Value) error
Unit(string) (Unit, error)
UpdateModelConfig(map[string]interface{}, []string, ...state.ValidateConfigFunc) error
Expand Down
2 changes: 1 addition & 1 deletion apiserver/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -569,7 +569,7 @@ func (c *Client) SetModelAgentVersion(args params.SetModelAgentVersion) error {
}
}

return c.api.stateAccessor.SetModelAgentVersion(args.Version)
return c.api.stateAccessor.SetModelAgentVersion(args.Version, false)
}

// AbortCurrentUpgrade aborts and archives the current upgrade
Expand Down
2 changes: 1 addition & 1 deletion apiserver/modelmanager/modelmanager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -827,7 +827,7 @@ func (s *modelManagerStateSuite) TestCreateModelSameAgentVersion(c *gc.C) {
}

func (s *modelManagerStateSuite) TestCreateModelBadAgentVersion(c *gc.C) {
err := s.BackingState.SetModelAgentVersion(coretesting.FakeVersionNumber)
err := s.BackingState.SetModelAgentVersion(coretesting.FakeVersionNumber, false)
c.Assert(err, jc.ErrorIsNil)

admin := s.AdminUserTag(c)
Expand Down
2 changes: 1 addition & 1 deletion apiserver/upgrader/upgrader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ func (s *upgraderSuite) bumpDesiredAgentVersion(c *gc.C) version.Number {
s.rawMachine.SetAgentVersion(current)
newer := current
newer.Patch++
err := s.State.SetModelAgentVersion(newer.Number)
err := s.State.SetModelAgentVersion(newer.Number, false)
c.Assert(err, jc.ErrorIsNil)
cfg, err := s.State.ModelConfig()
c.Assert(err, jc.ErrorIsNil)
Expand Down
2 changes: 1 addition & 1 deletion cmd/jujud/agent/machine_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -523,7 +523,7 @@ func (s *MachineSuite) testUpgradeRequest(c *gc.C, agent runner, tag string, cur
newVers.Patch++
newTools := envtesting.AssertUploadFakeToolsVersions(
c, s.DefaultToolsStorage, s.Environ.Config().AgentStream(), s.Environ.Config().AgentStream(), newVers)[0]
err := s.State.SetModelAgentVersion(newVers.Number)
err := s.State.SetModelAgentVersion(newVers.Number, true)
c.Assert(err, jc.ErrorIsNil)
err = runWithTimeout(agent)
envtesting.CheckUpgraderReadyError(c, err, &upgrader.UpgradeReadyError{
Expand Down
111 changes: 111 additions & 0 deletions scripts/force-upgrade/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package main

import (
"fmt"
"os"
"path/filepath"

"github.com/juju/errors"
"github.com/juju/gnuflag"
"github.com/juju/loggo"
"github.com/juju/utils/clock"
"github.com/juju/version"
"gopkg.in/juju/names.v2"

"github.com/juju/juju/agent"
"github.com/juju/juju/mongo"
"github.com/juju/juju/state"
jversion "github.com/juju/juju/version"
)

var logger = loggo.GetLogger("forceagentversion")

func checkErr(label string, err error) {
if err != nil {
logger.Errorf("%s: %s", label, err)
os.Exit(1)
}
}

const dataDir = "/var/lib/juju"

func getState() (*state.State, error) {
tag, err := getCurrentMachineTag(dataDir)
if err != nil {
return nil, errors.Annotate(err, "finding machine tag")
}

logger.Infof("current machine tag: %s", tag)

config, err := getConfig(tag)
if err != nil {
return nil, errors.Annotate(err, "loading agent config")
}

mongoInfo, available := config.MongoInfo()
if !available {
return nil, errors.New("mongo info not available from agent config")
}
st, err := state.Open(state.OpenParams{
Clock: clock.WallClock,
ControllerTag: config.Controller(),
ControllerModelTag: config.Model(),
MongoInfo: mongoInfo,
MongoDialOpts: mongo.DefaultDialOpts(),
})
if err != nil {
return nil, errors.Annotate(err, "opening state connection")
}
return st, nil
}

func getCurrentMachineTag(datadir string) (names.MachineTag, error) {
var empty names.MachineTag
values, err := filepath.Glob(filepath.Join(datadir, "agents", "machine-*"))
if err != nil {
return empty, errors.Annotate(err, "problem globbing")
}
switch len(values) {
case 0:
return empty, errors.Errorf("no machines found")
case 1:
return names.ParseMachineTag(filepath.Base(values[0]))
default:
return empty, errors.Errorf("too many options: %v", values)
}
}

func getConfig(tag names.MachineTag) (agent.ConfigSetterWriter, error) {
path := agent.ConfigPath("/var/lib/juju", tag)
return agent.ReadConfig(path)
}

func main() {
loggo.GetLogger("").SetLogLevel(loggo.TRACE)
gnuflag.Usage = func() {
fmt.Printf("Usage: %s <model-uuid> <version>\n", os.Args[0])
os.Exit(1)
}

gnuflag.Parse(true)

args := gnuflag.Args()
if len(args) < 2 {
gnuflag.Usage()
}

modelUUID := args[0]
agentVersion := version.MustParse(args[1])
if agentVersion.Compare(jversion.Current) < 0 {
// Force the client to think it is at least as new as the desired version
jversion.Current = agentVersion
}

st, err := getState()
checkErr("getting state connection", err)
defer st.Close()

modelSt, err := st.ForModel(names.NewModelTag(modelUUID))
checkErr("open model", err)
checkErr("set model agent version", modelSt.SetModelAgentVersion(agentVersion, true))
}
8 changes: 5 additions & 3 deletions state/state.go
Original file line number Diff line number Diff line change
Expand Up @@ -650,7 +650,7 @@ func IsUpgradeInProgressError(err error) bool {
// given version, only if the model is in a stable state (all agents are
// running the current version). If this is a hosted model, newVersion
// cannot be higher than the controller version.
func (st *State) SetModelAgentVersion(newVersion version.Number) (err error) {
func (st *State) SetModelAgentVersion(newVersion version.Number, force bool) (err error) {
if newVersion.Compare(jujuversion.Current) > 0 && !st.IsController() {
return errors.Errorf("a hosted model cannot have a higher version than the server model: %s > %s",
newVersion.String(),
Expand All @@ -676,8 +676,10 @@ func (st *State) SetModelAgentVersion(newVersion version.Number) (err error) {
return nil, jujutxn.ErrNoOperations
}

if err := st.checkCanUpgrade(currentVersion, newVersion.String()); err != nil {
return nil, errors.Trace(err)
if !force {
if err := st.checkCanUpgrade(currentVersion, newVersion.String()); err != nil {
return nil, errors.Trace(err)
}
}

ops := []txn.Op{
Expand Down
60 changes: 39 additions & 21 deletions state/state_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3519,7 +3519,7 @@ func (s *StateSuite) TestIsUpgradeInProgressError(c *gc.C) {
c.Assert(state.IsUpgradeInProgressError(errors.Trace(state.UpgradeInProgressError)), jc.IsTrue)
}

func (s *StateSuite) TestSetEnvironAgentVersionErrors(c *gc.C) {
func (s *StateSuite) TestSetModelAgentVersionErrors(c *gc.C) {
// Get the agent-version set in the model.
envConfig, err := s.State.ModelConfig()
c.Assert(err, jc.ErrorIsNil)
Expand All @@ -3546,7 +3546,7 @@ func (s *StateSuite) TestSetEnvironAgentVersionErrors(c *gc.C) {
c.Assert(err, jc.ErrorIsNil)

// Verify machine0 and machine1 are reported as error.
err = s.State.SetModelAgentVersion(version.MustParse("4.5.6"))
err = s.State.SetModelAgentVersion(version.MustParse("4.5.6"), false)
expectErr := fmt.Sprintf("some agents have not upgraded to the current model version %s: machine-0, machine-1", stringVersion)
c.Assert(err, gc.ErrorMatches, expectErr)
c.Assert(err, jc.Satisfies, state.IsVersionInconsistentError)
Expand All @@ -3573,7 +3573,7 @@ func (s *StateSuite) TestSetEnvironAgentVersionErrors(c *gc.C) {

// Verify unit0 and unit1 are reported as error, along with the
// machines from before.
err = s.State.SetModelAgentVersion(version.MustParse("4.5.6"))
err = s.State.SetModelAgentVersion(version.MustParse("4.5.6"), false)
expectErr = fmt.Sprintf("some agents have not upgraded to the current model version %s: machine-0, machine-1, unit-wordpress-0, unit-wordpress-1", stringVersion)
c.Assert(err, gc.ErrorMatches, expectErr)
c.Assert(err, jc.Satisfies, state.IsVersionInconsistentError)
Expand All @@ -3587,7 +3587,7 @@ func (s *StateSuite) TestSetEnvironAgentVersionErrors(c *gc.C) {
}

// Verify only the units are reported as error.
err = s.State.SetModelAgentVersion(version.MustParse("4.5.6"))
err = s.State.SetModelAgentVersion(version.MustParse("4.5.6"), false)
expectErr = fmt.Sprintf("some agents have not upgraded to the current model version %s: unit-wordpress-0, unit-wordpress-1", stringVersion)
c.Assert(err, gc.ErrorMatches, expectErr)
c.Assert(err, jc.Satisfies, state.IsVersionInconsistentError)
Expand Down Expand Up @@ -3631,7 +3631,7 @@ func assertAgentVersion(c *gc.C, st *state.State, vers string) {
c.Assert(agentVersion.String(), gc.Equals, vers)
}

func (s *StateSuite) TestSetEnvironAgentVersionRetriesOnConfigChange(c *gc.C) {
func (s *StateSuite) TestSetModelAgentVersionRetriesOnConfigChange(c *gc.C) {
envConfig, _ := s.prepareAgentVersionTests(c, s.State)

// Set up a transaction hook to change something
Expand All @@ -3642,12 +3642,12 @@ func (s *StateSuite) TestSetEnvironAgentVersionRetriesOnConfigChange(c *gc.C) {
}).Check()

// Change the agent-version and ensure it has changed.
err := s.State.SetModelAgentVersion(version.MustParse("4.5.6"))
err := s.State.SetModelAgentVersion(version.MustParse("4.5.6"), false)
c.Assert(err, jc.ErrorIsNil)
assertAgentVersion(c, s.State, "4.5.6")
}

func (s *StateSuite) TestSetEnvironAgentVersionSucceedsWithSameVersion(c *gc.C) {
func (s *StateSuite) TestSetModelAgentVersionSucceedsWithSameVersion(c *gc.C) {
envConfig, _ := s.prepareAgentVersionTests(c, s.State)

// Set up a transaction hook to change the version
Expand All @@ -3658,12 +3658,12 @@ func (s *StateSuite) TestSetEnvironAgentVersionSucceedsWithSameVersion(c *gc.C)
}).Check()

// Change the agent-version and verify.
err := s.State.SetModelAgentVersion(version.MustParse("4.5.6"))
err := s.State.SetModelAgentVersion(version.MustParse("4.5.6"), false)
c.Assert(err, jc.ErrorIsNil)
assertAgentVersion(c, s.State, "4.5.6")
}

func (s *StateSuite) TestSetEnvironAgentVersionOnOtherEnviron(c *gc.C) {
func (s *StateSuite) TestSetModelAgentVersionOnOtherModel(c *gc.C) {
current := version.MustParseBinary("1.24.7-trusty-amd64")
s.PatchValue(&jujuversion.Current, current.Number)
s.PatchValue(&arch.HostArch, func() string { return current.Arch })
Expand All @@ -3675,26 +3675,26 @@ func (s *StateSuite) TestSetEnvironAgentVersionOnOtherEnviron(c *gc.C) {
higher := version.MustParseBinary("1.25.0-trusty-amd64")
lower := version.MustParseBinary("1.24.6-trusty-amd64")

// Set other environ version to < server environ version
err := otherSt.SetModelAgentVersion(lower.Number)
// Set other model version to < controller model version
err := otherSt.SetModelAgentVersion(lower.Number, false)
c.Assert(err, jc.ErrorIsNil)
assertAgentVersion(c, otherSt, lower.Number.String())

// Set other environ version == server environ version
err = otherSt.SetModelAgentVersion(jujuversion.Current)
// Set other model version == controller version
err = otherSt.SetModelAgentVersion(jujuversion.Current, false)
c.Assert(err, jc.ErrorIsNil)
assertAgentVersion(c, otherSt, jujuversion.Current.String())

// Set other environ version to > server environ version
err = otherSt.SetModelAgentVersion(higher.Number)
// Set other model version to > server version
err = otherSt.SetModelAgentVersion(higher.Number, false)
expected := fmt.Sprintf("a hosted model cannot have a higher version than the server model: %s > %s",
higher.Number,
jujuversion.Current,
)
c.Assert(err, gc.ErrorMatches, expected)
}

func (s *StateSuite) TestSetEnvironAgentVersionExcessiveContention(c *gc.C) {
func (s *StateSuite) TestSetModelAgentVersionExcessiveContention(c *gc.C) {
envConfig, currentVersion := s.prepareAgentVersionTests(c, s.State)

// Set a hook to change the config 3 times
Expand All @@ -3705,13 +3705,31 @@ func (s *StateSuite) TestSetEnvironAgentVersionExcessiveContention(c *gc.C) {
func() { s.changeEnviron(c, envConfig, "default-series", "3") },
}
defer state.SetBeforeHooks(c, s.State, changeFuncs...).Check()
err := s.State.SetModelAgentVersion(version.MustParse("4.5.6"))
err := s.State.SetModelAgentVersion(version.MustParse("4.5.6"), false)
c.Assert(errors.Cause(err), gc.Equals, txn.ErrExcessiveContention)
// Make sure the version remained the same.
assertAgentVersion(c, s.State, currentVersion)
}

func (s *StateSuite) TestSetModelAgentFailsIfUpgrading(c *gc.C) {
func (s *StateSuite) TestSetModelAgentVerisonMixedVersions(c *gc.C) {
_, currentVersion := s.prepareAgentVersionTests(c, s.State)
machine, err := s.State.Machine("0")
c.Assert(err, jc.ErrorIsNil)
// Force this to something old that should not match current versions
err = machine.SetAgentVersion(version.MustParseBinary("1.0.1-quantal-amd64"))
c.Assert(err, jc.ErrorIsNil)
// This should be refused because an agent doesn't match "currentVersion"
err = s.State.SetModelAgentVersion(version.MustParse("4.5.6"), false)
c.Check(err, gc.ErrorMatches, "some agents have not upgraded to the current model version .*: machine-0")
// Version hasn't changed
assertAgentVersion(c, s.State, currentVersion)
// But we can force it
err = s.State.SetModelAgentVersion(version.MustParse("4.5.6"), true)
c.Assert(err, jc.ErrorIsNil)
assertAgentVersion(c, s.State, "4.5.6")
}

func (s *StateSuite) TestSetModelAgentVersionFailsIfUpgrading(c *gc.C) {
// Get the agent-version set in the model.
modelConfig, err := s.State.ModelConfig()
c.Assert(err, jc.ErrorIsNil)
Expand All @@ -3732,11 +3750,11 @@ func (s *StateSuite) TestSetModelAgentFailsIfUpgrading(c *gc.C) {
_, err = s.State.EnsureUpgradeInfo(machine.Tag().Id(), agentVersion, nextVersion)
c.Assert(err, jc.ErrorIsNil)

err = s.State.SetModelAgentVersion(nextVersion)
err = s.State.SetModelAgentVersion(nextVersion, false)
c.Assert(err, jc.Satisfies, state.IsUpgradeInProgressError)
}

func (s *StateSuite) TestSetEnvironAgentFailsReportsCorrectError(c *gc.C) {
func (s *StateSuite) TestSetModelAgentVersionFailsReportsCorrectError(c *gc.C) {
// Ensure that the correct error is reported if an upgrade is
// progress but that isn't the reason for the
// SetModelAgentVersion call failing.
Expand All @@ -3761,7 +3779,7 @@ func (s *StateSuite) TestSetEnvironAgentFailsReportsCorrectError(c *gc.C) {
_, err = s.State.EnsureUpgradeInfo(machine.Tag().Id(), agentVersion, nextVersion)
c.Assert(err, jc.ErrorIsNil)

err = s.State.SetModelAgentVersion(nextVersion)
err = s.State.SetModelAgentVersion(nextVersion, false)
c.Assert(err, gc.ErrorMatches, "some agents have not upgraded to the current model version.+")
}

Expand Down
2 changes: 1 addition & 1 deletion worker/upgradesteps/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ func (w *upgradesteps) prepareForUpgrade() (*state.UpgradeInfo, error) {
if w.isMaster {
logger.Errorf("downgrading model agent version to %v due to aborted upgrade",
w.fromVersion)
if rollbackErr := w.st.SetModelAgentVersion(w.fromVersion); rollbackErr != nil {
if rollbackErr := w.st.SetModelAgentVersion(w.fromVersion, true); rollbackErr != nil {
logger.Errorf("rollback failed: %v", rollbackErr)
return nil, errors.Annotate(rollbackErr, "failed to roll back desired agent version")
}
Expand Down
2 changes: 1 addition & 1 deletion worker/upgradesteps/worker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ func (s *UpgradeSuite) TestAbortWhenOtherControllerDoesntStartUpgrade(c *gc.C) {
// This test checks when a controller is upgrading and one of
// the other controllers doesn't signal it is ready in time.

err := s.State.SetModelAgentVersion(jujuversion.Current)
err := s.State.SetModelAgentVersion(jujuversion.Current, false)
c.Assert(err, jc.ErrorIsNil)

// The master controller in this scenario is functionally tested
Expand Down

0 comments on commit c8cfee0

Please sign in to comment.