Merge pull request juju#14636 from wallyworld/secret-migration

juju#14636 Secrets are now included in a model migration. ## Checklist - [X] Code style: imports ordered, good names, simple structure, etc - [X] Comments saying why design decisions were made - [X] Go unit tests, with comments saying what you're testing - ~[ ] [Integration tests](https://github.com/juju/juju/tree/develop/tests), with comments saying what you're testing~ - ~[ ] [doc.go](https://discourse.charmhub.io/t/readme-in-packages/451) added or updated in changed packages~ ## QA steps *Commands to run to verify that the change works.* bootstrap 2 controllers on c1, create a model and a secret and share it with another app run secret-get from the other app's unit to ensure a consumer record is created migrate the model to c2 run secret-get again to see the secret has been migrated
bnhassin · Sep 21, 2022 · bcce3af · bcce3af
2 parents bf780c5 + 548a7f0
commit bcce3af
Show file tree

Hide file tree

Showing 12 changed files with 554 additions and 27 deletions.
diff --git a/apiserver/facades/agent/secretsmanager/secrets.go b/apiserver/facades/agent/secretsmanager/secrets.go
@@ -538,7 +538,7 @@ func (s *SecretsManagerAPI) SecretsRotated(args params.SecretRotatedArgs) (param
 		willExpire := md.LatestExpireTime != nil && md.LatestExpireTime.Before(nextRotateTime)
 		forcedRotateTime := lastRotateTime.Add(coresecrets.RotateRetryDelay)
 		if willExpire {
-			logger.Warningf("secret %q rev %d will expire before next scheduled rotation", uri.String(), md.LatestExpireTime.UTC().Format(time.RFC3339))
+			logger.Warningf("secret %q rev %d will expire before next scheduled rotation", uri.String(), md.LatestRevision)
 		}
 		if willExpire && forcedRotateTime.Before(*md.LatestExpireTime) || !arg.Skip && md.LatestRevision == arg.OriginalRevision {
 			nextRotateTime = forcedRotateTime

diff --git a/apiserver/facades/client/bundle/state.go b/apiserver/facades/client/bundle/state.go
@@ -6,6 +6,7 @@ package bundle
 import (
 	"github.com/juju/charm/v9"
 	"github.com/juju/description/v3"
+
 	"github.com/juju/juju/state"
 )
 
@@ -39,6 +40,7 @@ func (m *stateShim) GetExportConfig() state.ExportConfig {
 	cfg.SkipUnitAgentBinaries = true
 	cfg.SkipInstanceData = true
 	cfg.SkipExternalControllers = true
+	cfg.SkipSecrets = true
 
 	return cfg
 }

diff --git a/go.mod b/go.mod
@@ -54,7 +54,7 @@ require (
 	github.com/juju/clock v1.0.2
 	github.com/juju/cmd/v3 v3.0.2
 	github.com/juju/collections v1.0.0
-	github.com/juju/description/v3 v3.0.1
+	github.com/juju/description/v3 v3.0.2
 	github.com/juju/errors v1.0.0
 	github.com/juju/featureflag v1.0.0
 	github.com/juju/gnuflag v1.0.0

diff --git a/go.sum b/go.sum
@@ -742,8 +742,8 @@ github.com/juju/collections v0.0.0-20200605021417-0d0ec82b7271/go.mod h1:5XgO71d
 github.com/juju/collections v0.0.0-20220203020748-febd7cad8a7a/go.mod h1:JWeZdyttIEbkR51z2S13+J+aCuHVe0F6meRy+P0YGDo=
 github.com/juju/collections v1.0.0 h1:iuWddKN/SMbjLoyOpcnAl8XyBCavZm6coeFBcXQldbw=
 github.com/juju/collections v1.0.0/go.mod h1:JWeZdyttIEbkR51z2S13+J+aCuHVe0F6meRy+P0YGDo=
-github.com/juju/description/v3 v3.0.1 h1:vFvjhSulX+DtTAVDs9FoLnCQcr8CiFD+/SwGYS4dYAg=
-github.com/juju/description/v3 v3.0.1/go.mod h1:i2ZbXp3mj6EnOgexunf9DsnsZuyOB5oT0y0Lek8vBSs=
+github.com/juju/description/v3 v3.0.2 h1:HE8qZthoYyJTefjBxzHlUVB3CYJLY9/pwDgEjFKMNjw=
+github.com/juju/description/v3 v3.0.2/go.mod h1:gPRoYLLAYVXhlR5UHLvcfC5JvQHrF6GLryozBEk0s8g=
 github.com/juju/errors v0.0.0-20150916125642-1b5e39b83d18/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
 github.com/juju/errors v0.0.0-20200330140219-3fe23663418f/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
 github.com/juju/errors v0.0.0-20210818161939-5560c4c073ff/go.mod h1:i1eL7XREII6aHpQ2gApI/v6FkVUDEBremNkcBCKYAcY=

diff --git a/secrets/provider/vault/secrets.go b/secrets/provider/vault/secrets.go
@@ -233,7 +233,7 @@ func (p vaultProvider) StoreConfig(m provider.Model, adminUser bool, owned []*se
 	modelUUID := m.UUID()
 	var policies []string
 	if adminUser {
-		// For admin users, add secrets for the model can be read.
+		// For admin users, all secrets for the model can be read.
 		rule := fmt.Sprintf(`path "%s/*" {capabilities = ["read"]}`, modelUUID)
 		policyName := fmt.Sprintf("model-%s-read", modelUUID)
 		err = sys.PutPolicyWithContext(ctx, policyName, rule)

diff --git a/state/migration_export.go b/state/migration_export.go
@@ -25,6 +25,7 @@ import (
 	"github.com/juju/juju/core/network"
 	"github.com/juju/juju/core/payloads"
 	"github.com/juju/juju/core/resources"
+	"github.com/juju/juju/core/secrets"
 	"github.com/juju/juju/feature"
 	"github.com/juju/juju/state/migrations"
 	"github.com/juju/juju/storage/poolmanager"
@@ -75,6 +76,7 @@ type ExportConfig struct {
 	SkipApplicationOffers    bool
 	SkipOfferConnections     bool
 	SkipExternalControllers  bool
+	SkipSecrets              bool
 }
 
 // ExportPartial the current model for the State optionally skipping
@@ -224,6 +226,9 @@ func (st *State) exportImpl(cfg ExportConfig) (description.Model, error) {
 	if err := export.externalControllers(); err != nil {
 		return nil, errors.Trace(err)
 	}
+	if err := export.secrets(); err != nil {
+		return nil, errors.Trace(err)
+	}
 
 	// If we are doing a partial export, it doesn't really make sense
 	// to validate the model.
@@ -1417,6 +1422,9 @@ func (s externalControllerShim) AllRemoteApplications() ([]migrations.MigrationR
 }
 
 func (e *exporter) externalControllers() error {
+	if e.cfg.SkipExternalControllers {
+		return nil
+	}
 	e.logger.Debugf("reading external controllers")
 	migration := &ExportStateMigration{
 		src: e.st,
@@ -1717,6 +1725,100 @@ func (e *exporter) operations() error {
 	return nil
 }
 
+func (e *exporter) secrets() error {
+	if e.cfg.SkipSecrets {
+		return nil
+	}
+	store := NewSecrets(e.st)
+
+	allSecrets, err := store.ListSecrets(SecretsFilter{})
+	if err != nil {
+		return errors.Trace(err)
+	}
+	e.logger.Debugf("read %d secrets", len(allSecrets))
+	allRevisions, err := store.allSecretRevisions()
+	if err != nil {
+		return errors.Trace(err)
+	}
+	revisionArgsByID := make(map[string][]description.SecretRevisionArgs)
+	for _, rev := range allRevisions {
+		id, _ := splitSecretRevision(e.st.localID(rev.DocID))
+		revArg := description.SecretRevisionArgs{
+			Number:     rev.Revision,
+			Created:    rev.CreateTime,
+			Updated:    rev.UpdateTime,
+			ExpireTime: rev.ExpireTime,
+			ProviderId: rev.ProviderId,
+		}
+		if len(rev.Data) > 0 {
+			revArg.Content = make(secrets.SecretData)
+			for k, v := range rev.Data {
+				revArg.Content[k] = fmt.Sprintf("%v", v)
+			}
+		}
+		revisionArgsByID[id] = append(revisionArgsByID[id], revArg)
+	}
+	allPermissions, err := store.allSecretPermissions()
+	if err != nil {
+		return errors.Trace(err)
+	}
+	accessArgsByID := make(map[string]map[string]description.SecretAccessArgs)
+	for _, perm := range allPermissions {
+		id := strings.Split(e.st.localID(perm.DocID), "#")[0]
+		accessArg := description.SecretAccessArgs{
+			Scope: perm.Scope,
+			Role:  perm.Role,
+		}
+		access, ok := accessArgsByID[id]
+		if !ok {
+			access = make(map[string]description.SecretAccessArgs)
+			accessArgsByID[id] = access
+		}
+		access[perm.Subject] = accessArg
+	}
+	allConsumers, err := store.allSecretConsumers()
+	if err != nil {
+		return errors.Trace(err)
+	}
+	consumersByID := make(map[string][]description.SecretConsumerArgs)
+	for _, info := range allConsumers {
+		consumer, err := names.ParseTag(info.ConsumerTag)
+		if err != nil {
+			return errors.Trace(err)
+		}
+		id := strings.Split(e.st.localID(info.DocID), "#")[0]
+		consumerArg := description.SecretConsumerArgs{
+			Consumer:        consumer,
+			Label:           info.Label,
+			CurrentRevision: info.CurrentRevision,
+		}
+		consumersByID[id] = append(consumersByID[id], consumerArg)
+	}
+
+	for _, md := range allSecrets {
+		owner, err := names.ParseTag(md.OwnerTag)
+		if err != nil {
+			return errors.Trace(err)
+		}
+		arg := description.SecretArgs{
+			ID:             md.URI.ID,
+			Version:        md.Version,
+			Description:    md.Description,
+			Label:          md.Label,
+			RotatePolicy:   md.RotatePolicy.String(),
+			Owner:          owner,
+			Created:        md.CreateTime,
+			Updated:        md.UpdateTime,
+			NextRotateTime: md.NextRotateTime,
+			Revisions:      revisionArgsByID[md.URI.ID],
+			ACL:            accessArgsByID[md.URI.ID],
+			Consumers:      consumersByID[md.URI.ID],
+		}
+		e.model.AddSecret(arg)
+	}
+	return nil
+}
+
 func (e *exporter) readAllRelationScopes() (set.Strings, error) {
 	relationScopes, closer := e.st.db().GetCollection(relationScopesC)
 	defer closer()

diff --git a/state/migration_export_test.go b/state/migration_export_test.go
@@ -34,6 +34,7 @@ import (
 	"github.com/juju/juju/core/permission"
 	"github.com/juju/juju/core/resources"
 	resourcetesting "github.com/juju/juju/core/resources/testing"
+	"github.com/juju/juju/core/secrets"
 	coreseries "github.com/juju/juju/core/series"
 	"github.com/juju/juju/core/status"
 	"github.com/juju/juju/environs"
@@ -2666,3 +2667,79 @@ func (s *MigrationExportSuite) TestRemoteRelationSettingsForLocalUnitInCMR(c *gc
 		}
 	}
 }
+
+func (s *MigrationExportSuite) TestSecrets(c *gc.C) {
+	store := state.NewSecrets(s.State)
+	owner := s.Factory.MakeApplication(c, nil)
+	uri := secrets.NewURI()
+	createTime := time.Now().UTC().Round(time.Second)
+	next := createTime.Add(time.Minute).Round(time.Second).UTC()
+	expire := createTime.Add(2 * time.Hour).Round(time.Second).UTC()
+	p := state.CreateSecretParams{
+		Version: 1,
+		Owner:   owner.Tag(),
+		UpdateSecretParams: state.UpdateSecretParams{
+			LeaderToken:    &fakeToken{},
+			RotatePolicy:   ptr(secrets.RotateDaily),
+			NextRotateTime: ptr(next),
+			Description:    ptr("my secret"),
+			Label:          ptr("foobar"),
+			ExpireTime:     ptr(expire),
+			Params:         nil,
+			Data:           map[string]string{"foo": "bar"},
+		},
+	}
+	md, err := store.CreateSecret(uri, p)
+	c.Assert(err, jc.ErrorIsNil)
+	md, err = store.UpdateSecret(md.URI, state.UpdateSecretParams{
+		LeaderToken: &fakeToken{},
+		ProviderId:  ptr("provider-id"),
+	})
+	c.Assert(err, jc.ErrorIsNil)
+	err = s.State.GrantSecretAccess(uri, state.SecretAccessParams{
+		LeaderToken: &fakeToken{},
+		Scope:       owner.Tag(),
+		Subject:     owner.Tag(),
+		Role:        secrets.RoleManage,
+	})
+	c.Assert(err, jc.ErrorIsNil)
+
+	consumer := s.Factory.MakeApplication(c, &factory.ApplicationParams{
+		Charm: s.Factory.MakeCharm(c, &factory.CharmParams{
+			Name: "wordpress",
+		}),
+	})
+	err = s.State.SaveSecretConsumer(uri, consumer.Tag(), &secrets.SecretConsumerMetadata{
+		Label:           "consumer label",
+		CurrentRevision: 666,
+	})
+	c.Assert(err, jc.ErrorIsNil)
+
+	model, err := s.State.Export()
+	c.Assert(err, jc.ErrorIsNil)
+
+	allSecrets := model.Secrets()
+	c.Assert(allSecrets, gc.HasLen, 1)
+	secret := allSecrets[0]
+	c.Assert(secret.Id(), gc.Equals, uri.ID)
+	c.Assert(secret.Description(), gc.Equals, "my secret")
+	c.Assert(secret.NextRotateTime(), jc.DeepEquals, ptr(next))
+	entity, err := secret.Owner()
+	c.Assert(err, jc.ErrorIsNil)
+	c.Assert(entity.Id(), gc.Equals, "mysql")
+	access, ok := secret.ACL()["application-mysql"]
+	c.Assert(ok, jc.IsTrue)
+	c.Assert(access.Role(), gc.Equals, "manage")
+	revisions := secret.Revisions()
+	c.Assert(revisions, gc.HasLen, 2)
+	c.Assert(revisions[0].Content(), jc.DeepEquals, map[string]string{"foo": "bar"})
+	c.Assert(revisions[0].ExpireTime(), jc.DeepEquals, ptr(expire))
+	c.Assert(revisions[1].ProviderId(), jc.DeepEquals, ptr("provider-id"))
+	consumers := secret.Consumers()
+	c.Assert(consumers, gc.HasLen, 1)
+	info := consumers[0]
+	entity, err = info.Consumer()
+	c.Assert(err, jc.ErrorIsNil)
+	c.Assert(entity.Id(), gc.Equals, "wordpress")
+	c.Assert(info.CurrentRevision(), gc.Equals, 666)
+}
diff --git a/state/migration_import.go b/state/migration_import.go
@@ -217,6 +217,9 @@ func (ctrl *Controller) Import(model description.Model) (_ *Model, _ *State, err
 	if err := restore.storage(); err != nil {
 		return nil, nil, errors.Annotate(err, "storage")
 	}
+	if err := restore.secrets(); err != nil {
+		return nil, nil, errors.Annotate(err, "secrets")
+	}
 
 	// NOTE: at the end of the import make sure that the mode of the model
 	// is set to "imported" not "active" (or whatever we call it). This way
@@ -1657,7 +1660,7 @@ func (i *importer) firewallRules() error {
 	if err := migration.Run(); err != nil {
 		return errors.Trace(err)
 	}
-	i.logger.Debugf("importing remote applications succeeded")
+	i.logger.Debugf("importing firewall rules succeeded")
 	return nil
 }
 
@@ -2753,3 +2756,23 @@ func (i *importer) storagePools() error {
 	}
 	return nil
 }
+
+func (i *importer) secrets() error {
+	i.logger.Debugf("importing secrets")
+	migration := &ImportStateMigration{
+		src: i.model,
+		dst: i.st.db(),
+	}
+	migration.Add(func() error {
+		m := ImportSecrets{}
+		return m.Execute(stateModelNamspaceShim{
+			Model: migration.src,
+			st:    i.st,
+		}, migration.dst)
+	})
+	if err := migration.Run(); err != nil {
+		return errors.Trace(err)
+	}
+	i.logger.Debugf("importing secrets succeeded")
+	return nil
+}