Skip to content

Commit

Permalink
Merge pull request juju#13101 from hpidcock/aws-encrypted-ebs
Browse files Browse the repository at this point in the history 
juju#13101

Adds support to EC2 provider for:
- Encrypted root disks
- KMS key selection for encrypted disks via "kms-key-id" key
- Add support for GP3 and IO2 volume types
- Add support for "throughput" key on GP3 volumes

## QA steps

```
juju bootstrap aws \n --storage-pool name=ebs-encrypted \n --storage-pool type=ebs \n --storage-pool encrypted=true \n --storage-pool kms-key-id="arn:aws:kms:us-east-1:<snip>" \n --storage-pool volume-type=gp3 \n --storage-pool throughput=500M \n --bootstrap-constraints="root-disk-source=ebs-encrypted"
```

## Documentation changes

Updates required for AWS documentation in storage docs.
https://discourse.charmhub.io/t/use-juju-storage/1079

## Bug reference

https://bugs.launchpad.net/juju/+bug/1931139
  • Loading branch information
@jujubot
jujubot authored Jun 23, 2021
2 parents 4da92bf + af373fc commit 87cc371
Show file tree
Hide file tree
Showing 5 changed files with 231 additions and 40 deletions.
111 changes: 88 additions & 23 deletions provider/ec2/ebs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,17 +38,24 @@ const (
// "gp2" for General Purpose (SSD) volumes
// "io1" for Provisioned IOPS (SSD) volumes,
// "standard" for Magnetic volumes.
// see volumes types below for more.
EBS_VolumeType = "volume-type"

// EBS_IOPS is the number of I/O operations per second (IOPS) per GiB
// to provision for the volume. Only valid for Provisioned
// IOPS (SSD) volumes.
// to provision for the volume. Only valid for io1 io2 and gp3 volumes.
EBS_IOPS = "iops"

// EBS_Throughput is the max transfer troughput for gp3 volumes.
EBS_Throughput = "throughput"

// EBS_Encrypted specifies whether the volume should be encrypted.
EBS_Encrypted = "encrypted"

// EBS_KMSKeyID specifies what encryption key to use for the EBS volume.
EBS_KMSKeyID = "kms-key-id"

// Volume Aliases
// TODO(juju3): remove volume aliases and use the raw AWS names.
volumeAliasMagnetic = "magnetic" // standard
volumeAliasOptimizedHDD = "optimized-hdd" // sc1
volumeAliasColdStorage = "cold-storage" // sc1
Expand All @@ -58,7 +65,9 @@ const (
// Volume types
volumeTypeStandard = "standard"
volumeTypeGP2 = "gp2"
volumeTypeGP3 = "gp3"
volumeTypeIO1 = "io1"
volumeTypeIO2 = "io2"
volumeTypeST1 = "st1"
volumeTypeSC1 = "sc1"

Expand Down Expand Up @@ -178,27 +187,35 @@ var ebsConfigFields = schema.Fields{
schema.Const(volumeAliasProvisionedIops),
schema.Const(volumeTypeStandard),
schema.Const(volumeTypeGP2),
schema.Const(volumeTypeGP3),
schema.Const(volumeTypeIO1),
schema.Const(volumeTypeIO2),
schema.Const(volumeTypeST1),
schema.Const(volumeTypeSC1),
),
EBS_IOPS: schema.ForceInt(),
EBS_Encrypted: schema.Bool(),
EBS_IOPS: schema.ForceInt(),
EBS_Encrypted: schema.Bool(),
EBS_KMSKeyID: schema.String(),
EBS_Throughput: schema.String(),
}

var ebsConfigChecker = schema.FieldMap(
ebsConfigFields,
schema.Defaults{
EBS_VolumeType: volumeAliasSSD,
EBS_VolumeType: volumeTypeGP2,
EBS_IOPS: schema.Omit,
EBS_Encrypted: false,
EBS_KMSKeyID: schema.Omit,
EBS_Throughput: schema.Omit,
},
)

type ebsConfig struct {
volumeType string
iops int
encrypted bool
volumeType string
iops int
encrypted bool
kmsKeyID string
throughputMB int
}

func newEbsConfig(attrs map[string]interface{}) (*ebsConfig, error) {
Expand All @@ -209,10 +226,13 @@ func newEbsConfig(attrs map[string]interface{}) (*ebsConfig, error) {
coerced := out.(map[string]interface{})
iops, _ := coerced[EBS_IOPS].(int)
volumeType := coerced[EBS_VolumeType].(string)
kmsKeyID, _ := coerced[EBS_KMSKeyID].(string)
throughput, _ := coerced[EBS_Throughput].(string)
ebsConfig := &ebsConfig{
volumeType: volumeType,
iops: iops,
encrypted: coerced[EBS_Encrypted].(bool),
kmsKeyID: kmsKeyID,
}
switch ebsConfig.volumeType {
case volumeAliasMagnetic:
Expand All @@ -226,10 +246,27 @@ func newEbsConfig(attrs map[string]interface{}) (*ebsConfig, error) {
case volumeAliasProvisionedIops:
ebsConfig.volumeType = volumeTypeIO1
}
if ebsConfig.iops > 0 && ebsConfig.volumeType != volumeTypeIO1 {
return nil, errors.Errorf("IOPS specified, but volume type is %q", volumeType)
} else if ebsConfig.iops == 0 && ebsConfig.volumeType == volumeTypeIO1 {
return nil, errors.Errorf("volume type is %q, IOPS unspecified or zero", volumeTypeIO1)
if throughput != "" {
throughputMB, err := utils.ParseSize(throughput)
if err != nil {
return nil, errors.Annotatef(err, "parsing %q", EBS_Throughput)
}
ebsConfig.throughputMB = int(throughputMB)
}
switch ebsConfig.volumeType {
case volumeTypeIO1, volumeTypeIO2:
if ebsConfig.iops == 0 {
return nil, errors.Errorf("volume type is %q, IOPS unspecified or zero", volumeTypeIO1)
}
case volumeTypeGP3:
// iops is optional
default:
if ebsConfig.iops > 0 {
return nil, errors.Errorf("IOPS specified, but volume type is %q", volumeType)
}
}
if ebsConfig.throughputMB != 0 && ebsConfig.volumeType != volumeTypeGP3 {
return nil, errors.Errorf("%q cannot be specified when volume type is %q", EBS_Throughput, volumeType)
}
return ebsConfig, nil
}
Expand Down Expand Up @@ -298,10 +335,6 @@ func parseVolumeOptions(size uint64, attrs map[string]interface{}) (_ ec2.Create
if err != nil {
return ec2.CreateVolumeInput{}, errors.Trace(err)
}
if ebsConfig.iops > 0 && ebsConfig.volumeType != volumeTypeIO1 {
return ec2.CreateVolumeInput{}, errors.Errorf(
"specifying iops is not supported for %q volumes", ebsConfig.volumeType)
}
if ebsConfig.iops > maxProvisionedIopsSizeRatio {
return ec2.CreateVolumeInput{}, errors.Errorf(
"specified IOPS ratio is %d/GiB, maximum is %d/GiB",
Expand All @@ -320,9 +353,15 @@ func parseVolumeOptions(size uint64, attrs map[string]interface{}) (_ ec2.Create
VolumeType: types.VolumeType(ebsConfig.volumeType),
Encrypted: aws.Bool(ebsConfig.encrypted),
}
if ebsConfig.kmsKeyID != "" {
vol.KmsKeyId = aws.String(ebsConfig.kmsKeyID)
}
if iops > 0 {
vol.Iops = aws.Int32(int32(iops))
}
if ebsConfig.throughputMB > 0 {
vol.Throughput = aws.Int32(int32(ebsConfig.throughputMB))
}
return vol, nil
}

Expand Down Expand Up @@ -688,10 +727,11 @@ func (v *ebsVolumeSource) ValidateVolumeParams(params storage.VolumeParams) erro
case volumeTypeStandard:
minVolumeSize = minMagneticVolumeSizeGiB
maxVolumeSize = maxMagneticVolumeSizeGiB
case volumeTypeGP2:
case volumeTypeGP2, types.VolumeTypeGp3:
minVolumeSize = minSSDVolumeSizeGiB
maxVolumeSize = maxSSDVolumeSizeGiB
case volumeTypeIO1:
case volumeTypeIO1, types.VolumeTypeIo2:
// TODO(juju3): check io2 max disk size re: io2 block express on r5b instances.
minVolumeSize = minProvisionedIopsVolumeSizeGiB
maxVolumeSize = maxProvisionedIopsVolumeSizeGiB
case volumeTypeST1:
Expand Down Expand Up @@ -1069,7 +1109,8 @@ func getBlockDeviceMappings(
cons constraints.Value,
series string,
controller bool,
) []types.BlockDeviceMapping {
rootDisk *storage.VolumeParams,
) ([]types.BlockDeviceMapping, error) {
minRootDiskSizeMiB := minRootDiskSizeMiB(series)
rootDiskSizeMiB := minRootDiskSizeMiB
if controller {
Expand All @@ -1086,13 +1127,37 @@ func getBlockDeviceMappings(
)
}
}
// The first block device is for the root disk.
blockDeviceMappings := []types.BlockDeviceMapping{{

rootDiskMapping := types.BlockDeviceMapping{
DeviceName: aws.String(rootDiskDeviceName),
Ebs: &types.EbsBlockDevice{
VolumeSize: aws.Int32(int32(mibToGib(rootDiskSizeMiB))),
},
}}
}
if rootDisk != nil {
config, err := newEbsConfig(rootDisk.Attributes)
if err != nil {
return nil, errors.Annotatef(err, "parsing root disk attributes")
}
if config.encrypted {
rootDiskMapping.Ebs.Encrypted = aws.Bool(config.encrypted)
}
if config.iops > 0 {
rootDiskMapping.Ebs.Iops = aws.Int32(int32(config.iops))
}
if config.volumeType != "" {
rootDiskMapping.Ebs.VolumeType = types.VolumeType(config.volumeType)
}
if config.kmsKeyID != "" {
rootDiskMapping.Ebs.KmsKeyId = aws.String(config.kmsKeyID)
}
if config.throughputMB > 0 {
rootDiskMapping.Ebs.Throughput = aws.Int32(int32(config.throughputMB))
}
}

// The first block device is for the root disk.
blockDeviceMappings := []types.BlockDeviceMapping{rootDiskMapping}

// Not all machines have this many instance stores.
// Instances will be started with as many of the
Expand All @@ -1111,7 +1176,7 @@ func getBlockDeviceMappings(
DeviceName: aws.String("/dev/sde"),
}}...)

return blockDeviceMappings
return blockDeviceMappings, nil
}

// mibToGib converts mebibytes to gibibytes.
Expand Down
Loading

0 comments on commit 87cc371

Please sign in to comment.