Skip to content

Commit 8d1be26

Browse files
eedorenkoDavid Tesar
eedorenko
authored and
David Tesar
committed
Canary deployment (microsoft#125)
1 parent 2a22e75 commit 8d1be26

20 files changed

Lines changed: 505 additions & 2 deletions

‎.pipelines/azdo-abtest-pipeline.yml‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
pr: none
2+
trigger:
3+
branches:
4+
include:
5+
- master
6+
paths:
7+
exclude:
8+
- docs/
9+
- environment_setup/
10+
- ml_service/util/create_scoring_image.*
11+
- ml_service/util/smoke_test_scoring_service.py
12+
13+
variables:
14+
- group: 'devopsforai-aml-vg'
15+
- name: 'helmVersion'
16+
value: 'v3.0.1'
17+
- name: 'helmDownloadURL'
18+
value: 'https://get.helm.sh/helm-$HELM_VERSION-linux-amd64.tar.gz'
19+
- name: 'blueReleaseName'
20+
value: 'model-blue'
21+
- name: 'greenReleaseName'
22+
value: 'model-green'
23+
- name: 'SCORE_SCRIPT'
24+
value: 'scoreA.py'
25+
26+
27+
stages:
28+
- stage: 'Building'
29+
jobs:
30+
- job: "Build_Scoring_image"
31+
timeoutInMinutes: 0
32+
pool:
33+
vmImage: 'ubuntu-latest'
34+
container: mcr.microsoft.com/mlops/python:latest
35+
steps:
36+
- task: AzureCLI@1
37+
inputs:
38+
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
39+
scriptLocation: inlineScript
40+
inlineScript: |
41+
set -e
42+
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
43+
python ml_service/util/create_scoring_image.py --output_image_location_file image_location.txt
44+
displayName: 'Create Scoring Image'
45+
name: 'buildscoringimage'
46+
47+
- publish: image_location.txt
48+
artifact: image_location
49+
50+
- publish: $(System.DefaultWorkingDirectory)/charts
51+
artifact: allcharts
52+
53+
54+
- stage: 'Blue_Staging'
55+
jobs:
56+
- deployment: "Deploy_to_Staging"
57+
timeoutInMinutes: 0
58+
environment: abtestenv
59+
strategy:
60+
runOnce:
61+
deploy:
62+
steps:
63+
- script: |
64+
IMAGE_LOCATION="$(cat $(Pipeline.Workspace)/image_location/image_location.txt)"
65+
echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION"
66+
displayName: 'Get Image Location'
67+
- template: azdo-helm-upgrade.yml
68+
parameters:
69+
chartPath: '$(Pipeline.Workspace)/allcharts/abtest-model'
70+
releaseName: $(blueReleaseName)
71+
overrideValues: 'deployment.name=$(blueReleaseName),deployment.bluegreen=blue,deployment.image.name=$(IMAGE_LOCATION)'
72+
73+
- stage: 'Blue_50'
74+
jobs:
75+
- job: 'Blue_Rollout_50'
76+
displayName: 50 50 rollout to blue environment
77+
timeoutInMinutes: 0
78+
steps:
79+
- template: azdo-helm-upgrade.yml
80+
parameters:
81+
chartPath: '$(System.DefaultWorkingDirectory)/charts/abtest-istio'
82+
releaseName: 'abtest-istio'
83+
overrideValues: 'weight.blue=50,weight.green=50'
84+
85+
- stage: 'Blue_100'
86+
jobs:
87+
- deployment: 'blue_Rollout_100'
88+
timeoutInMinutes: 0
89+
environment: abtestenv
90+
strategy:
91+
runOnce:
92+
deploy:
93+
steps:
94+
- template: azdo-helm-upgrade.yml
95+
parameters:
96+
chartPath: '$(Pipeline.Workspace)/allcharts/abtest-istio'
97+
releaseName: 'abtest-istio'
98+
overrideValues: 'weight.blue=100,weight.green=0'
99+
100+
- stage: 'Rollback'
101+
dependsOn: 'Blue_100'
102+
condition: failed()
103+
jobs:
104+
- deployment: 'Roll_Back'
105+
displayName: 'Roll Back after failure'
106+
environment: abtestenv
107+
strategy:
108+
runOnce:
109+
deploy:
110+
steps:
111+
- template: azdo-helm-upgrade.yml
112+
parameters:
113+
chartPath: '$(Pipeline.Workspace)/allcharts/abtest-istio'
114+
releaseName: 'abtest-istio'
115+
overrideValues: 'weight.blue=0,weight.green=100'
116+
117+
- stage: 'Set_Production_Tag'
118+
dependsOn: 'Blue_100'
119+
condition: succeeded()
120+
jobs:
121+
- deployment: 'green_blue_tagging'
122+
timeoutInMinutes: 0
123+
environment: abtestenv
124+
strategy:
125+
runOnce:
126+
deploy:
127+
steps:
128+
- script: |
129+
IMAGE_LOCATION="$(cat $(Pipeline.Workspace)/image_location/image_location.txt)"
130+
echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION"
131+
displayName: 'Get Image Location'
132+
- template: azdo-helm-upgrade.yml
133+
parameters:
134+
chartPath: '$(Pipeline.Workspace)/allcharts/abtest-model'
135+
releaseName: $(greenReleaseName)
136+
overrideValues: 'deployment.name=$(greenReleaseName),deployment.bluegreen=green,deployment.image.name=$(IMAGE_LOCATION)'
137+
138+
- stage: 'Green_100'
139+
jobs:
140+
- job: 'Prod_Rollout_100'
141+
timeoutInMinutes: 0
142+
steps:
143+
- template: azdo-helm-upgrade.yml
144+
parameters:
145+
chartPath: '$(System.DefaultWorkingDirectory)/charts/abtest-istio'
146+
releaseName: 'abtest-istio'
147+
overrideValues: 'weight.blue=0,weight.green=100'
148+
149+
- stage: 'Disable_blue'
150+
condition: always()
151+
jobs:
152+
- job: 'blue_disable'
153+
timeoutInMinutes: 0
154+
steps:
155+
- template: azdo-helm-install.yml
156+
- task: HelmDeploy@0
157+
displayName: 'helm uninstall blue'
158+
inputs:
159+
connectionType: 'Kubernetes Service Connection'
160+
kubernetesServiceConnection: $(K8S_AB_SERVICE_CONNECTION)
161+
command: delete
162+
arguments: $(blueReleaseName) --namespace $(K8S_AB_NAMESPACE)

‎.pipelines/azdo-ci-image.yml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
pr: none
2+
trigger:
3+
branches:
4+
include:
5+
- master
6+
paths:
7+
include:
8+
- ml_service/util/create_scoring_image.py
9+
- ml_service/util/Dockerfile
10+
- code/scoring/
11+
exclude:
12+
- code/scoring/deployment_config_aci.yml
13+
- code/scoring/deployment_config_aks.yml
14+
15+
pool:
16+
vmImage: 'ubuntu-latest'
17+
18+
container: mcr.microsoft.com/mlops/python:latest
19+
20+
variables:
21+
- group: devopsforai-aml-vg
22+
- name: 'SCORE_SCRIPT'
23+
value: 'scoreB.py'
24+
25+
steps:
26+
27+
- task: AzureCLI@1
28+
inputs:
29+
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
30+
scriptLocation: inlineScript
31+
inlineScript: |
32+
set -e
33+
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
34+
python3 $(Build.SourcesDirectory)/ml_service/util/create_scoring_image.py
35+
displayName: 'Create Scoring Image'
36+

‎.pipelines/azdo-helm-install.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
steps:
2+
- task: Bash@3
3+
displayName: 'Install Helm $(helmVersion)'
4+
inputs:
5+
targetType: inline
6+
script: wget -q $(helmDownloadURL) -O /tmp/$FILENAME && tar -zxvf /tmp/$FILENAME -C /tmp && sudo mv /tmp/linux-amd64/helm /usr/local/bin/helm
7+
env:
8+
HELM_VERSION: $(helmVersion)
9+
FILENAME: helm-$(helmVersion)-linux-amd64.tar.gz

‎.pipelines/azdo-helm-upgrade.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
parameters:
2+
chartPath: ''
3+
releaseName: ''
4+
overrideValues: ''
5+
6+
steps:
7+
- template: azdo-helm-install.yml
8+
- task: HelmDeploy@0
9+
displayName: 'helm upgrade'
10+
inputs:
11+
connectionType: 'Kubernetes Service Connection'
12+
kubernetesServiceConnection: $(K8S_AB_SERVICE_CONNECTION)
13+
command: upgrade
14+
chartType: FilePath
15+
chartPath: ${{ parameters.chartPath }}
16+
releaseName: ${{ parameters.releaseName }}
17+
overrideValues: ${{ parameters.overrideValues }}
18+
install: true
19+
arguments: --namespace $(K8S_AB_NAMESPACE)

‎.pipelines/azdo-variables.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,4 +37,6 @@ variables:
3737
value: 'mltrained'
3838
# Optional. Used by a training pipeline with R on Databricks
3939
- name: DB_CLUSTER_ID
40-
value: ''
40+
value: ''
41+
- name: SCORE_SCRIPT
42+
value: score.py

‎charts/abtest-istio/Chart.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
apiVersion: v1
2+
appVersion: "1.0"
3+
description: A Helm chart for Kubernetes
4+
name: abtest-istio
5+
version: 0.1.0

‎charts/abtest-istio/templates/istio-canary.yaml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
apiVersion: networking.istio.io/v1alpha3
2+
kind: Gateway
3+
metadata:
4+
name: mlmodel-gateway
5+
namespace: abtesting
6+
spec:
7+
selector:
8+
istio: ingressgateway
9+
servers:
10+
- port:
11+
number: {{ .Values.ingress.port }}
12+
name: http
13+
protocol: HTTP
14+
hosts:
15+
- "*"
16+
---
17+
apiVersion: networking.istio.io/v1alpha3
18+
kind: VirtualService
19+
metadata:
20+
name: mlmodel-virtualservice
21+
namespace: abtesting
22+
spec:
23+
gateways:
24+
- mlmodel-gateway
25+
hosts:
26+
- '*'
27+
http:
28+
- match:
29+
- uri:
30+
prefix: /score
31+
headers:
32+
x-api-version:
33+
exact: 'blue'
34+
route:
35+
- destination:
36+
host: {{ .Values.svc.name }}-blue.abtesting.svc.cluster.local
37+
port:
38+
number: {{ .Values.svc.port }}
39+
- match:
40+
- uri:
41+
prefix: /score
42+
headers:
43+
x-api-version:
44+
exact: 'green'
45+
route:
46+
- destination:
47+
host: {{ .Values.svc.name }}-green.abtesting.svc.cluster.local
48+
port:
49+
number: {{ .Values.svc.port }}
50+
- route:
51+
- destination:
52+
host: {{ .Values.svc.name }}-green.abtesting.svc.cluster.local
53+
port:
54+
number: {{ .Values.svc.port }}
55+
weight: {{ .Values.weight.green }}
56+
- destination:
57+
host: {{ .Values.svc.name }}-blue.abtesting.svc.cluster.local
58+
port:
59+
number: {{ .Values.svc.port }}
60+
weight: {{ .Values.weight.blue }}

‎charts/abtest-istio/values.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
ingress:
2+
port: 80
3+
4+
svc:
5+
port: 5001
6+
name: model-svc
7+
8+
9+
weight:
10+
green: 50
11+
blue: 50
12+
13+
uri:
14+
prefix: /score
15+

‎charts/abtest-model/Chart.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
apiVersion: v1
2+
appVersion: "1.0"
3+
description: A Helm chart for Kubernetes
4+
name: abtest-model
5+
version: 0.1.0

‎charts/abtest-model/templates/deployment.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: {{ .Values.deployment.name }}
5+
namespace: {{ .Values.namespace }}
6+
labels:
7+
app: {{ .Values.appname }}
8+
model_version: {{ .Values.deployment.bluegreen }}
9+
spec:
10+
replicas: 1
11+
selector:
12+
matchLabels:
13+
app: {{ .Values.appname }}
14+
model_version: {{ .Values.deployment.bluegreen }}
15+
template:
16+
metadata:
17+
labels:
18+
app: {{ .Values.appname }}
19+
model_version: {{ .Values.deployment.bluegreen }}
20+
spec:
21+
containers:
22+
- name: {{ .Values.deployment.container.name }}
23+
image: "{{ .Values.deployment.image.name }}"
24+
imagePullPolicy: Always
25+
ports:
26+
- name: http
27+
containerPort: 5001
28+
- name: probe
29+
containerPort: 8086
30+
imagePullSecrets:
31+
- name: aks-secret

0 commit comments

Comments
 (0)