Merge pull request #1 from Merlion-Crew/ml_model_uc76

huynhbaotoan · web-flow · commit 347c776acebc · 2020-09-24T12:01:17.000+07:00
UC-90: Build ML models container image to Nexus
diff --git a/ml_service/pipelines/build-model-container-image.Jenkinsfile b/ml_service/pipelines/build-model-container-image.Jenkinsfile
@@ -0,0 +1,59 @@
+pipeline {
+    agent { label 'master' }
+    environment {
+        ML_IMAGE_FOLDER         = 'imagefiles'
+        IMAGE_NAME              = 'mlmodelimage'
+        MODEL_NAME = 'diabetes_regression_model.pkl'
+        MODEL_VERSION = '1'
+        SCORE_SCRIPT = 'scoring/score.py'
+        RESOURCE_GROUP = "${RESOURCE_GROUP}"
+        WORKSPACE_NAME = "${WORKSPACE_NAME}"
+        ML_CONTAINER_REGISTRY   = "${ML_CONTAINER_REGISTRY}"
+    }
+    stages {
+        stage('initialize') {
+            steps {
+                echo 'Remove the previous one!'
+            }
+            post {
+                always {
+                    deleteDir() /* clean up our workspace */
+                }
+            }
+        }
+        stage('generate_dockerfile') {
+            steps {
+                echo "Hello build ${env.BUILD_ID}"
+                /*checkout scm*/
+                checkout([$class: 'GitSCM', branches: [[name: '*/ml_model_uc76']],
+                    userRemoteConfigs: [[url: 'https://github.com/Merlion-Crew/MLOpsPython.git/']]])
+
+                azureCLI commands: [[exportVariablesString: '/id|SUBSCRIPTION_ID', script: "az account show"]], principalCredentialId: "${AZURE_SP}"
+                
+                sh '''#!/bin/bash -ex
+                    source /home/azureuser/anaconda3/bin/activate mlopspython_ci
+                    python3 -m ml_service.util.create_scoring_image
+                '''
+            }
+        }
+        stage('build_and_push') {
+            steps {
+                echo "Build docker images"
+
+                sh '''#!/bin/bash -ex
+                    az acr login --name $ML_CONTAINER_REGISTRY
+                    docker build -t $NEXUS_DOCKER_REGISTRY_URL/$IMAGE_NAME:$BUILD_ID ./diabetes_regression/scoring/$ML_IMAGE_FOLDER/ 
+                '''
+
+                withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'nexus-docker-repo',
+                    usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD']]) {
+                        
+                    sh '''#!/bin/bash -ex
+                        docker login -u $USERNAME --password $PASSWORD https://$NEXUS_DOCKER_REGISTRY_URL
+                        docker push $NEXUS_DOCKER_REGISTRY_URL/$IMAGE_NAME:$BUILD_ID
+                    '''
+                }
+            }
+        }
+    }
+}
diff --git a/ml_service/util/create_scoring_image.py b/ml_service/util/create_scoring_image.py
@@ -40,8 +40,19 @@
 scoring_env = Environment.from_conda_specification(name="scoringenv", file_path="conda_dependencies.yml")  # NOQA: E501
 inference_config = InferenceConfig(
     entry_script=score_file, environment=scoring_env)
-package = Model.package(ws, [model], inference_config)
+
+saved_package = os.environ.get("ML_IMAGE_FOLDER")
+saved_dockerfile = True
+if (saved_package is None):
+    saved_dockerfile = False
+package = Model.package(ws, [model], inference_config, generate_dockerfile=saved_dockerfile)
+
+
 package.wait_for_creation(show_output=True)
+
+if(saved_dockerfile):
+    package.save(saved_package)
+    
 # Display the package location/ACR path
 print(package.location)
 
