Continously release versions. (linkedin#12)

This is done via a custom GitHub workflow / action that uses shipkit github release and auto version plugins. This means we will publish a release on GitHub and bintray for every push of the master branch. For now autopublish to bintray is still disabled, we can enable it once we're sure we're happy with this pipeline.
jywadhwani · Nov 10, 2020 · 7cc3c4f · 7cc3c4f
1 parent fc886d1
commit 7cc3c4f
Show file tree

Hide file tree

Showing 7 changed files with 180 additions and 16 deletions.
diff --git a/.github/actions/release/action.yml b/.github/actions/release/action.yml
@@ -0,0 +1,18 @@
+name: 'Publish Releases'
+description: 'Runs gradle tasks to publish versions to Bintray and GitHub.'
+outputs:
+  version:
+    description: "The version tag this created."
+    value: ${{ steps.get-tag.outputs.tag }}
+runs:
+  using: "composite"
+  steps:
+    - id: get-tag
+      run: echo "::set-output name=tag::$(./gradlew -q getVersion)"
+      shell: bash
+    - run: echo Got version ${{ steps.get-tag.outputs.tag }}
+      shell: bash
+    - run: ./gradlew build publishToMavenLocal bintrayUploadAll -Pbintray.dryRun
+      shell: bash
+    - run: ./gradlew ciPerformRelease
+      shell: bash
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,23 @@
+name: Create GitHub & Bintray Release
+on:
+  push:
+    branches:
+      - 'master'
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          # Needed to get all tags
+          fetch-depth: 0
+      - name: Release
+        uses: ./.github/actions/release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BINTRAY_USER: ${{ secrets.BINTRAY_USER }}
+          BINTRAY_KEY: ${{ secrets.BINTRAY_KEY }}
diff --git a/build.gradle b/build.gradle
@@ -11,7 +11,8 @@ buildscript {
     classpath 'com.linkedin.pegasus:gradle-plugins:' + pegasusVersion
     classpath 'org.springframework.boot:spring-boot-gradle-plugin:2.1.2.RELEASE'
     classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.+'
-    classpath "org.shipkit:shipkit-auto-version:0.0.22"
+    classpath "org.shipkit:shipkit-auto-version:0.0.48"
+    classpath "org.shipkit:shipkit-changelog:0.0.45"
   }
 }
 
@@ -60,7 +61,7 @@ project.ext.externalDependency = [
 ]
 
 apply plugin: 'com.diffplug.spotless'
-apply plugin: 'org.shipkit.shipkit-auto-version'
+apply from: "./gradle/release.gradle"
 
 // TODO expand this to all projects and then delete this allow list. This list is letting us fix errors over time rather
 // than in one big change.
@@ -74,8 +75,6 @@ def wErrorProjects = [
 ]
 
 allprojects {
-  group = "com.linkedin.datahub-gma"
-
   apply plugin: 'idea'
   apply plugin: 'eclipse'
   apply plugin: 'checkstyle'
@@ -92,10 +91,6 @@ allprojects {
   }
 }
 
-if (!project.hasProperty('disableShipkit')) {
-  apply from: file("gradle/ci-release.gradle")
-}
-
 subprojects { sp ->
   apply plugin: 'maven'
 
@@ -140,4 +135,4 @@ spotless {
     targetExclude '.github/**' // GitHub renders these documents differently...
     prettier().config([printWidth: 120, proseWrap: 'always'])
   }
-}
+}
diff --git a/docs/how/releases/continuous-releases.md b/docs/how/releases/continuous-releases.md
@@ -0,0 +1,36 @@
+# GMA Continuous Releases
+
+We create [GitHub releases](https://github.com/linkedin/datahub-gma/releases) and publish our artifacts to
+[Bintray](https://bintray.com/linkedin/maven/datahub-gma).
+
+We use [shipkit-auto-version](https://github.com/shipkit/shipkit-auto-version),
+[shipkit-changelog](https://github.com/shipkit/shipkit-changelog), and shipkit-gh-release (part of shipkit-changelog) to
+help create our continuous releases.
+
+## Automatic Versioning
+
+We follow the semantic versioning 2.0 scheme as defined at [https://semver.org](https://semver.org).
+
+Versioning is determined by the [shipkit-auto-version plugin](https://github.com/shipkit/shipkit-auto-version). The
+current version is defined in the [version.properties](../../../version.properties) file.
+
+We automatically create a new release for every push of the master branch. We feel this simplifies development and makes
+it clear what the latest state of the repository and releases are, as well as making it very easy for contributors to
+pick up their changes once submitted. We do not currently support continuous beta releases.
+
+Note that when we make a release, we also tag the commit with the version, making it easy to go back through the git
+history to a specific version if needed.
+
+## Automatic Changelog
+
+We use shipkit changelog to create the release notes of each version. See
+[shipkit-changelog](https://github.com/shipkit/shipkit-changelog) for more details.
+
+## Automatic Publishing
+
+Automation is done with a [GitHub workflow](../../.github/workflows/gh-version.yml) that will create the GitHub release
+and then publish to bintray for every push of the `main` branch.
+
+This workflow relies on the `BINTRAY_USER` and `BINTRAY_KEY` secrets, which should be a user on Bintray with permissions
+to our project, and their api key, respectively. The exact user and key don't really matter, it is very easy to change
+in the event that we need to.
diff --git a/gradle/ci-release.gradle b/gradle/ci-release.gradle
@@ -1,9 +1,4 @@
-// TODO actually hook up to a CI. Until we hook up a CI follow these steps:
-// 1. Check out the repo and sync to the commit you want to release.
-// 2. Tag the commit by running `git tag -a -m "Release v$version" "v$version"`, replacing $version with the semver.
-// 3. Push the tag by running `git push --tags`
-// 4. Upload to bintray by running `./gradlew bintrayUploadAll`.
-// 5. Go to GitHub and make a release from the tag.
+// CI done via ./github/workflows/gh-release.yml.
 
 // Task to fail fast and clearly if required properties are not set.
 task verifyBintrayProperties {
@@ -21,11 +16,23 @@ task verifyBintrayProperties {
 
 task bintrayUploadAll {
   description = "Runs 'bintrayUpload' tasks from all projects"
+  mustRunAfter "githubRelease" // github release is easier to rollback so we run it first
 }
 
 allprojects {
   tasks.matching { it.name == "bintrayUpload" }.all {
     it.dependsOn verifyBintrayProperties
     bintrayUploadAll.dependsOn it
   }
-}
+}
+
+task ciPerformRelease {
+  description = "Performs the release, intended to be ran on CI"
+  dependsOn "githubRelease", "bintrayUploadAll"
+}
+
+task getVersion() {
+  doFirst {
+    println version
+  }
+}
diff --git a/gradle/gradle-push.sh b/gradle/gradle-push.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+tag="v$1"
+echo "Applying and pushing tag $tag"
+
+# For debugging
+echo "current branch" "$(git branch --show-current)"
+echo "=== recent history"
+git log --oneline -n 10
+echo "==="
+
+EXISTING_TAGS=$(git tag --points-at HEAD)
+
+if [[ "$EXISTING_TAGS" ]]; then
+  echo "Tags already exist on HEAD: $EXISTING_TAGS"
+  exit 1
+fi
+
+echo "Tagging commit"
+
+git tag "$tag"
+
+commit=$(git rev-parse HEAD)
+full_name=$GITHUB_REPOSITORY
+git_refs_url=$(jq .repository.git_refs_url $GITHUB_EVENT_PATH | tr -d '"' | sed 's/{\/sha}//g')
+echo "pushing tag $tag to repo $full_name"
+
+git_refs_response=$(
+curl -s -X POST $git_refs_url \
+-H "Authorization: token $GITHUB_TOKEN" \
+-d @- << EOF
+{
+  "ref": "refs/tags/$tag",
+  "sha": "$commit"
+}
+EOF
+)
+
+git_ref_posted=$( echo "${git_refs_response}" | jq .ref | tr -d '"' )
+
+echo "::debug::${git_refs_response}"
+if [ "${git_ref_posted}" = "refs/tags/${tag}" ]; then
+  exit 0
+else
+  echo "::error::Tag was not created properly."
+  exit 1
+fi
diff --git a/gradle/release.gradle b/gradle/release.gradle
@@ -0,0 +1,38 @@
+apply plugin: 'org.shipkit.shipkit-auto-version'
+apply plugin: 'org.shipkit.shipkit-changelog'
+apply plugin: 'org.shipkit.shipkit-gh-release'
+
+
+if (!project.hasProperty('disableShipkit')) {
+  task checkGitHubToken {
+    doFirst {
+      if (System.getenv("GITHUB_TOKEN") == null) {
+        throw new Exception("Environment variable GITHUB_TOKEN not set.");
+      }
+      println "Using repository " + System.getenv("GITHUB_REPOSITORY")
+    }
+  }
+
+  tasks.named("generateChangelog") {
+    dependsOn checkGitHubToken
+    previousRevision = "v" + project.ext.'shipkit-auto-version.previous-version'
+    readOnlyToken = System.getenv("GITHUB_TOKEN")
+    repository = System.getenv("GITHUB_REPOSITORY")
+  }
+
+  tasks.named("githubRelease") {
+    dependsOn tasks.named("generateChangelog")
+    dependsOn checkGitHubToken
+    repository = System.getenv("GITHUB_REPOSITORY")
+    changelog = tasks.named("generateChangelog").get().outputFile
+    writeToken = System.getenv("GITHUB_TOKEN")
+  }
+}
+
+allprojects { p ->
+  group = "com.linkedin.datahub-gma"
+}
+
+if (!project.hasProperty('disableShipkit')) {
+  apply from: file("gradle/ci-release.gradle")
+}