Skip to content

Commit 63680ba

Browse files
adchiaadchia
authored
Moving Feast Java back into main repo under java/ package (feast-dev#1997)
* Moving Feast Java back into main repo under java/ package Signed-off-by: Danny Chiao <[email protected]> * Update infra scripts with new pom.xml structure Signed-off-by: Danny Chiao <[email protected]> * Fix test-java-sdk.sh Signed-off-by: Danny Chiao <[email protected]> * Remove unneeded mirror Signed-off-by: Danny Chiao <[email protected]> * Update workflows with TODOs Signed-off-by: Danny Chiao <[email protected]>
1 parent a9a3a16 commit 63680ba

287 files changed

Lines changed: 27701 additions & 18 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/build_feature_server.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ on:
44
push:
55
branches:
66
- master
7+
paths:
8+
- "protos/**"
9+
- "sdk/python**"
710

811
jobs:
912
build-docker-image:

‎.github/workflows/build_feature_transformation_server.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ on:
44
push:
55
branches:
66
- master
7+
paths:
8+
- "protos/**"
9+
- "sdk/python**"
710

811
jobs:
912
build-docker-image:

‎.github/workflows/java_complete.yml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
name: java_complete
2+
3+
on: [push, pull_request]
4+
5+
jobs:
6+
lint-java:
7+
container: gcr.io/kf-feast/feast-ci:latest
8+
runs-on: [ubuntu-latest]
9+
steps:
10+
- uses: actions/checkout@v2
11+
with:
12+
submodules: 'true'
13+
- name: Lint java
14+
run: make lint-java
15+
16+
unit-test-java:
17+
runs-on: ubuntu-latest
18+
needs: lint-java
19+
steps:
20+
- uses: actions/checkout@v2
21+
with:
22+
submodules: 'true'
23+
- name: Set up JDK 11
24+
uses: actions/setup-java@v1
25+
with:
26+
java-version: '11'
27+
java-package: jdk
28+
architecture: x64
29+
- uses: actions/cache@v2
30+
with:
31+
path: ~/.m2/repository
32+
key: ${{ runner.os }}-ut-maven-${{ hashFiles('**/pom.xml') }}
33+
restore-keys: |
34+
${{ runner.os }}-ut-maven-
35+
- name: Test java
36+
run: make test-java-with-coverage
37+
- uses: actions/upload-artifact@v2
38+
with:
39+
name: java-coverage-report
40+
path: ${{ github.workspace }}/docs/coverage/java/target/site/jacoco-aggregate/
41+
42+
integration-test:
43+
runs-on: ubuntu-latest
44+
needs: unit-test-java
45+
services:
46+
redis:
47+
image: redis
48+
ports:
49+
- 6389:6379
50+
options: >-
51+
--health-cmd "redis-cli ping"
52+
--health-interval 10s
53+
--health-timeout 5s
54+
--health-retries 5
55+
steps:
56+
- uses: actions/checkout@v2
57+
with:
58+
submodules: 'true'
59+
- name: Set up JDK 11
60+
uses: actions/setup-java@v1
61+
with:
62+
java-version: '11'
63+
java-package: jdk
64+
architecture: x64
65+
- uses: actions/setup-python@v2
66+
with:
67+
python-version: '3.7'
68+
architecture: 'x64'
69+
- uses: actions/cache@v2
70+
with:
71+
path: ~/.m2/repository
72+
key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }}
73+
restore-keys: |
74+
${{ runner.os }}-it-maven-
75+
- name: Run integration tests
76+
run: make test-java-integration
77+
- name: Save report
78+
uses: actions/upload-artifact@v2
79+
if: failure()
80+
with:
81+
name: it-report
82+
path: spark/ingestion/target/test-reports/TestSuite.txt
83+
retention-days: 5

‎.github/workflows/java_master_only.yml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
name: master only
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
tags:
8+
- 'v*.*.*'
9+
10+
jobs:
11+
build-docker-images:
12+
runs-on: ubuntu-latest
13+
strategy:
14+
matrix:
15+
component: [core, serving]
16+
env:
17+
MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
18+
steps:
19+
- uses: actions/checkout@v2
20+
with:
21+
submodules: 'true'
22+
- uses: google-github-actions/setup-gcloud@master
23+
with:
24+
version: '290.0.1'
25+
export_default_credentials: true
26+
project_id: ${{ secrets.GCP_PROJECT_ID }}
27+
service_account_key: ${{ secrets.GCP_SA_KEY }}
28+
- run: gcloud auth configure-docker --quiet
29+
- name: Get m2 cache
30+
run: |
31+
infra/scripts/download-maven-cache.sh \
32+
--archive-uri ${MAVEN_CACHE} \
33+
--output-dir .
34+
- name: Get version
35+
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
36+
- name: Build image
37+
run: make build-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA}
38+
- name: Push image
39+
run: make push-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA}
40+
- name: Push development Docker image
41+
run: |
42+
if [ ${GITHUB_REF#refs/*/} == "master" ]; then
43+
docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:develop
44+
docker push gcr.io/kf-feast/feast-${{ matrix.component }}:develop
45+
fi

‎.github/workflows/java_release.yml‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
name: release
2+
# TODO(adchia): update DockerHub + Maven repo paths so version can match existing Feast versions.
3+
on:
4+
push:
5+
tags:
6+
- 'v*.*.*'
7+
8+
jobs:
9+
get-version:
10+
runs-on: ubuntu-latest
11+
outputs:
12+
release_version: ${{ steps.get_release_version.outputs.release_version }}
13+
version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
14+
highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
15+
steps:
16+
- uses: actions/checkout@v2
17+
- name: Get release version
18+
id: get_release_version
19+
run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/}
20+
- name: Get release version without prefix
21+
id: get_release_version_without_prefix
22+
env:
23+
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
24+
run: |
25+
echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1}
26+
- name: Get highest semver
27+
id: get_highest_semver
28+
env:
29+
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
30+
run: |
31+
source infra/scripts/setup-common-functions.sh
32+
SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
33+
if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then
34+
echo ::set-output name=highest_semver_tag::$(get_tag_release -m)
35+
fi
36+
- name: Check output
37+
env:
38+
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
39+
VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
40+
HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
41+
run: |
42+
echo $RELEASE_VERSION
43+
echo $VERSION_WITHOUT_PREFIX
44+
echo $HIGHEST_SEMVER_TAG
45+
46+
build-publish-docker-images:
47+
runs-on: [ubuntu-latest]
48+
needs: get-version
49+
strategy:
50+
matrix:
51+
component: [core, serving]
52+
env:
53+
MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
54+
steps:
55+
- uses: actions/checkout@v2
56+
with:
57+
submodules: 'true'
58+
- name: Set up QEMU
59+
uses: docker/setup-qemu-action@v1
60+
- name: Set up Docker Buildx
61+
uses: docker/setup-buildx-action@v1
62+
- name: Login to DockerHub
63+
uses: docker/login-action@v1
64+
with:
65+
username: ${{ secrets.DOCKERHUB_USERNAME }}
66+
password: ${{ secrets.DOCKERHUB_TOKEN }}
67+
- uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
68+
with:
69+
version: '290.0.1'
70+
export_default_credentials: true
71+
project_id: ${{ secrets.GCP_PROJECT_ID }}
72+
service_account_key: ${{ secrets.GCP_SA_KEY }}
73+
- run: gcloud auth configure-docker --quiet
74+
- name: Get m2 cache
75+
run: |
76+
infra/scripts/download-maven-cache.sh \
77+
--archive-uri ${MAVEN_CACHE} \
78+
--output-dir .
79+
- name: Build and push versioned images
80+
env:
81+
RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }}
82+
VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
83+
HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }}
84+
run: |
85+
docker build --build-arg VERSION=$RELEASE_VERSION \
86+
-t gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} \
87+
-t gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} \
88+
-t feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} \
89+
-f infra/docker/${{ matrix.component }}/Dockerfile .
90+
docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
91+
docker push feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
92+
93+
echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG"
94+
if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ]
95+
then
96+
docker tag feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} feastdev/feast-${{ matrix.component }}:latest
97+
docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} gcr.io/kf-feast/feast-${{ matrix.component }}:latest
98+
docker push feastdev/feast-${{ matrix.component }}:latest
99+
docker push gcr.io/kf-feast/feast-${{ matrix.component }}:latest
100+
fi
101+
102+
publish-java-sdk:
103+
container: maven:3.6-jdk-11
104+
runs-on: [ubuntu-latest]
105+
needs: get-version
106+
steps:
107+
- uses: actions/checkout@v2
108+
with:
109+
submodules: 'true'
110+
- name: Set up JDK 11
111+
uses: actions/setup-java@v1
112+
with:
113+
java-version: '11'
114+
java-package: jdk
115+
architecture: x64
116+
- uses: actions/setup-python@v2
117+
with:
118+
python-version: '3.6'
119+
architecture: 'x64'
120+
- uses: actions/cache@v2
121+
with:
122+
path: ~/.m2/repository
123+
key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }}
124+
restore-keys: |
125+
${{ runner.os }}-it-maven-
126+
- name: Publish java sdk
127+
env:
128+
VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
129+
GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }}
130+
GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }}
131+
MAVEN_SETTINGS: ${{ secrets.MAVEN_SETTINGS }}
132+
run: |
133+
echo -n "$GPG_PUBLIC_KEY" > /root/public-key
134+
echo -n "$GPG_PRIVATE_KEY" > /root/private-key
135+
mkdir -p /root/.m2/
136+
echo -n "$MAVEN_SETTINGS" > /root/.m2/settings.xml
137+
infra/scripts/publish-java-sdk.sh --revision ${VERSION_WITHOUT_PREFIX} --gpg-key-import-dir /root
138+
139+
publish-helm-charts:
140+
runs-on: ubuntu-latest
141+
needs: get-version
142+
env:
143+
HELM_VERSION: v2.17.0
144+
VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
145+
steps:
146+
- uses: actions/checkout@v2
147+
- uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
148+
with:
149+
version: '290.0.1'
150+
export_default_credentials: true
151+
project_id: ${{ secrets.GCP_PROJECT_ID }}
152+
service_account_key: ${{ secrets.GCP_SA_KEY }}
153+
- run: gcloud auth configure-docker --quiet
154+
- name: Remove previous Helm
155+
run: sudo rm -rf $(which helm)
156+
- name: Install Helm
157+
run: ./infra/scripts/install-helm.sh
158+
- name: Validate all version consistency
159+
run: ./infra/scripts/validate-helm-chart-versions.sh $VERSION_WITHOUT_PREFIX
160+
- name: Publish Helm charts
161+
run: ./infra/scripts/push-helm-charts.sh $VERSION_WITHOUT_PREFIX

‎.prow.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ presubmits:
44
spec:
55
containers:
66
- image: maven:3.6-jdk-11
7-
command: ["infra/scripts/test-core-ingestion.sh"]
7+
command: ["infra/scripts/test-java-core-ingestion.sh"]
88
resources:
99
requests:
1010
cpu: "2000m"
@@ -18,7 +18,7 @@ presubmits:
1818
spec:
1919
containers:
2020
- image: maven:3.6-jdk-8
21-
command: ["infra/scripts/test-core-ingestion.sh"]
21+
command: ["infra/scripts/test-java-core-ingestion.sh"]
2222
resources:
2323
requests:
2424
cpu: "2000m"
@@ -31,7 +31,7 @@ presubmits:
3131
spec:
3232
containers:
3333
- image: maven:3.6-jdk-11
34-
command: ["infra/scripts/test-serving.sh"]
34+
command: ["infra/scripts/test-java-serving.sh"]
3535
skip_branches:
3636
- ^v0\.(3|4)-branch$
3737

@@ -41,7 +41,7 @@ presubmits:
4141
spec:
4242
containers:
4343
- image: maven:3.6-jdk-8
44-
command: ["infra/scripts/test-serving.sh"]
44+
command: ["infra/scripts/test-java-serving.sh"]
4545
branches:
4646
- ^v0\.(3|4)-branch$
4747

0 commit comments

Comments
 (0)