Skip to content

Commit 6ef7852

Browse files
tokokotokoko
authored
chore: Install python dependencies with uv in workflows (#4086)
* install dependencies in unit-tests with uv Signed-off-by: tokoko <[email protected]> * install dependencies in unit-tests with uv Signed-off-by: tokoko <[email protected]> * enable caching, change linter job Signed-off-by: tokoko <[email protected]> * change local integration tests to uv Signed-off-by: tokoko <[email protected]> * change all installs to uv Signed-off-by: tokoko <[email protected]> * try adding uv cache Signed-off-by: tokoko <[email protected]> * fix lambda cache step name Signed-off-by: tokoko <[email protected]> * reenable caches for uv Signed-off-by: tokoko <[email protected]> * remove dangling cache step Signed-off-by: tokoko <[email protected]> --------- Signed-off-by: tokoko <[email protected]>
1 parent 9537946 commit 6ef7852

File tree

7 files changed

+78
-111
lines changed

7 files changed

+78
-111
lines changed

‎.github/workflows/linter.yml‎

Lines changed: 4 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -11,31 +11,17 @@ jobs:
1111
- uses: actions/checkout@v3
1212
- name: Setup Python
1313
id: setup-python
14-
uses: actions/setup-python@v3
14+
uses: actions/setup-python@v5
1515
with:
1616
python-version: "3.9"
1717
architecture: x64
18-
- name: Get pip cache dir
19-
id: pip-cache
20-
run: |
21-
echo "::set-output name=dir::$(pip cache dir)"
22-
- name: pip cache
23-
uses: actions/cache@v2
24-
with:
25-
path: |
26-
${{ steps.pip-cache.outputs.dir }}
27-
/opt/hostedtoolcache/Python
28-
/Users/runner/hostedtoolcache/Python
29-
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
30-
restore-keys: |
31-
${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
3218
- name: Upgrade pip version
3319
run: |
3420
pip install --upgrade "pip>=21.3.1,<23.2"
35-
- name: Install pip-tools
36-
run: pip install pip-tools
21+
- name: Install uv
22+
run: pip install uv
3723
- name: Install dependencies
3824
run: |
39-
make install-python-ci-dependencies
25+
make install-python-ci-dependencies-uv
4026
- name: Lint python
4127
run: make lint-python

‎.github/workflows/master_only.yml‎

Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ jobs:
8181
--health-timeout 5s
8282
--health-retries 5
8383
steps:
84-
- uses: actions/checkout@v3
84+
- uses: actions/checkout@v5
8585
- name: Setup Python
8686
id: setup-python
8787
uses: actions/setup-python@v3
@@ -106,27 +106,22 @@ jobs:
106106
aws-region: us-west-2
107107
- name: Use AWS CLI
108108
run: aws sts get-caller-identity
109-
- name: Get pip cache dir
110-
id: pip-cache
111-
run: |
112-
echo "::set-output name=dir::$(pip cache dir)"
113-
- name: pip cache
114-
uses: actions/cache@v2
115-
with:
116-
path: |
117-
${{ steps.pip-cache.outputs.dir }}
118-
/opt/hostedtoolcache/Python
119-
/Users/runner/hostedtoolcache/Python
120-
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
121-
restore-keys: |
122-
${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
123109
- name: Upgrade pip version
124110
run: |
125111
pip install --upgrade "pip>=21.3.1,<23.2"
126-
- name: Install pip-tools
127-
run: pip install pip-tools
112+
- name: Install uv
113+
run: pip install uv
114+
- name: Get uv cache dir
115+
id: uv-cache
116+
run: |
117+
echo "::set-output name=dir::$(uv cache dir)"
118+
- name: uv cache
119+
uses: actions/cache@v4
120+
with:
121+
path: ${{ steps.uv-cache.outputs.dir }}
122+
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
128123
- name: Install dependencies
129-
run: make install-python-ci-dependencies
124+
run: make install-python-ci-dependencies-uv
130125
- name: Setup Redis Cluster
131126
run: |
132127
docker pull vishnunair/docker-redis-cluster:latest

‎.github/workflows/nightly-ci.yml‎

Lines changed: 16 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ jobs:
3333
with:
3434
ref: master
3535
- name: Setup Python
36-
uses: actions/setup-python@v3
36+
uses: actions/setup-python@v5
3737
id: setup-python
3838
with:
3939
python-version: "3.9"
@@ -90,18 +90,18 @@ jobs:
9090
run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`"
9191
- name: Cache Public ECR Image
9292
id: lambda_python_3_9
93-
uses: actions/cache@v2
93+
uses: actions/cache@v4
9494
with:
9595
path: ~/cache
9696
key: lambda_python_3_9
9797
- name: Handle Cache Miss (pull public ECR image & save it to tar file)
98-
if: steps.cache-primes.outputs.cache-hit != 'true'
98+
if: steps.lambda_python_3_9.outputs.cache-hit != 'true'
9999
run: |
100100
mkdir -p ~/cache
101101
docker pull public.ecr.aws/lambda/python:3.9
102102
docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar
103103
- name: Handle Cache Hit (load docker image from tar file)
104-
if: steps.cache-primes.outputs.cache-hit == 'true'
104+
if: steps.lambda_python_3_9.outputs.cache-hit == 'true'
105105
run: |
106106
docker load -i ~/cache/lambda_python_3_9.tar
107107
- name: Build and push
@@ -145,7 +145,7 @@ jobs:
145145
ref: master
146146
submodules: recursive
147147
- name: Setup Python
148-
uses: actions/setup-python@v3
148+
uses: actions/setup-python@v5
149149
id: setup-python
150150
with:
151151
python-version: ${{ matrix.python-version }}
@@ -173,25 +173,20 @@ jobs:
173173
aws-region: us-west-2
174174
- name: Use AWS CLI
175175
run: aws sts get-caller-identity
176-
- name: Get pip cache dir
177-
id: pip-cache
178-
run: |
179-
echo "::set-output name=dir::$(pip cache dir)"
180-
- name: pip cache
181-
uses: actions/cache@v2
182-
with:
183-
path: |
184-
${{ steps.pip-cache.outputs.dir }}
185-
/opt/hostedtoolcache/Python
186-
/Users/runner/hostedtoolcache/Python
187-
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
188-
restore-keys: |
189-
${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
190176
- name: Upgrade pip version
191177
run: |
192178
pip install --upgrade "pip>=21.3.1,<23.2"
193-
- name: Install pip-tools
194-
run: pip install pip-tools
179+
- name: Install uv
180+
run: pip install uv
181+
- name: Get uv cache dir
182+
id: uv-cache
183+
run: |
184+
echo "::set-output name=dir::$(uv cache dir)"
185+
- name: uv cache
186+
uses: actions/cache@v4
187+
with:
188+
path: ${{ steps.uv-cache.outputs.dir }}
189+
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
195190
- name: Install apache-arrow on ubuntu
196191
if: matrix.os == 'ubuntu-latest'
197192
run: |

‎.github/workflows/pr_integration_tests.yml‎

Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ jobs:
110110
ref: refs/pull/${{ github.event.pull_request.number }}/merge
111111
submodules: recursive
112112
- name: Setup Python
113-
uses: actions/setup-python@v4
113+
uses: actions/setup-python@v5
114114
id: setup-python
115115
with:
116116
python-version: ${{ matrix.python-version }}
@@ -133,27 +133,22 @@ jobs:
133133
aws-region: us-west-2
134134
- name: Use AWS CLI
135135
run: aws sts get-caller-identity
136-
- name: Get pip cache dir
137-
id: pip-cache
138-
run: |
139-
echo "::set-output name=dir::$(pip cache dir)"
140-
- name: pip cache
141-
uses: actions/cache@v2
142-
with:
143-
path: |
144-
${{ steps.pip-cache.outputs.dir }}
145-
/opt/hostedtoolcache/Python
146-
/Users/runner/hostedtoolcache/Python
147-
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
148-
restore-keys: |
149-
${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
150136
- name: Upgrade pip version
151137
run: |
152138
pip install --upgrade "pip>=21.3.1,<23.2"
153-
- name: Install pip-tools
154-
run: pip install pip-tools
139+
- name: Install uv
140+
run: pip install uv
141+
- name: Get uv cache dir
142+
id: uv-cache
143+
run: |
144+
echo "::set-output name=dir::$(uv cache dir)"
145+
- name: uv cache
146+
uses: actions/cache@v4
147+
with:
148+
path: ${{ steps.uv-cache.outputs.dir }}
149+
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
155150
- name: Install dependencies
156-
run: make install-python-ci-dependencies
151+
run: make install-python-ci-dependencies-uv
157152
- name: Setup Redis Cluster
158153
run: |
159154
docker pull vishnunair/docker-redis-cluster:latest

‎.github/workflows/pr_local_integration_tests.yml‎

Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -33,32 +33,27 @@ jobs:
3333
ref: refs/pull/${{ github.event.pull_request.number }}/merge
3434
submodules: recursive
3535
- name: Setup Python
36-
uses: actions/setup-python@v3
36+
uses: actions/setup-python@v5
3737
id: setup-python
3838
with:
3939
python-version: ${{ matrix.python-version }}
4040
architecture: x64
41-
- name: Get pip cache dir
42-
id: pip-cache
43-
run: |
44-
echo "::set-output name=dir::$(pip cache dir)"
45-
- name: pip cache
46-
uses: actions/cache@v2
47-
with:
48-
path: |
49-
${{ steps.pip-cache.outputs.dir }}
50-
/opt/hostedtoolcache/Python
51-
/Users/runner/hostedtoolcache/Python
52-
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
53-
restore-keys: |
54-
${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
5541
- name: Upgrade pip version
5642
run: |
5743
pip install --upgrade "pip>=21.3.1,<23.2"
58-
- name: Install pip-tools
59-
run: pip install pip-tools
44+
- name: Install uv
45+
run: pip install uv
46+
- name: Get uv cache dir
47+
id: uv-cache
48+
run: |
49+
echo "::set-output name=dir::$(uv cache dir)"
50+
- name: uv cache
51+
uses: actions/cache@v4
52+
with:
53+
path: ${{ steps.uv-cache.outputs.dir }}
54+
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
6055
- name: Install dependencies
61-
run: make install-python-ci-dependencies
56+
run: make install-python-ci-dependencies-uv
6257
- name: Test local integration tests
6358
if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak
6459
run: make test-python-integration-local

‎.github/workflows/unit_tests.yml‎

Lines changed: 14 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -19,31 +19,27 @@ jobs:
1919
- uses: actions/checkout@v3
2020
- name: Setup Python
2121
id: setup-python
22-
uses: actions/setup-python@v3
22+
uses: actions/setup-python@v5
2323
with:
2424
python-version: ${{ matrix.python-version }}
2525
architecture: x64
26-
- name: Get pip cache dir
27-
id: pip-cache
28-
run: |
29-
echo "::set-output name=dir::$(pip cache dir)"
30-
- name: pip cache
31-
uses: actions/cache@v2
32-
with:
33-
path: |
34-
${{ steps.pip-cache.outputs.dir }}
35-
/opt/hostedtoolcache/Python
36-
/Users/runner/hostedtoolcache/Python
37-
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
38-
restore-keys: |
39-
${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
4026
- name: Upgrade pip version
4127
run: |
4228
pip install --upgrade "pip>=21.3.1,<23.2"
43-
- name: Install pip-tools
44-
run: pip install pip-tools
29+
- name: Install uv
30+
run: |
31+
pip install uv
32+
- name: Get uv cache dir
33+
id: uv-cache
34+
run: |
35+
echo "::set-output name=dir::$(uv cache dir)"
36+
- name: uv cache
37+
uses: actions/cache@v4
38+
with:
39+
path: ${{ steps.uv-cache.outputs.dir }}
40+
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
4541
- name: Install dependencies
46-
run: make install-python-ci-dependencies
42+
run: make install-python-ci-dependencies-uv
4743
- name: Test Python
4844
run: make test-python-unit
4945

‎Makefile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,11 @@ install-python-ci-dependencies:
4141
pip install --no-deps -e .
4242
python setup.py build_python_protos --inplace
4343

44+
install-python-ci-dependencies-uv:
45+
uv pip sync --system sdk/python/requirements/py$(PYTHON)-ci-requirements.txt
46+
uv pip install --system --no-deps -e .
47+
python setup.py build_python_protos --inplace
48+
4449
lock-python-ci-dependencies:
4550
python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py$(PYTHON)-ci-requirements.txt
4651

0 commit comments

Comments
 (0)