Skip to content

Commit c2193a3

Browse files
fnesvedafnesveda
authored
Sync project setup with Python SDK (apify#119)
1 parent 0f65189 commit c2193a3

12 files changed

Lines changed: 133 additions & 17 deletions

File tree

‎.flake8‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,11 @@ filename =
55
./src/*.py,
66
./tests/*.py,
77
./setup.py
8-
max_line_length = 150
8+
per-file-ignores =
9+
docs/*: D
10+
scripts/*: D
11+
tests/*: D
12+
913
# Google docstring convention + D204 & D401
1014
docstring-convention = all
1115
ignore =
@@ -20,9 +24,9 @@ ignore =
2024
D409
2125
D413
2226
U101
27+
28+
max_line_length = 150
2329
unused-arguments-ignore-overload-functions = True
2430
unused-arguments-ignore-stub-functions = True
25-
per-file-ignores =
26-
docs/*: D
27-
scripts/*: D
28-
tests/*: D
31+
pytest-fixture-no-parentheses = True
32+
pytest-mark-no-parentheses = True

‎.github/workflows/integration_tests.yaml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
name: Integration tests
2+
3+
on:
4+
workflow_call:
5+
secrets:
6+
APIFY_TEST_USER_API_TOKEN:
7+
description: API token of the Python SDK testing user on Apify
8+
required: true
9+
10+
concurrency: # This is to make sure that only one run of this workflow is running at the same time, to not overshoot the test user limits
11+
group: integration_tests
12+
13+
jobs:
14+
integration_tests:
15+
name: Run integration tests
16+
runs-on: ubuntu-latest
17+
strategy:
18+
matrix:
19+
python-version: ["3.8", "3.9", "3.10", "3.11"]
20+
max-parallel: 1 # no concurrency on this level, to not overshoot the test user limits
21+
22+
steps:
23+
- name: Checkout repository
24+
uses: actions/checkout@v3
25+
26+
- name: Set up Python ${{ matrix.python-version }}
27+
uses: actions/setup-python@v4
28+
with:
29+
python-version: ${{ matrix.python-version }}
30+
31+
- name: Install dependencies
32+
run: make install-dev
33+
34+
- name: Run integration tests
35+
run: make INTEGRATION_TESTS_CONCURRENCY=8 integration-tests
36+
env:
37+
APIFY_TEST_USER_API_TOKEN: ${{ secrets.APIFY_TEST_USER_API_TOKEN }}

‎.github/workflows/release.yaml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,18 @@ jobs:
3232
name: Run unit tests
3333
uses: ./.github/workflows/unit_tests.yaml
3434

35+
integration_tests:
36+
name: Run integration tests
37+
uses: ./.github/workflows/integration_tests.yaml
38+
secrets: inherit
39+
3540
check_docs:
3641
name: Check whether the documentation is up to date
3742
uses: ./.github/workflows/check_docs.yaml
3843

3944
deploy:
4045
name: Publish to PyPI
41-
needs: [lint_and_type_checks, unit_tests, check_docs]
46+
needs: [lint_and_type_checks, unit_tests, integration_tests, check_docs]
4247
runs-on: ubuntu-latest
4348

4449
steps:

‎.github/workflows/run_checks.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,8 @@ jobs:
1717
name: Check whether the documentation is up to date
1818
uses: ./.github/workflows/check_docs.yaml
1919

20+
integration_tests:
21+
name: Run integration tests
22+
needs: [lint_and_type_checks, unit_tests, check_docs]
23+
uses: ./.github/workflows/integration_tests.yaml
24+
secrets: inherit

‎.pre-commit-config.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ repos:
1919
language: system
2020
pass_filenames: false
2121

22-
- id: unit-test
22+
- id: unit-tests
2323
name: "Run unit tests"
2424
entry: "make unit-tests"
2525
language: system

‎Makefile‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
.PHONY: clean install-dev lint unit-tests type-check check-code format docs check-docs check-async-docstrings fix-async-docstrings check-changelog-entry
1+
.PHONY: clean install-dev lint unit-tests integration-tests type-check check-code format docs check-docs check-async-docstrings fix-async-docstrings check-changelog-entry
2+
3+
# This is default for local testing, but GitHub workflows override it to a higher value in CI
4+
INTEGRATION_TESTS_CONCURRENCY = 1
25

36
clean:
47
rm -rf build dist .mypy_cache .pytest_cache src/*.egg-info __pycache__
@@ -15,6 +18,9 @@ lint:
1518
unit-tests:
1619
python3 -m pytest -n auto -ra tests/unit
1720

21+
integration-tests:
22+
python3 -m pytest -n $(INTEGRATION_TESTS_CONCURRENCY) -ra tests/integration
23+
1824
type-check:
1925
python3 -m mypy
2026

‎setup.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,15 +53,21 @@
5353
packages=find_packages(where='src'),
5454
package_data={'apify_client': ['py.typed']},
5555
python_requires='>=3.8',
56-
install_requires=['httpx ~= 0.23.0'],
56+
install_requires=[
57+
'httpx ~= 0.23.0',
58+
],
5759
extras_require={
5860
'dev': [
5961
'autopep8 ~= 2.0.1',
6062
'flake8 ~= 6.0.0',
6163
'flake8-bugbear ~= 23.1.20',
6264
'flake8-commas ~= 2.1.0',
65+
'flake8-comprehensions ~= 3.10.1',
66+
'flake8-datetimez ~= 20.10.0',
6367
'flake8-docstrings ~= 1.7.0',
6468
'flake8-isort ~= 6.0.0',
69+
'flake8-noqa ~= 1.3.0',
70+
'flake8-pytest-style ~= 1.7.2',
6571
'flake8-quotes ~= 3.3.1',
6672
'flake8-unused-arguments ~= 0.0.13',
6773
'isort ~= 5.12.0',
@@ -70,6 +76,7 @@
7076
'pre-commit ~= 3.0.1',
7177
'pytest ~= 7.2.0',
7278
'pytest-asyncio ~= 0.20.3',
79+
'pytest-only ~= 2.0.0',
7380
'pytest-randomly ~= 3.12.0',
7481
'pytest-timeout ~= 2.1.0',
7582
'pytest-xdist ~= 3.2.0',

‎src/apify_client/clients/base/actor_job_base_client.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import asyncio
22
import math
33
import time
4-
from datetime import datetime
4+
from datetime import datetime, timezone
55
from typing import Dict, Optional
66

77
from ..._errors import ApifyApiError
@@ -20,7 +20,7 @@ class ActorJobBaseClient(ResourceClient):
2020
"""Base sub-client class for actor runs and actor builds."""
2121

2222
def _wait_for_finish(self, wait_secs: Optional[int] = None) -> Optional[Dict]:
23-
started_at = datetime.now()
23+
started_at = datetime.now(timezone.utc)
2424
should_repeat = True
2525
job: Optional[Dict] = None
2626
seconds_elapsed = 0
@@ -38,7 +38,7 @@ def _wait_for_finish(self, wait_secs: Optional[int] = None) -> Optional[Dict]:
3838
)
3939
job = _parse_date_fields(_pluck_data(response.json()))
4040

41-
seconds_elapsed = math.floor(((datetime.now() - started_at).total_seconds()))
41+
seconds_elapsed = math.floor(((datetime.now(timezone.utc) - started_at).total_seconds()))
4242
if (
4343
ActorJobStatus(job['status'])._is_terminal or (wait_secs is not None and seconds_elapsed >= wait_secs)
4444
):
@@ -77,7 +77,7 @@ class ActorJobBaseClientAsync(ResourceClientAsync):
7777
"""Base async sub-client class for actor runs and actor builds."""
7878

7979
async def _wait_for_finish(self, wait_secs: Optional[int] = None) -> Optional[Dict]:
80-
started_at = datetime.now()
80+
started_at = datetime.now(timezone.utc)
8181
should_repeat = True
8282
job: Optional[Dict] = None
8383
seconds_elapsed = 0
@@ -95,7 +95,7 @@ async def _wait_for_finish(self, wait_secs: Optional[int] = None) -> Optional[Di
9595
)
9696
job = _parse_date_fields(_pluck_data(response.json()))
9797

98-
seconds_elapsed = math.floor(((datetime.now() - started_at).total_seconds()))
98+
seconds_elapsed = math.floor(((datetime.now(timezone.utc) - started_at).total_seconds()))
9999
if (
100100
ActorJobStatus(job['status'])._is_terminal or (wait_secs is not None and seconds_elapsed >= wait_secs)
101101
):

‎tests/integration/conftest.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
import os
2+
3+
import pytest
4+
5+
from apify_client import ApifyClient, ApifyClientAsync
6+
7+
TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN'
8+
API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL'
9+
10+
11+
@pytest.fixture
12+
def apify_client() -> ApifyClient:
13+
api_token = os.getenv(TOKEN_ENV_VAR)
14+
api_url = os.getenv(API_URL_ENV_VAR)
15+
16+
if not api_token:
17+
raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!')
18+
19+
return ApifyClient(api_token, api_url=api_url)
20+
21+
22+
# This fixture can't be session-scoped,
23+
# because then you start getting `RuntimeError: Event loop is closed` errors,
24+
# because `httpx.AsyncClient` in `ApifyClientAsync` tries to reuse the same event loop across requests,
25+
# but `pytest-asyncio` closes the event loop after each test,
26+
# and uses a new one for the next test.
27+
@pytest.fixture
28+
def apify_client_async() -> ApifyClientAsync:
29+
api_token = os.getenv(TOKEN_ENV_VAR)
30+
api_url = os.getenv(API_URL_ENV_VAR)
31+
32+
if not api_token:
33+
raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!')
34+
35+
return ApifyClientAsync(api_token, api_url=api_url)

‎tests/integration/test_basic.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
from apify_client import ApifyClient, ApifyClientAsync
2+
3+
4+
class TestBasicSync:
5+
def test_basic(self, apify_client: ApifyClient) -> None:
6+
me = apify_client.user('me').get()
7+
assert me is not None
8+
assert me.get('id') is not None
9+
assert me.get('username') is not None
10+
11+
12+
class TestBasicAsync:
13+
async def test_basic(self, apify_client_async: ApifyClientAsync) -> None:
14+
me = await apify_client_async.user('me').get()
15+
assert me is not None
16+
assert me.get('id') is not None
17+
assert me.get('username') is not None

0 commit comments

Comments
 (0)