Skip to content

Commit 80c74cf

Browse files
laipz8200gemini-code-assist[bot]
laipz8200
and
gemini-code-assist[bot]
authored
test: Consolidate API CI test runner (langgenius#29440)
Signed-off-by: -LAN- <[email protected]> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
1 parent 1e47ffb commit 80c74cf

File tree

17 files changed

+187
-115
lines changed

17 files changed

+187
-115
lines changed

‎.coveragerc‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
[run]
2+
omit =
3+
api/tests/*
4+
api/migrations/*
5+
api/core/rag/datasource/vdb/*

‎.github/workflows/api-tests.yml‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -71,18 +71,18 @@ jobs:
7171
run: |
7272
cp api/tests/integration_tests/.env.example api/tests/integration_tests/.env
7373
74-
- name: Run Workflow
75-
run: uv run --project api bash dev/pytest/pytest_workflow.sh
76-
77-
- name: Run Tool
78-
run: uv run --project api bash dev/pytest/pytest_tools.sh
79-
80-
- name: Run TestContainers
81-
run: uv run --project api bash dev/pytest/pytest_testcontainers.sh
82-
83-
- name: Run Unit tests
74+
- name: Run API Tests
75+
env:
76+
STORAGE_TYPE: opendal
77+
OPENDAL_SCHEME: fs
78+
OPENDAL_FS_ROOT: /tmp/dify-storage
8479
run: |
85-
uv run --project api bash dev/pytest/pytest_unit_tests.sh
80+
uv run --project api pytest \
81+
--timeout "${PYTEST_TIMEOUT:-180}" \
82+
api/tests/integration_tests/workflow \
83+
api/tests/integration_tests/tools \
84+
api/tests/test_containers_integration_tests \
85+
api/tests/unit_tests
8686
8787
- name: Coverage Summary
8888
run: |
@@ -94,4 +94,3 @@ jobs:
9494
echo "### Test Coverage Summary :test_tube:" >> $GITHUB_STEP_SUMMARY
9595
echo "Total Coverage: ${TOTAL_COVERAGE}%" >> $GITHUB_STEP_SUMMARY
9696
uv run --project api coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
97-

‎api/extensions/ext_blueprints.py‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,21 @@
99
EXPOSED_HEADERS: tuple[str, ...] = ("X-Version", "X-Env", "X-Trace-Id")
1010

1111

12-
def init_app(app: DifyApp):
13-
# register blueprint routers
12+
def _apply_cors_once(bp, /, **cors_kwargs):
13+
"""Make CORS idempotent so blueprints can be reused across multiple app instances."""
14+
15+
if getattr(bp, "_dify_cors_applied", False):
16+
return
1417

1518
from flask_cors import CORS
1619

20+
CORS(bp, **cors_kwargs)
21+
bp._dify_cors_applied = True
22+
23+
24+
def init_app(app: DifyApp):
25+
# register blueprint routers
26+
1727
from controllers.console import bp as console_app_bp
1828
from controllers.files import bp as files_bp
1929
from controllers.inner_api import bp as inner_api_bp
@@ -22,15 +32,15 @@ def init_app(app: DifyApp):
2232
from controllers.trigger import bp as trigger_bp
2333
from controllers.web import bp as web_bp
2434

25-
CORS(
35+
_apply_cors_once(
2636
service_api_bp,
2737
allow_headers=list(SERVICE_API_HEADERS),
2838
methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"],
2939
expose_headers=list(EXPOSED_HEADERS),
3040
)
3141
app.register_blueprint(service_api_bp)
3242

33-
CORS(
43+
_apply_cors_once(
3444
web_bp,
3545
resources={r"/*": {"origins": dify_config.WEB_API_CORS_ALLOW_ORIGINS}},
3646
supports_credentials=True,
@@ -40,7 +50,7 @@ def init_app(app: DifyApp):
4050
)
4151
app.register_blueprint(web_bp)
4252

43-
CORS(
53+
_apply_cors_once(
4454
console_app_bp,
4555
resources={r"/*": {"origins": dify_config.CONSOLE_CORS_ALLOW_ORIGINS}},
4656
supports_credentials=True,
@@ -50,7 +60,7 @@ def init_app(app: DifyApp):
5060
)
5161
app.register_blueprint(console_app_bp)
5262

53-
CORS(
63+
_apply_cors_once(
5464
files_bp,
5565
allow_headers=list(FILES_HEADERS),
5666
methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"],
@@ -62,7 +72,7 @@ def init_app(app: DifyApp):
6272
app.register_blueprint(mcp_bp)
6373

6474
# Register trigger blueprint with CORS for webhook calls
65-
CORS(
75+
_apply_cors_once(
6676
trigger_bp,
6777
allow_headers=["Content-Type", "Authorization", "X-App-Code"],
6878
methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH", "HEAD"],

‎api/pytest.ini‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[pytest]
2-
addopts = --cov=./api --cov-report=json --cov-report=xml
2+
addopts = --cov=./api --cov-report=json
33
env =
44
ANTHROPIC_API_KEY = sk-ant-api11-IamNotARealKeyJustForMockTestKawaiiiiiiiiii-NotBaka-ASkksz
55
AZURE_OPENAI_API_BASE = https://difyai-openai.openai.azure.com

‎api/tests/integration_tests/conftest.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import os
12
import pathlib
23
import random
34
import secrets
@@ -32,6 +33,10 @@ def _load_env():
3233

3334

3435
_load_env()
36+
# Override storage root to tmp to avoid polluting repo during local runs
37+
os.environ["OPENDAL_FS_ROOT"] = "/tmp/dify-storage"
38+
os.environ.setdefault("STORAGE_TYPE", "opendal")
39+
os.environ.setdefault("OPENDAL_SCHEME", "fs")
3540

3641
_CACHED_APP = create_app()
3742

‎api/tests/test_containers_integration_tests/conftest.py‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -138,9 +138,9 @@ def start_containers_with_env(self):
138138
logger.warning("Failed to create plugin database: %s", e)
139139

140140
# Set up storage environment variables
141-
os.environ["STORAGE_TYPE"] = "opendal"
142-
os.environ["OPENDAL_SCHEME"] = "fs"
143-
os.environ["OPENDAL_FS_ROOT"] = "storage"
141+
os.environ.setdefault("STORAGE_TYPE", "opendal")
142+
os.environ.setdefault("OPENDAL_SCHEME", "fs")
143+
os.environ.setdefault("OPENDAL_FS_ROOT", "/tmp/dify-storage")
144144

145145
# Start Redis container for caching and session management
146146
# Redis is used for storing session data, cache entries, and temporary data
@@ -348,6 +348,13 @@ def _create_app_with_containers() -> Flask:
348348
"""
349349
logger.info("Creating Flask application with test container configuration...")
350350

351+
# Ensure Redis client reconnects to the containerized Redis (no auth)
352+
from extensions import ext_redis
353+
354+
ext_redis.redis_client._client = None
355+
os.environ["REDIS_USERNAME"] = ""
356+
os.environ["REDIS_PASSWORD"] = ""
357+
351358
# Re-create the config after environment variables have been set
352359
from configs import dify_config
353360

@@ -486,3 +493,29 @@ def db_session_with_containers(flask_app_with_containers) -> Generator[Session,
486493
finally:
487494
session.close()
488495
logger.debug("Database session closed")
496+
497+
498+
@pytest.fixture(scope="package", autouse=True)
499+
def mock_ssrf_proxy_requests():
500+
"""
501+
Avoid outbound network during containerized tests by stubbing SSRF proxy helpers.
502+
"""
503+
504+
from unittest.mock import patch
505+
506+
import httpx
507+
508+
def _fake_request(method, url, **kwargs):
509+
request = httpx.Request(method=method, url=url)
510+
return httpx.Response(200, request=request, content=b"")
511+
512+
with (
513+
patch("core.helper.ssrf_proxy.make_request", side_effect=_fake_request),
514+
patch("core.helper.ssrf_proxy.get", side_effect=lambda url, **kw: _fake_request("GET", url, **kw)),
515+
patch("core.helper.ssrf_proxy.post", side_effect=lambda url, **kw: _fake_request("POST", url, **kw)),
516+
patch("core.helper.ssrf_proxy.put", side_effect=lambda url, **kw: _fake_request("PUT", url, **kw)),
517+
patch("core.helper.ssrf_proxy.patch", side_effect=lambda url, **kw: _fake_request("PATCH", url, **kw)),
518+
patch("core.helper.ssrf_proxy.delete", side_effect=lambda url, **kw: _fake_request("DELETE", url, **kw)),
519+
patch("core.helper.ssrf_proxy.head", side_effect=lambda url, **kw: _fake_request("HEAD", url, **kw)),
520+
):
521+
yield

‎api/tests/test_containers_integration_tests/libs/broadcast_channel/redis/test_sharded_channel.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -240,8 +240,7 @@ def consumer_thread() -> set[bytes]:
240240
for future in as_completed(producer_futures, timeout=30.0):
241241
sent_msgs.update(future.result())
242242

243-
subscription.close()
244-
consumer_received_msgs = consumer_future.result(timeout=30.0)
243+
consumer_received_msgs = consumer_future.result(timeout=60.0)
245244

246245
assert sent_msgs == consumer_received_msgs
247246

‎api/tests/unit_tests/conftest.py‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,29 @@
2626
redis_mock.hdel = MagicMock()
2727
redis_mock.incr = MagicMock(return_value=1)
2828

29+
# Ensure OpenDAL fs writes to tmp to avoid polluting workspace
30+
os.environ.setdefault("OPENDAL_SCHEME", "fs")
31+
os.environ.setdefault("OPENDAL_FS_ROOT", "/tmp/dify-storage")
32+
os.environ.setdefault("STORAGE_TYPE", "opendal")
33+
2934
# Add the API directory to Python path to ensure proper imports
3035
import sys
3136

3237
sys.path.insert(0, PROJECT_DIR)
3338

34-
# apply the mock to the Redis client in the Flask app
3539
from extensions import ext_redis
3640

37-
redis_patcher = patch.object(ext_redis, "redis_client", redis_mock)
38-
redis_patcher.start()
41+
42+
def _patch_redis_clients_on_loaded_modules():
43+
"""Ensure any module-level redis_client references point to the shared redis_mock."""
44+
45+
import sys
46+
47+
for module in list(sys.modules.values()):
48+
if module is None:
49+
continue
50+
if hasattr(module, "redis_client"):
51+
module.redis_client = redis_mock
3952

4053

4154
@pytest.fixture
@@ -49,6 +62,15 @@ def _provide_app_context(app: Flask):
4962
yield
5063

5164

65+
@pytest.fixture(autouse=True)
66+
def _patch_redis_clients():
67+
"""Patch redis_client to MagicMock only for unit test executions."""
68+
69+
with patch.object(ext_redis, "redis_client", redis_mock):
70+
_patch_redis_clients_on_loaded_modules()
71+
yield
72+
73+
5274
@pytest.fixture(autouse=True)
5375
def reset_redis_mock():
5476
"""reset the Redis mock before each test"""
@@ -63,3 +85,20 @@ def reset_redis_mock():
6385
redis_mock.hgetall.return_value = {}
6486
redis_mock.hdel.return_value = None
6587
redis_mock.incr.return_value = 1
88+
89+
# Keep any imported modules pointing at the mock between tests
90+
_patch_redis_clients_on_loaded_modules()
91+
92+
93+
@pytest.fixture(autouse=True)
94+
def reset_secret_key():
95+
"""Ensure SECRET_KEY-dependent logic sees an empty config value by default."""
96+
97+
from configs import dify_config
98+
99+
original = dify_config.SECRET_KEY
100+
dify_config.SECRET_KEY = ""
101+
try:
102+
yield
103+
finally:
104+
dify_config.SECRET_KEY = original

‎api/tests/unit_tests/oss/__mock/base.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,9 @@ def get_example_bucket() -> str:
1414

1515

1616
def get_opendal_bucket() -> str:
17-
return "./dify"
17+
import os
18+
19+
return os.environ.get("OPENDAL_FS_ROOT", "/tmp/dify-storage")
1820

1921

2022
def get_example_filename() -> str:

‎api/tests/unit_tests/oss/opendal/test_opendal.py‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,20 +21,16 @@ def setup_method(self, *args, **kwargs):
2121
)
2222

2323
@pytest.fixture(scope="class", autouse=True)
24-
def teardown_class(self, request):
24+
def teardown_class(self):
2525
"""Clean up after all tests in the class."""
2626

27-
def cleanup():
28-
folder = Path(get_opendal_bucket())
29-
if folder.exists() and folder.is_dir():
30-
for item in folder.iterdir():
31-
if item.is_file():
32-
item.unlink()
33-
elif item.is_dir():
34-
item.rmdir()
35-
folder.rmdir()
36-
37-
return cleanup()
27+
yield
28+
29+
folder = Path(get_opendal_bucket())
30+
if folder.exists() and folder.is_dir():
31+
import shutil
32+
33+
shutil.rmtree(folder, ignore_errors=True)
3834

3935
def test_save_and_exists(self):
4036
"""Test saving data and checking existence."""

0 commit comments

Comments
 (0)