Skip to content

Commit 03b9134

Browse files
AA-TurnerAA-Turner
authored
Short-circuit HTTP requests in tests (#12772)
1 parent 070f2c1 commit 03b9134

File tree

12 files changed

+91
-51
lines changed

12 files changed

+91
-51
lines changed

‎sphinx/environment/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,7 @@ def __init__(self, app: Sphinx) -> None:
248248
self.dlfiles: DownloadFiles = DownloadFiles()
249249

250250
# the original URI for images
251-
self.original_image_uri: dict[str, str] = {}
251+
self.original_image_uri: dict[_StrPath, str] = {}
252252

253253
# temporary data storage while reading a document
254254
self.temp_data: dict[str, Any] = {}

‎sphinx/environment/adapters/asset.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
"""Assets adapter for sphinx.environment."""
22

33
from sphinx.environment import BuildEnvironment
4+
from sphinx.util._pathlib import _StrPath
45

56

67
class ImageAdapter:
@@ -9,7 +10,7 @@ def __init__(self, env: BuildEnvironment) -> None:
910

1011
def get_original_image_uri(self, name: str) -> str:
1112
"""Get the original image URI."""
12-
while name in self.env.original_image_uri:
13-
name = self.env.original_image_uri[name]
13+
while _StrPath(name) in self.env.original_image_uri:
14+
name = self.env.original_image_uri[_StrPath(name)]
1415

1516
return name

‎sphinx/transforms/post_transforms/images.py‎

Lines changed: 59 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,15 @@
66
import re
77
from hashlib import sha1
88
from math import ceil
9+
from pathlib import Path
910
from typing import TYPE_CHECKING, Any
1011

1112
from docutils import nodes
1213

1314
from sphinx.locale import __
1415
from sphinx.transforms import SphinxTransform
1516
from sphinx.util import logging, requests
17+
from sphinx.util._pathlib import _StrPath
1618
from sphinx.util.http_date import epoch_to_rfc1123, rfc1123_to_epoch
1719
from sphinx.util.images import get_image_extension, guess_mimetype, parse_data_uri
1820
from sphinx.util.osutil import ensuredir
@@ -65,50 +67,58 @@ def handle(self, node: nodes.image) -> None:
6567
basename = CRITICAL_PATH_CHAR_RE.sub("_", basename)
6668

6769
uri_hash = sha1(node['uri'].encode(), usedforsecurity=False).hexdigest()
68-
ensuredir(os.path.join(self.imagedir, uri_hash))
69-
path = os.path.join(self.imagedir, uri_hash, basename)
70-
71-
headers = {}
72-
if os.path.exists(path):
73-
timestamp: float = ceil(os.stat(path).st_mtime)
74-
headers['If-Modified-Since'] = epoch_to_rfc1123(timestamp)
75-
76-
config = self.app.config
77-
r = requests.get(
78-
node['uri'], headers=headers,
79-
_user_agent=config.user_agent,
80-
_tls_info=(config.tls_verify, config.tls_cacerts),
81-
)
82-
if r.status_code >= 400:
83-
logger.warning(__('Could not fetch remote image: %s [%d]'),
84-
node['uri'], r.status_code)
85-
else:
86-
self.app.env.original_image_uri[path] = node['uri']
87-
88-
if r.status_code == 200:
89-
with open(path, 'wb') as f:
90-
f.write(r.content)
91-
92-
last_modified = r.headers.get('last-modified')
93-
if last_modified:
94-
timestamp = rfc1123_to_epoch(last_modified)
95-
os.utime(path, (timestamp, timestamp))
96-
97-
mimetype = guess_mimetype(path, default='*')
98-
if mimetype != '*' and os.path.splitext(basename)[1] == '':
99-
# append a suffix if URI does not contain suffix
100-
ext = get_image_extension(mimetype)
101-
newpath = os.path.join(self.imagedir, uri_hash, basename + ext)
102-
os.replace(path, newpath)
103-
self.app.env.original_image_uri.pop(path)
104-
self.app.env.original_image_uri[newpath] = node['uri']
105-
path = newpath
106-
node['candidates'].pop('?')
107-
node['candidates'][mimetype] = path
108-
node['uri'] = path
109-
self.app.env.images.add_file(self.env.docname, path)
70+
path = Path(self.imagedir, uri_hash, basename)
71+
path.parent.mkdir(parents=True, exist_ok=True)
72+
self._download_image(node, path)
73+
11074
except Exception as exc:
111-
logger.warning(__('Could not fetch remote image: %s [%s]'), node['uri'], exc)
75+
msg = __('Could not fetch remote image: %s [%s]')
76+
logger.warning(msg, node['uri'], exc)
77+
78+
def _download_image(self, node: nodes.image, path: Path) -> None:
79+
headers = {}
80+
if path.exists():
81+
timestamp: float = ceil(path.stat().st_mtime)
82+
headers['If-Modified-Since'] = epoch_to_rfc1123(timestamp)
83+
84+
config = self.app.config
85+
r = requests.get(
86+
node['uri'], headers=headers,
87+
_user_agent=config.user_agent,
88+
_tls_info=(config.tls_verify, config.tls_cacerts),
89+
)
90+
if r.status_code >= 400:
91+
msg = __('Could not fetch remote image: %s [%d]')
92+
logger.warning(msg, node['uri'], r.status_code)
93+
else:
94+
self.app.env.original_image_uri[_StrPath(path)] = node['uri']
95+
96+
if r.status_code == 200:
97+
path.write_bytes(r.content)
98+
if last_modified := r.headers.get('Last-Modified'):
99+
timestamp = rfc1123_to_epoch(last_modified)
100+
os.utime(path, (timestamp, timestamp))
101+
102+
self._process_image(node, path)
103+
104+
def _process_image(self, node: nodes.image, path: Path) -> None:
105+
str_path = _StrPath(path)
106+
self.app.env.original_image_uri[str_path] = node['uri']
107+
108+
mimetype = guess_mimetype(path, default='*')
109+
if mimetype != '*' and path.suffix == '':
110+
# append a suffix if URI does not contain suffix
111+
ext = get_image_extension(mimetype) or ''
112+
with_ext = path.with_name(path.name + ext)
113+
os.replace(path, with_ext)
114+
self.app.env.original_image_uri.pop(str_path)
115+
self.app.env.original_image_uri[_StrPath(with_ext)] = node['uri']
116+
path = with_ext
117+
path_str = str(path)
118+
node['candidates'].pop('?')
119+
node['candidates'][mimetype] = path_str
120+
node['uri'] = path_str
121+
self.app.env.images.add_file(self.env.docname, path_str)
112122

113123

114124
class DataURIExtractor(BaseImageConverter):
@@ -130,16 +140,17 @@ def handle(self, node: nodes.image) -> None:
130140

131141
ensuredir(os.path.join(self.imagedir, 'embeded'))
132142
digest = sha1(image.data, usedforsecurity=False).hexdigest()
133-
path = os.path.join(self.imagedir, 'embeded', digest + ext)
143+
path = _StrPath(self.imagedir, 'embeded', digest + ext)
134144
self.app.env.original_image_uri[path] = node['uri']
135145

136146
with open(path, 'wb') as f:
137147
f.write(image.data)
138148

149+
path_str = str(path)
139150
node['candidates'].pop('?')
140-
node['candidates'][image.mimetype] = path
141-
node['uri'] = path
142-
self.app.env.images.add_file(self.env.docname, path)
151+
node['candidates'][image.mimetype] = path_str
152+
node['uri'] = path_str
153+
self.app.env.images.add_file(self.env.docname, path_str)
143154

144155

145156
def get_filename_for(filename: str, mimetype: str) -> str:
@@ -258,7 +269,7 @@ def handle(self, node: nodes.image) -> None:
258269
node['candidates'][_to] = destpath
259270
node['uri'] = destpath
260271

261-
self.env.original_image_uri[destpath] = srcpath
272+
self.env.original_image_uri[_StrPath(destpath)] = srcpath
262273
self.env.images.add_file(self.env.docname, destpath)
263274

264275
def convert(self, _from: str, _to: str) -> bool:

‎tests/conftest.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import os
55
import sys
66
from pathlib import Path
7+
from types import SimpleNamespace
78
from typing import TYPE_CHECKING
89

910
import docutils
@@ -60,3 +61,20 @@ def _cleanup_docutils() -> Iterator[None]:
6061
sys.path[:] = saved_path
6162

6263
_clean_up_global_state()
64+
65+
66+
@pytest.fixture
67+
def _http_teapot(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
68+
"""Short-circuit HTTP requests.
69+
70+
Windows takes too long to fail on connections, hence this fixture.
71+
"""
72+
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/418
73+
response = SimpleNamespace(status_code=418)
74+
75+
def _request(*args, **kwargs):
76+
return response
77+
78+
with monkeypatch.context() as m:
79+
m.setattr('sphinx.util.requests._Session.request', _request)
80+
yield

‎tests/test_builders/test_build.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ def test_numbered_circular_toctree(app):
100100
) in warnings
101101

102102

103+
@pytest.mark.usefixtures('_http_teapot')
103104
@pytest.mark.sphinx('dummy', testroot='images')
104105
def test_image_glob(app):
105106
app.build(force_all=True)

‎tests/test_builders/test_build_epub.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,7 @@ def test_xml_name_pattern_check():
473473
assert not _XML_NAME_PATTERN.match('1bfda21')
474474

475475

476+
@pytest.mark.usefixtures('_http_teapot')
476477
@pytest.mark.sphinx('epub', testroot='images')
477478
def test_copy_images(app):
478479
app.build()

‎tests/test_builders/test_build_html.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,7 @@ def test_html_inventory(app):
266266
)
267267

268268

269+
@pytest.mark.usefixtures('_http_teapot')
269270
@pytest.mark.sphinx(
270271
'html',
271272
testroot='images',

‎tests/test_builders/test_build_html_image.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import pytest
66

77

8+
@pytest.mark.usefixtures('_http_teapot')
89
@pytest.mark.sphinx('html', testroot='images')
910
def test_html_remote_images(app):
1011
app.build(force_all=True)
@@ -77,6 +78,7 @@ def test_html_scaled_image_link(app):
7778
)
7879

7980

81+
@pytest.mark.usefixtures('_http_teapot')
8082
@pytest.mark.sphinx('html', testroot='images')
8183
def test_copy_images(app):
8284
app.build()

‎tests/test_builders/test_build_latex.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2157,6 +2157,7 @@ def test_latex_code_role(app):
21572157
) in content
21582158

21592159

2160+
@pytest.mark.usefixtures('_http_teapot')
21602161
@pytest.mark.sphinx('latex', testroot='images')
21612162
def test_copy_images(app):
21622163
app.build()

‎tests/test_builders/test_build_texinfo.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ def test_texinfo_samp_with_variable(app):
131131
assert '@code{Show @var{variable} in the middle}' in output
132132

133133

134+
@pytest.mark.usefixtures('_http_teapot')
134135
@pytest.mark.sphinx('texinfo', testroot='images')
135136
def test_copy_images(app):
136137
app.build()

0 commit comments

Comments
 (0)