Coverage for an_website/patches/__init__.py: 88.953%

1# This program is free software: you can redistribute it and/or modify 

2# it under the terms of the GNU Affero General Public License as 

3# published by the Free Software Foundation, either version 3 of the 

4# License, or (at your option) any later version. 

5# 

6# This program is distributed in the hope that it will be useful, 

7# but WITHOUT ANY WARRANTY; without even the implied warranty of 

8# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

9# GNU Affero General Public License for more details. 

10# 

11# You should have received a copy of the GNU Affero General Public License 

12# along with this program. If not, see <https://www.gnu.org/licenses/>. 

13# pylint: disable=protected-access 

14 

15"""Patches that improve everything.""" 

16 

17from __future__ import annotations 

18 

19import asyncio 

20import http.client 

21import json as stdlib_json # pylint: disable=preferred-module 

22import logging 

23import os 

24import sys 

25import warnings 

26from collections.abc import Callable 

27from configparser import RawConfigParser 

28from contextlib import suppress 

29from importlib import import_module 

30from pathlib import Path 

31from threading import Thread 

32from types import MethodType 

33from typing import Any 

34from urllib.parse import urlsplit 

35 

36import certifi 

37import defusedxml # type: ignore[import-untyped] 

38import jsonpickle # type: ignore[import-untyped] 

39import orjson 

40import pycurl 

41import tornado.httputil 

42import yaml 

43from emoji import EMOJI_DATA 

44from pillow_jxl import JpegXLImagePlugin # noqa: F401 

45from setproctitle import setthreadtitle 

46from tornado.httpclient import AsyncHTTPClient, HTTPRequest 

47from tornado.httputil import HTTPFile, HTTPHeaders, HTTPServerRequest 

48from tornado.log import gen_log 

49from tornado.web import GZipContentEncoding, RedirectHandler, RequestHandler 

50 

51from .. import CA_BUNDLE_PATH, MEDIA_TYPES 

52from . import braille, json # noqa: F401 # pylint: disable=reimported 

53 

54 

55def apply() -> None: 

56 """Improve.""" 

57 patch_asyncio() 

58 patch_certifi() 

59 patch_configparser() 

60 patch_emoji() 

61 patch_http() 

62 patch_json() 

63 patch_jsonpickle() 

64 patch_threading() 

65 patch_xml() 

66 

67 patch_tornado_418() 

68 patch_tornado_arguments() 

69 patch_tornado_gzip() 

70 patch_tornado_httpclient() 

71 patch_tornado_logs() 

72 patch_tornado_redirect() 

73 

74 

75def patch_asyncio() -> None: 

76 """Make stuff faster.""" 

77 if os.environ.get("DISABLE_UVLOOP") not in { 

78 "y", "yes", "t", "true", "on", "1" # fmt: skip 

79 }: 

80 with warnings.catch_warnings(): 

81 warnings.filterwarnings("ignore", category=DeprecationWarning) 

82 try: 

83 policy = import_module("uvloop").EventLoopPolicy() 

84 except ModuleNotFoundError: 

85 pass 

86 else: 

87 asyncio.set_event_loop_policy(policy) 

88 

89 

90def patch_certifi() -> None: 

91 """Make everything use our CA bundle.""" 

92 certifi.where = lambda: CA_BUNDLE_PATH 

93 certifi.contents = lambda: Path(certifi.where()).read_text("ASCII") 

94 

95 

96def patch_configparser() -> None: 

97 """Make configparser funky.""" 

98 RawConfigParser.BOOLEAN_STATES.update( # type: ignore[attr-defined] 

99 { 

100 "sure": True, 

101 "nope": False, 

102 "accept": True, 

103 "reject": False, 

104 "enabled": True, 

105 "disabled": False, 

106 } 

107 ) 

108 

109 

110def patch_emoji() -> None: 

111 """Add cool new emoji.""" 

112 EMOJI_DATA["🐱\u200D💻"] = { 

113 "de": ":hacker_katze:", 

114 "en": ":hacker_cat:", 

115 "status": 2, 

116 "E": 1, 

117 } 

118 for de_name, en_name, rect in ( 

119 ("rot", "red", "🟥"), 

120 ("blau", "blue", "🟦"), 

121 ("orang", "orange", "🟧"), 

122 ("gelb", "yellow", "🟨"), 

123 ("grün", "green", "🟩"), 

124 ("lilan", "purple", "🟪"), 

125 ("braun", "brown", "🟫"), 

126 ): 

127 EMOJI_DATA[f"🫙\u200D{rect}"] = { 

128 "de": f":{de_name}es_glas:", 

129 "en": f":{en_name}_jar:", 

130 "status": 2, 

131 "E": 14, 

132 } 

133 EMOJI_DATA[f"🏳\uFE0F\u200D{rect}"] = { 

134 "de": f":{de_name}e_flagge:", 

135 "en": f":{en_name}_flag:", 

136 "status": 2, 

137 "E": 11, 

138 } 

139 EMOJI_DATA[f"\u2691\uFE0F\u200D{rect}"] = { 

140 "de": f":tief{de_name}e_flagge:", 

141 "en": f":deep_{en_name}_flag:", 

142 "status": 2, 

143 "E": 11, 

144 } 

145 

146 

147def patch_http() -> None: 

148 """Add response code 420.""" 

149 http.client.responses[420] = "Enhance Your Calm" 

150 

151 

152def patch_json() -> None: 

153 """Replace json with orjson.""" 

154 if getattr(stdlib_json, "_omegajson", False) or sys.version_info < (3, 12): 

155 return 

156 stdlib_json.dumps = json.dumps 

157 stdlib_json.dump = json.dump # type: ignore[assignment] 

158 stdlib_json.loads = json.loads # type: ignore[assignment] 

159 stdlib_json.load = json.load 

160 

161 

162def patch_jsonpickle() -> None: 

163 """Make jsonpickle return bytes.""" 

164 jsonpickle.load_backend("orjson") 

165 jsonpickle.set_preferred_backend("orjson") 

166 jsonpickle.enable_fallthrough(False) 

167 

168 

169def patch_threading() -> None: 

170 """Set thread names.""" 

171 _bootstrap = Thread._bootstrap # type: ignore[attr-defined] 

172 

173 def bootstrap(self: Thread) -> None: 

174 with suppress(Exception): 

175 setthreadtitle(self.name) 

176 _bootstrap(self) 

177 

178 Thread._bootstrap = bootstrap # type: ignore[attr-defined] 

179 

180 

181def patch_tornado_418() -> None: 

182 """Add support for RFC 7168.""" 

183 RequestHandler.SUPPORTED_METHODS += ( 

184 "PROPFIND", 

185 "BREW", 

186 "WHEN", 

187 ) 

188 _ = RequestHandler._unimplemented_method 

189 RequestHandler.propfind = _ # type: ignore[attr-defined] 

190 RequestHandler.brew = _ # type: ignore[attr-defined] 

191 RequestHandler.when = _ # type: ignore[attr-defined] 

192 

193 

194def patch_tornado_arguments() -> None: # noqa: C901 

195 """Improve argument parsing.""" 

196 # pylint: disable=too-complex 

197 

198 def ensure_bytes(value: Any) -> bytes: 

199 """Return the value as bytes.""" 

200 if isinstance(value, bool): 

201 return b"true" if value else b"false" 

202 if isinstance(value, bytes): 

203 return value 

204 return str(value).encode("UTF-8") 

205 

206 def parse_body_arguments( 

207 content_type: str, 

208 body: bytes, 

209 arguments: dict[str, list[bytes]], 

210 files: dict[str, list[HTTPFile]], 

211 headers: None | HTTPHeaders = None, 

212 *, 

213 _: Callable[..., None] = tornado.httputil.parse_body_arguments, 

214 ) -> None: 

215 # pylint: disable=too-many-branches 

216 if content_type.startswith("application/json"): 

217 if headers and "Content-Encoding" in headers: 

218 gen_log.warning( 

219 "Unsupported Content-Encoding: %s", 

220 headers["Content-Encoding"], 

221 ) 

222 return 

223 try: 

224 spam = orjson.loads(body) 

225 except Exception as exc: # pylint: disable=broad-except 

226 gen_log.warning("Invalid JSON body: %s", exc) 

227 else: 

228 if not isinstance(spam, dict): 

229 return 

230 for key, value in spam.items(): 

231 if value is not None: 

232 arguments.setdefault(key, []).append( 

233 ensure_bytes(value) 

234 ) 

235 elif content_type.startswith("application/yaml"): 

236 if headers and "Content-Encoding" in headers: 

237 gen_log.warning( 

238 "Unsupported Content-Encoding: %s", 

239 headers["Content-Encoding"], 

240 ) 

241 return 

242 try: 

243 spam = yaml.safe_load(body) 

244 except Exception as exc: # pylint: disable=broad-except 

245 gen_log.warning("Invalid YAML body: %s", exc) 

246 else: 

247 if not isinstance(spam, dict): 

248 return 

249 for key, value in spam.items(): 

250 if value is not None: 

251 arguments.setdefault(key, []).append( 

252 ensure_bytes(value) 

253 ) 

254 else: 

255 _(content_type, body, arguments, files, headers) 

256 

257 parse_body_arguments.__doc__ = tornado.httputil.parse_body_arguments.__doc__ 

258 

259 tornado.httputil.parse_body_arguments = parse_body_arguments 

260 

261 

262def patch_tornado_gzip() -> None: 

263 """Use gzip for more content types.""" 

264 GZipContentEncoding.CONTENT_TYPES = { 

265 type for type, data in MEDIA_TYPES.items() if data.get("compressible") 

266 } 

267 

268 

269def patch_tornado_httpclient() -> None: # fmt: off 

270 """Make requests quick.""" 

271 BACON = 0x75800 # noqa: N806 # pylint: disable=invalid-name 

272 EGGS = 1 << 25 # noqa: N806 # pylint: disable=invalid-name 

273 

274 AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") 

275 

276 def prepare_curl_callback(self: HTTPRequest, curl: pycurl.Curl) -> None: 

277 # pylint: disable=c-extension-no-member, useless-suppression 

278 if urlsplit(self.url).scheme == "https": # noqa: SIM102 

279 if (ver := pycurl.version_info())[2] >= BACON and ver[4] & EGGS: 

280 curl.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_3) 

281 

282 original_request_init = HTTPRequest.__init__ 

283 

284 def request_init(self: HTTPRequest, *args: Any, **kwargs: Any) -> None: 

285 if len(args) < 18: # there are too many positional arguments here 

286 prepare_curl_method = MethodType(prepare_curl_callback, self) 

287 kwargs.setdefault("prepare_curl_callback", prepare_curl_method) 

288 original_request_init(self, *args, **kwargs) 

289 

290 request_init.__doc__ = HTTPRequest.__init__.__doc__ 

291 

292 HTTPRequest.__init__ = request_init # type: ignore[method-assign] 

293 

294 

295def patch_tornado_logs() -> None: 

296 """Anonymize Tornado logs.""" 

297 # pylint: disable=import-outside-toplevel 

298 from ..utils.utils import SUS_PATHS, anonymize_ip 

299 

300 RequestHandler._request_summary = ( # type: ignore[method-assign] 

301 lambda self: "%s %s (%s)" # pylint: disable=consider-using-f-string 

302 % ( 

303 self.request.method, 

304 self.request.uri, 

305 ( 

306 self.request.remote_ip 

307 if self.request.path == "/robots.txt" 

308 or self.request.path.lower() in SUS_PATHS 

309 else anonymize_ip(self.request.remote_ip, ignore_invalid=True) 

310 ), 

311 ) 

312 ) 

313 

314 HTTPServerRequest.__repr__ = ( # type: ignore[method-assign] 

315 lambda self: "%s(%s)" # pylint: disable=consider-using-f-string 

316 % ( 

317 self.__class__.__name__, 

318 ", ".join( 

319 [ 

320 "%s=%r" # pylint: disable=consider-using-f-string 

321 % ( 

322 n, 

323 getattr(self, n), 

324 ) 

325 for n in ("protocol", "host", "method", "uri", "version") 

326 ] 

327 ), 

328 ) 

329 ) 

330 

331 

332def patch_tornado_redirect() -> None: 

333 """Use modern redirect codes and support HEAD requests.""" 

334 

335 def redirect( 

336 self: RequestHandler, 

337 url: str, 

338 permanent: bool = False, 

339 status: None | int = None, 

340 ) -> None: 

341 if url == self.request.full_url(): 

342 logging.getLogger( 

343 f"{self.__class__.__module__}.{self.__class__.__qualname__}" 

344 ).critical("Infinite redirect to %r detected", url) 

345 if self._headers_written: 

346 # pylint: disable=broad-exception-raised 

347 raise Exception("Cannot redirect after headers have been written") 

348 if status is None: 

349 status = 308 if permanent else 307 

350 else: 

351 assert isinstance(status, int) and 300 <= status <= 399 # type: ignore[redundant-expr] # noqa: B950 

352 self.set_status(status) 

353 self.set_header("Location", url) 

354 self.finish() # type: ignore[unused-awaitable] 

355 

356 if RequestHandler.redirect.__doc__: 

357 # fmt: off 

358 redirect.__doc__ = ( 

359 RequestHandler.redirect.__doc__ 

360 .replace("301", "308") 

361 .replace("302", "307") 

362 ) 

363 # fmt: on 

364 

365 RequestHandler.redirect = redirect # type: ignore[method-assign] 

366 

367 RedirectHandler.head = RedirectHandler.get 

368 

369 

370def patch_xml() -> None: 

371 """Make XML safer.""" 

372 defusedxml.defuse_stdlib() 

373 defusedxml.xmlrpc.monkey_patch()