Coverage for an_website / patches / __init__.py: 92.353%

1# This program is free software: you can redistribute it and/or modify 

2# it under the terms of the GNU Affero General Public License as 

3# published by the Free Software Foundation, either version 3 of the 

4# License, or (at your option) any later version. 

5# 

6# This program is distributed in the hope that it will be useful, 

7# but WITHOUT ANY WARRANTY; without even the implied warranty of 

8# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

9# GNU Affero General Public License for more details. 

10# 

11# You should have received a copy of the GNU Affero General Public License 

12# along with this program. If not, see <https://www.gnu.org/licenses/>. 

13# pylint: disable=protected-access 

14 

15"""Patches that improve everything.""" 

16 

17import asyncio 

18import http.client 

19import json as stdlib_json # pylint: disable=preferred-module 

20import logging 

21import os 

22from collections.abc import Callable 

23from configparser import RawConfigParser 

24from contextlib import suppress 

25from importlib import import_module 

26from pathlib import Path 

27from threading import Thread 

28from types import MethodType 

29from typing import Any 

30from urllib.parse import urlsplit 

31from warnings import catch_warnings, simplefilter 

32 

33import certifi 

34import defusedxml # type: ignore[import-untyped] 

35import jsonpickle # type: ignore[import-untyped] 

36import multiprocessing_importlib_resources 

37import orjson 

38import pycurl 

39import tornado.httputil 

40import yaml 

41from emoji import EMOJI_DATA 

42from pillow_jxl import JpegXLImagePlugin # noqa: F401 

43from setproctitle import setthreadtitle 

44from tornado.httpclient import AsyncHTTPClient, HTTPRequest 

45from tornado.httputil import ( 

46 HTTPFile, 

47 HTTPHeaders, 

48 HTTPInputError, 

49 HTTPServerRequest, 

50 ParseBodyConfig, 

51) 

52from tornado.log import gen_log 

53from tornado.web import GZipContentEncoding, RedirectHandler, RequestHandler 

54 

55from .. import CA_BUNDLE_PATH, MEDIA_TYPES 

56from . import braille, json # noqa: F401 # pylint: disable=reimported 

57 

58 

59def apply() -> None: 

60 """Improve.""" 

61 multiprocessing_importlib_resources._patch() 

62 patch_asyncio() 

63 patch_certifi() 

64 patch_configparser() 

65 patch_emoji() 

66 patch_http() 

67 patch_json() 

68 patch_jsonpickle() 

69 patch_threading() 

70 patch_xml() 

71 

72 patch_tornado_418() 

73 patch_tornado_arguments() 

74 patch_tornado_gzip() 

75 patch_tornado_httpclient() 

76 patch_tornado_logs() 

77 patch_tornado_redirect() 

78 

79 

80def patch_asyncio() -> None: 

81 """Make stuff faster.""" 

82 if os.environ.get("DISABLE_UVLOOP") not in { 

83 "y", "yes", "t", "true", "on", "1" # fmt: skip 

84 }: 

85 with catch_warnings(): 

86 simplefilter("ignore", DeprecationWarning) 

87 with suppress(ModuleNotFoundError): 

88 asyncio.set_event_loop_policy( 

89 import_module("uvloop").EventLoopPolicy() 

90 ) 

91 

92 

93def patch_certifi() -> None: 

94 """Make everything use our CA bundle.""" 

95 certifi.where = lambda: CA_BUNDLE_PATH 

96 certifi.contents = lambda: Path(certifi.where()).read_text("ASCII") 

97 

98 

99def patch_configparser() -> None: 

100 """Make configparser funky.""" 

101 RawConfigParser.BOOLEAN_STATES.update( # type: ignore[attr-defined] 

102 { 

103 "sure": True, 

104 "nope": False, 

105 "accept": True, 

106 "reject": False, 

107 "enabled": True, 

108 "disabled": False, 

109 } 

110 ) 

111 

112 

113def patch_emoji() -> None: 

114 """Add cool new emoji.""" 

115 EMOJI_DATA["🐱\u200D💻"] = { 

116 "de": ":hacker_katze:", 

117 "en": ":hacker_cat:", 

118 "status": 2, 

119 "E": 1, 

120 } 

121 for de_name, en_name, rect in ( 

122 ("rot", "red", "🟥"), 

123 ("blau", "blue", "🟦"), 

124 ("orang", "orange", "🟧"), 

125 ("gelb", "yellow", "🟨"), 

126 ("grün", "green", "🟩"), 

127 ("lilan", "purple", "🟪"), 

128 ("braun", "brown", "🟫"), 

129 ): 

130 EMOJI_DATA[f"🫙\u200D{rect}"] = { 

131 "de": f":{de_name}es_glas:", 

132 "en": f":{en_name}_jar:", 

133 "status": 2, 

134 "E": 14, 

135 } 

136 EMOJI_DATA[f"🏳\uFE0F\u200D{rect}"] = { 

137 "de": f":{de_name}e_flagge:", 

138 "en": f":{en_name}_flag:", 

139 "status": 2, 

140 "E": 11, 

141 } 

142 EMOJI_DATA[f"\u2691\uFE0F\u200D{rect}"] = { 

143 "de": f":tief{de_name}e_flagge:", 

144 "en": f":deep_{en_name}_flag:", 

145 "status": 2, 

146 "E": 11, 

147 } 

148 

149 

150def patch_http() -> None: 

151 """Add response code 420.""" 

152 http.client.responses[420] = "Enhance Your Calm" 

153 

154 

155def patch_json() -> None: 

156 """Replace json with orjson.""" 

157 if getattr(stdlib_json, "_omegajson", False): 

158 return 

159 stdlib_json.dumps = json.dumps 

160 stdlib_json.dump = json.dump # type: ignore[assignment] 

161 stdlib_json.loads = json.loads # type: ignore[assignment] 

162 stdlib_json.load = json.load 

163 

164 

165def patch_jsonpickle() -> None: 

166 """Make jsonpickle return bytes.""" 

167 jsonpickle.load_backend("orjson") 

168 jsonpickle.set_preferred_backend("orjson") 

169 jsonpickle.enable_fallthrough(False) 

170 

171 

172def patch_threading() -> None: 

173 """Set thread names.""" 

174 _bootstrap = Thread._bootstrap # type: ignore[attr-defined] 

175 

176 def bootstrap(self: Thread) -> None: 

177 with suppress(Exception): 

178 setthreadtitle(self.name) 

179 _bootstrap(self) 

180 

181 Thread._bootstrap = bootstrap # type: ignore[attr-defined] 

182 

183 

184def patch_tornado_418() -> None: 

185 """Add support for RFC 7168.""" 

186 RequestHandler.SUPPORTED_METHODS += ( 

187 "PROPFIND", 

188 "BREW", 

189 "WHEN", 

190 ) 

191 _ = RequestHandler._unimplemented_method 

192 RequestHandler.propfind = _ # type: ignore[attr-defined] 

193 RequestHandler.brew = _ # type: ignore[attr-defined] 

194 RequestHandler.when = _ # type: ignore[attr-defined] 

195 

196 

197def patch_tornado_arguments() -> None: # noqa: C901 

198 """Improve argument parsing.""" 

199 # pylint: disable=too-complex 

200 

201 def ensure_bytes(value: Any) -> bytes: 

202 """Return the value as bytes.""" 

203 if isinstance(value, bool): 

204 return b"true" if value else b"false" 

205 if isinstance(value, bytes): 

206 return value 

207 return str(value).encode("UTF-8") 

208 

209 def parse_body_arguments( # pylint: disable=too-many-arguments 

210 content_type: str, 

211 body: bytes, 

212 arguments: dict[str, list[bytes]], 

213 files: dict[str, list[HTTPFile]], 

214 headers: None | HTTPHeaders = None, 

215 *, 

216 config: ParseBodyConfig | None = None, 

217 _: Callable[..., None] = tornado.httputil.parse_body_arguments, 

218 ) -> None: 

219 # pylint: disable=too-many-branches 

220 if content_type.startswith("application/json"): 

221 if headers and "Content-Encoding" in headers: 

222 gen_log.warning( 

223 "Unsupported Content-Encoding: %s", 

224 headers["Content-Encoding"], 

225 ) 

226 return 

227 try: 

228 spam = orjson.loads(body) 

229 except Exception as exc: 

230 raise HTTPInputError(f"Invalid JSON body: {exc}") from exc 

231 if not isinstance(spam, dict): 

232 return 

233 for key, value in spam.items(): 

234 if value is not None: 

235 arguments.setdefault(key, []).append(ensure_bytes(value)) 

236 elif content_type.startswith("application/yaml"): 

237 if headers and "Content-Encoding" in headers: 

238 gen_log.warning( 

239 "Unsupported Content-Encoding: %s", 

240 headers["Content-Encoding"], 

241 ) 

242 return 

243 try: 

244 spam = yaml.safe_load(body) 

245 except Exception as exc: 

246 raise HTTPInputError(f"Invalid YAML body: {exc}") from exc 

247 if not isinstance(spam, dict): 

248 return 

249 for key, value in spam.items(): 

250 if value is not None: 

251 arguments.setdefault(key, []).append(ensure_bytes(value)) 

252 else: 

253 _(content_type, body, arguments, files, headers, config=config) 

254 

255 parse_body_arguments.__doc__ = tornado.httputil.parse_body_arguments.__doc__ 

256 

257 tornado.httputil.parse_body_arguments = parse_body_arguments 

258 

259 

260def patch_tornado_gzip() -> None: 

261 """Use gzip for more content types.""" 

262 GZipContentEncoding.CONTENT_TYPES = { 

263 type for type, data in MEDIA_TYPES.items() if data.get("compressible") 

264 } 

265 

266 

267def patch_tornado_httpclient() -> None: # fmt: off 

268 """Make requests quick.""" 

269 BACON = 0x75800 # noqa: N806 # pylint: disable=invalid-name 

270 EGGS = 1 << 25 # noqa: N806 # pylint: disable=invalid-name 

271 

272 AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") 

273 

274 def prepare_curl_callback(self: HTTPRequest, curl: pycurl.Curl) -> None: 

275 # pylint: disable=c-extension-no-member, useless-suppression 

276 if urlsplit(self.url).scheme == "https": # noqa: SIM102 

277 if (ver := pycurl.version_info())[2] >= BACON and ver[4] & EGGS: 

278 curl.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_3) 

279 

280 original_request_init = HTTPRequest.__init__ 

281 

282 def request_init(self: HTTPRequest, *args: Any, **kwargs: Any) -> None: 

283 if len(args) < 18: # there are too many positional arguments here 

284 prepare_curl_method = MethodType(prepare_curl_callback, self) 

285 kwargs.setdefault("prepare_curl_callback", prepare_curl_method) 

286 original_request_init(self, *args, **kwargs) 

287 

288 request_init.__doc__ = HTTPRequest.__init__.__doc__ 

289 

290 HTTPRequest.__init__ = request_init # type: ignore[method-assign] 

291 

292 

293def patch_tornado_logs() -> None: 

294 """Anonymize Tornado logs.""" 

295 # pylint: disable=import-outside-toplevel 

296 from ..utils.utils import SUS_PATHS, anonymize_ip 

297 

298 RequestHandler._request_summary = ( # type: ignore[method-assign] 

299 lambda self: "%s %s (%s)" # pylint: disable=consider-using-f-string 

300 % ( 

301 self.request.method, 

302 self.request.uri, 

303 ( 

304 self.request.remote_ip 

305 if self.request.path == "/robots.txt" 

306 or self.request.path.lower() in SUS_PATHS 

307 else anonymize_ip(self.request.remote_ip, ignore_invalid=True) 

308 ), 

309 ) 

310 ) 

311 

312 HTTPServerRequest.__repr__ = ( # type: ignore[method-assign] 

313 lambda self: "%s(%s)" # pylint: disable=consider-using-f-string 

314 % ( 

315 self.__class__.__name__, 

316 ", ".join( 

317 [ 

318 "%s=%r" # pylint: disable=consider-using-f-string 

319 % ( 

320 n, 

321 getattr(self, n), 

322 ) 

323 for n in ("protocol", "host", "method", "uri", "version") 

324 ] 

325 ), 

326 ) 

327 ) 

328 

329 

330def patch_tornado_redirect() -> None: 

331 """Use modern redirect codes and support HEAD requests.""" 

332 

333 def redirect( 

334 self: RequestHandler, 

335 url: str, 

336 permanent: bool = False, 

337 status: None | int = None, 

338 ) -> None: 

339 if url == self.request.full_url(): 

340 logging.getLogger( 

341 f"{self.__class__.__module__}.{self.__class__.__qualname__}" 

342 ).critical("Infinite redirect to %r detected", url) 

343 if self._headers_written: 

344 # pylint: disable=broad-exception-raised 

345 raise Exception("Cannot redirect after headers have been written") 

346 if status is None: 

347 status = 308 if permanent else 307 

348 else: 

349 assert isinstance(status, int) and 300 <= status <= 399 # type: ignore[redundant-expr] # noqa: B950 

350 self.set_status(status) 

351 self.set_header("Location", url) 

352 self.finish() # type: ignore[unused-awaitable] 

353 

354 if RequestHandler.redirect.__doc__: 

355 # fmt: off 

356 redirect.__doc__ = ( 

357 RequestHandler.redirect.__doc__ 

358 .replace("301", "308") 

359 .replace("302", "307") 

360 ) 

361 # fmt: on 

362 

363 RequestHandler.redirect = redirect # type: ignore[method-assign] 

364 

365 RedirectHandler.head = RedirectHandler.get 

366 

367 

368def patch_xml() -> None: 

369 """Make XML safer.""" 

370 defusedxml.defuse_stdlib() 

371 defusedxml.xmlrpc.monkey_patch()