Coverage for an_website/utils/base_request_handler.py: 79.032%
496 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-09 14:51 +0000
1# This program is free software: you can redistribute it and/or modify
2# it under the terms of the GNU Affero General Public License as
3# published by the Free Software Foundation, either version 3 of the
4# License, or (at your option) any later version.
5#
6# This program is distributed in the hope that it will be useful,
7# but WITHOUT ANY WARRANTY; without even the implied warranty of
8# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9# GNU Affero General Public License for more details.
10#
11# You should have received a copy of the GNU Affero General Public License
12# along with this program. If not, see <https://www.gnu.org/licenses/>.
13# pylint: disable=too-many-lines
15"""
16The base request handler used by other modules.
18This should only contain the BaseRequestHandler class.
19"""
21import contextlib
22import inspect
23import logging
24import secrets
25import sys
26import traceback
27import uuid
28from asyncio import Future
29from base64 import b64decode
30from collections.abc import Awaitable, Callable, Coroutine, Mapping
31from contextvars import ContextVar
32from datetime import date, datetime, timedelta, timezone, tzinfo
33from functools import cached_property, partial, reduce
34from random import Random, choice as random_choice
35from types import TracebackType
36from typing import Any, ClassVar, Final, cast, override
37from urllib.parse import SplitResult, urlsplit, urlunsplit
38from zoneinfo import ZoneInfo
40import elasticapm
41import html2text
42import orjson as json
43import regex
44import tornado.web
45import yaml
46from accept_types import get_best_match # type: ignore[import-untyped]
47from ansi2html import Ansi2HTMLConverter
48from bs4 import BeautifulSoup
49from dateutil.easter import easter
50from elastic_transport import ApiError, TransportError
51from elasticsearch import AsyncElasticsearch
52from openmoji_dist import VERSION as OPENMOJI_VERSION
53from redis.asyncio import Redis
54from tornado.httputil import HTTPServerRequest
55from tornado.iostream import StreamClosedError
56from tornado.web import (
57 Finish,
58 GZipContentEncoding,
59 HTTPError,
60 MissingArgumentError,
61 OutputTransform,
62)
64from .. import (
65 EVENT_ELASTICSEARCH,
66 EVENT_REDIS,
67 GH_ORG_URL,
68 GH_PAGES_URL,
69 GH_REPO_URL,
70 NAME,
71 ORJSON_OPTIONS,
72 pytest_is_running,
73)
74from .decorators import is_authorized
75from .options import ColourScheme, Options
76from .static_file_handling import FILE_HASHES_DICT, fix_static_path
77from .themes import RANDOM_THEMES
78from .utils import (
79 ModuleInfo,
80 Permission,
81 add_args_to_url,
82 ansi_replace,
83 apply,
84 backspace_replace,
85 bool_to_str,
86 emoji2html,
87 geoip,
88 hash_bytes,
89 is_prime,
90 ratelimit,
91 str_to_bool,
92)
94LOGGER: Final = logging.getLogger(__name__)
96TEXT_CONTENT_TYPES: Final[set[str]] = {
97 "application/javascript",
98 "application/json",
99 "application/vnd.asozial.dynload+json",
100 "application/x-ndjson",
101 "application/xml",
102 "application/yaml",
103}
105CLACKS_OVERHEADS = (
106 "GNU Aaron Swartz",
107 "GNU Carol Angie Deborah Maltesi",
108 "GNU Charlotte Angie",
109 "GNU Terry Pratchett",
110)
112request_ctx_var: ContextVar[HTTPServerRequest] = ContextVar("current_request")
115class _RequestHandler(tornado.web.RequestHandler):
116 """Base for Tornado request handlers."""
118 crawler: bool = False
120 @override
121 async def _execute(
122 self, transforms: list[OutputTransform], *args: bytes, **kwargs: bytes
123 ) -> None:
124 request_ctx_var.set(self.request)
126 self.now = await self.get_time()
128 return await super()._execute(transforms, *args, **kwargs)
130 # pylint: disable-next=protected-access
131 _execute.__doc__ = tornado.web.RequestHandler._execute.__doc__
133 @property
134 def apm_client(self) -> None | elasticapm.Client:
135 """Get the APM client from the settings."""
136 return self.settings.get("ELASTIC_APM", {}).get("CLIENT") # type: ignore[no-any-return]
138 @property
139 def apm_enabled(self) -> bool:
140 """Return whether APM is enabled."""
141 return bool(self.settings.get("ELASTIC_APM", {}).get("ENABLED"))
143 @override
144 def data_received( # noqa: D102
145 self, chunk: bytes
146 ) -> None | Awaitable[None]:
147 pass
149 data_received.__doc__ = tornado.web.RequestHandler.data_received.__doc__
151 @property
152 def elasticsearch(self) -> AsyncElasticsearch:
153 """
154 Get the Elasticsearch client from the settings.
156 This is None if Elasticsearch is not enabled.
157 """
158 return cast(AsyncElasticsearch, self.settings.get("ELASTICSEARCH"))
160 @property
161 def elasticsearch_prefix(self) -> str:
162 """Get the Elasticsearch prefix from the settings."""
163 return self.settings.get( # type: ignore[no-any-return]
164 "ELASTICSEARCH_PREFIX", NAME
165 )
167 def geoip(
168 self,
169 ip: None | str = None,
170 database: str = geoip.__defaults__[0], # type: ignore[index]
171 *,
172 allow_fallback: bool = True,
173 ) -> Coroutine[None, None, None | dict[str, Any]]:
174 """Get GeoIP information."""
175 if not ip:
176 ip = self.request.remote_ip
177 if not EVENT_ELASTICSEARCH.is_set():
178 return geoip(ip, database)
179 return geoip(
180 ip, database, self.elasticsearch, allow_fallback=allow_fallback
181 )
183 async def get_time(self) -> datetime:
184 """Get the start time of the request in the users' timezone."""
185 tz: tzinfo = timezone.utc
186 try:
187 geoip = await self.geoip() # pylint: disable=redefined-outer-name
188 except ApiError, TransportError:
189 LOGGER.exception("Elasticsearch request failed")
190 if self.apm_client:
191 self.apm_client.capture_exception() # type: ignore[no-untyped-call]
192 else:
193 if geoip and "timezone" in geoip:
194 tz = ZoneInfo(geoip["timezone"])
195 return datetime.fromtimestamp(
196 self.request._start_time, tz=tz # pylint: disable=protected-access
197 )
199 def is_authorized(
200 self, permission: Permission, allow_cookie_auth: bool = True
201 ) -> bool | None:
202 """Check whether the request is authorized."""
203 return is_authorized(self, permission, allow_cookie_auth)
205 @override
206 def log_exception(
207 self,
208 typ: None | type[BaseException],
209 value: None | BaseException,
210 tb: None | TracebackType,
211 ) -> None:
212 if isinstance(value, HTTPError):
213 super().log_exception(typ, value, tb)
214 elif typ is StreamClosedError:
215 LOGGER.debug(
216 "Stream closed %s",
217 self._request_summary(),
218 exc_info=(typ, value, tb), # type: ignore[arg-type]
219 )
220 else:
221 LOGGER.error(
222 "Uncaught exception %s",
223 self._request_summary(),
224 exc_info=(typ, value, tb), # type: ignore[arg-type]
225 )
227 log_exception.__doc__ = tornado.web.RequestHandler.log_exception.__doc__
229 @cached_property
230 def now(self) -> datetime:
231 """Get the current time."""
232 if pytest_is_running():
233 raise AssertionError("Now accessed before it was set")
234 # if self.request.method in self.SUPPORTED_METHODS: # Why?
235 LOGGER.error("Now accessed before it was set", stacklevel=3)
236 return self.now_utc
238 @cached_property
239 def now_utc(self) -> datetime:
240 """Get the current time in the correct timezone."""
241 return datetime.fromtimestamp(
242 self.request._start_time, # pylint: disable=protected-access
243 tz=timezone.utc,
244 )
246 @override # pylint: disable-next=invalid-overridden-method
247 async def prepare(self) -> None:
248 """Check authorization and call self.ratelimit()."""
249 if crawler_secret := self.settings.get("CRAWLER_SECRET"):
250 self.crawler = crawler_secret in self.request.headers.get(
251 "User-Agent", ""
252 )
254 if (
255 self.request.method in {"GET", "HEAD"}
256 and self.redirect_to_canonical_domain()
257 ):
258 return
260 if self.request.method != "OPTIONS" and not await self.ratelimit(True):
261 await self.ratelimit()
263 async def ratelimit(self, global_ratelimit: bool = False) -> bool:
264 """Take b1nzy to space using Redis."""
265 if (
266 not self.settings.get("RATELIMITS")
267 or self.request.method == "OPTIONS"
268 or self.is_authorized(Permission.RATELIMITS)
269 or self.crawler
270 ):
271 return False
273 if not EVENT_REDIS.is_set():
274 LOGGER.warning(
275 (
276 "Ratelimits are enabled, but Redis is not available. "
277 "This can happen shortly after starting the website."
278 ),
279 )
280 raise HTTPError(503)
282 if global_ratelimit: # TODO: add to _RequestHandler
283 ratelimited, headers = await ratelimit(
284 self.redis,
285 self.redis_prefix,
286 str(self.request.remote_ip),
287 bucket=None,
288 max_burst=99, # limit = 100
289 count_per_period=20, # 20 requests per second
290 period=1,
291 tokens=10 if self.settings.get("UNDER_ATTACK") else 1,
292 )
293 else:
294 method = (
295 "GET" if self.request.method == "HEAD" else self.request.method
296 )
297 if not (limit := getattr(self, f"RATELIMIT_{method}_LIMIT", 0)):
298 return False
299 ratelimited, headers = await ratelimit(
300 self.redis,
301 self.redis_prefix,
302 str(self.request.remote_ip),
303 bucket=getattr(
304 self,
305 f"RATELIMIT_{method}_BUCKET",
306 self.__class__.__name__.lower(),
307 ),
308 max_burst=limit - 1,
309 count_per_period=getattr( # request count per period
310 self,
311 f"RATELIMIT_{method}_COUNT_PER_PERIOD",
312 30,
313 ),
314 period=getattr(
315 self, f"RATELIMIT_{method}_PERIOD", 60 # period in seconds
316 ),
317 tokens=1 if self.request.method != "HEAD" else 0,
318 )
320 for header, value in headers.items():
321 self.set_header(header, value)
323 if ratelimited:
324 if self.now.date() == date(self.now.year, 4, 20):
325 self.set_status(420)
326 self.write_error(420)
327 else:
328 self.set_status(429)
329 self.write_error(429)
331 return ratelimited
333 def redirect_to_canonical_domain(self) -> bool:
334 """Redirect to the canonical domain."""
335 if (
336 not (domain := self.settings.get("DOMAIN"))
337 or not self.request.headers.get("Host")
338 or self.request.host_name == domain
339 or self.request.host_name.endswith((".onion", ".i2p"))
340 or regex.fullmatch(r"/[\u2800-\u28FF]+/?", self.request.path)
341 ):
342 return False
343 port = urlsplit(f"//{self.request.headers['Host']}").port
344 self.redirect(
345 urlsplit(self.request.full_url())
346 ._replace(netloc=f"{domain}:{port}" if port else domain)
347 .geturl(),
348 permanent=True,
349 )
350 return True
352 @property
353 def redis(self) -> Redis[str]:
354 """
355 Get the Redis client from the settings.
357 This is None if Redis is not enabled.
358 """
359 return cast("Redis[str]", self.settings.get("REDIS"))
361 @property
362 def redis_prefix(self) -> str:
363 """Get the Redis prefix from the settings."""
364 return self.settings.get( # type: ignore[no-any-return]
365 "REDIS_PREFIX", NAME
366 )
369class BaseRequestHandler(_RequestHandler):
370 """The base request handler used by every page and API."""
372 # pylint: disable=too-many-instance-attributes, too-many-public-methods
374 ELASTIC_RUM_URL: ClassVar[str] = (
375 f"/@apm-rum/elastic-apm-rum.umd{'' if sys.flags.dev_mode else '.min'}.js"
376 "?v=5.12.0"
377 )
379 COMPUTE_ETAG: ClassVar[bool] = True
380 ALLOW_COMPRESSION: ClassVar[bool] = True
381 MAX_BODY_SIZE: ClassVar[None | int] = None
382 ALLOWED_METHODS: ClassVar[tuple[str, ...]] = ("GET",)
383 POSSIBLE_CONTENT_TYPES: ClassVar[tuple[str, ...]] = ()
385 module_info: ModuleInfo
386 # info about page, can be overridden in module_info
387 title: str = "Das Asoziale Netzwerk"
388 short_title: str = "Asoziales Netzwerk"
389 description: str = "Die tolle Webseite des Asozialen Netzwerks"
391 used_render: bool = False
393 active_origin_trials: set[str]
394 content_type: None | str = None
395 apm_script: None | str
396 nonce: str
398 def _finish(
399 self, chunk: None | str | bytes | dict[str, Any] = None
400 ) -> Future[None]:
401 if self._finished:
402 raise RuntimeError("finish() called twice")
404 if chunk is not None:
405 self.write(chunk)
407 if ( # pylint: disable=too-many-boolean-expressions
408 (content_type := self.content_type)
409 and (
410 content_type in TEXT_CONTENT_TYPES
411 or content_type.startswith("text/")
412 or content_type.endswith(("+xml", "+json"))
413 )
414 and self._write_buffer
415 and not self._write_buffer[-1].endswith(b"\n")
416 ):
417 self.write(b"\n")
419 return super().finish()
421 @override
422 def compute_etag(self) -> None | str:
423 """Compute ETag with Base85 encoding."""
424 if not self.COMPUTE_ETAG:
425 return None
426 return f'"{hash_bytes(*self._write_buffer)}"' # noqa: B907
428 @override
429 def decode_argument( # noqa: D102
430 self, value: bytes, name: str | None = None
431 ) -> str:
432 try:
433 return value.decode("UTF-8", "replace")
434 except UnicodeDecodeError as exc:
435 err_msg = f"Invalid unicode in {name or 'url'}: {value[:40]!r}"
436 LOGGER.exception(err_msg, exc_info=exc)
437 raise HTTPError(400, err_msg) from exc
439 @property
440 def dump(self) -> Callable[[Any], str | bytes]:
441 """Get the function for dumping the output."""
442 yaml_subset = self.content_type in {
443 "application/json",
444 "application/vnd.asozial.dynload+json",
445 }
447 if self.content_type == "application/yaml":
448 if self.now.timetuple()[2:0:-1] == (1, 4):
449 yaml_subset = True
450 else:
451 return lambda spam: yaml.dump(
452 spam,
453 width=self.get_int_argument("yaml_width", 80, min_=80),
454 )
456 if yaml_subset:
457 option = ORJSON_OPTIONS
458 if self.get_bool_argument("pretty", False):
459 option |= json.OPT_INDENT_2
460 return lambda spam: json.dumps(spam, option=option)
462 return lambda spam: spam
464 @override
465 def finish( # noqa: D102
466 self, chunk: None | str | bytes | dict[Any, Any] = None
467 ) -> Future[None]:
468 as_json = self.content_type == "application/vnd.asozial.dynload+json"
469 as_plain_text = self.content_type == "text/plain"
470 as_markdown = self.content_type == "text/markdown"
472 if (
473 not isinstance(chunk, bytes | str)
474 or self.content_type == "text/html"
475 or not self.used_render
476 or not (as_json or as_plain_text or as_markdown)
477 ):
478 return self._finish(chunk)
480 chunk = chunk.decode("UTF-8") if isinstance(chunk, bytes) else chunk
482 if as_markdown:
483 return self._finish(
484 f"# {self.title}\n\n"
485 + html2text.html2text(chunk, self.request.full_url()).strip()
486 )
488 soup = BeautifulSoup(chunk, features="lxml")
490 if as_plain_text:
491 return self._finish(soup.get_text("\n", True))
493 dictionary: dict[str, object] = {
494 "url": self.fix_url(include_protocol_and_host=True),
495 "title": self.title,
496 "short_title": (
497 self.short_title if self.title != self.short_title else None
498 ),
499 "body": "".join(
500 str(element)
501 for element in soup.find_all(name="main")[0].contents
502 ).strip(),
503 "scripts": [
504 {"script": script.string, **script.attrs}
505 for script in soup.find_all("script")
506 ],
507 "stylesheets": [
508 stylesheet.get("href")
509 for stylesheet in soup.find_all("link", rel="stylesheet")
510 ],
511 "css": "\n".join(
512 (style.string or "") for style in soup.find_all("style")
513 ),
514 }
516 return self._finish(dictionary)
518 finish.__doc__ = _RequestHandler.finish.__doc__
520 def finish_dict(self, **kwargs: Any) -> Future[None]:
521 """Finish the request with a dictionary."""
522 return self.finish(kwargs)
524 def fix_url(
525 self,
526 url: None | str | SplitResult = None,
527 new_path: None | str = None,
528 include_protocol_and_host: bool | str = False,
529 query_args: Mapping[str, None | str | bool | float] | None = None,
530 ) -> str:
531 """
532 Fix a URL and return it.
534 If the URL is from another website, link to it with the redirect page,
535 otherwise just return the URL with no_3rd_party appended.
536 """
537 query_args_d = dict(query_args or {})
538 del query_args
539 if url is None:
540 url = self.request.full_url()
541 if isinstance(url, str):
542 url = urlsplit(url)
543 if url.netloc and url.netloc.lower() != self.request.host.lower():
544 if (
545 not self.user_settings.ask_before_leaving
546 or not self.settings.get("REDIRECT_MODULE_LOADED")
547 ):
548 return url.geturl()
549 path = "/redirect"
550 query_args_d["to"] = url.geturl()
551 url = urlsplit(self.request.full_url())
552 else:
553 path = url.path if new_path is None else new_path
554 path = f"/{path.strip('/')}".lower()
555 if path == "/lolwut":
556 path = path.upper()
557 if path.startswith("/soundboard/files/") or path in FILE_HASHES_DICT:
558 query_args_d.update(
559 dict.fromkeys(self.user_settings.iter_option_names())
560 )
561 else:
562 for (
563 key,
564 value,
565 ) in self.user_settings.as_dict_with_str_values().items():
566 query_args_d.setdefault(key, value)
567 for key, value in self.user_settings.as_dict_with_str_values(
568 include_query_argument=False,
569 include_body_argument=self.request.path == "/einstellungen"
570 and self.get_bool_argument("save_in_cookie", False),
571 ).items():
572 if value == query_args_d[key]:
573 query_args_d[key] = None
575 result = add_args_to_url(
576 urlunsplit(
577 (
578 self.request.protocol,
579 self.request.host,
580 path,
581 url.query,
582 url.fragment,
583 )
584 ),
585 **query_args_d,
586 )
588 return (
589 result
590 if include_protocol_and_host
591 else result.removeprefix(
592 f"{self.request.protocol}://{self.request.host}"
593 )
594 )
596 @classmethod
597 def get_allowed_methods(cls) -> list[str]:
598 """Get allowed methods."""
599 methods = {"OPTIONS", *cls.ALLOWED_METHODS}
600 if "GET" in cls.ALLOWED_METHODS and cls.supports_head():
601 methods.add("HEAD")
602 return sorted(methods)
604 def get_bool_argument(
605 self,
606 name: str,
607 default: None | bool = None,
608 ) -> bool:
609 """Get an argument parsed as boolean."""
610 if default is not None:
611 return str_to_bool(self.get_argument(name, ""), default)
612 value = str(self.get_argument(name))
613 try:
614 return str_to_bool(value)
615 except ValueError as err:
616 raise HTTPError(400, f"{value} is not a boolean") from err
618 def get_display_scheme(self) -> ColourScheme:
619 """Get the scheme currently displayed."""
620 scheme = self.user_settings.scheme
621 if scheme == "random":
622 return ("light", "dark")[self.now.microsecond & 1]
623 return scheme
625 def get_display_theme(self) -> str:
626 """Get the theme currently displayed."""
627 theme = self.user_settings.theme
629 if theme == "default" and self.now.month == 12:
630 return "christmas"
632 if theme != "random":
633 return theme
635 return random_choice(RANDOM_THEMES) # nosec: B311
637 def get_error_message(self, **kwargs: Any) -> str:
638 """
639 Get the error message and return it.
641 If the serve_traceback setting is true (debug mode is activated),
642 the traceback gets returned.
643 """
644 if "exc_info" in kwargs and not issubclass(
645 kwargs["exc_info"][0], HTTPError
646 ):
647 if self.settings.get("serve_traceback") or self.is_authorized(
648 Permission.TRACEBACK
649 ):
650 return "".join(
651 traceback.format_exception(*kwargs["exc_info"])
652 ).strip()
653 return "".join(
654 traceback.format_exception_only(*kwargs["exc_info"][:2])
655 ).strip()
656 if "exc_info" in kwargs and issubclass(
657 kwargs["exc_info"][0], MissingArgumentError
658 ):
659 return cast(str, kwargs["exc_info"][1].log_message)
660 return str(self._reason)
662 def get_error_page_description(self, status_code: int) -> str:
663 """Get the description for the error page."""
664 # pylint: disable=too-many-return-statements
665 # https://developer.mozilla.org/docs/Web/HTTP/Status
666 if 100 <= status_code <= 199:
667 return "Hier gibt es eine total wichtige Information."
668 if 200 <= status_code <= 299:
669 return "Hier ist alles super! 🎶🎶"
670 if 300 <= status_code <= 399:
671 return "Eine Umleitung ist eingerichtet."
672 if 400 <= status_code <= 499:
673 if status_code == 404:
674 return f"{self.request.path} wurde nicht gefunden."
675 if status_code == 451:
676 return "Hier wäre bestimmt geiler Scheiß."
677 return "Ein Client-Fehler ist aufgetreten."
678 if 500 <= status_code <= 599:
679 return "Ein Server-Fehler ist aufgetreten."
680 raise ValueError(
681 f"{status_code} is not a valid HTTP response status code."
682 )
684 def get_int_argument(
685 self,
686 name: str,
687 default: None | int = None,
688 *,
689 max_: None | int = None,
690 min_: None | int = None,
691 ) -> int:
692 """Get an argument parsed as integer."""
693 if default is None:
694 str_value = self.get_argument(name)
695 try:
696 value = int(str_value, base=0)
697 except ValueError as err:
698 raise HTTPError(400, f"{str_value} is not an integer") from err
699 elif self.get_argument(name, ""):
700 try:
701 value = int(self.get_argument(name), base=0)
702 except ValueError:
703 value = default
704 else:
705 value = default
707 if max_ is not None:
708 value = min(max_, value)
709 if min_ is not None:
710 value = max(min_, value)
712 return value
714 def get_module_infos(self) -> tuple[ModuleInfo, ...]:
715 """Get the module infos."""
716 return self.settings.get("MODULE_INFOS") or ()
718 def get_reporting_api_endpoint(self) -> None | str:
719 """Get the endpoint for the Reporting API™️."""
720 if not self.settings.get("REPORTING"):
721 return None
722 endpoint = self.settings.get("REPORTING_ENDPOINT")
724 if not endpoint or not endpoint.startswith("/"):
725 return endpoint
727 return f"{self.request.protocol}://{self.request.host}{endpoint}"
729 @override
730 def get_template_namespace(self) -> dict[str, Any]:
731 """
732 Add useful things to the template namespace and return it.
734 They are mostly needed by most of the pages (like title,
735 description and no_3rd_party).
736 """
737 namespace = super().get_template_namespace()
738 ansi2html = partial(
739 Ansi2HTMLConverter(inline=True, scheme="xterm").convert, full=False
740 )
741 namespace.update(self.user_settings.as_dict())
742 namespace.update(
743 ansi2html=partial(
744 reduce, apply, (ansi2html, ansi_replace, backspace_replace)
745 ),
746 apm_script=(
747 self.settings["ELASTIC_APM"].get("INLINE_SCRIPT")
748 if self.apm_enabled
749 else None
750 ),
751 as_html=self.content_type == "text/html",
752 c=self.now.date() == date(self.now.year, 4, 1)
753 or str_to_bool(self.get_cookie("c", "f") or "f", False),
754 canonical_url=self.request.protocol
755 + "://"
756 + (self.settings["DOMAIN"] or self.request.host)
757 + self.fix_url(
758 self.request.full_url().upper()
759 if self.request.path.upper().startswith("/LOLWUT")
760 else self.request.full_url().lower()
761 )
762 .split("?")[0]
763 .removesuffix("/"),
764 description=self.description,
765 display_theme=self.get_display_theme(),
766 display_scheme=self.get_display_scheme(),
767 elastic_rum_url=self.ELASTIC_RUM_URL,
768 fix_static=lambda path: self.fix_url(fix_static_path(path)),
769 fix_url=self.fix_url,
770 emoji2html=(
771 emoji2html
772 if self.user_settings.openmoji == "img"
773 else (
774 (lambda emoji: f'<span class="openmoji">{emoji}</span>')
775 if self.user_settings.openmoji
776 else (lambda emoji: f"<span>{emoji}</span>")
777 )
778 ),
779 form_appendix=self.user_settings.get_form_appendix(),
780 GH_ORG_URL=GH_ORG_URL,
781 GH_PAGES_URL=GH_PAGES_URL,
782 GH_REPO_URL=GH_REPO_URL,
783 keywords="Asoziales Netzwerk, Känguru-Chroniken"
784 + (
785 f", {self.module_info.get_keywords_as_str(self.request.path)}"
786 if self.module_info # type: ignore[truthy-bool]
787 else ""
788 ),
789 lang="de", # TODO: add language support
790 nonce=self.nonce,
791 now=self.now,
792 openmoji_version=OPENMOJI_VERSION,
793 settings=self.settings,
794 short_title=self.short_title,
795 testing=pytest_is_running(),
796 title=self.title,
797 )
798 namespace.update(
799 {
800 "🥚": timedelta()
801 <= self.now.date() - easter(self.now.year)
802 < timedelta(days=2),
803 "🦘": is_prime(self.now.microsecond),
804 }
805 )
806 return namespace
808 def get_user_id(self) -> str:
809 """Get the user id saved in the cookie or create one."""
810 cookie = self.get_secure_cookie(
811 "user_id",
812 max_age_days=90,
813 min_version=2,
814 )
816 user_id = cookie.decode("UTF-8") if cookie else str(uuid.uuid4())
818 if not self.get_secure_cookie( # save it in cookie or reset expiry date
819 "user_id", max_age_days=30, min_version=2
820 ):
821 self.set_secure_cookie(
822 "user_id",
823 user_id,
824 expires_days=90,
825 path="/",
826 samesite="Strict",
827 )
829 return user_id
831 def handle_accept_header( # pylint: disable=inconsistent-return-statements
832 self, possible_content_types: tuple[str, ...], strict: bool = True
833 ) -> None:
834 """Handle the Accept header and set `self.content_type`."""
835 if not possible_content_types:
836 return
837 content_type = get_best_match(
838 self.request.headers.get("Accept") or "*/*",
839 possible_content_types,
840 )
841 if content_type is None:
842 if strict:
843 return self.handle_not_acceptable(possible_content_types)
844 content_type = possible_content_types[0]
845 self.content_type = content_type
846 self.set_content_type_header()
848 def handle_not_acceptable(
849 self, possible_content_types: tuple[str, ...]
850 ) -> None:
851 """Only call this if we cannot respect the Accept header."""
852 self.clear_header("Content-Type")
853 self.set_status(406)
854 raise Finish("\n".join(possible_content_types) + "\n")
856 def head(self, *args: Any, **kwargs: Any) -> None | Awaitable[None]:
857 """Handle HEAD requests."""
858 if self.get.__module__ == "tornado.web":
859 raise HTTPError(405)
860 if not self.supports_head():
861 raise HTTPError(501)
863 kwargs["head"] = True
864 return self.get(*args, **kwargs)
866 @override
867 def initialize(
868 self,
869 *,
870 module_info: ModuleInfo,
871 # default is true, because then empty args dicts are
872 # enough to specify that the defaults should be used
873 default_title: bool = True,
874 default_description: bool = True,
875 ) -> None:
876 """
877 Get title and description from the kwargs.
879 If title and description are present in the kwargs,
880 then they override self.title and self.description.
881 """
882 self.module_info = module_info
883 if not default_title:
884 page_info = self.module_info.get_page_info(self.request.path)
885 self.title = page_info.name
886 self.short_title = page_info.short_name or self.title
887 if not default_description:
888 self.description = self.module_info.get_page_info(
889 self.request.path
890 ).description
892 @override
893 async def options(self, *args: Any, **kwargs: Any) -> None:
894 """Handle OPTIONS requests."""
895 # pylint: disable=unused-argument
896 self.set_header("Allow", ", ".join(self.get_allowed_methods()))
897 self.set_status(204)
898 await self.finish()
900 def origin_trial(self, token: bytes | str) -> bool:
901 """Enable an experimental feature."""
902 # pylint: disable=protected-access
903 payload = json.loads(b64decode(token)[69:])
904 if payload["feature"] in self.active_origin_trials:
905 return True
906 origin = urlsplit(payload["origin"])
907 url = urlsplit(self.request.full_url())
908 if url.port is None and url.scheme in {"http", "https"}:
909 url = url._replace(
910 netloc=f"{url.hostname}:{443 if url.scheme == 'https' else 80}"
911 )
912 if self.request._start_time > payload["expiry"]:
913 return False
914 if url.scheme != origin.scheme:
915 return False
916 if url.netloc != origin.netloc and not (
917 payload.get("isSubdomain")
918 and url.netloc.endswith(f".{origin.netloc}")
919 ):
920 return False
921 self.add_header("Origin-Trial", token)
922 self.active_origin_trials.add(payload["feature"])
923 return True
925 @override
926 async def prepare(self) -> None:
927 """Check authorization and call self.ratelimit()."""
928 await super().prepare()
930 if self._finished:
931 return
933 if not self.ALLOW_COMPRESSION:
934 for transform in self._transforms:
935 if isinstance(transform, GZipContentEncoding):
936 # pylint: disable=protected-access
937 transform._gzipping = False
939 self.handle_accept_header(self.POSSIBLE_CONTENT_TYPES)
941 if self.request.method == "GET" and (
942 days := Random(self.now.timestamp()).randint(0, 31337)
943 ) in {
944 69,
945 420,
946 1337,
947 31337,
948 }:
949 self.set_cookie("c", "s", expires_days=days / 24, path="/")
951 if (
952 self.request.method != "OPTIONS"
953 and self.MAX_BODY_SIZE is not None
954 and len(self.request.body) > self.MAX_BODY_SIZE
955 ):
956 LOGGER.warning(
957 "%s > MAX_BODY_SIZE (%s)",
958 len(self.request.body),
959 self.MAX_BODY_SIZE,
960 )
961 raise HTTPError(413)
963 @override
964 def render( # noqa: D102
965 self, template_name: str, **kwargs: Any
966 ) -> Future[None]:
967 self.used_render = True
968 return super().render(template_name, **kwargs)
970 render.__doc__ = _RequestHandler.render.__doc__
972 def set_content_type_header(self) -> None:
973 """Set the Content-Type header based on `self.content_type`."""
974 if str(self.content_type).startswith("text/"): # RFC 2616 (3.7.1)
975 self.set_header(
976 "Content-Type", f"{self.content_type};charset=utf-8"
977 )
978 elif self.content_type is not None:
979 self.set_header("Content-Type", self.content_type)
981 @override
982 def set_cookie( # noqa: D102 # pylint: disable=too-many-arguments
983 self,
984 name: str,
985 value: str | bytes,
986 domain: None | str = None,
987 expires: None | float | tuple[int, ...] | datetime = None,
988 path: str = "/",
989 expires_days: None | float = 400, # changed
990 *,
991 secure: bool | None = None,
992 httponly: bool = True,
993 **kwargs: Any,
994 ) -> None:
995 if "samesite" not in kwargs:
996 # default for same site should be strict
997 kwargs["samesite"] = "Strict"
999 super().set_cookie(
1000 name,
1001 value,
1002 domain,
1003 expires,
1004 path,
1005 expires_days,
1006 secure=(
1007 self.request.protocol == "https" if secure is None else secure
1008 ),
1009 httponly=httponly,
1010 **kwargs,
1011 )
1013 set_cookie.__doc__ = _RequestHandler.set_cookie.__doc__
1015 def set_csp_header(self) -> None:
1016 """Set the Content-Security-Policy header."""
1017 self.nonce = secrets.token_urlsafe(16)
1019 script_src = ["'self'", f"'nonce-{self.nonce}'"]
1021 if (
1022 self.apm_enabled
1023 and "INLINE_SCRIPT_HASH" in self.settings["ELASTIC_APM"]
1024 ):
1025 script_src.extend(
1026 (
1027 f"'sha256-{self.settings['ELASTIC_APM']['INLINE_SCRIPT_HASH']}'",
1028 "'unsafe-inline'", # for browsers that don't support hash
1029 )
1030 )
1032 connect_src = ["'self'"]
1034 if self.apm_enabled and "SERVER_URL" in self.settings["ELASTIC_APM"]:
1035 rum_server_url = self.settings["ELASTIC_APM"].get("RUM_SERVER_URL")
1036 if rum_server_url:
1037 # the RUM agent needs to connect to rum_server_url
1038 connect_src.append(rum_server_url)
1039 elif rum_server_url is None:
1040 # the RUM agent needs to connect to ["ELASTIC_APM"]["SERVER_URL"]
1041 connect_src.append(self.settings["ELASTIC_APM"]["SERVER_URL"])
1043 connect_src.append( # fix for older browsers
1044 ("wss" if self.request.protocol == "https" else "ws")
1045 + f"://{self.request.host}"
1046 )
1048 self.set_header(
1049 "Content-Security-Policy",
1050 "default-src 'self';"
1051 f"script-src {' '.join(script_src)};"
1052 f"connect-src {' '.join(connect_src)};"
1053 "style-src 'self' 'unsafe-inline';"
1054 "img-src 'self' https://img.zeit.de https://github.asozial.org;"
1055 "frame-ancestors 'self';"
1056 "sandbox allow-downloads allow-same-origin allow-modals"
1057 " allow-popups-to-escape-sandbox allow-scripts allow-popups"
1058 " allow-top-navigation-by-user-activation allow-forms;"
1059 "report-to default;"
1060 "base-uri 'none';"
1061 + (
1062 f"report-uri {self.get_reporting_api_endpoint()};"
1063 if self.settings.get("REPORTING")
1064 else ""
1065 ),
1066 )
1068 @override
1069 def set_default_headers(self) -> None:
1070 """Set default headers."""
1071 self.set_csp_header()
1072 self.active_origin_trials = set()
1073 if self.settings.get("REPORTING"):
1074 endpoint = self.get_reporting_api_endpoint()
1075 self.set_header(
1076 "Reporting-Endpoints",
1077 f'default="{endpoint}"', # noqa: B907
1078 )
1079 self.set_header(
1080 "Report-To",
1081 json.dumps(
1082 {
1083 "group": "default",
1084 "max_age": 2592000,
1085 "endpoints": [{"url": endpoint}],
1086 },
1087 option=ORJSON_OPTIONS,
1088 ),
1089 )
1090 self.set_header("NEL", '{"report_to":"default","max_age":2592000}')
1091 self.set_header("X-Content-Type-Options", "nosniff")
1092 self.set_header("Access-Control-Max-Age", "7200")
1093 self.set_header("Access-Control-Allow-Origin", "*")
1094 self.set_header("Access-Control-Allow-Headers", "*")
1095 self.set_header(
1096 "Access-Control-Allow-Methods",
1097 ", ".join(self.get_allowed_methods()),
1098 )
1099 self.set_header("Cross-Origin-Resource-Policy", "cross-origin")
1100 self.set_header(
1101 "Permissions-Policy",
1102 "browsing-topics=(),"
1103 "identity-credentials-get=(),"
1104 "join-ad-interest-group=(),"
1105 "private-state-token-issuance=(),"
1106 "private-state-token-redemption=(),"
1107 "run-ad-auction=()",
1108 )
1109 self.set_header("Referrer-Policy", "same-origin")
1110 self.set_header(
1111 "Cross-Origin-Opener-Policy", "same-origin;report-to=default"
1112 )
1113 if self.request.path == "/kaenguru-comics-alt": # TODO: improve this
1114 self.set_header(
1115 "Cross-Origin-Embedder-Policy",
1116 "credentialless;report-to=default",
1117 )
1118 else:
1119 self.set_header(
1120 "Cross-Origin-Embedder-Policy",
1121 "require-corp;report-to=default",
1122 )
1123 if self.settings.get("HSTS"):
1124 self.set_header("Strict-Transport-Security", "max-age=63072000")
1125 if (
1126 onion_address := self.settings.get("ONION_ADDRESS")
1127 ) and not self.request.host_name.endswith(".onion"):
1128 self.set_header(
1129 "Onion-Location",
1130 onion_address
1131 + self.request.path
1132 + (f"?{self.request.query}" if self.request.query else ""),
1133 )
1134 if self.settings.get("debug"):
1135 self.set_header("X-Debug", bool_to_str(True))
1136 for permission in Permission:
1137 if permission.name:
1138 self.set_header(
1139 f"X-Permission-{permission.name}",
1140 bool_to_str(bool(self.is_authorized(permission))),
1141 )
1142 self.set_header(
1143 "X-Clacks-Overhead",
1144 CLACKS_OVERHEADS[
1145 int(self.now_utc.microsecond) % len(CLACKS_OVERHEADS)
1146 ],
1147 )
1148 self.set_header("Accept-CH", "Sec-CH-Prefers-Reduced-Motion")
1149 self.set_header("Critical-CH", "Sec-CH-Prefers-Reduced-Motion")
1150 self.set_header(
1151 "Vary", "Accept,Authorization,Cookie,Sec-CH-Prefers-Reduced-Motion"
1152 )
1154 set_default_headers.__doc__ = _RequestHandler.set_default_headers.__doc__
1156 def stanley(self) -> bool:
1157 """Stanley."""
1158 return self.user_settings.stanley is not False and (
1159 self.now.date() == date(self.now.year, 4, 27)
1160 or self.user_settings.stanley is True
1161 )
1163 def sub_stanley(self, text: str) -> str:
1164 """Sub Stanley."""
1165 return regex.sub(
1166 r"\b\p{Lu}\p{Ll}{4}\p{Ll}*\b",
1167 lambda match: (
1168 "Stanley"
1169 if Random(match[0]).randrange(5) == self.now.year % 5
1170 else match[0]
1171 ),
1172 text,
1173 )
1175 @classmethod
1176 def supports_head(cls) -> bool:
1177 """Check whether this request handler supports HEAD requests."""
1178 signature = inspect.signature(cls.get)
1179 return (
1180 "head" in signature.parameters
1181 and signature.parameters["head"].kind
1182 == inspect.Parameter.KEYWORD_ONLY
1183 )
1185 @cached_property
1186 def user_settings(self) -> Options:
1187 """Get the user settings."""
1188 return Options(self)
1190 @override
1191 def write(self, chunk: str | bytes | dict[str, Any]) -> None: # noqa: D102
1192 if self._finished:
1193 raise RuntimeError("Cannot write() after finish()")
1195 self.set_content_type_header()
1197 if isinstance(chunk, dict):
1198 chunk = self.dump(chunk)
1200 if self.stanley():
1201 if isinstance(chunk, bytes):
1202 with contextlib.suppress(UnicodeDecodeError):
1203 chunk = chunk.decode("UTF-8")
1204 if isinstance(chunk, str):
1205 chunk = self.sub_stanley(chunk)
1207 super().write(chunk)
1209 write.__doc__ = _RequestHandler.write.__doc__
1211 @override
1212 def write_error(self, status_code: int, **kwargs: Any) -> None:
1213 """Render the error page."""
1214 dict_content_types: tuple[str, str] = (
1215 "application/json",
1216 "application/yaml",
1217 )
1218 all_error_content_types: tuple[str, ...] = (
1219 # text/plain as first (default), to not screw up output in terminals
1220 "text/plain",
1221 "text/html",
1222 "text/markdown",
1223 *dict_content_types,
1224 "application/vnd.asozial.dynload+json",
1225 )
1227 if self.content_type not in all_error_content_types:
1228 # don't send 406, instead default with text/plain
1229 self.handle_accept_header(all_error_content_types, strict=False)
1231 if self.content_type == "text/html":
1232 self.render( # type: ignore[unused-awaitable]
1233 "error.html",
1234 status=status_code,
1235 reason=self.get_error_message(**kwargs),
1236 description=self.get_error_page_description(status_code),
1237 is_traceback="exc_info" in kwargs
1238 and not issubclass(kwargs["exc_info"][0], HTTPError)
1239 and (
1240 self.settings.get("serve_traceback")
1241 or self.is_authorized(Permission.TRACEBACK)
1242 ),
1243 )
1244 return
1246 if self.content_type in dict_content_types:
1247 self.finish( # type: ignore[unused-awaitable]
1248 {
1249 "status": status_code,
1250 "reason": self.get_error_message(**kwargs),
1251 }
1252 )
1253 return
1255 self.finish( # type: ignore[unused-awaitable]
1256 f"{status_code} {self.get_error_message(**kwargs)}\n"
1257 )
1259 write_error.__doc__ = _RequestHandler.write_error.__doc__