Coverage for an_website/main.py: 81.070%
243 statements
« prev ^ index » next coverage.py v7.8.2, created at 2025-06-01 08:32 +0000
1# This program is free software: you can redistribute it and/or modify
2# it under the terms of the GNU Affero General Public License as
3# published by the Free Software Foundation, either version 3 of the
4# License, or (at your option) any later version.
5#
6# This program is distributed in the hope that it will be useful,
7# but WITHOUT ANY WARRANTY; without even the implied warranty of
8# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9# GNU Affero General Public License for more details.
10#
11# You should have received a copy of the GNU Affero General Public License
12# along with this program. If not, see <https://www.gnu.org/licenses/>.
13# pylint: disable=import-private-name, too-many-lines
15"""
16The website of the AN.
18Loads config and modules and starts Tornado.
19"""
21from __future__ import annotations
23import asyncio
24import atexit
25import importlib
26import logging
27import os
28import platform
29import signal
30import ssl
31import sys
32import threading
33import time
34import types
35import uuid
36from asyncio import AbstractEventLoop
37from asyncio.runners import _cancel_all_tasks # type: ignore[attr-defined]
38from base64 import b64encode
39from collections.abc import Callable, Iterable, Mapping, MutableSequence
40from configparser import ConfigParser
41from functools import partial
42from hashlib import sha256
43from multiprocessing.process import _children # type: ignore[attr-defined]
44from pathlib import Path
45from socket import socket
46from typing import Any, Final, TypedDict, TypeGuard, cast
47from warnings import catch_warnings, simplefilter
48from zoneinfo import ZoneInfo
50import regex
51from Crypto.Hash import RIPEMD160
52from ecs_logging import StdlibFormatter
53from elasticapm.contrib.tornado import ElasticAPM
54from redis.asyncio import (
55 BlockingConnectionPool,
56 Redis,
57 SSLConnection,
58 UnixDomainSocketConnection,
59)
60from setproctitle import setproctitle
61from tornado.httpserver import HTTPServer
62from tornado.log import LogFormatter
63from tornado.netutil import bind_sockets, bind_unix_socket
64from tornado.process import fork_processes, task_id
65from tornado.web import Application, RedirectHandler
66from typed_stream import Stream
68from . import (
69 CA_BUNDLE_PATH,
70 DIR,
71 EVENT_SHUTDOWN,
72 NAME,
73 TEMPLATES_DIR,
74 UPTIME,
75 VERSION,
76 pytest_is_running,
77)
78from .contact.contact import apply_contact_stuff_to_app
79from .utils import background_tasks, static_file_handling
80from .utils.base_request_handler import BaseRequestHandler, request_ctx_var
81from .utils.better_config_parser import BetterConfigParser
82from .utils.elasticsearch_setup import setup_elasticsearch
83from .utils.logging import WebhookFormatter, WebhookHandler
84from .utils.request_handler import NotFoundHandler
85from .utils.static_file_from_traversable import TraversableStaticFileHandler
86from .utils.template_loader import TemplateLoader
87from .utils.utils import (
88 ArgparseNamespace,
89 Handler,
90 ModuleInfo,
91 Permission,
92 Timer,
93 create_argument_parser,
94 geoip,
95 get_arguments_without_help,
96 time_function,
97)
99try:
100 import perf8 # type: ignore[import, unused-ignore]
101except ModuleNotFoundError:
102 perf8 = None # pylint: disable=invalid-name
104IGNORED_MODULES: Final[set[str]] = {
105 "patches",
106 "static",
107 "templates",
108} | (set() if sys.flags.dev_mode or pytest_is_running() else {"example"})
110LOGGER: Final = logging.getLogger(__name__)
113# add all the information from the packages to a list
114# this calls the get_module_info function in every file
115# files and dirs starting with '_' get ignored
116def get_module_infos() -> str | tuple[ModuleInfo, ...]:
117 """Import the modules and return the loaded module infos in a tuple."""
118 module_infos: list[ModuleInfo] = []
119 loaded_modules: list[str] = []
120 errors: list[str] = []
122 for potential_module in DIR.iterdir():
123 if (
124 potential_module.name.startswith("_")
125 or potential_module.name in IGNORED_MODULES
126 or not potential_module.is_dir()
127 ):
128 continue
130 _module_infos = get_module_infos_from_module(
131 potential_module.name, errors, ignore_not_found=True
132 )
133 if _module_infos:
134 module_infos.extend(_module_infos)
135 loaded_modules.append(potential_module.name)
136 LOGGER.debug(
137 (
138 "Found module_infos in %s.__init__.py, "
139 "not searching in other modules in the package."
140 ),
141 potential_module,
142 )
143 continue
145 if f"{potential_module.name}.*" in IGNORED_MODULES:
146 continue
148 for potential_file in potential_module.iterdir():
149 module_name = f"{potential_module.name}.{potential_file.name[:-3]}"
150 if (
151 not potential_file.name.endswith(".py")
152 or module_name in IGNORED_MODULES
153 or potential_file.name.startswith("_")
154 ):
155 continue
156 _module_infos = get_module_infos_from_module(module_name, errors)
157 if _module_infos:
158 module_infos.extend(_module_infos)
159 loaded_modules.append(module_name)
161 if len(errors) > 0:
162 if sys.flags.dev_mode:
163 # exit to make sure it gets fixed
164 return "\n".join(errors)
165 # don't exit in production to keep stuff running
166 LOGGER.error("\n".join(errors))
168 LOGGER.info(
169 "Loaded %d modules: '%s'",
170 len(loaded_modules),
171 "', '".join(loaded_modules),
172 )
174 LOGGER.info(
175 "Ignored %d modules: '%s'",
176 len(IGNORED_MODULES),
177 "', '".join(IGNORED_MODULES),
178 )
180 sort_module_infos(module_infos)
182 # make module_infos immutable so it never changes
183 return tuple(module_infos)
186def get_module_infos_from_module(
187 module_name: str,
188 errors: MutableSequence[str], # gets modified
189 ignore_not_found: bool = False,
190) -> None | list[ModuleInfo]:
191 """Get the module infos based on a module."""
192 import_timer = Timer()
193 module = importlib.import_module(
194 f".{module_name}",
195 package="an_website",
196 )
197 if import_timer.stop() > 0.1:
198 LOGGER.warning(
199 "Import of %s took %ss. That's affecting the startup time.",
200 module_name,
201 import_timer.get(),
202 )
204 module_infos: list[ModuleInfo] = []
206 has_get_module_info = "get_module_info" in dir(module)
207 has_get_module_infos = "get_module_infos" in dir(module)
209 if not (has_get_module_info or has_get_module_infos):
210 if ignore_not_found:
211 return None
212 errors.append(
213 f"{module_name} has no 'get_module_info' and no 'get_module_infos' "
214 "method. Please add at least one of the methods or add "
215 f"'{module_name.rsplit('.', 1)[0]}.*' or {module_name!r} to "
216 "IGNORED_MODULES."
217 )
218 return None
220 if has_get_module_info and isinstance(
221 module_info := module.get_module_info(),
222 ModuleInfo,
223 ):
224 module_infos.append(module_info)
225 elif has_get_module_info:
226 errors.append(
227 f"'get_module_info' in {module_name} does not return ModuleInfo. "
228 "Please fix the returned value."
229 )
231 if not has_get_module_infos:
232 return module_infos or None
234 _module_infos = module.get_module_infos()
236 if not isinstance(_module_infos, Iterable):
237 errors.append(
238 f"'get_module_infos' in {module_name} does not return an Iterable. "
239 "Please fix the returned value."
240 )
241 return module_infos or None
243 for _module_info in _module_infos:
244 if isinstance(_module_info, ModuleInfo):
245 module_infos.append(_module_info)
246 else:
247 errors.append(
248 f"'get_module_infos' in {module_name} did return an Iterable "
249 f"with an element of type {type(_module_info)}. "
250 "Please fix the returned value."
251 )
253 return module_infos or None
256def sort_module_infos(module_infos: list[ModuleInfo]) -> None:
257 """Sort a list of module info and move the main page to the top."""
258 # sort it so the order makes sense
259 module_infos.sort()
261 # move the main page to the top
262 for i, info in enumerate(module_infos):
263 if info.path == "/":
264 module_infos.insert(0, module_infos.pop(i))
265 break
268def get_all_handlers(module_infos: Iterable[ModuleInfo]) -> list[Handler]:
269 """
270 Parse the module information and return the handlers in a tuple.
272 If a handler has only 2 elements a dict with title and description
273 gets added. This information is gotten from the module info.
274 """
275 handler: Handler | list[Any]
276 handlers: list[Handler] = static_file_handling.get_handlers()
278 # add all the normal handlers
279 for module_info in module_infos:
280 for handler in module_info.handlers:
281 handler = list(handler) # pylint: disable=redefined-loop-name
282 # if the handler is a request handler from us
283 # and not a built-in like StaticFileHandler & RedirectHandler
284 if issubclass(handler[1], BaseRequestHandler):
285 if len(handler) == 2:
286 # set "default_title" or "default_description" to False so
287 # that module_info.name & module_info.description get used
288 handler.append(
289 {
290 "default_title": False,
291 "default_description": False,
292 "module_info": module_info,
293 }
294 )
295 else:
296 handler[2]["module_info"] = module_info
297 handlers.append(tuple(handler))
299 # redirect handler, to make finding APIs easier
300 handlers.append((r"/(.+)/api/*", RedirectHandler, {"url": "/api/{0}"}))
302 handlers.append(
303 (
304 r"(?i)/\.well-known/(.*)",
305 TraversableStaticFileHandler,
306 {"root": Path(".well-known"), "hashes": {}},
307 )
308 )
310 LOGGER.debug("Loaded %d handlers", len(handlers))
312 return handlers
315def ignore_modules(config: BetterConfigParser) -> None:
316 """Read ignored modules from the config."""
317 IGNORED_MODULES.update(
318 config.getset("GENERAL", "IGNORED_MODULES", fallback=set())
319 )
322def get_normed_paths_from_module_infos(
323 module_infos: Iterable[ModuleInfo],
324) -> dict[str, str]:
325 """Get all paths from the module infos."""
327 def tuple_has_no_none(
328 value: tuple[str | None, str | None],
329 ) -> TypeGuard[tuple[str, str]]:
330 return None not in value
332 def info_to_paths(info: ModuleInfo) -> Stream[tuple[str, str]]:
333 return (
334 Stream(((info.path, info.path),))
335 .chain(
336 info.aliases.items()
337 if isinstance(info.aliases, Mapping)
338 else ((alias, info.path) for alias in info.aliases)
339 )
340 .chain(
341 Stream(info.sub_pages)
342 .map(lambda sub_info: sub_info.path)
343 .filter()
344 .map(lambda path: (path, path))
345 )
346 .filter(tuple_has_no_none)
347 )
349 return (
350 Stream(module_infos)
351 .flat_map(info_to_paths)
352 .filter(lambda p: p[0].startswith("/"))
353 .map(lambda p: (p[0].strip("/").lower(), p[1]))
354 .filter(lambda p: p[0])
355 .collect(dict)
356 )
359def make_app(config: ConfigParser) -> str | Application:
360 """Create the Tornado application and return it."""
361 module_infos, duration = time_function(get_module_infos)
362 if isinstance(module_infos, str):
363 return module_infos
364 if duration > 1:
365 LOGGER.warning(
366 "Getting the module infos took %ss. That's probably too long.",
367 duration,
368 )
369 handlers = get_all_handlers(module_infos)
370 return Application(
371 handlers,
372 MODULE_INFOS=module_infos,
373 SHOW_HAMBURGER_MENU=not Stream(module_infos)
374 .exclude(lambda info: info.hidden)
375 .filter(lambda info: info.path)
376 .empty(),
377 NORMED_PATHS=get_normed_paths_from_module_infos(module_infos),
378 HANDLERS=handlers,
379 # General settings
380 autoreload=False,
381 debug=sys.flags.dev_mode,
382 default_handler_class=NotFoundHandler,
383 compress_response=config.getboolean(
384 "GENERAL", "COMPRESS_RESPONSE", fallback=False
385 ),
386 websocket_ping_interval=10,
387 # Template settings
388 template_loader=TemplateLoader(
389 root=TEMPLATES_DIR, whitespace="oneline"
390 ),
391 )
394def apply_config_to_app(app: Application, config: BetterConfigParser) -> None:
395 """Apply the config (from the config.ini file) to the application."""
396 app.settings["CONFIG"] = config
398 app.settings["cookie_secret"] = config.get(
399 "GENERAL", "COOKIE_SECRET", fallback="xyzzy"
400 )
402 app.settings["CRAWLER_SECRET"] = config.get(
403 "APP_SEARCH", "CRAWLER_SECRET", fallback=None
404 )
406 app.settings["DOMAIN"] = config.get("GENERAL", "DOMAIN", fallback=None)
408 app.settings["ELASTICSEARCH_PREFIX"] = config.get(
409 "ELASTICSEARCH", "PREFIX", fallback=NAME
410 )
412 app.settings["HSTS"] = config.getboolean("TLS", "HSTS", fallback=False)
414 app.settings["NETCUP"] = config.getboolean(
415 "GENERAL", "NETCUP", fallback=False
416 )
418 onion_address = config.get("GENERAL", "ONION_ADDRESS", fallback=None)
419 app.settings["ONION_ADDRESS"] = onion_address
420 if onion_address is None:
421 app.settings["ONION_PROTOCOL"] = None
422 else:
423 app.settings["ONION_PROTOCOL"] = onion_address.split("://")[0]
425 app.settings["RATELIMITS"] = config.getboolean(
426 "GENERAL",
427 "RATELIMITS",
428 fallback=config.getboolean("REDIS", "ENABLED", fallback=False),
429 )
431 app.settings["REDIS_PREFIX"] = config.get("REDIS", "PREFIX", fallback=NAME)
433 app.settings["REPORTING"] = config.getboolean(
434 "REPORTING", "ENABLED", fallback=True
435 )
437 app.settings["REPORTING_BUILTIN"] = config.getboolean(
438 "REPORTING", "BUILTIN", fallback=sys.flags.dev_mode
439 )
441 app.settings["REPORTING_ENDPOINT"] = config.get(
442 "REPORTING",
443 "ENDPOINT",
444 fallback=(
445 "/api/reports"
446 if app.settings["REPORTING_BUILTIN"]
447 else "https://asozial.org/api/reports"
448 ),
449 )
451 app.settings["TRUSTED_API_SECRETS"] = {
452 key_perms[0]: Permission(
453 int(key_perms[1])
454 if len(key_perms) > 1
455 else (1 << len(Permission)) - 1 # should be all permissions
456 )
457 for secret in config.getset(
458 "GENERAL", "TRUSTED_API_SECRETS", fallback={"xyzzy"}
459 )
460 if (key_perms := [part.strip() for part in secret.split("=")])
461 if key_perms[0]
462 }
464 app.settings["AUTH_TOKEN_SECRET"] = config.get(
465 "GENERAL", "AUTH_TOKEN_SECRET", fallback=None
466 )
467 if not app.settings["AUTH_TOKEN_SECRET"]:
468 node = uuid.getnode().to_bytes(6, "big")
469 secret = RIPEMD160.new(node).digest().decode("BRAILLE")
470 LOGGER.warning(
471 "AUTH_TOKEN_SECRET is unset, implicitly setting it to %r",
472 secret,
473 )
474 app.settings["AUTH_TOKEN_SECRET"] = secret
476 app.settings["UNDER_ATTACK"] = config.getboolean(
477 "GENERAL", "UNDER_ATTACK", fallback=False
478 )
480 apply_contact_stuff_to_app(app, config)
483def get_ssl_context( # pragma: no cover
484 config: ConfigParser,
485) -> None | ssl.SSLContext:
486 """Create SSL context and configure using the config."""
487 if config.getboolean("TLS", "ENABLED", fallback=False):
488 ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
489 ssl_ctx.load_cert_chain(
490 config.get("TLS", "CERTFILE"),
491 config.get("TLS", "KEYFILE", fallback=None),
492 config.get("TLS", "PASSWORD", fallback=None),
493 )
494 return ssl_ctx
495 return None
498def setup_logging( # pragma: no cover
499 config: ConfigParser,
500 force: bool = False,
501) -> None:
502 """Setup logging.""" # noqa: D401
503 root_logger = logging.getLogger()
505 if root_logger.handlers:
506 if not force:
507 return
508 for handler in root_logger.handlers[:]:
509 root_logger.removeHandler(handler)
510 handler.close()
512 debug = config.getboolean("LOGGING", "DEBUG", fallback=sys.flags.dev_mode)
514 logging.captureWarnings(True)
516 root_logger.setLevel(logging.DEBUG if debug else logging.INFO)
517 logging.getLogger("tornado.curl_httpclient").setLevel(logging.INFO)
518 logging.getLogger("elasticsearch").setLevel(logging.INFO)
520 stream_handler = logging.StreamHandler()
521 if sys.flags.dev_mode:
522 spam = regex.sub(r"%\((end_)?color\)s", "", LogFormatter.DEFAULT_FORMAT)
523 formatter = logging.Formatter(spam, LogFormatter.DEFAULT_DATE_FORMAT)
524 else:
525 formatter = LogFormatter()
526 stream_handler.setFormatter(formatter)
527 root_logger.addHandler(stream_handler)
529 if path := config.get("LOGGING", "PATH", fallback=None):
530 os.makedirs(path, 0o755, True)
531 file_handler = logging.handlers.TimedRotatingFileHandler(
532 os.path.join(path, f"{NAME}.log"),
533 encoding="UTF-8",
534 when="midnight",
535 backupCount=30,
536 utc=True,
537 )
538 file_handler.setFormatter(StdlibFormatter())
539 root_logger.addHandler(file_handler)
542class WebhookLoggingOptions: # pylint: disable=too-few-public-methods
543 """Webhook logging options."""
545 __slots__ = (
546 "url",
547 "content_type",
548 "body_format",
549 "timestamp_format",
550 "timestamp_timezone",
551 "escape_message",
552 "max_message_length",
553 )
555 url: str | None
556 content_type: str
557 body_format: str
558 timestamp_format: str | None
559 timestamp_timezone: str | None
560 escape_message: bool
561 max_message_length: int | None
563 def __init__(self, config: ConfigParser) -> None:
564 """Initialize Webhook logging options."""
565 self.url = config.get("LOGGING", "WEBHOOK_URL", fallback=None)
566 self.content_type = config.get(
567 "LOGGING",
568 "WEBHOOK_CONTENT_TYPE",
569 fallback="application/json",
570 )
571 spam = regex.sub(r"%\((end_)?color\)s", "", LogFormatter.DEFAULT_FORMAT)
572 self.body_format = config.get(
573 "LOGGING",
574 "WEBHOOK_BODY_FORMAT",
575 fallback='{"text":"' + spam + '"}',
576 )
577 self.timestamp_format = config.get(
578 "LOGGING",
579 "WEBHOOK_TIMESTAMP_FORMAT",
580 fallback=None,
581 )
582 self.timestamp_timezone = config.get(
583 "LOGGING", "WEBHOOK_TIMESTAMP_TIMEZONE", fallback=None
584 )
585 self.escape_message = config.getboolean(
586 "LOGGING",
587 "WEBHOOK_ESCAPE_MESSAGE",
588 fallback=True,
589 )
590 self.max_message_length = config.getint(
591 "LOGGING", "WEBHOOK_MAX_MESSAGE_LENGTH", fallback=None
592 )
595def setup_webhook_logging( # pragma: no cover
596 options: WebhookLoggingOptions,
597 loop: asyncio.AbstractEventLoop,
598) -> None:
599 """Setup Webhook logging.""" # noqa: D401
600 if not options.url:
601 return
603 LOGGER.info("Setting up Webhook logging")
605 root_logger = logging.getLogger()
607 webhook_content_type = options.content_type
608 webhook_handler = WebhookHandler(
609 logging.ERROR,
610 loop=loop,
611 url=options.url,
612 content_type=webhook_content_type,
613 )
614 formatter = WebhookFormatter(
615 options.body_format,
616 options.timestamp_format,
617 )
618 formatter.timezone = (
619 None
620 if options.timestamp_format is None
621 else ZoneInfo(options.timestamp_format)
622 )
623 formatter.escape_message = options.escape_message
624 formatter.max_message_length = options.max_message_length
625 webhook_handler.setFormatter(formatter)
626 root_logger.addHandler(webhook_handler)
628 info_handler = WebhookHandler(
629 logging.INFO,
630 loop=loop,
631 url=options.url,
632 content_type=webhook_content_type,
633 )
634 info_handler.setFormatter(formatter)
635 logging.getLogger("an_website.quotes.create").addHandler(info_handler)
638def setup_apm(app: Application) -> None: # pragma: no cover
639 """Setup APM.""" # noqa: D401
640 config: BetterConfigParser = app.settings["CONFIG"]
641 app.settings["ELASTIC_APM"] = {
642 "ENABLED": config.getboolean("ELASTIC_APM", "ENABLED", fallback=False),
643 "SERVER_URL": config.get(
644 "ELASTIC_APM", "SERVER_URL", fallback="http://localhost:8200"
645 ),
646 "SECRET_TOKEN": config.get(
647 "ELASTIC_APM", "SECRET_TOKEN", fallback=None
648 ),
649 "API_KEY": config.get("ELASTIC_APM", "API_KEY", fallback=None),
650 "VERIFY_SERVER_CERT": config.getboolean(
651 "ELASTIC_APM", "VERIFY_SERVER_CERT", fallback=True
652 ),
653 "USE_CERTIFI": True, # doesn't actually use certifi
654 "SERVICE_NAME": NAME.removesuffix("-dev"),
655 "SERVICE_VERSION": VERSION,
656 "ENVIRONMENT": (
657 "production" if not sys.flags.dev_mode else "development"
658 ),
659 "DEBUG": True,
660 "CAPTURE_BODY": "errors",
661 "TRANSACTION_IGNORE_URLS": [
662 "/api/ping",
663 "/static/*",
664 "/favicon.png",
665 ],
666 "TRANSACTIONS_IGNORE_PATTERNS": ["^OPTIONS "],
667 "PROCESSORS": [
668 "an_website.utils.utils.apm_anonymization_processor",
669 "elasticapm.processors.sanitize_stacktrace_locals",
670 "elasticapm.processors.sanitize_http_request_cookies",
671 "elasticapm.processors.sanitize_http_headers",
672 "elasticapm.processors.sanitize_http_wsgi_env",
673 "elasticapm.processors.sanitize_http_request_body",
674 ],
675 "RUM_SERVER_URL": config.get(
676 "ELASTIC_APM", "RUM_SERVER_URL", fallback=None
677 ),
678 "RUM_SERVER_URL_PREFIX": config.get(
679 "ELASTIC_APM", "RUM_SERVER_URL_PREFIX", fallback=None
680 ),
681 }
683 script_options = [
684 f"serviceName:{app.settings['ELASTIC_APM']['SERVICE_NAME']!r}",
685 f"serviceVersion:{app.settings['ELASTIC_APM']['SERVICE_VERSION']!r}",
686 f"environment:{app.settings['ELASTIC_APM']['ENVIRONMENT']!r}",
687 ]
689 rum_server_url = app.settings["ELASTIC_APM"]["RUM_SERVER_URL"]
691 if rum_server_url is None:
692 script_options.append(
693 f"serverUrl:{app.settings['ELASTIC_APM']['SERVER_URL']!r}"
694 )
695 elif rum_server_url:
696 script_options.append(f"serverUrl:{rum_server_url!r}")
697 else:
698 script_options.append("serverUrl:window.location.origin")
700 if app.settings["ELASTIC_APM"]["RUM_SERVER_URL_PREFIX"]:
701 script_options.append(
702 f"serverUrlPrefix:{app.settings['ELASTIC_APM']['RUM_SERVER_URL_PREFIX']!r}"
703 )
705 app.settings["ELASTIC_APM"]["INLINE_SCRIPT"] = (
706 "elasticApm.init({" + ",".join(script_options) + "})"
707 )
709 app.settings["ELASTIC_APM"]["INLINE_SCRIPT_HASH"] = b64encode(
710 sha256(
711 app.settings["ELASTIC_APM"]["INLINE_SCRIPT"].encode("ASCII")
712 ).digest()
713 ).decode("ASCII")
715 if app.settings["ELASTIC_APM"]["ENABLED"]:
716 app.settings["ELASTIC_APM"]["CLIENT"] = ElasticAPM(app).client
719def setup_app_search(app: Application) -> None: # pragma: no cover
720 """Setup Elastic App Search.""" # noqa: D401
721 with catch_warnings():
722 simplefilter("ignore", DeprecationWarning)
723 # pylint: disable-next=import-outside-toplevel
724 from elastic_enterprise_search import (
725 AppSearch, # type: ignore[import-untyped]
726 )
728 config: BetterConfigParser = app.settings["CONFIG"]
729 host = config.get("APP_SEARCH", "HOST", fallback=None)
730 key = config.get("APP_SEARCH", "SEARCH_KEY", fallback=None)
731 verify_certs = config.getboolean(
732 "APP_SEARCH", "VERIFY_CERTS", fallback=True
733 )
734 app.settings["APP_SEARCH"] = (
735 AppSearch(
736 host,
737 bearer_auth=key,
738 verify_certs=verify_certs,
739 ca_certs=CA_BUNDLE_PATH,
740 )
741 if host
742 else None
743 )
744 app.settings["APP_SEARCH_HOST"] = host
745 app.settings["APP_SEARCH_KEY"] = key
746 app.settings["APP_SEARCH_ENGINE"] = config.get(
747 "APP_SEARCH", "ENGINE_NAME", fallback=NAME.removesuffix("-dev")
748 )
751def setup_redis(app: Application) -> None | Redis[str]:
752 """Setup Redis.""" # noqa: D401
753 config: BetterConfigParser = app.settings["CONFIG"]
755 class Kwargs(TypedDict, total=False):
756 """Kwargs of BlockingConnectionPool constructor."""
758 db: int
759 username: None | str
760 password: None | str
761 retry_on_timeout: bool
762 connection_class: type[UnixDomainSocketConnection] | type[SSLConnection]
763 path: str
764 host: str
765 port: int
766 ssl_ca_certs: str
767 ssl_keyfile: None | str
768 ssl_certfile: None | str
769 ssl_check_hostname: bool
770 ssl_cert_reqs: str
772 kwargs: Kwargs = {
773 "db": config.getint("REDIS", "DB", fallback=0),
774 "username": config.get("REDIS", "USERNAME", fallback=None),
775 "password": config.get("REDIS", "PASSWORD", fallback=None),
776 "retry_on_timeout": config.getboolean(
777 "REDIS", "RETRY_ON_TIMEOUT", fallback=False
778 ),
779 }
780 redis_ssl_kwargs: Kwargs = {
781 "connection_class": SSLConnection,
782 "ssl_ca_certs": CA_BUNDLE_PATH,
783 "ssl_keyfile": config.get("REDIS", "SSL_KEYFILE", fallback=None),
784 "ssl_certfile": config.get("REDIS", "SSL_CERTFILE", fallback=None),
785 "ssl_cert_reqs": config.get(
786 "REDIS", "SSL_CERT_REQS", fallback="required"
787 ),
788 "ssl_check_hostname": config.getboolean(
789 "REDIS", "SSL_CHECK_HOSTNAME", fallback=False
790 ),
791 }
792 redis_host_port_kwargs: Kwargs = {
793 "host": config.get("REDIS", "HOST", fallback="localhost"),
794 "port": config.getint("REDIS", "PORT", fallback=6379),
795 }
796 redis_use_ssl = config.getboolean("REDIS", "SSL", fallback=False)
797 redis_unix_socket_path = config.get(
798 "REDIS", "UNIX_SOCKET_PATH", fallback=None
799 )
801 if redis_unix_socket_path is not None:
802 if redis_use_ssl:
803 LOGGER.warning(
804 "SSL is enabled for Redis, but a UNIX socket is used"
805 )
806 if config.has_option("REDIS", "HOST"):
807 LOGGER.warning(
808 "A host is configured for Redis, but a UNIX socket is used"
809 )
810 if config.has_option("REDIS", "PORT"):
811 LOGGER.warning(
812 "A port is configured for Redis, but a UNIX socket is used"
813 )
814 kwargs.update(
815 {
816 "connection_class": UnixDomainSocketConnection,
817 "path": redis_unix_socket_path,
818 }
819 )
820 else:
821 kwargs.update(redis_host_port_kwargs)
822 if redis_use_ssl:
823 kwargs.update(redis_ssl_kwargs)
825 if not config.getboolean("REDIS", "ENABLED", fallback=False):
826 app.settings["REDIS"] = None
827 return None
828 connection_pool = BlockingConnectionPool(
829 client_name=NAME,
830 decode_responses=True,
831 **kwargs,
832 )
833 redis = cast("Redis[str]", Redis(connection_pool=connection_pool))
834 app.settings["REDIS"] = redis
835 return redis
838def signal_handler( # noqa: D103 # pragma: no cover
839 signalnum: int, frame: None | types.FrameType
840) -> None:
841 # pylint: disable=unused-argument, missing-function-docstring
842 if signalnum in {signal.SIGINT, signal.SIGTERM}:
843 EVENT_SHUTDOWN.set()
844 if signalnum == getattr(signal, "SIGHUP", None):
845 EVENT_SHUTDOWN.set()
848def install_signal_handler() -> None: # pragma: no cover
849 """Install the signal handler."""
850 signal.signal(signal.SIGINT, signal_handler)
851 signal.signal(signal.SIGTERM, signal_handler)
852 if hasattr(signal, "SIGHUP"):
853 signal.signal(signal.SIGHUP, signal_handler)
856def supervise(loop: AbstractEventLoop) -> None:
857 """Supervise."""
858 while foobarbaz := background_tasks.HEARTBEAT: # pylint: disable=while-used
859 if time.monotonic() - foobarbaz >= 10:
860 worker = task_id()
861 pid = os.getpid()
863 task = asyncio.current_task(loop)
864 request = task.get_context().get(request_ctx_var) if task else None
866 LOGGER.fatal(
867 "Heartbeat timed out for worker %s (pid %d), "
868 "current request: %s, current task: %s",
869 worker,
870 pid,
871 request,
872 task,
873 )
874 atexit._run_exitfuncs() # pylint: disable=protected-access
875 os.abort()
876 time.sleep(1)
879def main( # noqa: C901 # pragma: no cover
880 config: BetterConfigParser | None = None,
881) -> int | str:
882 """
883 Start everything.
885 This is the main function that is called when running this program.
886 """
887 # pylint: disable=too-complex, too-many-branches
888 # pylint: disable=too-many-locals, too-many-statements
889 setproctitle(NAME)
891 install_signal_handler()
893 parser = create_argument_parser()
894 args, _ = parser.parse_known_args(
895 get_arguments_without_help(), ArgparseNamespace()
896 )
898 if args.version:
899 print("Version:", VERSION)
900 if args.verbose:
901 # pylint: disable-next=import-outside-toplevel
902 from .version.version import (
903 get_file_hashes,
904 get_hash_of_file_hashes,
905 )
907 print()
908 print("Hash der Datei-Hashes:")
909 print(get_hash_of_file_hashes())
911 if args.verbose > 1:
912 print()
913 print("Datei-Hashes:")
914 print(get_file_hashes())
916 return 0
918 config = config or BetterConfigParser.from_path(*args.config)
919 assert config is not None
920 config.add_override_argument_parser(parser)
922 setup_logging(config)
924 LOGGER.info("Starting %s %s", NAME, VERSION)
926 if platform.system() == "Windows":
927 LOGGER.warning(
928 "Running %s on Windows is not officially supported",
929 NAME.removesuffix("-dev"),
930 )
932 ignore_modules(config)
933 app = make_app(config)
934 if isinstance(app, str):
935 return app
937 apply_config_to_app(app, config)
938 setup_elasticsearch(app)
939 setup_app_search(app)
940 setup_redis(app)
941 setup_apm(app)
943 behind_proxy = config.getboolean("GENERAL", "BEHIND_PROXY", fallback=False)
945 server = HTTPServer(
946 app,
947 body_timeout=3600,
948 decompress_request=True,
949 max_body_size=1_000_000_000,
950 ssl_options=get_ssl_context(config),
951 xheaders=behind_proxy,
952 )
954 socket_factories: list[Callable[[], Iterable[socket]]] = []
956 port = config.getint("GENERAL", "PORT", fallback=None)
958 if port:
959 socket_factories.append(
960 partial(
961 bind_sockets,
962 port,
963 "localhost" if behind_proxy else "",
964 )
965 )
967 unix_socket_path = config.get(
968 "GENERAL",
969 "UNIX_SOCKET_PATH",
970 fallback=None,
971 )
973 if unix_socket_path:
974 os.makedirs(unix_socket_path, 0o755, True)
975 socket_factories.append(
976 lambda: (
977 bind_unix_socket(
978 os.path.join(unix_socket_path, f"{NAME}.sock"),
979 mode=0o666,
980 ),
981 )
982 )
984 processes = config.getint(
985 "GENERAL",
986 "PROCESSES",
987 fallback=hasattr(os, "fork") * (2 if sys.flags.dev_mode else -1),
988 )
990 if processes < 0:
991 processes = (
992 os.process_cpu_count() # type: ignore[attr-defined]
993 if sys.version_info >= (3, 13)
994 else os.cpu_count()
995 ) or 0
997 worker: None | int = None
999 run_supervisor_thread = config.getboolean(
1000 "GENERAL", "SUPERVISE", fallback=False
1001 )
1002 elasticsearch_is_enabled = config.getboolean(
1003 "ELASTICSEARCH", "ENABLED", fallback=False
1004 )
1005 redis_is_enabled = config.getboolean("REDIS", "ENABLED", fallback=False)
1006 webhook_logging_options = WebhookLoggingOptions(config)
1007 # all config options should be read before forking
1008 if args.save_config_to:
1009 with open(args.save_config_to, "w", encoding="UTF-8") as file:
1010 config.write(file)
1011 config.set_all_options_should_be_parsed()
1012 del config
1013 # show help message if --help is given (after reading config, before forking)
1014 parser.parse_args()
1016 if not socket_factories:
1017 LOGGER.warning("No sockets configured")
1018 return 0
1020 # create sockets after checking for --help
1021 sockets: list[socket] = (
1022 Stream(socket_factories).flat_map(lambda fun: fun()).collect(list)
1023 )
1025 UPTIME.reset()
1026 main_pid = os.getpid()
1028 if processes:
1029 setproctitle(f"{NAME} - Master")
1031 worker = fork_processes(processes)
1033 setproctitle(f"{NAME} - Worker {worker}")
1035 # yeet all children (there should be none, but do it regardless, just in case)
1036 _children.clear()
1038 if "an_website.quotes" in sys.modules:
1039 from .quotes.utils import ( # pylint: disable=import-outside-toplevel
1040 AUTHORS_CACHE,
1041 QUOTES_CACHE,
1042 WRONG_QUOTES_CACHE,
1043 )
1045 del AUTHORS_CACHE.control.created_by_ultra # type: ignore[attr-defined]
1046 del QUOTES_CACHE.control.created_by_ultra # type: ignore[attr-defined]
1047 del WRONG_QUOTES_CACHE.control.created_by_ultra # type: ignore[attr-defined]
1048 del geoip.__kwdefaults__["caches"].control.created_by_ultra
1050 if unix_socket_path:
1051 sockets.append(
1052 bind_unix_socket(
1053 os.path.join(unix_socket_path, f"{NAME}.{worker}.sock"),
1054 mode=0o666,
1055 )
1056 )
1058 # get loop after forking
1059 # if not forking allow loop to be set in advance by external code
1060 loop: None | asyncio.AbstractEventLoop
1061 try:
1062 with catch_warnings(): # TODO: remove after dropping support for 3.13
1063 simplefilter("ignore", DeprecationWarning)
1064 loop = asyncio.get_event_loop()
1065 if loop.is_closed():
1066 loop = None
1067 except RuntimeError:
1068 loop = None
1070 if loop is None:
1071 loop = asyncio.new_event_loop()
1072 asyncio.set_event_loop(loop)
1074 if sys.version_info >= (3, 13) and not loop.get_task_factory():
1075 loop.set_task_factory(asyncio.eager_task_factory)
1077 if perf8 and "PERF8" in os.environ:
1078 loop.run_until_complete(perf8.enable())
1080 setup_webhook_logging(webhook_logging_options, loop)
1082 server.add_sockets(sockets)
1084 tasks = background_tasks.start_background_tasks( # noqa: F841
1085 module_infos=app.settings["MODULE_INFOS"],
1086 loop=loop,
1087 main_pid=main_pid,
1088 app=app,
1089 processes=processes,
1090 elasticsearch_is_enabled=elasticsearch_is_enabled,
1091 redis_is_enabled=redis_is_enabled,
1092 worker=worker,
1093 )
1095 if run_supervisor_thread:
1096 background_tasks.HEARTBEAT = time.monotonic()
1097 threading.Thread(
1098 target=supervise, args=(loop,), name="supervisor", daemon=True
1099 ).start()
1101 try:
1102 loop.run_forever()
1103 EVENT_SHUTDOWN.set()
1104 finally:
1105 try: # pylint: disable=too-many-try-statements
1106 server.stop()
1107 loop.run_until_complete(asyncio.sleep(1))
1108 loop.run_until_complete(server.close_all_connections())
1109 if perf8 and "PERF8" in os.environ:
1110 loop.run_until_complete(perf8.disable())
1111 if redis := app.settings.get("REDIS"):
1112 loop.run_until_complete(
1113 redis.aclose(close_connection_pool=True)
1114 )
1115 if elasticsearch := app.settings.get("ELASTICSEARCH"):
1116 loop.run_until_complete(elasticsearch.close())
1117 finally:
1118 try:
1119 _cancel_all_tasks(loop)
1120 loop.run_until_complete(loop.shutdown_asyncgens())
1121 loop.run_until_complete(loop.shutdown_default_executor())
1122 finally:
1123 loop.close()
1124 background_tasks.HEARTBEAT = 0
1126 return len(tasks)