Coverage for an_website/main.py: 81.224%
245 statements
« prev ^ index » next coverage.py v7.8.0, created at 2025-05-07 13:44 +0000
1# This program is free software: you can redistribute it and/or modify
2# it under the terms of the GNU Affero General Public License as
3# published by the Free Software Foundation, either version 3 of the
4# License, or (at your option) any later version.
5#
6# This program is distributed in the hope that it will be useful,
7# but WITHOUT ANY WARRANTY; without even the implied warranty of
8# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9# GNU Affero General Public License for more details.
10#
11# You should have received a copy of the GNU Affero General Public License
12# along with this program. If not, see <https://www.gnu.org/licenses/>.
13# pylint: disable=import-private-name, too-many-lines
15"""
16The website of the AN.
18Loads config and modules and starts Tornado.
19"""
21from __future__ import annotations
23import asyncio
24import atexit
25import importlib
26import logging
27import os
28import platform
29import signal
30import ssl
31import sys
32import threading
33import time
34import types
35import uuid
36from asyncio import AbstractEventLoop
37from asyncio.runners import _cancel_all_tasks # type: ignore[attr-defined]
38from base64 import b64encode
39from collections.abc import Callable, Iterable, Mapping, MutableSequence
40from configparser import ConfigParser
41from functools import partial
42from hashlib import sha256
43from multiprocessing.process import _children # type: ignore[attr-defined]
44from pathlib import Path
45from socket import socket
46from typing import Any, Final, TypedDict, TypeGuard, cast
47from warnings import catch_warnings, simplefilter
48from zoneinfo import ZoneInfo
50import regex
51from Crypto.Hash import RIPEMD160
52from ecs_logging import StdlibFormatter
53from elastic_enterprise_search import AppSearch # type: ignore[import-untyped]
54from elasticapm.contrib.tornado import ElasticAPM
55from redis.asyncio import (
56 BlockingConnectionPool,
57 Redis,
58 SSLConnection,
59 UnixDomainSocketConnection,
60)
61from setproctitle import setproctitle
62from tornado.httpserver import HTTPServer
63from tornado.log import LogFormatter
64from tornado.netutil import bind_sockets, bind_unix_socket
65from tornado.process import fork_processes, task_id
66from tornado.web import Application, RedirectHandler
67from typed_stream import Stream
69from . import (
70 CA_BUNDLE_PATH,
71 DIR,
72 EVENT_SHUTDOWN,
73 NAME,
74 TEMPLATES_DIR,
75 UPTIME,
76 VERSION,
77 pytest_is_running,
78)
79from .contact.contact import apply_contact_stuff_to_app
80from .utils import background_tasks, static_file_handling
81from .utils.base_request_handler import BaseRequestHandler, request_ctx_var
82from .utils.better_config_parser import BetterConfigParser
83from .utils.elasticsearch_setup import setup_elasticsearch
84from .utils.logging import WebhookFormatter, WebhookHandler
85from .utils.request_handler import NotFoundHandler
86from .utils.static_file_from_traversable import TraversableStaticFileHandler
87from .utils.template_loader import TemplateLoader
88from .utils.utils import (
89 ArgparseNamespace,
90 Handler,
91 ModuleInfo,
92 Permission,
93 Timer,
94 create_argument_parser,
95 geoip,
96 get_arguments_without_help,
97 time_function,
98)
100try:
101 import perf8 # type: ignore[import, unused-ignore]
102except ModuleNotFoundError:
103 perf8 = None # pylint: disable=invalid-name
105IGNORED_MODULES: Final[set[str]] = {
106 "patches",
107 "static",
108 "templates",
109} | (set() if sys.flags.dev_mode or pytest_is_running() else {"example"})
111LOGGER: Final = logging.getLogger(__name__)
114# add all the information from the packages to a list
115# this calls the get_module_info function in every file
116# files and dirs starting with '_' get ignored
117def get_module_infos() -> str | tuple[ModuleInfo, ...]:
118 """Import the modules and return the loaded module infos in a tuple."""
119 module_infos: list[ModuleInfo] = []
120 loaded_modules: list[str] = []
121 errors: list[str] = []
123 for potential_module in DIR.iterdir():
124 if (
125 potential_module.name.startswith("_")
126 or potential_module.name in IGNORED_MODULES
127 or not potential_module.is_dir()
128 ):
129 continue
131 _module_infos = get_module_infos_from_module(
132 potential_module.name, errors, ignore_not_found=True
133 )
134 if _module_infos:
135 module_infos.extend(_module_infos)
136 loaded_modules.append(potential_module.name)
137 LOGGER.debug(
138 (
139 "Found module_infos in %s.__init__.py, "
140 "not searching in other modules in the package."
141 ),
142 potential_module,
143 )
144 continue
146 if f"{potential_module.name}.*" in IGNORED_MODULES:
147 continue
149 for potential_file in potential_module.iterdir():
150 module_name = f"{potential_module.name}.{potential_file.name[:-3]}"
151 if (
152 not potential_file.name.endswith(".py")
153 or module_name in IGNORED_MODULES
154 or potential_file.name.startswith("_")
155 ):
156 continue
157 _module_infos = get_module_infos_from_module(module_name, errors)
158 if _module_infos:
159 module_infos.extend(_module_infos)
160 loaded_modules.append(module_name)
162 if len(errors) > 0:
163 if sys.flags.dev_mode:
164 # exit to make sure it gets fixed
165 return "\n".join(errors)
166 # don't exit in production to keep stuff running
167 LOGGER.error("\n".join(errors))
169 LOGGER.info(
170 "Loaded %d modules: '%s'",
171 len(loaded_modules),
172 "', '".join(loaded_modules),
173 )
175 LOGGER.info(
176 "Ignored %d modules: '%s'",
177 len(IGNORED_MODULES),
178 "', '".join(IGNORED_MODULES),
179 )
181 sort_module_infos(module_infos)
183 # make module_infos immutable so it never changes
184 return tuple(module_infos)
187def get_module_infos_from_module(
188 module_name: str,
189 errors: MutableSequence[str], # gets modified
190 ignore_not_found: bool = False,
191) -> None | list[ModuleInfo]:
192 """Get the module infos based on a module."""
193 import_timer = Timer()
194 module = importlib.import_module(
195 f".{module_name}",
196 package="an_website",
197 )
198 if import_timer.stop() > 0.1:
199 LOGGER.warning(
200 "Import of %s took %ss. That's affecting the startup time.",
201 module_name,
202 import_timer.get(),
203 )
205 module_infos: list[ModuleInfo] = []
207 has_get_module_info = "get_module_info" in dir(module)
208 has_get_module_infos = "get_module_infos" in dir(module)
210 if not (has_get_module_info or has_get_module_infos):
211 if ignore_not_found:
212 return None
213 errors.append(
214 f"{module_name} has no 'get_module_info' and no 'get_module_infos' "
215 "method. Please add at least one of the methods or add "
216 f"'{module_name.rsplit('.', 1)[0]}.*' or {module_name!r} to "
217 "IGNORED_MODULES."
218 )
219 return None
221 if has_get_module_info and isinstance(
222 module_info := module.get_module_info(),
223 ModuleInfo,
224 ):
225 module_infos.append(module_info)
226 elif has_get_module_info:
227 errors.append(
228 f"'get_module_info' in {module_name} does not return ModuleInfo. "
229 "Please fix the returned value."
230 )
232 if not has_get_module_infos:
233 return module_infos or None
235 _module_infos = module.get_module_infos()
237 if not isinstance(_module_infos, Iterable):
238 errors.append(
239 f"'get_module_infos' in {module_name} does not return an Iterable. "
240 "Please fix the returned value."
241 )
242 return module_infos or None
244 for _module_info in _module_infos:
245 if isinstance(_module_info, ModuleInfo):
246 module_infos.append(_module_info)
247 else:
248 errors.append(
249 f"'get_module_infos' in {module_name} did return an Iterable "
250 f"with an element of type {type(_module_info)}. "
251 "Please fix the returned value."
252 )
254 return module_infos or None
257def sort_module_infos(module_infos: list[ModuleInfo]) -> None:
258 """Sort a list of module info and move the main page to the top."""
259 # sort it so the order makes sense
260 module_infos.sort()
262 # move the main page to the top
263 for i, info in enumerate(module_infos):
264 if info.path == "/":
265 module_infos.insert(0, module_infos.pop(i))
266 break
269def get_all_handlers(module_infos: Iterable[ModuleInfo]) -> list[Handler]:
270 """
271 Parse the module information and return the handlers in a tuple.
273 If a handler has only 2 elements a dict with title and description
274 gets added. This information is gotten from the module info.
275 """
276 handler: Handler | list[Any]
277 handlers: list[Handler] = static_file_handling.get_handlers()
279 # add all the normal handlers
280 for module_info in module_infos:
281 for handler in module_info.handlers:
282 handler = list(handler) # pylint: disable=redefined-loop-name
283 # if the handler is a request handler from us
284 # and not a built-in like StaticFileHandler & RedirectHandler
285 if issubclass(handler[1], BaseRequestHandler):
286 if len(handler) == 2:
287 # set "default_title" or "default_description" to False so
288 # that module_info.name & module_info.description get used
289 handler.append(
290 {
291 "default_title": False,
292 "default_description": False,
293 "module_info": module_info,
294 }
295 )
296 else:
297 handler[2]["module_info"] = module_info
298 handlers.append(tuple(handler))
300 # redirect handler, to make finding APIs easier
301 handlers.append((r"/(.+)/api/*", RedirectHandler, {"url": "/api/{0}"}))
303 handlers.append(
304 (
305 r"(?i)/\.well-known/(.*)",
306 TraversableStaticFileHandler,
307 {"root": Path(".well-known"), "hashes": {}},
308 )
309 )
311 LOGGER.debug("Loaded %d handlers", len(handlers))
313 return handlers
316def ignore_modules(config: BetterConfigParser) -> None:
317 """Read ignored modules from the config."""
318 IGNORED_MODULES.update(
319 config.getset("GENERAL", "IGNORED_MODULES", fallback=set())
320 )
323def get_normed_paths_from_module_infos(
324 module_infos: Iterable[ModuleInfo],
325) -> dict[str, str]:
326 """Get all paths from the module infos."""
328 def tuple_has_no_none(
329 value: tuple[str | None, str | None],
330 ) -> TypeGuard[tuple[str, str]]:
331 return None not in value
333 def info_to_paths(info: ModuleInfo) -> Stream[tuple[str, str]]:
334 return (
335 Stream(((info.path, info.path),))
336 .chain(
337 info.aliases.items()
338 if isinstance(info.aliases, Mapping)
339 else ((alias, info.path) for alias in info.aliases)
340 )
341 .chain(
342 Stream(info.sub_pages)
343 .map(lambda sub_info: sub_info.path)
344 .filter()
345 .map(lambda path: (path, path))
346 )
347 .filter(tuple_has_no_none)
348 )
350 return (
351 Stream(module_infos)
352 .flat_map(info_to_paths)
353 .filter(lambda p: p[0].startswith("/"))
354 .map(lambda p: (p[0].strip("/").lower(), p[1]))
355 .filter(lambda p: p[0])
356 .collect(dict)
357 )
360def make_app(config: ConfigParser) -> str | Application:
361 """Create the Tornado application and return it."""
362 module_infos, duration = time_function(get_module_infos)
363 if isinstance(module_infos, str):
364 return module_infos
365 if duration > 1:
366 LOGGER.warning(
367 "Getting the module infos took %ss. That's probably too long.",
368 duration,
369 )
370 handlers = get_all_handlers(module_infos)
371 return Application(
372 handlers, # type: ignore[arg-type]
373 MODULE_INFOS=module_infos,
374 SHOW_HAMBURGER_MENU=not Stream(module_infos)
375 .exclude(lambda info: info.hidden)
376 .filter(lambda info: info.path)
377 .empty(),
378 NORMED_PATHS=get_normed_paths_from_module_infos(module_infos),
379 HANDLERS=handlers,
380 # General settings
381 autoreload=False,
382 debug=sys.flags.dev_mode,
383 default_handler_class=NotFoundHandler,
384 compress_response=config.getboolean(
385 "GENERAL", "COMPRESS_RESPONSE", fallback=False
386 ),
387 websocket_ping_interval=10,
388 # Template settings
389 template_loader=TemplateLoader(
390 root=TEMPLATES_DIR, whitespace="oneline"
391 ),
392 )
395def apply_config_to_app(app: Application, config: BetterConfigParser) -> None:
396 """Apply the config (from the config.ini file) to the application."""
397 app.settings["CONFIG"] = config
399 app.settings["cookie_secret"] = config.get(
400 "GENERAL", "COOKIE_SECRET", fallback="xyzzy"
401 )
403 app.settings["CRAWLER_SECRET"] = config.get(
404 "APP_SEARCH", "CRAWLER_SECRET", fallback=None
405 )
407 app.settings["DOMAIN"] = config.get("GENERAL", "DOMAIN", fallback=None)
409 app.settings["ELASTICSEARCH_PREFIX"] = config.get(
410 "ELASTICSEARCH", "PREFIX", fallback=NAME
411 )
413 app.settings["HSTS"] = config.getboolean("TLS", "HSTS", fallback=False)
415 app.settings["NETCUP"] = config.getboolean(
416 "GENERAL", "NETCUP", fallback=False
417 )
419 app.settings["COMMITMENT_URI"] = config.get(
420 "GENERAL",
421 "COMMITMENT_URI",
422 fallback="https://github.asozial.org/an-website/commitment.txt",
423 )
425 onion_address = config.get("GENERAL", "ONION_ADDRESS", fallback=None)
426 app.settings["ONION_ADDRESS"] = onion_address
427 if onion_address is None:
428 app.settings["ONION_PROTOCOL"] = None
429 else:
430 app.settings["ONION_PROTOCOL"] = onion_address.split("://")[0]
432 app.settings["RATELIMITS"] = config.getboolean(
433 "GENERAL",
434 "RATELIMITS",
435 fallback=config.getboolean("REDIS", "ENABLED", fallback=False),
436 )
438 app.settings["REDIS_PREFIX"] = config.get("REDIS", "PREFIX", fallback=NAME)
440 app.settings["REPORTING"] = config.getboolean(
441 "REPORTING", "ENABLED", fallback=True
442 )
444 app.settings["REPORTING_BUILTIN"] = config.getboolean(
445 "REPORTING", "BUILTIN", fallback=sys.flags.dev_mode
446 )
448 app.settings["REPORTING_ENDPOINT"] = config.get(
449 "REPORTING",
450 "ENDPOINT",
451 fallback=(
452 "/api/reports"
453 if app.settings["REPORTING_BUILTIN"]
454 else "https://asozial.org/api/reports"
455 ),
456 )
458 app.settings["TRUSTED_API_SECRETS"] = {
459 key_perms[0]: Permission(
460 int(key_perms[1])
461 if len(key_perms) > 1
462 else (1 << len(Permission)) - 1 # should be all permissions
463 )
464 for secret in config.getset(
465 "GENERAL", "TRUSTED_API_SECRETS", fallback={"xyzzy"}
466 )
467 if (key_perms := [part.strip() for part in secret.split("=")])
468 if key_perms[0]
469 }
471 app.settings["AUTH_TOKEN_SECRET"] = config.get(
472 "GENERAL", "AUTH_TOKEN_SECRET", fallback=None
473 )
474 if not app.settings["AUTH_TOKEN_SECRET"]:
475 node = uuid.getnode().to_bytes(6, "big")
476 secret = RIPEMD160.new(node).digest().decode("BRAILLE")
477 LOGGER.warning(
478 "AUTH_TOKEN_SECRET is unset, implicitly setting it to %r",
479 secret,
480 )
481 app.settings["AUTH_TOKEN_SECRET"] = secret
483 app.settings["UNDER_ATTACK"] = config.getboolean(
484 "GENERAL", "UNDER_ATTACK", fallback=False
485 )
487 apply_contact_stuff_to_app(app, config)
490def get_ssl_context( # pragma: no cover
491 config: ConfigParser,
492) -> None | ssl.SSLContext:
493 """Create SSL context and configure using the config."""
494 if config.getboolean("TLS", "ENABLED", fallback=False):
495 ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
496 ssl_ctx.load_cert_chain(
497 config.get("TLS", "CERTFILE"),
498 config.get("TLS", "KEYFILE", fallback=None),
499 config.get("TLS", "PASSWORD", fallback=None),
500 )
501 return ssl_ctx
502 return None
505def setup_logging( # pragma: no cover
506 config: ConfigParser,
507 force: bool = False,
508) -> None:
509 """Setup logging.""" # noqa: D401
510 root_logger = logging.getLogger()
512 if root_logger.handlers:
513 if not force:
514 return
515 for handler in root_logger.handlers[:]:
516 root_logger.removeHandler(handler)
517 handler.close()
519 debug = config.getboolean("LOGGING", "DEBUG", fallback=sys.flags.dev_mode)
521 logging.captureWarnings(True)
523 root_logger.setLevel(logging.DEBUG if debug else logging.INFO)
524 logging.getLogger("tornado.curl_httpclient").setLevel(logging.INFO)
525 logging.getLogger("elasticsearch").setLevel(logging.INFO)
527 stream_handler = logging.StreamHandler()
528 if sys.flags.dev_mode:
529 spam = regex.sub(r"%\((end_)?color\)s", "", LogFormatter.DEFAULT_FORMAT)
530 formatter = logging.Formatter(spam, LogFormatter.DEFAULT_DATE_FORMAT)
531 else:
532 formatter = LogFormatter()
533 stream_handler.setFormatter(formatter)
534 root_logger.addHandler(stream_handler)
536 if path := config.get("LOGGING", "PATH", fallback=None):
537 os.makedirs(path, 0o755, True)
538 file_handler = logging.handlers.TimedRotatingFileHandler(
539 os.path.join(path, f"{NAME}.log"),
540 encoding="UTF-8",
541 when="midnight",
542 backupCount=30,
543 utc=True,
544 )
545 file_handler.setFormatter(StdlibFormatter())
546 root_logger.addHandler(file_handler)
549class WebhookLoggingOptions: # pylint: disable=too-few-public-methods
550 """Webhook logging options."""
552 __slots__ = (
553 "url",
554 "content_type",
555 "body_format",
556 "timestamp_format",
557 "timestamp_timezone",
558 "escape_message",
559 "max_message_length",
560 )
562 url: str | None
563 content_type: str
564 body_format: str
565 timestamp_format: str | None
566 timestamp_timezone: str | None
567 escape_message: bool
568 max_message_length: int | None
570 def __init__(self, config: ConfigParser) -> None:
571 """Initialize Webhook logging options."""
572 self.url = config.get("LOGGING", "WEBHOOK_URL", fallback=None)
573 self.content_type = config.get(
574 "LOGGING",
575 "WEBHOOK_CONTENT_TYPE",
576 fallback="application/json",
577 )
578 spam = regex.sub(r"%\((end_)?color\)s", "", LogFormatter.DEFAULT_FORMAT)
579 self.body_format = config.get(
580 "LOGGING",
581 "WEBHOOK_BODY_FORMAT",
582 fallback='{"text":"' + spam + '"}',
583 )
584 self.timestamp_format = config.get(
585 "LOGGING",
586 "WEBHOOK_TIMESTAMP_FORMAT",
587 fallback=None,
588 )
589 self.timestamp_timezone = config.get(
590 "LOGGING", "WEBHOOK_TIMESTAMP_TIMEZONE", fallback=None
591 )
592 self.escape_message = config.getboolean(
593 "LOGGING",
594 "WEBHOOK_ESCAPE_MESSAGE",
595 fallback=True,
596 )
597 self.max_message_length = config.getint(
598 "LOGGING", "WEBHOOK_MAX_MESSAGE_LENGTH", fallback=None
599 )
602def setup_webhook_logging( # pragma: no cover
603 options: WebhookLoggingOptions,
604 loop: asyncio.AbstractEventLoop,
605) -> None:
606 """Setup Webhook logging.""" # noqa: D401
607 if not options.url:
608 return
610 LOGGER.info("Setting up Webhook logging")
612 root_logger = logging.getLogger()
614 webhook_content_type = options.content_type
615 webhook_handler = WebhookHandler(
616 logging.ERROR,
617 loop=loop,
618 url=options.url,
619 content_type=webhook_content_type,
620 )
621 formatter = WebhookFormatter(
622 options.body_format,
623 options.timestamp_format,
624 )
625 formatter.timezone = (
626 None
627 if options.timestamp_format is None
628 else ZoneInfo(options.timestamp_format)
629 )
630 formatter.escape_message = options.escape_message
631 formatter.max_message_length = options.max_message_length
632 webhook_handler.setFormatter(formatter)
633 root_logger.addHandler(webhook_handler)
635 info_handler = WebhookHandler(
636 logging.INFO,
637 loop=loop,
638 url=options.url,
639 content_type=webhook_content_type,
640 )
641 info_handler.setFormatter(formatter)
642 logging.getLogger("an_website.quotes.create").addHandler(info_handler)
645def setup_apm(app: Application) -> None: # pragma: no cover
646 """Setup APM.""" # noqa: D401
647 config: BetterConfigParser = app.settings["CONFIG"]
648 app.settings["ELASTIC_APM"] = {
649 "ENABLED": config.getboolean("ELASTIC_APM", "ENABLED", fallback=False),
650 "SERVER_URL": config.get(
651 "ELASTIC_APM", "SERVER_URL", fallback="http://localhost:8200"
652 ),
653 "SECRET_TOKEN": config.get(
654 "ELASTIC_APM", "SECRET_TOKEN", fallback=None
655 ),
656 "API_KEY": config.get("ELASTIC_APM", "API_KEY", fallback=None),
657 "VERIFY_SERVER_CERT": config.getboolean(
658 "ELASTIC_APM", "VERIFY_SERVER_CERT", fallback=True
659 ),
660 "USE_CERTIFI": True, # doesn't actually use certifi
661 "SERVICE_NAME": NAME.removesuffix("-dev"),
662 "SERVICE_VERSION": VERSION,
663 "ENVIRONMENT": (
664 "production" if not sys.flags.dev_mode else "development"
665 ),
666 "DEBUG": True,
667 "CAPTURE_BODY": "errors",
668 "TRANSACTION_IGNORE_URLS": [
669 "/api/ping",
670 "/static/*",
671 "/favicon.png",
672 ],
673 "TRANSACTIONS_IGNORE_PATTERNS": ["^OPTIONS "],
674 "PROCESSORS": [
675 "an_website.utils.utils.apm_anonymization_processor",
676 "elasticapm.processors.sanitize_stacktrace_locals",
677 "elasticapm.processors.sanitize_http_request_cookies",
678 "elasticapm.processors.sanitize_http_headers",
679 "elasticapm.processors.sanitize_http_wsgi_env",
680 "elasticapm.processors.sanitize_http_request_body",
681 ],
682 "RUM_SERVER_URL": config.get(
683 "ELASTIC_APM", "RUM_SERVER_URL", fallback=None
684 ),
685 "RUM_SERVER_URL_PREFIX": config.get(
686 "ELASTIC_APM", "RUM_SERVER_URL_PREFIX", fallback=None
687 ),
688 }
690 script_options = [
691 f"serviceName:{app.settings['ELASTIC_APM']['SERVICE_NAME']!r}",
692 f"serviceVersion:{app.settings['ELASTIC_APM']['SERVICE_VERSION']!r}",
693 f"environment:{app.settings['ELASTIC_APM']['ENVIRONMENT']!r}",
694 ]
696 rum_server_url = app.settings["ELASTIC_APM"]["RUM_SERVER_URL"]
698 if rum_server_url is None:
699 script_options.append(
700 f"serverUrl:{app.settings['ELASTIC_APM']['SERVER_URL']!r}"
701 )
702 elif rum_server_url:
703 script_options.append(f"serverUrl:{rum_server_url!r}")
704 else:
705 script_options.append("serverUrl:window.location.origin")
707 if app.settings["ELASTIC_APM"]["RUM_SERVER_URL_PREFIX"]:
708 script_options.append(
709 f"serverUrlPrefix:{app.settings['ELASTIC_APM']['RUM_SERVER_URL_PREFIX']!r}"
710 )
712 app.settings["ELASTIC_APM"]["INLINE_SCRIPT"] = (
713 "elasticApm.init({" + ",".join(script_options) + "})"
714 )
716 app.settings["ELASTIC_APM"]["INLINE_SCRIPT_HASH"] = b64encode(
717 sha256(
718 app.settings["ELASTIC_APM"]["INLINE_SCRIPT"].encode("ASCII")
719 ).digest()
720 ).decode("ASCII")
722 if app.settings["ELASTIC_APM"]["ENABLED"]:
723 app.settings["ELASTIC_APM"]["CLIENT"] = ElasticAPM(app).client
726def setup_app_search(app: Application) -> None: # pragma: no cover
727 """Setup Elastic App Search.""" # noqa: D401
728 config: BetterConfigParser = app.settings["CONFIG"]
729 host = config.get("APP_SEARCH", "HOST", fallback=None)
730 key = config.get("APP_SEARCH", "SEARCH_KEY", fallback=None)
731 verify_certs = config.getboolean(
732 "APP_SEARCH", "VERIFY_CERTS", fallback=True
733 )
734 app.settings["APP_SEARCH"] = (
735 AppSearch(
736 host,
737 bearer_auth=key,
738 verify_certs=verify_certs,
739 ca_certs=CA_BUNDLE_PATH,
740 )
741 if host
742 else None
743 )
744 app.settings["APP_SEARCH_HOST"] = host
745 app.settings["APP_SEARCH_KEY"] = key
746 app.settings["APP_SEARCH_ENGINE"] = config.get(
747 "APP_SEARCH", "ENGINE_NAME", fallback=NAME.removesuffix("-dev")
748 )
751def setup_redis(app: Application) -> None | Redis[str]:
752 """Setup Redis.""" # noqa: D401
753 config: BetterConfigParser = app.settings["CONFIG"]
755 class Kwargs(TypedDict, total=False):
756 """Kwargs of BlockingConnectionPool constructor."""
758 db: int
759 username: None | str
760 password: None | str
761 retry_on_timeout: bool
762 connection_class: type[UnixDomainSocketConnection] | type[SSLConnection]
763 path: str
764 host: str
765 port: int
766 ssl_ca_certs: str
767 ssl_keyfile: None | str
768 ssl_certfile: None | str
769 ssl_check_hostname: bool
770 ssl_cert_reqs: str
772 kwargs: Kwargs = {
773 "db": config.getint("REDIS", "DB", fallback=0),
774 "username": config.get("REDIS", "USERNAME", fallback=None),
775 "password": config.get("REDIS", "PASSWORD", fallback=None),
776 "retry_on_timeout": config.getboolean(
777 "REDIS", "RETRY_ON_TIMEOUT", fallback=False
778 ),
779 }
780 redis_ssl_kwargs: Kwargs = {
781 "connection_class": SSLConnection,
782 "ssl_ca_certs": CA_BUNDLE_PATH,
783 "ssl_keyfile": config.get("REDIS", "SSL_KEYFILE", fallback=None),
784 "ssl_certfile": config.get("REDIS", "SSL_CERTFILE", fallback=None),
785 "ssl_cert_reqs": config.get(
786 "REDIS", "SSL_CERT_REQS", fallback="required"
787 ),
788 "ssl_check_hostname": config.getboolean(
789 "REDIS", "SSL_CHECK_HOSTNAME", fallback=False
790 ),
791 }
792 redis_host_port_kwargs: Kwargs = {
793 "host": config.get("REDIS", "HOST", fallback="localhost"),
794 "port": config.getint("REDIS", "PORT", fallback=6379),
795 }
796 redis_use_ssl = config.getboolean("REDIS", "SSL", fallback=False)
797 redis_unix_socket_path = config.get(
798 "REDIS", "UNIX_SOCKET_PATH", fallback=None
799 )
801 if redis_unix_socket_path is not None:
802 if redis_use_ssl:
803 LOGGER.warning(
804 "SSL is enabled for Redis, but a UNIX socket is used"
805 )
806 if config.has_option("REDIS", "HOST"):
807 LOGGER.warning(
808 "A host is configured for Redis, but a UNIX socket is used"
809 )
810 if config.has_option("REDIS", "PORT"):
811 LOGGER.warning(
812 "A port is configured for Redis, but a UNIX socket is used"
813 )
814 kwargs.update(
815 {
816 "connection_class": UnixDomainSocketConnection,
817 "path": redis_unix_socket_path,
818 }
819 )
820 else:
821 kwargs.update(redis_host_port_kwargs)
822 if redis_use_ssl:
823 kwargs.update(redis_ssl_kwargs)
825 if not config.getboolean("REDIS", "ENABLED", fallback=False):
826 app.settings["REDIS"] = None
827 return None
828 connection_pool = BlockingConnectionPool(
829 client_name=NAME,
830 decode_responses=True,
831 **kwargs,
832 )
833 redis = cast("Redis[str]", Redis(connection_pool=connection_pool))
834 app.settings["REDIS"] = redis
835 return redis
838def signal_handler( # noqa: D103 # pragma: no cover
839 signalnum: int, frame: None | types.FrameType
840) -> None:
841 # pylint: disable=unused-argument, missing-function-docstring
842 if signalnum in {signal.SIGINT, signal.SIGTERM}:
843 EVENT_SHUTDOWN.set()
844 if signalnum == getattr(signal, "SIGHUP", None):
845 EVENT_SHUTDOWN.set()
848def install_signal_handler() -> None: # pragma: no cover
849 """Install the signal handler."""
850 signal.signal(signal.SIGINT, signal_handler)
851 signal.signal(signal.SIGTERM, signal_handler)
852 if hasattr(signal, "SIGHUP"):
853 signal.signal(signal.SIGHUP, signal_handler)
856def supervise(loop: AbstractEventLoop) -> None:
857 """Supervise."""
858 while foobarbaz := background_tasks.HEARTBEAT: # pylint: disable=while-used
859 if time.monotonic() - foobarbaz >= 10:
860 worker = task_id()
861 pid = os.getpid()
863 task = asyncio.current_task(loop)
864 request = task.get_context().get(request_ctx_var) if task else None
866 LOGGER.fatal(
867 "Heartbeat timed out for worker %s (pid %d), "
868 "current request: %s, current task: %s",
869 worker,
870 pid,
871 request,
872 task,
873 )
874 atexit._run_exitfuncs() # pylint: disable=protected-access
875 os.abort()
876 time.sleep(1)
879def main( # noqa: C901 # pragma: no cover
880 config: BetterConfigParser | None = None,
881) -> int | str:
882 """
883 Start everything.
885 This is the main function that is called when running this program.
886 """
887 # pylint: disable=too-complex, too-many-branches
888 # pylint: disable=too-many-locals, too-many-statements
889 setproctitle(NAME)
891 install_signal_handler()
893 parser = create_argument_parser()
894 args, _ = parser.parse_known_args(
895 get_arguments_without_help(), ArgparseNamespace()
896 )
898 if args.version:
899 print("Version:", VERSION)
900 if args.verbose:
901 # pylint: disable-next=import-outside-toplevel
902 from .version.version import (
903 get_file_hashes,
904 get_hash_of_file_hashes,
905 )
907 print()
908 print("Hash der Datei-Hashes:")
909 print(get_hash_of_file_hashes())
911 if args.verbose > 1:
912 print()
913 print("Datei-Hashes:")
914 print(get_file_hashes())
916 return 0
918 config = config or BetterConfigParser.from_path(*args.config)
919 assert config is not None
920 config.add_override_argument_parser(parser)
922 setup_logging(config)
924 LOGGER.info("Starting %s %s", NAME, VERSION)
926 if platform.system() == "Windows":
927 LOGGER.warning(
928 "Running %s on Windows is not officially supported",
929 NAME.removesuffix("-dev"),
930 )
932 ignore_modules(config)
933 app = make_app(config)
934 if isinstance(app, str):
935 return app
937 apply_config_to_app(app, config)
938 setup_elasticsearch(app)
939 setup_app_search(app)
940 setup_redis(app)
941 setup_apm(app)
943 behind_proxy = config.getboolean("GENERAL", "BEHIND_PROXY", fallback=False)
945 server = HTTPServer(
946 app,
947 body_timeout=3600,
948 decompress_request=True,
949 max_body_size=1_000_000_000,
950 ssl_options=get_ssl_context(config),
951 xheaders=behind_proxy,
952 )
954 socket_factories: list[Callable[[], Iterable[socket]]] = []
956 port = config.getint("GENERAL", "PORT", fallback=None)
958 if port:
959 socket_factories.append(
960 partial(
961 bind_sockets,
962 port,
963 "localhost" if behind_proxy else "",
964 )
965 )
967 unix_socket_path = config.get(
968 "GENERAL",
969 "UNIX_SOCKET_PATH",
970 fallback=None,
971 )
973 if unix_socket_path:
974 os.makedirs(unix_socket_path, 0o755, True)
975 socket_factories.append(
976 lambda: (
977 bind_unix_socket(
978 os.path.join(unix_socket_path, f"{NAME}.sock"),
979 mode=0o666,
980 ),
981 )
982 )
984 processes = config.getint(
985 "GENERAL",
986 "PROCESSES",
987 fallback=hasattr(os, "fork") * (2 if sys.flags.dev_mode else -1),
988 )
990 if processes < 0:
991 processes = (
992 os.process_cpu_count() # type: ignore[attr-defined]
993 if sys.version_info >= (3, 13)
994 else os.cpu_count()
995 ) or 0
997 worker: None | int = None
999 run_supervisor_thread = config.getboolean(
1000 "GENERAL", "SUPERVISE", fallback=False
1001 )
1002 elasticsearch_is_enabled = config.getboolean(
1003 "ELASTICSEARCH", "ENABLED", fallback=False
1004 )
1005 redis_is_enabled = config.getboolean("REDIS", "ENABLED", fallback=False)
1006 webhook_logging_options = WebhookLoggingOptions(config)
1007 # all config options should be read before forking
1008 if args.save_config_to:
1009 with open(args.save_config_to, "w", encoding="UTF-8") as file:
1010 config.write(file)
1011 config.set_all_options_should_be_parsed()
1012 del config
1013 # show help message if --help is given (after reading config, before forking)
1014 parser.parse_args()
1016 if not socket_factories:
1017 LOGGER.warning("No sockets configured")
1018 return 0
1020 # create sockets after checking for --help
1021 sockets: list[socket] = (
1022 Stream(socket_factories).flat_map(lambda fun: fun()).collect(list)
1023 )
1025 UPTIME.reset()
1026 main_pid = os.getpid()
1028 if processes:
1029 setproctitle(f"{NAME} - Master")
1031 worker = fork_processes(processes)
1033 setproctitle(f"{NAME} - Worker {worker}")
1035 # yeet all children (there should be none, but do it regardless, just in case)
1036 _children.clear()
1038 if "an_website.quotes" in sys.modules:
1039 from .quotes.utils import ( # pylint: disable=import-outside-toplevel
1040 AUTHORS_CACHE,
1041 QUOTES_CACHE,
1042 WRONG_QUOTES_CACHE,
1043 )
1045 del AUTHORS_CACHE.control.created_by_ultra # type: ignore[attr-defined]
1046 del QUOTES_CACHE.control.created_by_ultra # type: ignore[attr-defined]
1047 del WRONG_QUOTES_CACHE.control.created_by_ultra # type: ignore[attr-defined]
1048 del geoip.__kwdefaults__["caches"].control.created_by_ultra
1050 if unix_socket_path:
1051 sockets.append(
1052 bind_unix_socket(
1053 os.path.join(unix_socket_path, f"{NAME}.{worker}.sock"),
1054 mode=0o666,
1055 )
1056 )
1058 # get loop after forking
1059 # if not forking allow loop to be set in advance by external code
1060 loop: None | asyncio.AbstractEventLoop
1061 try:
1062 with catch_warnings(): # TODO: remove after dropping support for 3.13
1063 simplefilter("ignore", DeprecationWarning)
1064 loop = asyncio.get_event_loop()
1065 if loop.is_closed():
1066 loop = None
1067 except RuntimeError:
1068 loop = None
1070 if loop is None:
1071 loop = asyncio.new_event_loop()
1072 asyncio.set_event_loop(loop)
1074 if sys.version_info >= (3, 13) and not loop.get_task_factory():
1075 loop.set_task_factory(asyncio.eager_task_factory)
1077 if perf8 and "PERF8" in os.environ:
1078 loop.run_until_complete(perf8.enable())
1080 setup_webhook_logging(webhook_logging_options, loop)
1082 server.add_sockets(sockets)
1084 tasks = background_tasks.start_background_tasks( # noqa: F841
1085 module_infos=app.settings["MODULE_INFOS"],
1086 loop=loop,
1087 main_pid=main_pid,
1088 app=app,
1089 processes=processes,
1090 elasticsearch_is_enabled=elasticsearch_is_enabled,
1091 redis_is_enabled=redis_is_enabled,
1092 worker=worker,
1093 )
1095 if run_supervisor_thread:
1096 background_tasks.HEARTBEAT = time.monotonic()
1097 threading.Thread(
1098 target=supervise, args=(loop,), name="supervisor", daemon=True
1099 ).start()
1101 try:
1102 loop.run_forever()
1103 EVENT_SHUTDOWN.set()
1104 finally:
1105 try: # pylint: disable=too-many-try-statements
1106 server.stop()
1107 loop.run_until_complete(asyncio.sleep(1))
1108 loop.run_until_complete(server.close_all_connections())
1109 if perf8 and "PERF8" in os.environ:
1110 loop.run_until_complete(perf8.disable())
1111 if redis := app.settings.get("REDIS"):
1112 loop.run_until_complete(
1113 redis.aclose(close_connection_pool=True)
1114 )
1115 if elasticsearch := app.settings.get("ELASTICSEARCH"):
1116 loop.run_until_complete(elasticsearch.close())
1117 finally:
1118 try:
1119 _cancel_all_tasks(loop)
1120 loop.run_until_complete(loop.shutdown_asyncgens())
1121 loop.run_until_complete(loop.shutdown_default_executor())
1122 finally:
1123 loop.close()
1124 background_tasks.HEARTBEAT = 0
1126 return len(tasks)