Coverage for an_website/main.py: 81.070%

243 statements  

« prev     ^ index     » next       coverage.py v7.9.1, created at 2025-06-26 12:07 +0000

1# This program is free software: you can redistribute it and/or modify 

2# it under the terms of the GNU Affero General Public License as 

3# published by the Free Software Foundation, either version 3 of the 

4# License, or (at your option) any later version. 

5# 

6# This program is distributed in the hope that it will be useful, 

7# but WITHOUT ANY WARRANTY; without even the implied warranty of 

8# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

9# GNU Affero General Public License for more details. 

10# 

11# You should have received a copy of the GNU Affero General Public License 

12# along with this program. If not, see <https://www.gnu.org/licenses/>. 

13# pylint: disable=import-private-name, too-many-lines 

14 

15""" 

16The website of the AN. 

17 

18Loads config and modules and starts Tornado. 

19""" 

20 

21from __future__ import annotations 

22 

23import asyncio 

24import atexit 

25import importlib 

26import logging 

27import os 

28import platform 

29import signal 

30import ssl 

31import sys 

32import threading 

33import time 

34import types 

35import uuid 

36from asyncio import AbstractEventLoop 

37from asyncio.runners import _cancel_all_tasks # type: ignore[attr-defined] 

38from base64 import b64encode 

39from collections.abc import Callable, Iterable, Mapping, MutableSequence 

40from configparser import ConfigParser 

41from functools import partial 

42from hashlib import sha256 

43from multiprocessing.process import _children # type: ignore[attr-defined] 

44from pathlib import Path 

45from socket import socket 

46from typing import Any, Final, TypedDict, TypeGuard, cast 

47from warnings import catch_warnings, simplefilter 

48from zoneinfo import ZoneInfo 

49 

50import regex 

51from Crypto.Hash import RIPEMD160 

52from ecs_logging import StdlibFormatter 

53from elasticapm.contrib.tornado import ElasticAPM 

54from redis.asyncio import ( 

55 BlockingConnectionPool, 

56 Redis, 

57 SSLConnection, 

58 UnixDomainSocketConnection, 

59) 

60from setproctitle import setproctitle 

61from tornado.httpserver import HTTPServer 

62from tornado.log import LogFormatter 

63from tornado.netutil import bind_sockets, bind_unix_socket 

64from tornado.process import fork_processes, task_id 

65from tornado.web import Application, RedirectHandler 

66from typed_stream import Stream 

67 

68from . import ( 

69 CA_BUNDLE_PATH, 

70 DIR, 

71 EVENT_SHUTDOWN, 

72 NAME, 

73 TEMPLATES_DIR, 

74 UPTIME, 

75 VERSION, 

76 pytest_is_running, 

77) 

78from .contact.contact import apply_contact_stuff_to_app 

79from .utils import background_tasks, static_file_handling 

80from .utils.base_request_handler import BaseRequestHandler, request_ctx_var 

81from .utils.better_config_parser import BetterConfigParser 

82from .utils.elasticsearch_setup import setup_elasticsearch 

83from .utils.logging import WebhookFormatter, WebhookHandler 

84from .utils.request_handler import NotFoundHandler 

85from .utils.static_file_from_traversable import TraversableStaticFileHandler 

86from .utils.template_loader import TemplateLoader 

87from .utils.utils import ( 

88 ArgparseNamespace, 

89 Handler, 

90 ModuleInfo, 

91 Permission, 

92 Timer, 

93 create_argument_parser, 

94 geoip, 

95 get_arguments_without_help, 

96 time_function, 

97) 

98 

99try: 

100 import perf8 # type: ignore[import, unused-ignore] 

101except ModuleNotFoundError: 

102 perf8 = None # pylint: disable=invalid-name 

103 

104IGNORED_MODULES: Final[set[str]] = { 

105 "patches", 

106 "static", 

107 "templates", 

108} | (set() if sys.flags.dev_mode or pytest_is_running() else {"example"}) 

109 

110LOGGER: Final = logging.getLogger(__name__) 

111 

112 

113# add all the information from the packages to a list 

114# this calls the get_module_info function in every file 

115# files and dirs starting with '_' get ignored 

116def get_module_infos() -> str | tuple[ModuleInfo, ...]: 

117 """Import the modules and return the loaded module infos in a tuple.""" 

118 module_infos: list[ModuleInfo] = [] 

119 loaded_modules: list[str] = [] 

120 errors: list[str] = [] 

121 

122 for potential_module in DIR.iterdir(): 

123 if ( 

124 potential_module.name.startswith("_") 

125 or potential_module.name in IGNORED_MODULES 

126 or not potential_module.is_dir() 

127 ): 

128 continue 

129 

130 _module_infos = get_module_infos_from_module( 

131 potential_module.name, errors, ignore_not_found=True 

132 ) 

133 if _module_infos: 

134 module_infos.extend(_module_infos) 

135 loaded_modules.append(potential_module.name) 

136 LOGGER.debug( 

137 ( 

138 "Found module_infos in %s.__init__.py, " 

139 "not searching in other modules in the package." 

140 ), 

141 potential_module, 

142 ) 

143 continue 

144 

145 if f"{potential_module.name}.*" in IGNORED_MODULES: 

146 continue 

147 

148 for potential_file in potential_module.iterdir(): 

149 module_name = f"{potential_module.name}.{potential_file.name[:-3]}" 

150 if ( 

151 not potential_file.name.endswith(".py") 

152 or module_name in IGNORED_MODULES 

153 or potential_file.name.startswith("_") 

154 ): 

155 continue 

156 _module_infos = get_module_infos_from_module(module_name, errors) 

157 if _module_infos: 

158 module_infos.extend(_module_infos) 

159 loaded_modules.append(module_name) 

160 

161 if len(errors) > 0: 

162 if sys.flags.dev_mode: 

163 # exit to make sure it gets fixed 

164 return "\n".join(errors) 

165 # don't exit in production to keep stuff running 

166 LOGGER.error("\n".join(errors)) 

167 

168 LOGGER.info( 

169 "Loaded %d modules: '%s'", 

170 len(loaded_modules), 

171 "', '".join(loaded_modules), 

172 ) 

173 

174 LOGGER.info( 

175 "Ignored %d modules: '%s'", 

176 len(IGNORED_MODULES), 

177 "', '".join(IGNORED_MODULES), 

178 ) 

179 

180 sort_module_infos(module_infos) 

181 

182 # make module_infos immutable so it never changes 

183 return tuple(module_infos) 

184 

185 

186def get_module_infos_from_module( 

187 module_name: str, 

188 errors: MutableSequence[str], # gets modified 

189 ignore_not_found: bool = False, 

190) -> None | list[ModuleInfo]: 

191 """Get the module infos based on a module.""" 

192 import_timer = Timer() 

193 module = importlib.import_module( 

194 f".{module_name}", 

195 package="an_website", 

196 ) 

197 if import_timer.stop() > 0.1: 

198 LOGGER.warning( 

199 "Import of %s took %ss. That's affecting the startup time.", 

200 module_name, 

201 import_timer.get(), 

202 ) 

203 

204 module_infos: list[ModuleInfo] = [] 

205 

206 has_get_module_info = "get_module_info" in dir(module) 

207 has_get_module_infos = "get_module_infos" in dir(module) 

208 

209 if not (has_get_module_info or has_get_module_infos): 

210 if ignore_not_found: 

211 return None 

212 errors.append( 

213 f"{module_name} has no 'get_module_info' and no 'get_module_infos' " 

214 "method. Please add at least one of the methods or add " 

215 f"'{module_name.rsplit('.', 1)[0]}.*' or {module_name!r} to " 

216 "IGNORED_MODULES." 

217 ) 

218 return None 

219 

220 if has_get_module_info and isinstance( 

221 module_info := module.get_module_info(), 

222 ModuleInfo, 

223 ): 

224 module_infos.append(module_info) 

225 elif has_get_module_info: 

226 errors.append( 

227 f"'get_module_info' in {module_name} does not return ModuleInfo. " 

228 "Please fix the returned value." 

229 ) 

230 

231 if not has_get_module_infos: 

232 return module_infos or None 

233 

234 _module_infos = module.get_module_infos() 

235 

236 if not isinstance(_module_infos, Iterable): 

237 errors.append( 

238 f"'get_module_infos' in {module_name} does not return an Iterable. " 

239 "Please fix the returned value." 

240 ) 

241 return module_infos or None 

242 

243 for _module_info in _module_infos: 

244 if isinstance(_module_info, ModuleInfo): 

245 module_infos.append(_module_info) 

246 else: 

247 errors.append( 

248 f"'get_module_infos' in {module_name} did return an Iterable " 

249 f"with an element of type {type(_module_info)}. " 

250 "Please fix the returned value." 

251 ) 

252 

253 return module_infos or None 

254 

255 

256def sort_module_infos(module_infos: list[ModuleInfo]) -> None: 

257 """Sort a list of module info and move the main page to the top.""" 

258 # sort it so the order makes sense 

259 module_infos.sort() 

260 

261 # move the main page to the top 

262 for i, info in enumerate(module_infos): 

263 if info.path == "/": 

264 module_infos.insert(0, module_infos.pop(i)) 

265 break 

266 

267 

268def get_all_handlers(module_infos: Iterable[ModuleInfo]) -> list[Handler]: 

269 """ 

270 Parse the module information and return the handlers in a tuple. 

271 

272 If a handler has only 2 elements a dict with title and description 

273 gets added. This information is gotten from the module info. 

274 """ 

275 handler: Handler | list[Any] 

276 handlers: list[Handler] = static_file_handling.get_handlers() 

277 

278 # add all the normal handlers 

279 for module_info in module_infos: 

280 for handler in module_info.handlers: 

281 handler = list(handler) # pylint: disable=redefined-loop-name 

282 # if the handler is a request handler from us 

283 # and not a built-in like StaticFileHandler & RedirectHandler 

284 if issubclass(handler[1], BaseRequestHandler): 

285 if len(handler) == 2: 

286 # set "default_title" or "default_description" to False so 

287 # that module_info.name & module_info.description get used 

288 handler.append( 

289 { 

290 "default_title": False, 

291 "default_description": False, 

292 "module_info": module_info, 

293 } 

294 ) 

295 else: 

296 handler[2]["module_info"] = module_info 

297 handlers.append(tuple(handler)) 

298 

299 # redirect handler, to make finding APIs easier 

300 handlers.append((r"/(.+)/api/*", RedirectHandler, {"url": "/api/{0}"})) 

301 

302 handlers.append( 

303 ( 

304 r"(?i)/\.well-known/(.*)", 

305 TraversableStaticFileHandler, 

306 { 

307 "root": Path(".well-known"), 

308 "headers": (("Access-Control-Allow-Origin", "*"),), 

309 }, 

310 ) 

311 ) 

312 

313 LOGGER.debug("Loaded %d handlers", len(handlers)) 

314 

315 return handlers 

316 

317 

318def ignore_modules(config: BetterConfigParser) -> None: 

319 """Read ignored modules from the config.""" 

320 IGNORED_MODULES.update( 

321 config.getset("GENERAL", "IGNORED_MODULES", fallback=set()) 

322 ) 

323 

324 

325def get_normed_paths_from_module_infos( 

326 module_infos: Iterable[ModuleInfo], 

327) -> dict[str, str]: 

328 """Get all paths from the module infos.""" 

329 

330 def tuple_has_no_none( 

331 value: tuple[str | None, str | None], 

332 ) -> TypeGuard[tuple[str, str]]: 

333 return None not in value 

334 

335 def info_to_paths(info: ModuleInfo) -> Stream[tuple[str, str]]: 

336 return ( 

337 Stream(((info.path, info.path),)) 

338 .chain( 

339 info.aliases.items() 

340 if isinstance(info.aliases, Mapping) 

341 else ((alias, info.path) for alias in info.aliases) 

342 ) 

343 .chain( 

344 Stream(info.sub_pages) 

345 .map(lambda sub_info: sub_info.path) 

346 .filter() 

347 .map(lambda path: (path, path)) 

348 ) 

349 .filter(tuple_has_no_none) 

350 ) 

351 

352 return ( 

353 Stream(module_infos) 

354 .flat_map(info_to_paths) 

355 .filter(lambda p: p[0].startswith("/")) 

356 .map(lambda p: (p[0].strip("/").lower(), p[1])) 

357 .filter(lambda p: p[0]) 

358 .collect(dict) 

359 ) 

360 

361 

362def make_app(config: ConfigParser) -> str | Application: 

363 """Create the Tornado application and return it.""" 

364 module_infos, duration = time_function(get_module_infos) 

365 if isinstance(module_infos, str): 

366 return module_infos 

367 if duration > 1: 

368 LOGGER.warning( 

369 "Getting the module infos took %ss. That's probably too long.", 

370 duration, 

371 ) 

372 handlers = get_all_handlers(module_infos) 

373 return Application( 

374 handlers, 

375 MODULE_INFOS=module_infos, 

376 SHOW_HAMBURGER_MENU=not Stream(module_infos) 

377 .exclude(lambda info: info.hidden) 

378 .filter(lambda info: info.path) 

379 .empty(), 

380 NORMED_PATHS=get_normed_paths_from_module_infos(module_infos), 

381 HANDLERS=handlers, 

382 # General settings 

383 autoreload=False, 

384 debug=sys.flags.dev_mode, 

385 default_handler_class=NotFoundHandler, 

386 compress_response=config.getboolean( 

387 "GENERAL", "COMPRESS_RESPONSE", fallback=False 

388 ), 

389 websocket_ping_interval=10, 

390 # Template settings 

391 template_loader=TemplateLoader( 

392 root=TEMPLATES_DIR, whitespace="oneline" 

393 ), 

394 ) 

395 

396 

397def apply_config_to_app(app: Application, config: BetterConfigParser) -> None: 

398 """Apply the config (from the config.ini file) to the application.""" 

399 app.settings["CONFIG"] = config 

400 

401 app.settings["cookie_secret"] = config.get( 

402 "GENERAL", "COOKIE_SECRET", fallback="xyzzy" 

403 ) 

404 

405 app.settings["CRAWLER_SECRET"] = config.get( 

406 "APP_SEARCH", "CRAWLER_SECRET", fallback=None 

407 ) 

408 

409 app.settings["DOMAIN"] = config.get("GENERAL", "DOMAIN", fallback=None) 

410 

411 app.settings["ELASTICSEARCH_PREFIX"] = config.get( 

412 "ELASTICSEARCH", "PREFIX", fallback=NAME 

413 ) 

414 

415 app.settings["HSTS"] = config.getboolean("TLS", "HSTS", fallback=False) 

416 

417 app.settings["NETCUP"] = config.getboolean( 

418 "GENERAL", "NETCUP", fallback=False 

419 ) 

420 

421 onion_address = config.get("GENERAL", "ONION_ADDRESS", fallback=None) 

422 app.settings["ONION_ADDRESS"] = onion_address 

423 if onion_address is None: 

424 app.settings["ONION_PROTOCOL"] = None 

425 else: 

426 app.settings["ONION_PROTOCOL"] = onion_address.split("://")[0] 

427 

428 app.settings["RATELIMITS"] = config.getboolean( 

429 "GENERAL", 

430 "RATELIMITS", 

431 fallback=config.getboolean("REDIS", "ENABLED", fallback=False), 

432 ) 

433 

434 app.settings["REDIS_PREFIX"] = config.get("REDIS", "PREFIX", fallback=NAME) 

435 

436 app.settings["REPORTING"] = config.getboolean( 

437 "REPORTING", "ENABLED", fallback=True 

438 ) 

439 

440 app.settings["REPORTING_BUILTIN"] = config.getboolean( 

441 "REPORTING", "BUILTIN", fallback=sys.flags.dev_mode 

442 ) 

443 

444 app.settings["REPORTING_ENDPOINT"] = config.get( 

445 "REPORTING", 

446 "ENDPOINT", 

447 fallback=( 

448 "/api/reports" 

449 if app.settings["REPORTING_BUILTIN"] 

450 else "https://asozial.org/api/reports" 

451 ), 

452 ) 

453 

454 app.settings["TRUSTED_API_SECRETS"] = { 

455 key_perms[0]: Permission( 

456 int(key_perms[1]) 

457 if len(key_perms) > 1 

458 else (1 << len(Permission)) - 1 # should be all permissions 

459 ) 

460 for secret in config.getset( 

461 "GENERAL", "TRUSTED_API_SECRETS", fallback={"xyzzy"} 

462 ) 

463 if (key_perms := [part.strip() for part in secret.split("=")]) 

464 if key_perms[0] 

465 } 

466 

467 app.settings["AUTH_TOKEN_SECRET"] = config.get( 

468 "GENERAL", "AUTH_TOKEN_SECRET", fallback=None 

469 ) 

470 if not app.settings["AUTH_TOKEN_SECRET"]: 

471 node = uuid.getnode().to_bytes(6, "big") 

472 secret = RIPEMD160.new(node).digest().decode("BRAILLE") 

473 LOGGER.warning( 

474 "AUTH_TOKEN_SECRET is unset, implicitly setting it to %r", 

475 secret, 

476 ) 

477 app.settings["AUTH_TOKEN_SECRET"] = secret 

478 

479 app.settings["UNDER_ATTACK"] = config.getboolean( 

480 "GENERAL", "UNDER_ATTACK", fallback=False 

481 ) 

482 

483 apply_contact_stuff_to_app(app, config) 

484 

485 

486def get_ssl_context( # pragma: no cover 

487 config: ConfigParser, 

488) -> None | ssl.SSLContext: 

489 """Create SSL context and configure using the config.""" 

490 if config.getboolean("TLS", "ENABLED", fallback=False): 

491 ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) 

492 ssl_ctx.load_cert_chain( 

493 config.get("TLS", "CERTFILE"), 

494 config.get("TLS", "KEYFILE", fallback=None), 

495 config.get("TLS", "PASSWORD", fallback=None), 

496 ) 

497 return ssl_ctx 

498 return None 

499 

500 

501def setup_logging( # pragma: no cover 

502 config: ConfigParser, 

503 force: bool = False, 

504) -> None: 

505 """Setup logging.""" # noqa: D401 

506 root_logger = logging.getLogger() 

507 

508 if root_logger.handlers: 

509 if not force: 

510 return 

511 for handler in root_logger.handlers[:]: 

512 root_logger.removeHandler(handler) 

513 handler.close() 

514 

515 debug = config.getboolean("LOGGING", "DEBUG", fallback=sys.flags.dev_mode) 

516 

517 logging.captureWarnings(True) 

518 

519 root_logger.setLevel(logging.DEBUG if debug else logging.INFO) 

520 logging.getLogger("tornado.curl_httpclient").setLevel(logging.INFO) 

521 logging.getLogger("elasticsearch").setLevel(logging.INFO) 

522 

523 stream_handler = logging.StreamHandler() 

524 if sys.flags.dev_mode: 

525 spam = regex.sub(r"%\((end_)?color\)s", "", LogFormatter.DEFAULT_FORMAT) 

526 formatter = logging.Formatter(spam, LogFormatter.DEFAULT_DATE_FORMAT) 

527 else: 

528 formatter = LogFormatter() 

529 stream_handler.setFormatter(formatter) 

530 root_logger.addHandler(stream_handler) 

531 

532 if path := config.get("LOGGING", "PATH", fallback=None): 

533 os.makedirs(path, 0o755, True) 

534 file_handler = logging.handlers.TimedRotatingFileHandler( 

535 os.path.join(path, f"{NAME}.log"), 

536 encoding="UTF-8", 

537 when="midnight", 

538 backupCount=30, 

539 utc=True, 

540 ) 

541 file_handler.setFormatter(StdlibFormatter()) 

542 root_logger.addHandler(file_handler) 

543 

544 

545class WebhookLoggingOptions: # pylint: disable=too-few-public-methods 

546 """Webhook logging options.""" 

547 

548 __slots__ = ( 

549 "url", 

550 "content_type", 

551 "body_format", 

552 "timestamp_format", 

553 "timestamp_timezone", 

554 "escape_message", 

555 "max_message_length", 

556 ) 

557 

558 url: str | None 

559 content_type: str 

560 body_format: str 

561 timestamp_format: str | None 

562 timestamp_timezone: str | None 

563 escape_message: bool 

564 max_message_length: int | None 

565 

566 def __init__(self, config: ConfigParser) -> None: 

567 """Initialize Webhook logging options.""" 

568 self.url = config.get("LOGGING", "WEBHOOK_URL", fallback=None) 

569 self.content_type = config.get( 

570 "LOGGING", 

571 "WEBHOOK_CONTENT_TYPE", 

572 fallback="application/json", 

573 ) 

574 spam = regex.sub(r"%\((end_)?color\)s", "", LogFormatter.DEFAULT_FORMAT) 

575 self.body_format = config.get( 

576 "LOGGING", 

577 "WEBHOOK_BODY_FORMAT", 

578 fallback='{"text":"' + spam + '"}', 

579 ) 

580 self.timestamp_format = config.get( 

581 "LOGGING", 

582 "WEBHOOK_TIMESTAMP_FORMAT", 

583 fallback=None, 

584 ) 

585 self.timestamp_timezone = config.get( 

586 "LOGGING", "WEBHOOK_TIMESTAMP_TIMEZONE", fallback=None 

587 ) 

588 self.escape_message = config.getboolean( 

589 "LOGGING", 

590 "WEBHOOK_ESCAPE_MESSAGE", 

591 fallback=True, 

592 ) 

593 self.max_message_length = config.getint( 

594 "LOGGING", "WEBHOOK_MAX_MESSAGE_LENGTH", fallback=None 

595 ) 

596 

597 

598def setup_webhook_logging( # pragma: no cover 

599 options: WebhookLoggingOptions, 

600 loop: asyncio.AbstractEventLoop, 

601) -> None: 

602 """Setup Webhook logging.""" # noqa: D401 

603 if not options.url: 

604 return 

605 

606 LOGGER.info("Setting up Webhook logging") 

607 

608 root_logger = logging.getLogger() 

609 

610 webhook_content_type = options.content_type 

611 webhook_handler = WebhookHandler( 

612 logging.ERROR, 

613 loop=loop, 

614 url=options.url, 

615 content_type=webhook_content_type, 

616 ) 

617 formatter = WebhookFormatter( 

618 options.body_format, 

619 options.timestamp_format, 

620 ) 

621 formatter.timezone = ( 

622 None 

623 if options.timestamp_format is None 

624 else ZoneInfo(options.timestamp_format) 

625 ) 

626 formatter.escape_message = options.escape_message 

627 formatter.max_message_length = options.max_message_length 

628 webhook_handler.setFormatter(formatter) 

629 root_logger.addHandler(webhook_handler) 

630 

631 info_handler = WebhookHandler( 

632 logging.INFO, 

633 loop=loop, 

634 url=options.url, 

635 content_type=webhook_content_type, 

636 ) 

637 info_handler.setFormatter(formatter) 

638 logging.getLogger("an_website.quotes.create").addHandler(info_handler) 

639 

640 

641def setup_apm(app: Application) -> None: # pragma: no cover 

642 """Setup APM.""" # noqa: D401 

643 config: BetterConfigParser = app.settings["CONFIG"] 

644 app.settings["ELASTIC_APM"] = { 

645 "ENABLED": config.getboolean("ELASTIC_APM", "ENABLED", fallback=False), 

646 "SERVER_URL": config.get( 

647 "ELASTIC_APM", "SERVER_URL", fallback="http://localhost:8200" 

648 ), 

649 "SECRET_TOKEN": config.get( 

650 "ELASTIC_APM", "SECRET_TOKEN", fallback=None 

651 ), 

652 "API_KEY": config.get("ELASTIC_APM", "API_KEY", fallback=None), 

653 "VERIFY_SERVER_CERT": config.getboolean( 

654 "ELASTIC_APM", "VERIFY_SERVER_CERT", fallback=True 

655 ), 

656 "USE_CERTIFI": True, # doesn't actually use certifi 

657 "SERVICE_NAME": NAME.removesuffix("-dev"), 

658 "SERVICE_VERSION": VERSION, 

659 "ENVIRONMENT": ( 

660 "production" if not sys.flags.dev_mode else "development" 

661 ), 

662 "DEBUG": True, 

663 "CAPTURE_BODY": "errors", 

664 "TRANSACTION_IGNORE_URLS": [ 

665 "/api/ping", 

666 "/static/*", 

667 "/favicon.png", 

668 ], 

669 "TRANSACTIONS_IGNORE_PATTERNS": ["^OPTIONS "], 

670 "PROCESSORS": [ 

671 "an_website.utils.utils.apm_anonymization_processor", 

672 "elasticapm.processors.sanitize_stacktrace_locals", 

673 "elasticapm.processors.sanitize_http_request_cookies", 

674 "elasticapm.processors.sanitize_http_headers", 

675 "elasticapm.processors.sanitize_http_wsgi_env", 

676 "elasticapm.processors.sanitize_http_request_body", 

677 ], 

678 "RUM_SERVER_URL": config.get( 

679 "ELASTIC_APM", "RUM_SERVER_URL", fallback=None 

680 ), 

681 "RUM_SERVER_URL_PREFIX": config.get( 

682 "ELASTIC_APM", "RUM_SERVER_URL_PREFIX", fallback=None 

683 ), 

684 } 

685 

686 script_options = [ 

687 f"serviceName:{app.settings['ELASTIC_APM']['SERVICE_NAME']!r}", 

688 f"serviceVersion:{app.settings['ELASTIC_APM']['SERVICE_VERSION']!r}", 

689 f"environment:{app.settings['ELASTIC_APM']['ENVIRONMENT']!r}", 

690 ] 

691 

692 rum_server_url = app.settings["ELASTIC_APM"]["RUM_SERVER_URL"] 

693 

694 if rum_server_url is None: 

695 script_options.append( 

696 f"serverUrl:{app.settings['ELASTIC_APM']['SERVER_URL']!r}" 

697 ) 

698 elif rum_server_url: 

699 script_options.append(f"serverUrl:{rum_server_url!r}") 

700 else: 

701 script_options.append("serverUrl:window.location.origin") 

702 

703 if app.settings["ELASTIC_APM"]["RUM_SERVER_URL_PREFIX"]: 

704 script_options.append( 

705 f"serverUrlPrefix:{app.settings['ELASTIC_APM']['RUM_SERVER_URL_PREFIX']!r}" 

706 ) 

707 

708 app.settings["ELASTIC_APM"]["INLINE_SCRIPT"] = ( 

709 "elasticApm.init({" + ",".join(script_options) + "})" 

710 ) 

711 

712 app.settings["ELASTIC_APM"]["INLINE_SCRIPT_HASH"] = b64encode( 

713 sha256( 

714 app.settings["ELASTIC_APM"]["INLINE_SCRIPT"].encode("ASCII") 

715 ).digest() 

716 ).decode("ASCII") 

717 

718 if app.settings["ELASTIC_APM"]["ENABLED"]: 

719 app.settings["ELASTIC_APM"]["CLIENT"] = ElasticAPM(app).client 

720 

721 

722def setup_app_search(app: Application) -> None: # pragma: no cover 

723 """Setup Elastic App Search.""" # noqa: D401 

724 with catch_warnings(): 

725 simplefilter("ignore", DeprecationWarning) 

726 # pylint: disable-next=import-outside-toplevel 

727 from elastic_enterprise_search import ( # type: ignore[import-untyped] 

728 AppSearch, 

729 ) 

730 

731 config: BetterConfigParser = app.settings["CONFIG"] 

732 host = config.get("APP_SEARCH", "HOST", fallback=None) 

733 key = config.get("APP_SEARCH", "SEARCH_KEY", fallback=None) 

734 verify_certs = config.getboolean( 

735 "APP_SEARCH", "VERIFY_CERTS", fallback=True 

736 ) 

737 app.settings["APP_SEARCH"] = ( 

738 AppSearch( 

739 host, 

740 bearer_auth=key, 

741 verify_certs=verify_certs, 

742 ca_certs=CA_BUNDLE_PATH, 

743 ) 

744 if host 

745 else None 

746 ) 

747 app.settings["APP_SEARCH_HOST"] = host 

748 app.settings["APP_SEARCH_KEY"] = key 

749 app.settings["APP_SEARCH_ENGINE"] = config.get( 

750 "APP_SEARCH", "ENGINE_NAME", fallback=NAME.removesuffix("-dev") 

751 ) 

752 

753 

754def setup_redis(app: Application) -> None | Redis[str]: 

755 """Setup Redis.""" # noqa: D401 

756 config: BetterConfigParser = app.settings["CONFIG"] 

757 

758 class Kwargs(TypedDict, total=False): 

759 """Kwargs of BlockingConnectionPool constructor.""" 

760 

761 db: int 

762 username: None | str 

763 password: None | str 

764 retry_on_timeout: bool 

765 connection_class: type[UnixDomainSocketConnection] | type[SSLConnection] 

766 path: str 

767 host: str 

768 port: int 

769 ssl_ca_certs: str 

770 ssl_keyfile: None | str 

771 ssl_certfile: None | str 

772 ssl_check_hostname: bool 

773 ssl_cert_reqs: str 

774 

775 kwargs: Kwargs = { 

776 "db": config.getint("REDIS", "DB", fallback=0), 

777 "username": config.get("REDIS", "USERNAME", fallback=None), 

778 "password": config.get("REDIS", "PASSWORD", fallback=None), 

779 "retry_on_timeout": config.getboolean( 

780 "REDIS", "RETRY_ON_TIMEOUT", fallback=False 

781 ), 

782 } 

783 redis_ssl_kwargs: Kwargs = { 

784 "connection_class": SSLConnection, 

785 "ssl_ca_certs": CA_BUNDLE_PATH, 

786 "ssl_keyfile": config.get("REDIS", "SSL_KEYFILE", fallback=None), 

787 "ssl_certfile": config.get("REDIS", "SSL_CERTFILE", fallback=None), 

788 "ssl_cert_reqs": config.get( 

789 "REDIS", "SSL_CERT_REQS", fallback="required" 

790 ), 

791 "ssl_check_hostname": config.getboolean( 

792 "REDIS", "SSL_CHECK_HOSTNAME", fallback=False 

793 ), 

794 } 

795 redis_host_port_kwargs: Kwargs = { 

796 "host": config.get("REDIS", "HOST", fallback="localhost"), 

797 "port": config.getint("REDIS", "PORT", fallback=6379), 

798 } 

799 redis_use_ssl = config.getboolean("REDIS", "SSL", fallback=False) 

800 redis_unix_socket_path = config.get( 

801 "REDIS", "UNIX_SOCKET_PATH", fallback=None 

802 ) 

803 

804 if redis_unix_socket_path is not None: 

805 if redis_use_ssl: 

806 LOGGER.warning( 

807 "SSL is enabled for Redis, but a UNIX socket is used" 

808 ) 

809 if config.has_option("REDIS", "HOST"): 

810 LOGGER.warning( 

811 "A host is configured for Redis, but a UNIX socket is used" 

812 ) 

813 if config.has_option("REDIS", "PORT"): 

814 LOGGER.warning( 

815 "A port is configured for Redis, but a UNIX socket is used" 

816 ) 

817 kwargs.update( 

818 { 

819 "connection_class": UnixDomainSocketConnection, 

820 "path": redis_unix_socket_path, 

821 } 

822 ) 

823 else: 

824 kwargs.update(redis_host_port_kwargs) 

825 if redis_use_ssl: 

826 kwargs.update(redis_ssl_kwargs) 

827 

828 if not config.getboolean("REDIS", "ENABLED", fallback=False): 

829 app.settings["REDIS"] = None 

830 return None 

831 connection_pool = BlockingConnectionPool( 

832 client_name=NAME, 

833 decode_responses=True, 

834 **kwargs, 

835 ) 

836 redis = cast("Redis[str]", Redis(connection_pool=connection_pool)) 

837 app.settings["REDIS"] = redis 

838 return redis 

839 

840 

841def signal_handler( # noqa: D103 # pragma: no cover 

842 signalnum: int, frame: None | types.FrameType 

843) -> None: 

844 # pylint: disable=unused-argument, missing-function-docstring 

845 if signalnum in {signal.SIGINT, signal.SIGTERM}: 

846 EVENT_SHUTDOWN.set() 

847 if signalnum == getattr(signal, "SIGHUP", None): 

848 EVENT_SHUTDOWN.set() 

849 

850 

851def install_signal_handler() -> None: # pragma: no cover 

852 """Install the signal handler.""" 

853 signal.signal(signal.SIGINT, signal_handler) 

854 signal.signal(signal.SIGTERM, signal_handler) 

855 if hasattr(signal, "SIGHUP"): 

856 signal.signal(signal.SIGHUP, signal_handler) 

857 

858 

859def supervise(loop: AbstractEventLoop) -> None: 

860 """Supervise.""" 

861 while foobarbaz := background_tasks.HEARTBEAT: # pylint: disable=while-used 

862 if time.monotonic() - foobarbaz >= 10: 

863 worker = task_id() 

864 pid = os.getpid() 

865 

866 task = asyncio.current_task(loop) 

867 request = task.get_context().get(request_ctx_var) if task else None 

868 

869 LOGGER.fatal( 

870 "Heartbeat timed out for worker %s (pid %d), " 

871 "current request: %s, current task: %s", 

872 worker, 

873 pid, 

874 request, 

875 task, 

876 ) 

877 atexit._run_exitfuncs() # pylint: disable=protected-access 

878 os.abort() 

879 time.sleep(1) 

880 

881 

882def main( # noqa: C901 # pragma: no cover 

883 config: BetterConfigParser | None = None, 

884) -> int | str: 

885 """ 

886 Start everything. 

887 

888 This is the main function that is called when running this program. 

889 """ 

890 # pylint: disable=too-complex, too-many-branches 

891 # pylint: disable=too-many-locals, too-many-statements 

892 setproctitle(NAME) 

893 

894 install_signal_handler() 

895 

896 parser = create_argument_parser() 

897 args, _ = parser.parse_known_args( 

898 get_arguments_without_help(), ArgparseNamespace() 

899 ) 

900 

901 if args.version: 

902 print("Version:", VERSION) 

903 if args.verbose: 

904 # pylint: disable-next=import-outside-toplevel 

905 from .version.version import ( 

906 get_file_hashes, 

907 get_hash_of_file_hashes, 

908 ) 

909 

910 print() 

911 print("Hash der Datei-Hashes:") 

912 print(get_hash_of_file_hashes()) 

913 

914 if args.verbose > 1: 

915 print() 

916 print("Datei-Hashes:") 

917 print(get_file_hashes()) 

918 

919 return 0 

920 

921 config = config or BetterConfigParser.from_path(*args.config) 

922 assert config is not None 

923 config.add_override_argument_parser(parser) 

924 

925 setup_logging(config) 

926 

927 LOGGER.info("Starting %s %s", NAME, VERSION) 

928 

929 if platform.system() == "Windows": 

930 LOGGER.warning( 

931 "Running %s on Windows is not officially supported", 

932 NAME.removesuffix("-dev"), 

933 ) 

934 

935 ignore_modules(config) 

936 app = make_app(config) 

937 if isinstance(app, str): 

938 return app 

939 

940 apply_config_to_app(app, config) 

941 setup_elasticsearch(app) 

942 setup_app_search(app) 

943 setup_redis(app) 

944 setup_apm(app) 

945 

946 behind_proxy = config.getboolean("GENERAL", "BEHIND_PROXY", fallback=False) 

947 

948 server = HTTPServer( 

949 app, 

950 body_timeout=3600, 

951 decompress_request=True, 

952 max_body_size=1_000_000_000, 

953 ssl_options=get_ssl_context(config), 

954 xheaders=behind_proxy, 

955 ) 

956 

957 socket_factories: list[Callable[[], Iterable[socket]]] = [] 

958 

959 port = config.getint("GENERAL", "PORT", fallback=None) 

960 

961 if port: 

962 socket_factories.append( 

963 partial( 

964 bind_sockets, 

965 port, 

966 "localhost" if behind_proxy else "", 

967 ) 

968 ) 

969 

970 unix_socket_path = config.get( 

971 "GENERAL", 

972 "UNIX_SOCKET_PATH", 

973 fallback=None, 

974 ) 

975 

976 if unix_socket_path: 

977 os.makedirs(unix_socket_path, 0o755, True) 

978 socket_factories.append( 

979 lambda: ( 

980 bind_unix_socket( 

981 os.path.join(unix_socket_path, f"{NAME}.sock"), 

982 mode=0o666, 

983 ), 

984 ) 

985 ) 

986 

987 processes = config.getint( 

988 "GENERAL", 

989 "PROCESSES", 

990 fallback=hasattr(os, "fork") * (2 if sys.flags.dev_mode else -1), 

991 ) 

992 

993 if processes < 0: 

994 processes = ( 

995 os.process_cpu_count() # type: ignore[attr-defined] 

996 if sys.version_info >= (3, 13) 

997 else os.cpu_count() 

998 ) or 0 

999 

1000 worker: None | int = None 

1001 

1002 run_supervisor_thread = config.getboolean( 

1003 "GENERAL", "SUPERVISE", fallback=False 

1004 ) 

1005 elasticsearch_is_enabled = config.getboolean( 

1006 "ELASTICSEARCH", "ENABLED", fallback=False 

1007 ) 

1008 redis_is_enabled = config.getboolean("REDIS", "ENABLED", fallback=False) 

1009 webhook_logging_options = WebhookLoggingOptions(config) 

1010 # all config options should be read before forking 

1011 if args.save_config_to: 

1012 with open(args.save_config_to, "w", encoding="UTF-8") as file: 

1013 config.write(file) 

1014 config.set_all_options_should_be_parsed() 

1015 del config 

1016 # show help message if --help is given (after reading config, before forking) 

1017 parser.parse_args() 

1018 

1019 if not socket_factories: 

1020 LOGGER.warning("No sockets configured") 

1021 return 0 

1022 

1023 # create sockets after checking for --help 

1024 sockets: list[socket] = ( 

1025 Stream(socket_factories).flat_map(lambda fun: fun()).collect(list) 

1026 ) 

1027 

1028 UPTIME.reset() 

1029 main_pid = os.getpid() 

1030 

1031 if processes: 

1032 setproctitle(f"{NAME} - Master") 

1033 

1034 worker = fork_processes(processes) 

1035 

1036 setproctitle(f"{NAME} - Worker {worker}") 

1037 

1038 # yeet all children (there should be none, but do it regardless, just in case) 

1039 _children.clear() 

1040 

1041 if "an_website.quotes" in sys.modules: 

1042 from .quotes.utils import ( # pylint: disable=import-outside-toplevel 

1043 AUTHORS_CACHE, 

1044 QUOTES_CACHE, 

1045 WRONG_QUOTES_CACHE, 

1046 ) 

1047 

1048 del AUTHORS_CACHE.control.created_by_ultra # type: ignore[attr-defined] 

1049 del QUOTES_CACHE.control.created_by_ultra # type: ignore[attr-defined] 

1050 del WRONG_QUOTES_CACHE.control.created_by_ultra # type: ignore[attr-defined] 

1051 del geoip.__kwdefaults__["caches"].control.created_by_ultra 

1052 

1053 if unix_socket_path: 

1054 sockets.append( 

1055 bind_unix_socket( 

1056 os.path.join(unix_socket_path, f"{NAME}.{worker}.sock"), 

1057 mode=0o666, 

1058 ) 

1059 ) 

1060 

1061 # get loop after forking 

1062 # if not forking allow loop to be set in advance by external code 

1063 loop: None | asyncio.AbstractEventLoop 

1064 try: 

1065 with catch_warnings(): # TODO: remove after dropping support for 3.13 

1066 simplefilter("ignore", DeprecationWarning) 

1067 loop = asyncio.get_event_loop() 

1068 if loop.is_closed(): 

1069 loop = None 

1070 except RuntimeError: 

1071 loop = None 

1072 

1073 if loop is None: 

1074 loop = asyncio.new_event_loop() 

1075 asyncio.set_event_loop(loop) 

1076 

1077 if sys.version_info >= (3, 13) and not loop.get_task_factory(): 

1078 loop.set_task_factory(asyncio.eager_task_factory) 

1079 

1080 if perf8 and "PERF8" in os.environ: 

1081 loop.run_until_complete(perf8.enable()) 

1082 

1083 setup_webhook_logging(webhook_logging_options, loop) 

1084 

1085 server.add_sockets(sockets) 

1086 

1087 tasks = background_tasks.start_background_tasks( # noqa: F841 

1088 module_infos=app.settings["MODULE_INFOS"], 

1089 loop=loop, 

1090 main_pid=main_pid, 

1091 app=app, 

1092 processes=processes, 

1093 elasticsearch_is_enabled=elasticsearch_is_enabled, 

1094 redis_is_enabled=redis_is_enabled, 

1095 worker=worker, 

1096 ) 

1097 

1098 if run_supervisor_thread: 

1099 background_tasks.HEARTBEAT = time.monotonic() 

1100 threading.Thread( 

1101 target=supervise, args=(loop,), name="supervisor", daemon=True 

1102 ).start() 

1103 

1104 try: 

1105 loop.run_forever() 

1106 EVENT_SHUTDOWN.set() 

1107 finally: 

1108 try: # pylint: disable=too-many-try-statements 

1109 server.stop() 

1110 loop.run_until_complete(asyncio.sleep(1)) 

1111 loop.run_until_complete(server.close_all_connections()) 

1112 if perf8 and "PERF8" in os.environ: 

1113 loop.run_until_complete(perf8.disable()) 

1114 if redis := app.settings.get("REDIS"): 

1115 loop.run_until_complete( 

1116 redis.aclose(close_connection_pool=True) 

1117 ) 

1118 if elasticsearch := app.settings.get("ELASTICSEARCH"): 

1119 loop.run_until_complete(elasticsearch.close()) 

1120 finally: 

1121 try: 

1122 _cancel_all_tasks(loop) 

1123 loop.run_until_complete(loop.shutdown_asyncgens()) 

1124 loop.run_until_complete(loop.shutdown_default_executor()) 

1125 finally: 

1126 loop.close() 

1127 background_tasks.HEARTBEAT = 0 

1128 

1129 return len(tasks)