Coverage for an_website / utils / utils.py: 71.465%

1# This program is free software: you can redistribute it and/or modify 

2# it under the terms of the GNU Affero General Public License as 

3# published by the Free Software Foundation, either version 3 of the 

4# License, or (at your option) any later version. 

5# 

6# This program is distributed in the hope that it will be useful, 

7# but WITHOUT ANY WARRANTY; without even the implied warranty of 

8# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

9# GNU Affero General Public License for more details. 

10# 

11# You should have received a copy of the GNU Affero General Public License 

12# along with this program. If not, see <https://www.gnu.org/licenses/>. 

13 

14"""A module with many useful things used by other modules.""" 

15 

16from __future__ import annotations 

17 

18import argparse 

19import asyncio 

20import bisect 

21import contextlib 

22import logging 

23import random 

24import sys 

25import time 

26from base64 import b85encode 

27from collections.abc import ( 

28 Awaitable, 

29 Callable, 

30 Collection, 

31 Generator, 

32 Iterable, 

33 Mapping, 

34 Set, 

35) 

36from dataclasses import dataclass, field 

37from datetime import datetime, timezone 

38from enum import IntFlag 

39from functools import cache, partial 

40from hashlib import sha1 

41from importlib.resources.abc import Traversable 

42from ipaddress import IPv4Address, IPv6Address, ip_address, ip_network 

43from pathlib import Path 

44from typing import ( 

45 IO, 

46 TYPE_CHECKING, 

47 Any, 

48 Final, 

49 Literal, 

50 TypeAlias, 

51 Union, 

52 cast, 

53 get_args, 

54) 

55from urllib.parse import SplitResult, parse_qsl, urlencode, urlsplit, urlunsplit 

56 

57import elasticapm 

58import regex 

59from blake3 import blake3 

60from elastic_transport import ApiError, TransportError 

61from elasticsearch import AsyncElasticsearch 

62from geoip import geolite2 # type: ignore[import-untyped] 

63from openmoji_dist import VERSION as OPENMOJI_VERSION 

64from rapidfuzz.distance.Levenshtein import distance 

65from redis.asyncio import Redis 

66from tornado.web import HTTPError, RequestHandler 

67from typed_stream import Stream 

68from UltraDict import UltraDict # type: ignore[import-untyped] 

69 

70from .. import DIR as ROOT_DIR, pytest_is_running 

71 

72if TYPE_CHECKING: 

73 from .background_tasks import BackgroundTask 

74 

75LOGGER: Final = logging.getLogger(__name__) 

76 

77# pylint: disable-next=consider-alternative-union-syntax 

78type Handler = Union[ 

79 tuple[str, type[RequestHandler]], 

80 tuple[str, type[RequestHandler], dict[str, Any]], 

81 tuple[str, type[RequestHandler], dict[str, Any], str], 

82] 

83 

84type OpenMojiValue = Literal[False, "img", "glyf_colr1", "glyf_colr0"] 

85BumpscosityValue: TypeAlias = Literal[0, 1, 12, 50, 76, 100, 1000] 

86BUMPSCOSITY_VALUES: Final[tuple[BumpscosityValue, ...]] = get_args( 

87 BumpscosityValue 

88) 

89 

90PRINT = int.from_bytes((ROOT_DIR / "primes.bin").read_bytes(), "big") 

91 

92IP_HASH_SALT: Final = { 

93 "date": datetime.now(timezone.utc).date(), 

94 "hasher": blake3( 

95 blake3( 

96 datetime.now(timezone.utc).date().isoformat().encode("ASCII") 

97 ).digest() 

98 ), 

99} 

100 

101SUS_PATHS: Final[Set[str]] = { 

102 "/-profiler/phpinfo", 

103 "/.aws/credentials", 

104 "/.env", 

105 "/.env.bak", 

106 "/.ftpconfig", 

107 "/admin/controller/extension/extension", 

108 "/assets/filemanager/dialog", 

109 "/assets/vendor/server/php", 

110 "/aws.yml", 

111 "/boaform/admin/formlogin", 

112 "/phpinfo", 

113 "/public/assets/jquery-file-upload/server/php", 

114 "/root", 

115 "/settings/aws.yml", 

116 "/uploads", 

117 "/vendor/phpunit/phpunit/src/util/php/eval-stdin", 

118 "/wordpress", 

119 "/wp", 

120 "/wp-admin", 

121 "/wp-admin/css", 

122 "/wp-includes", 

123 "/wp-login", 

124 "/wp-upload", 

125} 

126 

127 

128class ArgparseNamespace(argparse.Namespace): 

129 """A class to fake type hints for argparse.Namespace.""" 

130 

131 # pylint: disable=too-few-public-methods 

132 __slots__ = ("config", "save_config_to", "version", "verbose") 

133 

134 config: list[Path] 

135 save_config_to: Path | None 

136 version: bool 

137 verbose: int 

138 

139 

140class AwaitableValue[T](Awaitable[T]): 

141 # pylint: disable=too-few-public-methods 

142 """An awaitable that always returns the same value.""" 

143 

144 def __await__(self) -> Generator[None, None, T]: 

145 """Return the value.""" 

146 yield 

147 return self._value 

148 

149 def __init__(self, value: T) -> None: 

150 """Set the value.""" 

151 self._value = value 

152 

153 

154class Permission(IntFlag): 

155 """Permissions for accessing restricted stuff.""" 

156 

157 RATELIMITS = 1 

158 TRACEBACK = 2 

159 BACKDOOR = 4 

160 UPDATE = 8 

161 REPORTING = 16 

162 SHORTEN = 32 

163 UPLOAD = 64 

164 

165 

166class Timer: 

167 """Timer class used for timing stuff.""" 

168 

169 __slots__ = ("_execution_time", "_start_time") 

170 

171 _execution_time: int 

172 

173 def __init__(self) -> None: 

174 """Start the timer.""" 

175 self._start_time = time.perf_counter_ns() 

176 

177 def get(self) -> float: 

178 """Get the execution time in seconds.""" 

179 return self.get_ns() / 1_000_000_000 

180 

181 def get_ns(self) -> int: 

182 """Get the execution time in nanoseconds.""" 

183 assert hasattr(self, "_execution_time"), "Timer not stopped yet" 

184 return self._execution_time 

185 

186 def stop(self) -> float: 

187 """Stop the timer and get the execution time in seconds.""" 

188 return self.stop_ns() / 1_000_000_000 

189 

190 def stop_ns(self) -> int: 

191 """Stop the timer and get the execution time in nanoseconds.""" 

192 assert not hasattr(self, "_execution_time"), "Timer already stopped" 

193 self._execution_time = time.perf_counter_ns() - self._start_time 

194 return self._execution_time 

195 

196 

197@cache 

198def add_args_to_url(url: str | SplitResult, **kwargs: object) -> str: 

199 """Add query arguments to a URL.""" 

200 if isinstance(url, str): 

201 url = urlsplit(url) 

202 

203 if not kwargs: 

204 return url.geturl() 

205 

206 url_args: dict[str, str] = dict( 

207 parse_qsl(url.query, keep_blank_values=True) 

208 ) 

209 

210 for key, value in kwargs.items(): 

211 if value is None: 

212 if key in url_args: 

213 del url_args[key] 

214 # pylint: disable-next=confusing-consecutive-elif 

215 elif isinstance(value, bool): 

216 url_args[key] = bool_to_str(value) 

217 else: 

218 url_args[key] = str(value) 

219 

220 return urlunsplit( 

221 ( 

222 url.scheme, 

223 url.netloc, 

224 url.path, 

225 urlencode(url_args), 

226 url.fragment, 

227 ) 

228 ) 

229 

230 

231def anonymize_ip[ # noqa: D103 

232 A: (str, None, str | None) 

233](address: A, *, ignore_invalid: bool = False) -> A: 

234 """Anonymize an IP address.""" 

235 if address is None: 

236 return None 

237 

238 address = address.strip() 

239 

240 try: 

241 version = ip_address(address).version 

242 except ValueError: 

243 if ignore_invalid: 

244 return address 

245 raise 

246 

247 if version == 4: 

248 return str(ip_network(address + "/24", strict=False).network_address) 

249 if version == 6: 

250 return str(ip_network(address + "/48", strict=False).network_address) 

251 

252 raise HTTPError(reason="ERROR: -41") 

253 

254 

255ansi_replace = partial(regex.sub, "\033" + r"\[-?\d+[a-zA-Z]", "") 

256ansi_replace.__doc__ = "Remove ANSI escape sequences from a string." 

257 

258 

259def apm_anonymization_processor( 

260 # pylint: disable-next=unused-argument 

261 client: elasticapm.Client, 

262 event: dict[str, Any], 

263) -> dict[str, Any]: 

264 """Anonymize an APM event.""" 

265 if "context" in event and "request" in event["context"]: 

266 request = event["context"]["request"] 

267 if "url" in request and "pathname" in request["url"]: 

268 path = request["url"]["pathname"] 

269 if path == "/robots.txt" or path.lower() in SUS_PATHS: 

270 return event 

271 if "socket" in request and "remote_address" in request["socket"]: 

272 request["socket"]["remote_address"] = anonymize_ip( 

273 request["socket"]["remote_address"] 

274 ) 

275 if "headers" in request: 

276 headers = request["headers"] 

277 if "X-Forwarded-For" in headers: 

278 headers["X-Forwarded-For"] = ", ".join( 

279 anonymize_ip(ip.strip(), ignore_invalid=True) 

280 for ip in headers["X-Forwarded-For"].split(",") 

281 ) 

282 for header in headers: 

283 if "ip" in header.lower().split("-"): 

284 headers[header] = anonymize_ip( 

285 headers[header], ignore_invalid=True 

286 ) 

287 return event 

288 

289 

290def apply[V, Ret](value: V, fun: Callable[[V], Ret]) -> Ret: # noqa: D103 

291 """Apply a function to a value and return the result.""" 

292 return fun(value) 

293 

294 

295backspace_replace = partial(regex.sub, ".?\x08", "") 

296backspace_replace.__doc__ = "Remove backspaces from a string." 

297 

298 

299def bool_to_str(val: bool) -> str: 

300 """Convert a boolean to sure/nope.""" 

301 return "sure" if val else "nope" 

302 

303 

304def bounded_edit_distance(s1: str, s2: str, /, k: int) -> int: 

305 """Return a bounded edit distance between two strings. 

306 

307 k is the maximum number returned 

308 """ 

309 if (dist := distance(s1, s2, score_cutoff=k)) == k + 1: 

310 return k 

311 return dist 

312 

313 

314def country_code_to_flag(code: str) -> str: 

315 """Convert a two-letter ISO country code to a flag emoji.""" 

316 return "".join(chr(ord(char) + 23 * 29 * 191) for char in code.upper()) 

317 

318 

319def create_argument_parser() -> argparse.ArgumentParser: 

320 """Parse command line arguments.""" 

321 parser = argparse.ArgumentParser() 

322 parser.add_argument( 

323 "--version", 

324 help="show the version of the website", 

325 action="store_true", 

326 default=False, 

327 ) 

328 parser.add_argument( 

329 "--verbose", 

330 action="count", 

331 default=0, 

332 ) 

333 parser.add_argument( 

334 "-c", 

335 "--config", 

336 default=[Path("config.ini")], 

337 help="the path to the config file", 

338 metavar="PATH", 

339 nargs="*", 

340 type=Path, 

341 ) 

342 parser.add_argument( 

343 "--save-config-to", 

344 default=None, 

345 help="save the configuration to a file", 

346 metavar="Path", 

347 nargs="?", 

348 type=Path, 

349 ) 

350 return parser 

351 

352 

353def emoji2html(emoji: str) -> str: 

354 """Convert an emoji to HTML.""" 

355 return f"<img src={emoji2url(emoji)!r} alt={emoji!r} class='emoji'>" 

356 

357 

358def emoji2url(emoji: str) -> str: 

359 """Convert an emoji to an URL.""" 

360 if len(emoji) == 2: 

361 emoji = emoji.removesuffix("\uFE0F") 

362 code = "-".join(f"{ord(c):04x}" for c in emoji) 

363 return f"/static/openmoji/svg/{code.upper()}.svg?v={OPENMOJI_VERSION}" 

364 

365 

366if sys.flags.dev_mode and not pytest_is_running(): 

367 __origignal_emoji2url = emoji2url 

368 

369 def emoji2url(emoji: str) -> str: # pylint: disable=function-redefined 

370 """Convert an emoji to an URL.""" 

371 import openmoji_dist # pylint: disable=import-outside-toplevel 

372 from emoji import is_emoji # pylint: disable=import-outside-toplevel 

373 

374 assert is_emoji(emoji), f"{emoji} needs to be emoji" 

375 result = __origignal_emoji2url(emoji) 

376 file = ( 

377 openmoji_dist.get_openmoji_data() 

378 / result.removeprefix("/static/openmoji/").split("?")[0] 

379 ) 

380 assert file.is_file(), f"{file} needs to exist" 

381 return result 

382 

383 

384EMOJI_MAPPING: Final[Mapping[str, str]] = { 

385 "⁉": "⁉", 

386 "‼": "‼", 

387 "?": "❓", 

388 "!": "❗", 

389 "-": "➖", 

390 "+": "➕", 

391 "\U0001F51F": "\U0001F51F", 

392} 

393 

394 

395def emojify(string: str) -> Iterable[str]: 

396 """Emojify a given string.""" 

397 non_emojis: list[str] = [] 

398 for ch in ( 

399 replace_umlauts(string) 

400 .replace("!?", "⁉") 

401 .replace("!!", "‼") 

402 .replace("10", "\U0001F51F") 

403 ): 

404 emoji: str | None = None 

405 if ch.isascii(): 

406 if ch.isdigit() or ch in "#*": 

407 emoji = f"{ch}\uFE0F\u20E3" 

408 elif ch.isalpha(): 

409 emoji = country_code_to_flag(ch) 

410 emoji = EMOJI_MAPPING.get(ch, emoji) 

411 

412 if emoji is None: 

413 non_emojis.append(ch) 

414 else: 

415 if non_emojis: 

416 yield "".join(non_emojis) 

417 non_emojis.clear() 

418 yield emoji 

419 

420 if non_emojis: 

421 yield "".join(non_emojis) 

422 

423 

424async def geoip( 

425 ip: None | str, 

426 database: str = "GeoLite2-City.mmdb", 

427 elasticsearch: None | AsyncElasticsearch = None, 

428 *, 

429 allow_fallback: bool = True, 

430 caches: dict[str, dict[str, dict[str, Any]]] = UltraDict(), # noqa: B008 

431) -> None | dict[str, Any]: 

432 """Get GeoIP information.""" 

433 # pylint: disable=too-complex 

434 if not ip: 

435 return None 

436 

437 # pylint: disable-next=redefined-outer-name 

438 cache = caches.get(ip, {}) 

439 if database not in cache: 

440 if not elasticsearch: 

441 if allow_fallback and database in { 

442 "GeoLite2-City.mmdb", 

443 "GeoLite2-Country.mmdb", 

444 }: 

445 return geoip_fallback( 

446 ip, country=database == "GeoLite2-City.mmdb" 

447 ) 

448 return None 

449 

450 properties: None | tuple[str, ...] 

451 if database == "GeoLite2-City.mmdb": 

452 properties = ( 

453 "continent_name", 

454 "country_iso_code", 

455 "country_name", 

456 "region_iso_code", 

457 "region_name", 

458 "city_name", 

459 "location", 

460 "timezone", 

461 ) 

462 elif database == "GeoLite2-Country.mmdb": 

463 properties = ( 

464 "continent_name", 

465 "country_iso_code", 

466 "country_name", 

467 ) 

468 elif database == "GeoLite2-ASN.mmdb": 

469 properties = ("asn", "network", "organization_name") 

470 else: 

471 properties = None 

472 

473 try: 

474 cache[database] = ( 

475 await elasticsearch.ingest.simulate( 

476 pipeline={ 

477 "processors": [ 

478 { 

479 "geoip": { 

480 "field": "ip", 

481 "database_file": database, 

482 "properties": properties, 

483 } 

484 } 

485 ] 

486 }, 

487 docs=[{"_source": {"ip": ip}}], 

488 filter_path="docs.doc._source", 

489 ) 

490 )["docs"][0]["doc"]["_source"].get("geoip", {}) 

491 except (ApiError, TransportError): 

492 if allow_fallback and database in { 

493 "GeoLite2-City.mmdb", 

494 "GeoLite2-Country.mmdb", 

495 }: 

496 return geoip_fallback( 

497 ip, country=database == "GeoLite2-City.mmdb" 

498 ) 

499 raise 

500 

501 if "country_iso_code" in cache[database]: 

502 cache[database]["country_flag"] = country_code_to_flag( 

503 cache[database]["country_iso_code"] 

504 ) 

505 

506 caches[ip] = cache 

507 return cache[database] 

508 

509 

510def geoip_fallback(ip: str, country: bool = False) -> None | dict[str, Any]: 

511 """Get GeoIP information without using Elasticsearch.""" 

512 if not (info := geolite2.lookup(ip)): 

513 return None 

514 

515 info_dict = info.get_info_dict() 

516 

517 continent_name = info_dict.get("continent", {}).get("names", {}).get("en") 

518 country_iso_code = info_dict.get("country", {}).get("iso_code") 

519 country_name = info_dict.get("country", {}).get("names", {}).get("en") 

520 

521 data = { 

522 "continent_name": continent_name, 

523 "country_iso_code": country_iso_code, 

524 "country_name": country_name, 

525 } 

526 

527 if data["country_iso_code"]: 

528 data["country_flag"] = country_code_to_flag(data["country_iso_code"]) 

529 

530 if country: 

531 for key, value in tuple(data.items()): 

532 if not value: 

533 del data[key] 

534 

535 return data 

536 

537 latitude = info_dict.get("location", {}).get("latitude") 

538 longitude = info_dict.get("location", {}).get("longitude") 

539 location = (latitude, longitude) if latitude and longitude else None 

540 time_zone = info_dict.get("location", {}).get("time_zone") 

541 

542 data.update({"location": location, "timezone": time_zone}) 

543 

544 for key, value in tuple(data.items()): 

545 if not value: 

546 del data[key] 

547 

548 return data 

549 

550 

551def get_arguments_without_help() -> tuple[str, ...]: 

552 """Get arguments without help.""" 

553 return tuple(arg for arg in sys.argv[1:] if arg not in {"-h", "--help"}) 

554 

555 

556def get_close_matches( # based on difflib.get_close_matches 

557 word: str, 

558 possibilities: Iterable[str], 

559 count: int = 3, 

560 cutoff: float = 0.5, 

561) -> tuple[str, ...]: 

562 """Use normalized_distance to return list of the best "good enough" matches. 

563 

564 word is a sequence for which close matches are desired (typically a string). 

565 

566 possibilities is a list of sequences against which to match word 

567 (typically a list of strings). 

568 

569 Optional arg count (default 3) is the maximum number of close matches to 

570 return. count must be > 0. 

571 

572 Optional arg cutoff (default 0.5) is a float in [0, 1]. Possibilities 

573 that don't score at least that similar to word are ignored. 

574 

575 The best (no more than count) matches among the possibilities are returned 

576 in a tuple, sorted by similarity score, most similar first. 

577 """ 

578 if count <= 0: 

579 raise ValueError(f"count must be > 0: {count}") 

580 if not 0.0 <= cutoff <= 1.0: 

581 raise ValueError(f"cutoff must be in [0.0, 1.0]: {cutoff}") 

582 word_len = len(word) 

583 if not word_len: 

584 if cutoff < 1.0: 

585 return () 

586 return Stream(possibilities).limit(count).collect(tuple) 

587 result: list[tuple[float, str]] = [] 

588 for possibility in possibilities: 

589 if max_dist := max(word_len, len(possibility)): 

590 dist = bounded_edit_distance( 

591 possibility, word, 1 + int(cutoff * max_dist) 

592 ) 

593 if (ratio := dist / max_dist) <= cutoff: 

594 bisect.insort(result, (ratio, possibility)) 

595 if len(result) > count: 

596 result.pop(-1) 

597 # Strip scores for the best count matches 

598 return tuple(word for score, word in result) 

599 

600 

601def hash_bytes(*args: bytes, hasher: Any = None, size: int = 32) -> str: 

602 """Hash bytes and return the Base85 representation.""" 

603 digest: bytes 

604 if not hasher: 

605 hasher = blake3() 

606 for arg in args: 

607 hasher.update(arg) 

608 digest = ( 

609 hasher.digest(size) 

610 if isinstance(hasher, blake3) 

611 else hasher.digest()[:size] 

612 ) 

613 return b85encode(digest).decode("ASCII") 

614 

615 

616def hash_ip( 

617 address: None | str | IPv4Address | IPv6Address, size: int = 32 

618) -> str: 

619 """Hash an IP address.""" 

620 if isinstance(address, str): 

621 address = ip_address(address) 

622 if IP_HASH_SALT["date"] != (date := datetime.now(timezone.utc).date()): 

623 IP_HASH_SALT["hasher"] = blake3( 

624 blake3(date.isoformat().encode("ASCII")).digest() 

625 ) 

626 IP_HASH_SALT["date"] = date 

627 return hash_bytes( 

628 address.packed if address else b"", 

629 hasher=IP_HASH_SALT["hasher"].copy(), # type: ignore[attr-defined] 

630 size=size, 

631 ) 

632 

633 

634def is_in_european_union(ip: None | str) -> None | bool: 

635 """Return whether the specified address is in the EU.""" 

636 if not (ip and (info := geolite2.lookup(ip))): 

637 return None 

638 

639 return cast(bool, info.get_info_dict().get("is_in_european_union", False)) 

640 

641 

642def is_prime(number: int) -> bool: 

643 """Return whether the specified number is prime.""" 

644 if not number % 2: 

645 return number == 2 

646 return bool(PRINT & (1 << (number // 2))) 

647 

648 

649def length_of_match(match: regex.Match[Any]) -> int: 

650 """Calculate the length of the regex match and return it.""" 

651 return match.end() - match.start() 

652 

653 

654def n_from_set[T](set_: Set[T], n: int) -> set[T]: # noqa: D103 

655 """Get and return n elements of the set as a new set.""" 

656 new_set = set() 

657 for i, element in enumerate(set_): 

658 if i >= n: 

659 break 

660 new_set.add(element) 

661 return new_set 

662 

663 

664def name_to_id(val: str) -> str: 

665 """Replace umlauts and whitespaces in a string to get a valid HTML id.""" 

666 return regex.sub( 

667 r"[^a-z0-9]+", 

668 "-", 

669 replace_umlauts(val).lower(), 

670 ).strip("-") 

671 

672 

673def none_to_default[T, D](value: None | T, default: D) -> D | T: # noqa: D103 

674 """Like ?? in ECMAScript.""" 

675 return default if value is None else value 

676 

677 

678def parse_bumpscosity(value: str | int | None) -> BumpscosityValue: 

679 """Parse a string to a valid bumpscosity value.""" 

680 if isinstance(value, str): 

681 with contextlib.suppress(ValueError): 

682 value = int(value, base=0) 

683 if value in BUMPSCOSITY_VALUES: 

684 return cast(BumpscosityValue, value) 

685 return random.Random(repr(value)).choice(BUMPSCOSITY_VALUES) 

686 

687 

688def parse_openmoji_arg(value: str, default: OpenMojiValue) -> OpenMojiValue: 

689 """Parse the openmoji arg into a Literal.""" 

690 value = value.lower() 

691 if value == "glyf_colr0": 

692 return "glyf_colr0" 

693 if value == "glyf_colr1": 

694 return "glyf_colr1" 

695 if value in {"i", "img"}: 

696 return "img" 

697 if value in {"n", "nope"}: 

698 return False 

699 return default 

700 

701 

702# pylint: disable-next=too-many-arguments 

703async def ratelimit( 

704 redis: Redis[str], 

705 redis_prefix: str, 

706 remote_ip: str, 

707 *, 

708 bucket: None | str, 

709 max_burst: int, 

710 count_per_period: int, 

711 period: int, 

712 tokens: int, 

713) -> tuple[bool, dict[str, str]]: 

714 """Take b1nzy to space using Redis.""" 

715 remote_ip = hash_bytes(remote_ip.encode("ASCII")) 

716 key = f"{redis_prefix}:ratelimit:{remote_ip}" 

717 if bucket: 

718 key = f"{key}:{bucket}" 

719 

720 # see: https://github.com/brandur/redis-cell#usage 

721 result = await redis.execute_command( 

722 # type: ignore[no-untyped-call] 

723 "CL.THROTTLE", 

724 key, 

725 max_burst, 

726 count_per_period, 

727 period, 

728 tokens, 

729 ) 

730 

731 now = time.time() 

732 

733 headers: dict[str, str] = {} 

734 

735 if result[0]: 

736 headers["Retry-After"] = str(result[3]) 

737 if not bucket: 

738 headers["X-RateLimit-Global"] = "true" 

739 

740 if bucket: 

741 headers["X-RateLimit-Limit"] = str(result[1]) 

742 headers["X-RateLimit-Remaining"] = str(result[2]) 

743 headers["X-RateLimit-Reset"] = str(now + result[4]) 

744 headers["X-RateLimit-Reset-After"] = str(result[4]) 

745 headers["X-RateLimit-Bucket"] = hash_bytes(bucket.encode("ASCII")) 

746 

747 return bool(result[0]), headers 

748 

749 

750def remove_suffix_ignore_case(string: str, suffix: str) -> str: 

751 """Remove a suffix without caring about the case.""" 

752 if string.lower().endswith(suffix.lower()): 

753 return string[: -len(suffix)] 

754 return string 

755 

756 

757def replace_umlauts(string: str) -> str: 

758 """Replace Ä, Ö, Ü, ẞ, ä, ö, ü, ß in string.""" 

759 if string.isupper(): 

760 return ( 

761 string.replace("Ä", "AE") 

762 .replace("Ö", "OE") 

763 .replace("Ü", "UE") 

764 .replace("ẞ", "SS") 

765 ) 

766 if " " in string: 

767 return " ".join(replace_umlauts(word) for word in string.split(" ")) 

768 return ( 

769 string.replace("ä", "ae") 

770 .replace("ö", "oe") 

771 .replace("ü", "ue") 

772 .replace("ß", "ss") 

773 .replace("Ä", "Ae") 

774 .replace("Ö", "Oe") 

775 .replace("Ü", "Ue") 

776 .replace("ẞ", "SS") 

777 ) 

778 

779 

780async def run( 

781 program: str, 

782 *args: str, 

783 stdin: int | IO[Any] = asyncio.subprocess.DEVNULL, 

784 stdout: None | int | IO[Any] = asyncio.subprocess.PIPE, 

785 stderr: None | int | IO[Any] = asyncio.subprocess.PIPE, 

786 **kwargs: Any, 

787) -> tuple[None | int, bytes, bytes]: 

788 """Run a programm and return the exit code, stdout and stderr as tuple.""" 

789 proc = await asyncio.create_subprocess_exec( 

790 program, 

791 *args, 

792 stdin=stdin, 

793 stdout=stdout, 

794 stderr=stderr, 

795 **kwargs, 

796 ) 

797 output = await proc.communicate() 

798 return proc.returncode, *output 

799 

800 

801def size_of_file(file: Traversable) -> int: 

802 """Calculate the size of a file.""" 

803 if isinstance(file, Path): 

804 return file.stat().st_size 

805 

806 with file.open("rb") as data: 

807 return sum(map(len, data)) # pylint: disable=bad-builtin 

808 

809 

810def str_to_bool(val: None | str | bool, default: None | bool = None) -> bool: 

811 """Convert a string representation of truth to True or False.""" 

812 if isinstance(val, bool): 

813 return val 

814 if isinstance(val, str): 

815 val = val.lower() 

816 if val in { 

817 "1", 

818 "a",