Coverage for an_website/utils/utils.py: 71.538%

1# This program is free software: you can redistribute it and/or modify 

2# it under the terms of the GNU Affero General Public License as 

3# published by the Free Software Foundation, either version 3 of the 

4# License, or (at your option) any later version. 

5# 

6# This program is distributed in the hope that it will be useful, 

7# but WITHOUT ANY WARRANTY; without even the implied warranty of 

8# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

9# GNU Affero General Public License for more details. 

10# 

11# You should have received a copy of the GNU Affero General Public License 

12# along with this program. If not, see <https://www.gnu.org/licenses/>. 

13 

14"""A module with many useful things used by other modules.""" 

15 

16from __future__ import annotations 

17 

18import argparse 

19import asyncio 

20import bisect 

21import contextlib 

22import logging 

23import pathlib 

24import random 

25import sys 

26import time 

27from base64 import b85encode 

28from collections.abc import ( 

29 Awaitable, 

30 Callable, 

31 Collection, 

32 Generator, 

33 Iterable, 

34 Mapping, 

35 Set, 

36) 

37from dataclasses import dataclass, field 

38from datetime import datetime, timezone 

39from enum import IntFlag 

40from functools import cache, partial 

41from hashlib import sha1 

42from importlib.resources.abc import Traversable 

43from ipaddress import IPv4Address, IPv6Address, ip_address, ip_network 

44from pathlib import Path 

45from typing import ( 

46 IO, 

47 TYPE_CHECKING, 

48 Any, 

49 Final, 

50 Literal, 

51 TypeAlias, 

52 Union, 

53 cast, 

54 get_args, 

55) 

56from urllib.parse import SplitResult, parse_qsl, urlencode, urlsplit, urlunsplit 

57 

58import elasticapm 

59import regex 

60from blake3 import blake3 

61from elastic_transport import ApiError, TransportError 

62from elasticsearch import AsyncElasticsearch 

63from geoip import geolite2 # type: ignore[import-untyped] 

64from openmoji_dist import VERSION as OPENMOJI_VERSION 

65from rapidfuzz.distance.Levenshtein import distance 

66from redis.asyncio import Redis 

67from tornado.web import HTTPError, RequestHandler 

68from typed_stream import Stream 

69from UltraDict import UltraDict # type: ignore[import-untyped] 

70 

71from .. import DIR as ROOT_DIR, pytest_is_running 

72 

73if TYPE_CHECKING: 

74 from .background_tasks import BackgroundTask 

75 

76LOGGER: Final = logging.getLogger(__name__) 

77 

78# pylint: disable-next=consider-alternative-union-syntax 

79type Handler = Union[ 

80 tuple[str, type[RequestHandler]], 

81 tuple[str, type[RequestHandler], dict[str, Any]], 

82 tuple[str, type[RequestHandler], dict[str, Any], str], 

83] 

84 

85type OpenMojiValue = Literal[False, "img", "glyf_colr1", "glyf_colr0"] 

86BumpscosityValue: TypeAlias = Literal[0, 1, 12, 50, 76, 100, 1000] 

87BUMPSCOSITY_VALUES: Final[tuple[BumpscosityValue, ...]] = get_args( 

88 BumpscosityValue 

89) 

90 

91PRINT = int.from_bytes((ROOT_DIR / "primes.bin").read_bytes(), "big") 

92 

93IP_HASH_SALT: Final = { 

94 "date": datetime.now(timezone.utc).date(), 

95 "hasher": blake3( 

96 blake3( 

97 datetime.now(timezone.utc).date().isoformat().encode("ASCII") 

98 ).digest() 

99 ), 

100} 

101 

102SUS_PATHS: Final[Set[str]] = { 

103 "/-profiler/phpinfo", 

104 "/.aws/credentials", 

105 "/.env", 

106 "/.env.bak", 

107 "/.ftpconfig", 

108 "/admin/controller/extension/extension", 

109 "/assets/filemanager/dialog", 

110 "/assets/vendor/server/php", 

111 "/aws.yml", 

112 "/boaform/admin/formlogin", 

113 "/phpinfo", 

114 "/public/assets/jquery-file-upload/server/php", 

115 "/root", 

116 "/settings/aws.yml", 

117 "/uploads", 

118 "/vendor/phpunit/phpunit/src/util/php/eval-stdin", 

119 "/wordpress", 

120 "/wp", 

121 "/wp-admin", 

122 "/wp-admin/css", 

123 "/wp-includes", 

124 "/wp-login", 

125 "/wp-upload", 

126} 

127 

128 

129class ArgparseNamespace(argparse.Namespace): 

130 """A class to fake type hints for argparse.Namespace.""" 

131 

132 # pylint: disable=too-few-public-methods 

133 __slots__ = ("config", "save_config_to", "version", "verbose") 

134 

135 config: list[pathlib.Path] 

136 save_config_to: pathlib.Path | None 

137 version: bool 

138 verbose: int 

139 

140 

141class AwaitableValue[T](Awaitable[T]): 

142 # pylint: disable=too-few-public-methods 

143 """An awaitable that always returns the same value.""" 

144 

145 def __await__(self) -> Generator[None, None, T]: 

146 """Return the value.""" 

147 yield 

148 return self._value 

149 

150 def __init__(self, value: T) -> None: 

151 """Set the value.""" 

152 self._value = value 

153 

154 

155class Permission(IntFlag): 

156 """Permissions for accessing restricted stuff.""" 

157 

158 RATELIMITS = 1 

159 TRACEBACK = 2 

160 BACKDOOR = 4 

161 UPDATE = 8 

162 REPORTING = 16 

163 SHORTEN = 32 

164 UPLOAD = 64 

165 

166 

167class Timer: 

168 """Timer class used for timing stuff.""" 

169 

170 __slots__ = ("_execution_time", "_start_time") 

171 

172 _execution_time: int 

173 

174 def __init__(self) -> None: 

175 """Start the timer.""" 

176 self._start_time = time.perf_counter_ns() 

177 

178 def get(self) -> float: 

179 """Get the execution time in seconds.""" 

180 return self.get_ns() / 1_000_000_000 

181 

182 def get_ns(self) -> int: 

183 """Get the execution time in nanoseconds.""" 

184 assert hasattr(self, "_execution_time"), "Timer not stopped yet" 

185 return self._execution_time 

186 

187 def stop(self) -> float: 

188 """Stop the timer and get the execution time in seconds.""" 

189 return self.stop_ns() / 1_000_000_000 

190 

191 def stop_ns(self) -> int: 

192 """Stop the timer and get the execution time in nanoseconds.""" 

193 assert not hasattr(self, "_execution_time"), "Timer already stopped" 

194 self._execution_time = time.perf_counter_ns() - self._start_time 

195 return self._execution_time 

196 

197 

198@cache 

199def add_args_to_url(url: str | SplitResult, **kwargs: object) -> str: 

200 """Add query arguments to a URL.""" 

201 if isinstance(url, str): 

202 url = urlsplit(url) 

203 

204 if not kwargs: 

205 return url.geturl() 

206 

207 url_args: dict[str, str] = dict( 

208 parse_qsl(url.query, keep_blank_values=True) 

209 ) 

210 

211 for key, value in kwargs.items(): 

212 if value is None: 

213 if key in url_args: 

214 del url_args[key] 

215 # pylint: disable-next=confusing-consecutive-elif 

216 elif isinstance(value, bool): 

217 url_args[key] = bool_to_str(value) 

218 else: 

219 url_args[key] = str(value) 

220 

221 return urlunsplit( 

222 ( 

223 url.scheme, 

224 url.netloc, 

225 url.path, 

226 urlencode(url_args), 

227 url.fragment, 

228 ) 

229 ) 

230 

231 

232def anonymize_ip[ # noqa: D103 

233 A: (str, None, str | None) 

234](address: A, *, ignore_invalid: bool = False) -> A: 

235 """Anonymize an IP address.""" 

236 if address is None: 

237 return None 

238 

239 address = address.strip() 

240 

241 try: 

242 version = ip_address(address).version 

243 except ValueError: 

244 if ignore_invalid: 

245 return address 

246 raise 

247 

248 if version == 4: 

249 return str(ip_network(address + "/24", strict=False).network_address) 

250 if version == 6: 

251 return str(ip_network(address + "/48", strict=False).network_address) 

252 

253 raise HTTPError(reason="ERROR: -41") 

254 

255 

256ansi_replace = partial(regex.sub, "\033" + r"\[-?\d+[a-zA-Z]", "") 

257ansi_replace.__doc__ = "Remove ANSI escape sequences from a string." 

258 

259 

260def apm_anonymization_processor( 

261 # pylint: disable-next=unused-argument 

262 client: elasticapm.Client, 

263 event: dict[str, Any], 

264) -> dict[str, Any]: 

265 """Anonymize an APM event.""" 

266 if "context" in event and "request" in event["context"]: 

267 request = event["context"]["request"] 

268 if "url" in request and "pathname" in request["url"]: 

269 path = request["url"]["pathname"] 

270 if path == "/robots.txt" or path.lower() in SUS_PATHS: 

271 return event 

272 if "socket" in request and "remote_address" in request["socket"]: 

273 request["socket"]["remote_address"] = anonymize_ip( 

274 request["socket"]["remote_address"] 

275 ) 

276 if "headers" in request: 

277 headers = request["headers"] 

278 if "X-Forwarded-For" in headers: 

279 headers["X-Forwarded-For"] = ", ".join( 

280 anonymize_ip(ip.strip(), ignore_invalid=True) 

281 for ip in headers["X-Forwarded-For"].split(",") 

282 ) 

283 for header in headers: 

284 if "ip" in header.lower().split("-"): 

285 headers[header] = anonymize_ip( 

286 headers[header], ignore_invalid=True 

287 ) 

288 return event 

289 

290 

291def apply[V, Ret](value: V, fun: Callable[[V], Ret]) -> Ret: # noqa: D103 

292 """Apply a function to a value and return the result.""" 

293 return fun(value) 

294 

295 

296backspace_replace = partial(regex.sub, ".?\x08", "") 

297backspace_replace.__doc__ = "Remove backspaces from a string." 

298 

299 

300def bool_to_str(val: bool) -> str: 

301 """Convert a boolean to sure/nope.""" 

302 return "sure" if val else "nope" 

303 

304 

305def bounded_edit_distance(s1: str, s2: str, /, k: int) -> int: 

306 """Return a bounded edit distance between two strings. 

307 

308 k is the maximum number returned 

309 """ 

310 if (dist := distance(s1, s2, score_cutoff=k)) == k + 1: 

311 return k 

312 return dist 

313 

314 

315def country_code_to_flag(code: str) -> str: 

316 """Convert a two-letter ISO country code to a flag emoji.""" 

317 return "".join(chr(ord(char) + 23 * 29 * 191) for char in code.upper()) 

318 

319 

320def create_argument_parser() -> argparse.ArgumentParser: 

321 """Parse command line arguments.""" 

322 parser = argparse.ArgumentParser() 

323 parser.add_argument( 

324 "--version", 

325 help="show the version of the website", 

326 action="store_true", 

327 default=False, 

328 ) 

329 parser.add_argument( 

330 "--verbose", 

331 action="count", 

332 default=0, 

333 ) 

334 parser.add_argument( 

335 "-c", 

336 "--config", 

337 default=[pathlib.Path("config.ini")], 

338 help="the path to the config file", 

339 metavar="PATH", 

340 nargs="*", 

341 type=pathlib.Path, 

342 ) 

343 parser.add_argument( 

344 "--save-config-to", 

345 default=None, 

346 help="save the configuration to a file", 

347 metavar="Path", 

348 nargs="?", 

349 type=pathlib.Path, 

350 ) 

351 return parser 

352 

353 

354def emoji2html(emoji: str) -> str: 

355 """Convert an emoji to HTML.""" 

356 return f"<img src={emoji2url(emoji)!r} alt={emoji!r} class='emoji'>" 

357 

358 

359def emoji2url(emoji: str) -> str: 

360 """Convert an emoji to an URL.""" 

361 if len(emoji) == 2: 

362 emoji = emoji.removesuffix("\uFE0F") 

363 code = "-".join(f"{ord(c):04x}" for c in emoji) 

364 return f"/static/openmoji/svg/{code.upper()}.svg?v={OPENMOJI_VERSION}" 

365 

366 

367if sys.flags.dev_mode or pytest_is_running(): 

368 __origignal_emoji2url = emoji2url 

369 

370 def emoji2url(emoji: str) -> str: # pylint: disable=function-redefined 

371 """Convert an emoji to an URL.""" 

372 import openmoji_dist # pylint: disable=import-outside-toplevel 

373 from emoji import is_emoji # pylint: disable=import-outside-toplevel 

374 

375 assert is_emoji(emoji), f"{emoji} needs to be emoji" 

376 result = __origignal_emoji2url(emoji) 

377 file = ( 

378 openmoji_dist.get_openmoji_data() 

379 / result.removeprefix("/static/openmoji/").split("?")[0] 

380 ) 

381 assert file.is_file(), f"{file} needs to exist" 

382 return result 

383 

384 

385EMOJI_MAPPING: Final[Mapping[str, str]] = { 

386 "⁉": "⁉", 

387 "‼": "‼", 

388 "?": "❓", 

389 "!": "❗", 

390 "-": "➖", 

391 "+": "➕", 

392 "\U0001F51F": "\U0001F51F", 

393} 

394 

395 

396def emojify(string: str) -> Iterable[str]: 

397 """Emojify a given string.""" 

398 non_emojis: list[str] = [] 

399 for ch in ( 

400 replace_umlauts(string) 

401 .replace("!?", "⁉") 

402 .replace("!!", "‼") 

403 .replace("10", "\U0001F51F") 

404 ): 

405 emoji: str | None = None 

406 if ch.isascii(): 

407 if ch.isdigit() or ch in "#*": 

408 emoji = f"{ch}\uFE0F\u20E3" 

409 elif ch.isalpha(): 

410 emoji = country_code_to_flag(ch) 

411 emoji = EMOJI_MAPPING.get(ch, emoji) 

412 

413 if emoji is None: 

414 non_emojis.append(ch) 

415 else: 

416 if non_emojis: 

417 yield "".join(non_emojis) 

418 non_emojis.clear() 

419 yield emoji 

420 

421 if non_emojis: 

422 yield "".join(non_emojis) 

423 

424 

425async def geoip( 

426 ip: None | str, 

427 database: str = "GeoLite2-City.mmdb", 

428 elasticsearch: None | AsyncElasticsearch = None, 

429 *, 

430 allow_fallback: bool = True, 

431 caches: dict[str, dict[str, dict[str, Any]]] = UltraDict(), # noqa: B008 

432) -> None | dict[str, Any]: 

433 """Get GeoIP information.""" 

434 # pylint: disable=too-complex 

435 if not ip: 

436 return None 

437 

438 # pylint: disable-next=redefined-outer-name 

439 cache = caches.get(ip, {}) 

440 if database not in cache: 

441 if not elasticsearch: 

442 if allow_fallback and database in { 

443 "GeoLite2-City.mmdb", 

444 "GeoLite2-Country.mmdb", 

445 }: 

446 return geoip_fallback( 

447 ip, country=database == "GeoLite2-City.mmdb" 

448 ) 

449 return None 

450 

451 properties: None | tuple[str, ...] 

452 if database == "GeoLite2-City.mmdb": 

453 properties = ( 

454 "continent_name", 

455 "country_iso_code", 

456 "country_name", 

457 "region_iso_code", 

458 "region_name", 

459 "city_name", 

460 "location", 

461 "timezone", 

462 ) 

463 elif database == "GeoLite2-Country.mmdb": 

464 properties = ( 

465 "continent_name", 

466 "country_iso_code", 

467 "country_name", 

468 ) 

469 elif database == "GeoLite2-ASN.mmdb": 

470 properties = ("asn", "network", "organization_name") 

471 else: 

472 properties = None 

473 

474 try: 

475 cache[database] = ( 

476 await elasticsearch.ingest.simulate( 

477 pipeline={ 

478 "processors": [ 

479 { 

480 "geoip": { 

481 "field": "ip", 

482 "database_file": database, 

483 "properties": properties, 

484 } 

485 } 

486 ] 

487 }, 

488 docs=[{"_source": {"ip": ip}}], 

489 filter_path="docs.doc._source", 

490 ) 

491 )["docs"][0]["doc"]["_source"].get("geoip", {}) 

492 except (ApiError, TransportError): 

493 if allow_fallback and database in { 

494 "GeoLite2-City.mmdb", 

495 "GeoLite2-Country.mmdb", 

496 }: 

497 return geoip_fallback( 

498 ip, country=database == "GeoLite2-City.mmdb" 

499 ) 

500 raise 

501 

502 if "country_iso_code" in cache[database]: 

503 cache[database]["country_flag"] = country_code_to_flag( 

504 cache[database]["country_iso_code"] 

505 ) 

506 

507 caches[ip] = cache 

508 return cache[database] 

509 

510 

511def geoip_fallback(ip: str, country: bool = False) -> None | dict[str, Any]: 

512 """Get GeoIP information without using Elasticsearch.""" 

513 if not (info := geolite2.lookup(ip)): 

514 return None 

515 

516 info_dict = info.get_info_dict() 

517 

518 continent_name = info_dict.get("continent", {}).get("names", {}).get("en") 

519 country_iso_code = info_dict.get("country", {}).get("iso_code") 

520 country_name = info_dict.get("country", {}).get("names", {}).get("en") 

521 

522 data = { 

523 "continent_name": continent_name, 

524 "country_iso_code": country_iso_code, 

525 "country_name": country_name, 

526 } 

527 

528 if data["country_iso_code"]: 

529 data["country_flag"] = country_code_to_flag(data["country_iso_code"]) 

530 

531 if country: 

532 for key, value in tuple(data.items()): 

533 if not value: 

534 del data[key] 

535 

536 return data 

537 

538 latitude = info_dict.get("location", {}).get("latitude") 

539 longitude = info_dict.get("location", {}).get("longitude") 

540 location = (latitude, longitude) if latitude and longitude else None 

541 time_zone = info_dict.get("location", {}).get("time_zone") 

542 

543 data.update({"location": location, "timezone": time_zone}) 

544 

545 for key, value in tuple(data.items()): 

546 if not value: 

547 del data[key] 

548 

549 return data 

550 

551 

552def get_arguments_without_help() -> tuple[str, ...]: 

553 """Get arguments without help.""" 

554 return tuple(arg for arg in sys.argv[1:] if arg not in {"-h", "--help"}) 

555 

556 

557def get_close_matches( # based on difflib.get_close_matches 

558 word: str, 

559 possibilities: Iterable[str], 

560 count: int = 3, 

561 cutoff: float = 0.5, 

562) -> tuple[str, ...]: 

563 """Use normalized_distance to return list of the best "good enough" matches. 

564 

565 word is a sequence for which close matches are desired (typically a string). 

566 

567 possibilities is a list of sequences against which to match word 

568 (typically a list of strings). 

569 

570 Optional arg count (default 3) is the maximum number of close matches to 

571 return. count must be > 0. 

572 

573 Optional arg cutoff (default 0.5) is a float in [0, 1]. Possibilities 

574 that don't score at least that similar to word are ignored. 

575 

576 The best (no more than count) matches among the possibilities are returned 

577 in a tuple, sorted by similarity score, most similar first. 

578 """ 

579 if count <= 0: 

580 raise ValueError(f"count must be > 0: {count}") 

581 if not 0.0 <= cutoff <= 1.0: 

582 raise ValueError(f"cutoff must be in [0.0, 1.0]: {cutoff}") 

583 word_len = len(word) 

584 if not word_len: 

585 if cutoff < 1.0: 

586 return () 

587 return Stream(possibilities).limit(count).collect(tuple) 

588 result: list[tuple[float, str]] = [] 

589 for possibility in possibilities: 

590 if max_dist := max(word_len, len(possibility)): 

591 dist = bounded_edit_distance( 

592 possibility, word, 1 + int(cutoff * max_dist) 

593 ) 

594 if (ratio := dist / max_dist) <= cutoff: 

595 bisect.insort(result, (ratio, possibility)) 

596 if len(result) > count: 

597 result.pop(-1) 

598 # Strip scores for the best count matches 

599 return tuple(word for score, word in result) 

600 

601 

602def hash_bytes(*args: bytes, hasher: Any = None, size: int = 32) -> str: 

603 """Hash bytes and return the Base85 representation.""" 

604 digest: bytes 

605 if not hasher: 

606 hasher = blake3() 

607 for arg in args: 

608 hasher.update(arg) 

609 digest = ( 

610 hasher.digest(size) 

611 if isinstance(hasher, blake3) 

612 else hasher.digest()[:size] 

613 ) 

614 return b85encode(digest).decode("ASCII") 

615 

616 

617def hash_ip( 

618 address: None | str | IPv4Address | IPv6Address, size: int = 32 

619) -> str: 

620 """Hash an IP address.""" 

621 if isinstance(address, str): 

622 address = ip_address(address) 

623 if IP_HASH_SALT["date"] != (date := datetime.now(timezone.utc).date()): 

624 IP_HASH_SALT["hasher"] = blake3( 

625 blake3(date.isoformat().encode("ASCII")).digest() 

626 ) 

627 IP_HASH_SALT["date"] = date 

628 return hash_bytes( 

629 address.packed if address else b"", 

630 hasher=IP_HASH_SALT["hasher"].copy(), # type: ignore[attr-defined] 

631 size=size, 

632 ) 

633 

634 

635def is_in_european_union(ip: None | str) -> None | bool: 

636 """Return whether the specified address is in the EU.""" 

637 if not (ip and (info := geolite2.lookup(ip))): 

638 return None 

639 

640 return cast(bool, info.get_info_dict().get("is_in_european_union", False)) 

641 

642 

643def is_prime(number: int) -> bool: 

644 """Return whether the specified number is prime.""" 

645 if not number % 2: 

646 return number == 2 

647 return bool(PRINT & (1 << (number // 2))) 

648 

649 

650def length_of_match(match: regex.Match[Any]) -> int: 

651 """Calculate the length of the regex match and return it.""" 

652 return match.end() - match.start() 

653 

654 

655def n_from_set[T](set_: Set[T], n: int) -> set[T]: # noqa: D103 

656 """Get and return n elements of the set as a new set.""" 

657 new_set = set() 

658 for i, element in enumerate(set_): 

659 if i >= n: 

660 break 

661 new_set.add(element) 

662 return new_set 

663 

664 

665def name_to_id(val: str) -> str: 

666 """Replace umlauts and whitespaces in a string to get a valid HTML id.""" 

667 return regex.sub( 

668 r"[^a-z0-9]+", 

669 "-", 

670 replace_umlauts(val).lower(), 

671 ).strip("-") 

672 

673 

674def none_to_default[T, D](value: None | T, default: D) -> D | T: # noqa: D103 

675 """Like ?? in ECMAScript.""" 

676 return default if value is None else value 

677 

678 

679def parse_bumpscosity(value: str | int | None) -> BumpscosityValue: 

680 """Parse a string to a valid bumpscosity value.""" 

681 if isinstance(value, str): 

682 with contextlib.suppress(ValueError): 

683 value = int(value, base=0) 

684 if value in BUMPSCOSITY_VALUES: 

685 return cast(BumpscosityValue, value) 

686 return random.Random(repr(value)).choice(BUMPSCOSITY_VALUES) 

687 

688 

689def parse_openmoji_arg(value: str, default: OpenMojiValue) -> OpenMojiValue: 

690 """Parse the openmoji arg into a Literal.""" 

691 value = value.lower() 

692 if value == "glyf_colr0": 

693 return "glyf_colr0" 

694 if value == "glyf_colr1": 

695 return "glyf_colr1" 

696 if value in {"i", "img"}: 

697 return "img" 

698 if value in {"n", "nope"}: 

699 return False 

700 return default 

701 

702 

703# pylint: disable-next=too-many-arguments 

704async def ratelimit( 

705 redis: Redis[str], 

706 redis_prefix: str, 

707 remote_ip: str, 

708 *, 

709 bucket: None | str, 

710 max_burst: int, 

711 count_per_period: int, 

712 period: int, 

713 tokens: int, 

714) -> tuple[bool, dict[str, str]]: 

715 """Take b1nzy to space using Redis.""" 

716 remote_ip = hash_bytes(remote_ip.encode("ASCII")) 

717 key = f"{redis_prefix}:ratelimit:{remote_ip}" 

718 if bucket: 

719 key = f"{key}:{bucket}" 

720 

721 # see: https://github.com/brandur/redis-cell#usage 

722 result = await redis.execute_command( 

723 # type: ignore[no-untyped-call] 

724 "CL.THROTTLE", 

725 key, 

726 max_burst, 

727 count_per_period, 

728 period, 

729 tokens, 

730 ) 

731 

732 now = time.time() 

733 

734 headers: dict[str, str] = {} 

735 

736 if result[0]: 

737 headers["Retry-After"] = str(result[3]) 

738 if not bucket: 

739 headers["X-RateLimit-Global"] = "true" 

740 

741 if bucket: 

742 headers["X-RateLimit-Limit"] = str(result[1]) 

743 headers["X-RateLimit-Remaining"] = str(result[2]) 

744 headers["X-RateLimit-Reset"] = str(now + result[4]) 

745 headers["X-RateLimit-Reset-After"] = str(result[4]) 

746 headers["X-RateLimit-Bucket"] = hash_bytes(bucket.encode("ASCII")) 

747 

748 return bool(result[0]), headers 

749 

750 

751def remove_suffix_ignore_case(string: str, suffix: str) -> str: 

752 """Remove a suffix without caring about the case.""" 

753 if string.lower().endswith(suffix.lower()): 

754 return string[: -len(suffix)] 

755 return string 

756 

757 

758def replace_umlauts(string: str) -> str: 

759 """Replace Ä, Ö, Ü, ẞ, ä, ö, ü, ß in string.""" 

760 if string.isupper(): 

761 return ( 

762 string.replace("Ä", "AE") 

763 .replace("Ö", "OE") 

764 .replace("Ü", "UE") 

765 .replace("ẞ", "SS") 

766 ) 

767 if " " in string: 

768 return " ".join(replace_umlauts(word) for word in string.split(" ")) 

769 return ( 

770 string.replace("ä", "ae") 

771 .replace("ö", "oe") 

772 .replace("ü", "ue") 

773 .replace("ß", "ss") 

774 .replace("Ä", "Ae") 

775 .replace("Ö", "Oe") 

776 .replace("Ü", "Ue") 

777 .replace("ẞ", "SS") 

778 ) 

779 

780 

781async def run( 

782 program: str, 

783 *args: str, 

784 stdin: int | IO[Any] = asyncio.subprocess.DEVNULL, 

785 stdout: None | int | IO[Any] = asyncio.subprocess.PIPE, 

786 stderr: None | int | IO[Any] = asyncio.subprocess.PIPE, 

787 **kwargs: Any, 

788) -> tuple[None | int, bytes, bytes]: 

789 """Run a programm and return the exit code, stdout and stderr as tuple.""" 

790 proc = await asyncio.create_subprocess_exec( 

791 program, 

792 *args, 

793 stdin=stdin, 

794 stdout=stdout, 

795 stderr=stderr, 

796 **kwargs, 

797 ) 

798 output = await proc.communicate() 

799 return proc.returncode, *output 

800 

801 

802def size_of_file(file: Traversable) -> int: 

803 """Calculate the size of a file.""" 

804 if isinstance(file, Path): 

805 return file.stat().st_size 

806 

807 with file.open("rb") as data: 

808 return sum(map(len, data)) # pylint: disable=bad-builtin 

809 

810 

811def str_to_bool(val: None | str | bool, default: None | bool = None) -> bool: 

812 """Convert a string representation of truth to True or False.""" 

813 if isinstance(val, bool): 

814 return val 

815 if isinstance(val, str): 

816 val = val.lower() 

817 if val in {