Coverage for an_website/quotes/utils.py: 53.721%

1# This program is free software: you can redistribute it and/or modify 

2# it under the terms of the GNU Affero General Public License as 

3# published by the Free Software Foundation, either version 3 of the 

4# License, or (at your option) any later version. 

5# 

6# This program is distributed in the hope that it will be useful, 

7# but WITHOUT ANY WARRANTY; without even the implied warranty of 

8# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

9# GNU Affero General Public License for more details. 

10# 

11# You should have received a copy of the GNU Affero General Public License 

12# along with this program. If not, see <https://www.gnu.org/licenses/>. 

13 

14"""A page with wrong quotes.""" 

15 

16import abc 

17import asyncio 

18import contextlib 

19import logging 

20import multiprocessing.synchronize 

21import random 

22import sys 

23import time 

24from collections.abc import ( 

25 Callable, 

26 Container, 

27 Iterable, 

28 Mapping, 

29 MutableMapping, 

30 Sequence, 

31) 

32from dataclasses import dataclass 

33from datetime import date 

34from typing import Any, Final, Literal, cast 

35from urllib.parse import urlencode 

36 

37import dill # type: ignore[import-untyped] # nosec: B403 

38import elasticapm 

39import orjson as json 

40import typed_stream 

41from redis.asyncio import Redis 

42from tornado.httpclient import AsyncHTTPClient 

43from tornado.web import Application, HTTPError 

44from UltraDict import UltraDict # type: ignore[import-untyped] 

45 

46from .. import ( 

47 CA_BUNDLE_PATH, 

48 DIR as ROOT_DIR, 

49 EVENT_REDIS, 

50 EVENT_SHUTDOWN, 

51 NAME, 

52 ORJSON_OPTIONS, 

53) 

54from ..utils.request_handler import HTMLRequestHandler 

55from ..utils.utils import ModuleInfo, Permission, ratelimit 

56 

57DIR: Final = ROOT_DIR / "quotes" 

58 

59LOGGER: Final = logging.getLogger(__name__) 

60 

61# TODO: make this configurable and move it into app settings 

62API_URL: str = "https://zitate.prapsschnalinen.de/api" 

63 

64WRONGQUOTE_DELETED: Final[int] = -2 

65WRONGQUOTE_UNKNOWN: Final[int] = -1 

66 

67SPAM_QUOTES: Final[Container[int]] = frozenset( 

68 { 

69 *range(1368, 1428 + 1), 

70 # curl -sSf https://zitate.prapsschnalinen.de/api/quotes \ 

71 # | jq '.[] | select(.quote | match("https://")) | .id' 

72 *{1154, 1193, 1237, 1268, 1269, 1313, 1314, 1328, 1329}, 

73 *{1433, 1434, 1447, 1448, 1458, 1459, 1490, 1491, 1647}, 

74 *{1651, 1652, 1653, 1655, 1657, 1658, 1659, 1660}, 

75 } 

76) 

77SPAM_AUTHORS: Final[Container[int]] = frozenset( 

78 { 

79 *range(1859, 1947 + 1), 

80 # curl -sSf https://zitate.prapsschnalinen.de/api/wrongquotes \ 

81 # | jq '.[] | select(.quote.quote | match("https://")) 

82 # | [.author, .quote.author].[] 

83 # | select(.author | length == 6) | .id' \ 

84 # | sort -hu 

85 *{1595, 1596, 1639, 1640, 1701, 1702, 1755, 1756, 1809, 1810}, 

86 *{1818, 1819, 1954, 1955, 1982, 1983, 1995, 1996, 2032, 2033}, 

87 *{2297, 2298, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2310}, 

88 *{2312, 2313, 2315, 2316, 2317, 2318, 2319, 2320}, 

89 } 

90) 

91 

92 

93# pylint: disable-next=too-few-public-methods 

94class UltraDictType[K, V](MutableMapping[K, V], abc.ABC): 

95 """The type of the shared dictionaries.""" 

96 

97 lock: multiprocessing.synchronize.RLock 

98 

99 

100QUOTES_CACHE: Final[UltraDictType[int, Quote]] = UltraDict( 

101 buffer_size=1024**2, serializer=dill 

102) 

103AUTHORS_CACHE: Final[UltraDictType[int, Author]] = UltraDict( 

104 buffer_size=1024**2, serializer=dill 

105) 

106WRONG_QUOTES_CACHE: Final[UltraDictType[tuple[int, int], WrongQuote]] = ( 

107 UltraDict(buffer_size=1024**2, serializer=dill) 

108) 

109 

110 

111@dataclass(init=False, slots=True) 

112class QuotesObjBase(abc.ABC): 

113 """An object with an id.""" 

114 

115 id: int 

116 

117 @classmethod 

118 @abc.abstractmethod 

119 def fetch_all_endpoint(cls) -> Literal["quotes", "authors", "wrongquotes"]: 

120 """Endpoint to fetch all of this type.""" 

121 raise NotImplementedError 

122 

123 @abc.abstractmethod 

124 async def fetch_new_data(self) -> QuotesObjBase | None: 

125 """Fetch new data from the API.""" 

126 raise NotImplementedError 

127 

128 # pylint: disable=unused-argument 

129 def get_id_as_str(self, minify: bool = False) -> str: 

130 """Get the id of the object as a string.""" 

131 return str(self.id) 

132 

133 @abc.abstractmethod 

134 def get_path(self) -> str: 

135 """Return the path to the Object.""" 

136 raise NotImplementedError 

137 

138 

139@dataclass(slots=True) 

140class Author(QuotesObjBase): 

141 """The author object with a name.""" 

142 

143 name: str 

144 # tuple(url_to_info, info_str, creation_date) 

145 info: None | tuple[str, None | str, date] 

146 

147 def __str__(self) -> str: 

148 """Return the name of the author.""" 

149 return self.name 

150 

151 @classmethod 

152 def fetch_all_endpoint(cls) -> Literal["authors"]: 

153 """Endpoint to fetch all authors.""" 

154 return "authors" 

155 

156 async def fetch_new_data(self) -> Author | None: 

157 """Fetch new data from the API.""" 

158 data = await make_api_request( 

159 f"authors/{self.id}", entity_should_exist=True 

160 ) 

161 if data is None: 

162 del AUTHORS_CACHE[self.id] 

163 return None 

164 return parse_author(data) 

165 

166 def get_path(self) -> str: 

167 """Return the path to the author info.""" 

168 return f"/zitate/info/a/{self.id}" 

169 

170 def to_json(self) -> dict[str, Any]: 

171 """Get the author as JSON.""" 

172 return { 

173 "id": self.id, 

174 "name": str(self), 

175 "path": self.get_path(), 

176 "info": ( 

177 { 

178 "source": self.info[0], 

179 "text": self.info[1], 

180 "date": self.info[2].isoformat(), 

181 } 

182 if self.info 

183 else None 

184 ), 

185 } 

186 

187 

188@dataclass(slots=True) 

189class Quote(QuotesObjBase): 

190 """The quote object with a quote text and an author.""" 

191 

192 quote: str 

193 author_id: int 

194 

195 def __str__(self) -> str: 

196 """Return the content of the quote.""" 

197 return self.quote.strip() 

198 

199 @property 

200 def author(self) -> Author: 

201 """Get the corresponding author object.""" 

202 try: 

203 return AUTHORS_CACHE[self.author_id] 

204 except KeyError as err: 

205 LOGGER.error("Author %d was not in cache", self.author_id) 

206 raise HTTPError(404) from err 

207 

208 @classmethod 

209 def fetch_all_endpoint(cls) -> Literal["quotes"]: 

210 """Endpoint to fetch all quotes.""" 

211 return "quotes" 

212 

213 async def fetch_new_data(self) -> Quote | None: 

214 """Fetch new data from the API.""" 

215 data = await make_api_request( 

216 f"quotes/{self.id}", entity_should_exist=True 

217 ) 

218 if data is None: 

219 del QUOTES_CACHE[self.id] 

220 return None 

221 return parse_quote(data, self) 

222 

223 def get_path(self) -> str: 

224 """Return the path to the quote info.""" 

225 return f"/zitate/info/z/{self.id}" 

226 

227 def to_json(self) -> dict[str, Any]: 

228 """Get the quote as JSON.""" 

229 return { 

230 "id": self.id, 

231 "quote": str(self), 

232 "author": self.author.to_json(), 

233 "path": self.get_path(), 

234 } 

235 

236 

237@dataclass(slots=True) 

238class WrongQuote(QuotesObjBase): 

239 """The wrong quote object with a quote, an author and a rating.""" 

240 

241 quote_id: int 

242 author_id: int 

243 rating: int 

244 

245 def __str__(self) -> str: 

246 r""" 

247 Return the wrong quote. 

248 

249 like: '»quote« - author'. 

250 """ 

251 return f"»{self.quote}« - {self.author}" 

252 

253 @property 

254 def author(self) -> Author: 

255 """Get the corresponding author object.""" 

256 try: 

257 return AUTHORS_CACHE[self.author_id] 

258 except KeyError as err: 

259 LOGGER.error("Author %d was not in cache", self.author_id) 

260 raise HTTPError(404) from err 

261 

262 @classmethod 

263 def fetch_all_endpoint(cls) -> Literal["wrongquotes"]: 

264 """Endpoint to fetch all wrong quotes.""" 

265 return "wrongquotes" 

266 

267 async def fetch_new_data(self) -> WrongQuote: 

268 """Fetch new data from the API.""" 

269 if self.id == WRONGQUOTE_UNKNOWN: 

270 api_data = await make_api_request( 

271 "wrongquotes", 

272 { 

273 "quote": str(self.quote_id), 

274 "simulate": "true", 

275 "author": str(self.author_id), 

276 }, 

277 entity_should_exist=True, 

278 ) 

279 if api_data: 

280 api_data = api_data[0] 

281 # pylint: disable-next=confusing-consecutive-elif 

282 elif self.id == WRONGQUOTE_DELETED: 

283 api_data = None 

284 else: 

285 api_data = await make_api_request( 

286 f"wrongquotes/{self.id}", entity_should_exist=True 

287 ) 

288 if not api_data: 

289 self.id = WRONGQUOTE_DELETED 

290 author = await self.author.fetch_new_data() 

291 quote = await self.quote.fetch_new_data() 

292 if author and quote: 

293 self.id = WRONGQUOTE_UNKNOWN 

294 else: 

295 del WRONG_QUOTES_CACHE[(self.quote_id, self.author_id)] 

296 return self 

297 return parse_wrong_quote(api_data, self) 

298 

299 def get_id(self) -> tuple[int, int]: 

300 """ 

301 Get the id of the quote and the author in a tuple. 

302 

303 :return tuple(quote_id, author_id) 

304 """ 

305 return self.quote_id, self.author_id 

306 

307 def get_id_as_str(self, minify: bool = False) -> str: 

308 """ 

309 Get the id of the wrong quote as a string. 

310 

311 Format: quote_id-author_id 

312 """ 

313 if minify and self.id != WRONGQUOTE_UNKNOWN: 

314 return str(self.id) 

315 return f"{self.quote_id}-{self.author_id}" 

316 

317 def get_path(self) -> str: 

318 """Return the path to the wrong quote.""" 

319 return f"/zitate/{self.get_id_as_str()}" 

320 

321 @property 

322 def quote(self) -> Quote: 

323 """Get the corresponding quote object.""" 

324 try: 

325 return QUOTES_CACHE[self.quote_id] 

326 except KeyError as err: 

327 LOGGER.error("Quote %d was not in cache", self.quote_id) 

328 raise HTTPError(404) from err 

329 

330 def to_json(self) -> dict[str, Any]: 

331 """Get the wrong quote as JSON.""" 

332 return { 

333 "id": self.get_id_as_str(), 

334 "quote": self.quote.to_json(), 

335 "author": self.author.to_json(), 

336 "rating": self.rating, 

337 "path": self.get_path(), 

338 } 

339 

340 async def vote( 

341 # pylint: disable=unused-argument 

342 self, 

343 vote: Literal[-1, 1], 

344 lazy: bool = False, 

345 ) -> WrongQuote | None: 

346 """Vote for the wrong quote.""" 

347 if self.id == WRONGQUOTE_UNKNOWN: 

348 raise ValueError("Can't vote for a not existing quote.") 

349 # if lazy: # simulate the vote and do the actual voting later 

350 # self.rating += vote 

351 # asyncio.get_running_loop().call_soon_threadsafe( 

352 # self.vote, 

353 # vote, 

354 # ) 

355 # return self 

356 # do the voting 

357 data = await make_api_request( 

358 f"wrongquotes/{self.id}", 

359 method="POST", 

360 body={"vote": str(vote)}, 

361 entity_should_exist=True, 

362 ) 

363 if data is None: 

364 return await self.fetch_new_data() 

365 

366 return parse_wrong_quote( 

367 data, 

368 self, 

369 ) 

370 

371 

372def get_wrong_quotes( 

373 filter_fun: None | Callable[[WrongQuote], bool] = None, 

374 *, 

375 sort: bool = False, # sorted by rating 

376 filter_real_quotes: bool = True, 

377 shuffle: bool = False, 

378 include_spam: bool = False, 

379) -> Sequence[WrongQuote]: 

380 """Get cached wrong quotes.""" 

381 if shuffle and sort: 

382 raise ValueError("Sort and shuffle can't be both true.") 

383 

384 iterable: Iterable[WrongQuote] = WRONG_QUOTES_CACHE.values() 

385 if not include_spam: 

386 iterable = ( 

387 wq 

388 for wq in iterable 

389 if wq.author_id not in SPAM_AUTHORS 

390 if wq.quote_id not in SPAM_QUOTES 

391 ) 

392 if filter_fun: 

393 iterable = filter(filter_fun, iterable) # pylint: disable=bad-builtin 

394 if filter_real_quotes: 

395 iterable = (wq for wq in iterable if wq.quote.author_id != wq.author_id) 

396 wqs = list(iterable) 

397 

398 if shuffle: 

399 random.shuffle(wqs) 

400 elif sort: 

401 wqs.sort(key=lambda wq: wq.rating, reverse=True) 

402 return wqs 

403 

404 

405def get_quotes( 

406 filter_fun: None | Callable[[Quote], bool] = None, 

407 shuffle: bool = False, 

408) -> list[Quote]: 

409 """Get cached quotes.""" 

410 quotes: list[Quote] = list(QUOTES_CACHE.values()) 

411 if filter_fun: 

412 for i in reversed(range(len(quotes))): 

413 if not filter_fun(quotes[i]): 

414 del quotes[i] 

415 if shuffle: 

416 random.shuffle(quotes) 

417 return quotes 

418 

419 

420def get_authors( 

421 filter_fun: None | Callable[[Author], bool] = None, 

422 shuffle: bool = False, 

423) -> list[Author]: 

424 """Get cached authors.""" 

425 authors: list[Author] = list(AUTHORS_CACHE.values()) 

426 if filter_fun: 

427 for i in reversed(range(len(authors))): 

428 if not filter_fun(authors[i]): 

429 del authors[i] 

430 if shuffle: 

431 random.shuffle(authors) 

432 return authors 

433 

434 

435# pylint: disable-next=too-many-arguments 

436async def make_api_request( 

437 endpoint: str, 

438 args: Mapping[str, str] | None = None, 

439 *, 

440 # pylint: disable-next=unused-argument 

441 entity_should_exist: bool, 

442 method: Literal["GET", "POST"] = "GET", 

443 body: None | Mapping[str, str | int] = None, 

444 request_timeout: float | None = None, 

445) -> Any | None: # TODO: list[dict[str, Any]] | dict[str, Any] | None 

446 """Make API request and return the result as dict.""" 

447 query = f"?{urlencode(args)}" if args else "" 

448 url = f"{API_URL}/{endpoint}{query}" 

449 body_str = urlencode(body) if body else body 

450 response = await AsyncHTTPClient().fetch( 

451 url, 

452 method=method, 

453 headers={"Content-Type": "application/x-www-form-urlencoded"}, 

454 body=body_str, 

455 raise_error=False, 

456 ca_certs=CA_BUNDLE_PATH, 

457 request_timeout=request_timeout, 

458 ) 

459 if response.code != 200: 

460 if response.code == 404: 

461 return None 

462 LOGGER.log( 

463 logging.ERROR if response.code >= 500 else logging.WARNING, 

464 "%s request to %r with body=%r failed with code=%d and reason=%r", 

465 method, 

466 url, 

467 body_str, 

468 response.code, 

469 response.reason, 

470 ) 

471 raise HTTPError( 

472 503, 

473 reason=f"{url} returned: {response.code} {response.reason}", 

474 ) 

475 return json.loads(response.body) 

476 

477 

478def fix_author_name(name: str) -> str: 

479 """Fix common mistakes in authors.""" 

480 if len(name) > 2 and name.startswith("(") and name.endswith(")"): 

481 # remove () from author name, that shouldn't be there 

482 name = name[1:-1] 

483 return name.strip() 

484 

485 

486def parse_author(json_data: Mapping[str, Any]) -> Author: 

487 """Parse an author from JSON data.""" 

488 id_ = int(json_data["id"]) 

489 name = fix_author_name(json_data["author"]) 

490 

491 with AUTHORS_CACHE.lock: 

492 author = AUTHORS_CACHE.get(id_) 

493 if author is None: 

494 # pylint: disable-next=too-many-function-args 

495 author = Author(id_, name, None) 

496 elif author.name != name: 

497 author.name = name 

498 author.info = None # reset info 

499 

500 AUTHORS_CACHE[author.id] = author 

501 

502 return author 

503 

504 

505def fix_quote_str(quote_str: str) -> str: 

506 """Fix common mistakes in quotes.""" 

507 if ( 

508 len(quote_str) > 2 

509 and quote_str.startswith(('"', "„", "“")) 

510 and quote_str.endswith(('"', "“", "”")) 

511 ): 

512 # remove quotation marks from quote, that shouldn't be there 

513 quote_str = quote_str[1:-1] 

514 

515 return quote_str.strip() 

516 

517 

518def parse_quote( 

519 json_data: Mapping[str, Any], quote: None | Quote = None 

520) -> Quote: 

521 """Parse a quote from JSON data.""" 

522 quote_id = int(json_data["id"]) 

523 author = parse_author(json_data["author"]) # update author 

524 quote_str = fix_quote_str(json_data["quote"]) 

525 

526 with QUOTES_CACHE.lock: 

527 if quote is None: # no quote supplied, try getting it from cache 

528 quote = QUOTES_CACHE.get(quote_id) 

529 if quote is None: # new quote 

530 # pylint: disable=too-many-function-args 

531 quote = Quote(quote_id, quote_str, author.id) 

532 else: # quote was already saved 

533 quote.quote = quote_str 

534 quote.author_id = author.id 

535 

536 QUOTES_CACHE[quote.id] = quote 

537 

538 return quote 

539 

540 

541def parse_wrong_quote( 

542 json_data: Mapping[str, Any], wrong_quote: None | WrongQuote = None 

543) -> WrongQuote: 

544 """Parse a wrong quote and update the cache.""" 

545 quote = parse_quote(json_data["quote"]) 

546 author = parse_author(json_data["author"]) 

547 

548 id_tuple = (quote.id, author.id) 

549 rating = json_data["rating"] 

550 wrong_quote_id = int(json_data.get("id") or WRONGQUOTE_UNKNOWN) 

551 

552 if wrong_quote: 

553 wrong_quote.id = wrong_quote_id 

554 wrong_quote.rating = rating 

555 

556 with WRONG_QUOTES_CACHE.lock: 

557 wrong_quote = WRONG_QUOTES_CACHE.get(id_tuple, wrong_quote) 

558 if wrong_quote is None: 

559 wrong_quote = WrongQuote( # pylint: disable=unexpected-keyword-arg 

560 id=wrong_quote_id, 

561 quote_id=quote.id, 

562 author_id=author.id, 

563 rating=rating, 

564 ) 

565 else: 

566 wrong_quote.id = wrong_quote_id 

567 wrong_quote.rating = rating 

568 WRONG_QUOTES_CACHE[id_tuple] = wrong_quote 

569 

570 return wrong_quote 

571 

572 

573async def parse_list_of_quote_data[Q: QuotesObjBase]( # noqa: D103 

574 json_list: str | Iterable[Mapping[str, Any]], 

575 parse_fun: Callable[[Mapping[str, Any]], Q], 

576) -> tuple[Q, ...]: 

577 """Parse a list of quote data.""" 

578 if not json_list: 

579 return () 

580 if isinstance(json_list, str): 

581 json_list = cast(list[dict[str, Any]], json.loads(json_list)) 

582 return_list = [] 

583 for json_data in json_list: 

584 _ = parse_fun(json_data) 

585 await asyncio.sleep(0) 

586 return_list.append(_) 

587 return tuple(return_list) 

588 

589 

590async def update_cache_periodically( 

591 app: Application, worker: int | None 

592) -> None: 

593 """Start updating the cache every hour.""" 

594 # pylint: disable=too-complex, too-many-branches 

595 if "/troet" in typed_stream.Stream( 

596 cast(Iterable[ModuleInfo], app.settings.get("MODULE_INFOS", ())) 

597 ).map(lambda m: m.path): 

598 app.settings["SHOW_SHARING_ON_MASTODON"] = True 

599 if worker: 

600 return 

601 with contextlib.suppress(asyncio.TimeoutError): 

602 await asyncio.wait_for(EVENT_REDIS.wait(), 5) 

603 redis: Redis[str] = cast("Redis[str]", app.settings.get("REDIS")) 

604 prefix: str = app.settings.get("REDIS_PREFIX", NAME).removesuffix("-dev") 

605 apm: None | elasticapm.Client 

606 if EVENT_REDIS.is_set(): # pylint: disable=too-many-nested-blocks 

607 await parse_list_of_quote_data( 

608 await redis.get(f"{prefix}:cached-quote-data:authors"), # type: ignore[arg-type] # noqa: B950 

609 parse_author, 

610 ) 

611 await parse_list_of_quote_data( 

612 await redis.get(f"{prefix}:cached-quote-data:quotes"), # type: ignore[arg-type] # noqa: B950 

613 parse_quote, 

614 ) 

615 await parse_list_of_quote_data( 

616 await redis.get(f"{prefix}:cached-quote-data:wrongquotes"), # type: ignore[arg-type] # noqa: B950 

617 parse_wrong_quote, 

618 ) 

619 if QUOTES_CACHE and AUTHORS_CACHE and WRONG_QUOTES_CACHE: 

620 last_update = await redis.get( 

621 f"{prefix}:cached-quote-data:last-update" 

622 ) 

623 if last_update: 

624 last_update_int = int(last_update) 

625 since_last_update = int(time.time()) - last_update_int 

626 if 0 <= since_last_update < 60 * 60: 

627 # wait until the last update is at least one hour old 

628 update_cache_in = 60 * 60 - since_last_update 

629 if not sys.flags.dev_mode and update_cache_in > 60: 

630 # if in production mode update wrong quotes just to be sure 

631 try: 

632 await update_cache( 

633 app, update_quotes=False, update_authors=False 

634 ) 

635 except Exception: # pylint: disable=broad-except 

636 LOGGER.exception("Updating quotes cache failed") 

637 apm = app.settings.get("ELASTIC_APM", {}).get( 

638 "CLIENT" 

639 ) 

640 if apm: 

641 apm.capture_exception() 

642 else: 

643 LOGGER.info("Updated quotes cache successfully") 

644 LOGGER.info( 

645 "Next update of quotes cache in %d seconds", 

646 update_cache_in, 

647 ) 

648 await asyncio.sleep(update_cache_in) 

649 

650 # update the cache every hour 

651 failed = 0 

652 while not EVENT_SHUTDOWN.is_set(): # pylint: disable=while-used 

653 try: 

654 await update_cache(app) 

655 except Exception: # pylint: disable=broad-except 

656 LOGGER.exception("Updating quotes cache failed") 

657 if apm := app.settings.get("ELASTIC_APM", {}).get("CLIENT"): 

658 apm.capture_exception() 

659 failed += 1 

660 await asyncio.sleep(pow(min(failed * 2, 60), 2)) # 4,16,...,60*60 

661 else: 

662 LOGGER.info("Updated quotes cache successfully") 

663 failed = 0 

664 await asyncio.sleep(60 * 60) 

665 

666 

667async def update_cache( # pylint: disable=too-complex,too-many-branches,too-many-locals,too-many-statements # noqa: B950,C901 

668 app: Application, 

669 update_wrong_quotes: bool = True, 

670 update_quotes: bool = True, 

671 update_authors: bool = True, 

672) -> None: 

673 """Fill the cache with all data from the API.""" 

674 LOGGER.info("Updating quotes cache") 

675 redis: Redis[str] = cast("Redis[str]", app.settings.get("REDIS")) 

676 prefix: str = app.settings.get("REDIS_PREFIX", NAME).removesuffix("-dev") 

677 redis_available = EVENT_REDIS.is_set() 

678 exceptions: list[Exception] = [] 

679 

680 if update_wrong_quotes: 

681 try: 

682 await _update_cache(WrongQuote, parse_wrong_quote, redis, prefix) 

683 except Exception as err: # pylint: disable=broad-exception-caught 

684 exceptions.append(err) 

685 

686 deleted_quotes: set[int] = set() 

687 

688 if update_quotes: 

689 try: 

690 quotes = await _update_cache(Quote, parse_quote, redis, prefix) 

691 except Exception as err: # pylint: disable=broad-exception-caught 

692 exceptions.append(err) 

693 else: 

694 with QUOTES_CACHE.lock: 

695 all_quote_ids = {q.id for q in quotes} 

696 max_quote_id = max(all_quote_ids) 

697 old_ids_in_cache = { 

698 _id for _id in QUOTES_CACHE if _id <= max_quote_id 

699 } 

700 deleted_quotes = old_ids_in_cache - all_quote_ids 

701 for _id in deleted_quotes: 

702 del QUOTES_CACHE[_id] 

703 

704 if len(QUOTES_CACHE) < len(quotes): 

705 LOGGER.error("Cache has less elements than just fetched") 

706 

707 deleted_authors: set[int] = set() 

708 

709 if update_authors: 

710 try: 

711 authors = await _update_cache(Author, parse_author, redis, prefix) 

712 except Exception as err: # pylint: disable=broad-exception-caught 

713 exceptions.append(err) 

714 else: 

715 with AUTHORS_CACHE.lock: 

716 all_author_ids = {q.id for q in authors} 

717 max_author_id = max(all_author_ids) 

718 old_ids_in_cache = { 

719 _id for _id in AUTHORS_CACHE if _id <= max_author_id 

720 } 

721 deleted_authors = old_ids_in_cache - all_author_ids 

722 for _id in deleted_authors: 

723 del AUTHORS_CACHE[_id] 

724 

725 if len(AUTHORS_CACHE) < len(authors): 

726 LOGGER.error("Cache has less elements than just fetched") 

727 

728 if deleted_authors or deleted_quotes: 

729 deleted_wrong_quotes: set[tuple[int, int]] = set() 

730 with WRONG_QUOTES_CACHE.lock: 

731 for qid, aid in tuple(WRONG_QUOTES_CACHE): 

732 if qid in deleted_quotes or aid in deleted_authors: 

733 deleted_wrong_quotes.add((qid, aid)) 

734 del WRONG_QUOTES_CACHE[(qid, aid)] 

735 LOGGER.warning( 

736 "Deleted %d wrong quotes: %r", 

737 len(deleted_wrong_quotes), 

738 deleted_wrong_quotes, 

739 ) 

740 

741 if exceptions: 

742 raise ExceptionGroup("Cache could not be updated", exceptions) 

743 

744 if ( 

745 redis_available 

746 and update_wrong_quotes 

747 and update_quotes 

748 and update_authors 

749 ): 

750 await redis.setex( 

751 f"{prefix}:cached-quote-data:last-update", 

752 60 * 60 * 24 * 30, 

753 int(time.time()), 

754 ) 

755 

756 

757async def _update_cache[Q: QuotesObjBase]( 

758 klass: type[Q], 

759 parse: Callable[[Mapping[str, Any]], Q], 

760 redis: Redis[str], 

761 redis_prefix: str, 

762) -> tuple[Q, ...]: 

763 wq_data = await make_api_request( 

764 klass.fetch_all_endpoint(), entity_should_exist=True 

765 ) 

766 if wq_data is None: 

767 LOGGER.error("%s returned 404", klass.fetch_all_endpoint()) 

768 return () 

769 parsed_data = await parse_list_of_quote_data( 

770 wq_data, 

771 parse, 

772 ) 

773 if wq_data and EVENT_REDIS.is_set(): 

774 await redis.setex( 

775 f"{redis_prefix}:cached-quote-data:{klass.fetch_all_endpoint()}", 

776 60 * 60 * 24 * 30, 

777 json.dumps(wq_data, option=ORJSON_OPTIONS), 

778 ) 

779 return parsed_data 

780 

781 

782async def get_author_by_id(author_id: int) -> Author | None: 

783 """Get an author by its id.""" 

784 author = AUTHORS_CACHE.get(author_id) 

785 if author is not None: 

786 return author 

787 data = await make_api_request( 

788 f"authors/{author_id}", entity_should_exist=False 

789 ) 

790 if data is None: 

791 return None 

792 return parse_author(data) 

793 

794 

795async def get_quote_by_id(quote_id: int) -> Quote | None: 

796 """Get a quote by its id.""" 

797 quote = QUOTES_CACHE.get(quote_id) 

798 if quote is not None: 

799 return quote 

800 data = await make_api_request( 

801 f"quotes/{quote_id}", entity_should_exist=False 

802 ) 

803 if data is None: 

804 return None 

805 return parse_quote(data) 

806 

807 

808async def get_wrong_quote( 

809 quote_id: int, author_id: int, use_cache: bool = True