1  """.. include:: ../README.md"""
   2  
   3  from __future__ import annotations
   4  
   5  import logging
   6  import time
   7  import itertools
   8  import feedparser
   9  import os
  10  import math
  11  import re
  12  import requests
  13  import warnings
  14  
  15  from urllib.parse import urlencode, urlparse
  16  from urllib.request import urlretrieve
  17  from datetime import datetime, timedelta, timezone
  18  from calendar import timegm
  19  
  20  from enum import Enum
  21  from typing import Dict, Generator, List, Optional
  22  
  23  logger = logging.getLogger(__name__)
  24  
  25  _DEFAULT_TIME = datetime.min
  26  
  27  
  28  class Result(object):
  29      """
  30      An entry in an arXiv query results feed.
  31  
  32      See [the arXiv API User's Manual: Details of Atom Results
  33      Returned](https://arxiv.org/help/api/user-manual#_details_of_atom_results_returned).
  34      """
  35  
  36      entry_id: str
  37      """A url of the form `https://arxiv.org/abs/{id}`."""
  38      updated: datetime
  39      """When the result was last updated."""
  40      published: datetime
  41      """When the result was originally published."""
  42      title: str
  43      """The title of the result."""
  44      authors: List[Author]
  45      """The result's authors."""
  46      summary: str
  47      """The result abstract."""
  48      comment: Optional[str]
  49      """The authors' comment if present."""
  50      journal_ref: Optional[str]
  51      """A journal reference if present."""
  52      doi: Optional[str]
  53      """A URL for the resolved DOI to an external resource if present."""
  54      primary_category: str
  55      """
  56      The result's primary arXiv category. See [arXiv: Category
  57      Taxonomy](https://arxiv.org/category_taxonomy).
  58      """
  59      categories: List[str]
  60      """
  61      All of the result's categories. See [arXiv: Category
  62      Taxonomy](https://arxiv.org/category_taxonomy).
  63      """
  64      links: List[Link]
  65      """Up to three URLs associated with this result."""
  66      pdf_url: Optional[str]
  67      """The URL of a PDF version of this result if present among links."""
  68      _raw: feedparser.FeedParserDict
  69      """
  70      The raw feedparser result object if this Result was constructed with
  71      Result._from_feed_entry.
  72      """
  73  
  74      def __init__(
  75          self,
  76          entry_id: str,
  77          updated: datetime = _DEFAULT_TIME,
  78          published: datetime = _DEFAULT_TIME,
  79          title: str = "",
  80          authors: List[Author] = [],
  81          summary: str = "",
  82          comment: str = "",
  83          journal_ref: str = "",
  84          doi: str = "",
  85          primary_category: str = "",
  86          categories: List[str] = [],
  87          links: List[Link] = [],
  88          _raw: feedparser.FeedParserDict = None,
  89      ):
  90          """
  91          Constructs an arXiv search result item.
  92  
  93          In most cases, prefer using `Result._from_feed_entry` to parsing and
  94          constructing `Result`s yourself.
  95          """
  96          self.entry_id = entry_id
  97          self.updated = updated
  98          self.published = published
  99          self.title = title
 100          self.authors = authors
 101          self.summary = summary
 102          self.comment = comment
 103          self.journal_ref = journal_ref
 104          self.doi = doi
 105          self.primary_category = primary_category
 106          self.categories = categories
 107          self.links = links
 108          # Calculated members
 109          self.pdf_url = Result._get_pdf_url(links)
 110          # Debugging
 111          self._raw = _raw
 112  
 113      def _from_feed_entry(entry: feedparser.FeedParserDict) -> Result:
 114          """
 115          Converts a feedparser entry for an arXiv search result feed into a
 116          Result object.
 117          """
 118          if not hasattr(entry, "id"):
 119              raise Result.MissingFieldError("id")
 120          # Title attribute may be absent for certain titles. Defaulting to "0" as
 121          # it's the only title observed to cause this bug.
 122          # https://github.com/lukasschwab/arxiv.py/issues/71
 123          # title = entry.title if hasattr(entry, "title") else "0"
 124          title = "0"
 125          if hasattr(entry, "title"):
 126              title = entry.title
 127          else:
 128              logger.warning("Result %s is missing title attribute; defaulting to '0'", entry.id)
 129          return Result(
 130              entry_id=entry.id,
 131              updated=Result._to_datetime(entry.updated_parsed),
 132              published=Result._to_datetime(entry.published_parsed),
 133              title=re.sub(r"\s+", " ", title),
 134              authors=[Result.Author._from_feed_author(a) for a in entry.authors],
 135              summary=entry.summary,
 136              comment=entry.get("arxiv_comment"),
 137              journal_ref=entry.get("arxiv_journal_ref"),
 138              doi=entry.get("arxiv_doi"),
 139              primary_category=entry.arxiv_primary_category.get("term"),
 140              categories=[tag.get("term") for tag in entry.tags],
 141              links=[Result.Link._from_feed_link(link) for link in entry.links],
 142              _raw=entry,
 143          )
 144  
 145      def __str__(self) -> str:
 146          return self.entry_id
 147  
 148      def __repr__(self) -> str:
 149          return (
 150              "{}(entry_id={}, updated={}, published={}, title={}, authors={}, "
 151              "summary={}, comment={}, journal_ref={}, doi={}, "
 152              "primary_category={}, categories={}, links={})"
 153          ).format(
 154              _classname(self),
 155              repr(self.entry_id),
 156              repr(self.updated),
 157              repr(self.published),
 158              repr(self.title),
 159              repr(self.authors),
 160              repr(self.summary),
 161              repr(self.comment),
 162              repr(self.journal_ref),
 163              repr(self.doi),
 164              repr(self.primary_category),
 165              repr(self.categories),
 166              repr(self.links),
 167          )
 168  
 169      def __eq__(self, other) -> bool:
 170          if isinstance(other, Result):
 171              return self.entry_id == other.entry_id
 172          return False
 173  
 174      def get_short_id(self) -> str:
 175          """
 176          Returns the short ID for this result.
 177  
 178          + If the result URL is `"https://arxiv.org/abs/2107.05580v1"`,
 179          `result.get_short_id()` returns `2107.05580v1`.
 180  
 181          + If the result URL is `"https://arxiv.org/abs/quant-ph/0201082v1"`,
 182          `result.get_short_id()` returns `"quant-ph/0201082v1"` (the pre-March
 183          2007 arXiv identifier format).
 184  
 185          For an explanation of the difference between arXiv's legacy and current
 186          identifiers, see [Understanding the arXiv
 187          identifier](https://arxiv.org/help/arxiv_identifier).
 188          """
 189          return self.entry_id.split("arxiv.org/abs/")[-1]
 190  
 191      def _get_default_filename(self, extension: str = "pdf") -> str:
 192          """
 193          A default `to_filename` function for the extension given.
 194          """
 195          nonempty_title = self.title if self.title else "UNTITLED"
 196          return ".".join(
 197              [
 198                  self.get_short_id().replace("/", "_"),
 199                  re.sub(r"[^\w]", "_", nonempty_title),
 200                  extension,
 201              ]
 202          )
 203  
 204      def download_pdf(
 205          self,
 206          dirpath: str = "./",
 207          filename: str = "",
 208          download_domain: str = "export.arxiv.org",
 209      ) -> str:
 210          """
 211          Downloads the PDF for this result to the specified directory.
 212  
 213          The filename is generated by calling `to_filename(self)`.
 214          """
 215          if not filename:
 216              filename = self._get_default_filename()
 217          path = os.path.join(dirpath, filename)
 218          pdf_url = Result._substitute_domain(self.pdf_url, download_domain)
 219          written_path, _ = urlretrieve(pdf_url, path)
 220          return written_path
 221  
 222      def download_source(
 223          self,
 224          dirpath: str = "./",
 225          filename: str = "",
 226          download_domain: str = "export.arxiv.org",
 227      ) -> str:
 228          """
 229          Downloads the source tarfile for this result to the specified
 230          directory.
 231  
 232          The filename is generated by calling `to_filename(self)`.
 233          """
 234          if not filename:
 235              filename = self._get_default_filename("tar.gz")
 236          path = os.path.join(dirpath, filename)
 237          pdf_url = Result._substitute_domain(self.pdf_url, download_domain)
 238          # Bodge: construct the source URL from the PDF URL.
 239          src_url = pdf_url.replace("/pdf/", "/src/")
 240          written_path, _ = urlretrieve(src_url, path)
 241          return written_path
 242  
 243      def _get_pdf_url(links: List[Link]) -> str:
 244          """
 245          Finds the PDF link among a result's links and returns its URL.
 246  
 247          Should only be called once for a given `Result`, in its constructor.
 248          After construction, the URL should be available in `Result.pdf_url`.
 249          """
 250          pdf_urls = [link.href for link in links if link.title == "pdf"]
 251          if len(pdf_urls) == 0:
 252              return None
 253          elif len(pdf_urls) > 1:
 254              logger.warning("Result has multiple PDF links; using %s", pdf_urls[0])
 255          return pdf_urls[0]
 256  
 257      def _to_datetime(ts: time.struct_time) -> datetime:
 258          """
 259          Converts a UTC time.struct_time into a time-zone-aware datetime.
 260  
 261          This will be replaced with feedparser functionality [when it becomes
 262          available](https://github.com/kurtmckee/feedparser/issues/212).
 263          """
 264          return datetime.fromtimestamp(timegm(ts), tz=timezone.utc)
 265  
 266      def _substitute_domain(url: str, domain: str) -> str:
 267          """
 268          Replaces the domain of the given URL with the specified domain.
 269  
 270          This is useful for testing purposes.
 271          """
 272          parsed_url = urlparse(url)
 273          return parsed_url._replace(netloc=domain).geturl()
 274  
 275      class Author(object):
 276          """
 277          A light inner class for representing a result's authors.
 278          """
 279  
 280          name: str
 281          """The author's name."""
 282  
 283          def __init__(self, name: str):
 284              """
 285              Constructs an `Author` with the specified name.
 286  
 287              In most cases, prefer using `Author._from_feed_author` to parsing
 288              and constructing `Author`s yourself.
 289              """
 290              self.name = name
 291  
 292          def _from_feed_author(feed_author: feedparser.FeedParserDict) -> Result.Author:
 293              """
 294              Constructs an `Author` with the name specified in an author object
 295              from a feed entry.
 296  
 297              See usage in `Result._from_feed_entry`.
 298              """
 299              return Result.Author(feed_author.name)
 300  
 301          def __str__(self) -> str:
 302              return self.name
 303  
 304          def __repr__(self) -> str:
 305              return "{}({})".format(_classname(self), repr(self.name))
 306  
 307          def __eq__(self, other) -> bool:
 308              if isinstance(other, Result.Author):
 309                  return self.name == other.name
 310              return False
 311  
 312      class Link(object):
 313          """
 314          A light inner class for representing a result's links.
 315          """
 316  
 317          href: str
 318          """The link's `href` attribute."""
 319          title: Optional[str]
 320          """The link's title."""
 321          rel: str
 322          """The link's relationship to the `Result`."""
 323          content_type: str
 324          """The link's HTTP content type."""
 325  
 326          def __init__(
 327              self,
 328              href: str,
 329              title: str = None,
 330              rel: str = None,
 331              content_type: str = None,
 332          ):
 333              """
 334              Constructs a `Link` with the specified link metadata.
 335  
 336              In most cases, prefer using `Link._from_feed_link` to parsing and
 337              constructing `Link`s yourself.
 338              """
 339              self.href = href
 340              self.title = title
 341              self.rel = rel
 342              self.content_type = content_type
 343  
 344          def _from_feed_link(feed_link: feedparser.FeedParserDict) -> Result.Link:
 345              """
 346              Constructs a `Link` with link metadata specified in a link object
 347              from a feed entry.
 348  
 349              See usage in `Result._from_feed_entry`.
 350              """
 351              return Result.Link(
 352                  href=feed_link.href,
 353                  title=feed_link.get("title"),
 354                  rel=feed_link.get("rel"),
 355                  content_type=feed_link.get("content_type"),
 356              )
 357  
 358          def __str__(self) -> str:
 359              return self.href
 360  
 361          def __repr__(self) -> str:
 362              return "{}({}, title={}, rel={}, content_type={})".format(
 363                  _classname(self),
 364                  repr(self.href),
 365                  repr(self.title),
 366                  repr(self.rel),
 367                  repr(self.content_type),
 368              )
 369  
 370          def __eq__(self, other) -> bool:
 371              if isinstance(other, Result.Link):
 372                  return self.href == other.href
 373              return False
 374  
 375      class MissingFieldError(Exception):
 376          """
 377          An error indicating an entry is unparseable because it lacks required
 378          fields.
 379          """
 380  
 381          missing_field: str
 382          """The required field missing from the would-be entry."""
 383          message: str
 384          """Message describing what caused this error."""
 385  
 386          def __init__(self, missing_field):
 387              self.missing_field = missing_field
 388              self.message = "Entry from arXiv missing required info"
 389  
 390          def __repr__(self) -> str:
 391              return "{}({})".format(_classname(self), repr(self.missing_field))
 392  
 393  
 394  class SortCriterion(Enum):
 395      """
 396      A SortCriterion identifies a property by which search results can be
 397      sorted.
 398  
 399      See [the arXiv API User's Manual: sort order for return
 400      results](https://arxiv.org/help/api/user-manual#sort).
 401      """
 402  
 403      Relevance = "relevance"
 404      LastUpdatedDate = "lastUpdatedDate"
 405      SubmittedDate = "submittedDate"
 406  
 407  
 408  class SortOrder(Enum):
 409      """
 410      A SortOrder indicates order in which search results are sorted according
 411      to the specified arxiv.SortCriterion.
 412  
 413      See [the arXiv API User's Manual: sort order for return
 414      results](https://arxiv.org/help/api/user-manual#sort).
 415      """
 416  
 417      Ascending = "ascending"
 418      Descending = "descending"
 419  
 420  
 421  class Search(object):
 422      """
 423      A specification for a search of arXiv's database.
 424  
 425      To run a search, use `Search.run` to use a default client or `Client.run`
 426      with a specific client.
 427      """
 428  
 429      query: str
 430      """
 431      A query string.
 432  
 433      This should be unencoded. Use `au:del_maestro AND ti:checkerboard`, not
 434      `au:del_maestro+AND+ti:checkerboard`.
 435  
 436      See [the arXiv API User's Manual: Details of Query
 437      Construction](https://arxiv.org/help/api/user-manual#query_details).
 438      """
 439      id_list: List[str]
 440      """
 441      A list of arXiv article IDs to which to limit the search.
 442  
 443      See [the arXiv API User's
 444      Manual](https://arxiv.org/help/api/user-manual#search_query_and_id_list)
 445      for documentation of the interaction between `query` and `id_list`.
 446      """
 447      max_results: int | None
 448      """
 449      The maximum number of results to be returned in an execution of this
 450      search. To fetch every result available, set `max_results=None`.
 451  
 452      The API's limit is 300,000 results per query.
 453      """
 454      sort_by: SortCriterion
 455      """The sort criterion for results."""
 456      sort_order: SortOrder
 457      """The sort order for results."""
 458  
 459      def __init__(
 460          self,
 461          query: str = "",
 462          id_list: List[str] = [],
 463          max_results: int | None = None,
 464          sort_by: SortCriterion = SortCriterion.Relevance,
 465          sort_order: SortOrder = SortOrder.Descending,
 466      ):
 467          """
 468          Constructs an arXiv API search with the specified criteria.
 469          """
 470          self.query = query
 471          self.id_list = id_list
 472          # Handle deprecated v1 default behavior.
 473          self.max_results = None if max_results == math.inf else max_results
 474          self.sort_by = sort_by
 475          self.sort_order = sort_order
 476  
 477      def __str__(self) -> str:
 478          # TODO: develop a more informative string representation.
 479          return repr(self)
 480  
 481      def __repr__(self) -> str:
 482          return ("{}(query={}, id_list={}, max_results={}, sort_by={}, sort_order={})").format(
 483              _classname(self),
 484              repr(self.query),
 485              repr(self.id_list),
 486              repr(self.max_results),
 487              repr(self.sort_by),
 488              repr(self.sort_order),
 489          )
 490  
 491      def _url_args(self) -> Dict[str, str]:
 492          """
 493          Returns a dict of search parameters that should be included in an API
 494          request for this search.
 495          """
 496          return {
 497              "search_query": self.query,
 498              "id_list": ",".join(self.id_list),
 499              "sortBy": self.sort_by.value,
 500              "sortOrder": self.sort_order.value,
 501          }
 502  
 503      def results(self, offset: int = 0) -> Generator[Result, None, None]:
 504          """
 505          Executes the specified search using a default arXiv API client. For info
 506          on default behavior, see `Client.__init__` and `Client.results`.
 507  
 508          **Deprecated** after 2.0.0; use `Client.results`.
 509          """
 510          warnings.warn(
 511              "The 'Search.results' method is deprecated, use 'Client.results' instead",
 512              DeprecationWarning,
 513              stacklevel=2,
 514          )
 515          return Client().results(self, offset=offset)
 516  
 517  
 518  class Client(object):
 519      """
 520      Specifies a strategy for fetching results from arXiv's API.
 521  
 522      This class obscures pagination and retry logic, and exposes
 523      `Client.results`.
 524      """
 525  
 526      query_url_format = "https://export.arxiv.org/api/query?{}"
 527      """
 528      The arXiv query API endpoint format.
 529      """
 530      page_size: int
 531      """
 532      Maximum number of results fetched in a single API request. Smaller pages can
 533      be retrieved faster, but may require more round-trips.
 534  
 535      The API's limit is 2000 results per page.
 536      """
 537      delay_seconds: float
 538      """
 539      Number of seconds to wait between API requests.
 540  
 541      [arXiv's Terms of Use](https://arxiv.org/help/api/tou) ask that you "make no
 542      more than one request every three seconds."
 543      """
 544      num_retries: int
 545      """
 546      Number of times to retry a failing API request before raising an Exception.
 547      """
 548  
 549      _last_request_dt: datetime
 550      _session: requests.Session
 551  
 552      def __init__(self, page_size: int = 100, delay_seconds: float = 3.0, num_retries: int = 3):
 553          """
 554          Constructs an arXiv API client with the specified options.
 555  
 556          Note: the default parameters should provide a robust request strategy
 557          for most use cases. Extreme page sizes, delays, or retries risk
 558          violating the arXiv [API Terms of Use](https://arxiv.org/help/api/tou),
 559          brittle behavior, and inconsistent results.
 560          """
 561          self.page_size = page_size
 562          self.delay_seconds = delay_seconds
 563          self.num_retries = num_retries
 564          self._last_request_dt = None
 565          self._session = requests.Session()
 566  
 567      def __str__(self) -> str:
 568          # TODO: develop a more informative string representation.
 569          return repr(self)
 570  
 571      def __repr__(self) -> str:
 572          return "{}(page_size={}, delay_seconds={}, num_retries={})".format(
 573              _classname(self),
 574              repr(self.page_size),
 575              repr(self.delay_seconds),
 576              repr(self.num_retries),
 577          )
 578  
 579      def results(self, search: Search, offset: int = 0) -> Generator[Result, None, None]:
 580          """
 581          Uses this client configuration to fetch one page of the search results
 582          at a time, yielding the parsed `Result`s, until `max_results` results
 583          have been yielded or there are no more search results.
 584  
 585          If all tries fail, raises an `UnexpectedEmptyPageError` or `HTTPError`.
 586  
 587          Setting a nonzero `offset` discards leading records in the result set.
 588          When `offset` is greater than or equal to `search.max_results`, the full
 589          result set is discarded.
 590  
 591          For more on using generators, see
 592          [Generators](https://wiki.python.org/moin/Generators).
 593          """
 594          limit = search.max_results - offset if search.max_results else None
 595          if limit and limit < 0:
 596              return iter(())
 597          return itertools.islice(self._results(search, offset), limit)
 598  
 599      def _results(self, search: Search, offset: int = 0) -> Generator[Result, None, None]:
 600          page_url = self._format_url(search, offset, self.page_size)
 601          feed = self._parse_feed(page_url, first_page=True)
 602          if not feed.entries:
 603              logger.info("Got empty first page; stopping generation")
 604              return
 605          total_results = int(feed.feed.opensearch_totalresults)
 606          logger.info(
 607              "Got first page: %d of %d total results",
 608              len(feed.entries),
 609              total_results,
 610          )
 611  
 612          while feed.entries:
 613              for entry in feed.entries:
 614                  try:
 615                      yield Result._from_feed_entry(entry)
 616                  except Result.MissingFieldError as e:
 617                      logger.warning("Skipping partial result: %s", e)
 618              offset += len(feed.entries)
 619              if offset >= total_results:
 620                  break
 621              page_url = self._format_url(search, offset, self.page_size)
 622              feed = self._parse_feed(page_url, first_page=False)
 623  
 624      def _format_url(self, search: Search, start: int, page_size: int) -> str:
 625          """
 626          Construct a request API for search that returns up to `page_size`
 627          results starting with the result at index `start`.
 628          """
 629          url_args = search._url_args()
 630          url_args.update(
 631              {
 632                  "start": start,
 633                  "max_results": page_size,
 634              }
 635          )
 636          return self.query_url_format.format(urlencode(url_args))
 637  
 638      def _parse_feed(
 639          self, url: str, first_page: bool = True, _try_index: int = 0
 640      ) -> feedparser.FeedParserDict:
 641          """
 642          Fetches the specified URL and parses it with feedparser.
 643  
 644          If a request fails or is unexpectedly empty, retries the request up to
 645          `self.num_retries` times.
 646          """
 647          try:
 648              return self.__try_parse_feed(url, first_page=first_page, try_index=_try_index)
 649          except (
 650              HTTPError,
 651              UnexpectedEmptyPageError,
 652              requests.exceptions.ConnectionError,
 653          ) as err:
 654              if _try_index < self.num_retries:
 655                  logger.debug("Got error (try %d): %s", _try_index, err)
 656                  return self._parse_feed(url, first_page=first_page, _try_index=_try_index + 1)
 657              logger.debug("Giving up (try %d): %s", _try_index, err)
 658              raise err
 659  
 660      def __try_parse_feed(
 661          self,
 662          url: str,
 663          first_page: bool,
 664          try_index: int,
 665      ) -> feedparser.FeedParserDict:
 666          """
 667          Recursive helper for _parse_feed. Enforces `self.delay_seconds`: if that
 668          number of seconds has not passed since `_parse_feed` was last called,
 669          sleeps until delay_seconds seconds have passed.
 670          """
 671          # If this call would violate the rate limit, sleep until it doesn't.
 672          if self._last_request_dt is not None:
 673              required = timedelta(seconds=self.delay_seconds)
 674              since_last_request = datetime.now() - self._last_request_dt
 675              if since_last_request < required:
 676                  to_sleep = (required - since_last_request).total_seconds()
 677                  logger.info("Sleeping: %f seconds", to_sleep)
 678                  time.sleep(to_sleep)
 679  
 680          logger.info("Requesting page (first: %r, try: %d): %s", first_page, try_index, url)
 681  
 682          resp = self._session.get(url, headers={"user-agent": "arxiv.py/2.2.0"})
 683          self._last_request_dt = datetime.now()
 684          if resp.status_code != requests.codes.OK:
 685              raise HTTPError(url, try_index, resp.status_code)
 686  
 687          feed = feedparser.parse(resp.content)
 688          if len(feed.entries) == 0 and not first_page:
 689              raise UnexpectedEmptyPageError(url, try_index, feed)
 690  
 691          if feed.bozo:
 692              logger.warning(
 693                  "Bozo feed; consider handling: %s",
 694                  feed.bozo_exception if "bozo_exception" in feed else None,
 695              )
 696  
 697          return feed
 698  
 699  
 700  class ArxivError(Exception):
 701      """This package's base Exception class."""
 702  
 703      url: str
 704      """The feed URL that could not be fetched."""
 705      retry: int
 706      """
 707      The request try number which encountered this error; 0 for the initial try,
 708      1 for the first retry, and so on.
 709      """
 710      message: str
 711      """Message describing what caused this error."""
 712  
 713      def __init__(self, url: str, retry: int, message: str):
 714          """
 715          Constructs an `ArxivError` encountered while fetching the specified URL.
 716          """
 717          self.url = url
 718          self.retry = retry
 719          self.message = message
 720          super().__init__(self.message)
 721  
 722      def __str__(self) -> str:
 723          return "{} ({})".format(self.message, self.url)
 724  
 725  
 726  class UnexpectedEmptyPageError(ArxivError):
 727      """
 728      An error raised when a page of results that should be non-empty is empty.
 729  
 730      This should never happen in theory, but happens sporadically due to
 731      brittleness in the underlying arXiv API; usually resolved by retries.
 732  
 733      See `Client.results` for usage.
 734      """
 735  
 736      raw_feed: feedparser.FeedParserDict
 737      """
 738      The raw output of `feedparser.parse`. Sometimes this contains useful
 739      diagnostic information, e.g. in 'bozo_exception'.
 740      """
 741  
 742      def __init__(self, url: str, retry: int, raw_feed: feedparser.FeedParserDict):
 743          """
 744          Constructs an `UnexpectedEmptyPageError` encountered for the specified
 745          API URL after `retry` tries.
 746          """
 747          self.url = url
 748          self.raw_feed = raw_feed
 749          super().__init__(url, retry, "Page of results was unexpectedly empty")
 750  
 751      def __repr__(self) -> str:
 752          return "{}({}, {}, {})".format(
 753              _classname(self), repr(self.url), repr(self.retry), repr(self.raw_feed)
 754          )
 755  
 756  
 757  class HTTPError(ArxivError):
 758      """
 759      A non-200 status encountered while fetching a page of results.
 760  
 761      See `Client.results` for usage.
 762      """
 763  
 764      status: int
 765      """The HTTP status reported by feedparser."""
 766  
 767      def __init__(self, url: str, retry: int, status: int):
 768          """
 769          Constructs an `HTTPError` for the specified status code, encountered for
 770          the specified API URL after `retry` tries.
 771          """
 772          self.url = url
 773          self.status = status
 774          super().__init__(
 775              url,
 776              retry,
 777              "Page request resulted in HTTP {}".format(self.status),
 778          )
 779  
 780      def __repr__(self) -> str:
 781          return "{}({}, {}, {})".format(
 782              _classname(self), repr(self.url), repr(self.retry), repr(self.status)
 783          )
 784  
 785  
 786  def _classname(o):
 787      """A helper function for use in __repr__ methods: arxiv.Result.Link."""
 788      return "arxiv.{}".format(o.__class__.__qualname__)
 789