1  # Copyright 2010-2023 Kurt McKee <contactme@kurtmckee.org>
   2  # Copyright 2002-2008 Mark Pilgrim
   3  # All rights reserved.
   4  #
   5  # This file is a part of feedparser.
   6  #
   7  # Redistribution and use in source and binary forms, with or without
   8  # modification, are permitted provided that the following conditions are met:
   9  #
  10  # * Redistributions of source code must retain the above copyright notice,
  11  #   this list of conditions and the following disclaimer.
  12  # * Redistributions in binary form must reproduce the above copyright notice,
  13  #   this list of conditions and the following disclaimer in the documentation
  14  #   and/or other materials provided with the distribution.
  15  #
  16  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
  17  # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  20  # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  21  # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  22  # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  23  # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  24  # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  25  # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  26  # POSSIBILITY OF SUCH DAMAGE.
  27  
  28  import re
  29  import urllib.parse
  30  
  31  from .html import _BaseHTMLProcessor
  32  
  33  # If you want feedparser to allow all URL schemes, set this to ()
  34  # List culled from Python's urlparse documentation at:
  35  #   http://docs.python.org/library/urlparse.html
  36  # as well as from "URI scheme" at Wikipedia:
  37  #   https://secure.wikimedia.org/wikipedia/en/wiki/URI_scheme
  38  # Many more will likely need to be added!
  39  ACCEPTABLE_URI_SCHEMES = (
  40      'file', 'ftp', 'gopher', 'h323', 'hdl', 'http', 'https', 'imap', 'magnet',
  41      'mailto', 'mms', 'news', 'nntp', 'prospero', 'rsync', 'rtsp', 'rtspu',
  42      'sftp', 'shttp', 'sip', 'sips', 'snews', 'svn', 'svn+ssh', 'telnet',
  43      'wais',
  44      # Additional common-but-unofficial schemes
  45      'aim', 'callto', 'cvs', 'facetime', 'feed', 'git', 'gtalk', 'irc', 'ircs',
  46      'irc6', 'itms', 'mms', 'msnim', 'skype', 'ssh', 'smb', 'svn', 'ymsg',
  47  )
  48  
  49  _urifixer = re.compile('^([A-Za-z][A-Za-z0-9+-.]*://)(/*)(.*?)')
  50  
  51  
  52  def _urljoin(base, uri):
  53      uri = _urifixer.sub(r'\1\3', uri)
  54      try:
  55          uri = urllib.parse.urljoin(base, uri)
  56      except ValueError:
  57          uri = ''
  58      return uri
  59  
  60  
  61  def convert_to_idn(url):
  62      """Convert a URL to IDN notation"""
  63      # this function should only be called with a unicode string
  64      # strategy: if the host cannot be encoded in ascii, then
  65      # it'll be necessary to encode it in idn form
  66      parts = list(urllib.parse.urlsplit(url))
  67      try:
  68          parts[1].encode('ascii')
  69      except UnicodeEncodeError:
  70          # the url needs to be converted to idn notation
  71          host = parts[1].rsplit(':', 1)
  72          newhost = []
  73          port = ''
  74          if len(host) == 2:
  75              port = host.pop()
  76          for h in host[0].split('.'):
  77              newhost.append(h.encode('idna').decode('utf-8'))
  78          parts[1] = '.'.join(newhost)
  79          if port:
  80              parts[1] += ':' + port
  81          return urllib.parse.urlunsplit(parts)
  82      else:
  83          return url
  84  
  85  
  86  def make_safe_absolute_uri(base, rel=None):
  87      # bail if ACCEPTABLE_URI_SCHEMES is empty
  88      if not ACCEPTABLE_URI_SCHEMES:
  89          return _urljoin(base, rel or '')
  90      if not base:
  91          return rel or ''
  92      if not rel:
  93          try:
  94              scheme = urllib.parse.urlparse(base)[0]
  95          except ValueError:
  96              return ''
  97          if not scheme or scheme in ACCEPTABLE_URI_SCHEMES:
  98              return base
  99          return ''
 100      uri = _urljoin(base, rel)
 101      if uri.strip().split(':', 1)[0] not in ACCEPTABLE_URI_SCHEMES:
 102          return ''
 103      return uri
 104  
 105  
 106  class RelativeURIResolver(_BaseHTMLProcessor):
 107      relative_uris = {
 108          ('a', 'href'),
 109          ('applet', 'codebase'),
 110          ('area', 'href'),
 111          ('audio', 'src'),
 112          ('blockquote', 'cite'),
 113          ('body', 'background'),
 114          ('del', 'cite'),
 115          ('form', 'action'),
 116          ('frame', 'longdesc'),
 117          ('frame', 'src'),
 118          ('iframe', 'longdesc'),
 119          ('iframe', 'src'),
 120          ('head', 'profile'),
 121          ('img', 'longdesc'),
 122          ('img', 'src'),
 123          ('img', 'usemap'),
 124          ('input', 'src'),
 125          ('input', 'usemap'),
 126          ('ins', 'cite'),
 127          ('link', 'href'),
 128          ('object', 'classid'),
 129          ('object', 'codebase'),
 130          ('object', 'data'),
 131          ('object', 'usemap'),
 132          ('q', 'cite'),
 133          ('script', 'src'),
 134          ('source', 'src'),
 135          ('video', 'poster'),
 136          ('video', 'src'),
 137      }
 138  
 139      def __init__(self, baseuri, encoding, _type):
 140          _BaseHTMLProcessor.__init__(self, encoding, _type)
 141          self.baseuri = baseuri
 142  
 143      def resolve_uri(self, uri):
 144          return make_safe_absolute_uri(self.baseuri, uri.strip())
 145  
 146      def unknown_starttag(self, tag, attrs):
 147          attrs = self.normalize_attrs(attrs)
 148          attrs = [(key, ((tag, key) in self.relative_uris) and self.resolve_uri(value) or value) for key, value in attrs]
 149          super(RelativeURIResolver, self).unknown_starttag(tag, attrs)
 150  
 151  
 152  def resolve_relative_uris(html_source, base_uri, encoding, type_):
 153      p = RelativeURIResolver(base_uri, encoding, type_)
 154      p.feed(html_source)
 155      return p.output()
 156