1 # Character encoding routines 2 # Copyright 2010-2023 Kurt McKee <contactme@kurtmckee.org> 3 # Copyright 2002-2008 Mark Pilgrim 4 # All rights reserved. 5 # 6 # This file is a part of feedparser. 7 # 8 # Redistribution and use in source and binary forms, with or without modification, 9 # are permitted provided that the following conditions are met: 10 # 11 # * Redistributions of source code must retain the above copyright notice, 12 # this list of conditions and the following disclaimer. 13 # * Redistributions in binary form must reproduce the above copyright notice, 14 # this list of conditions and the following disclaimer in the documentation 15 # and/or other materials provided with the distribution. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' 18 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 # POSSIBILITY OF SUCH DAMAGE. 28 29 import codecs 30 import re 31 import typing as t 32 33 try: 34 try: 35 import cchardet as chardet 36 except ImportError: 37 import chardet 38 except ImportError: 39 chardet = None 40 lazy_chardet_encoding = None 41 else: 42 def lazy_chardet_encoding(data): 43 return chardet.detect(data)['encoding'] or '' 44 45 from .exceptions import ( 46 CharacterEncodingOverride, 47 CharacterEncodingUnknown, 48 NonXMLContentType, 49 ) 50 51 52 # Each marker represents some of the characters of the opening XML 53 # processing instruction ('<?xm') in the specified encoding. 54 EBCDIC_MARKER = b'\x4C\x6F\xA7\x94' 55 UTF16BE_MARKER = b'\x00\x3C\x00\x3F' 56 UTF16LE_MARKER = b'\x3C\x00\x3F\x00' 57 UTF32BE_MARKER = b'\x00\x00\x00\x3C' 58 UTF32LE_MARKER = b'\x3C\x00\x00\x00' 59 60 ZERO_BYTES = '\x00\x00' 61 62 # Match the opening XML declaration. 63 # Example: <?xml version="1.0" encoding="utf-8"?> 64 RE_XML_DECLARATION = re.compile(r'^<\?xml[^>]*?>') 65 66 # Capture the value of the XML processing instruction's encoding attribute. 67 # Example: <?xml version="1.0" encoding="utf-8"?> 68 RE_XML_PI_ENCODING = re.compile(br'^<\?.*encoding=[\'"](.*?)[\'"].*\?>') 69 70 71 def parse_content_type(line: str) -> t.Tuple[str, str]: 72 """Parse an HTTP Content-Type header. 73 74 The return value will be a tuple of strings: 75 the MIME type, and the value of the "charset" (if any). 76 77 This is a custom replacement for Python's cgi.parse_header(). 78 The cgi module will be removed in Python 3.13. 79 """ 80 81 chunks = line.split(";") 82 if not chunks: 83 return "", "" 84 85 mime_type = chunks[0].strip() 86 charset_value = "" 87 for chunk in chunks[1:]: 88 key, _, value = chunk.partition("=") 89 if key.strip().lower() == "charset": 90 charset_value = value.strip().strip("\"'") 91 92 return mime_type, charset_value 93 94 95 def convert_to_utf8(http_headers, data, result): 96 """Detect and convert the character encoding to UTF-8. 97 98 http_headers is a dictionary 99 data is a raw string (not Unicode)""" 100 101 # This is so much trickier than it sounds, it's not even funny. 102 # According to RFC 3023 ('XML Media Types'), if the HTTP Content-Type 103 # is application/xml, application/*+xml, 104 # application/xml-external-parsed-entity, or application/xml-dtd, 105 # the encoding given in the charset parameter of the HTTP Content-Type 106 # takes precedence over the encoding given in the XML prefix within the 107 # document, and defaults to 'utf-8' if neither are specified. But, if 108 # the HTTP Content-Type is text/xml, text/*+xml, or 109 # text/xml-external-parsed-entity, the encoding given in the XML prefix 110 # within the document is ALWAYS IGNORED and only the encoding given in 111 # the charset parameter of the HTTP Content-Type header should be 112 # respected, and it defaults to 'us-ascii' if not specified. 113 114 # Furthermore, discussion on the atom-syntax mailing list with the 115 # author of RFC 3023 leads me to the conclusion that any document 116 # served with a Content-Type of text/* and no charset parameter 117 # must be treated as us-ascii. (We now do this.) And also that it 118 # must always be flagged as non-well-formed. (We now do this too.) 119 120 # If Content-Type is unspecified (input was local file or non-HTTP source) 121 # or unrecognized (server just got it totally wrong), then go by the 122 # encoding given in the XML prefix of the document and default to 123 # 'iso-8859-1' as per the HTTP specification (RFC 2616). 124 125 # Then, assuming we didn't find a character encoding in the HTTP headers 126 # (and the HTTP Content-type allowed us to look in the body), we need 127 # to sniff the first few bytes of the XML data and try to determine 128 # whether the encoding is ASCII-compatible. Section F of the XML 129 # specification shows the way here: 130 # http://www.w3.org/TR/REC-xml/#sec-guessing-no-ext-info 131 132 # If the sniffed encoding is not ASCII-compatible, we need to make it 133 # ASCII compatible so that we can sniff further into the XML declaration 134 # to find the encoding attribute, which will tell us the true encoding. 135 136 # Of course, none of this guarantees that we will be able to parse the 137 # feed in the declared character encoding (assuming it was declared 138 # correctly, which many are not). iconv_codec can help a lot; 139 # you should definitely install it if you can. 140 # http://cjkpython.i18n.org/ 141 142 bom_encoding = '' 143 xml_encoding = '' 144 145 # Look at the first few bytes of the document to guess what 146 # its encoding may be. We only need to decode enough of the 147 # document that we can use an ASCII-compatible regular 148 # expression to search for an XML encoding declaration. 149 # The heuristic follows the XML specification, section F: 150 # http://www.w3.org/TR/REC-xml/#sec-guessing-no-ext-info 151 # Check for BOMs first. 152 if data[:4] == codecs.BOM_UTF32_BE: 153 bom_encoding = 'utf-32be' 154 data = data[4:] 155 elif data[:4] == codecs.BOM_UTF32_LE: 156 bom_encoding = 'utf-32le' 157 data = data[4:] 158 elif data[:2] == codecs.BOM_UTF16_BE and data[2:4] != ZERO_BYTES: 159 bom_encoding = 'utf-16be' 160 data = data[2:] 161 elif data[:2] == codecs.BOM_UTF16_LE and data[2:4] != ZERO_BYTES: 162 bom_encoding = 'utf-16le' 163 data = data[2:] 164 elif data[:3] == codecs.BOM_UTF8: 165 bom_encoding = 'utf-8' 166 data = data[3:] 167 # Check for the characters '<?xm' in several encodings. 168 elif data[:4] == EBCDIC_MARKER: 169 bom_encoding = 'cp037' 170 elif data[:4] == UTF16BE_MARKER: 171 bom_encoding = 'utf-16be' 172 elif data[:4] == UTF16LE_MARKER: 173 bom_encoding = 'utf-16le' 174 elif data[:4] == UTF32BE_MARKER: 175 bom_encoding = 'utf-32be' 176 elif data[:4] == UTF32LE_MARKER: 177 bom_encoding = 'utf-32le' 178 179 tempdata = data 180 try: 181 if bom_encoding: 182 tempdata = data.decode(bom_encoding).encode('utf-8') 183 except (UnicodeDecodeError, LookupError): 184 # feedparser recognizes UTF-32 encodings that aren't 185 # available in Python 2.4 and 2.5, so it's possible to 186 # encounter a LookupError during decoding. 187 xml_encoding_match = None 188 else: 189 xml_encoding_match = RE_XML_PI_ENCODING.match(tempdata) 190 191 if xml_encoding_match: 192 xml_encoding = xml_encoding_match.groups()[0].decode('utf-8').lower() 193 # Normalize the xml_encoding if necessary. 194 if bom_encoding and (xml_encoding in ( 195 'u16', 'utf-16', 'utf16', 'utf_16', 196 'u32', 'utf-32', 'utf32', 'utf_32', 197 'iso-10646-ucs-2', 'iso-10646-ucs-4', 198 'csucs4', 'csunicode', 'ucs-2', 'ucs-4' 199 )): 200 xml_encoding = bom_encoding 201 202 # Find the HTTP Content-Type and, hopefully, a character 203 # encoding provided by the server. The Content-Type is used 204 # to choose the "correct" encoding among the BOM encoding, 205 # XML declaration encoding, and HTTP encoding, following the 206 # heuristic defined in RFC 3023. 207 http_content_type = http_headers.get('content-type') or '' 208 http_content_type, http_encoding = parse_content_type(http_content_type) 209 210 acceptable_content_type = 0 211 application_content_types = ('application/xml', 'application/xml-dtd', 212 'application/xml-external-parsed-entity') 213 text_content_types = ('text/xml', 'text/xml-external-parsed-entity') 214 if ( 215 http_content_type in application_content_types 216 or ( 217 http_content_type.startswith('application/') 218 and http_content_type.endswith('+xml') 219 ) 220 ): 221 acceptable_content_type = 1 222 rfc3023_encoding = http_encoding or xml_encoding or 'utf-8' 223 elif ( 224 http_content_type in text_content_types 225 or ( 226 http_content_type.startswith('text/') 227 and http_content_type.endswith('+xml') 228 ) 229 ): 230 acceptable_content_type = 1 231 rfc3023_encoding = http_encoding or 'us-ascii' 232 elif http_content_type.startswith('text/'): 233 rfc3023_encoding = http_encoding or 'us-ascii' 234 elif http_headers and 'content-type' not in http_headers: 235 rfc3023_encoding = xml_encoding or 'iso-8859-1' 236 else: 237 rfc3023_encoding = xml_encoding or 'utf-8' 238 # gb18030 is a superset of gb2312, so always replace gb2312 239 # with gb18030 for greater compatibility. 240 if rfc3023_encoding.lower() == 'gb2312': 241 rfc3023_encoding = 'gb18030' 242 if xml_encoding.lower() == 'gb2312': 243 xml_encoding = 'gb18030' 244 245 # there are four encodings to keep track of: 246 # - http_encoding is the encoding declared in the Content-Type HTTP header 247 # - xml_encoding is the encoding declared in the <?xml declaration 248 # - bom_encoding is the encoding sniffed from the first 4 bytes of the XML data 249 # - rfc3023_encoding is the actual encoding, as per RFC 3023 and a variety of other conflicting specifications 250 error = None 251 252 if http_headers and (not acceptable_content_type): 253 if 'content-type' in http_headers: 254 msg = '%s is not an XML media type' % http_headers['content-type'] 255 else: 256 msg = 'no Content-type specified' 257 error = NonXMLContentType(msg) 258 259 # determine character encoding 260 known_encoding = 0 261 tried_encodings = [] 262 # try: HTTP encoding, declared XML encoding, encoding sniffed from BOM 263 for proposed_encoding in (rfc3023_encoding, xml_encoding, bom_encoding, 264 lazy_chardet_encoding, 'utf-8', 'windows-1252', 'iso-8859-2'): 265 if callable(proposed_encoding): 266 proposed_encoding = proposed_encoding(data) 267 if not proposed_encoding: 268 continue 269 if proposed_encoding in tried_encodings: 270 continue 271 tried_encodings.append(proposed_encoding) 272 try: 273 data = data.decode(proposed_encoding) 274 except (UnicodeDecodeError, LookupError): 275 pass 276 else: 277 known_encoding = 1 278 # Update the encoding in the opening XML processing instruction. 279 new_declaration = '''<?xml version='1.0' encoding='utf-8'?>''' 280 if RE_XML_DECLARATION.search(data): 281 data = RE_XML_DECLARATION.sub(new_declaration, data) 282 else: 283 data = new_declaration + '\n' + data 284 data = data.encode('utf-8') 285 break 286 # if still no luck, give up 287 if not known_encoding: 288 error = CharacterEncodingUnknown( 289 'document encoding unknown, I tried ' + 290 '%s, %s, utf-8, windows-1252, and iso-8859-2 but nothing worked' % 291 (rfc3023_encoding, xml_encoding)) 292 rfc3023_encoding = '' 293 elif proposed_encoding != rfc3023_encoding: 294 error = CharacterEncodingOverride( 295 'document declared as %s, but parsed as %s' % 296 (rfc3023_encoding, proposed_encoding)) 297 rfc3023_encoding = proposed_encoding 298 299 result['encoding'] = rfc3023_encoding 300 if error: 301 result['bozo'] = True 302 result['bozo_exception'] = error 303 return data 304