1 # Copyright 2010-2023 Kurt McKee <contactme@kurtmckee.org> 2 # Copyright 2002-2008 Mark Pilgrim 3 # All rights reserved. 4 # 5 # This file is a part of feedparser. 6 # 7 # Redistribution and use in source and binary forms, with or without 8 # modification, are permitted provided that the following conditions are met: 9 # 10 # * Redistributions of source code must retain the above copyright notice, 11 # this list of conditions and the following disclaimer. 12 # * Redistributions in binary form must reproduce the above copyright notice, 13 # this list of conditions and the following disclaimer in the documentation 14 # and/or other materials provided with the distribution. 15 # 16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' 17 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 # POSSIBILITY OF SUCH DAMAGE. 27 28 import base64 29 import binascii 30 import copy 31 import html.entities 32 import re 33 import xml.sax.saxutils 34 35 from .html import _cp1252 36 from .namespaces import _base, cc, dc, georss, itunes, mediarss, psc 37 from .sanitizer import _sanitize_html, _HTMLSanitizer 38 from .util import FeedParserDict 39 from .urls import _urljoin, make_safe_absolute_uri, resolve_relative_uris 40 41 42 class _FeedParserMixin( 43 _base.Namespace, 44 cc.Namespace, 45 dc.Namespace, 46 georss.Namespace, 47 itunes.Namespace, 48 mediarss.Namespace, 49 psc.Namespace, 50 ): 51 namespaces = { 52 '': '', 53 'http://backend.userland.com/rss': '', 54 'http://blogs.law.harvard.edu/tech/rss': '', 55 'http://purl.org/rss/1.0/': '', 56 'http://my.netscape.com/rdf/simple/0.9/': '', 57 'http://example.com/newformat#': '', 58 'http://example.com/necho': '', 59 'http://purl.org/echo/': '', 60 'uri/of/echo/namespace#': '', 61 'http://purl.org/pie/': '', 62 'http://purl.org/atom/ns#': '', 63 'http://www.w3.org/2005/Atom': '', 64 'http://purl.org/rss/1.0/modules/rss091#': '', 65 66 'http://webns.net/mvcb/': 'admin', 67 'http://purl.org/rss/1.0/modules/aggregation/': 'ag', 68 'http://purl.org/rss/1.0/modules/annotate/': 'annotate', 69 'http://media.tangent.org/rss/1.0/': 'audio', 70 'http://backend.userland.com/blogChannelModule': 'blogChannel', 71 'http://creativecommons.org/ns#license': 'cc', 72 'http://web.resource.org/cc/': 'cc', 73 'http://cyber.law.harvard.edu/rss/creativeCommonsRssModule.html': 'creativeCommons', 74 'http://backend.userland.com/creativeCommonsRssModule': 'creativeCommons', 75 'http://purl.org/rss/1.0/modules/company': 'co', 76 'http://purl.org/rss/1.0/modules/content/': 'content', 77 'http://my.theinfo.org/changed/1.0/rss/': 'cp', 78 'http://purl.org/dc/elements/1.1/': 'dc', 79 'http://purl.org/dc/terms/': 'dcterms', 80 'http://purl.org/rss/1.0/modules/email/': 'email', 81 'http://purl.org/rss/1.0/modules/event/': 'ev', 82 'http://rssnamespace.org/feedburner/ext/1.0': 'feedburner', 83 'http://freshmeat.net/rss/fm/': 'fm', 84 'http://xmlns.com/foaf/0.1/': 'foaf', 85 'http://www.w3.org/2003/01/geo/wgs84_pos#': 'geo', 86 'http://www.georss.org/georss': 'georss', 87 'http://www.opengis.net/gml': 'gml', 88 'http://postneo.com/icbm/': 'icbm', 89 'http://purl.org/rss/1.0/modules/image/': 'image', 90 'http://www.itunes.com/DTDs/PodCast-1.0.dtd': 'itunes', 91 'http://example.com/DTDs/PodCast-1.0.dtd': 'itunes', 92 'http://purl.org/rss/1.0/modules/link/': 'l', 93 'http://search.yahoo.com/mrss': 'media', 94 # Version 1.1.2 of the Media RSS spec added the trailing slash on the namespace 95 'http://search.yahoo.com/mrss/': 'media', 96 'http://madskills.com/public/xml/rss/module/pingback/': 'pingback', 97 'http://prismstandard.org/namespaces/1.2/basic/': 'prism', 98 'http://www.w3.org/1999/02/22-rdf-syntax-ns#': 'rdf', 99 'http://www.w3.org/2000/01/rdf-schema#': 'rdfs', 100 'http://purl.org/rss/1.0/modules/reference/': 'ref', 101 'http://purl.org/rss/1.0/modules/richequiv/': 'reqv', 102 'http://purl.org/rss/1.0/modules/search/': 'search', 103 'http://purl.org/rss/1.0/modules/slash/': 'slash', 104 'http://schemas.xmlsoap.org/soap/envelope/': 'soap', 105 'http://purl.org/rss/1.0/modules/servicestatus/': 'ss', 106 'http://hacks.benhammersley.com/rss/streaming/': 'str', 107 'http://purl.org/rss/1.0/modules/subscription/': 'sub', 108 'http://purl.org/rss/1.0/modules/syndication/': 'sy', 109 'http://schemas.pocketsoap.com/rss/myDescModule/': 'szf', 110 'http://purl.org/rss/1.0/modules/taxonomy/': 'taxo', 111 'http://purl.org/rss/1.0/modules/threading/': 'thr', 112 'http://purl.org/rss/1.0/modules/textinput/': 'ti', 113 'http://madskills.com/public/xml/rss/module/trackback/': 'trackback', 114 'http://wellformedweb.org/commentAPI/': 'wfw', 115 'http://purl.org/rss/1.0/modules/wiki/': 'wiki', 116 'http://www.w3.org/1999/xhtml': 'xhtml', 117 'http://www.w3.org/1999/xlink': 'xlink', 118 'http://www.w3.org/XML/1998/namespace': 'xml', 119 'http://podlove.org/simple-chapters': 'psc', 120 } 121 _matchnamespaces = {} 122 123 can_be_relative_uri = { 124 'comments', 125 'docs', 126 'href', 127 'icon', 128 'id', 129 'link', 130 'logo', 131 'url', 132 'wfw_comment', 133 'wfw_commentrss', 134 } 135 136 can_contain_relative_uris = { 137 'content', 138 'copyright', 139 'description', 140 'info', 141 'rights', 142 'subtitle', 143 'summary', 144 'tagline', 145 'title', 146 } 147 148 can_contain_dangerous_markup = { 149 'content', 150 'copyright', 151 'description', 152 'info', 153 'rights', 154 'subtitle', 155 'summary', 156 'tagline', 157 'title', 158 } 159 160 html_types = { 161 'application/xhtml+xml', 162 'text/html', 163 } 164 165 def __init__(self): 166 if not self._matchnamespaces: 167 for k, v in self.namespaces.items(): 168 self._matchnamespaces[k.lower()] = v 169 self.feeddata = FeedParserDict() # feed-level data 170 self.entries = [] # list of entry-level data 171 self.version = '' # feed type/version, see SUPPORTED_VERSIONS 172 self.namespaces_in_use = {} # dictionary of namespaces defined by the feed 173 174 # the following are used internally to track state; 175 # this is really out of control and should be refactored 176 self.infeed = 0 177 self.inentry = 0 178 self.incontent = 0 179 self.intextinput = 0 180 self.inimage = 0 181 self.inauthor = 0 182 self.incontributor = 0 183 self.inpublisher = 0 184 self.insource = 0 185 186 self.sourcedata = FeedParserDict() 187 self.contentparams = FeedParserDict() 188 self._summaryKey = None 189 self.namespacemap = {} 190 self.elementstack = [] 191 self.basestack = [] 192 self.langstack = [] 193 self.svgOK = 0 194 self.title_depth = -1 195 self.depth = 0 196 self.hasContent = 0 197 if self.lang: 198 self.feeddata['language'] = self.lang.replace('_', '-') 199 200 # A map of the following form: 201 # { 202 # object_that_value_is_set_on: { 203 # property_name: depth_of_node_property_was_extracted_from, 204 # other_property: depth_of_node_property_was_extracted_from, 205 # }, 206 # } 207 self.property_depth_map = {} 208 super(_FeedParserMixin, self).__init__() 209 210 def _normalize_attributes(self, kv): 211 raise NotImplementedError 212 213 def unknown_starttag(self, tag, attrs): 214 # increment depth counter 215 self.depth += 1 216 217 # normalize attrs 218 attrs = [self._normalize_attributes(attr) for attr in attrs] 219 220 # track xml:base and xml:lang 221 attrs_d = dict(attrs) 222 baseuri = attrs_d.get('xml:base', attrs_d.get('base')) or self.baseuri 223 if isinstance(baseuri, bytes): 224 baseuri = baseuri.decode(self.encoding, 'ignore') 225 # ensure that self.baseuri is always an absolute URI that 226 # uses a whitelisted URI scheme (e.g. not `javscript:`) 227 if self.baseuri: 228 self.baseuri = make_safe_absolute_uri(self.baseuri, baseuri) or self.baseuri 229 else: 230 self.baseuri = _urljoin(self.baseuri, baseuri) 231 lang = attrs_d.get('xml:lang', attrs_d.get('lang')) 232 if lang == '': 233 # xml:lang could be explicitly set to '', we need to capture that 234 lang = None 235 elif lang is None: 236 # if no xml:lang is specified, use parent lang 237 lang = self.lang 238 if lang: 239 if tag in ('feed', 'rss', 'rdf:RDF'): 240 self.feeddata['language'] = lang.replace('_', '-') 241 self.lang = lang 242 self.basestack.append(self.baseuri) 243 self.langstack.append(lang) 244 245 # track namespaces 246 for prefix, uri in attrs: 247 if prefix.startswith('xmlns:'): 248 self.track_namespace(prefix[6:], uri) 249 elif prefix == 'xmlns': 250 self.track_namespace(None, uri) 251 252 # track inline content 253 if self.incontent and not self.contentparams.get('type', 'xml').endswith('xml'): 254 if tag in ('xhtml:div', 'div'): 255 return # typepad does this 10/2007 256 # element declared itself as escaped markup, but it isn't really 257 self.contentparams['type'] = 'application/xhtml+xml' 258 if self.incontent and self.contentparams.get('type') == 'application/xhtml+xml': 259 if tag.find(':') != -1: 260 prefix, tag = tag.split(':', 1) 261 namespace = self.namespaces_in_use.get(prefix, '') 262 if tag == 'math' and namespace == 'http://www.w3.org/1998/Math/MathML': 263 attrs.append(('xmlns', namespace)) 264 if tag == 'svg' and namespace == 'http://www.w3.org/2000/svg': 265 attrs.append(('xmlns', namespace)) 266 if tag == 'svg': 267 self.svgOK += 1 268 return self.handle_data('<%s%s>' % (tag, self.strattrs(attrs)), escape=0) 269 270 # match namespaces 271 if tag.find(':') != -1: 272 prefix, suffix = tag.split(':', 1) 273 else: 274 prefix, suffix = '', tag 275 prefix = self.namespacemap.get(prefix, prefix) 276 if prefix: 277 prefix = prefix + '_' 278 279 # Special hack for better tracking of empty textinput/image elements in 280 # illformed feeds. 281 if (not prefix) and tag not in ('title', 'link', 'description', 'name'): 282 self.intextinput = 0 283 if (not prefix) and tag not in ('title', 'link', 'description', 'url', 'href', 'width', 'height'): 284 self.inimage = 0 285 286 # call special handler (if defined) or default handler 287 methodname = '_start_' + prefix + suffix 288 try: 289 method = getattr(self, methodname) 290 return method(attrs_d) 291 except AttributeError: 292 # Since there's no handler or something has gone wrong we 293 # explicitly add the element and its attributes. 294 unknown_tag = prefix + suffix 295 if len(attrs_d) == 0: 296 # No attributes so merge it into the enclosing dictionary 297 return self.push(unknown_tag, 1) 298 else: 299 # Has attributes so create it in its own dictionary 300 context = self._get_context() 301 context[unknown_tag] = attrs_d 302 303 def unknown_endtag(self, tag): 304 # match namespaces 305 if tag.find(':') != -1: 306 prefix, suffix = tag.split(':', 1) 307 else: 308 prefix, suffix = '', tag 309 prefix = self.namespacemap.get(prefix, prefix) 310 if prefix: 311 prefix = prefix + '_' 312 if suffix == 'svg' and self.svgOK: 313 self.svgOK -= 1 314 315 # call special handler (if defined) or default handler 316 methodname = '_end_' + prefix + suffix 317 try: 318 if self.svgOK: 319 raise AttributeError() 320 method = getattr(self, methodname) 321 method() 322 except AttributeError: 323 self.pop(prefix + suffix) 324 325 # track inline content 326 if self.incontent and not self.contentparams.get('type', 'xml').endswith('xml'): 327 # element declared itself as escaped markup, but it isn't really 328 if tag in ('xhtml:div', 'div'): 329 return # typepad does this 10/2007 330 self.contentparams['type'] = 'application/xhtml+xml' 331 if self.incontent and self.contentparams.get('type') == 'application/xhtml+xml': 332 tag = tag.split(':')[-1] 333 self.handle_data('</%s>' % tag, escape=0) 334 335 # track xml:base and xml:lang going out of scope 336 if self.basestack: 337 self.basestack.pop() 338 if self.basestack and self.basestack[-1]: 339 self.baseuri = self.basestack[-1] 340 if self.langstack: 341 self.langstack.pop() 342 if self.langstack: # and (self.langstack[-1] is not None): 343 self.lang = self.langstack[-1] 344 345 self.depth -= 1 346 347 def handle_charref(self, ref): 348 # Called for each character reference, e.g. for ' ', ref is '160' 349 if not self.elementstack: 350 return 351 ref = ref.lower() 352 if ref in ('34', '38', '39', '60', '62', 'x22', 'x26', 'x27', 'x3c', 'x3e'): 353 text = '&#%s;' % ref 354 else: 355 if ref[0] == 'x': 356 c = int(ref[1:], 16) 357 else: 358 c = int(ref) 359 text = chr(c).encode('utf-8') 360 self.elementstack[-1][2].append(text) 361 362 def handle_entityref(self, ref): 363 # Called for each entity reference, e.g. for '©', ref is 'copy' 364 if not self.elementstack: 365 return 366 if ref in ('lt', 'gt', 'quot', 'amp', 'apos'): 367 text = '&%s;' % ref 368 elif ref in self.entities: 369 text = self.entities[ref] 370 if text.startswith('&#') and text.endswith(';'): 371 return self.handle_entityref(text) 372 else: 373 try: 374 html.entities.name2codepoint[ref] 375 except KeyError: 376 text = '&%s;' % ref 377 else: 378 text = chr(html.entities.name2codepoint[ref]).encode('utf-8') 379 self.elementstack[-1][2].append(text) 380 381 def handle_data(self, text, escape=1): 382 # Called for each block of plain text, i.e. outside of any tag and 383 # not containing any character or entity references 384 if not self.elementstack: 385 return 386 if escape and self.contentparams.get('type') == 'application/xhtml+xml': 387 text = xml.sax.saxutils.escape(text) 388 self.elementstack[-1][2].append(text) 389 390 def handle_comment(self, text): 391 # Called for each comment, e.g. <!-- insert message here --> 392 pass 393 394 def handle_pi(self, text): 395 # Called for each processing instruction, e.g. <?instruction> 396 pass 397 398 def handle_decl(self, text): 399 pass 400 401 def parse_declaration(self, i): 402 # Override internal declaration handler to handle CDATA blocks. 403 if self.rawdata[i:i+9] == '<![CDATA[': 404 k = self.rawdata.find(']]>', i) 405 if k == -1: 406 # CDATA block began but didn't finish 407 k = len(self.rawdata) 408 return k 409 self.handle_data(xml.sax.saxutils.escape(self.rawdata[i+9:k]), 0) 410 return k+3 411 else: 412 k = self.rawdata.find('>', i) 413 if k >= 0: 414 return k+1 415 else: 416 # We have an incomplete CDATA block. 417 return k 418 419 @staticmethod 420 def map_content_type(content_type): 421 content_type = content_type.lower() 422 if content_type == 'text' or content_type == 'plain': 423 content_type = 'text/plain' 424 elif content_type == 'html': 425 content_type = 'text/html' 426 elif content_type == 'xhtml': 427 content_type = 'application/xhtml+xml' 428 return content_type 429 430 def track_namespace(self, prefix, uri): 431 loweruri = uri.lower() 432 if not self.version: 433 if (prefix, loweruri) == (None, 'http://my.netscape.com/rdf/simple/0.9/'): 434 self.version = 'rss090' 435 elif loweruri == 'http://purl.org/rss/1.0/': 436 self.version = 'rss10' 437 elif loweruri == 'http://www.w3.org/2005/atom': 438 self.version = 'atom10' 439 if loweruri.find('backend.userland.com/rss') != -1: 440 # match any backend.userland.com namespace 441 uri = 'http://backend.userland.com/rss' 442 loweruri = uri 443 if loweruri in self._matchnamespaces: 444 self.namespacemap[prefix] = self._matchnamespaces[loweruri] 445 self.namespaces_in_use[self._matchnamespaces[loweruri]] = uri 446 else: 447 self.namespaces_in_use[prefix or ''] = uri 448 449 def resolve_uri(self, uri): 450 return _urljoin(self.baseuri or '', uri) 451 452 @staticmethod 453 def decode_entities(element, data): 454 return data 455 456 @staticmethod 457 def strattrs(attrs): 458 return ''.join( 459 ' %s="%s"' % (t[0], xml.sax.saxutils.escape(t[1], {'"': '"'})) 460 for t in attrs 461 ) 462 463 def push(self, element, expecting_text): 464 self.elementstack.append([element, expecting_text, []]) 465 466 def pop(self, element, strip_whitespace=1): 467 if not self.elementstack: 468 return 469 if self.elementstack[-1][0] != element: 470 return 471 472 element, expecting_text, pieces = self.elementstack.pop() 473 474 # Ensure each piece is a str for Python 3 475 for (i, v) in enumerate(pieces): 476 if isinstance(v, bytes): 477 pieces[i] = v.decode('utf-8') 478 479 if self.version == 'atom10' and self.contentparams.get('type', 'text') == 'application/xhtml+xml': 480 # remove enclosing child element, but only if it is a <div> and 481 # only if all the remaining content is nested underneath it. 482 # This means that the divs would be retained in the following: 483 # <div>foo</div><div>bar</div> 484 while pieces and len(pieces) > 1 and not pieces[-1].strip(): 485 del pieces[-1] 486 while pieces and len(pieces) > 1 and not pieces[0].strip(): 487 del pieces[0] 488 if pieces and (pieces[0] == '<div>' or pieces[0].startswith('<div ')) and pieces[-1] == '</div>': 489 depth = 0 490 for piece in pieces[:-1]: 491 if piece.startswith('</'): 492 depth -= 1 493 if depth == 0: 494 break 495 elif piece.startswith('<') and not piece.endswith('/>'): 496 depth += 1 497 else: 498 pieces = pieces[1:-1] 499 500 output = ''.join(pieces) 501 if strip_whitespace: 502 output = output.strip() 503 if not expecting_text: 504 return output 505 506 # decode base64 content 507 if base64 and self.contentparams.get('base64', 0): 508 try: 509 output = base64.decodebytes(output.encode('utf8')).decode('utf8') 510 except (binascii.Error, binascii.Incomplete, UnicodeDecodeError): 511 pass 512 513 # resolve relative URIs 514 if (element in self.can_be_relative_uri) and output: 515 # do not resolve guid elements with isPermalink="false" 516 if not element == 'id' or self.guidislink: 517 output = self.resolve_uri(output) 518 519 # decode entities within embedded markup 520 if not self.contentparams.get('base64', 0): 521 output = self.decode_entities(element, output) 522 523 # some feed formats require consumers to guess 524 # whether the content is html or plain text 525 if not self.version.startswith('atom') and self.contentparams.get('type') == 'text/plain': 526 if self.looks_like_html(output): 527 self.contentparams['type'] = 'text/html' 528 529 # remove temporary cruft from contentparams 530 try: 531 del self.contentparams['mode'] 532 except KeyError: 533 pass 534 try: 535 del self.contentparams['base64'] 536 except KeyError: 537 pass 538 539 is_htmlish = self.map_content_type(self.contentparams.get('type', 'text/html')) in self.html_types 540 # resolve relative URIs within embedded markup 541 if is_htmlish and self.resolve_relative_uris: 542 if element in self.can_contain_relative_uris: 543 output = resolve_relative_uris(output, self.baseuri, self.encoding, self.contentparams.get('type', 'text/html')) 544 545 # sanitize embedded markup 546 if is_htmlish and self.sanitize_html: 547 if element in self.can_contain_dangerous_markup: 548 output = _sanitize_html(output, self.encoding, self.contentparams.get('type', 'text/html')) 549 550 if self.encoding and isinstance(output, bytes): 551 output = output.decode(self.encoding, 'ignore') 552 553 # address common error where people take data that is already 554 # utf-8, presume that it is iso-8859-1, and re-encode it. 555 if self.encoding in ('utf-8', 'utf-8_INVALID_PYTHON_3') and not isinstance(output, bytes): 556 try: 557 output = output.encode('iso-8859-1').decode('utf-8') 558 except (UnicodeEncodeError, UnicodeDecodeError): 559 pass 560 561 # map win-1252 extensions to the proper code points 562 if not isinstance(output, bytes): 563 output = output.translate(_cp1252) 564 565 # categories/tags/keywords/whatever are handled in _end_category or 566 # _end_tags or _end_itunes_keywords 567 if element in ('category', 'tags', 'itunes_keywords'): 568 return output 569 570 if element == 'title' and -1 < self.title_depth <= self.depth: 571 return output 572 573 # store output in appropriate place(s) 574 if self.inentry and not self.insource: 575 if element == 'content': 576 self.entries[-1].setdefault(element, []) 577 contentparams = copy.deepcopy(self.contentparams) 578 contentparams['value'] = output 579 self.entries[-1][element].append(contentparams) 580 elif element == 'link': 581 if not self.inimage: 582 # query variables in urls in link elements are improperly 583 # converted from `?a=1&b=2` to `?a=1&b;=2` as if they're 584 # unhandled character references. fix this special case. 585 output = output.replace('&', '&') 586 output = re.sub("&([A-Za-z0-9_]+);", r"&\g<1>", output) 587 self.entries[-1][element] = output 588 if output: 589 self.entries[-1]['links'][-1]['href'] = output 590 else: 591 if element == 'description': 592 element = 'summary' 593 old_value_depth = self.property_depth_map.setdefault(self.entries[-1], {}).get(element) 594 if old_value_depth is None or self.depth <= old_value_depth: 595 self.property_depth_map[self.entries[-1]][element] = self.depth 596 self.entries[-1][element] = output 597 if self.incontent: 598 contentparams = copy.deepcopy(self.contentparams) 599 contentparams['value'] = output 600 self.entries[-1][element + '_detail'] = contentparams 601 elif self.infeed or self.insource: # and (not self.intextinput) and (not self.inimage): 602 context = self._get_context() 603 if element == 'description': 604 element = 'subtitle' 605 context[element] = output 606 if element == 'link': 607 # fix query variables; see above for the explanation 608 output = re.sub("&([A-Za-z0-9_]+);", r"&\g<1>", output) 609 context[element] = output 610 context['links'][-1]['href'] = output 611 elif self.incontent: 612 contentparams = copy.deepcopy(self.contentparams) 613 contentparams['value'] = output 614 context[element + '_detail'] = contentparams 615 return output 616 617 def push_content(self, tag, attrs_d, default_content_type, expecting_text): 618 self.incontent += 1 619 if self.lang: 620 self.lang = self.lang.replace('_', '-') 621 self.contentparams = FeedParserDict({ 622 'type': self.map_content_type(attrs_d.get('type', default_content_type)), 623 'language': self.lang, 624 'base': self.baseuri}) 625 self.contentparams['base64'] = self._is_base64(attrs_d, self.contentparams) 626 self.push(tag, expecting_text) 627 628 def pop_content(self, tag): 629 value = self.pop(tag) 630 self.incontent -= 1 631 self.contentparams.clear() 632 return value 633 634 # a number of elements in a number of RSS variants are nominally plain 635 # text, but this is routinely ignored. This is an attempt to detect 636 # the most common cases. As false positives often result in silent 637 # data loss, this function errs on the conservative side. 638 @staticmethod 639 def looks_like_html(s): 640 """ 641 :type s: str 642 :rtype: bool 643 """ 644 645 # must have a close tag or an entity reference to qualify 646 if not (re.search(r'</(\w+)>', s) or re.search(r'&#?\w+;', s)): 647 return False 648 649 # all tags must be in a restricted subset of valid HTML tags 650 if any((t for t in re.findall(r'</?(\w+)', s) if t.lower() not in _HTMLSanitizer.acceptable_elements)): 651 return False 652 653 # all entities must have been defined as valid HTML entities 654 if any((e for e in re.findall(r'&(\w+);', s) if e not in html.entities.entitydefs)): 655 return False 656 657 return True 658 659 def _map_to_standard_prefix(self, name): 660 colonpos = name.find(':') 661 if colonpos != -1: 662 prefix = name[:colonpos] 663 suffix = name[colonpos+1:] 664 prefix = self.namespacemap.get(prefix, prefix) 665 name = prefix + ':' + suffix 666 return name 667 668 def _get_attribute(self, attrs_d, name): 669 return attrs_d.get(self._map_to_standard_prefix(name)) 670 671 def _is_base64(self, attrs_d, contentparams): 672 if attrs_d.get('mode', '') == 'base64': 673 return 1 674 if self.contentparams['type'].startswith('text/'): 675 return 0 676 if self.contentparams['type'].endswith('+xml'): 677 return 0 678 if self.contentparams['type'].endswith('/xml'): 679 return 0 680 return 1 681 682 @staticmethod 683 def _enforce_href(attrs_d): 684 href = attrs_d.get('url', attrs_d.get('uri', attrs_d.get('href', None))) 685 if href: 686 try: 687 del attrs_d['url'] 688 except KeyError: 689 pass 690 try: 691 del attrs_d['uri'] 692 except KeyError: 693 pass 694 attrs_d['href'] = href 695 return attrs_d 696 697 def _save(self, key, value, overwrite=False): 698 context = self._get_context() 699 if overwrite: 700 context[key] = value 701 else: 702 context.setdefault(key, value) 703 704 def _get_context(self): 705 if self.insource: 706 context = self.sourcedata 707 elif self.inimage and 'image' in self.feeddata: 708 context = self.feeddata['image'] 709 elif self.intextinput: 710 context = self.feeddata['textinput'] 711 elif self.inentry: 712 context = self.entries[-1] 713 else: 714 context = self.feeddata 715 return context 716 717 def _save_author(self, key, value, prefix='author'): 718 context = self._get_context() 719 context.setdefault(prefix + '_detail', FeedParserDict()) 720 context[prefix + '_detail'][key] = value 721 self._sync_author_detail() 722 context.setdefault('authors', [FeedParserDict()]) 723 context['authors'][-1][key] = value 724 725 def _save_contributor(self, key, value): 726 context = self._get_context() 727 context.setdefault('contributors', [FeedParserDict()]) 728 context['contributors'][-1][key] = value 729 730 def _sync_author_detail(self, key='author'): 731 context = self._get_context() 732 detail = context.get('%ss' % key, [FeedParserDict()])[-1] 733 if detail: 734 name = detail.get('name') 735 email = detail.get('email') 736 if name and email: 737 context[key] = '%s (%s)' % (name, email) 738 elif name: 739 context[key] = name 740 elif email: 741 context[key] = email 742 else: 743 author, email = context.get(key), None 744 if not author: 745 return 746 emailmatch = re.search(r'''(([a-zA-Z0-9\_\-\.\+]+)@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.)|(([a-zA-Z0-9\-]+\.)+))([a-zA-Z]{2,4}|[0-9]{1,3})(\]?))(\?subject=\S+)?''', author) 747 if emailmatch: 748 email = emailmatch.group(0) 749 # probably a better way to do the following, but it passes 750 # all the tests 751 author = author.replace(email, '') 752 author = author.replace('()', '') 753 author = author.replace('<>', '') 754 author = author.replace('<>', '') 755 author = author.strip() 756 if author and (author[0] == '('): 757 author = author[1:] 758 if author and (author[-1] == ')'): 759 author = author[:-1] 760 author = author.strip() 761 if author or email: 762 context.setdefault('%s_detail' % key, detail) 763 if author: 764 detail['name'] = author 765 if email: 766 detail['email'] = email 767 768 def _add_tag(self, term, scheme, label): 769 context = self._get_context() 770 tags = context.setdefault('tags', []) 771 if (not term) and (not scheme) and (not label): 772 return 773 value = FeedParserDict(term=term, scheme=scheme, label=label) 774 if value not in tags: 775 tags.append(value) 776 777 def _start_tags(self, attrs_d): 778 # This is a completely-made up element. Its semantics are determined 779 # only by a single feed that precipitated bug report 392 on Google Code. 780 # In short, this is junk code. 781 self.push('tags', 1) 782 783 def _end_tags(self): 784 for term in self.pop('tags').split(','): 785 self._add_tag(term.strip(), None, None) 786