diff options
| -rw-r--r-- | Lib/html/parser.py | 10 | ||||
| -rw-r--r-- | Lib/test/test_htmlparser.py | 15 | ||||
| -rw-r--r-- | Misc/NEWS | 6 |
3 files changed, 26 insertions, 5 deletions
diff --git a/Lib/html/parser.py b/Lib/html/parser.py index 390d4ccc488..43e6411b735 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -139,7 +139,15 @@ class HTMLParser(_markupbase.ParserBase): if self.convert_charrefs and not self.cdata_elem: j = rawdata.find('<', i) if j < 0: - if not end: + # if we can't find the next <, either we are at the end + # or there's more text incoming. If the latter is True, + # we can't pass the text to handle_data in case we have + # a charref cut in half at end. Try to determine if + # this is the case before proceding by looking for an + # & near the end and see if it's followed by a space or ;. + amppos = rawdata.rfind('&', max(i, n-34)) + if (amppos >= 0 and + not re.compile(r'[\s;]').search(rawdata, amppos)): break # wait till we get all the text j = n else: diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index de8f3e80d4b..11420b2c846 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -72,9 +72,6 @@ class EventCollectorExtra(EventCollector): class EventCollectorCharrefs(EventCollector): - def get_events(self): - return self.events - def handle_charref(self, data): self.fail('This should never be called with convert_charrefs=True') @@ -633,6 +630,18 @@ text ] self._run_check(html, expected) + def test_convert_charrefs_dropped_text(self): + # #23144: make sure that all the events are triggered when + # convert_charrefs is True, even if we don't call .close() + parser = EventCollector(convert_charrefs=True) + # before the fix, bar & baz was missing + parser.feed("foo <a>link</a> bar & baz") + self.assertEqual( + parser.get_events(), + [('data', 'foo '), ('starttag', 'a', []), ('data', 'link'), + ('endtag', 'a'), ('data', ' bar & baz')] + ) + class AttributesTestCase(TestCaseBase): diff --git a/Misc/NEWS b/Misc/NEWS index 5f23605b23b..64c9af75bcb 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1,4 +1,4 @@ -+++++++++++ ++++++++++++ Python News +++++++++++ @@ -95,9 +95,13 @@ Core and Builtins Library ------- +- Issue #23144: Make sure that HTMLParser.feed() returns all the data, even + when convert_charrefs is True. + - Issue #24635: Fixed a bug in typing.py where isinstance([], typing.Iterable) would return True once, then False on subsequent calls. + - Issue #24989: Fixed buffer overread in BytesIO.readline() if a position is set beyond size. Based on patch by John Leitch. |