Skip to content

gh-77057: Fix handling of invalid markup declarations in HTMLParser #9295

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
May 10, 2025
Merged

gh-77057: Fix handling of invalid markup declarations in HTMLParser #9295

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5c0a5f3
bpo-32876: fix handling of invalid markup declarations.
ezio-melotti Sep 14, 2018
74cf400
bpo-32876: fix CDATA handling and add a couple of tests.
ezio-melotti Sep 14, 2018
d6561a0
Merge branch 'main' into issue32876-HTMLParser
ezio-melotti Apr 13, 2022
4fb2241
Merge branch 'main' into issue32876-HTMLParser
serhiy-storchaka May 7, 2025
7450a97
Apply suggestions from code review
serhiy-storchaka May 9, 2025
9c5e14a
Add a NEWS entry.
serhiy-storchaka May 9, 2025
57cdab2
Remove redundant empty lines.
serhiy-storchaka May 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Lib/html/parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ def parse_html_declaration(self, i):
if rawdata[i:i+4] == '<!--':
# this case is actually already handled in goahead()
return self.parse_comment(i)
elif rawdata[i:i+3] == '<![':
elif rawdata[i:i+9] == '<![CDATA[':
return self.parse_marked_section(i)
elif rawdata[i:i+9].lower() == '<!doctype':
# find the closing >
Expand All @@ -277,7 +277,7 @@ def parse_html_declaration(self, i):
def parse_bogus_comment(self, i, report=1):
rawdata = self.rawdata
assert rawdata[i:i+2] in ('<!', '</'), ('unexpected call to '
'parse_comment()')
'parse_bogus_comment()')
pos = rawdata.find('>', i+2)
if pos == -1:
return -1
Expand Down
81 changes: 64 additions & 17 deletions Lib/test/test_htmlparser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -539,52 +539,99 @@ def test_EOF_in_charref(self):
for html, expected in data:
self._run_check(html, expected)

def test_broken_comments(self):
def test_EOF_in_comments_or_decls(self):
data = [
('<!', [('data', '<!')]),
('<!-', [('data', '<!-')]),
('<!--', [('data', '<!--')]),
('<![', [('data', '<![')]),
('<![CDATA[', [('data', '<![CDATA[')]),
('<![CDATA[x', [('data', '<![CDATA[x')]),
('<!DOCTYPE', [('data', '<!DOCTYPE')]),
('<!DOCTYPE HTML', [('data', '<!DOCTYPE HTML')]),
]
for html, expected in data:
self._run_check(html, expected)
def test_bogus_comments(self):
html = ('<! not really a comment >'
'<! not a comment either -->'
'<! -- close enough -->'
'<!><!<-- this was an empty comment>'
'<!!! another bogus comment !!!>')
'<!!! another bogus comment !!!>'
# see #32876
'<![with square brackets]!>'
'<![\nmultiline\nbogusness\n]!>'
'<![more brackets]-[and a hyphen]!>'
'<![cdata[should be uppercase]]>'
'<![CDATA [whitespaces are not ignored]]>'
'<![CDATA]]>' # required '[' after CDATA
)
expected = [
('comment', ' not really a comment '),
('comment', ' not a comment either --'),
('comment', ' -- close enough --'),
('comment', ''),
('comment', '<-- this was an empty comment'),
('comment', '!! another bogus comment !!!'),
('comment', '[with square brackets]!'),
('comment', '[\nmultiline\nbogusness\n]!'),
('comment', '[more brackets]-[and a hyphen]!'),
('comment', '[cdata[should be uppercase]]'),
('comment', '[CDATA [whitespaces are not ignored]]'),
('comment', '[CDATA]]'),
]
self._run_check(html, expected)

def test_broken_condcoms(self):
# these condcoms are missing the '--' after '<!' and before the '>'
# and they are considered bogus comments according to
# "8.2.4.42. Markup declaration open state"
html = ('<![if !(IE)]>broken condcom<![endif]>'
'<![if ! IE]><link href="favicon.tiff"/><![endif]>'
'<![if !IE 6]><img src="firefox.png" /><![endif]>'
'<![if !ie 6]><b>foo</b><![endif]>'
'<![if (!IE)|(lt IE 9)]><img src="mammoth.bmp" /><![endif]>')
# According to the HTML5 specs sections "8.2.4.44 Bogus comment state"
# and "8.2.4.45 Markup declaration open state", comment tokens should
# be emitted instead of 'unknown decl', but calling unknown_decl
# provides more flexibility.
# See also Lib/_markupbase.py:parse_declaration
expected = [
('unknown decl', 'if !(IE)'),
('comment', '[if !(IE)]'),
('data', 'broken condcom'),
('unknown decl', 'endif'),
('unknown decl', 'if ! IE'),
('comment', '[endif]'),
('comment', '[if ! IE]'),
('startendtag', 'link', [('href', 'favicon.tiff')]),
('unknown decl', 'endif'),
('unknown decl', 'if !IE 6'),
('comment', '[endif]'),
('comment', '[if !IE 6]'),
('startendtag', 'img', [('src', 'firefox.png')]),
('unknown decl', 'endif'),
('unknown decl', 'if !ie 6'),
('comment', '[endif]'),
('comment', '[if !ie 6]'),
('starttag', 'b', []),
('data', 'foo'),
('endtag', 'b'),
('unknown decl', 'endif'),
('unknown decl', 'if (!IE)|(lt IE 9)'),
('comment', '[endif]'),
('comment', '[if (!IE)|(lt IE 9)]'),
('startendtag', 'img', [('src', 'mammoth.bmp')]),
('unknown decl', 'endif')
('comment', '[endif]')
]
self._run_check(html, expected)

def test_cdata_declarations(self):
# More tests should be added. See also "8.2.4.42. Markup
# declaration open state", "8.2.4.69. CDATA section state",
# and issue 32876
html = ('<![CDATA[just some plain text]]>')
expected = [('unknown decl', 'CDATA[just some plain text')]
self._run_check(html, expected)

def test_cdata_declarations_multiline(self):
html = ('<code><![CDATA['
' if (a < b && a > b) {'
' printf("[<marquee>How?</marquee>]");'
' }'
']]></code>')
expected = [
('starttag', 'code', []),
('unknown decl',
'CDATA[ if (a < b && a > b) { '
'printf("[<marquee>How?</marquee>]"); }'),
('endtag', 'code')
]
self._run_check(html, expected)

Expand Down
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Library/2025-05-09-15-50-00.gh-issue-77057.fV8SU-.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix handling of invalid markup declarations in
:class:`html.parser.HTMLParser`.
Loading