mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2024-10-04 08:32:56 +00:00
341 lines
16 KiB
Python
341 lines
16 KiB
Python
import textwrap
|
|
import unittest
|
|
|
|
from yt_dlp.compat import compat_HTMLParseError
|
|
from yt_dlp.parsing import (
|
|
MatchingElementParser,
|
|
HTMLCommentRanges,
|
|
HTMLTagParser,
|
|
)
|
|
|
|
extract_attributes = MatchingElementParser.extract_attributes
|
|
get_element_by_attribute = MatchingElementParser.get_element_by_attribute
|
|
get_element_by_class = MatchingElementParser.get_element_by_class
|
|
get_element_html_by_attribute = MatchingElementParser.get_element_html_by_attribute
|
|
get_element_html_by_class = MatchingElementParser.get_element_html_by_class
|
|
get_element_text_and_html_by_tag = MatchingElementParser.get_element_text_and_html_by_tag
|
|
get_elements_by_attribute = MatchingElementParser.get_elements_by_attribute
|
|
get_elements_by_class = MatchingElementParser.get_elements_by_class
|
|
get_elements_html_by_attribute = MatchingElementParser.get_elements_html_by_attribute
|
|
get_elements_html_by_class = MatchingElementParser.get_elements_html_by_class
|
|
get_elements_text_and_html_by_attribute = MatchingElementParser.get_elements_text_and_html_by_attribute
|
|
get_elements_text_and_html_by_tag = MatchingElementParser.get_elements_text_and_html_by_tag
|
|
|
|
|
|
class TestParsing(unittest.TestCase):
|
|
def test_extract_attributes(self):
|
|
self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})
|
|
self.assertEqual(extract_attributes("<e x='y'>"), {'x': 'y'})
|
|
self.assertEqual(extract_attributes('<e x=y>'), {'x': 'y'})
|
|
self.assertEqual(extract_attributes('<e x="a \'b\' c">'), {'x': "a 'b' c"})
|
|
self.assertEqual(extract_attributes('<e x=\'a "b" c\'>'), {'x': 'a "b" c'})
|
|
self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})
|
|
self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})
|
|
self.assertEqual(extract_attributes('<e x="&">'), {'x': '&'}) # XML
|
|
self.assertEqual(extract_attributes('<e x=""">'), {'x': '"'})
|
|
self.assertEqual(extract_attributes('<e x="£">'), {'x': '£'}) # HTML 3.2
|
|
self.assertEqual(extract_attributes('<e x="λ">'), {'x': 'λ'}) # HTML 4.0
|
|
self.assertEqual(extract_attributes('<e x="&foo">'), {'x': '&foo'})
|
|
self.assertEqual(extract_attributes('<e x="\'">'), {'x': "'"})
|
|
self.assertEqual(extract_attributes('<e x=\'"\'>'), {'x': '"'})
|
|
self.assertEqual(extract_attributes('<e x >'), {'x': None})
|
|
self.assertEqual(extract_attributes('<e x=y a>'), {'x': 'y', 'a': None})
|
|
self.assertEqual(extract_attributes('<e x= y>'), {'x': 'y'})
|
|
self.assertEqual(extract_attributes('<e x=1 y=2 x=3>'), {'y': '2', 'x': '3'})
|
|
self.assertEqual(extract_attributes('<e \nx=\ny\n>'), {'x': 'y'})
|
|
self.assertEqual(extract_attributes('<e \nx=\n"y"\n>'), {'x': 'y'})
|
|
self.assertEqual(extract_attributes("<e \nx=\n'y'\n>"), {'x': 'y'})
|
|
self.assertEqual(extract_attributes('<e \nx="\ny\n">'), {'x': '\ny\n'})
|
|
self.assertEqual(extract_attributes('<e CAPS=x>'), {'caps': 'x'}) # Names lowercased
|
|
self.assertEqual(extract_attributes('<e x=1 X=2>'), {'x': '2'})
|
|
self.assertEqual(extract_attributes('<e X=1 x=2>'), {'x': '2'})
|
|
self.assertEqual(extract_attributes('<e _:funny-name1=1>'), {'_:funny-name1': '1'})
|
|
self.assertEqual(extract_attributes('<e x="Fáilte 世界 \U0001f600">'), {'x': 'Fáilte 世界 \U0001f600'})
|
|
self.assertEqual(extract_attributes('<e x="décomposé">'), {'x': 'décompose\u0301'})
|
|
# "Narrow" Python builds don't support unicode code points outside BMP.
|
|
try:
|
|
chr(0x10000)
|
|
supports_outside_bmp = True
|
|
except ValueError:
|
|
supports_outside_bmp = False
|
|
if supports_outside_bmp:
|
|
self.assertEqual(extract_attributes('<e x="Smile 😀!">'), {'x': 'Smile \U0001f600!'})
|
|
# Malformed HTML should not break attributes extraction on older Python
|
|
self.assertEqual(extract_attributes('<mal"formed/>'), {})
|
|
|
|
GET_ELEMENT_BY_CLASS_TEST_STRING = '''
|
|
<span class="foo bar">nice</span>
|
|
<div class="foo bar">also nice</div>
|
|
'''
|
|
|
|
def test_get_element_by_class(self):
|
|
html = self.GET_ELEMENT_BY_CLASS_TEST_STRING
|
|
|
|
self.assertEqual(get_element_by_class('foo', html), 'nice')
|
|
self.assertEqual(get_element_by_class('no-such-class', html), None)
|
|
|
|
def test_get_element_html_by_class(self):
|
|
html = self.GET_ELEMENT_BY_CLASS_TEST_STRING
|
|
|
|
self.assertEqual(get_element_html_by_class('foo', html),
|
|
'<span class="foo bar">nice</span>')
|
|
self.assertEqual(get_element_by_class('no-such-class', html), None)
|
|
|
|
GET_ELEMENT_BY_ATTRIBUTE_TEST_STRING = '''
|
|
<div itemprop="author" itemscope>foo</div>
|
|
'''
|
|
|
|
def test_get_element_by_attribute(self):
|
|
html = self.GET_ELEMENT_BY_CLASS_TEST_STRING
|
|
|
|
self.assertEqual(get_element_by_attribute('class', 'foo bar', html), 'nice')
|
|
self.assertEqual(get_element_by_attribute('class', 'foo', html), None)
|
|
self.assertEqual(get_element_by_attribute('class', 'no-such-foo', html), None)
|
|
self.assertEqual(get_element_by_attribute('class', 'foo bar', html, tag='div'), 'also nice')
|
|
|
|
html = self.GET_ELEMENT_BY_ATTRIBUTE_TEST_STRING
|
|
|
|
self.assertEqual(get_element_by_attribute('itemprop', 'author', html), 'foo')
|
|
|
|
def test_get_element_html_by_attribute(self):
|
|
html = self.GET_ELEMENT_BY_CLASS_TEST_STRING
|
|
|
|
self.assertEqual(get_element_html_by_attribute('class', 'foo bar', html),
|
|
'<span class="foo bar">nice</span>')
|
|
self.assertEqual(get_element_html_by_attribute('class', 'foo', html), None)
|
|
self.assertEqual(get_element_html_by_attribute('class', 'no-such-foo', html), None)
|
|
|
|
html = self.GET_ELEMENT_BY_ATTRIBUTE_TEST_STRING
|
|
|
|
self.assertEqual(get_element_html_by_attribute('itemprop', 'author', html), html.strip())
|
|
|
|
GET_ELEMENTS_BY_CLASS_TEST_STRING = '''
|
|
<span class="foo bar">nice</span>
|
|
<span class="foo bar">also nice</span>
|
|
'''
|
|
GET_ELEMENTS_BY_CLASS_RES = [
|
|
'<span class="foo bar">nice</span>',
|
|
'<span class="foo bar">also nice</span>'
|
|
]
|
|
|
|
def test_get_elements_by_class(self):
|
|
html = self.GET_ELEMENTS_BY_CLASS_TEST_STRING
|
|
|
|
self.assertEqual(get_elements_by_class('foo', html), ['nice', 'also nice'])
|
|
self.assertEqual(get_elements_by_class('no-such-class', html), [])
|
|
|
|
def test_get_elements_html_by_class(self):
|
|
html = self.GET_ELEMENTS_BY_CLASS_TEST_STRING
|
|
|
|
self.assertEqual(get_elements_html_by_class('foo', html), self.GET_ELEMENTS_BY_CLASS_RES)
|
|
self.assertEqual(get_elements_html_by_class('no-such-class', html), [])
|
|
|
|
def test_get_elements_by_attribute(self):
|
|
html = self.GET_ELEMENTS_BY_CLASS_TEST_STRING
|
|
|
|
self.assertEqual(get_elements_by_attribute('class', 'foo bar', html), ['nice', 'also nice'])
|
|
self.assertEqual(get_elements_by_attribute('class', 'foo', html), [])
|
|
self.assertEqual(get_elements_by_attribute('class', 'no-such-foo', html), [])
|
|
|
|
def test_get_elements_html_by_attribute(self):
|
|
html = self.GET_ELEMENTS_BY_CLASS_TEST_STRING
|
|
|
|
self.assertEqual(get_elements_html_by_attribute('class', 'foo bar', html),
|
|
self.GET_ELEMENTS_BY_CLASS_RES)
|
|
self.assertEqual(get_elements_html_by_attribute('class', 'foo', html), [])
|
|
self.assertEqual(get_elements_html_by_attribute('class', 'no-such-foo', html), [])
|
|
|
|
def test_get_elements_text_and_html_by_attribute(self):
|
|
html = self.GET_ELEMENTS_BY_CLASS_TEST_STRING
|
|
|
|
self.assertEqual(
|
|
get_elements_text_and_html_by_attribute('class', 'foo bar', html),
|
|
list(zip(['nice', 'also nice'], self.GET_ELEMENTS_BY_CLASS_RES)))
|
|
self.assertEqual(get_elements_text_and_html_by_attribute('class', 'foo', html), [])
|
|
self.assertEqual(get_elements_text_and_html_by_attribute('class', 'no-such-foo', html), [])
|
|
|
|
self.assertEqual(get_elements_text_and_html_by_attribute(
|
|
'class', 'foo', '<a class="foo">nice</a><span class="foo">not nice</span>', tag='a'),
|
|
[('nice', '<a class="foo">nice</a>')])
|
|
|
|
def test_get_element_text_and_html_by_tag(self):
|
|
get_element_by_tag_test_string = '''
|
|
random text lorem ipsum</p>
|
|
<div>
|
|
this should be returned
|
|
<span>this should also be returned</span>
|
|
<div>
|
|
this should also be returned
|
|
</div>
|
|
closing tag above should not trick, so this should also be returned
|
|
</div>
|
|
but this text should not be returned
|
|
'''
|
|
html = textwrap.indent(textwrap.dedent(get_element_by_tag_test_string), ' ' * 4)
|
|
get_element_by_tag_res_outerdiv_html = html.strip()[32:276]
|
|
get_element_by_tag_res_outerdiv_text = get_element_by_tag_res_outerdiv_html[5:-6]
|
|
get_element_by_tag_res_innerspan_html = html.strip()[78:119]
|
|
get_element_by_tag_res_innerspan_text = get_element_by_tag_res_innerspan_html[6:-7]
|
|
|
|
self.assertEqual(
|
|
get_element_text_and_html_by_tag('div', html),
|
|
(get_element_by_tag_res_outerdiv_text, get_element_by_tag_res_outerdiv_html))
|
|
self.assertEqual(
|
|
get_element_text_and_html_by_tag('span', html),
|
|
(get_element_by_tag_res_innerspan_text, get_element_by_tag_res_innerspan_html))
|
|
self.assertIsNone(get_element_text_and_html_by_tag('article', html))
|
|
|
|
def test_get_elements_text_and_html_by_tag(self):
|
|
class StrictParser(MatchingElementParser):
|
|
STRICT = True
|
|
|
|
test_string = '''
|
|
<img src="a.png">
|
|
<img src="b.png" />
|
|
<span>ignore</span>
|
|
'''
|
|
items = get_elements_text_and_html_by_tag('img', test_string)
|
|
self.assertListEqual(items, [('', '<img src="a.png">'), ('', '<img src="b.png" />')])
|
|
|
|
self.assertEqual(
|
|
StrictParser.get_element_text_and_html_by_tag('use', '<use><img></use>'),
|
|
('<img>', '<use><img></use>'))
|
|
|
|
def test_get_element_text_and_html_by_tag_malformed(self):
|
|
inner_text = 'inner text'
|
|
malnested_elements = f'<malnested_a><malnested_b>{inner_text}</malnested_a></malnested_b>'
|
|
commented_html = '<!--<div>inner comment</div>-->'
|
|
outerdiv_html = f'<div>{malnested_elements}</div>'
|
|
html = f'{commented_html}{outerdiv_html}'
|
|
|
|
self.assertEqual(
|
|
get_element_text_and_html_by_tag('div', html), (malnested_elements, outerdiv_html))
|
|
self.assertEqual(
|
|
get_element_text_and_html_by_tag('malnested_a', html),
|
|
(f'<malnested_b>{inner_text}',
|
|
f'<malnested_a><malnested_b>{inner_text}</malnested_a>'))
|
|
self.assertEqual(
|
|
get_element_text_and_html_by_tag('malnested_b', html),
|
|
(f'{inner_text}</malnested_a>',
|
|
f'<malnested_b>{inner_text}</malnested_a></malnested_b>'))
|
|
self.assertEqual(
|
|
get_element_text_and_html_by_tag('orphan', f'<orphan>{html}'), ('', '<orphan>'))
|
|
self.assertIsNone(get_element_text_and_html_by_tag('orphan', f'{html}</orphan>'))
|
|
|
|
def test_strict_html_parsing(self):
|
|
class StrictTagParser(HTMLTagParser):
|
|
STRICT = True
|
|
|
|
parser = StrictTagParser()
|
|
with self.assertRaisesRegex(compat_HTMLParseError, "stray closing tag 'p'"):
|
|
parser.taglist('</p>', reset=True)
|
|
with self.assertRaisesRegex(compat_HTMLParseError, "unclosed tag 'p', 'div'"):
|
|
parser.taglist('<div><p>', reset=True)
|
|
with self.assertRaisesRegex(compat_HTMLParseError, "malnested closing tag 'div', expected after '</p>'"):
|
|
parser.taglist('<div><p></div></p>', reset=True)
|
|
with self.assertRaisesRegex(compat_HTMLParseError, "malnested closing tag 'div', expected after '</p>'"):
|
|
parser.taglist('<div><p>/p></div>', reset=True)
|
|
with self.assertRaisesRegex(compat_HTMLParseError, "malformed closing tag 'p<<'"):
|
|
parser.taglist('<div><p></p<< </div>', reset=True)
|
|
with self.assertRaisesRegex(compat_HTMLParseError, "stray closing tag 'img'"):
|
|
parser.taglist('<img>must be empty</img>', reset=True)
|
|
|
|
def test_relaxed_html_parsing(self):
|
|
Tag = HTMLTagParser.Tag
|
|
parser = HTMLTagParser()
|
|
|
|
self.assertEqual(parser.taglist('</p>', reset=True), [])
|
|
self.assertEqual(parser.taglist('<div><p>', reset=True), [Tag('div'), Tag('p')])
|
|
|
|
tags = parser.taglist('<div><p></div></p>', reset=True)
|
|
self.assertEqual(tags, [Tag('div'), Tag('p')])
|
|
|
|
tags = parser.taglist('<div><p>/p></div>', reset=True)
|
|
self.assertEqual(tags, [Tag('div'), Tag('p')])
|
|
|
|
tags = parser.taglist('<div><p>paragraph</p<ignored></div>', reset=True)
|
|
self.assertEqual(tags, [Tag('div'), Tag('p')])
|
|
self.assertEqual(tags[1].text_and_html(), ('paragraph', '<p>paragraph</p<ignored>'))
|
|
|
|
tags = parser.taglist('<img width="300px">must be empty</img>', reset=True)
|
|
self.assertEqual(tags, [Tag('img')])
|
|
self.assertEqual(tags[0].text_and_html(), ('', '<img width="300px">'))
|
|
|
|
def test_compliant_html_parsing(self):
|
|
# certain elements don't need to be closed (see HTMLTagParser.VOID_TAGS)
|
|
Tag = HTMLTagParser.Tag
|
|
html = '''
|
|
no error without closing tag: <img>
|
|
self closing is ok: <img />
|
|
'''
|
|
parser = HTMLTagParser()
|
|
tags = parser.taglist(html, reset=True)
|
|
self.assertEqual(tags, [Tag('img'), Tag('img')])
|
|
|
|
# don't get fooled by '>' in attributes
|
|
html = '''<img greater_a='1>0' greater_b="1>0">'''
|
|
tags = parser.taglist(html, reset=True)
|
|
self.assertEqual(tags[0].text_and_html(), ('', html))
|
|
|
|
def test_tag_return_order(self):
|
|
Tag = HTMLTagParser.Tag
|
|
html = '''
|
|
<t0>
|
|
<t1>
|
|
<t2>
|
|
<t3 /> <t4 />
|
|
</t2>
|
|
</t1>
|
|
<t5>
|
|
<t6 />
|
|
</t5>
|
|
</t0>
|
|
<t7>
|
|
<t8 />
|
|
</t7>
|
|
'''
|
|
parser = HTMLTagParser()
|
|
tags = parser.taglist(html, reset=True)
|
|
self.assertEqual(
|
|
str(tags), str([Tag('t0'), Tag('t1'), Tag('t2'), Tag('t3'), Tag('t4'),
|
|
Tag('t5'), Tag('t6'), Tag('t7'), Tag('t8')]))
|
|
|
|
tags = parser.taglist(html, reset=True, depth_first=True)
|
|
self.assertEqual(
|
|
str(tags), str([Tag('t3'), Tag('t4'), Tag('t2'), Tag('t1'), Tag('t6'),
|
|
Tag('t5'), Tag('t0'), Tag('t8'), Tag('t7')]))
|
|
|
|
# return tags in nested order
|
|
tags = parser.taglist(html, reset=True, depth_first=None)
|
|
self.assertEqual(
|
|
str(tags), str([
|
|
[Tag('t0'),
|
|
[Tag('t1'),
|
|
[Tag('t2'), Tag('t3'), Tag('t4')]],
|
|
[Tag('t5'), Tag('t6')]],
|
|
[Tag('t7'), Tag('t8')]]))
|
|
|
|
def test_within_html_comment(self):
|
|
def mark_comments(_string, char='^', nochar='-'):
|
|
cmts = HTMLCommentRanges(_string)
|
|
return "".join(char if _idx in cmts else nochar for _idx in range(len(_string)))
|
|
|
|
html_string = '''
|
|
no comments in this line
|
|
---------------------------------------------------------------------
|
|
<!-- whole line represents a comment -->
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
before <!-- comment --> after
|
|
-------^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^--------
|
|
here is <!-- a comment --> and <!-- another comment --> end
|
|
------------^^^^^^^^^^^^^^^^^^---------^^^^^^^^^^^^^^^^^^^^^^^^------
|
|
this <!-- nested <!-- comment --> ends here --> and not here
|
|
-----^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^----------------------------
|
|
stray --> comment closings --> are ignored <!-- but not <!-- openings
|
|
-------------------------------------------^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
'''
|
|
|
|
lines = textwrap.dedent(html_string).strip().splitlines()
|
|
for line, marker in zip(lines[0::2], lines[1::2]):
|
|
self.assertEqual((line, mark_comments(line)), (line, marker))
|