[youtube] Fix throttling by decrypting n-sig (#1437)

This commit is contained in:
pukkandan 2021-10-31 09:53:58 +05:30 committed by GitHub
parent cd9ea4104b
commit 404f611f1c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 547 additions and 147 deletions

1
.gitignore vendored
View File

@ -41,6 +41,7 @@ cookies
*.webp *.webp
*.annotations.xml *.annotations.xml
*.description *.description
.cache/
# Allow config/media files in testdata # Allow config/media files in testdata
!test/** !test/**

View File

@ -112,6 +112,56 @@ class TestJSInterpreter(unittest.TestCase):
''') ''')
self.assertEqual(jsi.call_function('z'), 5) self.assertEqual(jsi.call_function('z'), 5)
def test_for_loop(self):
jsi = JSInterpreter('''
function x() { a=0; for (i=0; i-10; i++) {a++} a }
''')
self.assertEqual(jsi.call_function('x'), 10)
def test_switch(self):
jsi = JSInterpreter('''
function x(f) { switch(f){
case 1:f+=1;
case 2:f+=2;
case 3:f+=3;break;
case 4:f+=4;
default:f=0;
} return f }
''')
self.assertEqual(jsi.call_function('x', 1), 7)
self.assertEqual(jsi.call_function('x', 3), 6)
self.assertEqual(jsi.call_function('x', 5), 0)
def test_try(self):
jsi = JSInterpreter('''
function x() { try{return 10} catch(e){return 5} }
''')
self.assertEqual(jsi.call_function('x'), 10)
def test_for_loop_continue(self):
jsi = JSInterpreter('''
function x() { a=0; for (i=0; i-10; i++) { continue; a++ } a }
''')
self.assertEqual(jsi.call_function('x'), 0)
def test_for_loop_break(self):
jsi = JSInterpreter('''
function x() { a=0; for (i=0; i-10; i++) { break; a++ } a }
''')
self.assertEqual(jsi.call_function('x'), 0)
def test_literal_list(self):
jsi = JSInterpreter('''
function x() { [1, 2, "asdf", [5, 6, 7]][3] }
''')
self.assertEqual(jsi.call_function('x'), [5, 6, 7])
def test_comma(self):
jsi = JSInterpreter('''
function x() { a=5; a -= 1, a+=3; return a }
''')
self.assertEqual(jsi.call_function('x'), 7)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

View File

@ -14,9 +14,10 @@ import string
from test.helper import FakeYDL, is_download_test from test.helper import FakeYDL, is_download_test
from yt_dlp.extractor import YoutubeIE from yt_dlp.extractor import YoutubeIE
from yt_dlp.jsinterp import JSInterpreter
from yt_dlp.compat import compat_str, compat_urlretrieve from yt_dlp.compat import compat_str, compat_urlretrieve
_TESTS = [ _SIG_TESTS = [
( (
'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js', 'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
86, 86,
@ -64,6 +65,13 @@ _TESTS = [
) )
] ]
_NSIG_TESTS = [
(
'https://www.youtube.com/s/player/9216d1f7/player_ias.vflset/en_US/base.js',
'SLp9F5bwjAdhE9F-', 'gWnb9IK2DJ8Q1w',
), # TODO: Add more tests
]
@is_download_test @is_download_test
class TestPlayerInfo(unittest.TestCase): class TestPlayerInfo(unittest.TestCase):
@ -97,35 +105,49 @@ class TestSignature(unittest.TestCase):
os.mkdir(self.TESTDATA_DIR) os.mkdir(self.TESTDATA_DIR)
def make_tfunc(url, sig_input, expected_sig): def t_factory(name, sig_func, url_pattern):
m = re.match(r'.*-([a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$', url) def make_tfunc(url, sig_input, expected_sig):
assert m, '%r should follow URL format' % url m = url_pattern.match(url)
test_id = m.group(1) assert m, '%r should follow URL format' % url
test_id = m.group('id')
def test_func(self): def test_func(self):
basename = 'player-%s.js' % test_id basename = f'player-{name}-{test_id}.js'
fn = os.path.join(self.TESTDATA_DIR, basename) fn = os.path.join(self.TESTDATA_DIR, basename)
if not os.path.exists(fn): if not os.path.exists(fn):
compat_urlretrieve(url, fn) compat_urlretrieve(url, fn)
with io.open(fn, encoding='utf-8') as testf:
jscode = testf.read()
self.assertEqual(sig_func(jscode, sig_input), expected_sig)
ydl = FakeYDL() test_func.__name__ = f'test_{name}_js_{test_id}'
ie = YoutubeIE(ydl) setattr(TestSignature, test_func.__name__, test_func)
with io.open(fn, encoding='utf-8') as testf: return make_tfunc
jscode = testf.read()
func = ie._parse_sig_js(jscode)
src_sig = (
compat_str(string.printable[:sig_input])
if isinstance(sig_input, int) else sig_input)
got_sig = func(src_sig)
self.assertEqual(got_sig, expected_sig)
test_func.__name__ = str('test_signature_js_' + test_id)
setattr(TestSignature, test_func.__name__, test_func)
for test_spec in _TESTS: def signature(jscode, sig_input):
make_tfunc(*test_spec) func = YoutubeIE(FakeYDL())._parse_sig_js(jscode)
src_sig = (
compat_str(string.printable[:sig_input])
if isinstance(sig_input, int) else sig_input)
return func(src_sig)
def n_sig(jscode, sig_input):
funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode)
return JSInterpreter(jscode).call_function(funcname, sig_input)
make_sig_test = t_factory(
'signature', signature, re.compile(r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$'))
for test_spec in _SIG_TESTS:
make_sig_test(*test_spec)
make_nsig_test = t_factory(
'nsig', n_sig, re.compile(r'.+/player/(?P<id>[a-zA-Z0-9_-]+)/.+.js$'))
for test_spec in _NSIG_TESTS:
make_nsig_test(*test_spec)
if __name__ == '__main__': if __name__ == '__main__':

View File

@ -1720,7 +1720,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
raise ExtractorError('Cannot identify player %r' % player_url) raise ExtractorError('Cannot identify player %r' % player_url)
return id_m.group('id') return id_m.group('id')
def _load_player(self, video_id, player_url, fatal=True) -> bool: def _load_player(self, video_id, player_url, fatal=True):
player_id = self._extract_player_info(player_url) player_id = self._extract_player_info(player_url)
if player_id not in self._code_cache: if player_id not in self._code_cache:
code = self._download_webpage( code = self._download_webpage(
@ -1729,7 +1729,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
errnote='Download of %s failed' % player_url) errnote='Download of %s failed' % player_url)
if code: if code:
self._code_cache[player_id] = code self._code_cache[player_id] = code
return player_id in self._code_cache return self._code_cache.get(player_id)
def _extract_signature_function(self, video_id, player_url, example_sig): def _extract_signature_function(self, video_id, player_url, example_sig):
player_id = self._extract_player_info(player_url) player_id = self._extract_player_info(player_url)
@ -1743,8 +1743,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
if cache_spec is not None: if cache_spec is not None:
return lambda s: ''.join(s[i] for i in cache_spec) return lambda s: ''.join(s[i] for i in cache_spec)
if self._load_player(video_id, player_url): code = self._load_player(video_id, player_url)
code = self._code_cache[player_id] if code:
res = self._parse_sig_js(code) res = self._parse_sig_js(code)
test_string = ''.join(map(compat_chr, range(len(example_sig)))) test_string = ''.join(map(compat_chr, range(len(example_sig))))
@ -1755,6 +1755,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
return res return res
def _print_sig_code(self, func, example_sig): def _print_sig_code(self, func, example_sig):
if not self.get_param('youtube_print_sig_code'):
return
def gen_sig_code(idxs): def gen_sig_code(idxs):
def _genslice(start, end, step): def _genslice(start, end, step):
starts = '' if start == 0 else str(start) starts = '' if start == 0 else str(start)
@ -1831,13 +1834,58 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
) )
self._player_cache[player_id] = func self._player_cache[player_id] = func
func = self._player_cache[player_id] func = self._player_cache[player_id]
if self.get_param('youtube_print_sig_code'): self._print_sig_code(func, s)
self._print_sig_code(func, s)
return func(s) return func(s)
except Exception as e: except Exception as e:
tb = traceback.format_exc() raise ExtractorError('Signature extraction failed: ' + traceback.format_exc(), cause=e)
raise ExtractorError(
'Signature extraction failed: ' + tb, cause=e) def _decrypt_nsig(self, s, video_id, player_url):
"""Turn the encrypted n field into a working signature"""
if player_url is None:
raise ExtractorError('Cannot decrypt nsig without player_url')
if player_url.startswith('//'):
player_url = 'https:' + player_url
elif not re.match(r'https?://', player_url):
player_url = compat_urlparse.urljoin(
'https://www.youtube.com', player_url)
sig_id = ('nsig_value', s)
if sig_id in self._player_cache:
return self._player_cache[sig_id]
try:
player_id = ('nsig', player_url)
if player_id not in self._player_cache:
self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
func = self._player_cache[player_id]
self._player_cache[sig_id] = func(s)
self.write_debug(f'Decrypted nsig {s} => {self._player_cache[sig_id]}')
return self._player_cache[sig_id]
except Exception as e:
raise ExtractorError(traceback.format_exc(), cause=e)
def _extract_n_function_name(self, jscode):
return self._search_regex(
(r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]{3})\([a-zA-Z0-9]\)',),
jscode, 'Initial JS player n function name', group='nfunc')
def _extract_n_function(self, video_id, player_url):
player_id = self._extract_player_info(player_url)
func_code = self._downloader.cache.load('youtube-nsig', player_id)
if func_code:
jsi = JSInterpreter(func_code)
else:
jscode = self._load_player(video_id, player_url)
funcname = self._extract_n_function_name(jscode)
jsi = JSInterpreter(jscode)
func_code = jsi.extract_function_code(funcname)
self._downloader.cache.store('youtube-nsig', player_id, func_code)
if self.get_param('youtube_print_sig_code'):
self.to_screen(f'Extracted nsig function from {player_id}:\n{func_code[1]}\n')
return lambda s: jsi.extract_function_from_code(*func_code)([s])
def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False): def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
""" """
@ -1856,9 +1904,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
raise ExtractorError(error_msg) raise ExtractorError(error_msg)
self.report_warning(error_msg) self.report_warning(error_msg)
return return
if self._load_player(video_id, player_url, fatal=fatal): code = self._load_player(video_id, player_url, fatal=fatal)
player_id = self._extract_player_info(player_url) if code:
code = self._code_cache[player_id]
sts = int_or_none(self._search_regex( sts = int_or_none(self._search_regex(
r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code, r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code,
'JS player signature timestamp', group='sts', fatal=fatal)) 'JS player signature timestamp', group='sts', fatal=fatal))
@ -2440,6 +2487,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
sp = try_get(sc, lambda x: x['sp'][0]) or 'signature' sp = try_get(sc, lambda x: x['sp'][0]) or 'signature'
fmt_url += '&' + sp + '=' + signature fmt_url += '&' + sp + '=' + signature
query = parse_qs(fmt_url)
throttled = False
if query.get('ratebypass') != ['yes'] and query.get('n'):
try:
fmt_url = update_url_query(fmt_url, {
'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
except ExtractorError as e:
self.report_warning(f'nsig extraction failed: You may experience throttling for some formats\n{e}', only_once=True)
throttled = True
if itag: if itag:
itags.append(itag) itags.append(itag)
stream_ids.append(stream_id) stream_ids.append(stream_id)
@ -2453,7 +2510,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'format_note': ', '.join(filter(None, ( 'format_note': ', '.join(filter(None, (
'%s%s' % (audio_track.get('displayName') or '', '%s%s' % (audio_track.get('displayName') or '',
' (default)' if audio_track.get('audioIsDefault') else ''), ' (default)' if audio_track.get('audioIsDefault') else ''),
fmt.get('qualityLabel') or quality.replace('audio_quality_', '')))), fmt.get('qualityLabel') or quality.replace('audio_quality_', ''),
throttled and 'THROTTLED'))),
'source_preference': -10 if not throttled else -1,
'fps': int_or_none(fmt.get('fps')), 'fps': int_or_none(fmt.get('fps')),
'height': height, 'height': height,
'quality': q(quality), 'quality': q(quality),
@ -2645,12 +2704,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
if reason: if reason:
self.raise_no_formats(reason, expected=True) self.raise_no_formats(reason, expected=True)
for f in formats:
if '&c=WEB&' in f['url'] and '&ratebypass=yes&' not in f['url']: # throttled
f['source_preference'] = -10
# TODO: this method is not reliable
f['format_note'] = format_field(f, 'format_note', '%s ') + '(maybe throttled)'
# Source is given priority since formats that throttle are given lower source_preference # Source is given priority since formats that throttle are given lower source_preference
# When throttling issue is fully fixed, remove this # When throttling issue is fully fixed, remove this
self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang')) self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang'))

View File

@ -1,5 +1,4 @@
from __future__ import unicode_literals from collections.abc import MutableMapping
import json import json
import operator import operator
import re import re
@ -22,11 +21,54 @@ _OPERATORS = [
('*', operator.mul), ('*', operator.mul),
] ]
_ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS] _ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]
_ASSIGN_OPERATORS.append(('=', lambda cur, right: right)) _ASSIGN_OPERATORS.append(('=', (lambda cur, right: right)))
_NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*' _NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
class JS_Break(ExtractorError):
def __init__(self):
ExtractorError.__init__(self, 'Invalid break')
class JS_Continue(ExtractorError):
def __init__(self):
ExtractorError.__init__(self, 'Invalid continue')
class LocalNameSpace(MutableMapping):
def __init__(self, *stack):
self.stack = tuple(stack)
def __getitem__(self, key):
for scope in self.stack:
if key in scope:
return scope[key]
raise KeyError(key)
def __setitem__(self, key, value):
for scope in self.stack:
if key in scope:
scope[key] = value
break
else:
self.stack[0][key] = value
return value
def __delitem__(self, key):
raise NotImplementedError('Deleting is not supported')
def __iter__(self):
for scope in self.stack:
yield from scope
def __len__(self, key):
return len(iter(self))
def __repr__(self):
return f'LocalNameSpace{self.stack}'
class JSInterpreter(object): class JSInterpreter(object):
def __init__(self, code, objects=None): def __init__(self, code, objects=None):
if objects is None: if objects is None:
@ -34,11 +76,58 @@ class JSInterpreter(object):
self.code = code self.code = code
self._functions = {} self._functions = {}
self._objects = objects self._objects = objects
self.__named_object_counter = 0
def _named_object(self, namespace, obj):
self.__named_object_counter += 1
name = f'__yt_dlp_jsinterp_obj{self.__named_object_counter}'
namespace[name] = obj
return name
@staticmethod
def _seperate(expr, delim=',', max_split=None):
if not expr:
return
parens = {'(': 0, '{': 0, '[': 0, ']': 0, '}': 0, ')': 0}
start, splits, pos, max_pos = 0, 0, 0, len(delim) - 1
for idx, char in enumerate(expr):
if char in parens:
parens[char] += 1
is_in_parens = (parens['['] - parens[']']
or parens['('] - parens[')']
or parens['{'] - parens['}'])
if char == delim[pos] and not is_in_parens:
if pos == max_pos:
pos = 0
yield expr[start: idx - max_pos]
start = idx + 1
splits += 1
if max_split and splits >= max_split:
break
else:
pos += 1
else:
pos = 0
yield expr[start:]
@staticmethod
def _seperate_at_paren(expr, delim):
seperated = list(JSInterpreter._seperate(expr, delim, 1))
if len(seperated) < 2:
raise ExtractorError(f'No terminating paren {delim} in {expr}')
return seperated[0][1:].strip(), seperated[1].strip()
def interpret_statement(self, stmt, local_vars, allow_recursion=100): def interpret_statement(self, stmt, local_vars, allow_recursion=100):
if allow_recursion < 0: if allow_recursion < 0:
raise ExtractorError('Recursion limit reached') raise ExtractorError('Recursion limit reached')
sub_statements = list(self._seperate(stmt, ';'))
stmt = (sub_statements or ['']).pop()
for sub_stmt in sub_statements:
ret, should_abort = self.interpret_statement(sub_stmt, local_vars, allow_recursion - 1)
if should_abort:
return ret
should_abort = False should_abort = False
stmt = stmt.lstrip() stmt = stmt.lstrip()
stmt_m = re.match(r'var\s', stmt) stmt_m = re.match(r'var\s', stmt)
@ -61,25 +150,118 @@ class JSInterpreter(object):
if expr == '': # Empty expression if expr == '': # Empty expression
return None return None
if expr.startswith('('): if expr.startswith('{'):
parens_count = 0 inner, outer = self._seperate_at_paren(expr, '}')
for m in re.finditer(r'[()]', expr): inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion - 1)
if m.group(0) == '(': if not outer or should_abort:
parens_count += 1 return inner
else:
parens_count -= 1
if parens_count == 0:
sub_expr = expr[1:m.start()]
sub_result = self.interpret_expression(
sub_expr, local_vars, allow_recursion)
remaining_expr = expr[m.end():].strip()
if not remaining_expr:
return sub_result
else:
expr = json.dumps(sub_result) + remaining_expr
break
else: else:
raise ExtractorError('Premature end of parens in %r' % expr) expr = json.dumps(inner) + outer
if expr.startswith('('):
inner, outer = self._seperate_at_paren(expr, ')')
inner = self.interpret_expression(inner, local_vars, allow_recursion)
if not outer:
return inner
else:
expr = json.dumps(inner) + outer
if expr.startswith('['):
inner, outer = self._seperate_at_paren(expr, ']')
name = self._named_object(local_vars, [
self.interpret_expression(item, local_vars, allow_recursion)
for item in self._seperate(inner)])
expr = name + outer
m = re.match(r'try\s*', expr)
if m:
if expr[m.end()] == '{':
try_expr, expr = self._seperate_at_paren(expr[m.end():], '}')
else:
try_expr, expr = expr[m.end() - 1:], ''
ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion - 1)
if should_abort:
return ret
return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
m = re.match(r'catch\s*\(', expr)
if m:
# We ignore the catch block
_, expr = self._seperate_at_paren(expr, '}')
return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
m = re.match(r'for\s*\(', expr)
if m:
constructor, remaining = self._seperate_at_paren(expr[m.end() - 1:], ')')
if remaining.startswith('{'):
body, expr = self._seperate_at_paren(remaining, '}')
else:
m = re.match(r'switch\s*\(', remaining) # FIXME
if m:
switch_val, remaining = self._seperate_at_paren(remaining[m.end() - 1:], ')')
body, expr = self._seperate_at_paren(remaining, '}')
body = 'switch(%s){%s}' % (switch_val, body)
else:
body, expr = remaining, ''
start, cndn, increment = self._seperate(constructor, ';')
if self.interpret_statement(start, local_vars, allow_recursion - 1)[1]:
raise ExtractorError(
f'Premature return in the initialization of a for loop in {constructor!r}')
while True:
if not self.interpret_expression(cndn, local_vars, allow_recursion):
break
try:
ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion - 1)
if should_abort:
return ret
except JS_Break:
break
except JS_Continue:
pass
if self.interpret_statement(increment, local_vars, allow_recursion - 1)[1]:
raise ExtractorError(
f'Premature return in the initialization of a for loop in {constructor!r}')
return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
m = re.match(r'switch\s*\(', expr)
if m:
switch_val, remaining = self._seperate_at_paren(expr[m.end() - 1:], ')')
switch_val = self.interpret_expression(switch_val, local_vars, allow_recursion)
body, expr = self._seperate_at_paren(remaining, '}')
body, default = body.split('default:') if 'default:' in body else (body, None)
items = body.split('case ')[1:]
if default:
items.append(f'default:{default}')
matched = False
for item in items:
case, stmt = [i.strip() for i in self._seperate(item, ':', 1)]
matched = matched or case == 'default' or switch_val == self.interpret_expression(case, local_vars, allow_recursion)
if matched:
try:
ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion - 1)
if should_abort:
return ret
except JS_Break:
break
return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
# Comma seperated statements
sub_expressions = list(self._seperate(expr))
expr = sub_expressions.pop().strip() if sub_expressions else ''
for sub_expr in sub_expressions:
self.interpret_expression(sub_expr, local_vars, allow_recursion)
for m in re.finditer(rf'''(?x)
(?P<pre_sign>\+\+|--)(?P<var1>{_NAME_RE})|
(?P<var2>{_NAME_RE})(?P<post_sign>\+\+|--)''', expr):
var = m.group('var1') or m.group('var2')
start, end = m.span()
sign = m.group('pre_sign') or m.group('post_sign')
ret = local_vars[var]
local_vars[var] += 1 if sign[0] == '+' else -1
if m.group('pre_sign'):
ret = local_vars[var]
expr = expr[:start] + json.dumps(ret) + expr[end:]
for op, opfunc in _ASSIGN_OPERATORS: for op, opfunc in _ASSIGN_OPERATORS:
m = re.match(r'''(?x) m = re.match(r'''(?x)
@ -88,14 +270,13 @@ class JSInterpreter(object):
(?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr) (?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr)
if not m: if not m:
continue continue
right_val = self.interpret_expression( right_val = self.interpret_expression(m.group('expr'), local_vars, allow_recursion)
m.group('expr'), local_vars, allow_recursion - 1)
if m.groupdict().get('index'): if m.groupdict().get('index'):
lvar = local_vars[m.group('out')] lvar = local_vars[m.group('out')]
idx = self.interpret_expression( idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
m.group('index'), local_vars, allow_recursion) if not isinstance(idx, int):
assert isinstance(idx, int) raise ExtractorError(f'List indices must be integers: {idx}')
cur = lvar[idx] cur = lvar[idx]
val = opfunc(cur, right_val) val = opfunc(cur, right_val)
lvar[idx] = val lvar[idx] = val
@ -109,8 +290,13 @@ class JSInterpreter(object):
if expr.isdigit(): if expr.isdigit():
return int(expr) return int(expr)
if expr == 'break':
raise JS_Break()
elif expr == 'continue':
raise JS_Continue()
var_m = re.match( var_m = re.match(
r'(?!if|return|true|false)(?P<name>%s)$' % _NAME_RE, r'(?!if|return|true|false|null)(?P<name>%s)$' % _NAME_RE,
expr) expr)
if var_m: if var_m:
return local_vars[var_m.group('name')] return local_vars[var_m.group('name')]
@ -124,91 +310,154 @@ class JSInterpreter(object):
r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr) r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
if m: if m:
val = local_vars[m.group('in')] val = local_vars[m.group('in')]
idx = self.interpret_expression( idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion)
m.group('idx'), local_vars, allow_recursion - 1)
return val[idx] return val[idx]
for op, opfunc in _OPERATORS:
seperated = list(self._seperate(expr, op))
if len(seperated) < 2:
continue
right_val = seperated.pop()
left_val = op.join(seperated)
left_val, should_abort = self.interpret_statement(
left_val, local_vars, allow_recursion - 1)
if should_abort:
raise ExtractorError(f'Premature left-side return of {op} in {expr!r}')
right_val, should_abort = self.interpret_statement(
right_val, local_vars, allow_recursion - 1)
if should_abort:
raise ExtractorError(f'Premature right-side return of {op} in {expr!r}')
return opfunc(left_val or 0, right_val)
m = re.match( m = re.match(
r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*(?:\(+(?P<args>[^()]*)\))?$' % _NAME_RE, r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*' % _NAME_RE,
expr) expr)
if m: if m:
variable = m.group('var') variable = m.group('var')
member = remove_quotes(m.group('member') or m.group('member2')) member = remove_quotes(m.group('member') or m.group('member2'))
arg_str = m.group('args') arg_str = expr[m.end():]
if arg_str.startswith('('):
if variable in local_vars: arg_str, remaining = self._seperate_at_paren(arg_str, ')')
obj = local_vars[variable]
else: else:
if variable not in self._objects: arg_str, remaining = None, arg_str
self._objects[variable] = self.extract_object(variable)
obj = self._objects[variable]
if arg_str is None: def assertion(cndn, msg):
# Member access """ assert, but without risk of getting optimized out """
if member == 'length': if not cndn:
return len(obj) raise ExtractorError(f'{member} {msg}: {expr}')
return obj[member]
assert expr.endswith(')') def eval_method():
# Function call nonlocal member
if arg_str == '': if variable == 'String':
argvals = tuple() obj = str
else: elif variable in local_vars:
argvals = tuple([ obj = local_vars[variable]
else:
if variable not in self._objects:
self._objects[variable] = self.extract_object(variable)
obj = self._objects[variable]
if arg_str is None:
# Member access
if member == 'length':
return len(obj)
return obj[member]
# Function call
argvals = [
self.interpret_expression(v, local_vars, allow_recursion) self.interpret_expression(v, local_vars, allow_recursion)
for v in arg_str.split(',')]) for v in self._seperate(arg_str)]
if member == 'split': if obj == str:
assert argvals == ('',) if member == 'fromCharCode':
return list(obj) assertion(argvals, 'takes one or more arguments')
if member == 'join': return ''.join(map(chr, argvals))
assert len(argvals) == 1 raise ExtractorError(f'Unsupported string method {member}')
return argvals[0].join(obj)
if member == 'reverse':
assert len(argvals) == 0
obj.reverse()
return obj
if member == 'slice':
assert len(argvals) == 1
return obj[argvals[0]:]
if member == 'splice':
assert isinstance(obj, list)
index, howMany = argvals
res = []
for i in range(index, min(index + howMany, len(obj))):
res.append(obj.pop(index))
return res
return obj[member](argvals) if member == 'split':
assertion(argvals, 'takes one or more arguments')
assertion(argvals == [''], 'with arguments is not implemented')
return list(obj)
elif member == 'join':
assertion(isinstance(obj, list), 'must be applied on a list')
assertion(len(argvals) == 1, 'takes exactly one argument')
return argvals[0].join(obj)
elif member == 'reverse':
assertion(not argvals, 'does not take any arguments')
obj.reverse()
return obj
elif member == 'slice':
assertion(isinstance(obj, list), 'must be applied on a list')
assertion(len(argvals) == 1, 'takes exactly one argument')
return obj[argvals[0]:]
elif member == 'splice':
assertion(isinstance(obj, list), 'must be applied on a list')
assertion(argvals, 'takes one or more arguments')
index, howMany = (argvals + [len(obj)])[:2]
if index < 0:
index += len(obj)
add_items = argvals[2:]
res = []
for i in range(index, min(index + howMany, len(obj))):
res.append(obj.pop(index))
for i, item in enumerate(add_items):
obj.insert(index + i, item)
return res
elif member == 'unshift':
assertion(isinstance(obj, list), 'must be applied on a list')
assertion(argvals, 'takes one or more arguments')
for item in reversed(argvals):
obj.insert(0, item)
return obj
elif member == 'pop':
assertion(isinstance(obj, list), 'must be applied on a list')
assertion(not argvals, 'does not take any arguments')
if not obj:
return
return obj.pop()
elif member == 'push':
assertion(argvals, 'takes one or more arguments')
obj.extend(argvals)
return obj
elif member == 'forEach':
assertion(argvals, 'takes one or more arguments')
assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
f, this = (argvals + [''])[:2]
return [f((item, idx, obj), this=this) for idx, item in enumerate(obj)]
elif member == 'indexOf':
assertion(argvals, 'takes one or more arguments')
assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
idx, start = (argvals + [0])[:2]
try:
return obj.index(idx, start)
except ValueError:
return -1
for op, opfunc in _OPERATORS: if isinstance(obj, list):
m = re.match(r'(?P<x>.+?)%s(?P<y>.+)' % re.escape(op), expr) member = int(member)
if not m: return obj[member](argvals)
continue
x, abort = self.interpret_statement(
m.group('x'), local_vars, allow_recursion - 1)
if abort:
raise ExtractorError(
'Premature left-side return of %s in %r' % (op, expr))
y, abort = self.interpret_statement(
m.group('y'), local_vars, allow_recursion - 1)
if abort:
raise ExtractorError(
'Premature right-side return of %s in %r' % (op, expr))
return opfunc(x, y)
m = re.match( if remaining:
r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr) return self.interpret_expression(
self._named_object(local_vars, eval_method()) + remaining,
local_vars, allow_recursion)
else:
return eval_method()
m = re.match(r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr)
if m: if m:
fname = m.group('func') fname = m.group('func')
argvals = tuple([ argvals = tuple([
int(v) if v.isdigit() else local_vars[v] int(v) if v.isdigit() else local_vars[v]
for v in m.group('args').split(',')]) if len(m.group('args')) > 0 else tuple() for v in self._seperate(m.group('args'))])
if fname not in self._functions: if fname in local_vars:
return local_vars[fname](argvals)
elif fname not in self._functions:
self._functions[fname] = self.extract_function(fname) self._functions[fname] = self.extract_function(fname)
return self._functions[fname](argvals) return self._functions[fname](argvals)
raise ExtractorError('Unsupported JS expression %r' % expr) if expr:
raise ExtractorError('Unsupported JS expression %r' % expr)
def extract_object(self, objname): def extract_object(self, objname):
_FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')''' _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
@ -233,30 +482,55 @@ class JSInterpreter(object):
return obj return obj
def extract_function(self, funcname): def extract_function_code(self, funcname):
""" @returns argnames, code """
func_m = re.search( func_m = re.search(
r'''(?x) r'''(?x)
(?:function\s+%s|[{;,]\s*%s\s*=\s*function|var\s+%s\s*=\s*function)\s* (?:function\s+%s|[{;,]\s*%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
\((?P<args>[^)]*)\)\s* \((?P<args>[^)]*)\)\s*
\{(?P<code>[^}]+)\}''' % ( (?P<code>\{(?:(?!};)[^"]|"([^"]|\\")*")+\})''' % (
re.escape(funcname), re.escape(funcname), re.escape(funcname)), re.escape(funcname), re.escape(funcname), re.escape(funcname)),
self.code) self.code)
code, _ = self._seperate_at_paren(func_m.group('code'), '}') # refine the match
if func_m is None: if func_m is None:
raise ExtractorError('Could not find JS function %r' % funcname) raise ExtractorError('Could not find JS function %r' % funcname)
argnames = func_m.group('args').split(',') return func_m.group('args').split(','), code
return self.build_function(argnames, func_m.group('code')) def extract_function(self, funcname):
return self.extract_function_from_code(*self.extract_function_code(funcname))
def extract_function_from_code(self, argnames, code, *global_stack):
local_vars = {}
while True:
mobj = re.search(r'function\((?P<args>[^)]*)\)\s*{', code)
if mobj is None:
break
start, body_start = mobj.span()
body, remaining = self._seperate_at_paren(code[body_start - 1:], '}')
name = self._named_object(
local_vars,
self.extract_function_from_code(
[str.strip(x) for x in mobj.group('args').split(',')],
body, local_vars, *global_stack))
code = code[:start] + name + remaining
return self.build_function(argnames, code, local_vars, *global_stack)
def call_function(self, funcname, *args): def call_function(self, funcname, *args):
f = self.extract_function(funcname) return self.extract_function(funcname)(args)
return f(args)
def build_function(self, argnames, code): def build_function(self, argnames, code, *global_stack):
def resf(args): global_stack = list(global_stack) or [{}]
local_vars = dict(zip(argnames, args)) local_vars = global_stack.pop(0)
for stmt in code.split(';'):
res, abort = self.interpret_statement(stmt, local_vars) def resf(args, **kwargs):
if abort: local_vars.update({
**dict(zip(argnames, args)),
**kwargs
})
var_stack = LocalNameSpace(local_vars, *global_stack)
for stmt in self._seperate(code.replace('\n', ''), ';'):
ret, should_abort = self.interpret_statement(stmt, var_stack)
if should_abort:
break break
return res return ret
return resf return resf