mirrors/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2024-11-30 12:01:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Youtube] Extract more formats for music.youtube URLs (#311)

Browse source 
Based on: https://github.com/ytdl-org/youtube-dl/pull/28778, https://github.com/ytdl-org/youtube-dl/pull/26160

Co-authored-by: craftingmod, colethedj, pukkandan
This commit is contained in:
coletdjnz 2021-05-16 02:38:47 +12:00 committed by GitHub
parent 774d79cc4c
commit 9297939ec3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 112 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

161
yt_dlp/extractor/youtube.py
View file

@ -398,6 +398,10 @@ def _generate_api_headers(self, ytcfg=None, identity_token=None, account_syncid=
headers['X-Origin'] = 'https://www.youtube.com' headers['X-Origin'] = 'https://www.youtube.com'
return headers return headers
@staticmethod
def is_music_url(url):
return re.match(r'https?://music\.youtube\.com/', url) is not None
def _extract_video(self, renderer): def _extract_video(self, renderer):
video_id = renderer.get('videoId') video_id = renderer.get('videoId')
title = try_get( title = try_get(
@ -521,7 +525,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
)? # all until now is optional -> you can pass the naked ID )? # all until now is optional -> you can pass the naked ID
(?P<id>[0-9A-Za-z_-]{11}) # here is it! the YouTube video ID (?P<id>[0-9A-Za-z_-]{11}) # here is it! the YouTube video ID
(?(1).+)? # if we found the ID, everything can follow (?(1).+)? # if we found the ID, everything can follow
$""" % { (?:\#|$)""" % {
'invidious': '|'.join(_INVIDIOUS_SITES), 'invidious': '|'.join(_INVIDIOUS_SITES),
} }
_PLAYER_INFO_RE = ( _PLAYER_INFO_RE = (
@ -1307,8 +1311,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
# Has multiple audio streams # Has multiple audio streams
'url': 'WaOKSUlf4TM', 'url': 'WaOKSUlf4TM',
'only_matching': True 'only_matching': True
}, }, {
{ # Requires Premium: has format 141 when requested using YTM url
'url': 'https://music.youtube.com/watch?v=XclachpHxis',
'only_matching': True
}, {
# multiple subtitles with same lang_code # multiple subtitles with same lang_code
'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug', 'url': 'https://www.youtube.com/watch?v=wsQiKKfKxug',
'only_matching': True, 'only_matching': True,
@ -1852,11 +1859,46 @@ def _extract_comments(self, ytcfg, video_id, contents, webpage, xsrf_token):
def _real_extract(self, url): def _real_extract(self, url):
url, smuggled_data = unsmuggle_url(url, {}) url, smuggled_data = unsmuggle_url(url, {})
video_id = self._match_id(url) video_id = self._match_id(url)
is_music_url = smuggled_data.get('is_music_url') or self.is_music_url(url)
base_url = self.http_scheme() + '//www.youtube.com/' base_url = self.http_scheme() + '//www.youtube.com/'
webpage_url = base_url + 'watch?v=' + video_id webpage_url = base_url + 'watch?v=' + video_id
webpage = self._download_webpage( webpage = self._download_webpage(
webpage_url + '&bpctr=9999999999&has_verified=1', video_id, fatal=False) webpage_url + '&bpctr=9999999999&has_verified=1', video_id, fatal=False)
def get_text(x):
if not x:
return
text = x.get('simpleText')
if text and isinstance(text, compat_str):
return text
runs = x.get('runs')
if not isinstance(runs, list):
return
return ''.join([r['text'] for r in runs if isinstance(r.get('text'), compat_str)])
ytm_streaming_data = {}
if is_music_url:
# we are forcing to use parse_json because 141 only appeared in get_video_info.
# el, c, cver, cplayer field required for 141(aac 256kbps) codec
# maybe paramter of youtube music player?
ytm_player_response = self._parse_json(try_get(compat_parse_qs(
self._download_webpage(
base_url + 'get_video_info', video_id,
'Fetching youtube-music info webpage',
'unable to download youtube-music info webpage', query={
'video_id': video_id,
'eurl': 'https://youtube.googleapis.com/v/' + video_id,
'el': 'detailpage',
'c': 'WEB_REMIX',
'cver': '0.1',
'cplayer': 'UNIPLAYER'
}, fatal=False)),
lambda x: x['player_response'][0],
compat_str) or '{}', video_id)
ytm_streaming_data = ytm_player_response.get('streamingData') or {}
player_response = None player_response = None
if webpage: if webpage:
player_response = self._extract_yt_initial_variable( player_response = self._extract_yt_initial_variable(
@ -1891,17 +1933,6 @@ def _real_extract(self, url):
return self.url_result( return self.url_result(
trailer_video_id, self.ie_key(), trailer_video_id) trailer_video_id, self.ie_key(), trailer_video_id)
def get_text(x):
if not x:
return
text = x.get('simpleText')
if text and isinstance(text, compat_str):
return text
runs = x.get('runs')
if not isinstance(runs, list):
return
return ''.join([r['text'] for r in runs if isinstance(r.get('text'), compat_str)])
search_meta = ( search_meta = (
lambda x: self._html_search_meta(x, webpage, default=None)) \ lambda x: self._html_search_meta(x, webpage, default=None)) \
if webpage else lambda x: None if webpage else lambda x: None
@ -1960,19 +1991,27 @@ def feed_entry(name):
else: else:
self.to_screen('Downloading just video %s because of --no-playlist' % video_id) self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
formats = [] formats, itags, stream_ids = [], [], []
itags = []
itag_qualities = {} itag_qualities = {}
player_url = None player_url = None
q = qualities(['tiny', 'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres']) q = qualities(['tiny', 'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'])
streaming_data = player_response.get('streamingData') or {} streaming_data = player_response.get('streamingData') or {}
streaming_formats = streaming_data.get('formats') or [] streaming_formats = streaming_data.get('formats') or []
streaming_formats.extend(streaming_data.get('adaptiveFormats') or []) streaming_formats.extend(streaming_data.get('adaptiveFormats') or [])
streaming_formats.extend(ytm_streaming_data.get('formats') or [])
streaming_formats.extend(ytm_streaming_data.get('adaptiveFormats') or [])
for fmt in streaming_formats: for fmt in streaming_formats:
if fmt.get('targetDurationSec') or fmt.get('drmFamilies'): if fmt.get('targetDurationSec') or fmt.get('drmFamilies'):
continue continue
itag = str_or_none(fmt.get('itag')) itag = str_or_none(fmt.get('itag'))
audio_track = fmt.get('audioTrack') or {}
stream_id = '%s.%s' % (itag or '', audio_track.get('id', ''))
if stream_id in stream_ids:
continue
quality = fmt.get('quality') quality = fmt.get('quality')
if itag and quality: if itag and quality:
itag_qualities[itag] = quality itag_qualities[itag] = quality
@ -2003,9 +2042,10 @@ def feed_entry(name):
if itag: if itag:
itags.append(itag) itags.append(itag)
stream_ids.append(stream_id)
tbr = float_or_none( tbr = float_or_none(
fmt.get('averageBitrate') or fmt.get('bitrate'), 1000) fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
audio_track = fmt.get('audioTrack') or {}
dct = { dct = {
'asr': int_or_none(fmt.get('audioSampleRate')), 'asr': int_or_none(fmt.get('audioSampleRate')),
'filesize': int_or_none(fmt.get('contentLength')), 'filesize': int_or_none(fmt.get('contentLength')),
@ -2041,35 +2081,37 @@ def feed_entry(name):
dct['container'] = dct['ext'] + '_dash' dct['container'] = dct['ext'] + '_dash'
formats.append(dct) formats.append(dct)
hls_manifest_url = streaming_data.get('hlsManifestUrl') for sd in (streaming_data, ytm_streaming_data):
if hls_manifest_url: hls_manifest_url = sd.get('hlsManifestUrl')
for f in self._extract_m3u8_formats( if hls_manifest_url:
hls_manifest_url, video_id, 'mp4', fatal=False): for f in self._extract_m3u8_formats(
itag = self._search_regex( hls_manifest_url, video_id, 'mp4', fatal=False):
r'/itag/(\d+)', f['url'], 'itag', default=None) itag = self._search_regex(
if itag: r'/itag/(\d+)', f['url'], 'itag', default=None)
f['format_id'] = itag if itag:
f['format_id'] = itag
formats.append(f) formats.append(f)
if self._downloader.params.get('youtube_include_dash_manifest', True): if self._downloader.params.get('youtube_include_dash_manifest', True):
dash_manifest_url = streaming_data.get('dashManifestUrl') for sd in (streaming_data, ytm_streaming_data):
if dash_manifest_url: dash_manifest_url = sd.get('dashManifestUrl')
for f in self._extract_mpd_formats( if dash_manifest_url:
dash_manifest_url, video_id, fatal=False): for f in self._extract_mpd_formats(
itag = f['format_id'] dash_manifest_url, video_id, fatal=False):
if itag in itags: itag = f['format_id']
continue if itag in itags:
if itag in itag_qualities: continue
# Not actually usefull since the sorting is already done with "quality,res,fps,codec" if itag in itag_qualities:
# but kept to maintain feature parity (and code similarity) with youtube-dl # Not actually usefull since the sorting is already done with "quality,res,fps,codec"
# Remove if this causes any issues with sorting in future # but kept to maintain feature parity (and code similarity) with youtube-dl
f['quality'] = q(itag_qualities[itag]) # Remove if this causes any issues with sorting in future
filesize = int_or_none(self._search_regex( f['quality'] = q(itag_qualities[itag])
r'/clen/(\d+)', f.get('fragment_base_url') filesize = int_or_none(self._search_regex(
or f['url'], 'file size', default=None)) r'/clen/(\d+)', f.get('fragment_base_url')
if filesize: or f['url'], 'file size', default=None))
f['filesize'] = filesize if filesize:
formats.append(f) f['filesize'] = filesize
formats.append(f)
if not formats: if not formats:
if not self._downloader.params.get('allow_unplayable_formats') and streaming_data.get('licenseInfos'): if not self._downloader.params.get('allow_unplayable_formats') and streaming_data.get('licenseInfos'):
@ -2831,6 +2873,10 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
}, { }, {
'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU', 'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
'only_matching': True, 'only_matching': True,
}, {
# Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist.
'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
'only_matching': True
}] }]
@classmethod @classmethod
@ -3493,7 +3539,23 @@ def _extract_webpage(self, url, item_id):
raise ExtractorError(last_error) raise ExtractorError(last_error)
return webpage, data return webpage, data
@staticmethod
def _smuggle_data(entries, data):
for entry in entries:
if data:
entry['url'] = smuggle_url(entry['url'], data)
yield entry
def _real_extract(self, url): def _real_extract(self, url):
url, smuggled_data = unsmuggle_url(url, {})
if self.is_music_url(url):
smuggled_data['is_music_url'] = True
info_dict = self.__real_extract(url)
if info_dict.get('entries'):
info_dict['entries'] = self._smuggle_data(info_dict['entries'], smuggled_data)
return info_dict
def __real_extract(self, url):
item_id = self._match_id(url) item_id = self._match_id(url)
url = compat_urlparse.urlunparse( url = compat_urlparse.urlunparse(
compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com')) compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
@ -3628,12 +3690,13 @@ def suitable(cls, url):
def _real_extract(self, url): def _real_extract(self, url):
playlist_id = self._match_id(url) playlist_id = self._match_id(url)
qs = parse_qs(url) is_music_url = self.is_music_url(url)
if not qs: url = update_url_query(
qs = {'list': playlist_id} 'https://www.youtube.com/playlist',
return self.url_result( parse_qs(url) or {'list': playlist_id})
update_url_query('https://www.youtube.com/playlist', qs), if is_music_url:
ie=YoutubeTabIE.ie_key(), video_id=playlist_id) url = smuggle_url(url, {'is_music_url': True})
return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
class YoutubeYtBeIE(InfoExtractor): class YoutubeYtBeIE(InfoExtractor):