Merge d07eb72f6f into 96da952504

Authored by: seproDev, pukkandan

yt_dlp/YoutubeDL.py

@@ -2136,6 +2136,11 @@ class YoutubeDL:
 
     def _check_formats(self, formats):
         for f in formats:
+            working = f.get('__working')
+            if working is not None:
+                if working:
+                    yield f
+                continue
             self.to_screen('[info] Testing format %s' % f['format_id'])
             path = self.get_output_path('temp')
             if not self._ensure_dir_exists(f'{path}/'):
@@ -2152,33 +2157,44 @@ class YoutubeDL:
                         os.remove(temp_file.name)
                     except OSError:
                         self.report_warning('Unable to delete temporary file "%s"' % temp_file.name)
+            f['__working'] = success
             if success:
                 yield f
             else:
                 self.to_screen('[info] Unable to download format %s. Skipping...' % f['format_id'])
 
+    def _select_formats(self, formats, selector):
+        return list(selector({
+            'formats': formats,
+            'has_merged_format': any('none' not in (f.get('acodec'), f.get('vcodec')) for f in formats),
+            'incomplete_formats': (all(f.get('vcodec') == 'none' for f in formats)  # No formats with video
+                                   or all(f.get('acodec') == 'none' for f in formats)),  # OR, No formats with audio
+        }))
+
     def _default_format_spec(self, info_dict, download=True):
+        download = download and not self.params.get('simulate')
+        prefer_best = download and (
+            self.params['outtmpl']['default'] == '-'
+            or info_dict.get('is_live') and not self.params.get('live_from_start'))
 
         def can_merge():
             merger = FFmpegMergerPP(self)
             return merger.available and merger.can_merge()
 
-        prefer_best = (
-            not self.params.get('simulate')
-            and download
-            and (
-                not can_merge()
-                or info_dict.get('is_live') and not self.params.get('live_from_start')
-                or self.params['outtmpl']['default'] == '-'))
-        compat = (
-            prefer_best
-            or self.params.get('allow_multiple_audio_streams', False)
-            or 'format-spec' in self.params['compat_opts'])
+        if not prefer_best and download and not can_merge():
+            prefer_best = True
+            formats = self._get_formats(info_dict)
+            evaluate_formats = lambda spec: self._select_formats(formats, self.build_format_selector(spec))
+            if evaluate_formats('b/bv+ba') != evaluate_formats('bv*+ba/b'):
+                self.report_warning('ffmpeg not found. The downloaded format may not be the best available. '
+                                    'Installing ffmpeg is strongly recommended: https://github.com/yt-dlp/yt-dlp#dependencies')
 
-        return (
-            'best/bestvideo+bestaudio' if prefer_best
-            else 'bestvideo*+bestaudio/best' if not compat
-            else 'bestvideo+bestaudio/best')
+        compat = (self.params.get('allow_multiple_audio_streams')
+                  or 'format-spec' in self.params['compat_opts'])
+
+        return ('best/bestvideo+bestaudio' if prefer_best
+                else 'bestvideo+bestaudio/best' if compat
+                else 'bestvideo*+bestaudio/best')
 
     def build_format_selector(self, format_spec):
         def syntax_error(note, start):
@@ -2928,12 +2944,7 @@ class YoutubeDL:
                 self.write_debug(f'Default format spec: {req_format}')
                 format_selector = self.build_format_selector(req_format)
 
-            formats_to_download = list(format_selector({
-                'formats': formats,
-                'has_merged_format': any('none' not in (f.get('acodec'), f.get('vcodec')) for f in formats),
-                'incomplete_formats': (all(f.get('vcodec') == 'none' for f in formats)  # No formats with video
-                                       or all(f.get('acodec') == 'none' for f in formats)),  # OR, No formats with audio
-            }))
+            formats_to_download = self._select_formats(formats, format_selector)
             if interactive_format_selection and not formats_to_download:
                 self.report_error('Requested format is not available', tb=False, is_error=False)
                 continue

yt_dlp/extractor/fox.py

@@ -10,17 +10,19 @@ from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
+    join_nonempty,
     parse_age_limit,
     parse_duration,
     traverse_obj,
     try_get,
     unified_timestamp,
     url_or_none,
+    urljoin,
 )
 
 
 class FOXIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?fox(?:sports)?\.com/(?:watch|replay)/(?P<id>[\da-fA-F]+)'
+    _VALID_URL = r'https?://(?:www\.)?fox(?:sports)?\.com/(?:watch|replay)/(?!play-)(?P<id>[\w-]+)'
     _TESTS = [{
         # clip
         'url': 'https://www.fox.com/watch/4b765a60490325103ea69888fb2bd4e8/',
@@ -33,8 +35,9 @@ class FOXIE(InfoExtractor):
             'duration': 102,
             'timestamp': 1504291893,
             'upload_date': '20170901',
-            'creator': 'FOX',
-            'series': 'Gotham',
+            'creators': ['FOX'],
+            # actual series name 'Gotham' is no longer returned by the API
+            'series': 'Aftermath: Bruce Wayne Develops Into The Dark Knight',
             'age_limit': 14,
             'episode': 'Aftermath: Bruce Wayne Develops Into The Dark Knight',
             'thumbnail': r're:^https?://.*\.jpg$',
@@ -42,6 +45,24 @@ class FOXIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # XML endpoint
+        'url': 'https://www.foxsports.com/watch/fmc-m2du80v5ewz11pbw',
+        'md5': '5451a633a5ca87b582a4d025df6852e6',
+        'info_dict': {
+            'id': 'fmc-m2du80v5ewz11pbw',
+            'ext': 'mp4',
+            'title': 'WWE FRIDAY NIGHT SMACKDOWN',
+            'description': 'From Fiserv Forum in Milwaukee, WI',
+            'duration': 5367,
+            'timestamp': 1698176671,
+            'upload_date': '20231024',
+            'creators': ['fox-digital'],
+            'series': 'WWE FRIDAY NIGHT SMACKDOWN',
+            'age_limit': 0,
+            'episode': 'WWE FRIDAY NIGHT SMACKDOWN',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
     }, {
         # episode, geo-restricted
         'url': 'https://www.fox.com/watch/087036ca7f33c8eb79b08152b4dd75c1/',
@@ -57,9 +78,13 @@ class FOXIE(InfoExtractor):
     }]
     _GEO_BYPASS = False
     _HOME_PAGE_URL = 'https://www.fox.com/'
-    _API_KEY = '6E9S4bmcoNnZwVLOHywOv8PJEdu76cM9'
+    _API_KEY = '6E9S4bmcoNnZwVLOHywOv8PJEdu76cM9'  # sports: 'cf289e299efdfa39fb6316f259d1de93'
     _access_token = None
     _device_id = compat_str(uuid.uuid4())
+    _XML_NS = {
+        'vmap': 'http://www.iab.net/videosuite/vmap',
+        'yospacenet': 'http://www.yospace.com/extension',
+    }
 
     def _call_api(self, path, video_id, data=None):
         headers = {
@@ -69,7 +94,7 @@ class FOXIE(InfoExtractor):
             headers['Authorization'] = 'Bearer ' + self._access_token
         try:
             return self._download_json(
-                'https://api3.fox.com/v2.0/' + path,
+                urljoin('https://api3.fox.com/v2.0/', path),
                 video_id, data=data, headers=headers)
         except ExtractorError as e:
             if isinstance(e.cause, HTTPError) and e.cause.status == 403:
@@ -103,8 +128,8 @@ class FOXIE(InfoExtractor):
             'previewpassmvpd?device_id=%s&mvpd_id=TempPass_fbcfox_60min' % self._device_id,
             video_id)['accessToken']
 
-        video = self._call_api('watch', video_id, data=json.dumps({
-            'capabilities': ['drm/widevine', 'fsdk/yo'],
+        video = self._call_api('https://prod.api.video.fox/v2.0/watch', video_id, data=json.dumps({
+            'capabilities': ['fsdk/yo/v3'],
             'deviceWidth': 1280,
             'deviceHeight': 720,
             'maxRes': '720p',
@@ -119,13 +144,16 @@ class FOXIE(InfoExtractor):
             'privacy': {'us': '1---'},
             'siteSection': '',
             'streamType': 'vod',
-            'streamId': video_id}).encode('utf-8'))
-
-        title = video['name']
-        release_url = video['url']
+            'streamId': video_id,
+        }).encode())
 
         try:
-            m3u8_url = self._download_json(release_url, video_id)['playURL']
+            if playback_url := traverse_obj(video, ('playbackUrl', {url_or_none})):
+                xml_data = self._download_xml(playback_url, video_id)
+                stream = xml_data.find('vmap:Extensions/vmap:Extension/yospacenet:Stream', self._XML_NS)
+                m3u8_url = join_nonempty('https://', stream.get('urlDomain'), stream.get('urlSuffix'), delim='')
+            else:
+                m3u8_url = self._download_json(video['url'], video_id)['playURL']
         except ExtractorError as e:
             if isinstance(e.cause, HTTPError) and e.cause.status == 403:
                 error = self._parse_json(e.cause.response.read().decode(), video_id)
@@ -133,9 +161,11 @@ class FOXIE(InfoExtractor):
                     self.raise_geo_restricted(countries=['US'])
                 raise ExtractorError(error['description'], expected=True)
             raise
-        formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, 'mp4',
-            entry_protocol='m3u8_native', m3u8_id='hls')
+
+        if not m3u8_url or m3u8_url == 'https://':
+            raise ExtractorError('Unable to extract m3u8 url')
+
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', m3u8_id='hls')
 
         data = try_get(
             video, lambda x: x['trackingData']['properties'], dict) or {}
@@ -160,7 +190,7 @@ class FOXIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': title,
+            'title': video.get('name'),
             'formats': formats,
             'description': video.get('description'),
             'duration': duration,

yt_dlp/extractor/foxsports.py

@@ -5,7 +5,7 @@ from ..utils import float_or_none, make_archive_id, smuggle_url
 
 
 class FoxSportsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?foxsports\.com/watch/(?P<id>[\w-]+)'
+    _VALID_URL = r'https?://(?:www\.)?foxsports\.com/watch/(?P<id>play-[\w-]+)'
     _TESTS = [{
         'url': 'https://www.foxsports.com/watch/play-612168c6700004b',
         'info_dict': {