From a403dcf9be20b49cbb3017328f4aaa352fb6d685 Mon Sep 17 00:00:00 2001
From: Mozi <29089388+pzhlkj6612@users.noreply.github.com>
Date: Mon, 4 Nov 2024 07:02:48 +0800
Subject: [PATCH] [ie/Dailymotion] Improve embed extraction (#10843)

Closes #8848, Closes #9432
Authored by: pzhlkj6612, bashonly

Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com>
---
 yt_dlp/extractor/dailymotion.py | 115 ++++++++++++++++++++++++++++----
 1 file changed, 101 insertions(+), 14 deletions(-)

diff --git a/yt_dlp/extractor/dailymotion.py b/yt_dlp/extractor/dailymotion.py
index 632335e5b0..4e99fdda7a 100644
--- a/yt_dlp/extractor/dailymotion.py
+++ b/yt_dlp/extractor/dailymotion.py
@@ -10,11 +10,14 @@
     OnDemandPagedList,
     age_restricted,
     clean_html,
+    extract_attributes,
     int_or_none,
     traverse_obj,
     try_get,
     unescapeHTML,
     unsmuggle_url,
+    update_url,
+    url_or_none,
     urlencode_postdata,
 )
 
@@ -99,11 +102,16 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
     _VALID_URL = r'''(?ix)
                     https?://
                         (?:
-                            (?:(?:www|touch|geo)\.)?dailymotion\.[a-z]{2,3}/(?:(?:(?:(?:embed|swf|\#)/)|player(?:/\w+)?\.html\?)?video|swf)|
-                            (?:www\.)?lequipe\.fr/video
+                            (?:(?:www|touch|geo)\.)?dailymotion\.[a-z]{2,3}|
+                            (?:www\.)?lequipe\.fr
+                        )/
+                        (?:
+                            swf/(?!video)|
+                            (?:(?:crawler|embed|swf)/)?video/|
+                            player(?:/[\da-z]+)?\.html\?(?:video|(?P<is_playlist>playlist))=
                         )
-                        [/=](?P<id>[^/?_&]+)(?:.+?\bplaylist=(?P<playlist_id>x[0-9a-z]+))?
-                    '''
+                    (?P<id>[^/?_&#]+)(?:[\w-]*\?playlist=(?P<playlist_id>x[0-9a-z]+))?
+    '''
     IE_NAME = 'dailymotion'
     _EMBED_REGEX = [r'<(?:(?:embed|iframe)[^>]+?src=|input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=)(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1']
     _TESTS = [{
@@ -217,6 +225,63 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
     }, {
         'url': 'https://geo.dailymotion.com/player/xakln.html?video=x8mjju4&customConfig%5BcustomParams%5D=%2Ffr-fr%2Ftennis%2Fwimbledon-mens-singles%2Farticles-video',
         'only_matching': True,
+    }, {  # playlist-only
+        'url': 'https://geo.dailymotion.com/player/xf7zn.html?playlist=x7wdsj',
+        'only_matching': True,
+    }, {
+        'url': 'https://geo.dailymotion.com/player/xmyye.html?video=x93blhi',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.dailymotion.com/crawler/video/x8u4owg',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.dailymotion.com/embed/video/x8u4owg',
+        'only_matching': True,
+    }]
+    _WEBPAGE_TESTS = [{
+        # https://geo.dailymotion.com/player/xmyye.html?video=x93blhi
+        'url': 'https://www.financialounge.com/video/2024/08/01/borse-europee-in-rosso-dopo-la-fed-a-milano-volano-mediobanca-e-tim-edizione-del-1-agosto/',
+        'info_dict': {
+            'id': 'x93blhi',
+            'ext': 'mp4',
+            'title': 'OnAir - 01/08/24',
+            'description': '',
+            'duration': 217,
+            'timestamp': 1722505658,
+            'upload_date': '20240801',
+            'uploader': 'Financialounge',
+            'uploader_id': 'x2vtgmm',
+            'age_limit': 0,
+            'tags': [],
+            'view_count': int,
+            'like_count': int,
+        },
+    }, {
+        # https://geo.dailymotion.com/player/xf7zn.html?playlist=x7wdsj
+        'url': 'https://www.cycleworld.com/blogs/ask-kevin/ducati-continues-to-evolve-with-v4/',
+        'info_dict': {
+            'id': 'x7wdsj',
+        },
+        'playlist_mincount': 50,
+    }, {
+        # https://www.dailymotion.com/crawler/video/x8u4owg
+        'url': 'https://www.leparisien.fr/environnement/video-le-veloto-la-voiture-a-pedales-qui-aimerait-se-faire-une-place-sur-les-routes-09-03-2024-KCYMCPM4WFHJXMSKBUI66UNFPU.php',
+        'info_dict': {
+            'id': 'x8u4owg',
+            'ext': 'mp4',
+            'like_count': int,
+            'uploader': 'Le Parisien',
+            'thumbnail': 'https://www.leparisien.fr/resizer/ho_GwveeYftNkLwg_cEta--5Bv4=/1200x675/cloudfront-eu-central-1.images.arcpublishing.com/leparisien/BFXJNEBN75EUNHGYJLORUC3TX4.jpg',
+            'upload_date': '20240309',
+            'view_count': int,
+            'timestamp': 1709997866,
+            'age_limit': 0,
+            'uploader_id': 'x32f7b',
+            'title': 'VIDÉO. Le «\xa0véloto\xa0», la voiture à pédales qui aimerait se faire une place sur les routes',
+            'duration': 428.0,
+            'description': 'À bord du « véloto », l’alternative à la voiture pour la campagne',
+            'tags': ['biclou', 'vélo', 'véloto', 'campagne', 'voiture', 'environnement', 'véhicules intermédiaires'],
+        },
     }]
     _GEO_BYPASS = False
     _COMMON_MEDIA_FIELDS = '''description
@@ -232,16 +297,35 @@ def _extract_embed_urls(cls, url, webpage):
         for mobj in re.finditer(
                 r'(?s)DM\.player\([^,]+,\s*{.*?video[\'"]?\s*:\s*["\']?(?P<id>[0-9a-zA-Z]+).+?}\s*\);', webpage):
             yield from 'https://www.dailymotion.com/embed/video/' + mobj.group('id')
+        for mobj in re.finditer(
+                r'(?s)<script [^>]*\bsrc=(["\'])(?:https?:)?//[\w-]+\.dailymotion\.com/player/(?:(?!\1).)+\1[^>]*>', webpage):
+            attrs = extract_attributes(mobj.group(0))
+            player_url = url_or_none(attrs.get('src'))
+            if not player_url:
+                continue
+            player_url = player_url.replace('.js', '.html')
+            if player_url.startswith('//'):
+                player_url = f'https:{player_url}'
+            if video_id := attrs.get('data-video'):
+                query_string = f'video={video_id}'
+            elif playlist_id := attrs.get('data-playlist'):
+                query_string = f'playlist={playlist_id}'
+            else:
+                continue
+            yield update_url(player_url, query=query_string)
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url)
-        video_id, playlist_id = self._match_valid_url(url).groups()
+        video_id, is_playlist, playlist_id = self._match_valid_url(url).group('id', 'is_playlist', 'playlist_id')
 
-        if playlist_id:
-            if self._yes_playlist(playlist_id, video_id):
-                return self.url_result(
-                    'http://www.dailymotion.com/playlist/' + playlist_id,
-                    'DailymotionPlaylist', playlist_id)
+        if is_playlist:  # We matched the playlist query param as video_id
+            playlist_id = video_id
+            video_id = None
+
+        if self._yes_playlist(playlist_id, video_id):
+            return self.url_result(
+                f'http://www.dailymotion.com/playlist/{playlist_id}',
+                'DailymotionPlaylist', playlist_id)
 
         password = self.get_param('videopassword')
         media = self._call_api(
@@ -282,6 +366,8 @@ def _real_extract(self, url):
         title = metadata['title']
         is_live = media.get('isOnAir')
         formats = []
+        subtitles = {}
+
         for quality, media_list in metadata['qualities'].items():
             for m in media_list:
                 media_url = m.get('url')
@@ -289,8 +375,10 @@ def _real_extract(self, url):
                 if not media_url or media_type == 'application/vnd.lumberjack.manifest':
                     continue
                 if media_type == 'application/x-mpegURL':
-                    formats.extend(self._extract_m3u8_formats(
-                        media_url, video_id, 'mp4', live=is_live, m3u8_id='hls', fatal=False))
+                    fmt, subs = self._extract_m3u8_formats_and_subtitles(
+                        media_url, video_id, 'mp4', live=is_live, m3u8_id='hls', fatal=False)
+                    formats.extend(fmt)
+                    self._merge_subtitles(subs, target=subtitles)
                 else:
                     f = {
                         'url': media_url,
@@ -310,7 +398,6 @@ def _real_extract(self, url):
             if not f.get('fps') and f['format_id'].endswith('@60'):
                 f['fps'] = 60
 
-        subtitles = {}
         subtitles_data = try_get(metadata, lambda x: x['subtitles']['data'], dict) or {}
         for subtitle_lang, subtitle in subtitles_data.items():
             subtitles[subtitle_lang] = [{
@@ -447,7 +534,7 @@ def _real_extract(self, url):
 
 class DailymotionUserIE(DailymotionPlaylistBaseIE):
     IE_NAME = 'dailymotion:user'
-    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?!(?:embed|swf|#|video|playlist|search)/)(?:(?:old/)?user/)?(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?!(?:embed|swf|#|video|playlist|search|crawler)/)(?:(?:old/)?user/)?(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'https://www.dailymotion.com/user/nqtv',
         'info_dict': {