From bc4ab17b38f01000d99c5c2bedec89721fee65ec Mon Sep 17 00:00:00 2001
From: barsnick <barsnick@users.noreply.github.com>
Date: Fri, 22 Dec 2023 02:32:29 +0100
Subject: [PATCH 01/17] [cleanup] Fix spelling of `IE_NAME` (#8810)

Authored by: barsnick
---
 yt_dlp/extractor/iheart.py | 2 +-
 yt_dlp/extractor/kinja.py  | 2 +-
 yt_dlp/extractor/nba.py    | 8 ++++----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/yt_dlp/extractor/iheart.py b/yt_dlp/extractor/iheart.py
index 2c6a5b6a1d..fb6f51e2ca 100644
--- a/yt_dlp/extractor/iheart.py
+++ b/yt_dlp/extractor/iheart.py
@@ -23,7 +23,7 @@ def _extract_episode(self, episode):
 
 
 class IHeartRadioIE(IHeartRadioBaseIE):
-    IENAME = 'iheartradio'
+    IE_NAME = 'iheartradio'
     _VALID_URL = r'(?:https?://(?:www\.)?iheart\.com/podcast/[^/]+/episode/(?P<display_id>[^/?&#]+)-|iheartradio:)(?P<id>\d+)'
     _TEST = {
         'url': 'https://www.iheart.com/podcast/105-behind-the-bastards-29236323/episode/part-one-alexander-lukashenko-the-dictator-70346499/?embed=true',
diff --git a/yt_dlp/extractor/kinja.py b/yt_dlp/extractor/kinja.py
index a225d0a0d2..f4e5c4c479 100644
--- a/yt_dlp/extractor/kinja.py
+++ b/yt_dlp/extractor/kinja.py
@@ -12,7 +12,7 @@
 
 
 class KinjaEmbedIE(InfoExtractor):
-    IENAME = 'kinja:embed'
+    IE_NAME = 'kinja:embed'
     _DOMAIN_REGEX = r'''(?:[^.]+\.)?
         (?:
             avclub|
diff --git a/yt_dlp/extractor/nba.py b/yt_dlp/extractor/nba.py
index d8fc82488d..81d11e3a50 100644
--- a/yt_dlp/extractor/nba.py
+++ b/yt_dlp/extractor/nba.py
@@ -97,7 +97,7 @@ def _extract_video(self, filter_key, filter_value):
 
 
 class NBAWatchEmbedIE(NBAWatchBaseIE):
-    IENAME = 'nba:watch:embed'
+    IE_NAME = 'nba:watch:embed'
     _VALID_URL = NBAWatchBaseIE._VALID_URL_BASE + r'embed\?.*?\bid=(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://watch.nba.com/embed?id=659395',
@@ -339,7 +339,7 @@ def _real_extract(self, url):
 
 
 class NBAEmbedIE(NBABaseIE):
-    IENAME = 'nba:embed'
+    IE_NAME = 'nba:embed'
     _VALID_URL = r'https?://secure\.nba\.com/assets/amp/include/video/(?:topI|i)frame\.html\?.*?\bcontentId=(?P<id>[^?#&]+)'
     _TESTS = [{
         'url': 'https://secure.nba.com/assets/amp/include/video/topIframe.html?contentId=teams/bulls/2020/12/04/3478774/1607105587854-20201204_SCHEDULE_RELEASE_FINAL_DRUPAL-3478774&team=bulls&adFree=false&profile=71&videoPlayerName=TAMPCVP&baseUrl=&videoAdsection=nba.com_mobile_web_teamsites_chicagobulls&ampEnv=',
@@ -361,7 +361,7 @@ def _real_extract(self, url):
 
 
 class NBAIE(NBABaseIE):
-    IENAME = 'nba'
+    IE_NAME = 'nba'
     _VALID_URL = NBABaseIE._VALID_URL_BASE + '(?!%s)video/(?P<id>(?:[^/]+/)*[^/?#&]+)' % NBABaseIE._CHANNEL_PATH_REGEX
     _TESTS = [{
         'url': 'https://www.nba.com/bulls/video/teams/bulls/2020/12/04/3478774/1607105587854-20201204schedulereleasefinaldrupal-3478774',
@@ -388,7 +388,7 @@ def _extract_url_results(self, team, content_id):
 
 
 class NBAChannelIE(NBABaseIE):
-    IENAME = 'nba:channel'
+    IE_NAME = 'nba:channel'
     _VALID_URL = NBABaseIE._VALID_URL_BASE + '(?:%s)/(?P<id>[^/?#&]+)' % NBABaseIE._CHANNEL_PATH_REGEX
     _TESTS = [{
         'url': 'https://www.nba.com/blazers/video/channel/summer_league',

From 0d531c35eca4c2eb36e160530a7a333edbc727cc Mon Sep 17 00:00:00 2001
From: Nicolas Dato <67328748+nicodato@users.noreply.github.com>
Date: Fri, 22 Dec 2023 18:52:07 -0300
Subject: [PATCH 02/17] [ie/RudoVideo] Add extractor (#8664)

Authored by: nicodato
---
 yt_dlp/extractor/_extractors.py |   1 +
 yt_dlp/extractor/rudovideo.py   | 135 ++++++++++++++++++++++++++++++++
 2 files changed, 136 insertions(+)
 create mode 100644 yt_dlp/extractor/rudovideo.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index d5f030c6b0..5c34bb7f4b 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1647,6 +1647,7 @@
     RumbleIE,
     RumbleChannelIE,
 )
+from .rudovideo import RudoVideoIE
 from .rutube import (
     RutubeIE,
     RutubeChannelIE,
diff --git a/yt_dlp/extractor/rudovideo.py b/yt_dlp/extractor/rudovideo.py
new file mode 100644
index 0000000000..1b8595593d
--- /dev/null
+++ b/yt_dlp/extractor/rudovideo.py
@@ -0,0 +1,135 @@
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    determine_ext,
+    js_to_json,
+    traverse_obj,
+    update_url_query,
+    url_or_none,
+)
+
+
+class RudoVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://rudo\.video/(?P<type>vod|podcast|live)/(?P<id>[^/?&#]+)'
+    _EMBED_REGEX = [r'<iframe[^>]+src=[\'"](?P<url>(?:https?:)//rudo\.video/(?:vod|podcast|live)/[^\'"]+)']
+    _TESTS = [{
+        'url': 'https://rudo.video/podcast/cz2wrUy8l0o',
+        'md5': '28ed82b477708dc5e12e072da2449221',
+        'info_dict': {
+            'id': 'cz2wrUy8l0o',
+            'title': 'Diego Cabot',
+            'ext': 'mp4',
+            'thumbnail': r're:^(?:https?:)?//.*\.(png|jpg)$',
+        },
+    }, {
+        'url': 'https://rudo.video/podcast/bQkt07',
+        'md5': '36b22a9863de0f47f00fc7532a32a898',
+        'info_dict': {
+            'id': 'bQkt07',
+            'title': 'Tubular Bells',
+            'ext': 'mp4',
+            'thumbnail': r're:^(?:https?:)?//.*\.(png|jpg)$',
+        },
+    }, {
+        'url': 'https://rudo.video/podcast/b42ZUznHX0',
+        'md5': 'b91c70d832938871367f8ad10c895821',
+        'info_dict': {
+            'id': 'b42ZUznHX0',
+            'title': 'Columna Ruperto Concha',
+            'ext': 'mp3',
+            'thumbnail': r're:^(?:https?:)?//.*\.(png|jpg)$',
+        },
+    }, {
+        'url': 'https://rudo.video/vod/bN5AaJ',
+        'md5': '01324a329227e2591530ecb4f555c881',
+        'info_dict': {
+            'id': 'bN5AaJ',
+            'title': 'Ucrania 19.03',
+            'creator': 'La Tercera',
+            'ext': 'mp4',
+            'thumbnail': r're:^(?:https?:)?//.*\.(png|jpg)$',
+        },
+    }, {
+        'url': 'https://rudo.video/live/bbtv',
+        'info_dict': {
+            'id': 'bbtv',
+            'ext': 'mp4',
+            'creator': 'BioBioTV',
+            'live_status': 'is_live',
+            'title': r're:^LIVE BBTV\s\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}$',
+            'thumbnail': r're:^(?:https?:)?//.*\.(png|jpg)$',
+        },
+    }, {
+        'url': 'https://rudo.video/live/c13',
+        'info_dict': {
+            'id': 'c13',
+            'title': 'CANAL13',
+            'ext': 'mp4',
+        },
+        'skip': 'Geo-restricted to Chile',
+    }, {
+        'url': 'https://rudo.video/live/t13-13cl',
+        'info_dict': {
+            'id': 't13-13cl',
+            'title': 'T13',
+            'ext': 'mp4',
+        },
+        'skip': 'Geo-restricted to Chile',
+    }]
+
+    def _real_extract(self, url):
+        video_id, type_ = self._match_valid_url(url).group('id', 'type')
+        is_live = type_ == 'live'
+
+        webpage = self._download_webpage(url, video_id)
+        if 'Streaming is not available in your area' in webpage:
+            self.raise_geo_restricted()
+
+        media_url = (
+            self._search_regex(
+                r'var\s+streamURL\s*=\s*[\'"]([^?\'"]+)', webpage, 'stream url', default=None)
+            # Source URL must be used only if streamURL is unavailable
+            or self._search_regex(
+                r'<source[^>]+src=[\'"]([^\'"]+)', webpage, 'source url', default=None))
+        if not media_url:
+            youtube_url = self._search_regex(r'file:\s*[\'"]((?:https?:)//(?:www\.)?youtube\.com[^\'"]+)',
+                                             webpage, 'youtube url', default=None)
+            if youtube_url:
+                return self.url_result(youtube_url, 'Youtube')
+            raise ExtractorError('Unable to extract stream url')
+
+        token_array = self._search_json(
+            r'<script>var\s+_\$_[a-zA-Z0-9]+\s*=', webpage, 'access token array', video_id,
+            contains_pattern=r'\[(?s:.+)\]', default=None, transform_source=js_to_json)
+        if token_array:
+            token_url = traverse_obj(token_array, (..., {url_or_none}), get_all=False)
+            if not token_url:
+                raise ExtractorError('Invalid access token array')
+            access_token = self._download_json(
+                token_url, video_id, note='Downloading access token')['data']['authToken']
+            media_url = update_url_query(media_url, {'auth-token': access_token})
+
+        ext = determine_ext(media_url)
+        if ext == 'm3u8':
+            formats = self._extract_m3u8_formats(media_url, video_id, live=is_live)
+        elif ext == 'mp3':
+            formats = [{
+                'url': media_url,
+                'vcodec': 'none',
+            }]
+        else:
+            formats = [{'url': media_url}]
+
+        return {
+            'id': video_id,
+            'title': (self._search_regex(r'var\s+titleVideo\s*=\s*[\'"]([^\'"]+)',
+                                         webpage, 'title', default=None)
+                      or self._og_search_title(webpage)),
+            'creator': self._search_regex(r'var\s+videoAuthor\s*=\s*[\'"]([^?\'"]+)',
+                                          webpage, 'videoAuthor', default=None),
+            'thumbnail': (self._search_regex(r'var\s+posterIMG\s*=\s*[\'"]([^?\'"]+)',
+                                             webpage, 'thumbnail', default=None)
+                          or self._og_search_thumbnail(webpage)),
+            'formats': formats,
+            'is_live': is_live,
+        }

From 13b3cb3c2b7169a1e17d6fc62593bf744170521c Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sun, 24 Dec 2023 00:11:10 +0100
Subject: [PATCH 03/17] [ci] Run core tests only for core changes (#8841)

Authored by: Grub4K
---
 .github/workflows/core.yml | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index b1ae4ae767..b22adb1b9d 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -1,5 +1,25 @@
 name: Core Tests
-on: [push, pull_request]
+on:
+  push:
+    paths:
+      - .github/**
+      - devscripts/**
+      - test/**
+      - yt_dlp/**.py
+      - '!yt_dlp/extractor/*.py'
+      - yt_dlp/extractor/__init__.py
+      - yt_dlp/extractor/common.py
+      - yt_dlp/extractor/extractors.py
+  pull_request:
+    paths:
+      - .github/**
+      - devscripts/**
+      - test/**
+      - yt_dlp/**.py
+      - '!yt_dlp/extractor/*.py'
+      - yt_dlp/extractor/__init__.py
+      - yt_dlp/extractor/common.py
+      - yt_dlp/extractor/extractors.py
 permissions:
   contents: read
 

From f45c4efcd928a173e1300a8f1ce4258e70c969b1 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 24 Dec 2023 10:33:16 -0600
Subject: [PATCH 04/17] [ie/litv] Fix premium content extraction (#8842)

Closes #8654
Authored by: bashonly
---
 yt_dlp/extractor/litv.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/yt_dlp/extractor/litv.py b/yt_dlp/extractor/litv.py
index 2c7c7175ea..1003fb2fd4 100644
--- a/yt_dlp/extractor/litv.py
+++ b/yt_dlp/extractor/litv.py
@@ -6,6 +6,7 @@
     int_or_none,
     smuggle_url,
     traverse_obj,
+    try_call,
     unsmuggle_url,
 )
 
@@ -96,13 +97,22 @@ def _real_extract(self, url):
             r'uiHlsUrl\s*=\s*testBackendData\(([^;]+)\);',
             webpage, 'video data', default='{}'), video_id)
         if not video_data:
-            payload = {
-                'assetId': program_info['assetId'],
-                'watchDevices': program_info['watchDevices'],
-                'contentType': program_info['contentType'],
-            }
+            payload = {'assetId': program_info['assetId']}
+            puid = try_call(lambda: self._get_cookies('https://www.litv.tv/')['PUID'].value)
+            if puid:
+                payload.update({
+                    'type': 'auth',
+                    'puid': puid,
+                })
+                endpoint = 'getUrl'
+            else:
+                payload.update({
+                    'watchDevices': program_info['watchDevices'],
+                    'contentType': program_info['contentType'],
+                })
+                endpoint = 'getMainUrlNoAuth'
             video_data = self._download_json(
-                'https://www.litv.tv/vod/ajax/getMainUrlNoAuth', video_id,
+                f'https://www.litv.tv/vod/ajax/{endpoint}', video_id,
                 data=json.dumps(payload).encode('utf-8'),
                 headers={'Content-Type': 'application/json'})
 

From 50eaea9fd7787546b53660e736325fa31c77765d Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 24 Dec 2023 10:40:03 -0600
Subject: [PATCH 05/17] [ie/instagram] Fix stories extraction (#8843)

Closes #8290
Authored by: bashonly
---
 yt_dlp/extractor/instagram.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/instagram.py b/yt_dlp/extractor/instagram.py
index bfc4b7b888..dbaa332c2a 100644
--- a/yt_dlp/extractor/instagram.py
+++ b/yt_dlp/extractor/instagram.py
@@ -10,6 +10,7 @@
     ExtractorError,
     decode_base_n,
     encode_base_n,
+    filter_dict,
     float_or_none,
     format_field,
     get_element_by_attribute,
@@ -703,28 +704,31 @@ def _real_extract(self, url):
         user_info = self._search_json(r'"user":', story_info, 'user info', story_id, fatal=False)
         if not user_info:
             self.raise_login_required('This content is unreachable')
-        user_id = user_info.get('id')
 
+        user_id = traverse_obj(user_info, 'pk', 'id', expected_type=str)
         story_info_url = user_id if username != 'highlights' else f'highlight:{story_id}'
+        if not story_info_url:  # user id is only mandatory for non-highlights
+            raise ExtractorError('Unable to extract user id')
+
         videos = traverse_obj(self._download_json(
             f'{self._API_BASE_URL}/feed/reels_media/?reel_ids={story_info_url}',
             story_id, errnote=False, fatal=False, headers=self._API_HEADERS), 'reels')
         if not videos:
             self.raise_login_required('You need to log in to access this content')
 
-        full_name = traverse_obj(videos, (f'highlight:{story_id}', 'user', 'full_name'), (str(user_id), 'user', 'full_name'))
+        full_name = traverse_obj(videos, (f'highlight:{story_id}', 'user', 'full_name'), (user_id, 'user', 'full_name'))
         story_title = traverse_obj(videos, (f'highlight:{story_id}', 'title'))
         if not story_title:
             story_title = f'Story by {username}'
 
-        highlights = traverse_obj(videos, (f'highlight:{story_id}', 'items'), (str(user_id), 'items'))
+        highlights = traverse_obj(videos, (f'highlight:{story_id}', 'items'), (user_id, 'items'))
         info_data = []
         for highlight in highlights:
             highlight_data = self._extract_product(highlight)
             if highlight_data.get('formats'):
                 info_data.append({
-                    **highlight_data,
                     'uploader': full_name,
                     'uploader_id': user_id,
+                    **filter_dict(highlight_data),
                 })
         return self.playlist_result(info_data, playlist_id=story_id, playlist_title=story_title)

From e7d22348e77367740da78a3db27167ecf894b7c9 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 24 Dec 2023 10:40:50 -0600
Subject: [PATCH 06/17] [ie/twitter] Prioritize m3u8 formats (#8826)

Closes #8117
Authored by: bashonly
---
 yt_dlp/extractor/twitter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index 932b478d44..5e8504aa15 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -1416,8 +1416,8 @@ def add_thumbnail(name, size):
                 'thumbnails': thumbnails,
                 'view_count': traverse_obj(media, ('mediaStats', 'viewCount', {int_or_none})),  # No longer available
                 'duration': float_or_none(traverse_obj(media, ('video_info', 'duration_millis')), 1000),
-                # The codec of http formats are unknown
-                '_format_sort_fields': ('res', 'br', 'size', 'proto'),
+                # Prioritize m3u8 formats for compat, see https://github.com/yt-dlp/yt-dlp/issues/8117
+                '_format_sort_fields': ('res', 'proto:m3u8', 'br', 'size'),  # http format codec is unknown
             }
 
         def extract_from_card_info(card):

From 116c268438ea4d3738f6fa502c169081ca8f0ee7 Mon Sep 17 00:00:00 2001
From: bashonly <88596187+bashonly@users.noreply.github.com>
Date: Sun, 24 Dec 2023 10:41:28 -0600
Subject: [PATCH 07/17] [ie/twitter] Work around API rate-limit (#8825)

Closes #8762
Authored by: bashonly
---
 yt_dlp/extractor/twitter.py | 73 +++++++++++++++++++++----------------
 1 file changed, 42 insertions(+), 31 deletions(-)

diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index 5e8504aa15..c3a6e406c1 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -10,6 +10,7 @@
     compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
 )
+from ..networking.exceptions import HTTPError
 from ..utils import (
     ExtractorError,
     dict_get,
@@ -1317,41 +1318,51 @@ def _build_graphql_query(self, media_id):
             }
         }
 
-    def _extract_status(self, twid):
-        if self.is_logged_in or self._selected_api == 'graphql':
-            status = self._graphql_to_legacy(self._call_graphql_api(self._GRAPHQL_ENDPOINT, twid), twid)
-
-        elif self._selected_api == 'legacy':
-            status = self._call_api(f'statuses/show/{twid}.json', twid, {
-                'cards_platform': 'Web-12',
-                'include_cards': 1,
-                'include_reply_count': 1,
-                'include_user_entities': 0,
-                'tweet_mode': 'extended',
+    def _call_syndication_api(self, twid):
+        self.report_warning(
+            'Not all metadata or media is available via syndication endpoint', twid, only_once=True)
+        status = self._download_json(
+            'https://cdn.syndication.twimg.com/tweet-result', twid, 'Downloading syndication JSON',
+            headers={'User-Agent': 'Googlebot'}, query={
+                'id': twid,
+                # TODO: token = ((Number(twid) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, '')
+                'token': ''.join(random.choices('123456789abcdefghijklmnopqrstuvwxyz', k=10)),
             })
+        if not status:
+            raise ExtractorError('Syndication endpoint returned empty JSON response')
+        # Transform the result so its structure matches that of legacy/graphql
+        media = []
+        for detail in traverse_obj(status, ((None, 'quoted_tweet'), 'mediaDetails', ..., {dict})):
+            detail['id_str'] = traverse_obj(detail, (
+                'video_info', 'variants', ..., 'url', {self._MEDIA_ID_RE.search}, 1), get_all=False) or twid
+            media.append(detail)
+        status['extended_entities'] = {'media': media}
 
-        elif self._selected_api == 'syndication':
-            self.report_warning(
-                'Not all metadata or media is available via syndication endpoint', twid, only_once=True)
-            status = self._download_json(
-                'https://cdn.syndication.twimg.com/tweet-result', twid, 'Downloading syndication JSON',
-                headers={'User-Agent': 'Googlebot'}, query={
-                    'id': twid,
-                    # TODO: token = ((Number(twid) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, '')
-                    'token': ''.join(random.choices('123456789abcdefghijklmnopqrstuvwxyz', k=10)),
+        return status
+
+    def _extract_status(self, twid):
+        if self._selected_api not in ('graphql', 'legacy', 'syndication'):
+            raise ExtractorError(f'{self._selected_api!r} is not a valid API selection', expected=True)
+
+        try:
+            if self.is_logged_in or self._selected_api == 'graphql':
+                status = self._graphql_to_legacy(self._call_graphql_api(self._GRAPHQL_ENDPOINT, twid), twid)
+            elif self._selected_api == 'legacy':
+                status = self._call_api(f'statuses/show/{twid}.json', twid, {
+                    'cards_platform': 'Web-12',
+                    'include_cards': 1,
+                    'include_reply_count': 1,
+                    'include_user_entities': 0,
+                    'tweet_mode': 'extended',
                 })
-            if not status:
-                raise ExtractorError('Syndication endpoint returned empty JSON response')
-            # Transform the result so its structure matches that of legacy/graphql
-            media = []
-            for detail in traverse_obj(status, ((None, 'quoted_tweet'), 'mediaDetails', ..., {dict})):
-                detail['id_str'] = traverse_obj(detail, (
-                    'video_info', 'variants', ..., 'url', {self._MEDIA_ID_RE.search}, 1), get_all=False) or twid
-                media.append(detail)
-            status['extended_entities'] = {'media': media}
+        except ExtractorError as e:
+            if not isinstance(e.cause, HTTPError) or not e.cause.status == 429:
+                raise
+            self.report_warning('Rate-limit exceeded; falling back to syndication endpoint')
+            status = self._call_syndication_api(twid)
 
-        else:
-            raise ExtractorError(f'"{self._selected_api}" is not a valid API selection', expected=True)
+        if self._selected_api == 'syndication':
+            status = self._call_syndication_api(twid)
 
         return traverse_obj(status, 'retweeted_status', None, expected_type=dict) or {}
 

From 00cdda4f6fe18712ced13dbc64b7ea10f323e268 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sun, 24 Dec 2023 22:09:01 +0100
Subject: [PATCH 08/17] [core] Fix format selection parse error for CPython
 3.12 (#8797)

Authored by: Grub4K
---
 test/test_YoutubeDL.py |  2 ++
 yt_dlp/YoutubeDL.py    | 11 +++++++++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 48c710e00c..8bff083145 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -140,6 +140,8 @@ def test(inp, *expected, multi=False):
         test('example-with-dashes', 'example-with-dashes')
         test('all', '2', '47', '45', 'example-with-dashes', '35')
         test('mergeall', '2+47+45+example-with-dashes+35', multi=True)
+        # See: https://github.com/yt-dlp/yt-dlp/pulls/8797
+        test('7_a/worst', '35')
 
     def test_format_selection_audio(self):
         formats = [
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 0c07866e49..5e28fd0e21 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -2465,9 +2465,16 @@ def final_selector(ctx):
                 return selector_function(ctx_copy)
             return final_selector
 
-        stream = io.BytesIO(format_spec.encode())
+        # HACK: Python 3.12 changed the underlying parser, rendering '7_a' invalid
+        #       Prefix numbers with random letters to avoid it being classified as a number
+        #       See: https://github.com/yt-dlp/yt-dlp/pulls/8797
+        # TODO: Implement parser not reliant on tokenize.tokenize
+        prefix = ''.join(random.choices(string.ascii_letters, k=32))
+        stream = io.BytesIO(re.sub(r'\d[_\d]*', rf'{prefix}\g<0>', format_spec).encode())
         try:
-            tokens = list(_remove_unused_ops(tokenize.tokenize(stream.readline)))
+            tokens = list(_remove_unused_ops(
+                token._replace(string=token.string.replace(prefix, ''))
+                for token in tokenize.tokenize(stream.readline)))
         except tokenize.TokenError:
             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
 

From 1f8bd8eba82ba10ddb49ee7cc0be4540dab103d5 Mon Sep 17 00:00:00 2001
From: Lars Strojny <lars@strojny.net>
Date: Sun, 24 Dec 2023 23:38:21 +0100
Subject: [PATCH 09/17] [ie/ARDBetaMediathek] Fix series extraction (#8687)

Closes #7666
Authored by: lstrojny
---
 yt_dlp/extractor/ard.py | 50 ++++++++++++++++++++++-------------------
 1 file changed, 27 insertions(+), 23 deletions(-)

diff --git a/yt_dlp/extractor/ard.py b/yt_dlp/extractor/ard.py
index ca1faa7d0b..8ac926c917 100644
--- a/yt_dlp/extractor/ard.py
+++ b/yt_dlp/extractor/ard.py
@@ -292,7 +292,7 @@ class ARDIE(InfoExtractor):
     _TESTS = [{
         # available till 7.12.2023
         'url': 'https://www.daserste.de/information/talk/maischberger/videos/maischberger-video-424.html',
-        'md5': 'a438f671e87a7eba04000336a119ccc4',
+        'md5': '94812e6438488fb923c361a44469614b',
         'info_dict': {
             'id': 'maischberger-video-424',
             'display_id': 'maischberger-video-424',
@@ -403,26 +403,25 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
     _VALID_URL = r'''(?x)https://
         (?:(?:beta|www)\.)?ardmediathek\.de/
         (?:(?P<client>[^/]+)/)?
-        (?:player|live|video|(?P<playlist>sendung|sammlung))/
+        (?:player|live|video|(?P<playlist>sendung|serie|sammlung))/
         (?:(?P<display_id>(?(playlist)[^?#]+?|[^?#]+))/)?
         (?P<id>(?(playlist)|Y3JpZDovL)[a-zA-Z0-9]+)
         (?(playlist)/(?P<season>\d+)?/?(?:[?#]|$))'''
 
     _TESTS = [{
-        'url': 'https://www.ardmediathek.de/video/filme-im-mdr/wolfsland-die-traurigen-schwestern/mdr-fernsehen/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy8xZGY0ZGJmZS00ZWQwLTRmMGItYjhhYy0wOGQ4ZmYxNjVhZDI',
-        'md5': '3fd5fead7a370a819341129c8d713136',
+        'url': 'https://www.ardmediathek.de/video/filme-im-mdr/liebe-auf-vier-pfoten/mdr-fernsehen/Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MjIwOTEtNDAyNTM0',
+        'md5': 'b6e8ab03f2bcc6e1f9e6cef25fcc03c4',
         'info_dict': {
-            'display_id': 'filme-im-mdr/wolfsland-die-traurigen-schwestern/mdr-fernsehen',
-            'id': '12172961',
-            'title': 'Wolfsland - Die traurigen Schwestern',
-            'description': r're:^Als der Polizeiobermeister Raaben',
-            'duration': 5241,
-            'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:efa186f7b0054957',
-            'timestamp': 1670710500,
-            'upload_date': '20221210',
+            'display_id': 'filme-im-mdr/liebe-auf-vier-pfoten/mdr-fernsehen',
+            'id': '12939099',
+            'title': 'Liebe auf vier Pfoten',
+            'description': r're:^Claudia Schmitt, Anwältin in Salzburg',
+            'duration': 5222,
+            'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:aee7cbf8f06de976?w=960&ch=ae4d0f2ee47d8b9b',
+            'timestamp': 1701343800,
+            'upload_date': '20231130',
             'ext': 'mp4',
-            'age_limit': 12,
-            'episode': 'Wolfsland - Die traurigen Schwestern',
+            'episode': 'Liebe auf vier Pfoten',
             'series': 'Filme im MDR'
         },
     }, {
@@ -454,7 +453,7 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
             'duration': 915,
             'episode': 'tagesschau, 20:00 Uhr',
             'series': 'tagesschau',
-            'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:fbb21142783b0a49',
+            'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:fbb21142783b0a49?w=960&ch=ee69108ae344f678',
         },
     }, {
         'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
@@ -475,6 +474,10 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
         # playlist of type 'sendung'
         'url': 'https://www.ardmediathek.de/ard/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw/',
         'only_matching': True,
+    }, {
+        # playlist of type 'serie'
+        'url': 'https://www.ardmediathek.de/serie/nachtstreife/staffel-1/Y3JpZDovL3N3ci5kZS9zZGIvc3RJZC8xMjQy/1',
+        'only_matching': True,
     }, {
         # playlist of type 'sammlung'
         'url': 'https://www.ardmediathek.de/ard/sammlung/team-muenster/5JpTzLSbWUAK8184IOvEir/',
@@ -487,10 +490,11 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
         'only_matching': True,
     }]
 
-    def _ARD_load_playlist_snipped(self, playlist_id, display_id, client, mode, pageNumber):
+    def _ARD_load_playlist_snippet(self, playlist_id, display_id, client, mode, page_number):
         """ Query the ARD server for playlist information
         and returns the data in "raw" format """
-        if mode == 'sendung':
+        assert mode in ('sendung', 'serie', 'sammlung')
+        if mode in ('sendung', 'serie'):
             graphQL = json.dumps({
                 'query': '''{
                     showPage(
@@ -507,7 +511,7 @@ def _ARD_load_playlist_snipped(self, playlist_id, display_id, client, mode, page
                             links { target { id href title } }
                             type
                         }
-                    }}''' % (client, playlist_id, pageNumber),
+                    }}''' % (client, playlist_id, page_number),
             }).encode()
         else:  # mode == 'sammlung'
             graphQL = json.dumps({
@@ -528,7 +532,7 @@ def _ARD_load_playlist_snipped(self, playlist_id, display_id, client, mode, page
                                 type
                             }
                         }
-                    }}''' % (client, playlist_id, pageNumber),
+                    }}''' % (client, playlist_id, page_number),
             }).encode()
         # Ressources for ARD graphQL debugging:
         # https://api-test.ardmediathek.de/public-gateway
@@ -538,7 +542,7 @@ def _ARD_load_playlist_snipped(self, playlist_id, display_id, client, mode, page
             data=graphQL,
             headers={'Content-Type': 'application/json'})['data']
         # align the structure of the returned data:
-        if mode == 'sendung':
+        if mode in ('sendung', 'serie'):
             show_page = show_page['showPage']
         else:  # mode == 'sammlung'
             show_page = show_page['morePage']['widget']
@@ -546,12 +550,12 @@ def _ARD_load_playlist_snipped(self, playlist_id, display_id, client, mode, page
 
     def _ARD_extract_playlist(self, url, playlist_id, display_id, client, mode):
         """ Collects all playlist entries and returns them as info dict.
-        Supports playlists of mode 'sendung' and 'sammlung', and also nested
-        playlists. """
+        Supports playlists of mode 'sendung', 'serie', and 'sammlung',
+        as well as nested playlists. """
         entries = []
         pageNumber = 0
         while True:  # iterate by pageNumber
-            show_page = self._ARD_load_playlist_snipped(
+            show_page = self._ARD_load_playlist_snippet(
                 playlist_id, display_id, client, mode, pageNumber)
             for teaser in show_page['teasers']:  # process playlist items
                 if '/compilation/' in teaser['links']['target']['href']:

From c39358a54bc6675ae0c50b81024e5a086e41656a Mon Sep 17 00:00:00 2001
From: kclauhk <78251477+kclauhk@users.noreply.github.com>
Date: Mon, 25 Dec 2023 06:43:35 +0800
Subject: [PATCH 10/17] [ie/Facebook] Fix Memories extraction (#8681)

- Support group /posts/ URLs
- Raise a proper error message if no formats are found

Closes #8669
Authored by: kclauhk
---
 yt_dlp/extractor/facebook.py | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index 58162cc5fc..a07a0d344d 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -52,7 +52,7 @@ class FacebookIE(InfoExtractor):
                             )\?(?:.*?)(?:v|video_id|story_fbid)=|
                             [^/]+/videos/(?:[^/]+/)?|
                             [^/]+/posts/|
-                            groups/[^/]+/permalink/|
+                            groups/[^/]+/(?:permalink|posts)/|
                             watchparty/
                         )|
                     facebook:
@@ -232,6 +232,21 @@ class FacebookIE(InfoExtractor):
             'uploader_id': '100013949973717',
         },
         'skip': 'Requires logging in',
+    }, {
+        # data.node.comet_sections.content.story.attachments[].throwbackStyles.attachment_target_renderer.attachment.target.attachments[].styles.attachment.media
+        'url': 'https://www.facebook.com/groups/1645456212344334/posts/3737828833107051/',
+        'info_dict': {
+            'id': '1569199726448814',
+            'ext': 'mp4',
+            'title': 'Pence MUST GO!',
+            'description': 'Vickie Gentry shared a memory.',
+            'timestamp': 1511548260,
+            'upload_date': '20171124',
+            'uploader': 'Vickie Gentry',
+            'uploader_id': 'pfbid0FuZhHCeWDAxWxEbr3yKPFaRstXvRxgsp9uCPG6GjD4J2AitB35NUAuJ4Q75KcjiDl',
+            'thumbnail': r're:^https?://.*',
+            'duration': 148.435,
+        },
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -612,9 +627,11 @@ def parse_attachment(attachment, key='media'):
                 nodes = variadic(traverse_obj(data, 'nodes', 'node') or [])
                 attachments = traverse_obj(nodes, (
                     ..., 'comet_sections', 'content', 'story', (None, 'attached_story'), 'attachments',
-                    ..., ('styles', 'style_type_renderer'), 'attachment'), expected_type=dict) or []
+                    ..., ('styles', 'style_type_renderer', ('throwbackStyles', 'attachment_target_renderer')),
+                    'attachment', {dict}))
                 for attachment in attachments:
-                    ns = try_get(attachment, lambda x: x['all_subattachments']['nodes'], list) or []
+                    ns = traverse_obj(attachment, ('all_subattachments', 'nodes', ..., {dict}),
+                                      ('target', 'attachments', ..., 'styles', 'attachment', {dict}))
                     for n in ns:
                         parse_attachment(n)
                     parse_attachment(attachment)
@@ -637,7 +654,7 @@ def parse_attachment(attachment, key='media'):
                 if len(entries) > 1:
                     return self.playlist_result(entries, video_id)
 
-                video_info = entries[0]
+                video_info = entries[0] if entries else {'id': video_id}
                 webpage_info = extract_metadata(webpage)
                 # honor precise duration in video info
                 if video_info.get('duration'):

From 65de7d204ce88c0225df1321060304baab85dbd8 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Tue, 26 Dec 2023 01:40:24 +0100
Subject: [PATCH 11/17] Update to ytdl-commit-be008e6 (#8836)

- [utils] Make restricted filenames ignore some Unicode categories (by dirkf)
- [ie/telewebion] Fix extraction (by Grub4K)
- [ie/imgur] Overhaul extractor (by bashonly, Grub4K)
- [ie/EpidemicSound] Add extractor (by Grub4K)

Authored by: bashonly, dirkf, Grub4K

Co-authored-by: bashonly <bashonly@protonmail.com>
---
 yt_dlp/extractor/_extractors.py   |   1 +
 yt_dlp/extractor/epidemicsound.py | 107 +++++++++
 yt_dlp/extractor/imgur.py         | 364 ++++++++++++++++++++++++------
 yt_dlp/extractor/telewebion.py    | 157 +++++++++----
 yt_dlp/utils/_utils.py            |   2 +-
 5 files changed, 518 insertions(+), 113 deletions(-)
 create mode 100644 yt_dlp/extractor/epidemicsound.py

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 5c34bb7f4b..62103f13c1 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -548,6 +548,7 @@
     EpiconIE,
     EpiconSeriesIE,
 )
+from .epidemicsound import EpidemicSoundIE
 from .eplus import EplusIbIE
 from .epoch import EpochIE
 from .eporner import EpornerIE
diff --git a/yt_dlp/extractor/epidemicsound.py b/yt_dlp/extractor/epidemicsound.py
new file mode 100644
index 0000000000..0d81b11c85
--- /dev/null
+++ b/yt_dlp/extractor/epidemicsound.py
@@ -0,0 +1,107 @@
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    orderedSet,
+    parse_iso8601,
+    parse_qs,
+    parse_resolution,
+    str_or_none,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class EpidemicSoundIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?epidemicsound\.com/track/(?P<id>[0-9a-zA-Z]+)'
+    _TESTS = [{
+        'url': 'https://www.epidemicsound.com/track/yFfQVRpSPz/',
+        'md5': 'd98ff2ddb49e8acab9716541cbc9dfac',
+        'info_dict': {
+            'id': '45014',
+            'display_id': 'yFfQVRpSPz',
+            'ext': 'mp3',
+            'title': 'Door Knock Door 1',
+            'alt_title': 'Door Knock Door 1',
+            'tags': ['foley', 'door', 'knock', 'glass', 'window', 'glass door knock'],
+            'categories': ['Misc. Door'],
+            'duration': 1,
+            'thumbnail': 'https://cdn.epidemicsound.com/curation-assets/commercial-release-cover-images/default-sfx/3000x3000.jpg',
+            'timestamp': 1415320353,
+            'upload_date': '20141107',
+        },
+    }, {
+        'url': 'https://www.epidemicsound.com/track/mj8GTTwsZd/',
+        'md5': 'c82b745890f9baf18dc2f8d568ee3830',
+        'info_dict': {
+            'id': '148700',
+            'display_id': 'mj8GTTwsZd',
+            'ext': 'mp3',
+            'title': 'Noplace',
+            'tags': ['liquid drum n bass', 'energetic'],
+            'categories': ['drum and bass'],
+            'duration': 237,
+            'timestamp': 1694426482,
+            'thumbnail': 'https://cdn.epidemicsound.com/curation-assets/commercial-release-cover-images/11138/3000x3000.jpg',
+            'upload_date': '20230911',
+            'release_timestamp': 1700535606,
+            'release_date': '20231121',
+        },
+    }]
+
+    @staticmethod
+    def _epidemic_parse_thumbnail(url: str):
+        if not url_or_none(url):
+            return None
+
+        return {
+            'url': url,
+            **(traverse_obj(url, ({parse_qs}, {
+                'width': ('width', 0, {int_or_none}),
+                'height': ('height', 0, {int_or_none}),
+            })) or parse_resolution(url)),
+        }
+
+    @staticmethod
+    def _epidemic_fmt_or_none(f):
+        if not f.get('format'):
+            f['format'] = f.get('format_id')
+        elif not f.get('format_id'):
+            f['format_id'] = f['format']
+        if not f['url'] or not f['format']:
+            return None
+        if f.get('format_note'):
+            f['format_note'] = f'track ID {f["format_note"]}'
+        if f['format'] != 'full':
+            f['preference'] = -2
+        return f
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        json_data = self._download_json(f'https://www.epidemicsound.com/json/track/{video_id}', video_id)
+
+        thumbnails = traverse_obj(json_data, [('imageUrl', 'cover')])
+        thumb_base_url = traverse_obj(json_data, ('coverArt', 'baseUrl', {url_or_none}))
+        if thumb_base_url:
+            thumbnails.extend(traverse_obj(json_data, (
+                'coverArt', 'sizes', ..., {thumb_base_url.__add__})))
+
+        return traverse_obj(json_data, {
+            'id': ('id', {str_or_none}),
+            'display_id': ('publicSlug', {str}),
+            'title': ('title', {str}),
+            'alt_title': ('oldTitle', {str}),
+            'duration': ('length', {float_or_none}),
+            'timestamp': ('added', {parse_iso8601}),
+            'release_timestamp': ('releaseDate', {parse_iso8601}),
+            'categories': ('genres', ..., 'tag', {str}),
+            'tags': ('metadataTags', ..., {str}),
+            'age_limit': ('isExplicit', {lambda b: 18 if b else None}),
+            'thumbnails': ({lambda _: thumbnails}, {orderedSet}, ..., {self._epidemic_parse_thumbnail}),
+            'formats': ('stems', {dict.items}, ..., {
+                'format': (0, {str_or_none}),
+                'format_note': (1, 's3TrackId', {str_or_none}),
+                'format_id': (1, 'stemType', {str}),
+                'url': (1, 'lqMp3Url', {url_or_none}),
+            }, {self._epidemic_fmt_or_none}),
+        })
diff --git a/yt_dlp/extractor/imgur.py b/yt_dlp/extractor/imgur.py
index bff6ed57f5..1fa0a2a791 100644
--- a/yt_dlp/extractor/imgur.py
+++ b/yt_dlp/extractor/imgur.py
@@ -1,99 +1,243 @@
+import functools
 import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
+    determine_ext,
+    float_or_none,
     int_or_none,
     js_to_json,
     mimetype2ext,
-    ExtractorError,
+    parse_iso8601,
+    str_or_none,
+    strip_or_none,
+    traverse_obj,
+    url_or_none,
 )
 
 
-class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?!(?:a|gallery|(?:t(?:opic)?|r)/[^/]+)/)(?P<id>[a-zA-Z0-9]+)'
+class ImgurBaseIE(InfoExtractor):
+    _CLIENT_ID = '546c25a59c58ad7'
+
+    @classmethod
+    def _imgur_result(cls, item_id):
+        return cls.url_result(f'https://imgur.com/{item_id}', ImgurIE, item_id)
+
+    def _call_api(self, endpoint, video_id, **kwargs):
+        return self._download_json(
+            f'https://api.imgur.com/post/v1/{endpoint}/{video_id}?client_id={self._CLIENT_ID}&include=media,account',
+            video_id, **kwargs)
+
+    @staticmethod
+    def get_description(s):
+        if 'Discover the magic of the internet at Imgur' in s:
+            return None
+        return s or None
+
+
+class ImgurIE(ImgurBaseIE):
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?!(?:a|gallery|t|topic|r)/)(?P<id>[a-zA-Z0-9]+)'
 
     _TESTS = [{
-        'url': 'https://i.imgur.com/A61SaA1.gifv',
+        'url': 'https://imgur.com/A61SaA1',
         'info_dict': {
             'id': 'A61SaA1',
             'ext': 'mp4',
-            'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
+            'title': 'MRW gifv is up and running without any bugs',
+            'timestamp': 1416446068,
+            'upload_date': '20141120',
+            'dislike_count': int,
+            'comment_count': int,
+            'release_timestamp': 1416446068,
+            'release_date': '20141120',
+            'like_count': int,
+            'thumbnail': 'https://i.imgur.com/A61SaA1h.jpg',
         },
     }, {
-        'url': 'https://imgur.com/A61SaA1',
+        'url': 'https://i.imgur.com/A61SaA1.gifv',
         'only_matching': True,
     }, {
         'url': 'https://i.imgur.com/crGpqCV.mp4',
         'only_matching': True,
     }, {
-        # no title
         'url': 'https://i.imgur.com/jxBXAMC.gifv',
-        'only_matching': True,
+        'info_dict': {
+            'id': 'jxBXAMC',
+            'ext': 'mp4',
+            'title': 'Fahaka puffer feeding',
+            'timestamp': 1533835503,
+            'upload_date': '20180809',
+            'release_date': '20180809',
+            'like_count': int,
+            'duration': 30.0,
+            'comment_count': int,
+            'release_timestamp': 1533835503,
+            'thumbnail': 'https://i.imgur.com/jxBXAMCh.jpg',
+            'dislike_count': int,
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        data = self._call_api('media', video_id)
+        if not traverse_obj(data, ('media', 0, (
+                ('type', {lambda t: t == 'video' or None}),
+                ('metadata', 'is_animated'))), get_all=False):
+            raise ExtractorError(f'{video_id} is not a video or animated image', expected=True)
         webpage = self._download_webpage(
-            'https://i.imgur.com/{id}.gifv'.format(id=video_id), video_id)
+            f'https://i.imgur.com/{video_id}.gifv', video_id, fatal=False) or ''
+        formats = []
 
-        width = int_or_none(self._og_search_property(
-            'video:width', webpage, default=None))
-        height = int_or_none(self._og_search_property(
-            'video:height', webpage, default=None))
+        media_fmt = traverse_obj(data, ('media', 0, {
+            'url': ('url', {url_or_none}),
+            'ext': ('ext', {str}),
+            'width': ('width', {int_or_none}),
+            'height': ('height', {int_or_none}),
+            'filesize': ('size', {int_or_none}),
+            'acodec': ('metadata', 'has_sound', {lambda b: None if b else 'none'}),
+        }))
+        media_url = media_fmt.get('url')
+        if media_url:
+            if not media_fmt.get('ext'):
+                media_fmt['ext'] = mimetype2ext(traverse_obj(
+                    data, ('media', 0, 'mime_type'))) or determine_ext(media_url)
+            if traverse_obj(data, ('media', 0, 'type')) == 'image':
+                media_fmt['acodec'] = 'none'
+                media_fmt.setdefault('preference', -10)
+            formats.append(media_fmt)
 
         video_elements = self._search_regex(
             r'(?s)<div class="video-elements">(.*?)</div>',
             webpage, 'video elements', default=None)
-        if not video_elements:
-            raise ExtractorError(
-                'No sources found for video %s. Maybe an image?' % video_id,
-                expected=True)
 
-        formats = []
-        for m in re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements):
-            formats.append({
-                'format_id': m.group('type').partition('/')[2],
-                'url': self._proto_relative_url(m.group('src')),
-                'ext': mimetype2ext(m.group('type')),
-                'width': width,
-                'height': height,
-                'http_headers': {
-                    'User-Agent': 'yt-dlp (like wget)',
-                },
-            })
+        if video_elements:
+            def og_get_size(media_type):
+                return {
+                    p: int_or_none(self._og_search_property(f'{media_type}:{p}', webpage, default=None))
+                    for p in ('width', 'height')
+                }
 
-        gif_json = self._search_regex(
-            r'(?s)var\s+videoItem\s*=\s*(\{.*?\})',
-            webpage, 'GIF code', fatal=False)
-        if gif_json:
-            gifd = self._parse_json(
-                gif_json, video_id, transform_source=js_to_json)
-            formats.append({
-                'format_id': 'gif',
-                'preference': -10,  # gifs are worse than videos
-                'width': width,
-                'height': height,
-                'ext': 'gif',
-                'acodec': 'none',
-                'vcodec': 'gif',
-                'container': 'gif',
-                'url': self._proto_relative_url(gifd['gifUrl']),
-                'filesize': gifd.get('size'),
-                'http_headers': {
-                    'User-Agent': 'yt-dlp (like wget)',
-                },
+            size = og_get_size('video')
+            if not any(size.values()):
+                size = og_get_size('image')
+
+            formats = traverse_obj(
+                re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements),
+                (..., {
+                    'format_id': ('type', {lambda s: s.partition('/')[2]}),
+                    'url': ('src', {self._proto_relative_url}),
+                    'ext': ('type', {mimetype2ext}),
+                }))
+            for f in formats:
+                f.update(size)
+
+            # We can get the original gif format from the webpage as well
+            gif_json = traverse_obj(self._search_json(
+                r'var\s+videoItem\s*=', webpage, 'GIF info', video_id,
+                transform_source=js_to_json, fatal=False), {
+                    'url': ('gifUrl', {self._proto_relative_url}),
+                    'filesize': ('size', {int_or_none}),
             })
+            if gif_json:
+                gif_json.update(size)
+                gif_json.update({
+                    'format_id': 'gif',
+                    'preference': -10,  # gifs < videos
+                    'ext': 'gif',
+                    'acodec': 'none',
+                    'vcodec': 'gif',
+                    'container': 'gif',
+                })
+                formats.append(gif_json)
+
+        search = functools.partial(self._html_search_meta, html=webpage, default=None)
+
+        twitter_fmt = {
+            'format_id': 'twitter',
+            'url': url_or_none(search('twitter:player:stream')),
+            'ext': mimetype2ext(search('twitter:player:stream:content_type')),
+            'width': int_or_none(search('twitter:width')),
+            'height': int_or_none(search('twitter:height')),
+        }
+        if twitter_fmt['url']:
+            formats.append(twitter_fmt)
+
+        if not formats:
+            self.raise_no_formats(
+                f'No sources found for video {video_id}. Maybe a plain image?', expected=True)
+        self._remove_duplicate_formats(formats)
 
         return {
+            'title': self._og_search_title(webpage, default=None),
+            'description': self.get_description(self._og_search_description(webpage, default='')),
+            **traverse_obj(data, {
+                'uploader_id': ('account_id', {lambda a: str(a) if int_or_none(a) else None}),
+                'uploader': ('account', 'username', {lambda x: strip_or_none(x) or None}),
+                'uploader_url': ('account', 'avatar_url', {url_or_none}),
+                'like_count': ('upvote_count', {int_or_none}),
+                'dislike_count': ('downvote_count', {int_or_none}),
+                'comment_count': ('comment_count', {int_or_none}),
+                'age_limit': ('is_mature', {lambda x: 18 if x else None}),
+                'timestamp': (('updated_at', 'created_at'), {parse_iso8601}),
+                'release_timestamp': ('created_at', {parse_iso8601}),
+            }, get_all=False),
+            **traverse_obj(data, ('media', 0, 'metadata', {
+                'title': ('title', {lambda x: strip_or_none(x) or None}),
+                'description': ('description', {self.get_description}),
+                'duration': ('duration', {float_or_none}),
+                'timestamp': (('updated_at', 'created_at'), {parse_iso8601}),
+                'release_timestamp': ('created_at', {parse_iso8601}),
+            }), get_all=False),
             'id': video_id,
             'formats': formats,
-            'title': self._og_search_title(webpage, default=video_id),
+            'thumbnail': url_or_none(search('thumbnailUrl')),
         }
 
 
-class ImgurGalleryIE(InfoExtractor):
+class ImgurGalleryBaseIE(ImgurBaseIE):
+    _GALLERY = True
+
+    def _real_extract(self, url):
+        gallery_id = self._match_id(url)
+
+        data = self._call_api('albums', gallery_id, fatal=False, expected_status=404)
+
+        info = traverse_obj(data, {
+            'title': ('title', {lambda x: strip_or_none(x) or None}),
+            'description': ('description', {self.get_description}),
+        })
+
+        if traverse_obj(data, 'is_album'):
+
+            def yield_media_ids():
+                for m_id in traverse_obj(data, (
+                        'media', lambda _, v: v.get('type') == 'video' or v['metadata']['is_animated'],
+                        'id', {lambda x: str_or_none(x) or None})):
+                    yield m_id
+
+            # if a gallery with exactly one video, apply album metadata to video
+            media_id = (
+                self._GALLERY
+                and traverse_obj(data, ('image_count', {lambda c: c == 1}))
+                and next(yield_media_ids(), None))
+
+            if not media_id:
+                result = self.playlist_result(
+                    map(self._imgur_result, yield_media_ids()), gallery_id)
+                result.update(info)
+                return result
+            gallery_id = media_id
+
+        result = self._imgur_result(gallery_id)
+        info['_type'] = 'url_transparent'
+        result.update(info)
+        return result
+
+
+class ImgurGalleryIE(ImgurGalleryBaseIE):
     IE_NAME = 'imgur:gallery'
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:gallery|(?:t(?:opic)?|r)/[^/]+)/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:gallery|(?:t(?:opic)?|r)/[^/?#]+)/(?P<id>[a-zA-Z0-9]+)'
 
     _TESTS = [{
         'url': 'http://imgur.com/gallery/Q95ko',
@@ -102,49 +246,121 @@ class ImgurGalleryIE(InfoExtractor):
             'title': 'Adding faces make every GIF better',
         },
         'playlist_count': 25,
+        'skip': 'Zoinks! You\'ve taken a wrong turn.',
     }, {
+        # TODO: static images - replace with animated/video gallery
         'url': 'http://imgur.com/topic/Aww/ll5Vk',
         'only_matching': True,
     }, {
         'url': 'https://imgur.com/gallery/YcAQlkx',
+        'add_ies': ['Imgur'],
         'info_dict': {
             'id': 'YcAQlkx',
             'ext': 'mp4',
             'title': 'Classic Steve Carell gif...cracks me up everytime....damn the repost downvotes....',
-        }
+            'timestamp': 1358554297,
+            'upload_date': '20130119',
+            'uploader_id': '1648642',
+            'uploader': 'wittyusernamehere',
+            'release_timestamp': 1358554297,
+            'thumbnail': 'https://i.imgur.com/YcAQlkxh.jpg',
+            'release_date': '20130119',
+            'uploader_url': 'https://i.imgur.com/u3R4I2S_d.png?maxwidth=290&fidelity=grand',
+            'comment_count': int,
+            'dislike_count': int,
+            'like_count': int,
+        },
     }, {
+        # TODO: static image - replace with animated/video gallery
         'url': 'http://imgur.com/topic/Funny/N8rOudd',
         'only_matching': True,
     }, {
         'url': 'http://imgur.com/r/aww/VQcQPhM',
-        'only_matching': True,
+        'add_ies': ['Imgur'],
+        'info_dict': {
+            'id': 'VQcQPhM',
+            'ext': 'mp4',
+            'title': 'The boss is here',
+            'timestamp': 1476494751,
+            'upload_date': '20161015',
+            'uploader_id': '19138530',
+            'uploader': 'thematrixcam',
+            'comment_count': int,
+            'dislike_count': int,
+            'uploader_url': 'https://i.imgur.com/qCjr5Pi_d.png?maxwidth=290&fidelity=grand',
+            'release_timestamp': 1476494751,
+            'like_count': int,
+            'release_date': '20161015',
+            'thumbnail': 'https://i.imgur.com/VQcQPhMh.jpg',
+        },
+    },
+        # from https://github.com/ytdl-org/youtube-dl/pull/16674
+        {
+        'url': 'https://imgur.com/t/unmuted/6lAn9VQ',
+        'info_dict': {
+            'id': '6lAn9VQ',
+            'title': 'Penguins !',
+        },
+        'playlist_count': 3,
+    }, {
+        'url': 'https://imgur.com/t/unmuted/kx2uD3C',
+        'add_ies': ['Imgur'],
+        'info_dict': {
+            'id': 'ZVMv45i',
+            'ext': 'mp4',
+            'title': 'Intruder',
+            'timestamp': 1528129683,
+            'upload_date': '20180604',
+            'release_timestamp': 1528129683,
+            'release_date': '20180604',
+            'like_count': int,
+            'dislike_count': int,
+            'comment_count': int,
+            'duration': 30.03,
+            'thumbnail': 'https://i.imgur.com/ZVMv45ih.jpg',
+        },
+    }, {
+        'url': 'https://imgur.com/t/unmuted/wXSK0YH',
+        'add_ies': ['Imgur'],
+        'info_dict': {
+            'id': 'JCAP4io',
+            'ext': 'mp4',
+            'title': 're:I got the blues$',
+            'description': 'Luka’s vocal stylings.\n\nFP edit: don’t encourage me. I’ll never stop posting Luka and friends.',
+            'timestamp': 1527809525,
+            'upload_date': '20180531',
+            'like_count': int,
+            'dislike_count': int,
+            'duration': 30.03,
+            'comment_count': int,
+            'release_timestamp': 1527809525,
+            'thumbnail': 'https://i.imgur.com/JCAP4ioh.jpg',
+            'release_date': '20180531',
+        },
     }]
 
-    def _real_extract(self, url):
-        gallery_id = self._match_id(url)
 
-        data = self._download_json(
-            'https://imgur.com/gallery/%s.json' % gallery_id,
-            gallery_id)['data']['image']
-
-        if data.get('is_album'):
-            entries = [
-                self.url_result('http://imgur.com/%s' % image['hash'], ImgurIE.ie_key(), image['hash'])
-                for image in data['album_images']['images'] if image.get('hash')]
-            return self.playlist_result(entries, gallery_id, data.get('title'), data.get('description'))
-
-        return self.url_result('http://imgur.com/%s' % gallery_id, ImgurIE.ie_key(), gallery_id)
-
-
-class ImgurAlbumIE(ImgurGalleryIE):  # XXX: Do not subclass from concrete IE
+class ImgurAlbumIE(ImgurGalleryBaseIE):
     IE_NAME = 'imgur:album'
     _VALID_URL = r'https?://(?:i\.)?imgur\.com/a/(?P<id>[a-zA-Z0-9]+)'
-
+    _GALLERY = False
     _TESTS = [{
+        # TODO: only static images - replace with animated/video gallery
         'url': 'http://imgur.com/a/j6Orj',
+        'only_matching': True,
+    },
+        # from https://github.com/ytdl-org/youtube-dl/pull/21693
+        {
+        'url': 'https://imgur.com/a/iX265HX',
         'info_dict': {
-            'id': 'j6Orj',
-            'title': 'A Literary Analysis of "Star Wars: The Force Awakens"',
+            'id': 'iX265HX',
+            'title': 'enen-no-shouboutai'
         },
-        'playlist_count': 12,
+        'playlist_count': 2,
+    }, {
+        'url': 'https://imgur.com/a/8pih2Ed',
+        'info_dict': {
+            'id': '8pih2Ed'
+        },
+        'playlist_mincount': 1,
     }]
diff --git a/yt_dlp/extractor/telewebion.py b/yt_dlp/extractor/telewebion.py
index 550549f056..9378ed0214 100644
--- a/yt_dlp/extractor/telewebion.py
+++ b/yt_dlp/extractor/telewebion.py
@@ -1,52 +1,133 @@
+from __future__ import annotations
+
+import json
+from functools import partial
+from textwrap import dedent
+
 from .common import InfoExtractor
+from ..utils import ExtractorError, format_field, int_or_none, parse_iso8601
+from ..utils.traversal import traverse_obj
+
+
+def _fmt_url(url):
+    return partial(format_field, template=url, default=None)
 
 
 class TelewebionIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?telewebion\.com/#!/episode/(?P<id>\d+)'
-
-    _TEST = {
-        'url': 'http://www.telewebion.com/#!/episode/1263668/',
+    _VALID_URL = r'https?://(?:www\.)?telewebion\.com/episode/(?P<id>(?:0x[a-fA-F\d]+|\d+))'
+    _TESTS = [{
+        'url': 'http://www.telewebion.com/episode/0x1b3139c/',
         'info_dict': {
-            'id': '1263668',
+            'id': '0x1b3139c',
             'ext': 'mp4',
-            'title': 'قرعه\u200cکشی لیگ قهرمانان اروپا',
-            'thumbnail': r're:^https?://.*\.jpg',
+            'title': 'قرعه‌کشی لیگ قهرمانان اروپا',
+            'series': '+ فوتبال',
+            'series_id': '0x1b2505c',
+            'channel': 'شبکه 3',
+            'channel_id': '0x1b1a761',
+            'channel_url': 'https://telewebion.com/live/tv3',
+            'timestamp': 1425522414,
+            'upload_date': '20150305',
+            'release_timestamp': 1425517020,
+            'release_date': '20150305',
+            'duration': 420,
             'view_count': int,
+            'tags': ['ورزشی', 'لیگ اروپا', 'اروپا'],
+            'thumbnail': 'https://static.telewebion.com/episodeImages/YjFhM2MxMDBkMDNiZTU0MjE5YjQ3ZDY0Mjk1ZDE0ZmUwZWU3OTE3OWRmMDAyODNhNzNkNjdmMWMzMWIyM2NmMA/default',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
+        'skip_download': 'm3u8',
+    }, {
+        'url': 'https://telewebion.com/episode/162175536',
+        'info_dict': {
+            'id': '0x9aa9a30',
+            'ext': 'mp4',
+            'title': 'کارما یعنی این !',
+            'series': 'پاورقی',
+            'series_id': '0x29a7426',
+            'channel': 'شبکه 2',
+            'channel_id': '0x1b1a719',
+            'channel_url': 'https://telewebion.com/live/tv2',
+            'timestamp': 1699979968,
+            'upload_date': '20231114',
+            'release_timestamp': 1699991638,
+            'release_date': '20231114',
+            'duration': 78,
+            'view_count': int,
+            'tags': ['کلیپ های منتخب', ' کلیپ طنز ', ' کلیپ سیاست ', 'پاورقی', 'ویژه فلسطین'],
+            'thumbnail': 'https://static.telewebion.com/episodeImages/871e9455-7567-49a5-9648-34c22c197f5f/default',
         },
-    }
+        'skip_download': 'm3u8',
+    }]
+
+    def _call_graphql_api(
+        self, operation, video_id, query,
+        variables: dict[str, tuple[str, str]] | None = None,
+        note='Downloading GraphQL JSON metadata',
+    ):
+        parameters = ''
+        if variables:
+            parameters = ', '.join(f'${name}: {type_}' for name, (type_, _) in variables.items())
+            parameters = f'({parameters})'
+
+        result = self._download_json('https://graph.telewebion.com/graphql', video_id, note, data=json.dumps({
+            'operationName': operation,
+            'query': f'query {operation}{parameters} @cacheControl(maxAge: 60) {{{query}\n}}\n',
+            'variables': {name: value for name, (_, value) in (variables or {}).items()}
+        }, separators=(',', ':')).encode(), headers={
+            'Content-Type': 'application/json',
+            'Accept': 'application/json',
+        })
+        if not result or traverse_obj(result, 'errors'):
+            message = ', '.join(traverse_obj(result, ('errors', ..., 'message', {str})))
+            raise ExtractorError(message or 'Unknown GraphQL API error')
+
+        return result['data']
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        if not video_id.startswith('0x'):
+            video_id = hex(int(video_id))
 
-        secure_token = self._download_webpage(
-            'http://m.s2.telewebion.com/op/op?action=getSecurityToken', video_id)
-        episode_details = self._download_json(
-            'http://m.s2.telewebion.com/op/op', video_id,
-            query={'action': 'getEpisodeDetails', 'episode_id': video_id})
+        episode_data = self._call_graphql_api('getEpisodeDetail', video_id, dedent('''
+            queryEpisode(filter: {EpisodeID: $EpisodeId}, first: 1) {
+              title
+              program {
+                ProgramID
+                title
+              }
+              image
+              view_count
+              duration
+              started_at
+              created_at
+              channel {
+                ChannelID
+                name
+                descriptor
+              }
+              tags {
+                name
+              }
+            }
+        '''), {'EpisodeId': ('[ID!]', video_id)})
 
-        m3u8_url = 'http://m.s1.telewebion.com/smil/%s.m3u8?filepath=%s&m3u8=1&secure_token=%s' % (
-            video_id, episode_details['file_path'], secure_token)
-        formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, ext='mp4', m3u8_id='hls')
-
-        picture_paths = [
-            episode_details.get('picture_path'),
-            episode_details.get('large_picture_path'),
-        ]
-
-        thumbnails = [{
-            'url': picture_path,
-            'preference': idx,
-        } for idx, picture_path in enumerate(picture_paths) if picture_path is not None]
-
-        return {
-            'id': video_id,
-            'title': episode_details['title'],
-            'formats': formats,
-            'thumbnails': thumbnails,
-            'view_count': episode_details.get('view_count'),
-        }
+        info_dict = traverse_obj(episode_data, ('queryEpisode', 0, {
+            'title': ('title', {str}),
+            'view_count': ('view_count', {int_or_none}),
+            'duration': ('duration', {int_or_none}),
+            'tags': ('tags', ..., 'name', {str}),
+            'release_timestamp': ('started_at', {parse_iso8601}),
+            'timestamp': ('created_at', {parse_iso8601}),
+            'series': ('program', 'title', {str}),
+            'series_id': ('program', 'ProgramID', {str}),
+            'channel': ('channel', 'name', {str}),
+            'channel_id': ('channel', 'ChannelID', {str}),
+            'channel_url': ('channel', 'descriptor', {_fmt_url('https://telewebion.com/live/%s')}),
+            'thumbnail': ('image', {_fmt_url('https://static.telewebion.com/episodeImages/%s/default')}),
+            'formats': (
+                'channel', 'descriptor', {str},
+                {_fmt_url(f'https://cdna.telewebion.com/%s/episode/{video_id}/playlist.m3u8')},
+                {partial(self._extract_m3u8_formats, video_id=video_id, ext='mp4', m3u8_id='hls')}),
+        }))
+        info_dict['id'] = video_id
+        return info_dict
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index f1d7cead66..361617c028 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -636,7 +636,7 @@ def replace_insane(char):
         elif char in '\\/|*<>':
             return '\0_'
         if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace() or ord(char) > 127):
-            return '\0_'
+            return '' if unicodedata.category(char)[0] in 'CM' else '\0_'
         return char
 
     # Replace look-alike Unicode glyphs

From 2d1d683a541d71f3d3bb999dfe8eeb1976fb91ce Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Tue, 26 Dec 2023 18:30:04 +0100
Subject: [PATCH 12/17] [devscripts] `run_tests`: Create Python script (#8720)

Authored by: Grub4K
---
 .github/workflows/core.yml       |  6 +--
 .github/workflows/download.yml   |  7 +---
 .github/workflows/quick-test.yml |  2 +-
 CONTRIBUTING.md                  | 31 +++++++-------
 devscripts/run_tests.bat         | 17 +-------
 devscripts/run_tests.py          | 70 ++++++++++++++++++++++++++++++++
 devscripts/run_tests.sh          | 14 +------
 7 files changed, 95 insertions(+), 52 deletions(-)
 create mode 100755 devscripts/run_tests.py

diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index b22adb1b9d..ded7e6d612 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -38,18 +38,14 @@ jobs:
         os: [ubuntu-latest]
         # CPython 3.11 is in quick-test
         python-version: ['3.8', '3.9', '3.10', '3.12', pypy-3.8, pypy-3.10]
-        run-tests-ext: [sh]
         include:
         # atleast one of each CPython/PyPy tests must be in windows
         - os: windows-latest
           python-version: '3.8'
-          run-tests-ext: bat
         - os: windows-latest
           python-version: '3.12'
-          run-tests-ext: bat
         - os: windows-latest
           python-version: pypy-3.9
-          run-tests-ext: bat
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
@@ -62,4 +58,4 @@ jobs:
       continue-on-error: False
       run: |
         python3 -m yt_dlp -v || true  # Print debug head
-        ./devscripts/run_tests.${{ matrix.run-tests-ext }} core
+        python3 ./devscripts/run_tests.py core
diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml
index 73b2f9ca3d..9f47d67187 100644
--- a/.github/workflows/download.yml
+++ b/.github/workflows/download.yml
@@ -18,7 +18,7 @@ jobs:
       run: pip install pytest -r requirements.txt
     - name: Run tests
       continue-on-error: true
-      run: ./devscripts/run_tests.sh download
+      run: python3 ./devscripts/run_tests.py download
 
   full:
     name: Full Download Tests
@@ -29,15 +29,12 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python-version: ['3.10', '3.11', '3.12', pypy-3.8, pypy-3.10]
-        run-tests-ext: [sh]
         include:
         # atleast one of each CPython/PyPy tests must be in windows
         - os: windows-latest
           python-version: '3.8'
-          run-tests-ext: bat
         - os: windows-latest
           python-version: pypy-3.9
-          run-tests-ext: bat
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
@@ -48,4 +45,4 @@ jobs:
       run: pip install pytest -r requirements.txt
     - name: Run tests
       continue-on-error: true
-      run: ./devscripts/run_tests.${{ matrix.run-tests-ext }} download
+      run: python3 ./devscripts/run_tests.py download
diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml
index edbdaffd74..1ccfbe836f 100644
--- a/.github/workflows/quick-test.yml
+++ b/.github/workflows/quick-test.yml
@@ -19,7 +19,7 @@ jobs:
     - name: Run tests
       run: |
         python3 -m yt_dlp -v || true
-        ./devscripts/run_tests.sh core
+        python3 ./devscripts/run_tests.py core
   flake8:
     name: Linter
     if: "!contains(github.event.head_commit.message, 'ci skip all')"
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c472f32514..248917bf55 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -140,12 +140,9 @@ # DEVELOPER INSTRUCTIONS
 
     python -m yt_dlp
 
-To run the test, simply invoke your favorite test runner, or execute a test file directly; any of the following work:
+To run all the available core tests, use:
 
-    python -m unittest discover
-    python test/test_download.py
-    nosetests
-    pytest
+    python devscripts/run_tests.py
 
 See item 6 of [new extractor tutorial](#adding-support-for-a-new-site) for how to run extractor specific test cases.
 
@@ -187,15 +184,21 @@ ## Adding support for a new site
             'url': 'https://yourextractor.com/watch/42',
             'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
             'info_dict': {
+                # For videos, only the 'id' and 'ext' fields are required to RUN the test:
                 'id': '42',
                 'ext': 'mp4',
-                'title': 'Video title goes here',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                # TODO more properties, either as:
-                # * A value
-                # * MD5 checksum; start the string with md5:
-                # * A regular expression; start the string with re:
-                # * Any Python type, e.g. int or float
+                # Then if the test run fails, it will output the missing/incorrect fields.
+                # Properties can be added as:
+                # * A value, e.g.
+                #     'title': 'Video title goes here',
+                # * MD5 checksum; start the string with 'md5:', e.g.
+                #     'description': 'md5:098f6bcd4621d373cade4e832627b4f6',
+                # * A regular expression; start the string with 're:', e.g.
+                #     'thumbnail': r're:^https?://.*\.jpg$',
+                # * A count of elements in a list; start the string with 'count:', e.g.
+                #     'tags': 'count:10',
+                # * Any Python type, e.g.
+                #     'view_count': int,
             }
         }]
 
@@ -215,8 +218,8 @@ ## Adding support for a new site
             }
     ```
 1. Add an import in [`yt_dlp/extractor/_extractors.py`](yt_dlp/extractor/_extractors.py). Note that the class name must end with `IE`.
-1. Run `python test/test_download.py TestDownload.test_YourExtractor` (note that `YourExtractor` doesn't end with `IE`). This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, the tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in. You can also run all the tests in one go with `TestDownload.test_YourExtractor_all`
-1. Make sure you have atleast one test for your extractor. Even if all videos covered by the extractor are expected to be inaccessible for automated testing, tests should still be added with a `skip` parameter indicating why the particular test is disabled from running.
+1. Run `python devscripts/run_tests.py YourExtractor`. This *may fail* at first, but you can continually re-run it until you're done. Upon failure, it will output the missing fields and/or correct values which you can copy. If you decide to add more than one test, the tests will then be named `YourExtractor`, `YourExtractor_1`, `YourExtractor_2`, etc. Note that tests with an `only_matching` key in the test's dict are not included in the count. You can also run all the tests in one go with `YourExtractor_all`
+1. Make sure you have at least one test for your extractor. Even if all videos covered by the extractor are expected to be inaccessible for automated testing, tests should still be added with a `skip` parameter indicating why the particular test is disabled from running.
 1. Have a look at [`yt_dlp/extractor/common.py`](yt_dlp/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](yt_dlp/extractor/common.py#L119-L440). Add tests and code for as many as you want.
 1. Make sure your code follows [yt-dlp coding conventions](#yt-dlp-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
 
diff --git a/devscripts/run_tests.bat b/devscripts/run_tests.bat
index 190d23918c..57b1f4bf46 100644
--- a/devscripts/run_tests.bat
+++ b/devscripts/run_tests.bat
@@ -1,17 +1,4 @@
-@setlocal
 @echo off
-cd /d %~dp0..
 
-if ["%~1"]==[""] (
-    set "test_set="test""
-) else if ["%~1"]==["core"] (
-    set "test_set="-m not download""
-) else if ["%~1"]==["download"] (
-    set "test_set="-m "download""
-) else (
-    echo.Invalid test type "%~1". Use "core" ^| "download"
-    exit /b 1
-)
-
-set PYTHONWARNINGS=error
-pytest %test_set%
+>&2 echo run_tests.bat is deprecated. Please use `devscripts/run_tests.py` instead
+python %~dp0run_tests.py %~1
diff --git a/devscripts/run_tests.py b/devscripts/run_tests.py
new file mode 100755
index 0000000000..b0c6ee67af
--- /dev/null
+++ b/devscripts/run_tests.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+import argparse
+import functools
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+
+fix_test_name = functools.partial(re.compile(r'IE(_all|_\d+)?$').sub, r'\1')
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Run selected yt-dlp tests')
+    parser.add_argument(
+        'test', help='a extractor tests, or one of "core" or "download"', nargs='*')
+    parser.add_argument(
+        '-k', help='run a test matching EXPRESSION. Same as "pytest -k"', metavar='EXPRESSION')
+    return parser.parse_args()
+
+
+def run_tests(*tests, pattern=None):
+    run_core = 'core' in tests or (not pattern and not tests)
+    run_download = 'download' in tests
+    tests = list(map(fix_test_name, tests))
+
+    arguments = ['pytest', '-Werror', '--tb', 'short']
+    if run_core:
+        arguments.extend(['-m', 'not download'])
+    elif run_download:
+        arguments.extend(['-m', 'download'])
+    elif pattern:
+        arguments.extend(['-k', pattern])
+    else:
+        arguments.extend(
+            f'test/test_download.py::TestDownload::test_{test}' for test in tests)
+
+    print(f'Running {arguments}')
+    try:
+        subprocess.run(arguments)
+        return
+    except FileNotFoundError:
+        pass
+
+    arguments = [sys.executable, '-Werror', '-m', 'unittest']
+    if run_core:
+        print('"pytest" needs to be installed to run core tests', file=sys.stderr)
+        return
+    elif run_download:
+        arguments.append('test.test_download')
+    elif pattern:
+        arguments.extend(['-k', pattern])
+    else:
+        arguments.extend(
+            f'test.test_download.TestDownload.test_{test}' for test in tests)
+
+    print(f'Running {arguments}')
+    subprocess.run(arguments)
+
+
+if __name__ == '__main__':
+    try:
+        args = parse_args()
+
+        os.chdir(Path(__file__).parent.parent)
+        run_tests(*args.test, pattern=args.k)
+    except KeyboardInterrupt:
+        pass
diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh
index faa642e96c..123ceb1ee4 100755
--- a/devscripts/run_tests.sh
+++ b/devscripts/run_tests.sh
@@ -1,14 +1,4 @@
 #!/usr/bin/env sh
 
-if [ -z "$1" ]; then
-    test_set='test'
-elif [ "$1" = 'core' ]; then
-    test_set="-m not download"
-elif [ "$1" = 'download' ]; then
-    test_set="-m download"
-else
-    echo 'Invalid test type "'"$1"'". Use "core" | "download"'
-    exit 1
-fi
-
-python3 -bb -Werror -m pytest "$test_set"
+>&2 echo 'run_tests.sh is deprecated. Please use `devscripts/run_tests.py` instead'
+python3 devscripts/run_tests.py "$1"

From 225cf2b830a1de2c5eacd257edd2a01aed1e1114 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Tue, 26 Dec 2023 19:55:30 +0100
Subject: [PATCH 13/17] Fix 2d1d683a541d71f3d3bb999dfe8eeb1976fb91ce

Authored by: Grub4K
---
 devscripts/run_tests.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/devscripts/run_tests.py b/devscripts/run_tests.py
index b0c6ee67af..6d638a9748 100755
--- a/devscripts/run_tests.py
+++ b/devscripts/run_tests.py
@@ -21,12 +21,14 @@ def parse_args():
     return parser.parse_args()
 
 
-def run_tests(*tests, pattern=None):
+def run_tests(*tests, pattern=None, ci=False):
     run_core = 'core' in tests or (not pattern and not tests)
     run_download = 'download' in tests
     tests = list(map(fix_test_name, tests))
 
-    arguments = ['pytest', '-Werror', '--tb', 'short']
+    arguments = ['pytest', '-Werror', '--tb=short']
+    if ci:
+        arguments.append('--color=yes')
     if run_core:
         arguments.extend(['-m', 'not download'])
     elif run_download:
@@ -37,17 +39,16 @@ def run_tests(*tests, pattern=None):
         arguments.extend(
             f'test/test_download.py::TestDownload::test_{test}' for test in tests)
 
-    print(f'Running {arguments}')
+    print(f'Running {arguments}', flush=True)
     try:
-        subprocess.run(arguments)
-        return
+        return subprocess.call(arguments)
     except FileNotFoundError:
         pass
 
     arguments = [sys.executable, '-Werror', '-m', 'unittest']
     if run_core:
-        print('"pytest" needs to be installed to run core tests', file=sys.stderr)
-        return
+        print('"pytest" needs to be installed to run core tests', file=sys.stderr, flush=True)
+        return 1
     elif run_download:
         arguments.append('test.test_download')
     elif pattern:
@@ -56,8 +57,8 @@ def run_tests(*tests, pattern=None):
         arguments.extend(
             f'test.test_download.TestDownload.test_{test}' for test in tests)
 
-    print(f'Running {arguments}')
-    subprocess.run(arguments)
+    print(f'Running {arguments}', flush=True)
+    return subprocess.call(arguments)
 
 
 if __name__ == '__main__':
@@ -65,6 +66,6 @@ def run_tests(*tests, pattern=None):
         args = parse_args()
 
         os.chdir(Path(__file__).parent.parent)
-        run_tests(*args.test, pattern=args.k)
+        sys.exit(run_tests(*args.test, pattern=args.k, ci=bool(os.getenv('CI'))))
     except KeyboardInterrupt:
         pass

From 5f009a094f0e8450792b097c4c8273622778052d Mon Sep 17 00:00:00 2001
From: sepro <4618135+seproDev@users.noreply.github.com>
Date: Sat, 30 Dec 2023 21:44:32 +0100
Subject: [PATCH 14/17] [ie/ARD] Overhaul extractors (#8878)

Closes #8731, Closes #6784, Closes #2366, Closes #2975, Closes #8760
Authored by: seproDev
---
 yt_dlp/extractor/_extractors.py |   2 +-
 yt_dlp/extractor/ard.py         | 623 +++++++++++++-------------------
 2 files changed, 250 insertions(+), 375 deletions(-)

diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 62103f13c1..6f7a1e4f10 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -134,8 +134,8 @@
 from .arkena import ArkenaIE
 from .ard import (
     ARDBetaMediathekIE,
+    ARDMediathekCollectionIE,
     ARDIE,
-    ARDMediathekIE,
 )
 from .arte import (
     ArteTVIE,
diff --git a/yt_dlp/extractor/ard.py b/yt_dlp/extractor/ard.py
index 8ac926c917..91d297e8ba 100644
--- a/yt_dlp/extractor/ard.py
+++ b/yt_dlp/extractor/ard.py
@@ -1,24 +1,23 @@
-import json
 import re
+from functools import partial
 
 from .common import InfoExtractor
-from .generic import GenericIE
 from ..utils import (
+    OnDemandPagedList,
     determine_ext,
-    ExtractorError,
     int_or_none,
+    join_nonempty,
+    make_archive_id,
     parse_duration,
-    qualities,
+    parse_iso8601,
+    remove_start,
     str_or_none,
-    try_get,
     unified_strdate,
-    unified_timestamp,
-    update_url,
     update_url_query,
     url_or_none,
     xpath_text,
 )
-from ..compat import compat_etree_fromstring
+from ..utils.traversal import traverse_obj
 
 
 class ARDMediathekBaseIE(InfoExtractor):
@@ -61,45 +60,6 @@ def _parse_media_info(self, media_info, video_id, fsk):
             'subtitles': subtitles,
         }
 
-    def _ARD_extract_episode_info(self, title):
-        """Try to extract season/episode data from the title."""
-        res = {}
-        if not title:
-            return res
-
-        for pattern in [
-            # Pattern for title like "Homo sapiens (S06/E07) - Originalversion"
-            # from: https://www.ardmediathek.de/one/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw
-            r'.*(?P<ep_info> \(S(?P<season_number>\d+)/E(?P<episode_number>\d+)\)).*',
-            # E.g.: title="Fritjof aus Norwegen (2) (AD)"
-            # from: https://www.ardmediathek.de/ard/sammlung/der-krieg-und-ich/68cMkqJdllm639Skj4c7sS/
-            r'.*(?P<ep_info> \((?:Folge |Teil )?(?P<episode_number>\d+)(?:/\d+)?\)).*',
-            r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:\:| -|) )\"(?P<episode>.+)\".*',
-            # E.g.: title="Folge 25/42: Symmetrie"
-            # from: https://www.ardmediathek.de/ard/video/grips-mathe/folge-25-42-symmetrie/ard-alpha/Y3JpZDovL2JyLmRlL3ZpZGVvLzMyYzI0ZjczLWQ1N2MtNDAxNC05ZmZhLTFjYzRkZDA5NDU5OQ/
-            # E.g.: title="Folge 1063 - Vertrauen"
-            # from: https://www.ardmediathek.de/ard/sendung/die-fallers/Y3JpZDovL3N3ci5kZS8yMzAyMDQ4/
-            r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:/\d+)?(?:\:| -|) ).*',
-        ]:
-            m = re.match(pattern, title)
-            if m:
-                groupdict = m.groupdict()
-                res['season_number'] = int_or_none(groupdict.get('season_number'))
-                res['episode_number'] = int_or_none(groupdict.get('episode_number'))
-                res['episode'] = str_or_none(groupdict.get('episode'))
-                # Build the episode title by removing numeric episode information:
-                if groupdict.get('ep_info') and not res['episode']:
-                    res['episode'] = str_or_none(
-                        title.replace(groupdict.get('ep_info'), ''))
-                if res['episode']:
-                    res['episode'] = res['episode'].strip()
-                break
-
-        # As a fallback use the whole title as the episode name:
-        if not res.get('episode'):
-            res['episode'] = title.strip()
-        return res
-
     def _extract_formats(self, media_info, video_id):
         type_ = media_info.get('_type')
         media_array = media_info.get('_mediaArray', [])
@@ -155,138 +115,6 @@ def _extract_formats(self, media_info, video_id):
         return formats
 
 
-class ARDMediathekIE(ARDMediathekBaseIE):
-    IE_NAME = 'ARD:mediathek'
-    _VALID_URL = r'^https?://(?:(?:(?:www|classic)\.)?ardmediathek\.de|mediathek\.(?:daserste|rbb-online)\.de|one\.ard\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
-
-    _TESTS = [{
-        # available till 26.07.2022
-        'url': 'http://www.ardmediathek.de/tv/S%C3%9CDLICHT/Was-ist-die-Kunst-der-Zukunft-liebe-Ann/BR-Fernsehen/Video?bcastId=34633636&documentId=44726822',
-        'info_dict': {
-            'id': '44726822',
-            'ext': 'mp4',
-            'title': 'Was ist die Kunst der Zukunft, liebe Anna McCarthy?',
-            'description': 'md5:4ada28b3e3b5df01647310e41f3a62f5',
-            'duration': 1740,
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        }
-    }, {
-        'url': 'https://one.ard.de/tv/Mord-mit-Aussicht/Mord-mit-Aussicht-6-39-T%C3%B6dliche-Nach/ONE/Video?bcastId=46384294&documentId=55586872',
-        'only_matching': True,
-    }, {
-        # audio
-        'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',
-        'only_matching': True,
-    }, {
-        'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
-        'only_matching': True,
-    }, {
-        # audio
-        'url': 'http://mediathek.rbb-online.de/radio/Hörspiel/Vor-dem-Fest/kulturradio/Audio?documentId=30796318&topRessort=radio&bcastId=9839158',
-        'only_matching': True,
-    }, {
-        'url': 'https://classic.ardmediathek.de/tv/Panda-Gorilla-Co/Panda-Gorilla-Co-Folge-274/Das-Erste/Video?bcastId=16355486&documentId=58234698',
-        'only_matching': True,
-    }]
-
-    @classmethod
-    def suitable(cls, url):
-        return False if ARDBetaMediathekIE.suitable(url) else super(ARDMediathekIE, cls).suitable(url)
-
-    def _real_extract(self, url):
-        # determine video id from url
-        m = self._match_valid_url(url)
-
-        document_id = None
-
-        numid = re.search(r'documentId=([0-9]+)', url)
-        if numid:
-            document_id = video_id = numid.group(1)
-        else:
-            video_id = m.group('video_id')
-
-        webpage = self._download_webpage(url, video_id)
-
-        ERRORS = (
-            ('>Leider liegt eine Störung vor.', 'Video %s is unavailable'),
-            ('>Der gewünschte Beitrag ist nicht mehr verfügbar.<',
-             'Video %s is no longer available'),
-        )
-
-        for pattern, message in ERRORS:
-            if pattern in webpage:
-                raise ExtractorError(message % video_id, expected=True)
-
-        if re.search(r'[\?&]rss($|[=&])', url):
-            doc = compat_etree_fromstring(webpage.encode('utf-8'))
-            if doc.tag == 'rss':
-                return GenericIE()._extract_rss(url, video_id, doc)
-
-        title = self._og_search_title(webpage, default=None) or self._html_search_regex(
-            [r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
-             r'<meta name="dcterms\.title" content="(.*?)"/>',
-             r'<h4 class="headline">(.*?)</h4>',
-             r'<title[^>]*>(.*?)</title>'],
-            webpage, 'title')
-        description = self._og_search_description(webpage, default=None) or self._html_search_meta(
-            'dcterms.abstract', webpage, 'description', default=None)
-        if description is None:
-            description = self._html_search_meta(
-                'description', webpage, 'meta description', default=None)
-        if description is None:
-            description = self._html_search_regex(
-                r'<p\s+class="teasertext">(.+?)</p>',
-                webpage, 'teaser text', default=None)
-
-        # Thumbnail is sometimes not present.
-        # It is in the mobile version, but that seems to use a different URL
-        # structure altogether.
-        thumbnail = self._og_search_thumbnail(webpage, default=None)
-
-        media_streams = re.findall(r'''(?x)
-            mediaCollection\.addMediaStream\([0-9]+,\s*[0-9]+,\s*"[^"]*",\s*
-            "([^"]+)"''', webpage)
-
-        if media_streams:
-            QUALITIES = qualities(['lo', 'hi', 'hq'])
-            formats = []
-            for furl in set(media_streams):
-                if furl.endswith('.f4m'):
-                    fid = 'f4m'
-                else:
-                    fid_m = re.match(r'.*\.([^.]+)\.[^.]+$', furl)
-                    fid = fid_m.group(1) if fid_m else None
-                formats.append({
-                    'quality': QUALITIES(fid),
-                    'format_id': fid,
-                    'url': furl,
-                })
-            info = {
-                'formats': formats,
-            }
-        else:  # request JSON file
-            if not document_id:
-                video_id = self._search_regex(
-                    (r'/play/(?:config|media|sola)/(\d+)', r'contentId["\']\s*:\s*(\d+)'),
-                    webpage, 'media id', default=None)
-            info = self._extract_media_info(
-                'http://www.ardmediathek.de/play/media/%s' % video_id,
-                webpage, video_id)
-
-        info.update({
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-        })
-        info.update(self._ARD_extract_episode_info(info['title']))
-
-        return info
-
-
 class ARDIE(InfoExtractor):
     _VALID_URL = r'(?P<mainurl>https?://(?:www\.)?daserste\.de/(?:[^/?#&]+/)+(?P<id>[^/?#&]+))\.html'
     _TESTS = [{
@@ -399,21 +227,23 @@ def _real_extract(self, url):
         }
 
 
-class ARDBetaMediathekIE(ARDMediathekBaseIE):
+class ARDBetaMediathekIE(InfoExtractor):
+    IE_NAME = 'ARDMediathek'
     _VALID_URL = r'''(?x)https://
         (?:(?:beta|www)\.)?ardmediathek\.de/
-        (?:(?P<client>[^/]+)/)?
-        (?:player|live|video|(?P<playlist>sendung|serie|sammlung))/
-        (?:(?P<display_id>(?(playlist)[^?#]+?|[^?#]+))/)?
-        (?P<id>(?(playlist)|Y3JpZDovL)[a-zA-Z0-9]+)
-        (?(playlist)/(?P<season>\d+)?/?(?:[?#]|$))'''
+        (?:[^/]+/)?
+        (?:player|live|video)/
+        (?:(?P<display_id>[^?#]+)/)?
+        (?P<id>[a-zA-Z0-9]+)
+        /?(?:[?#]|$)'''
+    _GEO_COUNTRIES = ['DE']
 
     _TESTS = [{
         'url': 'https://www.ardmediathek.de/video/filme-im-mdr/liebe-auf-vier-pfoten/mdr-fernsehen/Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MjIwOTEtNDAyNTM0',
         'md5': 'b6e8ab03f2bcc6e1f9e6cef25fcc03c4',
         'info_dict': {
             'display_id': 'filme-im-mdr/liebe-auf-vier-pfoten/mdr-fernsehen',
-            'id': '12939099',
+            'id': 'Y3JpZDovL21kci5kZS9zZW5kdW5nLzI4MjA0MC80MjIwOTEtNDAyNTM0',
             'title': 'Liebe auf vier Pfoten',
             'description': r're:^Claudia Schmitt, Anwältin in Salzburg',
             'duration': 5222,
@@ -422,7 +252,10 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
             'upload_date': '20231130',
             'ext': 'mp4',
             'episode': 'Liebe auf vier Pfoten',
-            'series': 'Filme im MDR'
+            'series': 'Filme im MDR',
+            'age_limit': 0,
+            'channel': 'MDR',
+            '_old_archive_ids': ['ardbetamediathek 12939099'],
         },
     }, {
         'url': 'https://www.ardmediathek.de/mdr/video/die-robuste-roswita/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy84MWMxN2MzZC0wMjkxLTRmMzUtODk4ZS0wYzhlOWQxODE2NGI/',
@@ -443,7 +276,7 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
         'url': 'https://www.ardmediathek.de/video/tagesschau-oder-tagesschau-20-00-uhr/das-erste/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXUvZmM4ZDUxMjgtOTE0ZC00Y2MzLTgzNzAtNDZkNGNiZWJkOTll',
         'md5': '1e73ded21cb79bac065117e80c81dc88',
         'info_dict': {
-            'id': '10049223',
+            'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXUvZmM4ZDUxMjgtOTE0ZC00Y2MzLTgzNzAtNDZkNGNiZWJkOTll',
             'ext': 'mp4',
             'title': 'tagesschau, 20:00 Uhr',
             'timestamp': 1636398000,
@@ -454,6 +287,26 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
             'episode': 'tagesschau, 20:00 Uhr',
             'series': 'tagesschau',
             'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:fbb21142783b0a49?w=960&ch=ee69108ae344f678',
+            'channel': 'ARD-Aktuell',
+            '_old_archive_ids': ['ardbetamediathek 10049223'],
+        },
+    }, {
+        'url': 'https://www.ardmediathek.de/video/7-tage/7-tage-unter-harten-jungs/hr-fernsehen/N2I2YmM5MzgtNWFlOS00ZGFlLTg2NzMtYzNjM2JlNjk4MDg3',
+        'md5': 'c428b9effff18ff624d4f903bda26315',
+        'info_dict': {
+            'id': 'N2I2YmM5MzgtNWFlOS00ZGFlLTg2NzMtYzNjM2JlNjk4MDg3',
+            'ext': 'mp4',
+            'duration': 2700,
+            'episode': '7 Tage ... unter harten Jungs',
+            'description': 'md5:0f215470dcd2b02f59f4bd10c963f072',
+            'upload_date': '20231005',
+            'timestamp': 1696491171,
+            'display_id': '7-tage/7-tage-unter-harten-jungs/hr-fernsehen',
+            'series': '7 Tage ...',
+            'channel': 'HR',
+            'thumbnail': 'https://api.ardmediathek.de/image-service/images/urn:ard:image:f6e6d5ffac41925c?w=960&ch=fa32ba69bc87989a',
+            'title': '7 Tage ... unter harten Jungs',
+            '_old_archive_ids': ['ardbetamediathek 94834686'],
         },
     }, {
         'url': 'https://beta.ardmediathek.de/ard/video/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
@@ -470,6 +323,176 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
     }, {
         'url': 'https://www.ardmediathek.de/swr/live/Y3JpZDovL3N3ci5kZS8xMzQ4MTA0Mg',
         'only_matching': True,
+    }, {
+        'url': 'https://www.ardmediathek.de/video/coronavirus-update-ndr-info/astrazeneca-kurz-lockdown-und-pims-syndrom-81/ndr/Y3JpZDovL25kci5kZS84NzE0M2FjNi0wMWEwLTQ5ODEtOTE5NS1mOGZhNzdhOTFmOTI/',
+        'only_matching': True,
+    }]
+
+    def _extract_episode_info(self, title):
+        patterns = [
+            # Pattern for title like "Homo sapiens (S06/E07) - Originalversion"
+            # from: https://www.ardmediathek.de/one/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw
+            r'.*(?P<ep_info> \(S(?P<season_number>\d+)/E(?P<episode_number>\d+)\)).*',
+            # E.g.: title="Fritjof aus Norwegen (2) (AD)"
+            # from: https://www.ardmediathek.de/ard/sammlung/der-krieg-und-ich/68cMkqJdllm639Skj4c7sS/
+            r'.*(?P<ep_info> \((?:Folge |Teil )?(?P<episode_number>\d+)(?:/\d+)?\)).*',
+            r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:\:| -|) )\"(?P<episode>.+)\".*',
+            # E.g.: title="Folge 25/42: Symmetrie"
+            # from: https://www.ardmediathek.de/ard/video/grips-mathe/folge-25-42-symmetrie/ard-alpha/Y3JpZDovL2JyLmRlL3ZpZGVvLzMyYzI0ZjczLWQ1N2MtNDAxNC05ZmZhLTFjYzRkZDA5NDU5OQ/
+            # E.g.: title="Folge 1063 - Vertrauen"
+            # from: https://www.ardmediathek.de/ard/sendung/die-fallers/Y3JpZDovL3N3ci5kZS8yMzAyMDQ4/
+            r'.*(?P<ep_info>Folge (?P<episode_number>\d+)(?:/\d+)?(?:\:| -|) ).*',
+            # As a fallback use the full title
+            r'(?P<title>.*)',
+        ]
+
+        return traverse_obj(patterns, (..., {partial(re.match, string=title)}, {
+            'season_number': ('season_number', {int_or_none}),
+            'episode_number': ('episode_number', {int_or_none}),
+            'episode': ((
+                ('episode', {str_or_none}),
+                ('ep_info', {lambda x: title.replace(x, '')}),
+                ('title', {str}),
+            ), {str.strip}),
+        }), get_all=False)
+
+    def _real_extract(self, url):
+        video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
+
+        page_data = self._download_json(
+            f'https://api.ardmediathek.de/page-gateway/pages/ard/item/{video_id}', video_id, query={
+                'embedded': 'false',
+                'mcV6': 'true',
+            })
+
+        player_data = traverse_obj(
+            page_data, ('widgets', lambda _, v: v['type'] in ('player_ondemand', 'player_live'), {dict}), get_all=False)
+        is_live = player_data.get('type') == 'player_live'
+        media_data = traverse_obj(player_data, ('mediaCollection', 'embedded', {dict}))
+
+        if player_data.get('blockedByFsk'):
+            self.raise_no_formats('This video is only available after 22:00', expected=True)
+
+        formats = []
+        subtitles = {}
+        for stream in traverse_obj(media_data, ('streams', ..., {dict})):
+            kind = stream.get('kind')
+            # Prioritize main stream over sign language and others
+            preference = 1 if kind == 'main' else None
+            for media in traverse_obj(stream, ('media', lambda _, v: url_or_none(v['url']))):
+                media_url = media['url']
+
+                audio_kind = traverse_obj(media, (
+                    'audios', 0, 'kind', {str}), default='').replace('standard', '')
+                lang_code = traverse_obj(media, ('audios', 0, 'languageCode', {str})) or 'deu'
+                lang = join_nonempty(lang_code, audio_kind)
+                language_preference = 10 if lang == 'deu' else -10
+
+                if determine_ext(media_url) == 'm3u8':
+                    fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                        media_url, video_id, m3u8_id=f'hls-{kind}', preference=preference, fatal=False, live=is_live)
+                    for f in fmts:
+                        f['language'] = lang
+                        f['language_preference'] = language_preference
+                    formats.extend(fmts)
+                    self._merge_subtitles(subs, target=subtitles)
+                else:
+                    formats.append({
+                        'url': media_url,
+                        'format_id': f'http-{kind}',
+                        'preference': preference,
+                        'language': lang,
+                        'language_preference': language_preference,
+                        **traverse_obj(media, {
+                            'format_note': ('forcedLabel', {str}),
+                            'width': ('maxHResolutionPx', {int_or_none}),
+                            'height': ('maxVResolutionPx', {int_or_none}),
+                            'vcodec': ('videoCodec', {str}),
+                        }),
+                    })
+
+        for sub in traverse_obj(media_data, ('subtitles', ..., {dict})):
+            for sources in traverse_obj(sub, ('sources', lambda _, v: url_or_none(v['url']))):
+                subtitles.setdefault(sub.get('languageCode') or 'deu', []).append({
+                    'url': sources['url'],
+                    'ext': {'webvtt': 'vtt', 'ebutt': 'ttml'}.get(sources.get('kind')),
+                })
+
+        age_limit = traverse_obj(page_data, ('fskRating', {lambda x: remove_start(x, 'FSK')}, {int_or_none}))
+        old_id = traverse_obj(page_data, ('tracking', 'atiCustomVars', 'contentId'))
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            'is_live': is_live,
+            'age_limit': age_limit,
+            **traverse_obj(media_data, ('meta', {
+                'title': 'title',
+                'description': 'synopsis',
+                'timestamp': ('broadcastedOnDateTime', {parse_iso8601}),
+                'series': 'seriesTitle',
+                'thumbnail': ('images', 0, 'url', {url_or_none}),
+                'duration': ('durationSeconds', {int_or_none}),
+                'channel': 'clipSourceName',
+            })),
+            **self._extract_episode_info(page_data.get('title')),
+            '_old_archive_ids': [make_archive_id(ARDBetaMediathekIE, old_id)],
+        }
+
+
+class ARDMediathekCollectionIE(InfoExtractor):
+    _VALID_URL = r'''(?x)https://
+        (?:(?:beta|www)\.)?ardmediathek\.de/
+        (?:[^/?#]+/)?
+        (?P<playlist>sendung|serie|sammlung)/
+        (?:(?P<display_id>[^?#]+?)/)?
+        (?P<id>[a-zA-Z0-9]+)
+        (?:/(?P<season>\d+)(?:/(?P<version>OV|AD))?)?/?(?:[?#]|$)'''
+    _GEO_COUNTRIES = ['DE']
+
+    _TESTS = [{
+        'url': 'https://www.ardmediathek.de/serie/quiz/staffel-1-originalversion/Y3JpZDovL3dkci5kZS9vbmUvcXVpeg/1/OV',
+        'info_dict': {
+            'id': 'Y3JpZDovL3dkci5kZS9vbmUvcXVpeg_1_OV',
+            'display_id': 'quiz/staffel-1-originalversion',
+            'title': 'Staffel 1 Originalversion',
+        },
+        'playlist_count': 3,
+    }, {
+        'url': 'https://www.ardmediathek.de/serie/babylon-berlin/staffel-4-mit-audiodeskription/Y3JpZDovL2Rhc2Vyc3RlLmRlL2JhYnlsb24tYmVybGlu/4/AD',
+        'info_dict': {
+            'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL2JhYnlsb24tYmVybGlu_4_AD',
+            'display_id': 'babylon-berlin/staffel-4-mit-audiodeskription',
+            'title': 'Staffel 4 mit Audiodeskription',
+        },
+        'playlist_count': 12,
+    }, {
+        'url': 'https://www.ardmediathek.de/serie/babylon-berlin/staffel-1/Y3JpZDovL2Rhc2Vyc3RlLmRlL2JhYnlsb24tYmVybGlu/1/',
+        'info_dict': {
+            'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL2JhYnlsb24tYmVybGlu_1',
+            'display_id': 'babylon-berlin/staffel-1',
+            'title': 'Staffel 1',
+        },
+        'playlist_count': 8,
+    }, {
+        'url': 'https://www.ardmediathek.de/sendung/tatort/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydA',
+        'info_dict': {
+            'id': 'Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydA',
+            'display_id': 'tatort',
+            'title': 'Tatort',
+        },
+        'playlist_mincount': 500,
+    }, {
+        'url': 'https://www.ardmediathek.de/sammlung/die-kirche-bleibt-im-dorf/5eOHzt8XB2sqeFXbIoJlg2',
+        'info_dict': {
+            'id': '5eOHzt8XB2sqeFXbIoJlg2',
+            'display_id': 'die-kirche-bleibt-im-dorf',
+            'title': 'Die Kirche bleibt im Dorf',
+            'description': 'Die Kirche bleibt im Dorf',
+        },
+        'playlist_count': 4,
     }, {
         # playlist of type 'sendung'
         'url': 'https://www.ardmediathek.de/ard/sendung/doctor-who/Y3JpZDovL3dkci5kZS9vbmUvZG9jdG9yIHdobw/',
@@ -482,196 +505,48 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
         # playlist of type 'sammlung'
         'url': 'https://www.ardmediathek.de/ard/sammlung/team-muenster/5JpTzLSbWUAK8184IOvEir/',
         'only_matching': True,
-    }, {
-        'url': 'https://www.ardmediathek.de/video/coronavirus-update-ndr-info/astrazeneca-kurz-lockdown-und-pims-syndrom-81/ndr/Y3JpZDovL25kci5kZS84NzE0M2FjNi0wMWEwLTQ5ODEtOTE5NS1mOGZhNzdhOTFmOTI/',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.ardmediathek.de/ard/player/Y3JpZDovL3dkci5kZS9CZWl0cmFnLWQ2NDJjYWEzLTMwZWYtNGI4NS1iMTI2LTU1N2UxYTcxOGIzOQ/tatort-duo-koeln-leipzig-ihr-kinderlein-kommet',
-        'only_matching': True,
     }]
 
-    def _ARD_load_playlist_snippet(self, playlist_id, display_id, client, mode, page_number):
-        """ Query the ARD server for playlist information
-        and returns the data in "raw" format """
-        assert mode in ('sendung', 'serie', 'sammlung')
-        if mode in ('sendung', 'serie'):
-            graphQL = json.dumps({
-                'query': '''{
-                    showPage(
-                        client: "%s"
-                        showId: "%s"
-                        pageNumber: %d
-                    ) {
-                        pagination {
-                            pageSize
-                            totalElements
-                        }
-                        teasers {        # Array
-                            mediumTitle
-                            links { target { id href title } }
-                            type
-                        }
-                    }}''' % (client, playlist_id, page_number),
-            }).encode()
-        else:  # mode == 'sammlung'
-            graphQL = json.dumps({
-                'query': '''{
-                    morePage(
-                        client: "%s"
-                        compilationId: "%s"
-                        pageNumber: %d
-                    ) {
-                        widget {
-                            pagination {
-                                pageSize
-                                totalElements
-                            }
-                            teasers {        # Array
-                                mediumTitle
-                                links { target { id href title } }
-                                type
-                            }
-                        }
-                    }}''' % (client, playlist_id, page_number),
-            }).encode()
-        # Ressources for ARD graphQL debugging:
-        # https://api-test.ardmediathek.de/public-gateway
-        show_page = self._download_json(
-            'https://api.ardmediathek.de/public-gateway',
-            '[Playlist] %s' % display_id,
-            data=graphQL,
-            headers={'Content-Type': 'application/json'})['data']
-        # align the structure of the returned data:
-        if mode in ('sendung', 'serie'):
-            show_page = show_page['showPage']
-        else:  # mode == 'sammlung'
-            show_page = show_page['morePage']['widget']
-        return show_page
-
-    def _ARD_extract_playlist(self, url, playlist_id, display_id, client, mode):
-        """ Collects all playlist entries and returns them as info dict.
-        Supports playlists of mode 'sendung', 'serie', and 'sammlung',
-        as well as nested playlists. """
-        entries = []
-        pageNumber = 0
-        while True:  # iterate by pageNumber
-            show_page = self._ARD_load_playlist_snippet(
-                playlist_id, display_id, client, mode, pageNumber)
-            for teaser in show_page['teasers']:  # process playlist items
-                if '/compilation/' in teaser['links']['target']['href']:
-                    # alternativ cond.: teaser['type'] == "compilation"
-                    # => This is an nested compilation, e.g. like:
-                    # https://www.ardmediathek.de/ard/sammlung/die-kirche-bleibt-im-dorf/5eOHzt8XB2sqeFXbIoJlg2/
-                    link_mode = 'sammlung'
-                else:
-                    link_mode = 'video'
-
-                item_url = 'https://www.ardmediathek.de/%s/%s/%s/%s/%s' % (
-                    client, link_mode, display_id,
-                    # perform HTLM quoting of episode title similar to ARD:
-                    re.sub('^-|-$', '',  # remove '-' from begin/end
-                           re.sub('[^a-zA-Z0-9]+', '-',  # replace special chars by -
-                                  teaser['links']['target']['title'].lower()
-                                  .replace('ä', 'ae').replace('ö', 'oe')
-                                  .replace('ü', 'ue').replace('ß', 'ss'))),
-                    teaser['links']['target']['id'])
-                entries.append(self.url_result(
-                    item_url,
-                    ie=ARDBetaMediathekIE.ie_key()))
-
-            if (show_page['pagination']['pageSize'] * (pageNumber + 1)
-               >= show_page['pagination']['totalElements']):
-                # we've processed enough pages to get all playlist entries
-                break
-            pageNumber = pageNumber + 1
-
-        return self.playlist_result(entries, playlist_id, playlist_title=display_id)
+    _PAGE_SIZE = 100
 
     def _real_extract(self, url):
-        video_id, display_id, playlist_type, client, season_number = self._match_valid_url(url).group(
-            'id', 'display_id', 'playlist', 'client', 'season')
-        display_id, client = display_id or video_id, client or 'ard'
+        playlist_id, display_id, playlist_type, season_number, version = self._match_valid_url(url).group(
+            'id', 'display_id', 'playlist', 'season', 'version')
 
-        if playlist_type:
-            # TODO: Extract only specified season
-            return self._ARD_extract_playlist(url, video_id, display_id, client, playlist_type)
+        def call_api(page_num):
+            api_path = 'compilations/ard' if playlist_type == 'sammlung' else 'widgets/ard/asset'
+            return self._download_json(
+                f'https://api.ardmediathek.de/page-gateway/{api_path}/{playlist_id}', playlist_id,
+                f'Downloading playlist page {page_num}', query={
+                    'pageNumber': page_num,
+                    'pageSize': self._PAGE_SIZE,
+                    **({
+                        'seasoned': 'true',
+                        'seasonNumber': season_number,
+                        'withOriginalversion': 'true' if version == 'OV' else 'false',
+                        'withAudiodescription': 'true' if version == 'AD' else 'false',
+                    } if season_number else {}),
+                })
 
-        player_page = self._download_json(
-            'https://api.ardmediathek.de/public-gateway',
-            display_id, data=json.dumps({
-                'query': '''{
-  playerPage(client:"%s", clipId: "%s") {
-    blockedByFsk
-    broadcastedOn
-    maturityContentRating
-    mediaCollection {
-      _duration
-      _geoblocked
-      _isLive
-      _mediaArray {
-        _mediaStreamArray {
-          _quality
-          _server
-          _stream
-        }
-      }
-      _previewImage
-      _subtitleUrl
-      _type
-    }
-    show {
-      title
-    }
-    image {
-      src
-    }
-    synopsis
-    title
-    tracking {
-      atiCustomVars {
-        contentId
-      }
-    }
-  }
-}''' % (client, video_id),
-            }).encode(), headers={
-                'Content-Type': 'application/json'
-            })['data']['playerPage']
-        title = player_page['title']
-        content_id = str_or_none(try_get(
-            player_page, lambda x: x['tracking']['atiCustomVars']['contentId']))
-        media_collection = player_page.get('mediaCollection') or {}
-        if not media_collection and content_id:
-            media_collection = self._download_json(
-                'https://www.ardmediathek.de/play/media/' + content_id,
-                content_id, fatal=False) or {}
-        info = self._parse_media_info(
-            media_collection, content_id or video_id,
-            player_page.get('blockedByFsk'))
-        age_limit = None
-        description = player_page.get('synopsis')
-        maturity_content_rating = player_page.get('maturityContentRating')
-        if maturity_content_rating:
-            age_limit = int_or_none(maturity_content_rating.lstrip('FSK'))
-        if not age_limit and description:
-            age_limit = int_or_none(self._search_regex(
-                r'\(FSK\s*(\d+)\)\s*$', description, 'age limit', default=None))
-        info.update({
-            'age_limit': age_limit,
-            'display_id': display_id,
-            'title': title,
-            'description': description,
-            'timestamp': unified_timestamp(player_page.get('broadcastedOn')),
-            'series': try_get(player_page, lambda x: x['show']['title']),
-            'thumbnail': (media_collection.get('_previewImage')
-                          or try_get(player_page, lambda x: update_url(x['image']['src'], query=None, fragment=None))
-                          or self.get_thumbnail_from_html(display_id, url)),
-        })
-        info.update(self._ARD_extract_episode_info(info['title']))
-        return info
+        def fetch_page(page_num):
+            for item in traverse_obj(call_api(page_num), ('teasers', ..., {dict})):
+                item_id = traverse_obj(item, ('links', 'target', ('urlId', 'id')), 'id', get_all=False)
+                if not item_id or item_id == playlist_id:
+                    continue
+                item_mode = 'sammlung' if item.get('type') == 'compilation' else 'video'
+                yield self.url_result(
+                    f'https://www.ardmediathek.de/{item_mode}/{item_id}',
+                    ie=(ARDMediathekCollectionIE if item_mode == 'sammlung' else ARDBetaMediathekIE),
+                    **traverse_obj(item, {
+                        'id': ('id', {str}),
+                        'title': ('longTitle', {str}),
+                        'duration': ('duration', {int_or_none}),
+                        'timestamp': ('broadcastedOn', {parse_iso8601}),
+                    }))
 
-    def get_thumbnail_from_html(self, display_id, url):
-        webpage = self._download_webpage(url, display_id, fatal=False) or ''
-        return (
-            self._og_search_thumbnail(webpage, default=None)
-            or self._html_search_meta('thumbnailUrl', webpage, default=None))
+        page_data = call_api(0)
+        full_id = join_nonempty(playlist_id, season_number, version, delim='_')
+
+        return self.playlist_result(
+            OnDemandPagedList(fetch_page, self._PAGE_SIZE), full_id, display_id=display_id,
+            title=page_data.get('title'), description=page_data.get('synopsis'))

From f9fb3ce86e3c6a0c3c33b45392b8d7288bceba76 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Sat, 30 Dec 2023 22:27:36 +0100
Subject: [PATCH 15/17] [cleanup] Misc (#8598)

Authored by: bashonly, pukkandan, seproDev, Grub4K

Co-authored-by: bashonly <bashonly@protonmail.com>
Co-authored-by: pukkandan <pukkandan.ytdlp@gmail.com>
Co-authored-by: sepro <4618135+seproDev@users.noreply.github.com>
---
 .github/workflows/core.yml         |  4 ++--
 .github/workflows/quick-test.yml   |  4 ++--
 Collaborators.md                   | 18 ++++++++++--------
 README.md                          |  3 ++-
 devscripts/changelog_override.json |  6 ++++++
 devscripts/make_changelog.py       | 18 ++----------------
 devscripts/make_issue_template.py  |  6 +-----
 requirements.txt                   |  1 -
 test/test_YoutubeDL.py             |  2 +-
 test/test_update.py                |  2 +-
 test/test_utils.py                 |  2 ++
 yt-dlp.cmd                         |  2 +-
 yt-dlp.sh                          |  2 +-
 yt_dlp/YoutubeDL.py                |  8 +++++++-
 yt_dlp/extractor/banbye.py         |  2 +-
 yt_dlp/extractor/duoplay.py        | 15 ---------------
 yt_dlp/extractor/floatplane.py     | 10 +++++-----
 yt_dlp/extractor/generic.py        |  2 +-
 yt_dlp/extractor/panopto.py        |  2 +-
 yt_dlp/extractor/radiofrance.py    |  6 +++---
 yt_dlp/extractor/wordpress.py      |  2 +-
 yt_dlp/extractor/youtube.py        |  1 +
 yt_dlp/networking/__init__.py      |  1 -
 yt_dlp/networking/_helper.py       |  8 ++++----
 yt_dlp/networking/_requests.py     |  4 +++-
 yt_dlp/networking/_websockets.py   | 14 ++++++++++----
 yt_dlp/networking/websocket.py     |  2 +-
 yt_dlp/socks.py                    |  2 +-
 yt_dlp/utils/_utils.py             |  8 ++++++--
 yt_dlp/utils/traversal.py          |  2 +-
 30 files changed, 77 insertions(+), 82 deletions(-)

diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index ded7e6d612..eaaf03dee4 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -36,8 +36,8 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        # CPython 3.11 is in quick-test
-        python-version: ['3.8', '3.9', '3.10', '3.12', pypy-3.8, pypy-3.10]
+        # CPython 3.8 is in quick-test
+        python-version: ['3.9', '3.10', '3.11', '3.12', pypy-3.8, pypy-3.10]
         include:
         # atleast one of each CPython/PyPy tests must be in windows
         - os: windows-latest
diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml
index 1ccfbe836f..84fca62d4d 100644
--- a/.github/workflows/quick-test.yml
+++ b/.github/workflows/quick-test.yml
@@ -10,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python 3.11
+    - name: Set up Python 3.8
       uses: actions/setup-python@v4
       with:
-        python-version: '3.11'
+        python-version: '3.8'
     - name: Install test requirements
       run: pip install pytest -r requirements.txt
     - name: Run tests
diff --git a/Collaborators.md b/Collaborators.md
index 70ab616f11..894a853c9b 100644
--- a/Collaborators.md
+++ b/Collaborators.md
@@ -29,6 +29,7 @@ ## [coletdjnz](https://github.com/coletdjnz)
 [![gh-sponsor](https://img.shields.io/badge/_-Github-white.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/coletdjnz)
 
 * Improved plugin architecture
+* Rewrote the networking infrastructure, implemented support for `requests`
 * YouTube improvements including: age-gate bypass, private playlists, multiple-clients (to avoid throttling) and a lot of under-the-hood improvements
 * Added support for new websites YoutubeWebArchive, MainStreaming, PRX, nzherald, Mediaklikk, StarTV etc
 * Improved/fixed support for Patreon, panopto, gfycat, itv, pbs, SouthParkDE etc
@@ -46,16 +47,17 @@ ## [Ashish0804](https://github.com/Ashish0804) <sub><sup>[Inactive]</sup></sub>
 
 ## [bashonly](https://github.com/bashonly)
 
-* `--update-to`, automated release, nightly builds
-* `--cookies-from-browser` support for Firefox containers
-* Added support for new websites Genius, Kick, NBCStations, Triller, VideoKen etc
-* Improved/fixed support for Anvato, Brightcove, Instagram, ParamountPlus, Reddit, SlidesLive, TikTok, Twitter, Vimeo etc
+* `--update-to`, self-updater rewrite, automated/nightly/master releases
+* `--cookies-from-browser` support for Firefox containers, external downloader cookie handling overhaul
+* Added support for new websites like Dacast, Kick, NBCStations, Triller, VideoKen, Weverse, WrestleUniverse etc
+* Improved/fixed support for Anvato, Brightcove, Reddit, SlidesLive, TikTok, Twitter, Vimeo etc
 
 
 ## [Grub4K](https://github.com/Grub4K)
 
-[![ko-fi](https://img.shields.io/badge/_-Ko--fi-red.svg?logo=kofi&labelColor=555555&style=for-the-badge)](https://ko-fi.com/Grub4K) [![gh-sponsor](https://img.shields.io/badge/_-Github-white.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/Grub4K)
+[![gh-sponsor](https://img.shields.io/badge/_-Github-white.svg?logo=github&labelColor=555555&style=for-the-badge)](https://github.com/sponsors/Grub4K) [![ko-fi](https://img.shields.io/badge/_-Ko--fi-red.svg?logo=kofi&labelColor=555555&style=for-the-badge)](https://ko-fi.com/Grub4K)
 
-* `--update-to`, automated release, nightly builds
-* Rework internals like `traverse_obj`, various core refactors and bugs fixes
-* Helped fix crunchyroll, Twitter, wrestleuniverse, wistia, slideslive etc
+* `--update-to`, self-updater rewrite, automated/nightly/master releases
+* Reworked internals like `traverse_obj`, various core refactors and bugs fixes
+* Implemented proper progress reporting for parallel downloads
+* Improved/fixed/added Bundestag, crunchyroll, pr0gramm, Twitter, WrestleUniverse etc
diff --git a/README.md b/README.md
index 06aceec027..4f8fcb6c9c 100644
--- a/README.md
+++ b/README.md
@@ -159,6 +159,7 @@ ### Differences in default behavior
 * yt-dlp versions between 2021.09.01 and 2023.01.02 applies `--match-filter` to nested playlists. This was an unintentional side-effect of [8f18ac](https://github.com/yt-dlp/yt-dlp/commit/8f18aca8717bb0dd49054555af8d386e5eda3a88) and is fixed in [d7b460](https://github.com/yt-dlp/yt-dlp/commit/d7b460d0e5fc710950582baed2e3fc616ed98a80). Use `--compat-options playlist-match-filter` to revert this
 * yt-dlp versions between 2021.11.10 and 2023.06.21 estimated `filesize_approx` values for fragmented/manifest formats. This was added for convenience in [f2fe69](https://github.com/yt-dlp/yt-dlp/commit/f2fe69c7b0d208bdb1f6292b4ae92bc1e1a7444a), but was reverted in [0dff8e](https://github.com/yt-dlp/yt-dlp/commit/0dff8e4d1e6e9fb938f4256ea9af7d81f42fd54f) due to the potentially extreme inaccuracy of the estimated values. Use `--compat-options manifest-filesize-approx` to keep extracting the estimated values
 * yt-dlp uses modern http client backends such as `requests`. Use `--compat-options prefer-legacy-http-handler` to prefer the legacy http handler (`urllib`) to be used for standard http requests.
+* The sub-module `swfinterp` is removed.
 
 For ease of use, a few more compat options are available:
 
@@ -299,7 +300,7 @@ ### Misc
 
 * [**pycryptodomex**](https://github.com/Legrandin/pycryptodome)\* - For decrypting AES-128 HLS streams and various other data. Licensed under [BSD-2-Clause](https://github.com/Legrandin/pycryptodome/blob/master/LICENSE.rst)
 * [**phantomjs**](https://github.com/ariya/phantomjs) - Used in extractors where javascript needs to be run. Licensed under [BSD-3-Clause](https://github.com/ariya/phantomjs/blob/master/LICENSE.BSD)
-* [**secretstorage**](https://github.com/mitya57/secretstorage) - For `--cookies-from-browser` to access the **Gnome** keyring while decrypting cookies of **Chromium**-based browsers on **Linux**. Licensed under [BSD-3-Clause](https://github.com/mitya57/secretstorage/blob/master/LICENSE)
+* [**secretstorage**](https://github.com/mitya57/secretstorage)\* - For `--cookies-from-browser` to access the **Gnome** keyring while decrypting cookies of **Chromium**-based browsers on **Linux**. Licensed under [BSD-3-Clause](https://github.com/mitya57/secretstorage/blob/master/LICENSE)
 * Any external downloader that you want to use with `--downloader`
 
 ### Deprecated
diff --git a/devscripts/changelog_override.json b/devscripts/changelog_override.json
index ca811cb650..8c52864322 100644
--- a/devscripts/changelog_override.json
+++ b/devscripts/changelog_override.json
@@ -114,5 +114,11 @@
         "action": "add",
         "when": "f04b5bedad7b281bee9814686bba1762bae092eb",
         "short": "[priority] Security: [[CVE-2023-46121](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-46121)] Patch [Generic Extractor MITM Vulnerability via Arbitrary Proxy Injection](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-3ch3-jhc6-5r8x)\n\t- Disallow smuggling of arbitrary `http_headers`; extractors now only use specific headers"
+    },
+    {
+        "action": "change",
+        "when": "15f22b4880b6b3f71f350c64d70976ae65b9f1ca",
+        "short": "[webvtt] Allow spaces before newlines for CueBlock (#7681)",
+        "authors": ["TSRBerry"]
     }
 ]
diff --git a/devscripts/make_changelog.py b/devscripts/make_changelog.py
index d0e893e581..123eebc2a9 100644
--- a/devscripts/make_changelog.py
+++ b/devscripts/make_changelog.py
@@ -40,20 +40,6 @@ def subgroup_lookup(cls):
         return {
             name: group
             for group, names in {
-                cls.CORE: {
-                    'aes',
-                    'cache',
-                    'compat_utils',
-                    'compat',
-                    'cookies',
-                    'dependencies',
-                    'formats',
-                    'jsinterp',
-                    'outtmpl',
-                    'plugins',
-                    'update',
-                    'utils',
-                },
                 cls.MISC: {
                     'build',
                     'ci',
@@ -404,9 +390,9 @@ def groups(self):
             if not group:
                 if self.EXTRACTOR_INDICATOR_RE.search(commit.short):
                     group = CommitGroup.EXTRACTOR
+                    logger.error(f'Assuming [ie] group for {commit.short!r}')
                 else:
-                    group = CommitGroup.POSTPROCESSOR
-                logger.warning(f'Failed to map {commit.short!r}, selected {group.name.lower()}')
+                    group = CommitGroup.CORE
 
             commit_info = CommitInfo(
                 details, sub_details, message.strip(),
diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py
index 6c85e200fe..a5d59f3c03 100644
--- a/devscripts/make_issue_template.py
+++ b/devscripts/make_issue_template.py
@@ -9,11 +9,7 @@
 
 import re
 
-from devscripts.utils import (
-    get_filename_args,
-    read_file,
-    write_file,
-)
+from devscripts.utils import get_filename_args, read_file, write_file
 
 VERBOSE_TMPL = '''
   - type: checkboxes
diff --git a/requirements.txt b/requirements.txt
index d983fa03ff..06ff82a800 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,5 @@
 mutagen
 pycryptodomex
-websockets
 brotli; implementation_name=='cpython'
 brotlicffi; implementation_name!='cpython'
 certifi
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 8bff083145..0087cbc941 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -730,7 +730,7 @@ def expect_same_infodict(out):
                 self.assertEqual(got_dict.get(info_field), expected, info_field)
             return True
 
-        test('%()j', (expect_same_infodict, str))
+        test('%()j', (expect_same_infodict, None))
 
         # NA placeholder
         NA_TEST_OUTTMPL = '%(uploader_date)s-%(width)d-%(x|def)s-%(id)s.%(ext)s'
diff --git a/test/test_update.py b/test/test_update.py
index a5a388c106..bc139562f4 100644
--- a/test/test_update.py
+++ b/test/test_update.py
@@ -9,7 +9,7 @@
 
 
 from test.helper import FakeYDL, report_warning
-from yt_dlp.update import Updater, UpdateInfo
+from yt_dlp.update import UpdateInfo, Updater
 
 
 # XXX: Keep in sync with yt_dlp.update.UPDATE_SOURCES
diff --git a/test/test_utils.py b/test/test_utils.py
index 6c8571f980..c3e387cd0d 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -2110,6 +2110,8 @@ def test_traverse_obj(self):
         self.assertEqual(traverse_obj(_TEST_DATA, (..., {str_or_none})),
                          [item for item in map(str_or_none, _TEST_DATA.values()) if item is not None],
                          msg='Function in set should be a transformation')
+        self.assertEqual(traverse_obj(_TEST_DATA, ('fail', {lambda _: 'const'})), 'const',
+                         msg='Function in set should always be called')
         if __debug__:
             with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
                 traverse_obj(_TEST_DATA, set())
diff --git a/yt-dlp.cmd b/yt-dlp.cmd
index aa4500f9f1..5537e0ea9c 100644
--- a/yt-dlp.cmd
+++ b/yt-dlp.cmd
@@ -1 +1 @@
-@py -bb -Werror -Xdev "%~dp0yt_dlp\__main__.py" %*
+@py -Werror -Xdev "%~dp0yt_dlp\__main__.py" %*
diff --git a/yt-dlp.sh b/yt-dlp.sh
index 22a69250c0..ce74df801f 100755
--- a/yt-dlp.sh
+++ b/yt-dlp.sh
@@ -1,2 +1,2 @@
 #!/usr/bin/env sh
-exec "${PYTHON:-python3}" -bb -Werror -Xdev "$(dirname "$(realpath "$0")")/yt_dlp/__main__.py" "$@"
+exec "${PYTHON:-python3}" -Werror -Xdev "$(dirname "$(realpath "$0")")/yt_dlp/__main__.py" "$@"
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 5e28fd0e21..8d96498a67 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -60,7 +60,13 @@
     get_postprocessor,
 )
 from .postprocessor.ffmpeg import resolve_mapping as resolve_recode_mapping
-from .update import REPOSITORY, _get_system_deprecation, _make_label, current_git_head, detect_variant
+from .update import (
+    REPOSITORY,
+    _get_system_deprecation,
+    _make_label,
+    current_git_head,
+    detect_variant,
+)
 from .utils import (
     DEFAULT_OUTTMPL,
     IDENTITY,
diff --git a/yt_dlp/extractor/banbye.py b/yt_dlp/extractor/banbye.py
index dfcc82f021..67af29a962 100644
--- a/yt_dlp/extractor/banbye.py
+++ b/yt_dlp/extractor/banbye.py
@@ -152,7 +152,7 @@ def page_func(page_num):
                 'sort': 'new',
                 'limit': self._PAGE_SIZE,
                 'offset': page_num * self._PAGE_SIZE,
-            }, note=f'Downloading page {page_num+1}')
+            }, note=f'Downloading page {page_num + 1}')
             return [
                 self.url_result(f"{self._VIDEO_BASE}/{video['_id']}", BanByeIE)
                 for video in data['items']
diff --git a/yt_dlp/extractor/duoplay.py b/yt_dlp/extractor/duoplay.py
index e57fa7924f..7d3f39942d 100644
--- a/yt_dlp/extractor/duoplay.py
+++ b/yt_dlp/extractor/duoplay.py
@@ -53,21 +53,6 @@ class DuoplayIE(InfoExtractor):
             'episode_id': 14,
             'release_year': 2010,
         },
-    }, {
-        'note': 'Movie',
-        'url': 'https://duoplay.ee/4325/naljamangud',
-        'md5': '2b0bcac4159a08b1844c2bfde06b1199',
-        'info_dict': {
-            'id': '4325',
-            'ext': 'mp4',
-            'title': 'Näljamängud',
-            'thumbnail': r're:https://.+\.jpg(?:\?c=\d+)?$',
-            'description': 'md5:fb35f5eb2ff46cdb82e4d5fbe7b49a13',
-            'cast': ['Jennifer Lawrence', 'Josh Hutcherson', 'Liam Hemsworth'],
-            'upload_date': '20231109',
-            'timestamp': 1699552800,
-            'release_year': 2012,
-        },
     }, {
         'note': 'Movie without expiry',
         'url': 'https://duoplay.ee/5501/pilvede-all.-neljas-ode',
diff --git a/yt_dlp/extractor/floatplane.py b/yt_dlp/extractor/floatplane.py
index 09abb40bf6..2cf4d4e648 100644
--- a/yt_dlp/extractor/floatplane.py
+++ b/yt_dlp/extractor/floatplane.py
@@ -173,8 +173,8 @@ def format_path(params):
                 'formats': formats,
             })
 
-        uploader_url = format_field(traverse_obj(
-            post_data, 'creator'), 'urlname', 'https://www.floatplane.com/channel/%s/home', default=None)
+        uploader_url = format_field(
+            post_data, [('creator', 'urlname')], 'https://www.floatplane.com/channel/%s/home') or None
         channel_url = urljoin(f'{uploader_url}/', traverse_obj(post_data, ('channel', 'urlname')))
 
         post_info = {
@@ -248,7 +248,7 @@ def _fetch_page(self, display_id, creator_id, channel_id, page):
         for post in page_data or []:
             yield self.url_result(
                 f'https://www.floatplane.com/post/{post["id"]}',
-                ie=FloatplaneIE, video_id=post['id'], video_title=post.get('title'),
+                FloatplaneIE, id=post['id'], title=post.get('title'),
                 release_timestamp=parse_iso8601(post.get('releaseDate')))
 
     def _real_extract(self, url):
@@ -264,5 +264,5 @@ def _real_extract(self, url):
 
         return self.playlist_result(OnDemandPagedList(functools.partial(
             self._fetch_page, display_id, creator_data['id'], channel_data.get('id')), self._PAGE_SIZE),
-            display_id, playlist_title=channel_data.get('title') or creator_data.get('title'),
-            playlist_description=channel_data.get('about') or creator_data.get('about'))
+            display_id, title=channel_data.get('title') or creator_data.get('title'),
+            description=channel_data.get('about') or creator_data.get('about'))
diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index 606b4f5d1e..1f0011c09f 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -35,8 +35,8 @@
     unified_timestamp,
     unsmuggle_url,
     update_url_query,
-    urlhandle_detect_ext,
     url_or_none,
+    urlhandle_detect_ext,
     urljoin,
     variadic,
     xpath_attr,
diff --git a/yt_dlp/extractor/panopto.py b/yt_dlp/extractor/panopto.py
index 5ab2b2bcec..ddea32d707 100644
--- a/yt_dlp/extractor/panopto.py
+++ b/yt_dlp/extractor/panopto.py
@@ -536,7 +536,7 @@ def _fetch_page(self, base_url, query_params, display_id, page):
         }
 
         response = self._call_api(
-            base_url, '/Services/Data.svc/GetSessions', f'{display_id} page {page+1}',
+            base_url, '/Services/Data.svc/GetSessions', f'{display_id} page {page + 1}',
             data={'queryParameters': params}, fatal=False)
 
         for result in get_first(response, 'Results', default=[]):
diff --git a/yt_dlp/extractor/radiofrance.py b/yt_dlp/extractor/radiofrance.py
index ec1b97631e..6bd6fe9b68 100644
--- a/yt_dlp/extractor/radiofrance.py
+++ b/yt_dlp/extractor/radiofrance.py
@@ -264,7 +264,7 @@ def _real_extract(self, url):
         }
 
 
-class RadioFrancePlaylistBase(RadioFranceBaseIE):
+class RadioFrancePlaylistBaseIE(RadioFranceBaseIE):
     """Subclasses must set _METADATA_KEY"""
 
     def _call_api(self, content_id, cursor, page_num):
@@ -308,7 +308,7 @@ def _real_extract(self, url):
             })})
 
 
-class RadioFrancePodcastIE(RadioFrancePlaylistBase):
+class RadioFrancePodcastIE(RadioFrancePlaylistBaseIE):
     _VALID_URL = rf'''(?x)
         {RadioFranceBaseIE._VALID_URL_BASE}
         /(?:{RadioFranceBaseIE._STATIONS_RE})
@@ -369,7 +369,7 @@ def _call_api(self, podcast_id, cursor, page_num):
             note=f'Downloading page {page_num}', query={'pageCursor': cursor})
 
 
-class RadioFranceProfileIE(RadioFrancePlaylistBase):
+class RadioFranceProfileIE(RadioFrancePlaylistBaseIE):
     _VALID_URL = rf'{RadioFranceBaseIE._VALID_URL_BASE}/personnes/(?P<id>[\w-]+)'
 
     _TESTS = [{
diff --git a/yt_dlp/extractor/wordpress.py b/yt_dlp/extractor/wordpress.py
index 53820b57a9..378d99dbcc 100644
--- a/yt_dlp/extractor/wordpress.py
+++ b/yt_dlp/extractor/wordpress.py
@@ -70,7 +70,7 @@ def _extract_from_webpage(self, url, webpage):
                 'height': int_or_none(traverse_obj(track, ('dimensions', 'original', 'height'))),
                 'width': int_or_none(traverse_obj(track, ('dimensions', 'original', 'width'))),
             } for track in traverse_obj(playlist_json, ('tracks', ...), expected_type=dict)]
-            yield self.playlist_result(entries, self._generic_id(url) + f'-wp-playlist-{i+1}', 'Wordpress Playlist')
+            yield self.playlist_result(entries, self._generic_id(url) + f'-wp-playlist-{i + 1}', 'Wordpress Playlist')
 
 
 class WordpressMiniAudioPlayerEmbedIE(InfoExtractor):
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 5b14b187a7..88126d11f0 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -5297,6 +5297,7 @@ def _extract_webpage(self, url, item_id, fatal=True):
             # See: https://github.com/yt-dlp/yt-dlp/issues/116
             if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
                 retry.error = ExtractorError('Incomplete yt initial data received')
+                data = None
                 continue
 
         return webpage, data
diff --git a/yt_dlp/networking/__init__.py b/yt_dlp/networking/__init__.py
index 96c5a0678f..acadc0147d 100644
--- a/yt_dlp/networking/__init__.py
+++ b/yt_dlp/networking/__init__.py
@@ -28,4 +28,3 @@
     pass
 except Exception as e:
     warnings.warn(f'Failed to import "websockets" request handler: {e}' + bug_reports_message())
-
diff --git a/yt_dlp/networking/_helper.py b/yt_dlp/networking/_helper.py
index a6fa3550bd..d79dd79530 100644
--- a/yt_dlp/networking/_helper.py
+++ b/yt_dlp/networking/_helper.py
@@ -219,7 +219,7 @@ def _socket_connect(ip_addr, timeout, source_address):
             sock.bind(source_address)
         sock.connect(sa)
         return sock
-    except socket.error:
+    except OSError:
         sock.close()
         raise
 
@@ -237,7 +237,7 @@ def create_socks_proxy_socket(dest_addr, proxy_args, proxy_ip_addr, timeout, sou
             sock.bind(source_address)
         sock.connect(dest_addr)
         return sock
-    except socket.error:
+    except OSError:
         sock.close()
         raise
 
@@ -255,7 +255,7 @@ def create_connection(
     host, port = address
     ip_addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
     if not ip_addrs:
-        raise socket.error('getaddrinfo returns an empty list')
+        raise OSError('getaddrinfo returns an empty list')
     if source_address is not None:
         af = socket.AF_INET if ':' not in source_address[0] else socket.AF_INET6
         ip_addrs = [addr for addr in ip_addrs if addr[0] == af]
@@ -272,7 +272,7 @@ def create_connection(
             # https://bugs.python.org/issue36820
             err = None
             return sock
-        except socket.error as e:
+        except OSError as e:
             err = e
 
     try:
diff --git a/yt_dlp/networking/_requests.py b/yt_dlp/networking/_requests.py
index 9fb1d75f4a..e129110ca4 100644
--- a/yt_dlp/networking/_requests.py
+++ b/yt_dlp/networking/_requests.py
@@ -188,6 +188,7 @@ class RequestsSession(requests.sessions.Session):
     """
     Ensure unified redirect method handling with our urllib redirect handler.
     """
+
     def rebuild_method(self, prepared_request, response):
         new_method = get_redirect_method(prepared_request.method, response.status_code)
 
@@ -218,6 +219,7 @@ def filter(self, record):
 
 class Urllib3LoggingHandler(logging.Handler):
     """Redirect urllib3 logs to our logger"""
+
     def __init__(self, logger, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._logger = logger
@@ -367,7 +369,7 @@ def _new_conn(self):
                 self, f'Connection to {self.host} timed out. (connect timeout={self.timeout})') from e
         except SocksProxyError as e:
             raise urllib3.exceptions.ProxyError(str(e), e) from e
-        except (OSError, socket.error) as e:
+        except OSError as e:
             raise urllib3.exceptions.NewConnectionError(
                 self, f'Failed to establish a new connection: {e}') from e
 
diff --git a/yt_dlp/networking/_websockets.py b/yt_dlp/networking/_websockets.py
index ad85554e45..ed64080d62 100644
--- a/yt_dlp/networking/_websockets.py
+++ b/yt_dlp/networking/_websockets.py
@@ -5,20 +5,26 @@
 import ssl
 import sys
 
-from ._helper import create_connection, select_proxy, make_socks_proxy_opts, create_socks_proxy_socket
-from .common import Response, register_rh, Features
+from ._helper import (
+    create_connection,
+    create_socks_proxy_socket,
+    make_socks_proxy_opts,
+    select_proxy,
+)
+from .common import Features, Response, register_rh
 from .exceptions import (
     CertificateVerifyError,
     HTTPError,
+    ProxyError,
     RequestError,
     SSLError,
-    TransportError, ProxyError,
+    TransportError,
 )
 from .websocket import WebSocketRequestHandler, WebSocketResponse
 from ..compat import functools
 from ..dependencies import websockets
-from ..utils import int_or_none
 from ..socks import ProxyError as SocksProxyError
+from ..utils import int_or_none
 
 if not websockets:
     raise ImportError('websockets is not installed')
diff --git a/yt_dlp/networking/websocket.py b/yt_dlp/networking/websocket.py
index 09fcf78ac2..0e7e73c9e2 100644
--- a/yt_dlp/networking/websocket.py
+++ b/yt_dlp/networking/websocket.py
@@ -2,7 +2,7 @@
 
 import abc
 
-from .common import Response, RequestHandler
+from .common import RequestHandler, Response
 
 
 class WebSocketResponse(Response):
diff --git a/yt_dlp/socks.py b/yt_dlp/socks.py
index e7f41d7e2a..b4957ac2ed 100644
--- a/yt_dlp/socks.py
+++ b/yt_dlp/socks.py
@@ -49,7 +49,7 @@ class Socks5AddressType:
     ATYP_IPV6 = 0x04
 
 
-class ProxyError(socket.error):
+class ProxyError(OSError):
     ERR_SUCCESS = 0x00
 
     def __init__(self, code=None, msg=None):
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index 361617c028..89a0d4cff1 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -558,7 +558,7 @@ def decode(self, s):
                     s = self._close_object(e)
                     if s is not None:
                         continue
-                raise type(e)(f'{e.msg} in {s[e.pos-10:e.pos+10]!r}', s, e.pos)
+                raise type(e)(f'{e.msg} in {s[e.pos - 10:e.pos + 10]!r}', s, e.pos)
         assert False, 'Too many attempts to decode JSON'
 
 
@@ -1885,6 +1885,7 @@ def setproctitle(title):
     buf = ctypes.create_string_buffer(len(title_bytes))
     buf.value = title_bytes
     try:
+        # PR_SET_NAME = 15      Ref: /usr/include/linux/prctl.h
         libc.prctl(15, buf, 0, 0, 0)
     except AttributeError:
         return  # Strange libc, just skip this
@@ -2260,6 +2261,9 @@ def __getitem__(self, idx):
             raise self.IndexError()
         return entries[0]
 
+    def __bool__(self):
+        return bool(self.getslice(0, 1))
+
 
 class OnDemandPagedList(PagedList):
     """Download pages until a page with less than maximum results"""
@@ -5070,7 +5074,7 @@ def truncate_string(s, left, right=0):
     assert left > 3 and right >= 0
     if s is None or len(s) <= left + right:
         return s
-    return f'{s[:left-3]}...{s[-right:] if right else ""}'
+    return f'{s[:left - 3]}...{s[-right:] if right else ""}'
 
 
 def orderedSet_from_options(options, alias_dict, *, use_regex=False, start=None):
diff --git a/yt_dlp/utils/traversal.py b/yt_dlp/utils/traversal.py
index ff5703198a..5a2f69fccd 100644
--- a/yt_dlp/utils/traversal.py
+++ b/yt_dlp/utils/traversal.py
@@ -23,7 +23,7 @@ def traverse_obj(
 
     >>> obj = [{}, {"key": "value"}]
     >>> traverse_obj(obj, (1, "key"))
-    "value"
+    'value'
 
     Each of the provided `paths` is tested and the first producing a valid result will be returned.
     The next path will also be tested if the path branched but no results could be found.

From f10589e3453009bb523f55849bba144c9b91cf2a Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@protonmail.com>
Date: Sat, 30 Dec 2023 15:39:06 -0600
Subject: [PATCH 16/17] [docs] Update youtube-dl merge commit in `README.md`

Authored by: bashonly
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4f8fcb6c9c..cfd861c56b 100644
--- a/README.md
+++ b/README.md
@@ -76,7 +76,7 @@
 
 # NEW FEATURES
 
-* Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@66ab08**](https://github.com/ytdl-org/youtube-dl/commit/66ab0814c4baa2dc79c2dd5287bc0ad61a37c5b9) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))
+* Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@be008e6**](https://github.com/ytdl-org/youtube-dl/commit/be008e657d79832642e2158557c899249c9e31cd) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))
 
 * **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in YouTube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API
 

From 9f40cd289665b2fb8a05ccaf9721b3b2ca0f39c7 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Sat, 30 Dec 2023 21:43:13 +0000
Subject: [PATCH 17/17] Release 2023.12.30

Created by: bashonly

:ci skip all :ci run dl
---
 CONTRIBUTORS      |  14 ++++
 Changelog.md      |  87 ++++++++++++++++++++
 supportedsites.md | 203 ++++++++++++----------------------------------
 yt_dlp/version.py |   6 +-
 4 files changed, 154 insertions(+), 156 deletions(-)

diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 8b6b3671eb..adcc921444 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -528,3 +528,17 @@ almx
 elivinsky
 starius
 TravisDupes
+amir16yp
+Fymyte
+Ganesh910
+hashFactory
+kclauhk
+Kyraminol
+lstrojny
+middlingphys
+NickCis
+nicodato
+prettykool
+S-Aarab
+sonmezberkay
+TSRBerry
diff --git a/Changelog.md b/Changelog.md
index 6115446cb8..30de9072e0 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -4,6 +4,93 @@ # Changelog
 # To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
 -->
 
+### 2023.12.30
+
+#### Core changes
+- [Fix format selection parse error for CPython 3.12](https://github.com/yt-dlp/yt-dlp/commit/00cdda4f6fe18712ced13dbc64b7ea10f323e268) ([#8797](https://github.com/yt-dlp/yt-dlp/issues/8797)) by [Grub4K](https://github.com/Grub4K)
+- [Let `read_stdin` obey `--quiet`](https://github.com/yt-dlp/yt-dlp/commit/a174c453ee1e853c584ceadeac17eef2bd433dc5) by [pukkandan](https://github.com/pukkandan)
+- [Merged with youtube-dl be008e6](https://github.com/yt-dlp/yt-dlp/commit/65de7d204ce88c0225df1321060304baab85dbd8) by [bashonly](https://github.com/bashonly), [dirkf](https://github.com/dirkf), [Grub4K](https://github.com/Grub4K)
+- [Parse `release_year` from `release_date`](https://github.com/yt-dlp/yt-dlp/commit/1732eccc0a40256e076bf0435a29f0f1d8419280) ([#8524](https://github.com/yt-dlp/yt-dlp/issues/8524)) by [seproDev](https://github.com/seproDev)
+- [Release workflow and Updater cleanup](https://github.com/yt-dlp/yt-dlp/commit/632b8ee54eb2df8ac6e20746a0bd95b7ebb053aa) ([#8640](https://github.com/yt-dlp/yt-dlp/issues/8640)) by [bashonly](https://github.com/bashonly)
+- [Remove Python 3.7 support](https://github.com/yt-dlp/yt-dlp/commit/f4b95acafcd69a50040730dfdf732e797278fdcc) ([#8361](https://github.com/yt-dlp/yt-dlp/issues/8361)) by [bashonly](https://github.com/bashonly)
+- [Support `NO_COLOR` environment variable](https://github.com/yt-dlp/yt-dlp/commit/a0b19d319a6ce8b7059318fa17a34b144fde1785) ([#8385](https://github.com/yt-dlp/yt-dlp/issues/8385)) by [Grub4K](https://github.com/Grub4K), [prettykool](https://github.com/prettykool)
+- **outtmpl**: [Support multiplication](https://github.com/yt-dlp/yt-dlp/commit/993edd3f6e17e966c763bc86dc34125445cec6b6) by [pukkandan](https://github.com/pukkandan)
+- **utils**: `traverse_obj`: [Move `is_user_input` into output template](https://github.com/yt-dlp/yt-dlp/commit/0b6f829b1dfda15d3c1d7d1fbe4ea6102c26dd24) ([#8673](https://github.com/yt-dlp/yt-dlp/issues/8673)) by [Grub4K](https://github.com/Grub4K)
+- **webvtt**: [Allow spaces before newlines for CueBlock](https://github.com/yt-dlp/yt-dlp/commit/15f22b4880b6b3f71f350c64d70976ae65b9f1ca) ([#7681](https://github.com/yt-dlp/yt-dlp/issues/7681)) by [TSRBerry](https://github.com/TSRBerry) (With fixes in [298230e](https://github.com/yt-dlp/yt-dlp/commit/298230e550886b746c266724dd701d842ca2696e) by [pukkandan](https://github.com/pukkandan))
+
+#### Extractor changes
+- [Add `media_type` field](https://github.com/yt-dlp/yt-dlp/commit/e370f9ec36972d06100a3db893b397bfc1b07b4d) by [trainman261](https://github.com/trainman261)
+- [Extract from `media` elements in SMIL manifests](https://github.com/yt-dlp/yt-dlp/commit/ddb2d7588bea48bae965dbfabe6df6550c9d3d43) ([#8504](https://github.com/yt-dlp/yt-dlp/issues/8504)) by [seproDev](https://github.com/seproDev)
+- **abematv**: [Fix season metadata](https://github.com/yt-dlp/yt-dlp/commit/cc07f5cc85d9e2a6cd0bedb9d961665eea0d6047) ([#8607](https://github.com/yt-dlp/yt-dlp/issues/8607)) by [middlingphys](https://github.com/middlingphys)
+- **allstar**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/3237f8ba29fe13bf95ff42b1e48b5b5109715feb) ([#8274](https://github.com/yt-dlp/yt-dlp/issues/8274)) by [S-Aarab](https://github.com/S-Aarab)
+- **altcensored**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/3f90813f0617e0d21302398010de7496c9ae36aa) ([#8291](https://github.com/yt-dlp/yt-dlp/issues/8291)) by [drzraf](https://github.com/drzraf)
+- **ard**: [Overhaul extractors](https://github.com/yt-dlp/yt-dlp/commit/5f009a094f0e8450792b097c4c8273622778052d) ([#8878](https://github.com/yt-dlp/yt-dlp/issues/8878)) by [seproDev](https://github.com/seproDev)
+- **ardbetamediathek**: [Fix series extraction](https://github.com/yt-dlp/yt-dlp/commit/1f8bd8eba82ba10ddb49ee7cc0be4540dab103d5) ([#8687](https://github.com/yt-dlp/yt-dlp/issues/8687)) by [lstrojny](https://github.com/lstrojny)
+- **bbc**
+    - [Extract more formats](https://github.com/yt-dlp/yt-dlp/commit/c919b68f7e79ea5010f75f648d3c9e45405a8011) ([#8321](https://github.com/yt-dlp/yt-dlp/issues/8321)) by [barsnick](https://github.com/barsnick), [dirkf](https://github.com/dirkf)
+    - [Fix JSON parsing bug](https://github.com/yt-dlp/yt-dlp/commit/19741ab8a401ec64d5e84fdbfcfb141d105e7bc8) by [bashonly](https://github.com/bashonly)
+- **bfmtv**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/4903f452b68efb62dadf22e81be8c7934fc743e7) ([#8651](https://github.com/yt-dlp/yt-dlp/issues/8651)) by [bashonly](https://github.com/bashonly)
+- **bilibili**: [Support courses and interactive videos](https://github.com/yt-dlp/yt-dlp/commit/9f09bdcfcb8e2b4b2decdc30d35d34b993bc7a94) ([#8343](https://github.com/yt-dlp/yt-dlp/issues/8343)) by [c-basalt](https://github.com/c-basalt)
+- **bitchute**: [Fix and improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/b1a1ec1540605d2ea7abdb63336ffb1c56bf6316) ([#8507](https://github.com/yt-dlp/yt-dlp/issues/8507)) by [SirElderling](https://github.com/SirElderling)
+- **box**: [Fix formats extraction](https://github.com/yt-dlp/yt-dlp/commit/5a230233d6fce06f4abd1fce0dc92b948e6f780b) ([#8649](https://github.com/yt-dlp/yt-dlp/issues/8649)) by [bashonly](https://github.com/bashonly)
+- **bundestag**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/00a3e47bf5440c96025a76e08337ff2a475ed83e) ([#8783](https://github.com/yt-dlp/yt-dlp/issues/8783)) by [Grub4K](https://github.com/Grub4K)
+- **drtv**: [Set default ext for m3u8 formats](https://github.com/yt-dlp/yt-dlp/commit/f96ab86cd837b1b5823baa87d144e15322ee9298) ([#8590](https://github.com/yt-dlp/yt-dlp/issues/8590)) by [seproDev](https://github.com/seproDev)
+- **duoplay**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/66a0127d45033c698bdbedf162cddc55d9e7b906) ([#8542](https://github.com/yt-dlp/yt-dlp/issues/8542)) by [glensc](https://github.com/glensc)
+- **eplus**: [Add login support and DRM detection](https://github.com/yt-dlp/yt-dlp/commit/d5d1517e7d838500800d193ac3234b06e89654cd) ([#8661](https://github.com/yt-dlp/yt-dlp/issues/8661)) by [pzhlkj6612](https://github.com/pzhlkj6612)
+- **facebook**
+    - [Fix Memories extraction](https://github.com/yt-dlp/yt-dlp/commit/c39358a54bc6675ae0c50b81024e5a086e41656a) ([#8681](https://github.com/yt-dlp/yt-dlp/issues/8681)) by [kclauhk](https://github.com/kclauhk)
+    - [Improve subtitles extraction](https://github.com/yt-dlp/yt-dlp/commit/9cafb9ff17e14475a35c9a58b5bb010c86c9db4b) ([#8296](https://github.com/yt-dlp/yt-dlp/issues/8296)) by [kclauhk](https://github.com/kclauhk)
+- **floatplane**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/628fa244bbce2ad39775a5959e99588f30cac152) ([#8639](https://github.com/yt-dlp/yt-dlp/issues/8639)) by [seproDev](https://github.com/seproDev)
+- **francetv**: [Improve metadata extraction](https://github.com/yt-dlp/yt-dlp/commit/71f28097fec1c9e029f74b68a4eadc8915399840) ([#8409](https://github.com/yt-dlp/yt-dlp/issues/8409)) by [Fymyte](https://github.com/Fymyte)
+- **instagram**: [Fix stories extraction](https://github.com/yt-dlp/yt-dlp/commit/50eaea9fd7787546b53660e736325fa31c77765d) ([#8843](https://github.com/yt-dlp/yt-dlp/issues/8843)) by [bashonly](https://github.com/bashonly)
+- **joqrag**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/db8b4edc7d0bd27da462f6fe82ff6e13e3d68a04) ([#8384](https://github.com/yt-dlp/yt-dlp/issues/8384)) by [pzhlkj6612](https://github.com/pzhlkj6612)
+- **litv**: [Fix premium content extraction](https://github.com/yt-dlp/yt-dlp/commit/f45c4efcd928a173e1300a8f1ce4258e70c969b1) ([#8842](https://github.com/yt-dlp/yt-dlp/issues/8842)) by [bashonly](https://github.com/bashonly)
+- **maariv**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/c5f01bf7d4b9426c87c3f8248de23934a56579e0) ([#8331](https://github.com/yt-dlp/yt-dlp/issues/8331)) by [amir16yp](https://github.com/amir16yp)
+- **mediastream**: [Fix authenticated format extraction](https://github.com/yt-dlp/yt-dlp/commit/b03c89309eb141be1a1eceeeb7475dd3b7529ad9) ([#8657](https://github.com/yt-dlp/yt-dlp/issues/8657)) by [NickCis](https://github.com/NickCis)
+- **nebula**: [Overhaul extractors](https://github.com/yt-dlp/yt-dlp/commit/45d82be65f71bb05506bd55376c6fdb36bc54142) ([#8566](https://github.com/yt-dlp/yt-dlp/issues/8566)) by [elyse0](https://github.com/elyse0), [pukkandan](https://github.com/pukkandan), [seproDev](https://github.com/seproDev)
+- **nintendo**: [Fix Nintendo Direct extraction](https://github.com/yt-dlp/yt-dlp/commit/1d24da6c899ef280d8b0a48a5e280ecd5d39cdf4) ([#8609](https://github.com/yt-dlp/yt-dlp/issues/8609)) by [Grub4K](https://github.com/Grub4K)
+- **ondemandkorea**: [Fix upgraded format extraction](https://github.com/yt-dlp/yt-dlp/commit/04a5e06350e3ef7c03f94f2f3f90dd96c6411152) ([#8677](https://github.com/yt-dlp/yt-dlp/issues/8677)) by [seproDev](https://github.com/seproDev)
+- **pr0gramm**: [Support variant formats and subtitles](https://github.com/yt-dlp/yt-dlp/commit/f98a3305eb124a0c375d03209d5c5a64fe1766c8) ([#8674](https://github.com/yt-dlp/yt-dlp/issues/8674)) by [Grub4K](https://github.com/Grub4K)
+- **rinsefm**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/c91af948e43570025e4aa887e248fd025abae394) ([#8778](https://github.com/yt-dlp/yt-dlp/issues/8778)) by [hashFactory](https://github.com/hashFactory)
+- **rudovideo**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/0d531c35eca4c2eb36e160530a7a333edbc727cc) ([#8664](https://github.com/yt-dlp/yt-dlp/issues/8664)) by [nicodato](https://github.com/nicodato)
+- **theguardian**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/1fa3f24d4b5d22176b11d78420f1f4b64a5af0a8) ([#8535](https://github.com/yt-dlp/yt-dlp/issues/8535)) by [SirElderling](https://github.com/SirElderling)
+- **theplatform**: [Extract more metadata](https://github.com/yt-dlp/yt-dlp/commit/7e09c147fdccb44806bbf601573adc4b77210a89) ([#8635](https://github.com/yt-dlp/yt-dlp/issues/8635)) by [trainman261](https://github.com/trainman261)
+- **twitcasting**: [Detect livestreams via API and `show` page](https://github.com/yt-dlp/yt-dlp/commit/585d0ed9abcfcb957f2b2684b8ad43c3af160383) ([#8601](https://github.com/yt-dlp/yt-dlp/issues/8601)) by [bashonly](https://github.com/bashonly), [JC-Chung](https://github.com/JC-Chung)
+- **twitcastinguser**: [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/ff2fde1b8f922fd34bae6172602008cd67c07c93) ([#8650](https://github.com/yt-dlp/yt-dlp/issues/8650)) by [bashonly](https://github.com/bashonly)
+- **twitter**
+    - [Extract stale tweets](https://github.com/yt-dlp/yt-dlp/commit/1c54a98e19d047e7c15184237b6ef8ad50af489c) ([#8724](https://github.com/yt-dlp/yt-dlp/issues/8724)) by [bashonly](https://github.com/bashonly)
+    - [Prioritize m3u8 formats](https://github.com/yt-dlp/yt-dlp/commit/e7d22348e77367740da78a3db27167ecf894b7c9) ([#8826](https://github.com/yt-dlp/yt-dlp/issues/8826)) by [bashonly](https://github.com/bashonly)
+    - [Work around API rate-limit](https://github.com/yt-dlp/yt-dlp/commit/116c268438ea4d3738f6fa502c169081ca8f0ee7) ([#8825](https://github.com/yt-dlp/yt-dlp/issues/8825)) by [bashonly](https://github.com/bashonly)
+    - broadcast: [Extract `concurrent_view_count`](https://github.com/yt-dlp/yt-dlp/commit/6fe82491ed622b948c512cf4aab46ac3a234ae0a) ([#8600](https://github.com/yt-dlp/yt-dlp/issues/8600)) by [sonmezberkay](https://github.com/sonmezberkay)
+- **vidly**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/34df1c1f60fa652c0a6a5c712b06c10e45daf6b7) ([#8612](https://github.com/yt-dlp/yt-dlp/issues/8612)) by [seproDev](https://github.com/seproDev)
+- **vocaroo**: [Do not use deprecated `getheader`](https://github.com/yt-dlp/yt-dlp/commit/f223b1b0789f65e06619dcc9fc9e74f50d259379) ([#8606](https://github.com/yt-dlp/yt-dlp/issues/8606)) by [qbnu](https://github.com/qbnu)
+- **vvvvid**: [Set user-agent to fix extraction](https://github.com/yt-dlp/yt-dlp/commit/1725e943b0e8a8b585305660d4611e684374409c) ([#8615](https://github.com/yt-dlp/yt-dlp/issues/8615)) by [Kyraminol](https://github.com/Kyraminol)
+- **youtube**
+    - [Fix `like_count` extraction](https://github.com/yt-dlp/yt-dlp/commit/6b5d93b0b0240e287389d1d43b2d5293e18aa4cc) ([#8763](https://github.com/yt-dlp/yt-dlp/issues/8763)) by [Ganesh910](https://github.com/Ganesh910)
+    - [Improve detection of faulty HLS formats](https://github.com/yt-dlp/yt-dlp/commit/bb5a54e6db2422bbd155d93a0e105b6616c09467) ([#8646](https://github.com/yt-dlp/yt-dlp/issues/8646)) by [bashonly](https://github.com/bashonly)
+    - [Return empty playlist when channel/tab has no videos](https://github.com/yt-dlp/yt-dlp/commit/044886c220620a7679109e92352890e18b6079e3) by [pukkandan](https://github.com/pukkandan)
+    - [Support cf.piped.video](https://github.com/yt-dlp/yt-dlp/commit/6a9c7a2b52655bacfa7ab2da24fd0d14a6fff495) ([#8514](https://github.com/yt-dlp/yt-dlp/issues/8514)) by [OIRNOIR](https://github.com/OIRNOIR)
+- **zingmp3**: [Add support for radio and podcasts](https://github.com/yt-dlp/yt-dlp/commit/64de1a4c25bada90374b88d7353754fe8fbfcc51) ([#7189](https://github.com/yt-dlp/yt-dlp/issues/7189)) by [hatienl0i261299](https://github.com/hatienl0i261299)
+
+#### Postprocessor changes
+- **ffmpegmetadata**: [Embed stream metadata in single format downloads](https://github.com/yt-dlp/yt-dlp/commit/deeb13eae82e60f82a2c0c5861f460399a997528) ([#8647](https://github.com/yt-dlp/yt-dlp/issues/8647)) by [bashonly](https://github.com/bashonly)
+
+#### Networking changes
+- [Strip whitespace around header values](https://github.com/yt-dlp/yt-dlp/commit/196eb0fe77b78e2e5ca02c506c3837c2b1a7964c) ([#8802](https://github.com/yt-dlp/yt-dlp/issues/8802)) by [coletdjnz](https://github.com/coletdjnz)
+- **Request Handler**: websockets: [Migrate websockets to networking framework](https://github.com/yt-dlp/yt-dlp/commit/ccfd70f4c24b579c72123ca76ab50164f8f122b7) ([#7720](https://github.com/yt-dlp/yt-dlp/issues/7720)) by [coletdjnz](https://github.com/coletdjnz)
+
+#### Misc. changes
+- **ci**
+    - [Concurrency optimizations](https://github.com/yt-dlp/yt-dlp/commit/f124fa458826308afc86cf364c509f857686ecfd) ([#8614](https://github.com/yt-dlp/yt-dlp/issues/8614)) by [Grub4K](https://github.com/Grub4K)
+    - [Run core tests only for core changes](https://github.com/yt-dlp/yt-dlp/commit/13b3cb3c2b7169a1e17d6fc62593bf744170521c) ([#8841](https://github.com/yt-dlp/yt-dlp/issues/8841)) by [Grub4K](https://github.com/Grub4K)
+- **cleanup**
+    - [Fix spelling of `IE_NAME`](https://github.com/yt-dlp/yt-dlp/commit/bc4ab17b38f01000d99c5c2bedec89721fee65ec) ([#8810](https://github.com/yt-dlp/yt-dlp/issues/8810)) by [barsnick](https://github.com/barsnick)
+    - [Remove dead extractors](https://github.com/yt-dlp/yt-dlp/commit/9751a457cfdb18bf99d9ee0d10e4e6a594502bbf) ([#8604](https://github.com/yt-dlp/yt-dlp/issues/8604)) by [seproDev](https://github.com/seproDev)
+    - Miscellaneous: [f9fb3ce](https://github.com/yt-dlp/yt-dlp/commit/f9fb3ce86e3c6a0c3c33b45392b8d7288bceba76) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K), [pukkandan](https://github.com/pukkandan), [seproDev](https://github.com/seproDev)
+- **devscripts**: `run_tests`: [Create Python script](https://github.com/yt-dlp/yt-dlp/commit/2d1d683a541d71f3d3bb999dfe8eeb1976fb91ce) ([#8720](https://github.com/yt-dlp/yt-dlp/issues/8720)) by [Grub4K](https://github.com/Grub4K) (With fixes in [225cf2b](https://github.com/yt-dlp/yt-dlp/commit/225cf2b830a1de2c5eacd257edd2a01aed1e1114))
+- **docs**: [Update youtube-dl merge commit in `README.md`](https://github.com/yt-dlp/yt-dlp/commit/f10589e3453009bb523f55849bba144c9b91cf2a) by [bashonly](https://github.com/bashonly)
+- **test**: networking: [Update tests for OpenSSL 3.2](https://github.com/yt-dlp/yt-dlp/commit/37755a037e612bfc608c3d4722e8ef2ce6a022ee) ([#8814](https://github.com/yt-dlp/yt-dlp/issues/8814)) by [bashonly](https://github.com/bashonly)
+
 ### 2023.11.16
 
 #### Extractor changes
diff --git a/supportedsites.md b/supportedsites.md
index 0e971c135e..96681c16b9 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -1,6 +1,4 @@
 # Supported sites
- - **0000studio:archive**
- - **0000studio:clip**
  - **17live**
  - **17live:clip**
  - **1News**: 1news.co.nz article videos
@@ -9,7 +7,6 @@ # Supported sites
  - **23video**
  - **247sports**
  - **24tv.ua**
- - **24video**
  - **3qsdn**: 3Q SDN
  - **3sat**
  - **4tube**
@@ -50,15 +47,18 @@ # Supported sites
  - **afreecatv**: [*afreecatv*](## "netrc machine") afreecatv.com
  - **afreecatv:live**: [*afreecatv*](## "netrc machine") afreecatv.com
  - **afreecatv:user**
- - **AirMozilla**
  - **AirTV**
  - **AitubeKZVideo**
  - **AliExpressLive**
  - **AlJazeera**
  - **Allocine**
+ - **Allstar**
+ - **AllstarProfile**
  - **AlphaPorno**
  - **Alsace20TV**
  - **Alsace20TVEmbed**
+ - **altcensored**
+ - **altcensored:channel**
  - **Alura**: [*alura*](## "netrc machine")
  - **AluraCourse**: [*aluracourse*](## "netrc machine")
  - **Amara**
@@ -79,7 +79,7 @@ # Supported sites
  - **ant1newsgr:embed**: ant1news.gr embedded videos
  - **antenna:watch**: antenna.gr and ant1news.gr videos
  - **Anvato**
- - **aol.com**: Yahoo screen and movies
+ - **aol.com**: Yahoo screen and movies (**Currently broken**)
  - **APA**
  - **Aparat**
  - **AppleConnect**
@@ -90,8 +90,8 @@ # Supported sites
  - **archive.org**: archive.org video and audio
  - **ArcPublishing**
  - **ARD**
- - **ARD:mediathek**
- - **ARDBetaMediathek**
+ - **ARDMediathek**
+ - **ARDMediathekCollection**
  - **Arkena**
  - **arte.sky.it**
  - **ArteTV**
@@ -100,7 +100,6 @@ # Supported sites
  - **ArteTVPlaylist**
  - **AtresPlayer**: [*atresplayer*](## "netrc machine")
  - **AtScaleConfEvent**
- - **ATTTechChannel**
  - **ATVAt**
  - **AudiMedia**
  - **AudioBoom**
@@ -140,12 +139,12 @@ # Supported sites
  - **BeatBumpVideo**
  - **Beatport**
  - **Beeg**
- - **BehindKink**
+ - **BehindKink**: (**Currently broken**)
  - **Bellator**
  - **BellMedia**
  - **BerufeTV**
- - **Bet**
- - **bfi:player**
+ - **Bet**: (**Currently broken**)
+ - **bfi:player**: (**Currently broken**)
  - **bfmtv**
  - **bfmtv:article**
  - **bfmtv:live**
@@ -162,6 +161,8 @@ # Supported sites
  - **BiliBiliBangumi**
  - **BiliBiliBangumiMedia**
  - **BiliBiliBangumiSeason**
+ - **BilibiliCheese**
+ - **BilibiliCheeseSeason**
  - **BilibiliCollectionList**
  - **BilibiliFavoritesList**
  - **BiliBiliPlayer**
@@ -176,11 +177,8 @@ # Supported sites
  - **BiliLive**
  - **BioBioChileTV**
  - **Biography**
- - **BIQLE**
  - **BitChute**
  - **BitChuteChannel**
- - **bitwave:replay**
- - **bitwave:stream**
  - **BlackboardCollaborate**
  - **BleacherReport**
  - **BleacherReportCMS**
@@ -193,7 +191,7 @@ # Supported sites
  - **Box**
  - **BoxCastVideo**
  - **Bpb**: Bundeszentrale für politische Bildung
- - **BR**: Bayerischer Rundfunk
+ - **BR**: Bayerischer Rundfunk (**Currently broken**)
  - **BrainPOP**: [*brainpop*](## "netrc machine")
  - **BrainPOPELL**: [*brainpop*](## "netrc machine")
  - **BrainPOPEsp**: [*brainpop*](## "netrc machine") BrainPOP Español
@@ -201,19 +199,18 @@ # Supported sites
  - **BrainPOPIl**: [*brainpop*](## "netrc machine") BrainPOP Hebrew
  - **BrainPOPJr**: [*brainpop*](## "netrc machine")
  - **BravoTV**
- - **Break**
  - **BreitBart**
  - **brightcove:legacy**
  - **brightcove:new**
  - **Brilliantpala:Classes**: [*brilliantpala*](## "netrc machine") VoD on classes.brilliantpala.org
  - **Brilliantpala:Elearn**: [*brilliantpala*](## "netrc machine") VoD on elearn.brilliantpala.org
- - **BRMediathek**: Bayerischer Rundfunk Mediathek
  - **bt:article**: Bergens Tidende Articles
  - **bt:vestlendingen**: Bergens Tidende - Vestlendingen
  - **Bundesliga**
+ - **Bundestag**
  - **BusinessInsider**
  - **BuzzFeed**
- - **BYUtv**
+ - **BYUtv**: (**Currently broken**)
  - **CableAV**
  - **Callin**
  - **Caltrans**
@@ -225,14 +222,11 @@ # Supported sites
  - **CamModels**
  - **Camsoda**
  - **CamtasiaEmbed**
- - **CamWithHer**
  - **Canal1**
  - **CanalAlpha**
  - **canalc2.tv**
  - **Canalplus**: mycanal.fr and piwiplus.fr
  - **CaracolTvPlay**: [*caracoltv-play*](## "netrc machine")
- - **CarambaTV**
- - **CarambaTVPage**
  - **CartoonNetwork**
  - **cbc.ca**
  - **cbc.ca:player**
@@ -254,16 +248,12 @@ # Supported sites
  - **Cellebrite**
  - **CeskaTelevize**
  - **CGTN**
- - **channel9**: Channel 9
  - **CharlieRose**
  - **Chaturbate**
  - **Chilloutzone**
  - **Chingari**
  - **ChingariUser**
- - **chirbit**
- - **chirbit:profile**
  - **cielotv.it**
- - **Cinchcast**
  - **Cinemax**
  - **CinetecaMilano**
  - **Cineverse**
@@ -276,14 +266,12 @@ # Supported sites
  - **cliphunter**
  - **Clippit**
  - **ClipRs**
- - **Clipsyndicate**
  - **ClipYouEmbed**
  - **CloserToTruth**
  - **CloudflareStream**
- - **Cloudy**
- - **Clubic**
+ - **Clubic**: (**Currently broken**)
  - **Clyp**
- - **cmt.com**
+ - **cmt.com**: (**Currently broken**)
  - **CNBC**
  - **CNBCVideo**
  - **CNN**
@@ -328,7 +316,6 @@ # Supported sites
  - **CybraryCourse**: [*cybrary*](## "netrc machine")
  - **DacastPlaylist**
  - **DacastVOD**
- - **Daftsex**
  - **DagelijkseKost**: dagelijksekost.een.be
  - **DailyMail**
  - **dailymotion**: [*dailymotion*](## "netrc machine")
@@ -347,13 +334,12 @@ # Supported sites
  - **DctpTv**
  - **DeezerAlbum**
  - **DeezerPlaylist**
- - **defense.gouv.fr**
  - **democracynow**
  - **DestinationAmerica**
  - **DetikEmbed**
  - **DeuxM**
  - **DeuxMNews**
- - **DHM**: Filmarchiv - Deutsches Historisches Museum
+ - **DHM**: Filmarchiv - Deutsches Historisches Museum (**Currently broken**)
  - **Digg**
  - **DigitalConcertHall**: [*digitalconcerthall*](## "netrc machine") DigitalConcertHall extractor
  - **DigitallySpeaking**
@@ -373,7 +359,6 @@ # Supported sites
  - **dlf:corpus**: DLF Multi-feed Archives
  - **dlive:stream**
  - **dlive:vod**
- - **Dotsub**
  - **Douyin**
  - **DouyuShow**
  - **DouyuTV**: 斗鱼直播
@@ -392,35 +377,29 @@ # Supported sites
  - **duboku**: www.duboku.io
  - **duboku:list**: www.duboku.io entire series
  - **Dumpert**
+ - **Duoplay**
  - **dvtv**: http://video.aktualne.cz/
  - **dw**
  - **dw:article**
  - **EaglePlatform**
  - **EbaumsWorld**
  - **Ebay**
- - **EchoMsk**
  - **egghead:course**: egghead.io course
  - **egghead:lesson**: egghead.io lesson
- - **ehftv**
- - **eHow**
  - **EinsUndEinsTV**: [*1und1tv*](## "netrc machine")
  - **EinsUndEinsTVLive**: [*1und1tv*](## "netrc machine")
  - **EinsUndEinsTVRecordings**: [*1und1tv*](## "netrc machine")
  - **Einthusan**
  - **eitb.tv**
- - **ElevenSports**
- - **EllenTube**
- - **EllenTubePlaylist**
- - **EllenTubeVideo**
  - **Elonet**
  - **ElPais**: El País
  - **ElTreceTV**: El Trece TV (Argentina)
  - **Embedly**
  - **EMPFlix**
- - **Engadget**
  - **Epicon**
  - **EpiconSeries**
- - **eplus:inbound**: e+ (イープラス) overseas
+ - **EpidemicSound**
+ - **eplus**: [*eplus*](## "netrc machine") e+ (イープラス)
  - **Epoch**
  - **Eporner**
  - **Erocast**
@@ -429,11 +408,9 @@ # Supported sites
  - **ertflix**: ERTFLIX videos
  - **ertflix:codename**: ERTFLIX videos by codename
  - **ertwebtv:embed**: ert.gr webtv embedded videos
- - **Escapist**
  - **ESPN**
  - **ESPNArticle**
  - **ESPNCricInfo**
- - **EsriVideo**
  - **EttuTv**
  - **Europa**
  - **EuroParlWebstream**
@@ -443,9 +420,7 @@ # Supported sites
  - **EWETV**: [*ewetv*](## "netrc machine")
  - **EWETVLive**: [*ewetv*](## "netrc machine")
  - **EWETVRecordings**: [*ewetv*](## "netrc machine")
- - **ExpoTV**
  - **Expressen**
- - **ExtremeTube**
  - **EyedoTV**
  - **facebook**: [*facebook*](## "netrc machine")
  - **facebook:reel**
@@ -465,6 +440,8 @@ # Supported sites
  - **FiveThirtyEight**
  - **FiveTV**
  - **Flickr**
+ - **Floatplane**
+ - **FloatplaneChannel**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
  - **FoodNetwork**
  - **FootyRoom**
@@ -472,7 +449,6 @@ # Supported sites
  - **FOX**
  - **FOX9**
  - **FOX9News**
- - **Foxgay**
  - **foxnews**: Fox News and Fox Business Video
  - **foxnews:article**
  - **FoxNewsVideo**
@@ -496,7 +472,6 @@ # Supported sites
  - **funimation:show**: [*funimation*](## "netrc machine")
  - **Funk**
  - **Funker530**
- - **Fusion**
  - **Fux**
  - **FuyinTV**
  - **Gab**
@@ -522,7 +497,6 @@ # Supported sites
  - **GeniusLyrics**
  - **Gettr**
  - **GettrStreaming**
- - **Gfycat**
  - **GiantBomb**
  - **Giga**
  - **GlattvisionTV**: [*glattvisiontv*](## "netrc machine")
@@ -564,7 +538,6 @@ # Supported sites
  - **HearThisAt**
  - **Heise**
  - **HellPorno**
- - **Helsinki**: helsinki.fi
  - **hetklokhuis**
  - **hgtv.com:show**
  - **HGTVDe**
@@ -573,8 +546,6 @@ # Supported sites
  - **HistoricFilms**
  - **history:player**
  - **history:topic**: History.com Topic
- - **hitbox**
- - **hitbox:live**
  - **HitRecord**
  - **hketv**: 香港教育局教育電視 (HKETV) Educational Television, Hong Kong Educational Bureau
  - **HollywoodReporter**
@@ -585,8 +556,6 @@ # Supported sites
  - **hotstar:playlist**
  - **hotstar:season**
  - **hotstar:series**
- - **Howcast**
- - **HowStuffWorks**
  - **hrfernsehen**
  - **HRTi**: [*hrti*](## "netrc machine")
  - **HRTiPlaylist**: [*hrti*](## "netrc machine")
@@ -608,7 +577,7 @@ # Supported sites
  - **ign.com**
  - **IGNArticle**
  - **IGNVideo**
- - **IHeartRadio**
+ - **iheartradio**
  - **iheartradio:podcast**
  - **Iltalehti**
  - **imdb**: Internet Movie Database trailers
@@ -638,7 +607,6 @@ # Supported sites
  - **IsraelNationalNews**
  - **ITProTV**
  - **ITProTVCourse**
- - **ITTF**
  - **ITV**
  - **ITVBTCC**
  - **ivi**: ivi.ru
@@ -658,6 +626,7 @@ # Supported sites
  - **JioSaavnAlbum**
  - **JioSaavnSong**
  - **Joj**
+ - **JoqrAg**: 超!A&G+ 文化放送 (f.k.a. AGQR) Nippon Cultural Broadcasting, Inc. (JOQR)
  - **Jove**
  - **JStream**
  - **JTBC**: jtbc.co.kr
@@ -670,7 +639,6 @@ # Supported sites
  - **Karaoketv**
  - **KarriereVideos**
  - **Katsomo**
- - **KeezMovies**
  - **KelbyOne**
  - **Ketnet**
  - **khanacademy**
@@ -679,7 +647,7 @@ # Supported sites
  - **Kicker**
  - **KickStarter**
  - **KickVOD**
- - **KinjaEmbed**
+ - **kinja:embed**
  - **KinoPoisk**
  - **Kommunetv**
  - **KompasVideo**
@@ -698,8 +666,6 @@ # Supported sites
  - **la7.it**
  - **la7.it:​pod:episode**
  - **la7.it:podcast**
- - **laola1tv**
- - **laola1tv:embed**
  - **LastFM**
  - **LastFMPlaylist**
  - **LastFMUser**
@@ -733,7 +699,6 @@ # Supported sites
  - **LinkedIn**: [*linkedin*](## "netrc machine")
  - **linkedin:learning**: [*linkedin*](## "netrc machine")
  - **linkedin:​learning:course**: [*linkedin*](## "netrc machine")
- - **LinuxAcademy**: [*linuxacademy*](## "netrc machine")
  - **Liputan6**
  - **ListenNotes**
  - **LiTV**
@@ -751,7 +716,7 @@ # Supported sites
  - **Lumni**
  - **lynda**: [*lynda*](## "netrc machine") lynda.com videos
  - **lynda:course**: [*lynda*](## "netrc machine") lynda.com online courses
- - **m6**
+ - **maariv.co.il**
  - **MagellanTV**
  - **MagentaMusik360**
  - **mailru**: Видео@Mail.Ru
@@ -793,11 +758,8 @@ # Supported sites
  - **megatvcom:embed**: megatv.com embedded videos
  - **Meipai**: 美拍
  - **MelonVOD**
- - **META**
- - **metacafe**
  - **Metacritic**
  - **mewatch**
- - **Mgoon**
  - **MiaoPai**
  - **MicrosoftEmbed**
  - **microsoftstream**: Microsoft Stream
@@ -810,7 +772,6 @@ # Supported sites
  - **minds:group**
  - **MinistryGrid**
  - **Minoto**
- - **miomio.tv**
  - **mirrativ**
  - **mirrativ:user**
  - **MirrorCoUK**
@@ -825,14 +786,10 @@ # Supported sites
  - **MLBTV**: [*mlb*](## "netrc machine")
  - **MLBVideo**
  - **MLSSoccer**
- - **Mnet**
  - **MNetTV**: [*mnettv*](## "netrc machine")
  - **MNetTVLive**: [*mnettv*](## "netrc machine")
  - **MNetTVRecordings**: [*mnettv*](## "netrc machine")
  - **MochaVideo**
- - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
- - **Mofosex**
- - **MofosexEmbed**
  - **Mojvideo**
  - **Monstercat**
  - **MonsterSirenHypergryphMusic**
@@ -843,13 +800,12 @@ # Supported sites
  - **Motorsport**: motorsport.com
  - **MotorTrend**
  - **MotorTrendOnDemand**
- - **MovieClips**
  - **MovieFap**
  - **Moviepilot**
  - **MoviewPlay**
  - **Moviezine**
  - **MovingImage**
- - **MSN**
+ - **MSN**: (**Currently broken**)
  - **mtg**: MTG services
  - **mtv**
  - **mtv.de**
@@ -871,18 +827,13 @@ # Supported sites
  - **MusicdexSong**
  - **mva**: Microsoft Virtual Academy videos
  - **mva:course**: Microsoft Virtual Academy courses
- - **Mwave**
- - **MwaveMeetGreet**
  - **Mxplayer**
  - **MxplayerShow**
- - **MyChannels**
  - **MySpace**
  - **MySpace:album**
  - **MySpass**
- - **Myvi**
  - **MyVideoGe**
  - **MyVidster**
- - **MyviEmbed**
  - **Mzaalo**
  - **n-tv.de**
  - **N1Info:article**
@@ -894,12 +845,12 @@ # Supported sites
  - **Naver**
  - **Naver:live**
  - **navernow**
- - **NBA**
+ - **nba**
+ - **nba:channel**
+ - **nba:embed**
  - **nba:watch**
  - **nba:​watch:collection**
- - **NBAChannel**
- - **NBAEmbed**
- - **NBAWatchEmbed**
+ - **nba:​watch:embed**
  - **NBC**
  - **NBCNews**
  - **nbcolympics**
@@ -914,6 +865,7 @@ # Supported sites
  - **NDTV**
  - **Nebula**: [*watchnebula*](## "netrc machine")
  - **nebula:channel**: [*watchnebula*](## "netrc machine")
+ - **nebula:class**: [*watchnebula*](## "netrc machine")
  - **nebula:subscriptions**: [*watchnebula*](## "netrc machine")
  - **NekoHacker**
  - **NerdCubedFeed**
@@ -935,7 +887,6 @@ # Supported sites
  - **Newgrounds:playlist**
  - **Newgrounds:user**
  - **NewsPicks**
- - **Newstube**
  - **Newsy**
  - **NextMedia**: 蘋果日報
  - **NextMediaActionNews**: 蘋果日報 - 動新聞
@@ -961,7 +912,6 @@ # Supported sites
  - **nick.de**
  - **nickelodeon:br**
  - **nickelodeonru**
- - **nicknight**
  - **niconico**: [*niconico*](## "netrc machine") ニコニコ動画
  - **niconico:history**: NicoNico user history or likes. Requires cookies.
  - **niconico:live**: ニコニコ生放送
@@ -984,9 +934,7 @@ # Supported sites
  - **NonkTube**
  - **NoodleMagazine**
  - **Noovo**
- - **Normalboots**
  - **NOSNLArticle**
- - **NosVideo**
  - **Nova**: TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz
  - **NovaEmbed**
  - **NovaPlay**
@@ -1009,7 +957,7 @@ # Supported sites
  - **NRKTVEpisodes**
  - **NRKTVSeason**
  - **NRKTVSeries**
- - **NRLTV**
+ - **NRLTV**: (**Currently broken**)
  - **ntv.ru**
  - **NubilesPorn**: [*nubiles-porn*](## "netrc machine")
  - **Nuvid**
@@ -1037,8 +985,6 @@ # Supported sites
  - **onet.tv:channel**
  - **OnetMVP**
  - **OnionStudios**
- - **Ooyala**
- - **OoyalaExternal**
  - **Opencast**
  - **OpencastPlaylist**
  - **openrec**
@@ -1060,7 +1006,6 @@ # Supported sites
  - **PalcoMP3:artist**
  - **PalcoMP3:song**
  - **PalcoMP3:video**
- - **pandora.tv**: 판도라TV
  - **Panopto**
  - **PanoptoList**
  - **PanoptoPlaylist**
@@ -1082,7 +1027,6 @@ # Supported sites
  - **PeerTube:Playlist**
  - **peloton**: [*peloton*](## "netrc machine")
  - **peloton:live**: Peloton Live
- - **People**
  - **PerformGroup**
  - **periscope**: Periscope
  - **periscope:user**: Periscope user videos
@@ -1104,14 +1048,11 @@ # Supported sites
  - **PlanetMarathi**
  - **Platzi**: [*platzi*](## "netrc machine")
  - **PlatziCourse**: [*platzi*](## "netrc machine")
- - **play.fm**
  - **player.sky.it**
  - **PlayPlusTV**: [*playplustv*](## "netrc machine")
  - **PlayStuff**
- - **PlaysTV**
  - **PlaySuisse**
  - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
- - **Playvid**
  - **PlayVids**
  - **Playwire**
  - **pluralsight**: [*pluralsight*](## "netrc machine")
@@ -1136,11 +1077,8 @@ # Supported sites
  - **Popcorntimes**
  - **PopcornTV**
  - **Pornbox**
- - **PornCom**
  - **PornerBros**
- - **Pornez**
  - **PornFlip**
- - **PornHd**
  - **PornHub**: [*pornhub*](## "netrc machine") PornHub and Thumbzilla
  - **PornHubPagedVideoList**: [*pornhub*](## "netrc machine")
  - **PornHubPlaylist**: [*pornhub*](## "netrc machine")
@@ -1182,7 +1120,6 @@ # Supported sites
  - **Radiko**
  - **RadikoRadio**
  - **radio.de**
- - **radiobremen**
  - **radiocanada**
  - **radiocanada:audiovideo**
  - **RadioComercial**
@@ -1222,7 +1159,6 @@ # Supported sites
  - **RCTIPlusSeries**
  - **RCTIPlusTV**
  - **RDS**: RDS.ca
- - **Recurbate**
  - **RedBull**
  - **RedBullEmbed**
  - **RedBullTV**
@@ -1239,7 +1175,7 @@ # Supported sites
  - **Reuters**
  - **ReverbNation**
  - **RheinMainTV**
- - **RICE**
+ - **RinseFM**
  - **RMCDecouverte**
  - **RockstarGames**
  - **Rokfin**: [*rokfin*](## "netrc machine")
@@ -1260,8 +1196,6 @@ # Supported sites
  - **rtl.lu:tele-vod**
  - **rtl.nl**: rtl.nl and rtlxl.nl
  - **rtl2**
- - **rtl2:you**
- - **rtl2:​you:series**
  - **RTLLuLive**
  - **RTLLuRadio**
  - **RTNews**
@@ -1276,10 +1210,9 @@ # Supported sites
  - **rtve.es:infantil**: RTVE infantil
  - **rtve.es:live**: RTVE.es live streams
  - **rtve.es:television**
- - **RTVNH**
  - **RTVS**
  - **rtvslo.si**
- - **RUHD**
+ - **RudoVideo**
  - **Rule34Video**
  - **Rumble**
  - **RumbleChannel**
@@ -1326,8 +1259,8 @@ # Supported sites
  - **ScrippsNetworks**
  - **scrippsnetworks:watch**
  - **Scrolller**
- - **SCTE**: [*scte*](## "netrc machine")
- - **SCTECourse**: [*scte*](## "netrc machine")
+ - **SCTE**: [*scte*](## "netrc machine") (**Currently broken**)
+ - **SCTECourse**: [*scte*](## "netrc machine") (**Currently broken**)
  - **Seeker**
  - **SenalColombiaLive**
  - **SenateGov**
@@ -1339,7 +1272,6 @@ # Supported sites
  - **SeznamZpravyArticle**
  - **Shahid**: [*shahid*](## "netrc machine")
  - **ShahidShow**
- - **Shared**: shared.sx
  - **ShareVideosEmbed**
  - **ShemarooMe**
  - **ShowRoomLive**
@@ -1391,7 +1323,6 @@ # Supported sites
  - **SovietsClosetPlaylist**
  - **SpankBang**
  - **SpankBangPlaylist**
- - **Spankwire**
  - **Spiegel**
  - **Sport5**
  - **SportBox**
@@ -1404,7 +1335,7 @@ # Supported sites
  - **SpreakerShowPage**
  - **SpringboardPlatform**
  - **Sprout**
- - **sr:mediathek**: Saarländischer Rundfunk
+ - **sr:mediathek**: Saarländischer Rundfunk (**Currently broken**)
  - **SRGSSR**
  - **SRGSSRPlay**: srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites
  - **StacommuLive**: [*stacommu*](## "netrc machine")
@@ -1421,7 +1352,6 @@ # Supported sites
  - **StoryFireSeries**
  - **StoryFireUser**
  - **Streamable**
- - **streamcloud.eu**
  - **StreamCZ**
  - **StreamFF**
  - **StreetVoice**
@@ -1437,7 +1367,6 @@ # Supported sites
  - **SVTPlay**: SVT Play and Öppet arkiv
  - **SVTSeries**
  - **SwearnetEpisode**
- - **SWRMediathek**
  - **Syfy**
  - **SYVDK**
  - **SztvHu**
@@ -1456,7 +1385,6 @@ # Supported sites
  - **TeachingChannel**
  - **Teamcoco**
  - **TeamTreeHouse**: [*teamtreehouse*](## "netrc machine")
- - **TechTalks**
  - **techtv.mit.edu**
  - **TedEmbed**
  - **TedPlaylist**
@@ -1486,6 +1414,8 @@ # Supported sites
  - **TFO**
  - **theatercomplextown:ppv**: [*theatercomplextown*](## "netrc machine")
  - **theatercomplextown:vod**: [*theatercomplextown*](## "netrc machine")
+ - **TheGuardianPodcast**
+ - **TheGuardianPodcastPlaylist**
  - **TheHoleTv**
  - **TheIntercept**
  - **ThePlatform**
@@ -1506,27 +1436,23 @@ # Supported sites
  - **tiktok:sound**: (**Currently broken**)
  - **tiktok:tag**: (**Currently broken**)
  - **tiktok:user**: (**Currently broken**)
- - **tinypic**: tinypic.com videos
  - **TLC**
  - **TMZ**
  - **TNAFlix**
  - **TNAFlixNetworkEmbed**
  - **toggle**
  - **toggo**
- - **Tokentube**
- - **Tokentube:channel**
  - **tokfm:audition**
  - **tokfm:podcast**
  - **ToonGoggles**
  - **tou.tv**: [*toutv*](## "netrc machine")
- - **Toypics**: Toypics video
- - **ToypicsUser**: Toypics user profile
+ - **Toypics**: Toypics video (**Currently broken**)
+ - **ToypicsUser**: Toypics user profile (**Currently broken**)
  - **TrailerAddict**: (**Currently broken**)
  - **TravelChannel**
  - **Triller**: [*triller*](## "netrc machine")
  - **TrillerShort**
  - **TrillerUser**: [*triller*](## "netrc machine")
- - **Trilulilu**
  - **Trovo**
  - **TrovoChannelClip**: All Clips of a trovo.live channel; "trovoclip:" prefix
  - **TrovoChannelVod**: All VODs of a trovo.live channel; "trovovod:" prefix
@@ -1536,7 +1462,7 @@ # Supported sites
  - **TruNews**
  - **Truth**
  - **TruTV**
- - **Tube8**
+ - **Tube8**: (**Currently broken**)
  - **TubeTuGraz**: [*tubetugraz*](## "netrc machine") tube.tugraz.at
  - **TubeTuGrazSeries**: [*tubetugraz*](## "netrc machine")
  - **TubiTv**: [*tubitv*](## "netrc machine")
@@ -1545,7 +1471,6 @@ # Supported sites
  - **TuneInPodcast**
  - **TuneInPodcastEpisode**
  - **TuneInStation**
- - **TunePk**
  - **Turbo**
  - **tv.dfb.de**
  - **TV2**
@@ -1569,14 +1494,7 @@ # Supported sites
  - **TVIPlayer**
  - **tvland.com**
  - **TVN24**
- - **TVNet**
  - **TVNoe**
- - **TVNow**
- - **TVNowAnnual**
- - **TVNowFilm**
- - **TVNowNew**
- - **TVNowSeason**
- - **TVNowShow**
  - **tvopengr:embed**: tvopen.gr embedded videos
  - **tvopengr:watch**: tvopen.gr (and ethnos.gr) videos
  - **tvp**: Telewizja Polska
@@ -1614,7 +1532,6 @@ # Supported sites
  - **umg:de**: Universal Music Deutschland
  - **Unistra**
  - **Unity**
- - **UnscriptedNewsVideo**
  - **uol.com.br**
  - **uplynk**
  - **uplynk:preplay**
@@ -1629,7 +1546,6 @@ # Supported sites
  - **Utreon**
  - **Varzesh3**
  - **Vbox7**
- - **VeeHD**
  - **Veo**
  - **Veoh**
  - **veoh:user**
@@ -1642,7 +1558,6 @@ # Supported sites
  - **vice**
  - **vice:article**
  - **vice:show**
- - **Vidbit**
  - **Viddler**
  - **Videa**
  - **video.arnes.si**: Arnes Video
@@ -1664,6 +1579,7 @@ # Supported sites
  - **VidioLive**: [*vidio*](## "netrc machine")
  - **VidioPremier**: [*vidio*](## "netrc machine")
  - **VidLii**
+ - **Vidly**
  - **viewlift**
  - **viewlift:embed**
  - **Viidea**
@@ -1683,7 +1599,6 @@ # Supported sites
  - **Vimm:stream**
  - **ViMP**
  - **ViMP:Playlist**
- - **Vimple**: Vimple - one-click video hosting
  - **Vine**
  - **vine:user**
  - **Viqeo**
@@ -1691,7 +1606,6 @@ # Supported sites
  - **viu:ott**: [*viu*](## "netrc machine")
  - **viu:playlist**
  - **ViuOTTIndonesia**
- - **Vivo**: vivo.sx
  - **vk**: [*vk*](## "netrc machine") VK
  - **vk:uservideos**: [*vk*](## "netrc machine") VK - User's Videos
  - **vk:wallpost**: [*vk*](## "netrc machine")
@@ -1699,37 +1613,27 @@ # Supported sites
  - **VKPlayLive**
  - **vm.tiktok**
  - **Vocaroo**
- - **Vodlocker**
  - **VODPl**
  - **VODPlatform**
- - **VoiceRepublic**
  - **voicy**
  - **voicy:channel**
  - **VolejTV**
- - **Voot**: [*voot*](## "netrc machine")
- - **VootSeries**: [*voot*](## "netrc machine")
+ - **Voot**: [*voot*](## "netrc machine") (**Currently broken**)
+ - **VootSeries**: [*voot*](## "netrc machine") (**Currently broken**)
  - **VoxMedia**
  - **VoxMediaVolume**
  - **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **vqq:series**
  - **vqq:video**
- - **Vrak**
  - **VRT**: VRT NWS, Flanders News, Flandern Info and Sporza
  - **VrtNU**: [*vrtnu*](## "netrc machine") VRT MAX
- - **vrv**: [*vrv*](## "netrc machine")
- - **vrv:series**
- - **VShare**
  - **VTM**
  - **VTXTV**: [*vtxtv*](## "netrc machine")
  - **VTXTVLive**: [*vtxtv*](## "netrc machine")
  - **VTXTVRecordings**: [*vtxtv*](## "netrc machine")
  - **VuClip**
- - **Vupload**
  - **VVVVID**
  - **VVVVIDShow**
- - **VyboryMos**
- - **Vzaar**
- - **Wakanim**
  - **Walla**
  - **WalyTV**: [*walytv*](## "netrc machine")
  - **WalyTVLive**: [*walytv*](## "netrc machine")
@@ -1740,9 +1644,7 @@ # Supported sites
  - **washingtonpost**
  - **washingtonpost:article**
  - **wat.tv**
- - **WatchBox**
  - **WatchESPN**
- - **WatchIndianPorn**: Watch Indian Porn
  - **WDR**
  - **wdr:mobile**: (**Currently broken**)
  - **WDRElefant**
@@ -1770,7 +1672,6 @@ # Supported sites
  - **whowatch**
  - **Whyp**
  - **wikimedia.org**
- - **Willow**
  - **Wimbledon**
  - **WimTV**
  - **WinSportsVideo**
@@ -1795,7 +1696,6 @@ # Supported sites
  - **wykop:post**
  - **wykop:​post:comment**
  - **Xanimu**
- - **XBef**
  - **XboxClips**
  - **XFileShare**: XFileShare based sites: Aparat, ClipWatching, GoUnlimited, GoVid, HolaVid, Streamty, TheVideoBee, Uqload, VidBom, vidlo, VidLocker, VidShare, VUp, WolfStream, XVideoSharing
  - **XHamster**
@@ -1807,9 +1707,6 @@ # Supported sites
  - **XMinus**
  - **XNXX**
  - **Xstream**
- - **XTube**
- - **XTubeUser**: XTube user profile
- - **Xuite**: 隨意窩Xuite影音
  - **XVideos**
  - **xvideos:quickies**
  - **XXXYMovies**
@@ -1826,10 +1723,7 @@ # Supported sites
  - **YapFiles**
  - **Yappy**
  - **YappyProfile**
- - **YesJapan**
- - **yinyuetai:video**: 音悦Tai
  - **YleAreena**
- - **Ynet**
  - **YouJizz**
  - **youku**: 优酷
  - **youku:show**
@@ -1877,6 +1771,9 @@ # Supported sites
  - **zingmp3:chart-home**
  - **zingmp3:chart-music-video**
  - **zingmp3:hub**
+ - **zingmp3:liveradio**
+ - **zingmp3:podcast**
+ - **zingmp3:podcast-episode**
  - **zingmp3:user**
  - **zingmp3:week-chart**
  - **zoom**
diff --git a/yt_dlp/version.py b/yt_dlp/version.py
index fd923fe45e..687ef8788f 100644
--- a/yt_dlp/version.py
+++ b/yt_dlp/version.py
@@ -1,8 +1,8 @@
 # Autogenerated by devscripts/update-version.py
 
-__version__ = '2023.11.16'
+__version__ = '2023.12.30'
 
-RELEASE_GIT_HEAD = '24f827875c6ba513f12ed09a3aef2bbed223760d'
+RELEASE_GIT_HEAD = 'f10589e3453009bb523f55849bba144c9b91cf2a'
 
 VARIANT = None
 
@@ -12,4 +12,4 @@
 
 ORIGIN = 'yt-dlp/yt-dlp'
 
-_pkg_version = '2023.11.16'
+_pkg_version = '2023.12.30'